polkadot.js api: How to find all nominators of a validator?

Question

How does the polkadot api keep track of the nominators per validator?

Here you can see there is a count of nominators.

api.query.staking.nominators(stash) provides a list of validators the stash has nominated.

I found this https://github.com/polkadot-js/api/issues/4124, which does not answer the question.

Do I have to query every address (is there an api that will list all addresses?), each block, and index the whole database?

Answer 1

It seems we have to parse the whole lot!

Here is the app code: https://github.com/polkadot-js/apps/blob/master/packages/page-staking/src/useNominations.ts#L12

Here is another example: https://github.com/w3f/polkadot-watcher-csv-exporter/blob/master/src/dataGatherer.ts#L72

How I managed to get the data:

const { ApiPromise, WsProvider } = require('@polkadot/api')
const { endpoints } = require('./test-endpoints')

const chain_id = 'kusama'
const endpoint = 'local';

const getNominatorStaking = async (api, apiChunkSize = 64) =>{

  console.debug(`getting the nominator entries...`)
  const nominators = await api.query.staking.nominators.entries();
  console.debug(`got ${nominators.length} entries !!`)
  const nominatorAddresses = nominators.map(([address]) => ""+address.toHuman()[0]);

  console.debug(`the nominator addresses size is ${nominatorAddresses.length}`)

  //A too big nominators set could make crush the API => Chunk splitting
  const size = apiChunkSize
  var nominatorAddressesChucked = []
  for (let i = 0; i < nominatorAddresses.length; i += size) {
    const chunk = nominatorAddresses.slice(i, i + size)
    nominatorAddressesChucked.push(chunk)
  } 

  const nominatorsStakings = []
  for (const chunk of nominatorAddressesChucked) {
    console.debug(`the handled chunk size is ${chunk.length}`)
    nominatorsStakings.push(...await api.derive.staking.accounts(chunk))
  }

  return nominatorsStakings
}

async function main () {

  const provider = new WsProvider(endpoints[chain_id][endpoint])
  const api = await ApiPromise.create({ provider: provider })

  // keep the chunk (batch) size **small** else you will flood the api!
  const nomintorStaking = await getNominatorStaking(api, 512)
  console.debug(nomintorStaking)
}

main().catch(err => {
  console.debug(err)
  process.exit(-1)
})

Answer 2

I came up with a slightly different solution which filter's out nominators for a specific set of validator. I hope this helps someone out!

Note: I wasn't able to parse the API's return value for

const nominators = await api.query.staking.nominators.entries();

so I wrote to json and parsed that instead as a temporary solution.

There is still a lot of parsing to do; if anybody comes up with a better solution please post it.

Here's my code:

/**
 * Extracts all staking data for nominators which support the target_validaotrs.
 * @param {ApiPromise} api the active polkadot.js api connection
 * @param {Collection} target_validators validators whose nominators will be filtered
 * @param {boolean} dump_json if true, write intermediate data to json and log filter message 
 * @returns a collection of staking data for each nominator supporting a target validator
 */
async function getNominatorStaking(api, target_validators, dump_json = false) {
    const nominators = await api.query.staking.nominators.entries();
    let nominatorData = nominators.map(entry => entry.map(data => data.toHuman()));

    // Filter nominators for those that support target validators
    const filteredNominators = nominatorData.filter(
        n => containsTargets(n[1].targets, target_validators)
    );

    const filteredNominatorAddresses = filteredNominators.map(entry => entry[0][0]);

    // Create chunks so as not to overload api
    let apiChunkSize = 64;
    const size = apiChunkSize;
    var nominatorAddressesChuncked = [];
    for (let i = 0; i < filteredNominatorAddresses.length; i += size) {
        const chunk = filteredNominatorAddresses.slice(i, i + size);
        nominatorAddressesChuncked.push(chunk);
    }

    // Fetch staking data for filtered nominators
    const nominatorStakings = [];
    for (const chunk of tdqm(nominatorAddressesChuncked)) {
        nominatorStakings.push(...await api.derive.staking.accounts(chunk));
    }

    // Dump data to json if requested 
    if (dump_json) {
        // Log information about filtering process
        console.log(
            filteredNominators.length +
            " addresses filtered out of " +
            nominatorData.length + ": " +
            Math.floor(filteredNominators.length / nominatorData.length * 100) +
            "%"
        );

        // Write json files 
        fs.writeFileSync(
            'intermediaries/nominators.json',
            JSON.stringify(nominatorData, null, 2)
        );

        fs.writeFileSync(
            'intermediaries/filtered_nominators.json',
            JSON.stringify(filteredNominators, null, 2)
        );

        fs.writeFileSync(
            'intermediaries/filtered_nominator_stakings.json',
            JSON.stringify(nominatorStakings, null, 2)
        );
    }

    return nominatorStakings;
}

Answer 3

You may use for the particular era:

    const particularEra=1319;
    const stakers = await api.query.staking.erasStakers(api.createType('EraIndex', paticularEra), VALIDATOR);

    console.log('validator and nominators inside (other):',
        JSON.parse(JSON.stringify(stakers)));

Output will be like (where others are validators):

  total: '0x0000000000000000005b9ef09bfade21',
  own: 100000000000000,
  others: [
    {
      who: '13yrrdCBQQ9RTy2dVFJ8yC6jQdJs5HqHrZZcVCYxsbJtCc82',
      value: 70458858722673

    },
...
   ]

Or for all eras and validators:

const stakers = await api.query.staking.erasStakers.entries();

stakers
    .forEach((x) => {
        console.log(`Validator: ======================\n\n`,
            x[0].toHuman());
        console.log(`Nominators:\n\n`,
            x[1].others.toJSON());
    });

Output will be like ('1,320' is era):

Validator: ======================

 [ '1,320', '138X7AdRKX7myWMwoFgWfHj1WUZKKZrKTr2CZ8pGUUXTrkDu' ]
Nominators:

 [
  {
    who: '15qunVeyLEybbNzwhG9Zo4Xz8dtbzc2r58eydd31M3wiz2y3',
    value: 1135000000000000
  },
...
]

Minor comments:

Spec saids about erasStakers:

Note: Deprecated since v14. Use `EraInfo` instead to work with exposures.
...
New `Exposure`s are stored in a paged manner in `ErasStakersPaged` instead.

But EraInfo and erasStakersPaged still not exists.