polkadot.js api: How to find all nominators of a validator?

Question

How does the polkadot api keep track of the nominators per validator?

Here you can see there is a count of nominators.

api.query.staking.nominators(stash) provides a list of validators the stash has nominated.

I found this https://github.com/polkadot-js/api/issues/4124, which does not answer the question.

Do I have to query every address (is there an api that will list all addresses?), each block, and index the whole database?

Answer 1

It seems we have to parse the whole lot!

Here is the app code: https://github.com/polkadot-js/apps/blob/master/packages/page-staking/src/useNominations.ts#L12

Here is another example: https://github.com/w3f/polkadot-watcher-csv-exporter/blob/master/src/dataGatherer.ts#L72

How I managed to get the data:

const { ApiPromise, WsProvider } = require('@polkadot/api')
const { endpoints } = require('./test-endpoints')

const chain_id = 'kusama'
const endpoint = 'local';

const getNominatorStaking = async (api, apiChunkSize = 64) =>{

  console.debug(`getting the nominator entries...`)
  const nominators = await api.query.staking.nominators.entries();
  console.debug(`got ${nominators.length} entries !!`)
  const nominatorAddresses = nominators.map(([address]) => ""+address.toHuman()[0]);

  console.debug(`the nominator addresses size is ${nominatorAddresses.length}`)

  //A too big nominators set could make crush the API => Chunk splitting
  const size = apiChunkSize
  var nominatorAddressesChucked = []
  for (let i = 0; i < nominatorAddresses.length; i += size) {
    const chunk = nominatorAddresses.slice(i, i + size)
    nominatorAddressesChucked.push(chunk)
  } 

  const nominatorsStakings = []
  for (const chunk of nominatorAddressesChucked) {
    console.debug(`the handled chunk size is ${chunk.length}`)
    nominatorsStakings.push(...await api.derive.staking.accounts(chunk))
  }

  return nominatorsStakings
}

async function main () {

  const provider = new WsProvider(endpoints[chain_id][endpoint])
  const api = await ApiPromise.create({ provider: provider })

  // keep the chunk (batch) size **small** else you will flood the api!
  const nomintorStaking = await getNominatorStaking(api, 512)
  console.debug(nomintorStaking)
}

main().catch(err => {
  console.debug(err)
  process.exit(-1)
})

Answer 2

I came up with a slightly different solution which filter's out nominators for a specific set of validator. I hope this helps someone out!

Note: I wasn't able to parse the API's return value for

const nominators = await api.query.staking.nominators.entries();

so I wrote to json and parsed that instead as a temporary solution.

There is still a lot of parsing to do; if anybody comes up with a better solution please post it.

Here's my code:

/**
 * Extracts all staking data for nominators which support the target_validaotrs.
 * @param {ApiPromise} api the active polkadot.js api connection
 * @param {Collection} target_validators validators whose nominators will be filtered
 * @param {boolean} dump_json if true, write intermediate data to json and log filter message 
 * @returns a collection of staking data for each nominator supporting a target validator
 */
async function getNominatorStaking(api, target_validators, dump_json = false) {
    const nominators = await api.query.staking.nominators.entries();
    let nominatorData = nominators.map(entry => entry.map(data => data.toHuman()));

    // Filter nominators for those that support target validators
    const filteredNominators = nominatorData.filter(
        n => containsTargets(n[1].targets, target_validators)
    );

    const filteredNominatorAddresses = filteredNominators.map(entry => entry[0][0]);

    // Create chunks so as not to overload api
    let apiChunkSize = 64;
    const size = apiChunkSize;
    var nominatorAddressesChuncked = [];
    for (let i = 0; i < filteredNominatorAddresses.length; i += size) {
        const chunk = filteredNominatorAddresses.slice(i, i + size);
        nominatorAddressesChuncked.push(chunk);
    }

    // Fetch staking data for filtered nominators
    const nominatorStakings = [];
    for (const chunk of tdqm(nominatorAddressesChuncked)) {
        nominatorStakings.push(...await api.derive.staking.accounts(chunk));
    }

    // Dump data to json if requested 
    if (dump_json) {
        // Log information about filtering process
        console.log(
            filteredNominators.length +
            " addresses filtered out of " +
            nominatorData.length + ": " +
            Math.floor(filteredNominators.length / nominatorData.length * 100) +
            "%"
        );

        // Write json files 
        fs.writeFileSync(
            'intermediaries/nominators.json',
            JSON.stringify(nominatorData, null, 2)
        );

        fs.writeFileSync(
            'intermediaries/filtered_nominators.json',
            JSON.stringify(filteredNominators, null, 2)
        );

        fs.writeFileSync(
            'intermediaries/filtered_nominator_stakings.json',
            JSON.stringify(nominatorStakings, null, 2)
        );
    }

    return nominatorStakings;
}