Loop through an api get request with variable URL

Because the for loop runs synchronously and your calls to needle() are asynchronous and therefore do not block, you end up attempting to start more than 100,000 network requests at once. This overwhelms either your local computer or the target server and you start getting socket errors.

For this many requests, you need to run them X at a time so no more than X are in flight at the same time. To maximize performance, you will have to figure out what value of X you want to use because it will depend upon the target server and how it handles lots of simultaneous requests. It is generally safe to start with a value of 5 and then increase it from there to test higher values.

If you were processing an array, there are a number of pre-built options to run X requests at once. The simplest is to use a pre-built concurrency management operation such as Bluebird. Or you can write your own. You can see examples of both here: Make several requests to an API that can only handle 20 request a minute

But, since you are not processing an array, but are just incrementing a number for each successive request, I couldn’t find a pre-built option that does that. So, I wrote a general purpose one where you can fill in the function that will increment your index:

// fn gets called on each iteration - must return a promise
// limit is max number of requests to be in flight at once
// cnt is number of times to call fn
// options is optional and can be {continueOnError: true}
// runN returns a promise that resolves with results array.  
// If continueOnError is set, then results array 
// contains error values too (presumed to be instanceof Error so caller can discern
// them from regular values)
function runN(fn, limit, cnt, options = {}) {
    return new Promise((resolve, reject) => {
        let inFlightCntr = 0;
        let results = [];
        let cntr = 0;
        let doneCnt = 0;

        function run() {
            while (inFlightCntr < limit && cntr < cnt) {
                let resultIndex = cntr++;
                ++inFlightCntr;
                fn().then(result => {
                    --inFlightCntr;
                    ++doneCnt;
                    results[resultIndex] = result;
                    run();          // run any more that still need to be run
                }).catch(err => {
                    --inFlightCntr;
                    ++doneCnt;
                    if (options.continueOnError) {
                        // assumes error is instanceof Error so caller can tell the
                        // difference between a genuine result and an error
                        results[resultIndex] = err;       
                        run();          // run any more that still need to be run
                    } else {
                        reject(err);
                    }
                });
            }
            if (doneCnt === cnt) {
                resolve(results);
            }
        }
        run();
    });
}

Then, you could use this like this:

const needle = require("needle");
const startIdx = 11059000;
const stopIdx  = 11211109;
const numConcurrent = 5;
let idxCntr = startIdx;

runN(function() {
    let idx = idxCntr++;
    return needle('get', "https://api.companieshouse.gov.uk/company/"+idx, { 
        username: key,password:"" 
    });
}, numConcurrent, stopIdx - startIdx + 1, {continueOnError: true}).then(results => {
    console.log(results);
}).catch(err => {
    console.log(err);
});

---

To minimize memory use, you can use a .then() handler on your call to needle() and trim down the response to only what you need in the final array:

const needle = require("needle");
const startIdx = 11059000;
const stopIdx  = 11211109;
const numConcurrent = 5;
let idxCntr = startIdx;

runN(function() {
    let idx = idxCntr++;
    return needle('get', "https://api.companieshouse.gov.uk/company/"+idx, { 
        username: key,password:"" 
    }).then(response => {
        // construct the smallest possible response here and then return it
        // to minimize memory use for your 100,000+ requests
        return response.someProperty;
    });
}, numConcurrent, stopIdx - startIdx + 1, {continueOnError: true}).then(results => {
    console.log(results);
}).catch(err => {
    console.log(err);
});