Make several requests to an API that can only handle 20 request a minute

If you can use the Bluebird promise library, it has a concurrency feature built in that lets you manage a group of async operations to at most N in flight at a time.

var Promise = require('bluebird');
const bigArray = [....];

Promise.map(bigArray, apiFetch, {concurrency: 20}).then(function(data) {
    // all done here
});

The nice thing about this interface is that it will keep 20 requests in flight. It will start up 20, then each time one finishes, it will start another. So, this is a potentially more efficient than sending 20, waiting for all to finish, sending 20 more, etc…

This also provides the results in the exact same order as bigArray so you can identify which result goes with which request.

You could, of course, code this yourself with generic promises using a counter, but since it is already built in the the Bluebird library, I thought I’d recommend that way.

The Async library also has a similar concurrency control though it is obviously not promise based.

---

Here’s a hand-coded version using only ES6 promises that maintains result order and keeps 20 requests in flight at all time (until there aren’t 20 left) for maximum throughput:

function pMap(array, fn, limit) {
    return new Promise(function(resolve, reject) {
        var index = 0, cnt = 0, stop = false, results = new Array(array.length);

        function run() {
            while (!stop && index < array.length && cnt < limit) {
                (function(i) {
                    ++cnt;
                    ++index;
                    fn(array[i]).then(function(data) {
                        results[i] = data;
                        --cnt;
                        // see if we are done or should run more requests
                        if (cnt === 0 && index === array.length) {
                            resolve(results);
                        } else {
                            run();
                        }
                    }, function(err) {
                        // set stop flag so no more requests will be sent
                        stop = true;
                        --cnt;
                        reject(err);
                    });
                })(index);
            }
        }
        run();
    });
}   

pMap(bigArray, apiFetch, 20).then(function(data) {
    // all done here
}, function(err) {
    // error here
});

Working demo here: http://jsfiddle.net/jfriend00/v98735uu/