Combine results from multiple Node.js API calls

Question

New to Node.js here. I'm looking for the correct way to make N asynchronous API calls from within another function, and combining their results to use further downstream. In my case, N would be reasonably small and blocking for their execution not too bad.

In synchronous execution the implementation for combine() below should work.

If I only needed the results from one API call it would be straightforward to implement the following logic in a callback function supplied to callAPI(). Where I stumble is when I need all the results combined before before executing foo(total, [...args]).

I looked into async.whilst but wasn't able to get that to work. I'm skeptical if that actually is the correct fit to my needs. I've also looked into Promises which seems to be the correct lead but it would be nice to get reassurances before crawling into that cavernous rabbit hole. Be it that Promises is the correct way, which module is the standard to use in Node.js projects?

var http = require('http');

function callAPI(id) {
    var options = {
        host: 'example.com',
        path: '/q/result/'.concat(id)
    }

    var req = http.get(options, (res) => {
        var body = [];
        res.on('data', (chunk) => {
            body.push(chunk);
        }).on('end', () => {
            body = Buffer.concat(body).toString();
            return body;
        }).on('error', (err) => {
            console.error(err);
        });
    });
}

function combine(inputs) {

    var total = 0;
    for (i=0; i < inputs.length; i++) {
        total += callAPI(inputs[i]['id']);
    };
    console.log(total);

    // call some function, foo(total, [...args])
}

Edit 1:

I attempted to follow samanime's answer below and modify the API call to return a Promise. See:

function callAPI(id) {
    return Promise((resolve, reject) => {
        var options = {
            host: 'example.com',
            path: '/q/result/'.concat(id)
        }

        var req = http.get(options, (res) => {
            var body = [];
            res.on('data', (chunk) => {
                body.push(chunk);
            }).on('end', () => {
                body = Buffer.concat(body).toString();
                resolve(body);
            }).on('error', (err) => {
                reject(err);
            });
        });
    });
}

function combine(inputs) {

    var combined = [];
    for (i=0; i < inputs.length; i++) {
        total += callAPI(inputs[i]['id']);
            .then(result => {
                combined.push(result);
            });
    };
    var total = combined.reduce((a, b) => a + b, 0);
    console.log(total);

    // call some function, foo(total, [...args])
}

This seems to get me halfway there. If I console.log(combined) inside the then() block I can see the list building up with results from the API calls. However, I still can't access the complete combined at the "end" of the for loop. Can I attach a callback to something to run after the full list has been built? Is there a better way?

Edit 2 (My solution - per Patrick Roberts suggestion)

function callAPI(id) {
    return Promise((resolve, reject) => {
        var options = {
            host: 'example.com',
            path: '/q/result/'.concat(id)
        }

        var req = http.get(options, (res) => {
            var body = [];
            res.on('data', (chunk) => {
                body.push(chunk);
            }).on('end', () => {
                body = parseInt(Buffer.concat(body));
                resolve(body);
            }).on('error', (err) => {
                reject(err);
            });
        });
    });
}

function combine(inputs) {
    var combined = [];
    Promise.all(inputs.map(input => callAPI(input.id)))
        .then((combined) => {
            var total = combined.reduce((a, b) => a + b, 0);
            // foo(total, [...args])
        });
};

try this stackoverflow.com/questions/44636542/… Its exactly what you are looking for. — Arpit Solanki
– Arpit Solanki, Commented Jun 20, 2017 at 18:59

samanime · Accepted Answer · 2017-06-20 19:01:24Z

3

It sounds like you can just chain together a bunch of promises, passing the data along.

Basically something like:

const combined = [];
asyncOne()
  .then(result => { combined.push(result); return asyncTwo())
  .then(result => { combined.push(result); return asyncThree())
  // and so on

As long as each function returns a promise, you'll be all set.

If you want to run them in parallel, use Promise.all(), which will do the same thing for you:

Promise.all([asyncOne(), asyncTwo(), asyncThree() /* , etc */])
  .then(combined => /* combined is an array with the results of each */)

This is by far the preferred pattern for this sort of thing.

answered Jun 20, 2017 at 19:01

samanime

26.8k17 gold badges99 silver badges155 bronze badges

Sign up to request clarification or add additional context in comments.

Comments

Patrick Roberts · Accepted Answer · 2017-06-20 20:19:09Z

0

Your edit is looking a lot better, but try this:

function callAPI(id) {
  return Promise((resolve, reject) => {
    var options = {
      host: 'example.com',
      path: '/q/result/' + id
    }

    http.get(options, (res) => {
      var body = [];
      res.on('data', (chunk) => {
        body.push(chunk);
      }).on('end', () => {
        body = Buffer.concat(body).toString();
        resolve(body);
      }).on('error', reject);
    });
  });
}

function combine(inputs) {
  Promise.all(inputs.map(input => callAPI(input.id))).then((combined) => {
    // completed array of bodies
    console.log(combined);
    // foo(combined.length, [...args]);
  }).catch((error) => {
    console.log(error);
  });
}

edited Jun 20, 2017 at 20:19

answered Jun 20, 2017 at 20:10

Patrick Roberts

52.4k10 gold badges120 silver badges166 bronze badges

Comments

Peheje · Accepted Answer · 2017-06-20 20:00:23Z

-1

I would add a counter that keeps track of remaining API calls. Whenever an API call finishes, decrement and if its 0, you're done.

const numCalls = 10;
let remaining = numCalls;
let data = [];

function getRandomInt(min, max) {
  min = Math.ceil(min);
  max = Math.floor(max);
  return Math.floor(Math.random() * (max - min)) + min;
}

function ajax() {
    // Simulate ajax with a setTimeout for random amount of time.
    setTimeout(() => {
        // This is the callback when calling http.get
        data.push(getRandomInt(0, 10)); // Some data from server
        if (--remaining <= 0) {
            // Am I the last call? Use data.
            console.log(data);
            console.log(data.length);
        }
    }, getRandomInt(1000, 3000));
}

for (let i = 0; i < numCalls; i++) {
    ajax();
}

answered Jun 20, 2017 at 20:00

Peheje

14.4k1 gold badge25 silver badges31 bronze badges

3 Comments

Patrick Roberts Over a year ago

It's typically bad practice to mix your application logic with asynchronous control-flow logic, this is why Promises were created, and before them, libraries like async.

Peheje Over a year ago

Sure. It all depends right. Use Promises if you can. I would argue however, that calling resolve/reject at certain points in your code is asynchronous control-flow logic (but neater) we're still mixing it with app. logic.

Patrick Roberts Over a year ago

I'm talking about counting internal callbacks, and conditional execution of consumer callbacks inside of your application logic. Of course you'll have to resolve(), reject(), or callback() or next() depending on the asynchronous control flow you choose to go with, but it's much cleaner than re-inventing the wheel with your own attempt at control-flow logic.

Collectives™ on Stack Overflow

Combine results from multiple Node.js API calls

3 Answers 3

Comments

Comments

3 Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

3 Answers 3

Comments

Comments

3 Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related