How to structure async flow using Node fs

Question

I'm having trouble getting the flow of a number of asynchronous operations to flow right. As my code below stands now, the outside readFile prints to the console first, then the names and years print next, and then finally the in readFile, when I'd like the entire readFile/forEach processing to happen before.

const fs = require('fs')
const names = {}
const years = {}

fs.readdir('./namesData', (err, files) => {
    files.forEach(file => {
        var year = file.substr(3, 4)
        var yearObj = {}
        fs.readFile(`./namesData/${file}`, 'utf8', (err, data) => {
            console.log('in readFile')
            if (err) throw new Error(err)
            var arr = data.split('\n')
            arr.forEach(record => {
                var recordArray = record.trim().split(',')
                var name = recordArray[0]
                var gender = recordArray[1]
                var score = recordArray[2]
                // populate 'names' object
                var nameObj = {year: year, gender: gender, score: score}
                if (names.hasOwnProperty(name)) {
                    names[name].push(nameObj)
                } else {
                    names[name] = [nameObj]
                }
                // populate 'years' object & add to yearObj and then years object
                var yearNameObj = {gender: gender, score: score}
                if (yearObj.hasOwnProperty(name)) {
                    yearObj[name].push(yearNameObj)
                } else {
                    yearObj[name] = [yearNameObj]
                }
            })
        })
        console.log('outside readFile')
        years[year] = yearObj
    })
    console.log(names)
    console.log(years)
})

Can you post the sample data how years and names will look like? — Sridhar
– Sridhar, Commented Jan 5, 2017 at 2:50

Sridhar · Accepted Answer · 2017-01-05 03:44:44Z

1

I tried this approach. Seems to suffice your need of printing names, years after the looping.

'use strict';

let _ = require('lodash');
const fs = require('fs');

function readFile(file) {
  try {
    console.log('in readFile');
    return fs.readFileSync(`./namesData/${file}`, 'utf-8');
  } catch (err) {
    console.log('err ', err.stack);
    return '';
  }
}

fs.readdir('./namesData', (err, files) => {
  let details = _.map(files, (file) => {
    let data = readFile(file);
    let year = file.substr(3, 4);
    return _.map(data.split('\n'), (row) => {
      if (_.isEmpty(row)) {
        return;
      }
      let words = row.trim().split(',');
      return {
        year: year,
        name: words[0],
        gender: words[1],
        score: words[2]
      };
    });
  });
  console.log('names', _.groupBy(details, 'name'));
  console.log('years', _.groupBy(details, 'year'));
});

Tested with hard coded details array

let details = [{
  year: 1998,
  name: 'A',
  gender: 'M',
  score: 20
}, {
  year: 1998,
  name: 'B',
  gender: 'M',
  score: 21
}, {
  year: 1999,
  name: 'A',
  gender: 'M',
  score: 100
}, {
  year: 1999,
  name: 'A',
  gender: 'M',
  score: 80
}];

answered Jan 5, 2017 at 3:44

Sridhar

11.8k5 gold badges42 silver badges48 bronze badges

Sign up to request clarification or add additional context in comments.

2 Comments

zahabba Over a year ago

Thank you. I was hoping to not use any of the synchronous fs methods; do you think it's possible to refactor using only fs.readFile()?

Sridhar Over a year ago

Ya. You could do that by wrapping readFile() in a promise. but then you need to call Promise.all() on it. Promise.all() is all or nothing. I stayed with sync version to avoid adding further complexity to it.

Collectives™ on Stack Overflow

How to structure async flow using Node fs

1 Answer 1

2 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

2 Comments

Your Answer

Sign up or log in

Post as a guest

Related