How do I create an array of Promises from the contents of an array in node.js?

Question

I am trying to read and parse a local folder consisting of CSV files asynchronously. I have managed to figure out how to do it by hardcoding the files paths

const fsPromises = require('fs').promises
const parse = require('csv-parse/lib/sync')
const fs = require('fs')

async function readfile() {
  try {
    const fileData = await Promise.all([
      fsPromises.readFile(__dirname + '/testdata/a1.csv').then((file) =>
        parse(file, {
          columns: ['Date', 'Name', 'Value'],
          relax_column_count: true,
          fromLine: 2,
        })
      ),
      fsPromises.readFile(__dirname + '/testdata/b1.csv').then((file) =>
        parse(file, {
          columns: ['Date', 'Name', 'Value'],
          relax_column_count: true,
          fromLine: 2,
        })
      ),
    ])

    fileDataFlattened = fileData.flat()
    console.log(fileDataFlattened)
  } catch (err) {
    console.log(err)
  }
}

readfile()

I would now like to do it without having to hardcode the files. I figured out you could do this by creating an array of promises which with the functionality to read and parse the files from an array of file paths and using Promise.all to resolve the promises.

Unfortunately, I can't seem to figure out how to create this array of promises. Please help.

const fsPromises = require('fs').promises
const parse = require('csv-parse/lib/sync')
const fs = require('fs')

// reading an array of filenames from the path

const filePaths = fs.readdirSync(__dirname + '/testdata')

// creating an array of promises to read and parse the file as per the above code

[array of promises] = .......

async function readDirectory() {
    .....
    const fileData = await Promise.all([array of promises])
    .....
}
    
readDirectory()

AKX · Accepted Answer · 2021-08-02 13:04:28Z

3

Use .map() to call your file-reading function for each path.

Then use Promise.all() to wait for all of those promises to resolve.

If you have lots of files, you may want to consider e.g. p-limit to not read every single file concurrently.

const fsPromises = require("fs").promises;
const parse = require("csv-parse/lib/sync");
const fs = require("fs");

async function parseCSVP(path) {
  const file = await fsPromises.readFile(path);
  // TODO: use async CSV parser too?
  return parse(file, {
    columns: ["Date", "Name", "Value"],
    relax_column_count: true,
    fromLine: 2,
  });
}

async function stuff() {
  const filePaths = fs.readdirSync(__dirname + "/testdata");
  const fileReadPromises = filePaths.map(parseCSVP);
  const fileReadResults = await Promise.all(fileReadPromises);
  console.log(fileReadResults);
}

answered Aug 2, 2021 at 13:04

AKX

171k16 gold badges146 silver badges228 bronze badges

Sign up to request clarification or add additional context in comments.

Collectives™ on Stack Overflow

How do I create an array of Promises from the contents of an array in node.js?

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related