How to merge two csv files rows in node js

Question

I have 2 csv files which have different different data but having a same header eg. FILE 1 data is

"CODE","NAME","SUB_USER","SCORE"
"01","TEST","1","5"
"01","TEST","2","6"

other file FILE2 have data like this

"CODE","NAME","SUB_USER","SCORE"
    "02","TEST2","3","5"
    "02","TEST2","4","6"

so i want to merge both file create FILE3 output like this

"CODE","NAME","SUB_USER","SCORE"
"01","TEST","1","5"
"01","TEST","2","6"
"02","TEST2","3","5"
"02","TEST2","4","6"

I have tried below code

var express = require('express');
var router = express.Router();
var fs = require('fs');
var parse = require('csv-parse');
var async = require('async');
var csv = require("fast-csv");

var file1 = appRoot + '\\csvFiles\\details1.csv';
var file2 = appRoot + '\\csvFiles\\details2.csv';
var stream = fs.createReadStream(file1);
var stream2 = fs.createReadStream(file2);
var fileData1 = [],
    fileData2 = [];
csv
    .fromStream(stream)
    .on("data", function(data) {
        fileData1.push(data);
    })
    .on("end", function() {
        console.log("done");
    });

csv
    .fromStream(stream2)
    .on("data", function(data) {
        fileData2.push(data);
    })
    .on("end", function() {
        console.log("done");
    });
var fileData3 = fileData1.concat(fileData2);

csv.writeToPath("outputfile.csv", fileData3).on("finish", function() {
    console.log("END");
});

But not working don't know why?? Please help me ///**********************************************************************// Thax for help but i got new problem here

After some changes above code start working

var file1 = appRoot + '\\csvFiles\\details1.csv';
var file2 = appRoot + '\\csvFiles\\idetails2.csv';
var stream = fs.createReadStream(file1);
var stream2 = fs.createReadStream(file2);
var fileData1 = [],
    fileData2 = [],
    i = 0;

csv.fromStream(stream).on("data", function(data) {
    fileData1.push(data);
}).on("end", function() {
    csv.fromStream(stream2).on("data", function(data) {
        if (i != 0) {
            fileData2.push(data);
        }
        i++;
    }).on("end", function() {
        console.log("done");
        var fileData3 = fileData1.concat(fileData2);
        csv.writeToPath("outputfile.csv", fileData3).on("finish", function() {
            res.send('Done merge');
        });
    });
});

But problem is that what if my number of file increase then how i will handle that thing

Hi, you could try this: parse the CSV as JSON, merge the JSON, convert JSON to CSV. There are plenty of NPM module which do each step for you :) P.S. where does your "csv" variable come from? It seems that it is undefined in the code block you posted — Anditthas
– Anditthas, Commented Jun 18, 2018 at 8:28

Anditthas · Accepted Answer · 2020-05-25 09:23:22Z

7

The biggest problem here is a quite common one. You do async tasks but you don't wait for them to finish before you are using their result.

You concat the file data before the "end" callback for each tasks was called. The solution is to wait for every callback to be called and THEN working with the data.

I created a small example using Promises

const file1 = 'one.csv';
const file2 = 'two.csv';
const stream = fs.createReadStream(file1);
const stream2 = fs.createReadStream(file2);
const fileData1 = [];
const fileData2 = [];

const file1Promise = new Promise((resolve) => {
  csv
      .parseFile(file1, {headers: true})
      .on('data', function(data) {
        fileData1.push(data);
      })
      .on('end', function() {
        console.log('done');
        resolve();
      });
});

const file2Promise = new Promise((resolve) => {
  csv
      .parseFile(file2, {headers: true})
      .on('data', function(data) {
        fileData2.push(data);
      })
      .on('end', function() {
        console.log('done');
        resolve();
      });
});

Promise.all([
  file1Promise,
  file2Promise,
])
    .then(() => {
      const fileData3 = fileData1.concat(fileData2);
      console.log(fileData3);

      const csvStream = csv.format({headers: true});
      const writableStream = fs.createWriteStream('outputfile.csv');

      writableStream.on('finish', function() {
        console.log('DONE!');
      });

      csvStream.pipe(writableStream);
      fileData3.forEach((data) => {
        csvStream.write(data);
      });
      csvStream.end();
    });

I created a function with which you can easily merge multiple files:

function concatCSVAndOutput(csvFilePaths, outputFilePath) {
  const promises = csvFilePaths.map((path) => {
    return new Promise((resolve) => {
      const dataArray = [];
      return csv
          .parseFile(path, {headers: true})
          .on('data', function(data) {
            dataArray.push(data);
          })
          .on('end', function() {
            resolve(dataArray);
          });
    });
  });

  return Promise.all(promises)
      .then((results) => {

        const csvStream = csv.format({headers: true});
        const writableStream = fs.createWriteStream(outputFilePath);

        writableStream.on('finish', function() {
          console.log('DONE!');
        });

        csvStream.pipe(writableStream);
        results.forEach((result) => {
          result.forEach((data) => {
            csvStream.write(data);
          });
        });
        csvStream.end();

      });
}

example usage

concatCSVAndOutput(['one.csv', 'two.csv'], 'outputfile.csv')
    .then(() => ...doStuff);

edited May 25, 2020 at 9:23

answered Jun 18, 2018 at 9:10

Anditthas

5411 gold badge3 silver badges11 bronze badges

Sign up to request clarification or add additional context in comments.

6 Comments

Dexter Over a year ago

What if i have multiple file in folder then how will i handle

Dexter Over a year ago

Awesome bro, thax for the help its really works like charms... :)

FerventHippo Over a year ago

2020 update: in recent versions of fast-csv , .fromPath has been replaced with .parseFile

Olavo Alexandrino Over a year ago

concatCSVAndOutput(entry,outputFile) .then(() => function(){ res.send("Success, file merged!"); }); hello! take a look at the code above. "Res.send" is never called. Only "done" is print at the console. Why?

Anditthas Over a year ago

@OlavoAlexandrino Hi, you accidentally created an anonymous function inside your promise handler instead of passing a callback the solution would be one of the following: concatCSVAndOutput(entry, outputFile) .then(function() { res.send('Success, file merged!') }); OR concatCSVAndOutput(entry, outputFile) .then(() => { res.send('Success, file merged!') })

|

Collectives™ on Stack Overflow

How to merge two csv files rows in node js

1 Answer 1

6 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

6 Comments

Your Answer

Sign up or log in

Post as a guest

Related