nodejs - streaming csv to string variable

Question

I have a code that accepts a list of nested objects, each of which should be converted to a log row.

The code goes through a loop on each object, then an inner loop on each property, and extracts its property (there are hundreds of properties), then puts all the information of a row - as a map of the object's name and its value, into a variable called returnVar. We use the 'fast-csv' library, with WriteStream that is named csvStream. also with a fs.createWriteStream pipe.

Finally, we loop over each object and write it with csvStream.write(), that will insert the properties name in the first line of the file, and the logs (in the same order) in the other lines.

I need to change the code so that instead of doing pipe to file stream, it will print to a string type variable.

This is the code:

let Promise = require('bluebird');
let csv = require('fast-csv');
let fs = Promise.promisifyAll(require('fs'));

...

return new Promise(function (resolve, reject) {
    var csvStream = csv.createWriteStream({ headers: propNames })
        .transform(function (item) { // every item is a nested object that contains data for a log line
            var returnVar = {}; // every returnVar will represents a map of property and value, that will be transform to a log line
            for (var prop in item) { 
                if (item.hasOwnProperty(prop)) {
                    if (propNames.indexOf(prop) >= 0) {
                        if (typeof item[prop] === 'object') {
                            returnVar[prop] = JSON.stringify(item[prop]);
                        }
                        else {
                            returnVar[prop] = item[prop];
                        }
                    }
                    //the object might be a complex item that contains some properties that we want to export...
                    else if (typeof item[prop] === 'object') {
                        var nestedItem = item[prop];
                        for (var nestedProp in nestedItem) {
                            if (propNames.indexOf(prop + "_" + nestedProp) >= 0) {
                                returnVar[prop + "_" + nestedProp] = nestedItem[nestedProp];
                            }
                        }
                    }
                }
            }

            return returnVar; // return log line
        });

    // create file path
    var fileId = "Report_" + cryptoService.generateRandomPassword(16) + ".csv";
    var filePath = tempPath + fileId;

    getOrCreateTempDirectory().then(function () {
        var writableStream = fs.createWriteStream(filePath);

        writableStream.on("finish", function () {
            resolve({
                fileId: fileId
            });
        });

        csvStream.pipe(writableStream);

        _.each(results.records, function (result) {
            // write line to file
            csvStream.write(result._source);
        });

        csvStream.end();
    });
});

user120242 · Accepted Answer · 2020-06-12 13:38:38Z

1

https://c2fo.io/fast-csv/docs/formatting/methods#writetobuffer
https://c2fo.io/fast-csv/docs/formatting/methods#writetostring

Change csvStream.write(result._source); to
csvStream.writeToString(result._source).then(data => console.log(data));

Promise.all(_.map(results.records, result => csvStream.writeToString(result._source)))
  .then(rows=>console.log(rows))
// rows should be an array of strings representing all the results

You can also use async/await

edited Jun 12, 2020 at 13:38

answered Jun 12, 2020 at 13:29

user120242

15.3k3 gold badges43 silver badges53 bronze badges

Sign up to request clarification or add additional context in comments.

Collectives™ on Stack Overflow

nodejs - streaming csv to string variable

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related