Parse a csv file in javascript and pull out a dictionary based on two particular rows

Question

I have some csv files that I need to parse in JS. The file consists of a bunch of rows, which I need to pull out two. One has the headers for this card, the next row is the data.

I am trying to use csv-parse and have managed to unpack the data into json objects, but can't for the life of me work out how to make this work.

I tried regex but of course csv-parse returns objects, not rows as strings.

The files in question have lots of stuff in them. The rows I'm interested in have rows like

H,TYPE1,TYPE2,field,field,field.....
followed by
D,TYPE1,TYPE2,value,value,value....

The result I'm looking for is a json object with field1:value1, field2:value2 etc.

Whats the best way to do this?

I thought about an arrayAll function to pull out the array with the right fields, but csv-parse returns objects not arrays.. sigh.

I'm sure I'm making this harder than it need to be.

Terry Lennox · Accepted Answer · 2020-02-28 12:34:29Z

1

You could simply pre-process the file to extract the relevant content (e.g. the two lines you're interested, then pass this to csv-parse),

For example:

const parse = require("csv-parse");
const fs = require("fs");

const filePath = "inline-headers.csv";
const csvContent = getRequiredCsvContent(filePath);

function getRequiredCsvContent(filePath) {
    // Replace with your actual regex pattern...
    let lineMatcher = /header/i;
    let fileLines = fs.readFileSync(filePath, "utf8").split("\n");
    let requiredLines = fileLines.reduce((contentArray, line, index) => { 
        if (lineMatcher.test(line) || (contentArray.length === 1)) {
            contentArray.push(line);
        }
        return contentArray;
    }, []);
    return requiredLines.join("\n");
}

console.log("Extracted csv content:\n", csvContent);
parse(csvContent, {
    delimiter: ",",
    columns: true
}, (err, records) => {
    if (err) {
        console.error("An error occurred:", err);
    } else { 
        console.log("Records:", records);
    }
});

I'm using a test file like so:

inline-headers.csv

someline01,someline02,someline03
someline11,someline12,someline13
header1,header2,header3
field1,field2,field3

I'm including a more specific answer here that's more tailored to your actual problem:

const parse = require("csv-parse");
const fs = require("fs");
const filePath = "inline-headers.csv";
const csvContent = getRequiredCsvContent(filePath);

function getRequiredCsvContent(filePath) {
    const lineMatcher = /^(I|D),DISPATCH,UNIT_SOLUTION/;
    let fileLines = fs.readFileSync(filePath, "utf8").split("\n");
    let requiredLines = fileLines.reduce((contentArray, line) => {
        if (lineMatcher.test(line) || (contentArray.length === 1)) {
            contentArray.push(line);
        }
        return contentArray;
    }, []);
    return requiredLines.join("\n")
}

console.log("Extracted csv content:\n", csvContent);
parse(csvContent, {
    delimiter: ",",
    columns: true,
    relax_column_count: true
}, (err, records) => {
    if (err) {
        console.error("Error:", err);
    } else {
        console.log("Records:", records);
    }
});

edited Feb 28, 2020 at 12:34

answered Feb 27, 2020 at 10:11

Terry Lennox

30.9k5 gold badges36 silver badges45 bronze badges

Sign up to request clarification or add additional context in comments.

7 Comments

Peter Nunn Over a year ago

I'm close. I have the correct two lines in csvContent but the parse command (I had to use parse = require("csv-parse") rather than "csv-parse/lib/sync" if that matters. Unfortunately rather than unpacking the full words for the header columns, it unpacked each character into its own if that makes sense.

Terry Lennox Over a year ago

I'll see if I can replicate that behaviour, then I might be able to find a workaround / fix!

Peter Nunn Over a year ago

Thanks Terry, I can send you the file I'm testing against with the code I've come up with if that helps.. not sure if you can pm on here though.

Terry Lennox Over a year ago

Ah yes, we can just do a trim on the fields and we should be ok. Reduce a very useful function, you can do an awful lot with it!

Peter Nunn Over a year ago

Yep.. that did it :)

|

Collectives™ on Stack Overflow

Parse a csv file in javascript and pull out a dictionary based on two particular rows

1 Answer 1

7 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

7 Comments

Your Answer

Sign up or log in

Post as a guest

Related