1

I noticed some similar questions here, namely question numbers: 23501241, 20122962, and 50691096 here on StackOverflow, but none solved my problem unfortunately.

I have an array of objects that look like this:

 [ {
    title: '2015 Jeep Wrangler Unlimited Sport SUV 4D',
    price: '$15,998',
    location: 'Tampa, Florida',
    miles: '135K miles',
    itemURL: '/marketplace/item/656602258232102/',
    imgUrl: 'https://example.com',
    seen: 0,
    created_date: 2020-05-16T14:51:30.000Z
  }
]

I have been trying, unsuccessfully, all day to insert that array into a MySQL database. Here is my latest code:

const saveNewJeeps = async function (entity) {
    var con = await dbConnection();
    let objLen = entity.length;

    // FOR EACH OBJECT IN ARRAY...
    for (var i = 0; i < objLen; i++) {
        var savedJeeps = con.query('INSERT INTO newjeeps SET ?', entity[i], function (err, result) {
            // Neat!
            console.log(result);
        });  
    }

    con.release();

}

What's interesting is, when I run the function independantly, by creating an object like the one above, and sending it through the function ... it works. But in my app workflow, it is failing.

I always get the same error, complaining about the length being undefined in the beginning of the loop.

Here is my entire code:

const puppeteer = require('puppeteer');
const jsonfile = require("jsonfile");
const _ = require("lodash");
var mysql = require('mysql');
const dbConnection = require("./dbConnection");

const getSavedItems = async function () {
    let con = await dbConnection();
    try {
        await con.query("START TRANSACTION");
        let savedItems = await con.query("SELECT * FROM jeeps");
        await con.query("COMMIT");
        //console.log(savedItems);
        return savedItems;
    } catch (ex) {
        console.log(ex);
        throw ex;
    } finally {
        await con.release();
        await con.destroy();
    }
}

const saveNewJeeps = async function (entity) {
    var con = await dbConnection();
    let objLen = entity.length;

    // FOR EACH OBJECT IN ARRAY...
    for (var i = 0; i < objLen; i++) {
        var savedJeeps = con.query('INSERT INTO newjeeps SET ?', entity[i], function (err, result) {
            // Neat!
            console.log(result);
        });  
    }

    con.release();

}

// Gets current items  Search Results
const getItems = async searchTerm => {

    browser = await puppeteer.launch({
        headless: true,
        timeout: 0,
        args: ["--no-sandbox"]
    });

    page = await browser.newPage();
    await page.goto(`https://facebook.com/marketplace/tampa/search/?query=${encodeURI(searchTerm)}&sort=created_date_descending&exact=false`);
    await autoScroll(page);

    const itemList = await page.waitForSelector('div > div > span > div > a[tabindex="0"]')
        .then(() => page.evaluate(() => {

            const itemArray = [];
            const itemNodeList = document.querySelectorAll('div > div > span > div > a[tabindex="0"]');

            itemNodeList.forEach(item => {

                const itemTitle = item.innerText;
                const itemURL = item.getAttribute('href');
                const itemImg = item.querySelector('div > div > span > div > a > div > div > div > div > div > div > img').getAttribute('src');

                var obj = ['price', 'title', 'location', 'miles',
                        ...itemTitle.split(/\n/)
                    ]
                    .reduce((a, c, i, t) => {
                        if (i < 4) a[c] = t[i + 4]
                        return a
                    }, {});

                obj.imgUrl = itemImg;
                obj.itemURL = itemURL;

                itemArray.push(obj);
            });

            return itemArray;

        }))
        .catch(() => console.log("Selector error."));

    return itemList;

}


// This takes care of the auto scrolling problem
async function autoScroll(page) {
    await page.evaluate(async () => {
        await new Promise(resolve => {
            var totalHeight = 0;
            var distance = 100;
            var timer = setInterval(() => {
                var scrollHeight = document.body.scrollHeight;
                window.scrollBy(0, distance);
                totalHeight += distance;

                if (totalHeight >= scrollHeight || scrollHeight > 9000) {
                    clearInterval(timer);
                    resolve();
                }
            }, 100);
        });
    });
}


const getDifferences = async function (objNew, objOld) {

    return _.difference(objNew, objOld);
}

const init = async function () {
    const newItems = await getItems("Jeep Wrangler");
    const oldItems = await getSavedItems();
    const finalArray = await getDifferences(newItems, oldItems);
    const saveSuccess = await saveNewJeeps(finalArray);

}


const myObj =  [ {
    title: '2015 Jeep Wrangler Unlimited Sport SUV 4D',
    price: '$15,998',
    location: 'Tampa, Florida',
    miles: '135K miles',
    itemURL: '/marketplace/item/656602258232102/',
    imgUrl: 'https://example.com',
    seen: 0
  }, 
  {
    title: '2020 BMW SUV 4D',
    price: '$55,998',
    location: 'gyyu, Florida',
    miles: '15K miles',
    itemURL: '/marketplace/item/6566102/',
    imgUrl: 'https://example2.com',
    seen: 0
  }
];

// This will work just fine.
saveNewJeeps(myObj);

// But going this way, it fails...
init();

Can anyone see why this is failing? Thank you for looking.

2 Answers 2

3
    currentLogs = [
 { socket_id: 'Server', message: 'Socketio online', data: 'Port  3333', logged: '2014-05-14 14:41:11' },
 { socket_id: 'Server', message: 'Waiting for Pi to connect...', data: 'Port: 8082', logged: '2014-05-14 14:41:11' }
];

console.warn(currentLogs.map(logs=>[ logs.socket_id , logs.message , logs.data , logs.logged ]));

Bulk inserts are possible by using nested array, see the github page

Nested arrays are turned into grouped lists (for bulk inserts), e.g. [['a', 'b'], ['c', 'd']] turns into ('a', 'b'), ('c', 'd')

You just insert a nested array of elements.

An example is given in here

var mysql = require('mysql');
var conn = mysql.createConnection({
    ...
});

var sql = "INSERT INTO Test (name, email, n) VALUES ?";
var values = [
    ['demian', '[email protected]', 1],
    ['john', '[email protected]', 2],
    ['mark', '[email protected]', 3],
    ['pete', '[email protected]', 4]
];
conn.query(sql, [values], function(err) {
    if (err) throw err;
    conn.end();
});

Note:

values

is an array of arrays wrapped in an array

[ [ [...], [...], [...] ] ]

There is also a totally different node-msql package for bulk insertion

hope it's helpful :)

answer from here

Sign up to request clarification or add additional context in comments.

Comments

0

I found the answer by taking a whole different route. Not sure if I should remove this or not.

Comments

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.