0

I created a simple web scraper with Node JS to extract watch prices, here is the code:

import axios from 'axios'
import cheerio from 'cheerio'
import * as fs from 'fs'

let Array1=[];
let a = 0;
let b=1;

axios.get("URL").then((response) => {
  let $ = cheerio.load(response.data);
  $(".article-title").each((i, e) => {
    let title = $(e).text();
    Array1[a] = title;
    a = a+2;
    
  }
  )
  
  $(".article-price").each((i, e) => {
    let price = $(e).text();
    Array1[b] = price;
    b = b+2;
  }
  )

  let data = JSON.stringify(Array1);
  fs.writeFileSync('sub.json', data);

});

However, the JSON output is the following:

["\n         Rolex Submariner 5513 Maxi MK5 Dial","\n            \n      \n         16 000 €\n   ","\n         Rolex Submariner Black Dial Date 41mm - 2021 - 126610LN","\n            \n      \n         14 839 €\n   ","\n         Rolex Black Dial Submariner Date 41mm - 2021 Card - 126610ln","\n            \n      \n         14 839 €\n   ","\n         Rolex Submariner Meter First  MK1","\n            \n      \n         15 416 €\n   ","\n         Rolex Submariner 16610 military NATO 1994 S series","\n            \n      \n         35 000 €\n   ","\n         Rolex Submariner 126610LV Starbucks NEW 06/2021 Full set","\n            \n      \n         17 400 €\n   ","\n         Rolex 06/2021 - Submariner Date 126610 LN 99%new","\n            \n      \n         13 300 €\n   ","\n         Rolex Submariner 6536 Gilt James Bond 1956","\n            \n      \n         33 000 €\n   ","\n         Rolex Submariner Date Starbucks Automatik Edelstahl Herrenuhr...","\n            \n      \n         17 770 €\n   ","\n         Rolex Submariner","\n            \n      \n         12 500 €\n   ","\n         Rolex Submariner Date NOS avec stickers","\n            \n      \n         17 900 €\n   ","\n         Rolex Submariner 5513 \"Meter first\"","\n            \n      \n         16 000 €\n   ","\n         Rolex Submariner","\n            \n      \n         9 700 €\n   ","\n         Rolex Submariner 16610LV Kermit Y series 2003 Fat Four Mark 1...","\n            \n      \n         38 000 €\n   ","\n         Rolex Submariner Date 116610LN Boite/Papiers 2020","\n            \n      \n         13 790 €\n   ","\n         Rolex NEW 2021 some stickers  - Submariner No Date 124060","\n            \n      \n         12 490 €\n   ","\n         Rolex Submariner 14060M Full Set","\n            \n      \n         10 900 €\n   ","\n         Rolex Submariner Date 18K Gold Automatik Herrenuhr Oyster...","\n            \n      \n         27 240 €\n   ","\n         Rolex \"Submariner \"\"Hulk\"\"\"Model 116610LV Full Set 2017","\n            \n      \n         19 950 €\n   ","\n         Rolex Submariner Date 116619LB Rolex Serviced 2017","\n            \n      \n         36 500 €\n   ","\n         Rolex Submariner 5513 Feet First","\n            \n      \n         16 000 €\n   ","\n         Rolex Submariner","\n            \n      \n         12 900 €\n   ","\n         Rolex Submariner 5513 Maxi dial 1979","\n            \n      \n         19 000 €\n   ","\n         Rolex Submariner Date 16610 Boite / Papiers","\n            \n      \n         11 750 €\n   ","\n         Rolex Mint 2021 KERMIT / STARBUCKS - Submariner Date 126610 LV...","\n            \n      \n         17 500 €\n   ","\n         Rolex Submariner 41 No-Date - 124060 - Full Set - 2021","\n            \n      \n         12 900 €\n   ","\n         Rolex Submariner Date Stahl / Gold Automatik Herrenuhr Oyster...","\n            \n      \n         9 550 €\n   ","\n         Rolex Submariner (No Date) 14060M 2 Lines Full set 2005","\n            \n      \n         11 200 €\n   ","\n         Rolex Submariner Date 41mm NEW 09/2021","\n            \n      \n         14 500 €\n   ","\n         Rolex Submariner 16610LV  \"Kermit\"","\n            \n      \n         17 500 €\n   ","\n         Rolex Submariner Céramique 40mm","\n            \n      \n         12 900 €\n   ","\n         Rolex Submariner 5513 Gilt  Glossy 1966 Rolex warranty","\n            \n      \n         48 000 €\n   ","\n         Rolex Submariner 14060M Boite / Papiers","\n            \n      \n         11 450 €\n   ","\n         Rolex Submariner date - Oyster Perpetual","\n            \n      \n         18 000 €\n   ","\n         Rolex Submariner Date - 116610LN- Full Set - 2012","\n            \n      \n         12 500 €\n   ","\n         Rolex Submariner No Date Automatik Stahl Oyster Perpetual Ref....","\n            \n      \n         10 650 €\n   ","\n         Rolex Submariner  116610 \"Hulk\" 2010 SERIE G FULL SET STICKERS","\n            \n      \n         22 500 €\n   ","\n         Rolex Submariner Date Or Blanc 41mm New 07/2021","\n            \n      \n         43 900 €\n   ","\n         Rolex Submariner Date 2000 Full set","\n            \n      \n         10 800 €\n   ","\n         Rolex Submariner","\n            \n      \n         11 900 €\n   ","\n         Rolex Submariner 16610 COMEX M series full set 2008","\n            \n      \n         89 000 €\n   ","\n         Rolex Submariner Date Purple dial","\n            \n      \n         12 290 €\n   ","\n         Rolex Submariner (No Date)","\n            \n      \n         11 000 €\n   ","\n         Rolex Submariner 5508 Tropical James Bond - 1958","\n            \n      \n         26 000 €\n   ","\n         Rolex Submariner Date Kermit Automatik Stahl Herrenuhr Ref....","\n            \n      \n         16 950 €\n   ","\n         Rolex montre rolex 5513 submariner 39 mm automatique en acier...","\n            \n      \n         11 120 €\n   ","\n         Rolex Submariner \"Kermit\"1661OLV","\n            \n      \n         17 000 €\n   ","\n         Rolex Submariner Date 1996 série W Full set","\n            \n      \n         10 900 €\n   ","\n         Rolex Submariner","\n            \n      \n         11 900 €\n   ","\n         Rolex Submariner 16610 S series 1994 tritium from Hawaii","\n            \n      \n         14 900 €\n   ","\n         Rolex Submariner Date Mk1 military NATO rare country code 905...","\n            \n      \n         13 290 €\n   ","\n         Rolex Submariner Date","\n            \n      \n         15 000 €\n   ","\n         Rolex Submariner 5513","\n            \n      \n         12 500 €\n   ","\n         Rolex Submariner Date Stahl / Gold Automatik Oyster Perpetual...","\n            \n      \n         14 250 €\n   ","\n         Rolex submariner 16613t 40 mm automatique or & acier gold new...","\n            \n      \n         12 845 €\n   ","\n         Rolex Submariner Date de 1988","\n            \n      \n         8 900 €\n   ","\n         Rolex Submariner No Date 14060 4 Lignes 2009 Full Set","\n            \n      \n         12 000 €\n   ","\n         Rolex Submariner \"Kermit\"","\n            \n      \n         17 000 €\n   ","\n         Rolex Submariner","\n            \n      \n         9 600 €\n   ","\n         Rolex Submariner Date 116613 LN","\n            \n      \n         13 500 €\n   ","\n         Rolex Submariner Date Blue Dial 40mm Gold steel FULL SET","\n            \n      \n         12 980 €\n   ","\n         Rolex Submariner No Date Stickers - 124060 - Full Set - 2020","\n            \n      \n         12 500 €\n   ","\n         Rolex Submariner Stahl / Gold Automatik Herrenuhr Ref....","\n            \n      \n         16 020 €\n   ","\n         Rolex Submariner Date","\n            \n      \n         11 150 €\n   "]

How can I make this much more nice in order to fetch data and display in a HTML table using jQuery? I would like to a JSON output that looks like this:

[
    {
        "title": "Rolex Submariner 5513 Maxi MK5 Dial",    
        "price": "16 000"
    },
    {
        "title": "Rolex Submariner Black Dial Date 41mm - 2021 - 126610LN",
        "price": "14 839"
    },

Many thanks!! Stuck here since days...

Improve this question
5
  • 1
    By the looks of it you should strip the newline characters from the values. title = title.replace(/[\n\r]/g, ''); stackoverflow.com/questions/25582820/… Commented Oct 15, 2021 at 8:10
  • Are you positively sure that it's a JSON encoding issue and not that just how your original data is? Commented Oct 15, 2021 at 8:10
  • Looks like your array is separating everything correctly, i.e [product1, price1, product2, price2, ...]. Then it's just a manner of extracting the text and numbers and creating a json object from it. You could extract the data with regular expressions or something similar. Or just remove the \n characters and surrounding white space. Commented Oct 15, 2021 at 8:11
  • I tried title = title.replace(/[\n\r]/g, ''); and my JSON output is now [" Rolex Submariner"," 10 900 € ", ; that's better without \n but my issue if having class name before data to display in html Commented Oct 15, 2021 at 8:34
  • I'd strip new lines and double-spacing (to not close single spaces off). Commented Oct 15, 2021 at 8:55

2 Answers 2

Reset to default
0 
Array1.push({title: $(e).text()});
Array1.push({price: $(e).text()});

you should place key and their value in object and then place it in array. you put the value in title variable and then pass it to array so it replaced with their value not the word:"title" and so ...

Improve this answer
Sign up to request clarification or add additional context in comments.

2 Comments

This is working! Many thanks!!! I get a lot of "undefined" cells when displayed in a HTML table but I will find a way to remove that I guess :)
great... the main problem was around made variables instead of strings (in composing an object), and then put the information in it. I will appreciate you find it useful, please choose this response as an answer
0

Here is an example of how you can transform your JSON output to the format that you need:

function jsonToPrettyJson(json) {
  const list = [];
  for (const line of json) {
    list.push(line.trim());
  }
  
  const prettyJson = [];
  for (let i = 0; i < list.length; i += 2) {
    prettyJson.push({ title: list[i], price: list[i + 1] });
  }

  return prettyJson;
}

const json = ["\n         Rolex Submariner 5513 Maxi MK5 Dial","\n            \n      \n         16 000 €\n   ","\n         Rolex Submariner Black Dial Date 41mm - 2021 - 126610LN","\n            \n      \n         14 839 €\n   ","\n         Rolex Black Dial Submariner Date 41mm - 2021 Card - 126610ln","\n            \n      \n         14 839 €\n   ","\n         Rolex Submariner Meter First  MK1","\n            \n      \n         15 416 €\n   ","\n         Rolex Submariner 16610 military NATO 1994 S series","\n            \n      \n         35 000 €\n   ","\n         Rolex Submariner 126610LV Starbucks NEW 06/2021 Full set","\n            \n      \n         17 400 €\n   ","\n         Rolex 06/2021 - Submariner Date 126610 LN 99%new","\n            \n      \n         13 300 €\n   ","\n         Rolex Submariner 6536 Gilt James Bond 1956","\n            \n      \n         33 000 €\n   ","\n         Rolex Submariner Date Starbucks Automatik Edelstahl Herrenuhr...","\n            \n      \n         17 770 €\n   ","\n         Rolex Submariner","\n            \n      \n         12 500 €\n   ","\n         Rolex Submariner Date NOS avec stickers","\n            \n      \n         17 900 €\n   ","\n         Rolex Submariner 5513 \"Meter first\"","\n            \n      \n         16 000 €\n   ","\n         Rolex Submariner","\n            \n      \n         9 700 €\n   ","\n         Rolex Submariner 16610LV Kermit Y series 2003 Fat Four Mark 1...","\n            \n      \n         38 000 €\n   ","\n         Rolex Submariner Date 116610LN Boite/Papiers 2020","\n            \n      \n         13 790 €\n   ","\n         Rolex NEW 2021 some stickers  - Submariner No Date 124060","\n            \n      \n         12 490 €\n   ","\n         Rolex Submariner 14060M Full Set","\n            \n      \n         10 900 €\n   ","\n         Rolex Submariner Date 18K Gold Automatik Herrenuhr Oyster...","\n            \n      \n         27 240 €\n   ","\n         Rolex \"Submariner \"\"Hulk\"\"\"Model 116610LV Full Set 2017","\n            \n      \n         19 950 €\n   ","\n         Rolex Submariner Date 116619LB Rolex Serviced 2017","\n            \n      \n         36 500 €\n   ","\n         Rolex Submariner 5513 Feet First","\n            \n      \n         16 000 €\n   ","\n         Rolex Submariner","\n            \n      \n         12 900 €\n   ","\n         Rolex Submariner 5513 Maxi dial 1979","\n            \n      \n         19 000 €\n   ","\n         Rolex Submariner Date 16610 Boite / Papiers","\n            \n      \n         11 750 €\n   ","\n         Rolex Mint 2021 KERMIT / STARBUCKS - Submariner Date 126610 LV...","\n            \n      \n         17 500 €\n   ","\n         Rolex Submariner 41 No-Date - 124060 - Full Set - 2021","\n            \n      \n         12 900 €\n   ","\n         Rolex Submariner Date Stahl / Gold Automatik Herrenuhr Oyster...","\n            \n      \n         9 550 €\n   ","\n         Rolex Submariner (No Date) 14060M 2 Lines Full set 2005","\n            \n      \n         11 200 €\n   ","\n         Rolex Submariner Date 41mm NEW 09/2021","\n            \n      \n         14 500 €\n   ","\n         Rolex Submariner 16610LV  \"Kermit\"","\n            \n      \n         17 500 €\n   ","\n         Rolex Submariner Céramique 40mm","\n            \n      \n         12 900 €\n   ","\n         Rolex Submariner 5513 Gilt  Glossy 1966 Rolex warranty","\n            \n      \n         48 000 €\n   ","\n         Rolex Submariner 14060M Boite / Papiers","\n            \n      \n         11 450 €\n   ","\n         Rolex Submariner date - Oyster Perpetual","\n            \n      \n         18 000 €\n   ","\n         Rolex Submariner Date - 116610LN- Full Set - 2012","\n            \n      \n         12 500 €\n   ","\n         Rolex Submariner No Date Automatik Stahl Oyster Perpetual Ref....","\n            \n      \n         10 650 €\n   ","\n         Rolex Submariner  116610 \"Hulk\" 2010 SERIE G FULL SET STICKERS","\n            \n      \n         22 500 €\n   ","\n         Rolex Submariner Date Or Blanc 41mm New 07/2021","\n            \n      \n         43 900 €\n   ","\n         Rolex Submariner Date 2000 Full set","\n            \n      \n         10 800 €\n   ","\n         Rolex Submariner","\n            \n      \n         11 900 €\n   ","\n         Rolex Submariner 16610 COMEX M series full set 2008","\n            \n      \n         89 000 €\n   ","\n         Rolex Submariner Date Purple dial","\n            \n      \n         12 290 €\n   ","\n         Rolex Submariner (No Date)","\n            \n      \n         11 000 €\n   ","\n         Rolex Submariner 5508 Tropical James Bond - 1958","\n            \n      \n         26 000 €\n   ","\n         Rolex Submariner Date Kermit Automatik Stahl Herrenuhr Ref....","\n            \n      \n         16 950 €\n   ","\n         Rolex montre rolex 5513 submariner 39 mm automatique en acier...","\n            \n      \n         11 120 €\n   ","\n         Rolex Submariner \"Kermit\"1661OLV","\n            \n      \n         17 000 €\n   ","\n         Rolex Submariner Date 1996 série W Full set","\n            \n      \n         10 900 €\n   ","\n         Rolex Submariner","\n            \n      \n         11 900 €\n   ","\n         Rolex Submariner 16610 S series 1994 tritium from Hawaii","\n            \n      \n         14 900 €\n   ","\n         Rolex Submariner Date Mk1 military NATO rare country code 905...","\n            \n      \n         13 290 €\n   ","\n         Rolex Submariner Date","\n            \n      \n         15 000 €\n   ","\n         Rolex Submariner 5513","\n            \n      \n         12 500 €\n   ","\n         Rolex Submariner Date Stahl / Gold Automatik Oyster Perpetual...","\n            \n      \n         14 250 €\n   ","\n         Rolex submariner 16613t 40 mm automatique or & acier gold new...","\n            \n      \n         12 845 €\n   ","\n         Rolex Submariner Date de 1988","\n            \n      \n         8 900 €\n   ","\n         Rolex Submariner No Date 14060 4 Lignes 2009 Full Set","\n            \n      \n         12 000 €\n   ","\n         Rolex Submariner \"Kermit\"","\n            \n      \n         17 000 €\n   ","\n         Rolex Submariner","\n            \n      \n         9 600 €\n   ","\n         Rolex Submariner Date 116613 LN","\n            \n      \n         13 500 €\n   ","\n         Rolex Submariner Date Blue Dial 40mm Gold steel FULL SET","\n            \n      \n         12 980 €\n   ","\n         Rolex Submariner No Date Stickers - 124060 - Full Set - 2020","\n            \n      \n         12 500 €\n   ","\n         Rolex Submariner Stahl / Gold Automatik Herrenuhr Ref....","\n            \n      \n         16 020 €\n   ","\n         Rolex Submariner Date","\n            \n      \n         11 150 €\n   "];
console.log(jsonToPrettyJson(json));

Output:

  {
    title: 'Rolex Submariner Stahl / Gold Automatik Herrenuhr Ref....',
    price: '16 020 €'
  },
  { title: 'Rolex Submariner Date', price: '11 150 €' }
...
Improve this answer

3 Comments

Many thanks for this - however, I get "(node:1636) UnhandledPromiseRejectionWarning: ReferenceError: json is not defined" as error when running - any idea why?
Maybe I don't put your code in the right place, is it right above let data = JSON.stringify(Array1); ? Thank you!
@Victor the jsonToPrettyJson function is synchronous, it does nothing with promises. Make sure you call the function after the promise has been resolved and you have the data fetched.

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service and acknowledge you have read our privacy policy.

Start asking to get answers

Find the answer to your question by asking.

Ask question

Explore related questions

See similar questions with these tags.