How to extract data from webpage using python

Question

can someone point me what i am doing wrong?

Enter Item name:Rockfish Traceback (most recent call last): File "C:\Users\partn_000\Desktop\sarvesh\Python Source Code\working\jellyneoscraper.py", line 45, in search(br, ITEMNAME) File "C:\Users\partn_000\Desktop\sarvesh\Python Source Code\working\jellyneoscraper.py", line 33, in search increment = increment[0] IndexError: list index out of range

This is the code i wrote

#Library Imports
import mechanize
import cookielib
import re
import sys
import time
import os.path
from operator import itemgetter
import ctypes
ctypes.windll.kernel32.SetConsoleTitleA("test")


def init_browser():
    br = mechanize.Browser()
    br.set_handle_equiv(True)
    br.set_handle_redirect(True)
    br.set_handle_referer(True)
    br.set_handle_robots(False)
    br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1)
    br.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36')]
    cj = cookielib.LWPCookieJar()
    br.set_cookiejar(cj)

    return br


def search(br, ITEMNAME):
    datapage = br.open('http://items.jellyneo.net/index.php?go=show_items&name=' +ITEMNAME +'&name_type=exact&desc=&cat=0&specialcat=0&status=0&rarity=0&sortby=name&numitems=20')
    f = open('search.html', 'w')
    f.write(datapage.read())
    f.close()
    value = re.findall('style="font-weight:bold;">(.+) NP</a></td>"',datapage.read())  #(.+) is replaced in place of required value
    value = value[0].replace(",","")
    value = int(value)
    print value
#http://items.jellyneo.net/index.php?go=show_items&name=Rockfish&name_type=exact&desc=&cat=0&specialcat=0&status=0&rarity=0&sortby=name&numitems=20


#('style="font-weight:bold;"> (.+) NP</a>"',search.read())


ITEMNAME = raw_input('Enter Item name:eg. Rockfish')

br = init_browser()
search(br, ITEMNAME)

DorElias · Accepted Answer · 2015-09-06 18:28:20Z

1

in your search method you read the entire page and save it to a file, then you try to reread it yo execute your regex but you are already at the end of the page so it returns empty string. you should add datapage.seek(0) before reading it again like this:

datapage = br.open('http://items.jellyneo.net/index.php?go=show_items&name=' +ITEMNAME +'&name_type=exact&desc=&cat=0&specialcat=0&status=0&rarity=0&sortby=name&numitems=20')
f = open('search.html', 'w')
f.write(datapage.read())
f.close()
datapage.seek(0)
value = re.findall('style="font-weight:bold;">(.+) NP</a></td>"',datapage.read())  #(.+) is replaced in place of required value
value = value[0].replace(",","")
value = int(value)

answered Sep 6, 2015 at 18:28

DorElias

2,31316 silver badges18 bronze badges

Sign up to request clarification or add additional context in comments.

Collectives™ on Stack Overflow

How to extract data from webpage using python

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related