Click see more till the end using selenium python

Question

I need to extract the articles from this link http://www.biography.com/people

so I am using selenium with python to press "see more" until download all the people piography so this is my code

from selenium import webdriver
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
chrome_path = r"./chromedriver"
driver = webdriver.Chrome(chrome_path)
driver.get("http://www.biography.com/people")
while(True):
    try:
        element = WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.XPATH, "//button[contains(@class, 'm-component-footer--loader') and contains(@class, 'm-button')]")))
        element.click()
    except TimeoutException:        
        break

but the problem this code working sometimes and hust for one time. and after that give me this exception.

Traceback (most recent call last):
  File "sel.py", line 17, in <module>
    element.click()
  File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webelement.py", line 77, in click
    self._execute(Command.CLICK_ELEMENT)
  File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webelement.py", line 493, in _execute
    return self._parent.execute(command, params)
  File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/webdriver.py", line 252, in execute
    self.error_handler.check_response(response)
  File "/usr/local/lib/python2.7/dist-packages/selenium/webdriver/remote/errorhandler.py", line 194, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.WebDriverException: Message: unknown error: Element <button class="m-component-footer--loader m-button" ng-show="properties.collection.hasMoreItems || loading" ng-click="buttonPressed()" phx-track-event="" phx-track-id="load more" ng-class="{'is-inverted': properties.background.inverted}" tabindex="0" aria-hidden="false">...</button> is not clickable at point (459, 883). Other element would receive the click: <div class="kskdDiv" style="position: fixed; overflow: hidden; height: 90px !important; z-index: 2000000; width: 728px !important;-webkit-transform-origin:0 100%;-moz-transform-origin:0 100%;-ms-transform-origin:0 100%;-o-transform-origin:0 100%;transform-origin:0 100%;left:50%;bottom:0px;-webkit-transform:scale(1) translateX(-50%);-moz-transform:scale(1) translateX(-50%);-ms-transform:scale(1) translateX(-50%);-o-transform:scale(1) translateX(-50%);transform:scale(1) translateX(-50%)" data-kiosked-role="boundary">...</div>
  (Session info: chrome=58.0.3029.81)
  (Driver info: chromedriver=2.29.461571 (8a88bbe0775e2a23afda0ceaf2ef7ee74e822cc5),platform=Linux 4.8.0-49-generic x86_64)

Edit 1: When I change the time wait from 20 to 9999999999999. it is download 4 pages and after that throw the same error.

Why can't you try Web scraping? more details : pythonforbeginners.com/python-on-the-web/… — rcubefather
– rcubefather, Commented May 3, 2017 at 13:24

Tobey · Accepted Answer · 2017-05-03 13:37:08Z

1

I prefer using css selectors

from selenium import webdriver
import time

chrome_path = r"./chromedriver"
driver = webdriver.Chrome(chrome_path)
driver.get("http://www.biography.com/people")
click_more = True
while click_more:
    time.sleep(1)
    element = driver.find_element_by_css_selector('button.m-component-footer--loader')
    if element:
        element.click()
    else:
        click_more = False

answered May 3, 2017 at 13:37

Tobey

1,4401 gold badge11 silver badges25 bronze badges

Sign up to request clarification or add additional context in comments.

2 Comments

user1927468 Over a year ago

Is it working with you? because I tried your code. and it throw exception after the second page.

Tobey Over a year ago

By second page you mean after 1 click? Yes it works for me.. only difference is I used the Firefox driver. What's the exception.. maybe increase the sleep time.

Joseph M Tsai · Accepted Answer · 2017-05-04 15:13:08Z

It work for me the point is the load should disappear then click the button.... customdriver.waitForElementInvisible(loading_loc) customdriver.clickElementsBySendKey(button,"\n")

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support import expected_conditions as EC

from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import Select


class SeleniumBaseClass(object):
    def __init__(self,driver):
        self.driver = driver
    def open(self,URL):
        self.driver.get(URL)
    def driverURLChange(self,URL):  
        print("change URL" + URL)
        self.driver.get(URL)
    def currentUrl(self):
        print("URL   " +  self.driver.current_url)
        return self.driver.current_url

    def locateElement(self, loc):
        try:
            print(loc)
            element = WebDriverWait(self.driver,10).until(EC.visibility_of_element_located(loc))
            return element
        except:
            print ("cannot find {0} element".format(loc))
        return None

    def waitForElementInvisible(self,loc):
        #load-spinner
        try:
            element = WebDriverWait(self.driver,10).until(EC.invisibility_of_element_located(loc))
            return True
        except:
            print ("cannot invisibility_of_element {0} element".format(loc))
        return False            

    def send_key_with_Element(self,loc,value):
        self.locateElement(loc).clear()
        self.locateElement(loc).send_keys(value)
    def click_with_Element(self,loc):
        self.locateElement(loc).click()
    def clickElementsBySendKey(self,loc,value):
        self.locateElement(loc).send_keys(value)  




customdriver = SeleniumBaseClass(webdriver.Chrome())

customdriver.open("http://www.biography.com/people")
button = (By.XPATH,"//button[contains(@class, 'm-component-footer--loader')]")
loading_loc = (By.XPATH,"//div[@aria-hidden='true' and @translate='FEED_LOADING']")

while True:
    customdriver.waitForElementInvisible(loading_loc)
    customdriver.clickElementsBySendKey(button,"\n")

S Nagendra · Accepted Answer · 2021-09-27 10:03:32Z

0

you should check for seemore availability on the screen .. from selenium import webdriver import time

chrome_path = r"./chromedriver"
driver = webdriver.Chrome(chrome_path)
driver.get("http://www.biography.com/people")
click_more = True
while click_more:
    time.sleep(1)
    element = driver.find_element_by_css_selector('button.m-component- 
        footer--loader')
    if element.is_displayed():
        element.click()
    else:
        click_more = False

answered Sep 27, 2021 at 10:03

S Nagendra

594 bronze badges

Collectives™ on Stack Overflow

Click see more till the end using selenium python

3 Answers 3

2 Comments

Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

3 Answers 3

2 Comments

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related