Since some time, I try to extract the following table from the webpage.
I am trying to get into the //tr objects for the distinct pages on the site (1-8). I managed to store and open the pages, and want to loop over the tables tbody, which holds the //tr objects representing the rows of information that I would like to access:
When running the following code, however, I only get the first entrance from all the 8 pages.
# Imports required for explicit waits
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import pandas as pd
import time
webside = 'https://www.xxx.dk/find-arkitekt?display_view=block_3&field_company_region=All'
driver = webdriver.Chrome(executable_path=r'C:\Users\KristerJens\Downloads\chromedriver_win32\chromedriver')
driver.implicitly_wait(10)
driver.get(webside)
wait = WebDriverWait(driver,10)
# Cookie Accept
driver.execute_script("arguments[0].click();", wait.until(EC.element_to_be_clickable((By.XPATH, "//button[@class='agree-button eu-cookie-compliance-default-button']"))))
# Get pages from products to Extract information from product web pages
pages = []
for page in driver.find_elements_by_xpath("//div[@class='pagination']//ul[@class='pager__items js-pager__items list-pages']//li"):
pages.append(page.find_element_by_tag_name('a').get_attribute("href"))
for i in pages:
driver.get(i)
options = driver.find_element_by_xpath("//div[@class='architect-view-table']//table[@class='cols-5 responsive-enabled']//tbody//tr")
virk=[]
post=[]
by=[]
web=[]
mail=[]
for opt in options:
print(opt)
virk.append(driver.find_element_by_xpath("//div[@class='architect-view-table']//table[@class='cols-5 responsive-enabled']//thead//th[@class='icon-markunread-mailbox priority-low views-align-left views-field views-field-field-company-zip views-align-left']").text)
post.append(driver.find_element_by_xpath("//div[@class='architect-view-table']//table[@class='cols-5 responsive-enabled']//thead//th[@class='icon-markunread-mailbox priority-low views-align-left views-field views-field-field-company-zip views-align-left']").text)
by.append(driver.find_element_by_xpath("//div[@class='architect-view-table']//table[@class='cols-5 responsive-enabled']//thead//th[@class='icon-markunread-mailbox priority-low views-align-left views-field views-field-field-company-zip views-align-left']").text)
web.append(driver.find_element_by_xpath("//div[@class='architect-view-table']//table[@class='cols-5 responsive-enabled']//thead//th[@class='icon-markunread-mailbox priority-low views-align-left views-field views-field-field-company-zip views-align-left']").text)
mail.append(driver.find_element_by_xpath("//div[@class='architect-view-table']//table[@class='cols-5 responsive-enabled']//thead//th[@class='icon-markunread-mailbox priority-low views-align-left views-field views-field-field-company-zip views-align-left']").text)
How can I access all the rows from the dinstinct tables with class="cols-5 responsive-enabled" on all the 8 pages?
I tried so long now so any help is much appreciated!
