When attempting Selenium web scraping, I encountered the following issues:
- Dropdown list did not change as expected
- Unable to scrape desired results
- Unsuccessful in resolving the problems
Here is a snippet of the code used:
'''
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.select import Select
import time
driver=webdriver.Chrome(executable_path=r'C:\Program Files\Python39\chromedriver.exe')
driver.maximize_window()
driver.get("https://www.gastite.com/locator/?cats=109")
for i in range(1,3,1):
state=driver.find_element(By.NAME, 'state')
stateDD=Select(state)
stateDD.select_by_index(i)
driver.find_element(By.XPATH,'//*[@id="content"]/div[3]/form/input[2]')
time.sleep(2)
lists=driver.find_elements_by_css_selector("div.repcontent > a")
for list in lists:
try:
company=list.find_element_by_class_name('namelink company_title').text
address=list.find_element_by_class_name('address').text
address1=list.find_element_by_class_name('address2').text
tel=list.find_element_by_tag_name('span').text
fax=list.find_element_by_tag_name('span').text
web=list.get_attribute('href')
print(company, address, address1, tel, fax, web)
except Exception as e:
print("Error: ", e)
'''