Currently, I am working on automating the scraping process for a specific webpage:
The main challenge I am facing involves dealing with a dropdown menu:
https://i.stack.imgur.com/BCO7y.jpg
This is the code snippet related to that particular section:
apart_or_house = Select(driver.find_element(By.NAME, 'oss-rent'))
apart_or_house.select_by_index(2)
Unfortunately, every time I execute this code, it throws an error message:
Traceback (most recent call last): File "c:\Users\Zemmente\Documents\Python\google maps scraper\test_scraping.py", line 41, in apart_or_house.select_by_index(2) File "C:\Users\Zemmente\AppData\Local\Programs\Python\Python311\Lib\site-packages\selenium\webdriver\support\select.py", line 98, in select_by_index self._set_selected(opt) File "C:\Users\Zemmente\AppData\Local\Programs\Python\Python311\Lib\site-packages\selenium\webdriver\support\select.py", line 213, in _set_selected raise NotImplementedError("You may not select a disabled option") NotImplementedError: You may not select a disabled option
Could anyone help me identify what could be causing this issue since the left dropdown menu functions correctly?
P.S. If needed, here is the complete code block:
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select, WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
URL = "https://www.immobilienscout24.de/"
driver = webdriver.Chrome()
driver.get(URL)
# Waiting for the page to load
time.sleep(4)
# Handling cookie consent (if required)
try:
def get_shadow_root(element):
return driver.execute_script('return arguments[0].shadowRoot', element)
shadow_host = driver.find_element(By.ID, 'usercentrics-root')
button = get_shadow_root(shadow_host).find_element(By.CSS_SELECTOR, '[data-testid=uc-accept-all-button]')
button.click()
except:
print("Accept cookies not found.")
# City input processing
city_inputs = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, '//*[@id="oss-location"]')))
city_input = [element for element in city_inputs if element.is_displayed()][0]
city_input.click()
city_input.send_keys(input('Enter the city name you want to search for: '))
time.sleep(1)
city_input.send_keys(Keys.ENTER)
# Selection between buying, renting, or building
while True:
buying_or_renting_inputs = WebDriverWait(driver, 10).until(EC.presence_of_all_elements_located((By.XPATH, '//*[@id="oss-form"]/article/div/div[1]/div/div[3]/div/div/select')))
buying_or_renting_input = [element for element in buying_or_renting_inputs if element.is_displayed()][0]
options = int(input('Enter the number of one of the options: 0-Rent, 1-Buying, 2-Building: '))
if options < 0 or options > 2:
options = int(input('Enter the number of one of the options: 0-Rent, 1-Buying, 2-Building: '))
else:
select_buy_or_rent = Select(buying_or_renting_input).select_by_index(options)
break
# Mapping user-selected options to corresponding names
options_mapping = {
0: 'oss-rent',
1: 'oss-buy',
2: 'oss-build'
}
# Dynamically updating the Apartmanent or house variable based on the selection
selected_option_name = options_mapping.get(options)
apart_or_house = driver.find_element(By.NAME, selected_option_name)
options_2 = Select(apart_or_house).select_by_index(0)
# Initiating the search
search_button = driver.find_element(By.XPATH, '//*[@id="oss-form"]/article/div/div[3]/button')
search_button.click()
# Addressing captcha hurdles
captcha = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.CLASS_NAME, 'geetest_radar_tip')))
captcha.click()
# Waiting for the results to load
time.sleep(5)
page_source = driver.page_source
driver.quit()
soup = BeautifulSoup(page_source, "html.parser")