Although it may not be easy, it is definitely achievable, and here's a method to tackle it:
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import pandas as pd
import time as t
from tqdm import tqdm ## if using Jupyter notebook, import as from tqdm.notebook import tqdm
pd.set_option('display.max_columns', None)
pd.set_option('display.max_colwidth', None)
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1920,1080")
webdriver_service = Service("chromedriver/chromedriver") ## provide the path where you saved chromedriver binary
driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)
wait = WebDriverWait(driver, 5)
restaurant_list = []
driver.get('https://www.kfc.de/find-a-kfc')
try:
wait.until(EC.element_to_be_clickable((By.ID, "onetrust-reject-all-handler"))).click()
print('cookies dismissed')
except Exception as e:
print('cookie button unavailable!')
header = wait.until(EC.element_to_be_clickable((By.TAG_NAME, "app-common-header")))
driver.execute_script("""
var element = arguments[0];
element.parentNode.removeChild(element);
""", header)
for x in tqdm(range(1, 21)):
kfc_rests = wait.until(EC.presence_of_all_elements_located((By.XPATH, '//div[@class="all-stores accordian ng-star-inserted"]//app-accordian[@class="card carryout-address ng-star-inserted"]')))
for k in kfc_rests:
k.location_once_scrolled_into_view
k.click()
name = k.find_element(By.TAG_NAME, 'strong').text
address = wait.until(EC.element_to_be_clickable((By.XPATH, '//address[@class="address-block"]/p'))).text.replace('\n', ' ').strip()
try:
services = ', '.join([x.text.strip() for x in wait.until(EC.presence_of_all_elements_located((By.XPATH, '//div[@class="services ng-star-inserted"]//li')))])
except Exception as e:
services = 'Not specified'
restaurant_list.append((name, address, services))
try:
next_page = wait.until(EC.element_to_be_clickable((By.XPATH, '//li[@class="pagination-next ng-star-inserted"]//a[@aria-label="Next page"]')))
next_page.location_once_scrolled_into_view
next_page.click()
except Exception as e:
print('end of list')
break
df = pd.DataFrame(restaurant_list, columns = ['Name', 'Address', 'Services'])
print(df)
The output in the terminal:
cookies dismissed
95%
19/20 [01:41<00:05, 5.91s/it]
end of list
Name Address Services
0 KFC BERLIN Grenzallee 37 12057 Berlin Delivery, Drive Thru, Free Refill, Credit Card Payment, Click & Collect
1 KFC BERLIN Gatower Straße 56 13595 Berlin Delivery, Drive Thru, Free Refill, Credit Card Payment, Click & Collect
2 KFC BERLIN Mall of Berlin Leipziger Platz 12 10117 Berlin Delivery, Free Refill, Credit Card Payment, Click & Collect
3 KFC BERLIN Klosterstraße 3 13581 Berlin Delivery, Credit Card Payment, Click & Collect
4 KFC BERLIN Schnellerstr. 18a 12439 Berlin Drive Thru, Free Refill, Credit Card Payment, Click & Collect
... ... ... ...
191 KFC SAARBRÜCKEN Wolfseck 6 66130 Saarbrücken Drive Thru, Free Refill, Credit Card Payment
192 KFC SAARLOUIS Provinzialstr. 246 66740 Saarlouis Drive Thru, Free Refill, Credit Card Payment
193 KFC OFFENBURG Heinrich-Hertz-Str. 3 77656 Offenburg Drive Thru, Free Refill, Credit Card Payment
194 KFC FREIBURG Tullastraße 68 79108 Freiburg Delivery, Drive Thru, Free Refill, Credit Card Payment
195 KFC FRANKFURT AIRPORT Tullastraße 68 79108 Freiburg Delivery, Drive Thru, Free Refill, Credit Card Payment
196 rows × 3 columns
To explore Selenium further, visit: https://www.selenium.dev/documentation/
For Pandas documentation: https://pandas.pydata.org/docs/
And for TQDM usage guide, head to https://pypi.org/project/tqdm/