Looking to extract Href links from a website and shuffle them, then iterate through each line in the list to scrape each webpage using Selenium in Python. I have come across information on how to do this with a notepad file, but I am looking for guidance on working with lists specifically.
Here is my code:
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import csv
import requests
import time
from selenium import webdriver
driver = webdriver.Chrome(executable_path=r'C:\Brother\chromedriver.exe')
driver.set_window_size(1024, 600)
driver.maximize_window()
driver.get('https://www.bookmaker.com.au/sports/soccer/')
SCROLL_PAUSE_TIME = 0.5
last_height = driver.execute_script("return document.body.scrollHeight")
while True:
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(SCROLL_PAUSE_TIME)
new_height = driver.execute_script("return document.body.scrollHeight")
if new_height == last_height:
break
last_height = new_height
time.sleep(5)
# Extract href list [Working]
elem = driver.find_elements_by_css_selector(".market-group a")
elem_href = []
for elem in elem:
print(elem.get_attribute("href"))
elem_href.append(elem.get_attribute("href"))
# Shuffle HREF (not working)
from random import shuffle
list1 = [link for link in elem_href]
shuffle(list1)
print(list1)
# Iterate through list... (Need help here)
# Driver.get…. (Read from..)
Driver.get(LINE FROM NOTEPAD HERE)
sections = driver.find_elements_by_css_selector(".fullbox")
with open('I AM HERE12345.csv', 'w') as file:
writer = csv.writer(file)
for section in sections:
link = section.find_element_by_css_selector("h3 a").get_attribute("href")
team_name = section.find_element_by_css_selector("tr.row[data-teamname]").get_attribute("data-teamname")
bet = section.find_element_by_css_selector("a.odds.quickbet").text
writer.writerow((bet, team_name, link))
# Looping.. (Not yet functioning)
driver.back()