Out of 3821 links, only 103 were provided to me. I tried applying the condition `window.scroll` to retrieve all the links, but unfortunately, it did not work as expected.
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import ssl
import undetected_chromedriver as uc
import requests
from bs4 import BeautifulSoup
import re
ssl._create_default_https_context = ssl._create_unverified_context
# ... Rest of your code ...
options = uc.ChromeOptions()
driver = uc.Chrome(options=options)
driver.get("http://www.servicealberta.gov.ab.ca/find-if-business-is-licenced.cfm")
# Click the button to load initial content
click_on_button = driver.find_element(By.CSS_SELECTOR, "td:nth-child(1) input:nth-child(1)")
click_on_button.click()
time.sleep(2)
base_url = "http://www.servicealberta.gov.ab.ca/"
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
data = BeautifulSoup(driver.page_source, "html.parser")
links = data.select("tbody td[colspan='4'] a")
for link in links:
url = base_url + link['href']
print(url)
print(len(url))