My current challenge involves executing a script that utilizes Selenium and specifically webdriver.
driver = webdriver.Firefox(executable_path='numpy-test/geckodriver', options=options, service_log_path ='/dev/null')
The problem I am facing is that the function requires geckodriver to be present in order to run. Geckodriver is stored in the zip file that I have uploaded to AWS, but I am unsure how to make the function access it on AWS. When running locally, everything works fine since geckodriver is in my directory.
Running the function via serverless results in the following error message:
{ "errorMessage": "Message: 'geckodriver' executable needs to be in PATH. \n", "errorType": "WebDriverException", "stackTrace": [ [ "/var/task/handler.py", 66, "main", "print(TatamiClearanceScrape())" ], [ "/var/task/handler.py", 28, "TatamiClearanceScrape", "driver = webdriver.Firefox(executable_path='numpy-test/geckodriver', options=options, service_log_path ='/dev/null')" ], [ "/var/task/selenium/webdriver/firefox/webdriver.py", 164, "init", "self.service.start()" ], [ "/var/task/selenium/webdriver/common/service.py", 83, "start", "os.path.basename(self.path), self.start_error_message)" ] ] }
Error --------------------------------------------------
The invoked function has failed
Any assistance on this matter would be greatly appreciated.
EDIT:
def TatamiClearanceScrape():
options = Options()
options.add_argument('--headless')
page_link = 'https://www.tatamifightwear.com/collections/clearance'
# this is the url that we've already determined is safe and legal to scrape from.
page_response = requests.get(page_link, timeout=5)
# here, we fetch the content from the url, using the requests library
page_content = BeautifulSoup(page_response.content, "html.parser")
driver = webdriver.Firefox(executable_path='numpy-test/geckodriver', options=options, service_log_path ='/dev/null')
driver.get('https://www.tatamifightwear.com/collections/clearance')
labtnx = driver.find_element_by_css_selector('a.btn.close')
labtnx.click()
time.sleep(10)
labtn = driver.find_element_by_css_selector('div.padding')
labtn.click()
time.sleep(5)
# wait(driver, 50).until(lambda x: len(driver.find_elements_by_css_selector("div.detailscontainer")) > 30)
html = driver.page_source
page_content = BeautifulSoup(html)
# we use the html parser to parse the url content and store it in a variable.
textContent = []
tags = page_content.findAll("a", class_="product-title")
product_title = page_content.findAll(attrs={'class': "product-title"}) # allocates all product titles from site
old_price = page_content.findAll(attrs={'class': "old-price"})
new_price = page_content.findAll(attrs={'class': "special-price"})
products = []
for i in range(len(product_title) - 2):
# groups all products together in list of dictionaries, with name, old price and new price
object = {"Product Name": product_title[i].get_text(strip=True),
"Old Price:": old_price[i].get_text(strip=True),
"New Price": new_price[i].get_text(), 'date': str(datetime.datetime.now())
}
products.append(object)
return products