I am looking to loop through all the category URLs and extract the content from each page. Although I have attempted to retrieve only the first category URL using
urls = [response.xpath('//ul[@class="flexboxesmain categorieslist"]/li/a/@href').extract()[0]]I am currently utilizing the scrapy_selenium library. However, the Selenium page source is not being passed to the 'scrape_it' function as expected. As a newcomer to the scrapy framework, I would appreciate it if you could review my code and point out any potential issues.
Below is the spider code snippet:
# -*- coding: utf-8 -*-
import scrapy
from scrapy import Request
from scrapy import Selector
from scrapy_selenium import SeleniumRequest
from ..items import CouponcollectItem
class Couponsite6SpiderSpider(scrapy.Spider):
name = 'couponSite6_spider'
allowed_domains = ['www.couponcodesme.com']
start_urls = ['https://www.couponcodesme.com/ae/categories']
def parse(self, response):
urls = [response.xpath('//ul[@class="flexboxesmain categorieslist"]/li/a/@href').extract()[0]]
for url in urls:
yield SeleniumRequest(
url=response.urljoin(url),
wait_time=3,
callback=self.parse_urls
)
def parse_urls(self, response):
driver = response.meta['driver']
while True:
next_page = driver.find_element_by_xpath('//a[@class="category_pagination_btn next_btn bottom_page_btn"]')
try:
html = driver.page_source
response_obj = Selector(text=html)
self.scrape_it(response_obj)
next_page.click()
except:
break
driver.close()
def scrape_it(self, response):
items = CouponcollectItem()
print('Hi there')
items['store_img_src'] = response.css('#temp1 > div > div.voucher_col_left.flexbox.spaceBetween > div.vouchercont.offerImg.flexbox.column1 > div.column.column1 > div > div > a > img::attr(src)').extract()
yield items
The following code has been added to the settings.py file:
DOWNLOADER_MIDDLEWARES = {
'scrapy_selenium.SeleniumMiddleware': 800
}
#SELENIUM
from shutil import which
SELENIUM_DRIVER_NAME = 'chrome'
SELENIUM_DRIVER_EXECUTABLE_PATH = which('chromedriver')
SELENIUM_DRIVER_ARGUMENTS=['-headless'] # '--headless' if using chrome instead of firefox
I have included a link to a terminal_output screenshot for reference. Thank you for your assistance in resolving this matter.