I am working with a CSV file that contains various URLs with different domain extensions such as .com, .eu, .org, and more. My goal is to only crawl domains with the .nl extension by using the condition if '.nl' in row: in Python 2.7.
from selenium import webdriver
import csv
fieldnames = ['Website', '@media', 'googleadservices.com/pagead/conversion']
def csv_writerheader(path):
with open(path, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames, lineterminator='\n')
writer.writeheader()
def csv_writer(dictdata, path):
with open(path, 'a') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=fieldnames, lineterminator='\n')
writer.writerow(dictdata)
csv_output_file = 'output!.csv'
driver = webdriver.Chrome(executable_path=r'C:\Users\Jacob\PycharmProjects\Testing\chromedriver_win32\chromedriver.exe')
keywords = ['@media', 'googleadservices.com/pagead/conversion']
csv_writerheader(csv_output_file)
with open('top1m-edited.csv') as example_file:
example_reader = csv.reader(example_file)
for row in example_reader:
# INITIALIZE DICT
data = {'Website': row}
if '.nl' in row: # FILTERING DOMAINS WITH .NL EXTENSION
try:
driver.get(row[0])
html = driver.page_source
for searchstring in keywords:
if searchstring.lower() in html.lower():
print (row, searchstring, 'FOUND!')
data[searchstring] = 'FOUND!'
else:
print (row, searchstring, 'not found')
data[searchstring] = 'not found'
csv_writer(data, csv_output_file)
except:
pass
Printed result:
C:\Python27\python.exe "C:/Users/Jacob/PycharmProjects/Testing/fooling around 2.py"
Process finished with exit code 0
Currently, my script is not producing any meaningful results and generating a CSV file with minimal output. However, when I remove the condition if '.nl' in row:, the script functions properly.
What modifications should be implemented to specifically target and scrape URLs with the .nl domain extension?