When it comes to conducting Google searches, there are several modules available such as google and Google-Search-API. However, if you need to perform numerous searches and send multiple requests, Google may block your access, resulting in error 503. In the past, alternative APIs like Bing and Yahoo were options, but they now come at a cost. The only free API for conducting internet searches is the FAROO API. Another option for executing Google searches is by utilizing the Selenium webdriver. Selenium is capable of mimicking browser actions and offers various webdrivers (such as Firefox, Chrome, Edge, or Safari) for use. Although Selenium will open a browser window during the search process, the annoyance can be resolved by using PhantomJS. Download PhantomJS from here, extract it, and refer to the example below on how to utilize it:
import time
from urllib.parse import quote_plus
from selenium import webdriver
class Browser:
def __init__(self, path, initiate=True, implicit_wait_time=10, explicit_wait_time=2):
self.path = path
self.implicit_wait_time = implicit_wait_time
self.explicit_wait_time = explicit_wait_time
if initiate:
self.start()
return
def start(self):
self.driver = webdriver.PhantomJS(path)
self.driver.implicitly_wait(self.implicit_wait_time)
return
def end(self):
self.driver.quit()
return
def go_to_url(self, url, wait_time=None):
if wait_time is None:
wait_time = self.explicit_wait_time
self.driver.get(url)
print('[*] Fetching results from: {}'.format(url))
time.sleep(wait_time)
return
def get_search_url(self, query, page_num=0, per_page=10, lang='en'):
query = quote_plus(query)
url = 'https://www.google.hr/search?q={}&num={}&start={}&nl={}'.format(query, per_page, page_num*per_page, lang)
return url
def scrape(self):
links = self.driver.find_elements_by_xpath("//h3[@class='r']/a[@href]")
results = []
for link in links:
d = {'url': link.get_attribute('href'),
'title': link.text}
results.append(d)
return results
def search(self, query, page_num=0, per_page=10, lang='en', wait_time=None):
if wait_time is None:
wait_time = self.explicit_wait_time
url = self.get_search_url(query, page_num, per_page, lang)
self.go_to_url(url, wait_time)
results = self.scrape()
return results
path = '<YOUR PATH TO PHANTOMJS>/phantomjs-2.1.1-windows/bin/phantomjs.exe'
br = Browser(path)
results = br.search('Python')
for r in results:
print(r)
br.end()