Utilizing python with selenium and firefox

Avoid setting up a proxy unless absolutely necessary. To retrieve outbound API requests, I implemented a solution similar to this one, but in python:

test = driver.execute_script("var performance = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {}; var network = performance.getEntries() || {}; return network;")

for item in test:
  print(item)

An array of dictionaries is returned.

This method allows me to monitor all network requests made. It helps me extract a parameter from one of the requests, which I can then utilize for making my own requests to the API.

Using python with selenium and Chrome

UPDATE: This approach garnered significant attention. Here's how I currently implement it with Chrome (adapted from undetected-chromedriver code):

chrome_options = webdriver.ChromeOptions()
chrome_options.set_capability(
                        "goog:loggingPrefs", {"performance": "ALL", "browser": "ALL"}
                    )
driver = webdriver.Chrome(options=chrome_options)


##visit your website, login, etc. then:
log_entries = driver.get_log("performance")

for entry in log_entries:

    try:
        obj_serialized: str = entry.get("message")
        obj = json.loads(obj_serialized)
        message = obj.get("message")
        method = message.get("method")
        if method in ['Network.requestWillBeSentExtraInfo' or 'Network.requestWillBeSent']:
            try:
                for c in message['params']['associatedCookies']:
                    if c['cookie']['name'] == 'authToken':
                        bearer_token = c['cookie']['value']
            except:
                pass
        print(type(message), method)
        print('--------------------------------------')
    except Exception as e:
        raise e from None

By utilizing this method, you can extract tokens, api keys, and other information that your browser transmits to the server.

Utilizing Python and ChromeDriver for Network Logs

If you want to access network logs, make sure to install BrowserMobProxy in addition to selenium in python

pip install browsermob-proxy

Next, download the browsermobproxy zip file from

Unzip the contents into a designated folder (e.g., path/to/extracted_folder). This folder contains the necessary binary files that need to be referenced when calling Server() in your python code.

You must start the browser proxy and configure it within the chrome driver options,

from browsermobproxy import Server
from selenium import webdriver

server = Server("path/to/extracted_folder/bin/browsermob-proxy")
server.start()
proxy = server.create_proxy()

# Configure the browser proxy in chrome options
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--proxy-server={0}".format(proxy.proxy))
browser = webdriver.Chrome(chrome_options = chrome_options)

#tag the har(network logs) with a name
proxy.new_har("google")

Proceed by navigating to a page using selenium

browser.get("http://www.google.co.in")

Upon navigation, retrieve the network logs in json format from the proxy

print(proxy.har) # returns a Network logs (HAR) as JSON 

Prior to quitting the driver, remember to stop the proxy server as well at the end,

server.stop()
browser.quit()

Consider using selenium-wire as it offers a more effective solution, including the use of undetected-chromedriver to help circumvent bot detection.

If you're currently using selenium version 4.11, the code below might be of assistance.

import json
from selenium import webdriver

# Initializing Chrome WebDriver with performance logging enabled
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--enable-logging')
chrome_options.add_argument('--log-level=0')
chrome_options.set_capability('goog:loggingPrefs', {'performance': 'ALL'})
driver = webdriver.Chrome(options=chrome_options)

# Navigating to the specified website
driver.get("https://your-website.com")

# Gathering network log entries
log_entries = driver.get_log("performance")

# Setting up variables to hold the last known URL
last_known_url = None

# Setting up lists for request and response headers
request_headers_data = []
response_headers_data = []

for entry in log_entries:
    try:
        obj_serialized = entry.get("message")
        obj = json.loads(obj_serialized)
        message = obj.get("message")
        method = message.get("method")
        url = message.get("params", {}).get("documentURL")

        # Updating last known URL if available
        if url:
            last_known_url = url

        if method == 'Network.requestWillBeSentExtraInfo' or method == 'Network.requestWillBeSent':
            try:
                request_payload = message['params'].get('request', {})
                request_headers = request_payload.get('headers', {})
                # Storing request headers and last known URL in request_headers_data
                request_headers_data.append({"url": last_known_url, "headers": request_headers})
            except KeyError:
                pass

        if method == 'Network.responseReceivedExtraInfo' or method == 'Network.responseReceived':
            try:
                response_payload = message['params'].get('response', {})
                response_headers = response_payload.get('headers', {})
                # Storing response headers and last known URL in response_headers_data
                response_headers_data.append({"url": last_known_url, "headers": response_headers})
            except KeyError:
                pass

        if method == 'Network.loadingFinished':
            # Network request has finished, now you can access request_headers_data and response_headers_data
            print("Request Headers:")
            for request_data in request_headers_data:
                print("URL:", request_data["url"])
                print(request_data["headers"])
            print("Response Headers:")
            for response_data in response_headers_data:
                print("URL:", response_data["url"])
                print(response_data["headers"])
            print('--------------------------------------')
    except Exception as e:
        raise e from None

# Closing the WebDriver
driver.quit()

If you prefer to view requests and responses in sequence, you can follow this approach.

import json
from selenium import webdriver

# Set up Chrome WebDriver with performance logging enabled
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--enable-logging')
chrome_options.add_argument('--log-level=0')
chrome_options.set_capability('goog:loggingPrefs', {'performance': 'ALL'})
driver = webdriver.Chrome(options=chrome_options)

# Access the desired website
driver.get("https://your-website.com")

# Gather network log entries
log_entries = driver.get_log("performance")

# Initialize dictionaries for storing request and response headers
requests_data = []
responses_data = []
latest_url = None  # Keep track of the most recent URL processed

for entry in log_entries:
    try:
        obj_serialized = entry.get("message")
        obj = json.loads(obj_serialized)
        message = obj.get("message")
        method = message.get("method")
        url = message.get("params", {}).get("documentURL")

        if method == 'Network.requestWillBeSentExtraInfo' or method == 'Network.requestWillBeSent':
            try:
                request_payload = message['params'].get('request', {})
                request_headers = request_payload.get('headers', {})
                # Store request headers and latest URL in requests_data
                requests_data.append({"url": url, "headers": request_headers})
                last_known_url = url
            except KeyError:
                pass

        if method == 'Network.responseReceivedExtraInfo' or method == 'Network.responseReceived':
            try:
                response_payload = message['params'].get('response', {})
                response_headers = response_payload.get('headers', {})
                # Store response headers and latest URL in responses_data
                responses_data.append({"url": url, "headers": response_headers})
                last_known_url = url
            except KeyError:
                pass

    except Exception as e:
        raise e from None

# Print out the headers sequentially
for request_data, response_data in zip(requests_data, responses_data):
    print("Request URL:", request_data["url"])
    print("Request Headers:", request_data["headers"])
    print("Response URL:", response_data["url"])
    print("Response Headers:", response_data["headers"])
    print('--------------------------------------')

# Close the WebDriver
driver.quit()