If you need information from XHR requests from testing page you can use logs from chrome driver.
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
# enable browser
logging d = DesiredCapabilities.CHROME
d['loggingPrefs'] = { 'browser':'ALL' }
driver = webdriver.Chrome(desired_capabilities=d)
# load the desired webpage
driver.get('http://foo.com')
# print messages for entry in
driver.get_log('browser'):
print(entry)
self.driver.get_log("performance")
self.driver.execute_cdp_cmd(
"Network.getResponseBody",
{"requestId":max(xhrs, key=lambda x:x['id']).get("id")})
Get details from an working example
def get_details(self, job):
logs_json = []
responses = []
self.driver.get(job.url)
sleep(2)
self.driver.execute_script("jQuery('.eu-cookie-compliance-default-button').click();")
sleep(2)
logs = self.driver.get_log("performance")
body = ""
for log in logs:
logs_json.append(json.loads(log["message"])["message"])
for log in logs_json:
if(log['method'] == 'Network.responseReceived'):
responses.append(log['params'])
api_url = job.url.replace("https://www.domain.be/", "https://www.domain.be/api/")
api_url = api_url + "?refresh=false"
for response in responses:
if(response['response']['url'].find(api_url) != -1): # if right request load body then
# Use second request first is expaired with cookies.
print(response['requestId'])
print(response['response']['url'])
try:
body = self.driver.execute_cdp_cmd("Network.getResponseBody", {"requestId": response['requestId']})
json_body = json.loads(body['body'])
except:
next
Troubleshooting
Sometimes you can sit for quite a while with an error where you cannot read the body of a request. In my experience that was a problem of multiple requests. If on a page multiple requests are made to the same URL then it could be that previous response is no longer readable and body cannot be found. Then this error happens. So you have to get your body not only from right request but on time.
-> body = self.driver.execute_cdp_cmd("Network.getResponseBody", {"requestId": response['requestId']})
(Pdb) response['requestId']
'73287.104'
(Pdb) body = self.driver.execute_cdp_cmd("Network.getResponseBody", {"requestId": response['requestId']})
* selenium.common.exceptions.WebDriverException: Message: unknown error: unhandled inspector error: {"code":-32000,"message":"No resource with given identifier found"}
(Session info: chrome=100.0.4896.127)
Comments
Post a Comment