bobseares-2/seleniumbase-website-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

write agent.py

Browse Source
This commit is contained in:
a2a-cloud
2026-06-07 23:15:50 +00:00
parent 5f7dd57c2b
commit d79395521c
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
agent.py
View File

@@ -280,7 +280,7 @@ def _robots_allowed(url: str, user_agent: str) -> dict[str, Any]:
urllib.request.Request(robots_url, headers={"User-Agent": user_agent}),
timeout=10,
) as response:
body = response.read(2_000_000).decode("utf-8", errors="ignore")
body = response.read(2000000).decode("utf-8", errors="ignore")
rp.parse(body.splitlines())
return {"allowed": bool(rp.can_fetch(user_agent, url)), "robots_url": robots_url, "warnings": []}
except urllib.error.HTTPError as exc:
@@ -349,7 +349,7 @@ def _render_scraper_script(payload: dict[str, Any]) -> str:
try:
req = urllib.request.Request(robots_url, headers={{"User-Agent": user_agent}})
with urllib.request.urlopen(req, timeout=10) as response:
body = response.read(2_000_000).decode("utf-8", errors="ignore")
body = response.read(2000000).decode("utf-8", errors="ignore")
rp.parse(body.splitlines())
return bool(rp.can_fetch(user_agent, url)), None
except Exception as exc: