diff --git a/agent.py b/agent.py index 6cb0145..b42d084 100644 --- a/agent.py +++ b/agent.py @@ -280,7 +280,7 @@ def _robots_allowed(url: str, user_agent: str) -> dict[str, Any]: urllib.request.Request(robots_url, headers={"User-Agent": user_agent}), timeout=10, ) as response: - body = response.read(2_000_000).decode("utf-8", errors="ignore") + body = response.read(2000000).decode("utf-8", errors="ignore") rp.parse(body.splitlines()) return {"allowed": bool(rp.can_fetch(user_agent, url)), "robots_url": robots_url, "warnings": []} except urllib.error.HTTPError as exc: @@ -349,7 +349,7 @@ def _render_scraper_script(payload: dict[str, Any]) -> str: try: req = urllib.request.Request(robots_url, headers={{"User-Agent": user_agent}}) with urllib.request.urlopen(req, timeout=10) as response: - body = response.read(2_000_000).decode("utf-8", errors="ignore") + body = response.read(2000000).decode("utf-8", errors="ignore") rp.parse(body.splitlines()) return bool(rp.can_fetch(user_agent, url)), None except Exception as exc: