From d79395521cd06905a7cea2666bc9a239f76ef1e1 Mon Sep 17 00:00:00 2001 From: a2a-cloud Date: Sun, 7 Jun 2026 23:15:50 +0000 Subject: [PATCH] write agent.py --- agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent.py b/agent.py index 6cb0145..b42d084 100644 --- a/agent.py +++ b/agent.py @@ -280,7 +280,7 @@ def _robots_allowed(url: str, user_agent: str) -> dict[str, Any]: urllib.request.Request(robots_url, headers={"User-Agent": user_agent}), timeout=10, ) as response: - body = response.read(2_000_000).decode("utf-8", errors="ignore") + body = response.read(2000000).decode("utf-8", errors="ignore") rp.parse(body.splitlines()) return {"allowed": bool(rp.can_fetch(user_agent, url)), "robots_url": robots_url, "warnings": []} except urllib.error.HTTPError as exc: @@ -349,7 +349,7 @@ def _render_scraper_script(payload: dict[str, Any]) -> str: try: req = urllib.request.Request(robots_url, headers={{"User-Agent": user_agent}}) with urllib.request.urlopen(req, timeout=10) as response: - body = response.read(2_000_000).decode("utf-8", errors="ignore") + body = response.read(2000000).decode("utf-8", errors="ignore") rp.parse(body.splitlines()) return bool(rp.can_fetch(user_agent, url)), None except Exception as exc: