bobseares-2/seleniumbase-website-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

write agent.py

Browse Source
This commit is contained in:
a2a-cloud
2026-06-08 22:49:21 +00:00
parent 0b79431716
commit 8134a9275e
1 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
agent.py
View File

@@ -353,13 +353,21 @@ class SeleniumbaseWebsiteScraper(A2AAgent[SeleniumbaseWebsiteScraperConfig, NoAu
current_url = start_url current_url = start_url
current_selectors = selectors or {"title": "title"} current_selectors = selectors or {"title": "title"}
for iteration in range(1, int(max_iterations) + 1): hard_cap_steps = max(1, min(int(max_steps), 3))
hard_cap_runtime = max(30, min(int(max_runtime_seconds), 840))
started_at = datetime.now(timezone.utc)
for step in range(1, hard_cap_steps + 1):
# Runtime budget check
if (datetime.now(timezone.utc) - started_at).total_seconds() > hard_cap_runtime:
warnings.append("max_runtime_seconds reached; stopping")
break
# Execute one browser pass # Execute one browser pass
run = await self.scrape_website( run = await self.scrape_website(
ctx=ctx, ctx=ctx,
url=current_url, url=current_url,
selectors=current_selectors, selectors=current_selectors,
max_pages=per_pass_max_pages, max_pages=max_pages,
wait_seconds=wait_seconds, wait_seconds=wait_seconds,
output_format=output_format, output_format=output_format,
headless=headless, headless=headless,