bobseares-2/seleniumbase-website-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

deploy

Browse Source
This commit is contained in:
a2a-platform
2026-06-27 01:15:21 +00:00
parent 0e622fe376
commit 6a6f5122b7
11 changed files with 1578 additions and 935 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.a2a-builder-state.json
View File

@@ -1,12 +0,0 @@
{
"agent": "seleniumbase-website-scraper",
"file_count": 9,
"last_sandbox": {
"exit_code": 0,
"source_hash": "800286217c3eb6945d8f9b09404a6da74147243f3a05ed4fb46f11b6c0ff4cf1",
"updated_at": 1781005877
},
"repo_head_sha": "59e28467b7e98c3ef0abe4da070be833ccacb7dd",
"source": "repo-sync",
"updated_at": 1781005822
}

351
.a2a/agent.dsl.json Normal file
View File

@@ -0,0 +1,351 @@
{
"schema_version": "2026-06-04",
"language": "python",
"name": "seleniumbase-website-scraper",
"description": "SeleniumBase-style browser agent that turns observed website traffic into OpenAPI specs, coverage reports, samples, and a replay client.",
"version": "0.2.0",
"entrypoint": {
"module": "agent",
"class_name": "BrowserToApiAgent",
"function": null,
"command": []
},
"skills": [
{
"name": "discover_api_from_browser",
"description": "Open a website in a browser, capture observable HTTP traffic, infer an OpenAPI 3.1 spec, and write a browsable API discovery report.",
"handler": "discover_api_from_browser",
"tags": [
"browser",
"seleniumbase",
"openapi",
"api-discovery"
],
"scopes": [],
"stream": false,
"policy": {
"timeout_seconds": 600.0,
"idempotent": false,
"max_retries": 0,
"cost_class": null,
"allow_scope_expansion": false,
"grant_mode": null,
"grant_allow_patterns": [],
"grant_deny_patterns": [],
"grant_outputs_prefix": null,
"grant_write_prefixes": [],
"grant_ttl_seconds": null,
"grant_run_timeout_seconds": null,
"grant_approval_timeout_seconds": null,
"grant_scope_approval_timeout_seconds": null
},
"input_schema": {
"type": "object",
"properties": {
"url": {
"type": "string"
},
"title": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
]
},
"origins": {
"anyOf": [
{
"items": {
"type": "string"
},
"type": "array"
},
{
"type": "null"
}
]
},
"include": {
"anyOf": [
{
"items": {
"type": "string"
},
"type": "array"
},
{
"type": "null"
}
]
},
"exclude": {
"anyOf": [
{
"items": {
"type": "string"
},
"type": "array"
},
{
"type": "null"
}
]
},
"wait_seconds": {
"type": "integer"
},
"max_clicks": {
"type": "integer"
},
"search_term": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
]
},
"min_samples": {
"type": "integer"
},
"redact": {
"anyOf": [
{
"items": {
"type": "string"
},
"type": "array"
},
{
"type": "null"
}
]
}
},
"required": [
"url"
],
"additionalProperties": false
},
"output_schema": {
"additionalProperties": true,
"type": "object"
}
},
{
"name": "discover_api_from_trace",
"description": "Replay request/response JSONL traffic into the same OpenAPI 3.1 discovery pipeline used by live browser capture.",
"handler": "discover_api_from_trace",
"tags": [
"trace",
"openapi",
"api-discovery"
],
"scopes": [],
"stream": false,
"policy": {
"timeout_seconds": 240.0,
"idempotent": false,
"max_retries": 0,
"cost_class": null,
"allow_scope_expansion": false,
"grant_mode": null,
"grant_allow_patterns": [],
"grant_deny_patterns": [],
"grant_outputs_prefix": null,
"grant_write_prefixes": [],
"grant_ttl_seconds": null,
"grant_run_timeout_seconds": null,
"grant_approval_timeout_seconds": null,
"grant_scope_approval_timeout_seconds": null
},
"input_schema": {
"type": "object",
"properties": {
"requests_jsonl": {
"type": "string"
},
"responses_jsonl": {
"type": "string"
},
"title": {
"type": "string"
},
"origins": {
"anyOf": [
{
"items": {
"type": "string"
},
"type": "array"
},
{
"type": "null"
}
]
},
"include": {
"anyOf": [
{
"items": {
"type": "string"
},
"type": "array"
},
{
"type": "null"
}
]
},
"exclude": {
"anyOf": [
{
"items": {
"type": "string"
},
"type": "array"
},
{
"type": "null"
}
]
},
"min_samples": {
"type": "integer"
},
"redact": {
"anyOf": [
{
"items": {
"type": "string"
},
"type": "array"
},
{
"type": "null"
}
]
}
},
"required": [
"requests_jsonl"
],
"additionalProperties": false
},
"output_schema": {
"additionalProperties": true,
"type": "object"
}
}
],
"capabilities": {
"browser_to_api": {
"capture": "Launches a browser, records XHR/fetch/document traffic with response bodies, then infers OpenAPI.",
"offline": "Can replay JSONL request/response captures into the same OpenAPI emitter.",
"outputs": [
"openapi.json",
"openapi.yaml",
"report.md",
"index.html",
"client.mjs",
"confidence.json",
"samples/*.json"
],
"noise_filtering": "Drops analytics, static assets, service workers, and obvious bot-defense plumbing by default.",
"schema_inference": "Infers request/response JSON schemas, query params, path params, GraphQL dispatches, and auth-shaped headers."
}
},
"input_modes": [
"application/json"
],
"output_modes": [
"application/json"
],
"required_secrets": [],
"required_env": [],
"consumer_setup": {
"fields": []
},
"runtime": {
"lifecycle": "ephemeral",
"availability": "on_demand",
"state": "none",
"sandbox": "microsandbox",
"resources": {
"cpu": "2000m",
"memory": "4Gi",
"gpu": 0,
"max_runtime_seconds": 900
},
"concurrency": 1,
"egress": {
"allow_hosts": [
"*"
],
"allow_internal_services": [],
"deny_internet_by_default": false
},
"tools_used": [
"seleniumbase",
"playwright",
"browser-to-api",
"openapi"
],
"llm_provisioning": "platform",
"pricing": {
"price_per_call_usd": 0.0,
"caller_pays_llm": false,
"notes": ""
},
"wants_cp_jwt": false,
"platform_resources": {
"memory": null,
"databases": []
},
"endpoints": [],
"apt_packages": []
},
"template_lineage": null,
"meta_agent_manifest": null,
"state_schema": null,
"workspace_access": {
"enabled": false,
"max_files": 0,
"allowed_modes": [],
"require_reason": true,
"deny_patterns": [],
"require_human_approval": false,
"max_total_size_bytes": 104857600
},
"config_schema": {
"properties": {},
"title": "BrowserToApiConfig",
"type": "object"
},
"auth": {
"model": "a2a_pack.auth.NoAuth",
"strategy": "public",
"principal_schema": {
"additionalProperties": false,
"description": "Public agent: no caller identity required.",
"properties": {},
"title": "NoAuth",
"type": "object"
},
"resolver": null,
"required": false
},
"metadata": {
"source": "python-a2a-pack",
"project_manifest": {
"name": "seleniumbase-website-scraper",
"version": "0.2.0",
"entrypoint": "agent:BrowserToApiAgent"
}
}
}

157
README.md
View File

@@ -1,145 +1,40 @@
# seleniumbase-website-scraper
Private A2A agent for compliant, authorized website scraping with SeleniumBase/browser automation where available.
Browser-to-API discovery agent.
## Safety policy
This upgrades the original SeleniumBase website scraper idea into a hosted API
discovery workflow:
Use this agent only for authorized scraping of public or user-owned sites. The agent must not bypass, solve, defeat, or evade CAPTCHAs, anti-bot protections, login gates, paywalls, `robots.txt`, ToS restrictions, or access controls. If a CAPTCHA, anti-bot challenge, login wall, paywall, or access-control indicator is detected, the run stops and returns a manual-intervention/blocked status.
- opens a site in a browser
- records observable XHR/fetch/document traffic
- captures JSON response bodies when the browser can read them
- filters analytics/static noise
- templates paths and query parameters
- splits GraphQL/JSON-RPC style dispatches
- infers request and response JSON schemas
- emits `openapi.json`, `openapi.yaml`, `report.md`, `index.html`,
`client.mjs`, `confidence.json`, and redacted samples
The agent does not use stealth/undetected-browser modes, CAPTCHA solvers, proxy rotation, credential injection, or access-control circumvention.
## Run Locally
## Skills
### scrape_website
Inputs:
- url — absolute http or https URL.
- selectors — mapping of output field names to CSS selectors.
- max_pages — maximum same-origin pages to visit.
- wait_seconds — wait after page open before capture/extraction.
- output_format — json or csv.
- headless — toggle browser headless mode.
- user_agent — optional custom single-line user agent.
- respect_robots — when true, checks robots.txt before fetch and pagination.
- rate_limit_seconds — polite delay between pagination requests.
- screenshot — save PNG screenshots for visited pages.
Outputs:
- status — ok, invalid_input, blocked_by_robots_txt, manual_intervention_required, browser_runtime_unavailable, or error.
- items — extracted records.
- saved_paths — output files under outputs/seleniumbase-website-scraper/.
- warnings — safety, robots, selector, browser, or runtime warnings.
- screenshots — screenshot file paths when requested.
- pages_visited and html_captures.
### iterate_scrape_goal (new)
Parameters:
- start_url, allowed_domains (default same-origin), seed_selectors, max_steps, max_pages,
max_runtime_seconds, wait_seconds, rate_limit_seconds, headless, user_agent,
respect_robots (default true), output_format (json/csv), screenshot.
Returns fields include: {status, items, pages_visited, html_captures, screenshots, saved_paths, warnings, steps, goal_summary}.
An LLM-powered planner that iterates scrapes until a goal is satisfied or a safety/stop condition occurs.
Inputs:
- goal — natural-language target (e.g., "Collect quotes text and authors from the first 2 pages").
- start_url — start page (absolute http/https). Same-origin only for all iterations unless allowed_domains is provided.
- seed_selectors — optional initial CSS selector map (defaults to {"title": "title"}).
- max_steps — planner steps (hard-capped at 3 improvements).
- max_pages — pages per scrape pass (default 1, capped by agent config).
- max_runtime_seconds — total runtime budget across steps.
- wait_seconds, output_format, headless, user_agent, respect_robots, rate_limit_seconds, screenshot — mirror scrape_website.
- allowed_domains — optional whitelist; else restricted to same-origin.
Outputs:
- status — ok | no_items | blocked_by_robots_txt | manual_intervention_required | browser_runtime_unavailable | error
- items — accumulated unique items across passes
- pages_visited — union of pages visited across passes
- html_captures — captures from passes
- screenshots — when requested
- saved_paths — includes an iter-plan manifest under outputs/seleniumbase-website-scraper/
- warnings — including planner parsing issues, safety stops, or budget stops
- iterations — number of planner decisions taken
- decisions — array of {iteration, action, reason, confidence, url?, selectors?}
Behavior and guarantees:
- Uses platform-provided ctx.llm with strict system prompt that forbids any evasion of CAPTCHAs, anti-bot, logins, paywalls, or robots.txt.
- Planner can only propose same-origin URLs; cross-origin plans are rejected and end the loop.
- Each pass runs the deterministic scrape_website skill; stops on any challenge or robots block.
- Produces a compact decisions trail and saves a small manifest under outputs/seleniumbase-website-scraper/.
## Selector syntax
Default extraction returns visible text:
```json
{"title": "h1", "summary": ".article-summary"}
```bash
python -m pip install -r requirements.txt
python -m playwright install chromium
python agent.py https://example.com --wait-seconds 8 --max-clicks 4
```
Supported suffixes:
- ::text — visible text.
- ::html — inner HTML.
- ::attr(name) — attribute extraction, e.g. a.card::attr(href).
## Run as A2A
Special pagination selector field names are not emitted as data fields and are used to find the next page:
- next
- _next
- __next__
- pagination_next
- next_page
Pagination is restricted to the same scheme and host as the start URL.
## Example payloads
Simple one-pass scrape:
```json
{
"url": "https://example.com/articles",
"selectors": {
"headline": "h2.article-title",
"links": "a.article-link::attr(href)",
"next": "a[rel='next']::attr(href)"
},
"max_pages": 3,
"wait_seconds": 2,
"output_format": "json",
"headless": true,
"respect_robots": true
}
```bash
a2a dev
a2a test --invoke --skill discover_api_from_browser --args-json '{"url":"https://example.com","wait_seconds":8,"max_clicks":2}'
```
Iterative planner:
Use `origins` to focus on the API host:
```json
{
"goal": "collect at least 30 product tiles with name, price, link",
"url": "https://example.com/store",
"selectors": {"name": ".tile .name", "price": ".tile .price", "href": ".tile a::attr(href)"},
"min_items": 30,
"max_iterations": 3,
"per_pass_max_pages": 1,
"respect_robots": true
}
{"url":"https://app.example.com","origins":["api.example.com"],"min_samples":2}
```
## Smoke test: quotes.toscrape.com
Goal: "Collect quotes text and authors from the first 2 pages"
- Expected: status=ok, >= 1 item, <= 2 iterations (often 1–2), a saved JSON manifest, no screenshots when disabled.
- Example call (schematic):
```json
{
"goal": "Collect quotes text and authors from the first 2 pages",
"url": "https://quotes.toscrape.com/",
"selectors": {"text": ".quote .text", "author": ".quote .author", "pagination_next": "li.next a::attr(href)"},
"min_items": 2,
"max_iterations": 2,
"per_pass_max_pages": 2,
"screenshot": false
}
```
## Deployment notes
The agent declares a larger runtime budget because browser work can be CPU/memory intensive. Actual browser execution is performed through the workspace sandbox helper using the SeleniumBase image, so files written under /workspace/outputs/seleniumbase-website-scraper/ are durable workspace outputs.
The replay skill accepts JSONL request/response captures and runs the same
OpenAPI emitter without opening a browser.

48
a2a.yaml
View File

@@ -1,14 +1,42 @@
# Project identity for `a2a deploy`. Most metadata (resources, scopes,
# secrets, workspace, etc.) lives on the Python class — this file only
# tells the CLI how to find it.
name: seleniumbase-website-scraper
version: 0.1.2
entrypoint: agent:SeleniumbaseWebsiteScraper
version: 0.2.0
entrypoint: agent:BrowserToApiAgent
expose:
public: false
public: true
runtime:
resources:
cpu: "2"
memory: 2Gi
max_runtime_seconds: 600
cpu: 2000m
memory: 4Gi
max_runtime_seconds: 900
egress:
allow_hosts:
- "*"
deny_internet_by_default: false
apt_packages:
- ca-certificates
- fonts-liberation
- libasound2
- libatk-bridge2.0-0
- libatk1.0-0
- libcairo2
- libcups2
- libdbus-1-3
- libdrm2
- libgbm1
- libglib2.0-0
- libgtk-3-0
- libnspr4
- libnss3
- libpango-1.0-0
- libx11-6
- libxcb1
- libxcomposite1
- libxdamage1
- libxext6
- libxfixes3
- libxkbcommon0
- libxrandr2
- xdg-utils
install_commands:
- python -m playwright install chromium

1743
agent.py
View File

File diff suppressed because it is too large Load Diff

4
requirements.txt
View File

@@ -1,3 +1,3 @@
# a2a-pack is auto-installed by the deploy build.
# This agent installs SeleniumBase at runtime inside the sandbox script.
# Leave this file empty to avoid CLI dependency conflicts during card build.
playwright>=1.44
seleniumbase==4.49.10

45
skills/compliant-browser-scraping/SKILL.md
View File

@@ -1,45 +0,0 @@
---
name: compliant-browser-scraping
description: "Safety and workflow guidance for authorized public or user-owned browser scraping with SeleniumBase. Use for website scraping runs, selector extraction, pagination, robots.txt checks, challenge detection, and artifact reporting."
---
# compliant-browser-scraping
# Compliant Browser Scraping
This agent is for authorized scraping of public or user-owned websites only.
## Mandatory safety rules
- Do not bypass, solve, defeat, evade, or work around CAPTCHAs, anti-bot protections, login gates, paywalls, robots.txt restrictions, ToS restrictions, IP blocks, access controls, or technical protection measures.
- If a CAPTCHA, anti-bot challenge, login wall, paywall, access denial, or equivalent challenge is detected, stop the run and return a clear `manual_intervention_required` or `blocked` status with warnings.
- Do not use stealth/undetected-browser modes, CAPTCHA solvers, proxy rotation, credential stuffing, cookie injection, or other circumvention techniques.
- Respect robots.txt when requested by the caller.
- Paginate only within the same scheme+host as the starting URL unless the caller starts a new scrape explicitly for another URL.
- Apply polite rate limiting between page visits.
## Selector mapping conventions
The public `selectors` input is a mapping from output field names to CSS selectors. Examples:
```json
{
"title": "h1",
"price": ".price",
"links": "a.result::attr(href)",
"next": "a[rel='next']::attr(href)"
}
```
Special selector names `next`, `_next`, `__next__`, `pagination_next`, or `next_page` are interpreted as pagination selectors and are not emitted as data fields.
Selector suffixes:
- `::text` extracts visible text (default behavior for element selectors).
- `::html` extracts innerHTML.
- `::attr(name)` extracts an attribute.
When multiple elements match a field, return a list. When one element matches, return a string. When nothing matches, return null.
## Output artifacts
Save machine-readable output under `outputs/seleniumbase-website-scraper/` as JSON or CSV. If screenshots are requested, save PNG screenshots in the same directory. Include warnings and status in the response.

31
skills/llm-iterative-scrape/SKILL.md
View File

@@ -1,31 +0,0 @@
---
name: llm-iterative-scrape
description: "Iterative compliant scraping workflow: plan -> bounded browser step -> evaluate -> refine until goal satisfied or safety/budget stops."
---
# llm-iterative-scrape
Use this skill for iterative, goal-driven scraping on authorized public or user-owned sites.
Loop:
1) Translate the natural-language goal into a minimal set of CSS selectors and a short plan for ONE bounded browser step.
2) Execute exactly one small browser step with a polite wait/rate limit via the host agent's primitives.
3) Evaluate observations vs. the goal. Refine minimally. Stop when satisfied or when any safety/budget condition triggers.
Hard safety constraints:
- Never solve or route around CAPTCHAs/anti-bot.
- Never log in or pass paywalls.
- Restrict pagination to same-origin as the start URL by default; also respect provided allowed_domains.
- When `respect_robots` is true, treat any robots.txt disallow or failure-to-check as a stop condition.
Selectors guidance:
- Use compact CSS selectors. Suffix conventions: `::text`, `::html`, `::attr(name)`.
- Use one pagination field: `pagination_next` or `next` for the next-page `href`.
- Prefer domain-agnostic patterns like `a[rel='next']::attr(href)` or `li.next a::attr(href)`.
Planning output contract:
Return strictly compact JSON only: {"plan_summary": str, "selectors": {str:str}, "continue": bool, "notes": str}. Keep under ~600 chars when possible.
Evaluation guidance:
- If extracted data matches the requested fields and count threshold, set continue=false and summarize.
- If pagination is needed for more pages and a next link exists, keep continue=true.
- On any safety trigger (captcha/login/paywall/robots), set continue=false and summarize the stop reason.

17
smoke_tests/quotes_toscrape.json
View File

@@ -1,17 +0,0 @@
{
"url": "https://quotes.toscrape.com/",
"selectors": {
"quotes": ".quote",
"text": ".text",
"author": ".author",
"tags": ".tag"
},
"max_pages": 1,
"wait_seconds": 1,
"output_format": "json",
"headless": true,
"user_agent": "",
"respect_robots": true,
"rate_limit_seconds": 1,
"screenshot": false
}

46
smoke_tests/test_iterative_quotes_plan.py
View File

@@ -1,46 +0,0 @@
from __future__ import annotations
import inspect
from pathlib import Path
def _load_agent_class():
import importlib.util
agent_path = Path(__file__).resolve().parents[1] / "agent.py"
spec = importlib.util.spec_from_file_location("agent", agent_path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return getattr(module, "SeleniumbaseWebsiteScraper")
def test_iterative_signature_has_required_params() -> None:
Agent = _load_agent_class()
fn = getattr(Agent, "iterative_scrape_goal")
sig = inspect.signature(fn)
params = set(sig.parameters.keys())
# skill method signature includes (self, ctx, ...)
required = {
"ctx",
"goal",
"start_url",
"allowed_domains",
"seed_selectors",
"max_steps",
"max_pages",
"max_runtime_seconds",
"wait_seconds",
"output_format",
"headless",
"user_agent",
"respect_robots",
"rate_limit_seconds",
"screenshot",
}
assert required.issubset(params)
if __name__ == "__main__":
test_iterative_signature_has_required_params()
print("iterative signature smoke ok")

59
smoke_tests/test_render_scraper_script.py
View File

@@ -1,59 +0,0 @@
from __future__ import annotations
import ast
import importlib.util
import re
from pathlib import Path
def _load_agent_module():
agent_path = Path(__file__).resolve().parents[1] / "agent.py"
spec = importlib.util.spec_from_file_location("agent", agent_path)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _render_script() -> str:
module = _load_agent_module()
return module._render_scraper_script(
{
"url": "https://example.com/",
"selectors": {"title": "title"},
"max_pages": 1,
"wait_seconds": 0,
"output_format": "json",
"headless": True,
"user_agent": "SmokeTest",
"respect_robots": True,
"rate_limit_seconds": 0,
"screenshot": False,
"output_dir": "/workspace/outputs/seleniumbase-website-scraper",
}
)
def test_render_scraper_script_parses_as_python_35() -> None:
script = _render_script()
ast.parse(script, feature_version=(3, 5))
def test_render_scraper_script_has_no_forbidden_modern_syntax_markers() -> None:
script = _render_script()
numeric_underscore_literal = re.compile(r"(?<![A-Za-z0-9_])\d+(?:_\d+)+(?![A-Za-z0-9_])")
f_string_prefix = re.compile(r"(?<![A-Za-z0-9_])(?:[fF]|[rR][fF]|[fF][rR])(['\"])")
assert numeric_underscore_literal.findall(script) == []
assert f_string_prefix.findall(script) == []
assert "2_000_000" not in script
assert "response.read(2000000)" in script
assert ":=" not in script
assert "pathlib" not in script
if __name__ == "__main__":
test_render_scraper_script_parses_as_python_35()
test_render_scraper_script_has_no_forbidden_modern_syntax_markers()
print("smoke ok: rendered scraper script parses as Python 3.5 and contains no forbidden markers")