deploy
This commit is contained in:
a2a-platform
@@ -1,12 +0,0 @@
|
|||||||
{
|
|
||||||
"agent": "seleniumbase-website-scraper",
|
|
||||||
"file_count": 9,
|
|
||||||
"last_sandbox": {
|
|
||||||
"exit_code": 0,
|
|
||||||
"source_hash": "800286217c3eb6945d8f9b09404a6da74147243f3a05ed4fb46f11b6c0ff4cf1",
|
|
||||||
"updated_at": 1781005877
|
|
||||||
},
|
|
||||||
"repo_head_sha": "59e28467b7e98c3ef0abe4da070be833ccacb7dd",
|
|
||||||
"source": "repo-sync",
|
|
||||||
"updated_at": 1781005822
|
|
||||||
}
|
|
||||||
351
.a2a/agent.dsl.json
Normal file
351
.a2a/agent.dsl.json
Normal file
@@ -0,0 +1,351 @@
|
|||||||
|
{
|
||||||
|
"schema_version": "2026-06-04",
|
||||||
|
"language": "python",
|
||||||
|
"name": "seleniumbase-website-scraper",
|
||||||
|
"description": "SeleniumBase-style browser agent that turns observed website traffic into OpenAPI specs, coverage reports, samples, and a replay client.",
|
||||||
|
"version": "0.2.0",
|
||||||
|
"entrypoint": {
|
||||||
|
"module": "agent",
|
||||||
|
"class_name": "BrowserToApiAgent",
|
||||||
|
"function": null,
|
||||||
|
"command": []
|
||||||
|
},
|
||||||
|
"skills": [
|
||||||
|
{
|
||||||
|
"name": "discover_api_from_browser",
|
||||||
|
"description": "Open a website in a browser, capture observable HTTP traffic, infer an OpenAPI 3.1 spec, and write a browsable API discovery report.",
|
||||||
|
"handler": "discover_api_from_browser",
|
||||||
|
"tags": [
|
||||||
|
"browser",
|
||||||
|
"seleniumbase",
|
||||||
|
"openapi",
|
||||||
|
"api-discovery"
|
||||||
|
],
|
||||||
|
"scopes": [],
|
||||||
|
"stream": false,
|
||||||
|
"policy": {
|
||||||
|
"timeout_seconds": 600.0,
|
||||||
|
"idempotent": false,
|
||||||
|
"max_retries": 0,
|
||||||
|
"cost_class": null,
|
||||||
|
"allow_scope_expansion": false,
|
||||||
|
"grant_mode": null,
|
||||||
|
"grant_allow_patterns": [],
|
||||||
|
"grant_deny_patterns": [],
|
||||||
|
"grant_outputs_prefix": null,
|
||||||
|
"grant_write_prefixes": [],
|
||||||
|
"grant_ttl_seconds": null,
|
||||||
|
"grant_run_timeout_seconds": null,
|
||||||
|
"grant_approval_timeout_seconds": null,
|
||||||
|
"grant_scope_approval_timeout_seconds": null
|
||||||
|
},
|
||||||
|
"input_schema": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"url": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"title": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"origins": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"include": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"exclude": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"wait_seconds": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"max_clicks": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"search_term": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"min_samples": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"redact": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"url"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
"output_schema": {
|
||||||
|
"additionalProperties": true,
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "discover_api_from_trace",
|
||||||
|
"description": "Replay request/response JSONL traffic into the same OpenAPI 3.1 discovery pipeline used by live browser capture.",
|
||||||
|
"handler": "discover_api_from_trace",
|
||||||
|
"tags": [
|
||||||
|
"trace",
|
||||||
|
"openapi",
|
||||||
|
"api-discovery"
|
||||||
|
],
|
||||||
|
"scopes": [],
|
||||||
|
"stream": false,
|
||||||
|
"policy": {
|
||||||
|
"timeout_seconds": 240.0,
|
||||||
|
"idempotent": false,
|
||||||
|
"max_retries": 0,
|
||||||
|
"cost_class": null,
|
||||||
|
"allow_scope_expansion": false,
|
||||||
|
"grant_mode": null,
|
||||||
|
"grant_allow_patterns": [],
|
||||||
|
"grant_deny_patterns": [],
|
||||||
|
"grant_outputs_prefix": null,
|
||||||
|
"grant_write_prefixes": [],
|
||||||
|
"grant_ttl_seconds": null,
|
||||||
|
"grant_run_timeout_seconds": null,
|
||||||
|
"grant_approval_timeout_seconds": null,
|
||||||
|
"grant_scope_approval_timeout_seconds": null
|
||||||
|
},
|
||||||
|
"input_schema": {
|
||||||
|
"type": "object",
|
||||||
|
"properties": {
|
||||||
|
"requests_jsonl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"responses_jsonl": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"title": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"origins": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"include": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"exclude": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"min_samples": {
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"redact": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"items": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"requests_jsonl"
|
||||||
|
],
|
||||||
|
"additionalProperties": false
|
||||||
|
},
|
||||||
|
"output_schema": {
|
||||||
|
"additionalProperties": true,
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"capabilities": {
|
||||||
|
"browser_to_api": {
|
||||||
|
"capture": "Launches a browser, records XHR/fetch/document traffic with response bodies, then infers OpenAPI.",
|
||||||
|
"offline": "Can replay JSONL request/response captures into the same OpenAPI emitter.",
|
||||||
|
"outputs": [
|
||||||
|
"openapi.json",
|
||||||
|
"openapi.yaml",
|
||||||
|
"report.md",
|
||||||
|
"index.html",
|
||||||
|
"client.mjs",
|
||||||
|
"confidence.json",
|
||||||
|
"samples/*.json"
|
||||||
|
],
|
||||||
|
"noise_filtering": "Drops analytics, static assets, service workers, and obvious bot-defense plumbing by default.",
|
||||||
|
"schema_inference": "Infers request/response JSON schemas, query params, path params, GraphQL dispatches, and auth-shaped headers."
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"input_modes": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"output_modes": [
|
||||||
|
"application/json"
|
||||||
|
],
|
||||||
|
"required_secrets": [],
|
||||||
|
"required_env": [],
|
||||||
|
"consumer_setup": {
|
||||||
|
"fields": []
|
||||||
|
},
|
||||||
|
"runtime": {
|
||||||
|
"lifecycle": "ephemeral",
|
||||||
|
"availability": "on_demand",
|
||||||
|
"state": "none",
|
||||||
|
"sandbox": "microsandbox",
|
||||||
|
"resources": {
|
||||||
|
"cpu": "2000m",
|
||||||
|
"memory": "4Gi",
|
||||||
|
"gpu": 0,
|
||||||
|
"max_runtime_seconds": 900
|
||||||
|
},
|
||||||
|
"concurrency": 1,
|
||||||
|
"egress": {
|
||||||
|
"allow_hosts": [
|
||||||
|
"*"
|
||||||
|
],
|
||||||
|
"allow_internal_services": [],
|
||||||
|
"deny_internet_by_default": false
|
||||||
|
},
|
||||||
|
"tools_used": [
|
||||||
|
"seleniumbase",
|
||||||
|
"playwright",
|
||||||
|
"browser-to-api",
|
||||||
|
"openapi"
|
||||||
|
],
|
||||||
|
"llm_provisioning": "platform",
|
||||||
|
"pricing": {
|
||||||
|
"price_per_call_usd": 0.0,
|
||||||
|
"caller_pays_llm": false,
|
||||||
|
"notes": ""
|
||||||
|
},
|
||||||
|
"wants_cp_jwt": false,
|
||||||
|
"platform_resources": {
|
||||||
|
"memory": null,
|
||||||
|
"databases": []
|
||||||
|
},
|
||||||
|
"endpoints": [],
|
||||||
|
"apt_packages": []
|
||||||
|
},
|
||||||
|
"template_lineage": null,
|
||||||
|
"meta_agent_manifest": null,
|
||||||
|
"state_schema": null,
|
||||||
|
"workspace_access": {
|
||||||
|
"enabled": false,
|
||||||
|
"max_files": 0,
|
||||||
|
"allowed_modes": [],
|
||||||
|
"require_reason": true,
|
||||||
|
"deny_patterns": [],
|
||||||
|
"require_human_approval": false,
|
||||||
|
"max_total_size_bytes": 104857600
|
||||||
|
},
|
||||||
|
"config_schema": {
|
||||||
|
"properties": {},
|
||||||
|
"title": "BrowserToApiConfig",
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"auth": {
|
||||||
|
"model": "a2a_pack.auth.NoAuth",
|
||||||
|
"strategy": "public",
|
||||||
|
"principal_schema": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"description": "Public agent: no caller identity required.",
|
||||||
|
"properties": {},
|
||||||
|
"title": "NoAuth",
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"resolver": null,
|
||||||
|
"required": false
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"source": "python-a2a-pack",
|
||||||
|
"project_manifest": {
|
||||||
|
"name": "seleniumbase-website-scraper",
|
||||||
|
"version": "0.2.0",
|
||||||
|
"entrypoint": "agent:BrowserToApiAgent"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
157
README.md
157
README.md
@@ -1,145 +1,40 @@
|
|||||||
# seleniumbase-website-scraper
|
# seleniumbase-website-scraper
|
||||||
|
|
||||||
Private A2A agent for compliant, authorized website scraping with SeleniumBase/browser automation where available.
|
Browser-to-API discovery agent.
|
||||||
|
|
||||||
## Safety policy
|
This upgrades the original SeleniumBase website scraper idea into a hosted API
|
||||||
|
discovery workflow:
|
||||||
|
|
||||||
Use this agent only for authorized scraping of public or user-owned sites. The agent must not bypass, solve, defeat, or evade CAPTCHAs, anti-bot protections, login gates, paywalls, `robots.txt`, ToS restrictions, or access controls. If a CAPTCHA, anti-bot challenge, login wall, paywall, or access-control indicator is detected, the run stops and returns a manual-intervention/blocked status.
|
- opens a site in a browser
|
||||||
|
- records observable XHR/fetch/document traffic
|
||||||
|
- captures JSON response bodies when the browser can read them
|
||||||
|
- filters analytics/static noise
|
||||||
|
- templates paths and query parameters
|
||||||
|
- splits GraphQL/JSON-RPC style dispatches
|
||||||
|
- infers request and response JSON schemas
|
||||||
|
- emits `openapi.json`, `openapi.yaml`, `report.md`, `index.html`,
|
||||||
|
`client.mjs`, `confidence.json`, and redacted samples
|
||||||
|
|
||||||
The agent does not use stealth/undetected-browser modes, CAPTCHA solvers, proxy rotation, credential injection, or access-control circumvention.
|
## Run Locally
|
||||||
|
|
||||||
## Skills
|
```bash
|
||||||
|
python -m pip install -r requirements.txt
|
||||||
### scrape_website
|
python -m playwright install chromium
|
||||||
|
python agent.py https://example.com --wait-seconds 8 --max-clicks 4
|
||||||
Inputs:
|
|
||||||
- url — absolute http or https URL.
|
|
||||||
- selectors — mapping of output field names to CSS selectors.
|
|
||||||
- max_pages — maximum same-origin pages to visit.
|
|
||||||
- wait_seconds — wait after page open before capture/extraction.
|
|
||||||
- output_format — json or csv.
|
|
||||||
- headless — toggle browser headless mode.
|
|
||||||
- user_agent — optional custom single-line user agent.
|
|
||||||
- respect_robots — when true, checks robots.txt before fetch and pagination.
|
|
||||||
- rate_limit_seconds — polite delay between pagination requests.
|
|
||||||
- screenshot — save PNG screenshots for visited pages.
|
|
||||||
|
|
||||||
Outputs:
|
|
||||||
- status — ok, invalid_input, blocked_by_robots_txt, manual_intervention_required, browser_runtime_unavailable, or error.
|
|
||||||
- items — extracted records.
|
|
||||||
- saved_paths — output files under outputs/seleniumbase-website-scraper/.
|
|
||||||
- warnings — safety, robots, selector, browser, or runtime warnings.
|
|
||||||
- screenshots — screenshot file paths when requested.
|
|
||||||
- pages_visited and html_captures.
|
|
||||||
|
|
||||||
### iterate_scrape_goal (new)
|
|
||||||
|
|
||||||
Parameters:
|
|
||||||
- start_url, allowed_domains (default same-origin), seed_selectors, max_steps, max_pages,
|
|
||||||
max_runtime_seconds, wait_seconds, rate_limit_seconds, headless, user_agent,
|
|
||||||
respect_robots (default true), output_format (json/csv), screenshot.
|
|
||||||
|
|
||||||
Returns fields include: {status, items, pages_visited, html_captures, screenshots, saved_paths, warnings, steps, goal_summary}.
|
|
||||||
|
|
||||||
An LLM-powered planner that iterates scrapes until a goal is satisfied or a safety/stop condition occurs.
|
|
||||||
Inputs:
|
|
||||||
- goal — natural-language target (e.g., "Collect quotes text and authors from the first 2 pages").
|
|
||||||
- start_url — start page (absolute http/https). Same-origin only for all iterations unless allowed_domains is provided.
|
|
||||||
- seed_selectors — optional initial CSS selector map (defaults to {"title": "title"}).
|
|
||||||
- max_steps — planner steps (hard-capped at 3 improvements).
|
|
||||||
- max_pages — pages per scrape pass (default 1, capped by agent config).
|
|
||||||
- max_runtime_seconds — total runtime budget across steps.
|
|
||||||
- wait_seconds, output_format, headless, user_agent, respect_robots, rate_limit_seconds, screenshot — mirror scrape_website.
|
|
||||||
- allowed_domains — optional whitelist; else restricted to same-origin.
|
|
||||||
|
|
||||||
Outputs:
|
|
||||||
- status — ok | no_items | blocked_by_robots_txt | manual_intervention_required | browser_runtime_unavailable | error
|
|
||||||
- items — accumulated unique items across passes
|
|
||||||
- pages_visited — union of pages visited across passes
|
|
||||||
- html_captures — captures from passes
|
|
||||||
- screenshots — when requested
|
|
||||||
- saved_paths — includes an iter-plan manifest under outputs/seleniumbase-website-scraper/
|
|
||||||
- warnings — including planner parsing issues, safety stops, or budget stops
|
|
||||||
- iterations — number of planner decisions taken
|
|
||||||
- decisions — array of {iteration, action, reason, confidence, url?, selectors?}
|
|
||||||
|
|
||||||
Behavior and guarantees:
|
|
||||||
- Uses platform-provided ctx.llm with strict system prompt that forbids any evasion of CAPTCHAs, anti-bot, logins, paywalls, or robots.txt.
|
|
||||||
- Planner can only propose same-origin URLs; cross-origin plans are rejected and end the loop.
|
|
||||||
- Each pass runs the deterministic scrape_website skill; stops on any challenge or robots block.
|
|
||||||
- Produces a compact decisions trail and saves a small manifest under outputs/seleniumbase-website-scraper/.
|
|
||||||
|
|
||||||
## Selector syntax
|
|
||||||
|
|
||||||
Default extraction returns visible text:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{"title": "h1", "summary": ".article-summary"}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Supported suffixes:
|
## Run as A2A
|
||||||
- ::text — visible text.
|
|
||||||
- ::html — inner HTML.
|
|
||||||
- ::attr(name) — attribute extraction, e.g. a.card::attr(href).
|
|
||||||
|
|
||||||
Special pagination selector field names are not emitted as data fields and are used to find the next page:
|
```bash
|
||||||
- next
|
a2a dev
|
||||||
- _next
|
a2a test --invoke --skill discover_api_from_browser --args-json '{"url":"https://example.com","wait_seconds":8,"max_clicks":2}'
|
||||||
- __next__
|
|
||||||
- pagination_next
|
|
||||||
- next_page
|
|
||||||
|
|
||||||
Pagination is restricted to the same scheme and host as the start URL.
|
|
||||||
|
|
||||||
## Example payloads
|
|
||||||
|
|
||||||
Simple one-pass scrape:
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"url": "https://example.com/articles",
|
|
||||||
"selectors": {
|
|
||||||
"headline": "h2.article-title",
|
|
||||||
"links": "a.article-link::attr(href)",
|
|
||||||
"next": "a[rel='next']::attr(href)"
|
|
||||||
},
|
|
||||||
"max_pages": 3,
|
|
||||||
"wait_seconds": 2,
|
|
||||||
"output_format": "json",
|
|
||||||
"headless": true,
|
|
||||||
"respect_robots": true
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Iterative planner:
|
Use `origins` to focus on the API host:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{"url":"https://app.example.com","origins":["api.example.com"],"min_samples":2}
|
||||||
"goal": "collect at least 30 product tiles with name, price, link",
|
|
||||||
"url": "https://example.com/store",
|
|
||||||
"selectors": {"name": ".tile .name", "price": ".tile .price", "href": ".tile a::attr(href)"},
|
|
||||||
"min_items": 30,
|
|
||||||
"max_iterations": 3,
|
|
||||||
"per_pass_max_pages": 1,
|
|
||||||
"respect_robots": true
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Smoke test: quotes.toscrape.com
|
The replay skill accepts JSONL request/response captures and runs the same
|
||||||
|
OpenAPI emitter without opening a browser.
|
||||||
Goal: "Collect quotes text and authors from the first 2 pages"
|
|
||||||
- Expected: status=ok, >= 1 item, <= 2 iterations (often 1–2), a saved JSON manifest, no screenshots when disabled.
|
|
||||||
- Example call (schematic):
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"goal": "Collect quotes text and authors from the first 2 pages",
|
|
||||||
"url": "https://quotes.toscrape.com/",
|
|
||||||
"selectors": {"text": ".quote .text", "author": ".quote .author", "pagination_next": "li.next a::attr(href)"},
|
|
||||||
"min_items": 2,
|
|
||||||
"max_iterations": 2,
|
|
||||||
"per_pass_max_pages": 2,
|
|
||||||
"screenshot": false
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Deployment notes
|
|
||||||
|
|
||||||
The agent declares a larger runtime budget because browser work can be CPU/memory intensive. Actual browser execution is performed through the workspace sandbox helper using the SeleniumBase image, so files written under /workspace/outputs/seleniumbase-website-scraper/ are durable workspace outputs.
|
|
||||||
|
|||||||
48
a2a.yaml
48
a2a.yaml
@@ -1,14 +1,42 @@
|
|||||||
# Project identity for `a2a deploy`. Most metadata (resources, scopes,
|
|
||||||
# secrets, workspace, etc.) lives on the Python class — this file only
|
|
||||||
# tells the CLI how to find it.
|
|
||||||
name: seleniumbase-website-scraper
|
name: seleniumbase-website-scraper
|
||||||
version: 0.1.2
|
version: 0.2.0
|
||||||
entrypoint: agent:SeleniumbaseWebsiteScraper
|
entrypoint: agent:BrowserToApiAgent
|
||||||
expose:
|
expose:
|
||||||
public: false
|
public: true
|
||||||
|
|
||||||
runtime:
|
runtime:
|
||||||
resources:
|
resources:
|
||||||
cpu: "2"
|
cpu: 2000m
|
||||||
memory: 2Gi
|
memory: 4Gi
|
||||||
max_runtime_seconds: 600
|
max_runtime_seconds: 900
|
||||||
|
egress:
|
||||||
|
allow_hosts:
|
||||||
|
- "*"
|
||||||
|
deny_internet_by_default: false
|
||||||
|
apt_packages:
|
||||||
|
- ca-certificates
|
||||||
|
- fonts-liberation
|
||||||
|
- libasound2
|
||||||
|
- libatk-bridge2.0-0
|
||||||
|
- libatk1.0-0
|
||||||
|
- libcairo2
|
||||||
|
- libcups2
|
||||||
|
- libdbus-1-3
|
||||||
|
- libdrm2
|
||||||
|
- libgbm1
|
||||||
|
- libglib2.0-0
|
||||||
|
- libgtk-3-0
|
||||||
|
- libnspr4
|
||||||
|
- libnss3
|
||||||
|
- libpango-1.0-0
|
||||||
|
- libx11-6
|
||||||
|
- libxcb1
|
||||||
|
- libxcomposite1
|
||||||
|
- libxdamage1
|
||||||
|
- libxext6
|
||||||
|
- libxfixes3
|
||||||
|
- libxkbcommon0
|
||||||
|
- libxrandr2
|
||||||
|
- xdg-utils
|
||||||
|
install_commands:
|
||||||
|
- python -m playwright install chromium
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
# a2a-pack is auto-installed by the deploy build.
|
# a2a-pack is auto-installed by the deploy build.
|
||||||
# This agent installs SeleniumBase at runtime inside the sandbox script.
|
playwright>=1.44
|
||||||
# Leave this file empty to avoid CLI dependency conflicts during card build.
|
seleniumbase==4.49.10
|
||||||
|
|||||||
@@ -1,45 +0,0 @@
|
|||||||
---
|
|
||||||
name: compliant-browser-scraping
|
|
||||||
description: "Safety and workflow guidance for authorized public or user-owned browser scraping with SeleniumBase. Use for website scraping runs, selector extraction, pagination, robots.txt checks, challenge detection, and artifact reporting."
|
|
||||||
---
|
|
||||||
# compliant-browser-scraping
|
|
||||||
|
|
||||||
# Compliant Browser Scraping
|
|
||||||
|
|
||||||
This agent is for authorized scraping of public or user-owned websites only.
|
|
||||||
|
|
||||||
## Mandatory safety rules
|
|
||||||
|
|
||||||
- Do not bypass, solve, defeat, evade, or work around CAPTCHAs, anti-bot protections, login gates, paywalls, robots.txt restrictions, ToS restrictions, IP blocks, access controls, or technical protection measures.
|
|
||||||
- If a CAPTCHA, anti-bot challenge, login wall, paywall, access denial, or equivalent challenge is detected, stop the run and return a clear `manual_intervention_required` or `blocked` status with warnings.
|
|
||||||
- Do not use stealth/undetected-browser modes, CAPTCHA solvers, proxy rotation, credential stuffing, cookie injection, or other circumvention techniques.
|
|
||||||
- Respect robots.txt when requested by the caller.
|
|
||||||
- Paginate only within the same scheme+host as the starting URL unless the caller starts a new scrape explicitly for another URL.
|
|
||||||
- Apply polite rate limiting between page visits.
|
|
||||||
|
|
||||||
## Selector mapping conventions
|
|
||||||
|
|
||||||
The public `selectors` input is a mapping from output field names to CSS selectors. Examples:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"title": "h1",
|
|
||||||
"price": ".price",
|
|
||||||
"links": "a.result::attr(href)",
|
|
||||||
"next": "a[rel='next']::attr(href)"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Special selector names `next`, `_next`, `__next__`, `pagination_next`, or `next_page` are interpreted as pagination selectors and are not emitted as data fields.
|
|
||||||
|
|
||||||
Selector suffixes:
|
|
||||||
|
|
||||||
- `::text` extracts visible text (default behavior for element selectors).
|
|
||||||
- `::html` extracts innerHTML.
|
|
||||||
- `::attr(name)` extracts an attribute.
|
|
||||||
|
|
||||||
When multiple elements match a field, return a list. When one element matches, return a string. When nothing matches, return null.
|
|
||||||
|
|
||||||
## Output artifacts
|
|
||||||
|
|
||||||
Save machine-readable output under `outputs/seleniumbase-website-scraper/` as JSON or CSV. If screenshots are requested, save PNG screenshots in the same directory. Include warnings and status in the response.
|
|
||||||
@@ -1,31 +0,0 @@
|
|||||||
---
|
|
||||||
name: llm-iterative-scrape
|
|
||||||
description: "Iterative compliant scraping workflow: plan -> bounded browser step -> evaluate -> refine until goal satisfied or safety/budget stops."
|
|
||||||
---
|
|
||||||
# llm-iterative-scrape
|
|
||||||
|
|
||||||
Use this skill for iterative, goal-driven scraping on authorized public or user-owned sites.
|
|
||||||
|
|
||||||
Loop:
|
|
||||||
1) Translate the natural-language goal into a minimal set of CSS selectors and a short plan for ONE bounded browser step.
|
|
||||||
2) Execute exactly one small browser step with a polite wait/rate limit via the host agent's primitives.
|
|
||||||
3) Evaluate observations vs. the goal. Refine minimally. Stop when satisfied or when any safety/budget condition triggers.
|
|
||||||
|
|
||||||
Hard safety constraints:
|
|
||||||
- Never solve or route around CAPTCHAs/anti-bot.
|
|
||||||
- Never log in or pass paywalls.
|
|
||||||
- Restrict pagination to same-origin as the start URL by default; also respect provided allowed_domains.
|
|
||||||
- When `respect_robots` is true, treat any robots.txt disallow or failure-to-check as a stop condition.
|
|
||||||
|
|
||||||
Selectors guidance:
|
|
||||||
- Use compact CSS selectors. Suffix conventions: `::text`, `::html`, `::attr(name)`.
|
|
||||||
- Use one pagination field: `pagination_next` or `next` for the next-page `href`.
|
|
||||||
- Prefer domain-agnostic patterns like `a[rel='next']::attr(href)` or `li.next a::attr(href)`.
|
|
||||||
|
|
||||||
Planning output contract:
|
|
||||||
Return strictly compact JSON only: {"plan_summary": str, "selectors": {str:str}, "continue": bool, "notes": str}. Keep under ~600 chars when possible.
|
|
||||||
|
|
||||||
Evaluation guidance:
|
|
||||||
- If extracted data matches the requested fields and count threshold, set continue=false and summarize.
|
|
||||||
- If pagination is needed for more pages and a next link exists, keep continue=true.
|
|
||||||
- On any safety trigger (captcha/login/paywall/robots), set continue=false and summarize the stop reason.
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
{
|
|
||||||
"url": "https://quotes.toscrape.com/",
|
|
||||||
"selectors": {
|
|
||||||
"quotes": ".quote",
|
|
||||||
"text": ".text",
|
|
||||||
"author": ".author",
|
|
||||||
"tags": ".tag"
|
|
||||||
},
|
|
||||||
"max_pages": 1,
|
|
||||||
"wait_seconds": 1,
|
|
||||||
"output_format": "json",
|
|
||||||
"headless": true,
|
|
||||||
"user_agent": "",
|
|
||||||
"respect_robots": true,
|
|
||||||
"rate_limit_seconds": 1,
|
|
||||||
"screenshot": false
|
|
||||||
}
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import inspect
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
def _load_agent_class():
|
|
||||||
import importlib.util
|
|
||||||
|
|
||||||
agent_path = Path(__file__).resolve().parents[1] / "agent.py"
|
|
||||||
spec = importlib.util.spec_from_file_location("agent", agent_path)
|
|
||||||
assert spec is not None and spec.loader is not None
|
|
||||||
module = importlib.util.module_from_spec(spec)
|
|
||||||
spec.loader.exec_module(module)
|
|
||||||
return getattr(module, "SeleniumbaseWebsiteScraper")
|
|
||||||
|
|
||||||
|
|
||||||
def test_iterative_signature_has_required_params() -> None:
|
|
||||||
Agent = _load_agent_class()
|
|
||||||
fn = getattr(Agent, "iterative_scrape_goal")
|
|
||||||
sig = inspect.signature(fn)
|
|
||||||
params = set(sig.parameters.keys())
|
|
||||||
# skill method signature includes (self, ctx, ...)
|
|
||||||
required = {
|
|
||||||
"ctx",
|
|
||||||
"goal",
|
|
||||||
"start_url",
|
|
||||||
"allowed_domains",
|
|
||||||
"seed_selectors",
|
|
||||||
"max_steps",
|
|
||||||
"max_pages",
|
|
||||||
"max_runtime_seconds",
|
|
||||||
"wait_seconds",
|
|
||||||
"output_format",
|
|
||||||
"headless",
|
|
||||||
"user_agent",
|
|
||||||
"respect_robots",
|
|
||||||
"rate_limit_seconds",
|
|
||||||
"screenshot",
|
|
||||||
}
|
|
||||||
assert required.issubset(params)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
test_iterative_signature_has_required_params()
|
|
||||||
print("iterative signature smoke ok")
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import ast
|
|
||||||
import importlib.util
|
|
||||||
import re
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
|
|
||||||
def _load_agent_module():
|
|
||||||
agent_path = Path(__file__).resolve().parents[1] / "agent.py"
|
|
||||||
spec = importlib.util.spec_from_file_location("agent", agent_path)
|
|
||||||
assert spec is not None and spec.loader is not None
|
|
||||||
module = importlib.util.module_from_spec(spec)
|
|
||||||
spec.loader.exec_module(module)
|
|
||||||
return module
|
|
||||||
|
|
||||||
|
|
||||||
def _render_script() -> str:
|
|
||||||
module = _load_agent_module()
|
|
||||||
return module._render_scraper_script(
|
|
||||||
{
|
|
||||||
"url": "https://example.com/",
|
|
||||||
"selectors": {"title": "title"},
|
|
||||||
"max_pages": 1,
|
|
||||||
"wait_seconds": 0,
|
|
||||||
"output_format": "json",
|
|
||||||
"headless": True,
|
|
||||||
"user_agent": "SmokeTest",
|
|
||||||
"respect_robots": True,
|
|
||||||
"rate_limit_seconds": 0,
|
|
||||||
"screenshot": False,
|
|
||||||
"output_dir": "/workspace/outputs/seleniumbase-website-scraper",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_render_scraper_script_parses_as_python_35() -> None:
|
|
||||||
script = _render_script()
|
|
||||||
ast.parse(script, feature_version=(3, 5))
|
|
||||||
|
|
||||||
|
|
||||||
def test_render_scraper_script_has_no_forbidden_modern_syntax_markers() -> None:
|
|
||||||
script = _render_script()
|
|
||||||
|
|
||||||
numeric_underscore_literal = re.compile(r"(?<![A-Za-z0-9_])\d+(?:_\d+)+(?![A-Za-z0-9_])")
|
|
||||||
f_string_prefix = re.compile(r"(?<![A-Za-z0-9_])(?:[fF]|[rR][fF]|[fF][rR])(['\"])")
|
|
||||||
|
|
||||||
assert numeric_underscore_literal.findall(script) == []
|
|
||||||
assert f_string_prefix.findall(script) == []
|
|
||||||
assert "2_000_000" not in script
|
|
||||||
assert "response.read(2000000)" in script
|
|
||||||
assert ":=" not in script
|
|
||||||
assert "pathlib" not in script
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
test_render_scraper_script_parses_as_python_35()
|
|
||||||
test_render_scraper_script_has_no_forbidden_modern_syntax_markers()
|
|
||||||
print("smoke ok: rendered scraper script parses as Python 3.5 and contains no forbidden markers")
|
|
||||||
Reference in New Issue
Block a user