deploy
This commit is contained in:
@@ -3,7 +3,7 @@
|
|||||||
"language": "python",
|
"language": "python",
|
||||||
"name": "seleniumbase-website-scraper",
|
"name": "seleniumbase-website-scraper",
|
||||||
"description": "SeleniumBase-style browser agent that turns observed website traffic into OpenAPI specs, coverage reports, samples, and a replay client.",
|
"description": "SeleniumBase-style browser agent that turns observed website traffic into OpenAPI specs, coverage reports, samples, and a replay client.",
|
||||||
"version": "0.2.3",
|
"version": "0.2.4",
|
||||||
"entrypoint": {
|
"entrypoint": {
|
||||||
"module": "agent",
|
"module": "agent",
|
||||||
"class_name": "BrowserToApiAgent",
|
"class_name": "BrowserToApiAgent",
|
||||||
@@ -364,7 +364,7 @@
|
|||||||
"source": "python-a2a-pack",
|
"source": "python-a2a-pack",
|
||||||
"project_manifest": {
|
"project_manifest": {
|
||||||
"name": "seleniumbase-website-scraper",
|
"name": "seleniumbase-website-scraper",
|
||||||
"version": "0.2.3",
|
"version": "0.2.4",
|
||||||
"entrypoint": "agent:BrowserToApiAgent"
|
"entrypoint": "agent:BrowserToApiAgent"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
2
a2a.yaml
2
a2a.yaml
@@ -1,5 +1,5 @@
|
|||||||
name: seleniumbase-website-scraper
|
name: seleniumbase-website-scraper
|
||||||
version: 0.2.3
|
version: 0.2.4
|
||||||
entrypoint: agent:BrowserToApiAgent
|
entrypoint: agent:BrowserToApiAgent
|
||||||
expose:
|
expose:
|
||||||
public: true
|
public: true
|
||||||
|
|||||||
32
agent.py
32
agent.py
@@ -127,7 +127,7 @@ class BrowserToApiAgent(A2AAgent[BrowserToApiConfig, NoAuth]):
|
|||||||
"SeleniumBase-style browser agent that turns observed website traffic "
|
"SeleniumBase-style browser agent that turns observed website traffic "
|
||||||
"into OpenAPI specs, coverage reports, samples, and a replay client."
|
"into OpenAPI specs, coverage reports, samples, and a replay client."
|
||||||
)
|
)
|
||||||
version = "0.2.3"
|
version = "0.2.4"
|
||||||
|
|
||||||
config_model = BrowserToApiConfig
|
config_model = BrowserToApiConfig
|
||||||
auth_model = NoAuth
|
auth_model = NoAuth
|
||||||
@@ -410,12 +410,8 @@ def seleniumbase_stealth_prep(
|
|||||||
|
|
||||||
if captcha_policy == "solve_if_allowed":
|
if captcha_policy == "solve_if_allowed":
|
||||||
if captcha_allowed:
|
if captcha_allowed:
|
||||||
try:
|
log.extend(_attempt_allowed_captcha_solve(sb))
|
||||||
sb.solve_captcha()
|
_seleniumbase_sleep(sb, 2)
|
||||||
log.append("seleniumbase: solve_captcha attempted for allowed domain")
|
|
||||||
_seleniumbase_sleep(sb, 2)
|
|
||||||
except Exception as exc: # noqa: BLE001
|
|
||||||
log.append(f"seleniumbase warning: solve_captcha failed: {type(exc).__name__}: {exc}")
|
|
||||||
else:
|
else:
|
||||||
host = urlparse(url).netloc
|
host = urlparse(url).netloc
|
||||||
log.append(f"seleniumbase: captcha solve skipped; host not allowlisted: {host}")
|
log.append(f"seleniumbase: captcha solve skipped; host not allowlisted: {host}")
|
||||||
@@ -435,6 +431,23 @@ def seleniumbase_stealth_prep(
|
|||||||
return cookies, log
|
return cookies, log
|
||||||
|
|
||||||
|
|
||||||
|
def _attempt_allowed_captcha_solve(sb: Any) -> list[str]:
|
||||||
|
log: list[str] = []
|
||||||
|
for method_name in ("solve_captcha", "uc_gui_click_captcha", "uc_gui_handle_captcha"):
|
||||||
|
method = getattr(sb, method_name, None)
|
||||||
|
if method is None:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
method()
|
||||||
|
log.append(f"seleniumbase: {method_name} attempted for allowed domain")
|
||||||
|
return log
|
||||||
|
except Exception as exc: # noqa: BLE001
|
||||||
|
log.append(f"seleniumbase warning: {method_name} failed: {type(exc).__name__}: {exc}")
|
||||||
|
return log
|
||||||
|
log.append("seleniumbase warning: no supported CAPTCHA helper found")
|
||||||
|
return log
|
||||||
|
|
||||||
|
|
||||||
def _seleniumbase_options() -> dict[str, Any]:
|
def _seleniumbase_options() -> dict[str, Any]:
|
||||||
options: dict[str, Any] = {
|
options: dict[str, Any] = {
|
||||||
"uc": True,
|
"uc": True,
|
||||||
@@ -1055,7 +1068,7 @@ def _result_payload(
|
|||||||
captcha_policy: str | None = None,
|
captcha_policy: str | None = None,
|
||||||
allowed_captcha_domains: list[str] | None = None,
|
allowed_captcha_domains: list[str] | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
bot_signals = _bot_defense_signals(samples, capture_log or [])
|
bot_signals = _bot_defense_signals(samples)
|
||||||
payload: dict[str, Any] = {
|
payload: dict[str, Any] = {
|
||||||
"captured_samples": len(samples),
|
"captured_samples": len(samples),
|
||||||
"included_samples": bundle["summary"]["included_samples"],
|
"included_samples": bundle["summary"]["included_samples"],
|
||||||
@@ -1085,7 +1098,7 @@ def _result_payload(
|
|||||||
return payload
|
return payload
|
||||||
|
|
||||||
|
|
||||||
def _bot_defense_signals(samples: list[TrafficSample], capture_log: list[str]) -> list[str]:
|
def _bot_defense_signals(samples: list[TrafficSample]) -> list[str]:
|
||||||
checks = {
|
checks = {
|
||||||
"captcha": (r"captcha",),
|
"captcha": (r"captcha",),
|
||||||
"perimeterx": (r"perimeterx", r"/px/", r"_px", r"px-captcha"),
|
"perimeterx": (r"perimeterx", r"/px/", r"_px", r"px-captcha"),
|
||||||
@@ -1097,7 +1110,6 @@ def _bot_defense_signals(samples: list[TrafficSample], capture_log: list[str]) -
|
|||||||
for sample in samples:
|
for sample in samples:
|
||||||
haystacks.extend(str(key) for key in sample.request_headers)
|
haystacks.extend(str(key) for key in sample.request_headers)
|
||||||
haystacks.extend(str(key) for key in sample.response_headers)
|
haystacks.extend(str(key) for key in sample.response_headers)
|
||||||
haystacks.extend(capture_log)
|
|
||||||
found: list[str] = []
|
found: list[str] = []
|
||||||
for name, patterns in checks.items():
|
for name, patterns in checks.items():
|
||||||
if any(_regex_search(pattern, haystack) for pattern in patterns for haystack in haystacks):
|
if any(_regex_search(pattern, haystack) for pattern in patterns for haystack in haystacks):
|
||||||
|
|||||||
Reference in New Issue
Block a user