deploy
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
"language": "python",
|
||||
"name": "seleniumbase-website-scraper",
|
||||
"description": "SeleniumBase-style browser agent that turns observed website traffic into OpenAPI specs, coverage reports, samples, and a replay client.",
|
||||
"version": "0.2.3",
|
||||
"version": "0.2.4",
|
||||
"entrypoint": {
|
||||
"module": "agent",
|
||||
"class_name": "BrowserToApiAgent",
|
||||
@@ -364,7 +364,7 @@
|
||||
"source": "python-a2a-pack",
|
||||
"project_manifest": {
|
||||
"name": "seleniumbase-website-scraper",
|
||||
"version": "0.2.3",
|
||||
"version": "0.2.4",
|
||||
"entrypoint": "agent:BrowserToApiAgent"
|
||||
}
|
||||
}
|
||||
|
||||
2
a2a.yaml
2
a2a.yaml
@@ -1,5 +1,5 @@
|
||||
name: seleniumbase-website-scraper
|
||||
version: 0.2.3
|
||||
version: 0.2.4
|
||||
entrypoint: agent:BrowserToApiAgent
|
||||
expose:
|
||||
public: true
|
||||
|
||||
30
agent.py
30
agent.py
@@ -127,7 +127,7 @@ class BrowserToApiAgent(A2AAgent[BrowserToApiConfig, NoAuth]):
|
||||
"SeleniumBase-style browser agent that turns observed website traffic "
|
||||
"into OpenAPI specs, coverage reports, samples, and a replay client."
|
||||
)
|
||||
version = "0.2.3"
|
||||
version = "0.2.4"
|
||||
|
||||
config_model = BrowserToApiConfig
|
||||
auth_model = NoAuth
|
||||
@@ -410,12 +410,8 @@ def seleniumbase_stealth_prep(
|
||||
|
||||
if captcha_policy == "solve_if_allowed":
|
||||
if captcha_allowed:
|
||||
try:
|
||||
sb.solve_captcha()
|
||||
log.append("seleniumbase: solve_captcha attempted for allowed domain")
|
||||
log.extend(_attempt_allowed_captcha_solve(sb))
|
||||
_seleniumbase_sleep(sb, 2)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.append(f"seleniumbase warning: solve_captcha failed: {type(exc).__name__}: {exc}")
|
||||
else:
|
||||
host = urlparse(url).netloc
|
||||
log.append(f"seleniumbase: captcha solve skipped; host not allowlisted: {host}")
|
||||
@@ -435,6 +431,23 @@ def seleniumbase_stealth_prep(
|
||||
return cookies, log
|
||||
|
||||
|
||||
def _attempt_allowed_captcha_solve(sb: Any) -> list[str]:
|
||||
log: list[str] = []
|
||||
for method_name in ("solve_captcha", "uc_gui_click_captcha", "uc_gui_handle_captcha"):
|
||||
method = getattr(sb, method_name, None)
|
||||
if method is None:
|
||||
continue
|
||||
try:
|
||||
method()
|
||||
log.append(f"seleniumbase: {method_name} attempted for allowed domain")
|
||||
return log
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.append(f"seleniumbase warning: {method_name} failed: {type(exc).__name__}: {exc}")
|
||||
return log
|
||||
log.append("seleniumbase warning: no supported CAPTCHA helper found")
|
||||
return log
|
||||
|
||||
|
||||
def _seleniumbase_options() -> dict[str, Any]:
|
||||
options: dict[str, Any] = {
|
||||
"uc": True,
|
||||
@@ -1055,7 +1068,7 @@ def _result_payload(
|
||||
captcha_policy: str | None = None,
|
||||
allowed_captcha_domains: list[str] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
bot_signals = _bot_defense_signals(samples, capture_log or [])
|
||||
bot_signals = _bot_defense_signals(samples)
|
||||
payload: dict[str, Any] = {
|
||||
"captured_samples": len(samples),
|
||||
"included_samples": bundle["summary"]["included_samples"],
|
||||
@@ -1085,7 +1098,7 @@ def _result_payload(
|
||||
return payload
|
||||
|
||||
|
||||
def _bot_defense_signals(samples: list[TrafficSample], capture_log: list[str]) -> list[str]:
|
||||
def _bot_defense_signals(samples: list[TrafficSample]) -> list[str]:
|
||||
checks = {
|
||||
"captcha": (r"captcha",),
|
||||
"perimeterx": (r"perimeterx", r"/px/", r"_px", r"px-captcha"),
|
||||
@@ -1097,7 +1110,6 @@ def _bot_defense_signals(samples: list[TrafficSample], capture_log: list[str]) -
|
||||
for sample in samples:
|
||||
haystacks.extend(str(key) for key in sample.request_headers)
|
||||
haystacks.extend(str(key) for key in sample.response_headers)
|
||||
haystacks.extend(capture_log)
|
||||
found: list[str] = []
|
||||
for name, patterns in checks.items():
|
||||
if any(_regex_search(pattern, haystack) for pattern in patterns for haystack in haystacks):
|
||||
|
||||
Reference in New Issue
Block a user