60 lines
1.9 KiB
Python
60 lines
1.9 KiB
Python
from __future__ import annotations
|
|
|
|
import ast
|
|
import importlib.util
|
|
import re
|
|
from pathlib import Path
|
|
|
|
|
|
def _load_agent_module():
|
|
agent_path = Path(__file__).resolve().parents[1] / "agent.py"
|
|
spec = importlib.util.spec_from_file_location("agent", agent_path)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _render_script() -> str:
|
|
module = _load_agent_module()
|
|
return module._render_scraper_script(
|
|
{
|
|
"url": "https://example.com/",
|
|
"selectors": {"title": "title"},
|
|
"max_pages": 1,
|
|
"wait_seconds": 0,
|
|
"output_format": "json",
|
|
"headless": True,
|
|
"user_agent": "SmokeTest",
|
|
"respect_robots": True,
|
|
"rate_limit_seconds": 0,
|
|
"screenshot": False,
|
|
"output_dir": "/workspace/outputs/seleniumbase-website-scraper",
|
|
}
|
|
)
|
|
|
|
|
|
def test_render_scraper_script_parses_as_python_35() -> None:
|
|
script = _render_script()
|
|
ast.parse(script, feature_version=(3, 5))
|
|
|
|
|
|
def test_render_scraper_script_has_no_forbidden_modern_syntax_markers() -> None:
|
|
script = _render_script()
|
|
|
|
numeric_underscore_literal = re.compile(r"(?<![A-Za-z0-9_])\d+(?:_\d+)+(?![A-Za-z0-9_])")
|
|
f_string_prefix = re.compile(r"(?<![A-Za-z0-9_])(?:[fF]|[rR][fF]|[fF][rR])(['\"])")
|
|
|
|
assert numeric_underscore_literal.findall(script) == []
|
|
assert f_string_prefix.findall(script) == []
|
|
assert "2_000_000" not in script
|
|
assert "response.read(2000000)" in script
|
|
assert ":=" not in script
|
|
assert "pathlib" not in script
|
|
|
|
|
|
if __name__ == "__main__":
|
|
test_render_scraper_script_parses_as_python_35()
|
|
test_render_scraper_script_has_no_forbidden_modern_syntax_markers()
|
|
print("smoke ok: rendered scraper script parses as Python 3.5 and contains no forbidden markers")
|