write smoke_tests/test_render_scraper_script.py

smoke_tests/test_render_scraper_script.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+import ast
+import importlib.util
+import re
+from pathlib import Path
+
+
+def _load_agent_module():
+    agent_path = Path(__file__).resolve().parents[1] / "agent.py"
+    spec = importlib.util.spec_from_file_location("agent", agent_path)
+    assert spec is not None and spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _render_script() -> str:
+    module = _load_agent_module()
+    return module._render_scraper_script(
+        {
+            "url": "https://example.com/",
+            "selectors": {"title": "title"},
+            "max_pages": 1,
+            "wait_seconds": 0,
+            "output_format": "json",
+            "headless": True,
+            "user_agent": "SmokeTest",
+            "respect_robots": True,
+            "rate_limit_seconds": 0,
+            "screenshot": False,
+            "output_dir": "/workspace/outputs/seleniumbase-website-scraper",
+        }
+    )
+
+
+def test_render_scraper_script_parses_as_python_35() -> None:
+    script = _render_script()
+    ast.parse(script, feature_version=(3, 5))
+
+
+def test_render_scraper_script_has_no_forbidden_modern_syntax_markers() -> None:
+    script = _render_script()
+
+    numeric_underscore_literal = re.compile(r"(?<![A-Za-z0-9_])\d+(?:_\d+)+(?![A-Za-z0-9_])")
+    f_string_prefix = re.compile(r"(?<![A-Za-z0-9_])(?:[fF]|[rR][fF]|[fF][rR])(['\"])")
+
+    assert numeric_underscore_literal.findall(script) == []
+    assert f_string_prefix.findall(script) == []
+    assert "2_000_000" not in script
+    assert "response.read(2000000)" in script
+    assert ":=" not in script
+    assert "pathlib" not in script
+
+
+if __name__ == "__main__":
+    test_render_scraper_script_parses_as_python_35()
+    test_render_scraper_script_has_no_forbidden_modern_syntax_markers()
+    print("smoke ok: rendered scraper script parses as Python 3.5 and contains no forbidden markers")