259 lines
11 KiB
Python
259 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
import unittest
|
|
|
|
from agent_builder.builder import (
|
|
BUILDER_SKILL_SOURCE,
|
|
BuilderContext,
|
|
SYSTEM_PROMPT,
|
|
_build_project_backend,
|
|
_builder_skill_files,
|
|
_with_builder_skills,
|
|
_workspace_backend_with_grep_compat,
|
|
)
|
|
from agent_builder.config import Settings
|
|
from agent_builder.tools import A2A_PACK_MIN_VERSION, _strip_unsupported_pricing_fields
|
|
from deepagents.backends import StateBackend
|
|
from agent import _build_failure_error
|
|
|
|
|
|
class BuilderPromptTests(unittest.TestCase):
|
|
def test_outer_agent_uses_500_recursion_limit(self) -> None:
|
|
agent_source = Path(__file__).resolve().parents[1].joinpath("agent.py").read_text()
|
|
|
|
self.assertIn("DEEPAGENTS_RECURSION_LIMIT = 500", agent_source)
|
|
self.assertIn(
|
|
"config={\"recursion_limit\": DEEPAGENTS_RECURSION_LIMIT}",
|
|
agent_source,
|
|
)
|
|
self.assertIn("stream=True", agent_source)
|
|
self.assertIn('grant_write_prefixes=("agents/{name}/",)', agent_source)
|
|
|
|
def test_outer_builder_accepts_user_creds_or_platform_llm_grants(self) -> None:
|
|
agent_source = Path(__file__).resolve().parents[1].joinpath("agent.py").read_text()
|
|
|
|
self.assertIn(
|
|
"llm_provisioning = LLMProvisioning.PLATFORM_OR_CALLER_PROVIDED",
|
|
agent_source,
|
|
)
|
|
self.assertIn("caller_pays_llm=False", agent_source)
|
|
self.assertIn("platform-scoped LiteLLM grant", agent_source)
|
|
self.assertIn("caller-selected LLM credentials", agent_source)
|
|
|
|
def test_builder_defaults_raise_timeout_budget(self) -> None:
|
|
config_source = Path(__file__).resolve().parents[1].joinpath(
|
|
"agent_builder/config.py"
|
|
).read_text()
|
|
|
|
self.assertIn('os.environ.get("A2A_SANDBOX_TIMEOUT_S")', config_source)
|
|
self.assertIn('os.environ.get("SANDBOX_TIMEOUT_S", "1200")', config_source)
|
|
self.assertIn('deploy_wait_timeout_s=int(os.environ.get("DEPLOY_WAIT_TIMEOUT_S", "900"))', config_source)
|
|
|
|
def test_builder_requires_frontend_capable_a2a_pack(self) -> None:
|
|
root = Path(__file__).resolve().parents[1]
|
|
requirements = root.joinpath("requirements.txt").read_text()
|
|
tools_source = root.joinpath("agent_builder/tools.py").read_text()
|
|
|
|
self.assertIn(f"a2a-pack>={A2A_PACK_MIN_VERSION}", requirements)
|
|
self.assertIn("a2a-pack>={A2A_PACK_MIN_VERSION}", tools_source)
|
|
|
|
def test_prompt_defaults_generated_agents_to_platform_llm_grants(self) -> None:
|
|
self.assertIn("LLMProvisioning.PLATFORM", SYSTEM_PROMPT)
|
|
self.assertIn("caller_pays_llm=False", SYSTEM_PROMPT)
|
|
self.assertIn("main-agent handoffs mint a scoped A2A LiteLLM grant", SYSTEM_PROMPT)
|
|
self.assertIn("ctx.llm.api_key", SYSTEM_PROMPT)
|
|
|
|
files = _builder_skill_files()
|
|
self.assertIn("LLMProvisioning.PLATFORM", files["a2apack-agent-authoring/SKILL.md"])
|
|
self.assertIn("caller_pays_llm=False", files["a2apack-agent-authoring/SKILL.md"])
|
|
self.assertIn("ctx.llm.api_key", files["agent-quality-review/SKILL.md"])
|
|
|
|
def test_prompt_rejects_platform_derived_pricing_fields(self) -> None:
|
|
self.assertIn("runtime.pricing.compute", SYSTEM_PROMPT)
|
|
self.assertIn("runtime.pricing.total_usd", SYSTEM_PROMPT)
|
|
self.assertIn("price_per_call_usd", SYSTEM_PROMPT)
|
|
|
|
files = _builder_skill_files()
|
|
for path in (
|
|
"a2apack-agent-authoring/SKILL.md",
|
|
"agent-quality-review/SKILL.md",
|
|
):
|
|
self.assertIn("runtime.pricing.compute", files[path])
|
|
self.assertIn("runtime.pricing.total_usd", files[path])
|
|
self.assertIn("compute=", files[path])
|
|
self.assertIn("total_usd=", files[path])
|
|
|
|
def test_pricing_sanitizer_removes_derived_agent_card_fields(self) -> None:
|
|
source = '''
|
|
from a2a_pack import Pricing
|
|
|
|
pricing = Pricing(
|
|
price_per_call_usd=0.05,
|
|
compute={
|
|
"cpu_usd": 0.000833,
|
|
"memory_usd": 0.00018,
|
|
},
|
|
total_usd=0.10125,
|
|
caller_pays_llm=False,
|
|
notes="caller gets the real billing projection from the platform",
|
|
)
|
|
|
|
inline = Pricing(price_per_call_usd=0.01, compute={"runtime_seconds": 600}, total_usd=0.02, caller_pays_llm=False)
|
|
'''
|
|
|
|
sanitized, removed = _strip_unsupported_pricing_fields(source)
|
|
|
|
self.assertEqual(set(removed), {"compute", "total_usd"})
|
|
self.assertIn("price_per_call_usd=0.05", sanitized)
|
|
self.assertIn("caller_pays_llm=False", sanitized)
|
|
self.assertNotIn("compute=", sanitized)
|
|
self.assertNotIn("total_usd=", sanitized)
|
|
compile(sanitized, "agent.py", "exec")
|
|
|
|
def test_prompt_requires_resource_mirror_for_heavy_agents(self) -> None:
|
|
self.assertIn("declare resources in BOTH places", SYSTEM_PROMPT)
|
|
self.assertIn("resources = Resources", SYSTEM_PROMPT)
|
|
self.assertIn("runtime:", SYSTEM_PROMPT)
|
|
self.assertIn("max_runtime_seconds", SYSTEM_PROMPT)
|
|
|
|
def test_prompt_requires_live_schema_check_and_manim_contract(self) -> None:
|
|
self.assertIn("live_skills[].input_schema", SYSTEM_PROMPT)
|
|
self.assertIn("class GeneratedDeck(Slide)", SYSTEM_PROMPT)
|
|
self.assertIn("manim-slides render --CE -ql", SYSTEM_PROMPT)
|
|
self.assertIn("render: bool = False", SYSTEM_PROMPT)
|
|
self.assertIn('config={"recursion_limit": 500}', SYSTEM_PROMPT)
|
|
|
|
def test_prompt_requires_workspace_mounted_execution_for_outputs(self) -> None:
|
|
self.assertIn("ctx.workspace_shell", SYSTEM_PROMPT)
|
|
self.assertIn("ctx.workspace_python", SYSTEM_PROMPT)
|
|
self.assertIn("outputs/rootfs-captures/...", SYSTEM_PROMPT)
|
|
self.assertIn("not workspace-mounted or rootfs-captured", SYSTEM_PROMPT)
|
|
|
|
def test_prompt_knows_packed_frontend_contract(self) -> None:
|
|
self.assertIn('init_agent_template(..., frontend="react")', SYSTEM_PROMPT)
|
|
self.assertIn("frontend:", SYSTEM_PROMPT)
|
|
self.assertIn("mount: /app", SYSTEM_PROMPT)
|
|
self.assertIn("/app/config.json", SYSTEM_PROMPT)
|
|
self.assertIn("/app/a2a-client.js", SYSTEM_PROMPT)
|
|
self.assertIn("https://docs.a2acloud.io/concepts/packed-frontends", SYSTEM_PROMPT)
|
|
self.assertIn("frontend/src/App.jsx", SYSTEM_PROMPT)
|
|
|
|
def test_prompt_and_backend_load_packaged_builder_skills(self) -> None:
|
|
expected = {
|
|
"deepagent-agent-design",
|
|
"a2apack-agent-authoring",
|
|
"deepagents-implementation-patterns",
|
|
"workspace-artifact-safety",
|
|
"agent-quality-review",
|
|
}
|
|
for skill_name in expected:
|
|
self.assertIn(skill_name, SYSTEM_PROMPT)
|
|
self.assertIn("deepagent-agent-design", SYSTEM_PROMPT)
|
|
self.assertIn("write_agent_skill", SYSTEM_PROMPT)
|
|
self.assertIn("workspace_access = WorkspaceAccess.dynamic", SYSTEM_PROMPT)
|
|
self.assertIn("ensure_read/ensure_write", SYSTEM_PROMPT)
|
|
|
|
files = _builder_skill_files()
|
|
self.assertTrue(
|
|
{path.split("/", 1)[0] for path in files}.issuperset(expected)
|
|
)
|
|
self.assertIn("skills=skill_sources or None", files["deepagent-agent-design/SKILL.md"])
|
|
self.assertIn("from a2a_pack import", files["a2apack-agent-authoring/SKILL.md"])
|
|
self.assertIn("ctx.workspace_shell", files["a2apack-agent-authoring/SKILL.md"])
|
|
self.assertIn("create_deep_agent", files["deepagents-implementation-patterns/SKILL.md"])
|
|
self.assertIn('config={"recursion_limit": 500}', files["deepagents-implementation-patterns/SKILL.md"])
|
|
self.assertIn("ctx.write_artifact", files["workspace-artifact-safety/SKILL.md"])
|
|
self.assertIn("ctx.workspace_shell", files["workspace-artifact-safety/SKILL.md"])
|
|
self.assertIn("write_prefixes", files["workspace-artifact-safety/SKILL.md"])
|
|
self.assertIn("outputs/rootfs-captures/...", files["workspace-artifact-safety/SKILL.md"])
|
|
self.assertIn("live_skills[].input_schema", files["agent-quality-review/SKILL.md"])
|
|
self.assertIn("ctx.workspace_python", files["agent-quality-review/SKILL.md"])
|
|
|
|
backend = _with_builder_skills(StateBackend())
|
|
listing = backend.ls(BUILDER_SKILL_SOURCE)
|
|
paths = [entry["path"] for entry in (listing.entries or [])]
|
|
for skill_name in expected:
|
|
self.assertIn(f"{BUILDER_SKILL_SOURCE}{skill_name}/", paths)
|
|
|
|
def test_build_failure_error_classifies_budget_exhaustion(self) -> None:
|
|
self.assertEqual(
|
|
_build_failure_error(RuntimeError("Budget has been exceeded! Key=abc")),
|
|
"LLM budget exceeded before deployment; retry with a larger budget "
|
|
"or a smaller build scope",
|
|
)
|
|
|
|
def test_project_backend_keeps_deepagents_artifacts_inside_project_grant(self) -> None:
|
|
backend = _build_project_backend(
|
|
BuilderContext(
|
|
bucket="user-1-files",
|
|
project_prefix="agents/svgwrite-agent",
|
|
workspace=object(),
|
|
),
|
|
settings=Settings(
|
|
sandbox_url="http://sandbox",
|
|
sandbox_timeout_s=1200,
|
|
sandbox_token=None,
|
|
deploy_wait_timeout_s=900,
|
|
litellm_url="http://litellm",
|
|
litellm_key="",
|
|
litellm_model="gpt-5.5",
|
|
cp_url="http://control-plane",
|
|
minio_endpoint="http://minio",
|
|
minio_access_key="",
|
|
minio_secret_key="",
|
|
image="python:3.11-slim",
|
|
),
|
|
)
|
|
|
|
self.assertEqual(
|
|
backend.artifacts_root,
|
|
"/agents/svgwrite-agent/.agent-builder",
|
|
)
|
|
|
|
def test_project_backend_grep_compat_qualifies_relative_glob(self) -> None:
|
|
class _LegacyBackend:
|
|
def __init__(self, workspace: object) -> None:
|
|
self.workspace = workspace
|
|
self.calls: list[tuple[str | None, str | None]] = []
|
|
|
|
def _norm(self, path: str | None) -> str:
|
|
return (path or "").strip("/")
|
|
|
|
def grep(
|
|
self,
|
|
pattern: str,
|
|
path: str | None = None,
|
|
glob: str | None = None,
|
|
):
|
|
self.calls.append((path, glob))
|
|
matches = (
|
|
[{"path": "/agents/github-repo-inspector/agent.py"}]
|
|
if glob == "agents/github-repo-inspector/agent.py"
|
|
else []
|
|
)
|
|
return type("GrepResult", (), {"error": None, "matches": matches})()
|
|
|
|
backend = _workspace_backend_with_grep_compat(_LegacyBackend, object())
|
|
|
|
result = backend.grep(
|
|
"_github_client",
|
|
path="/agents/github-repo-inspector",
|
|
glob="agent.py",
|
|
)
|
|
|
|
self.assertEqual(
|
|
backend.calls,
|
|
[
|
|
("/agents/github-repo-inspector", "agent.py"),
|
|
(
|
|
"/agents/github-repo-inspector",
|
|
"agents/github-repo-inspector/agent.py",
|
|
),
|
|
],
|
|
)
|
|
self.assertEqual(
|
|
result.matches,
|
|
[{"path": "/agents/github-repo-inspector/agent.py"}],
|
|
)
|