From aba68aaf2e12b0898ea81163f678e3710a460d89 Mon Sep 17 00:00:00 2001 From: robert Date: Sun, 7 Jun 2026 19:54:03 -0300 Subject: [PATCH] Exclude cache artifacts from builder source packages --- agent.py | 2 +- agent_builder/builder.py | 9 +++- agent_builder/tools.py | 98 ++++++++++++++++++++++++++++--------- requirements.txt | 2 +- tests/test_template_init.py | 73 +++++++++++++++++++++++++++ 5 files changed, 158 insertions(+), 26 deletions(-) diff --git a/agent.py b/agent.py index b548f89..2281c87 100644 --- a/agent.py +++ b/agent.py @@ -49,7 +49,7 @@ class AgentBuilder(A2AAgent[BuilderConfig, NoAuth]): "Writes the project into the user's workspace, validates it in a " "sandbox, then ships it via the control plane." ) - version = "0.1.2" + version = "0.1.3" config_model = BuilderConfig auth_model = NoAuth diff --git a/agent_builder/builder.py b/agent_builder/builder.py index f0269bc..2638321 100644 --- a/agent_builder/builder.py +++ b/agent_builder/builder.py @@ -12,7 +12,7 @@ from deepagents.backends.utils import create_file_data from langgraph.store.memory import InMemoryStore from .config import Settings, load_settings -from .tools import ToolContext, build_tools +from .tools import ToolContext, _should_include_source_path, build_tools BUILDER_SKILL_SOURCE = "/.agent-builder/skills/" @@ -430,6 +430,13 @@ def _workspace_backend_with_grep_compat( """ class BuilderWorkspaceBackend(workspace_backend_cls): # type: ignore[misc, valid-type] + def _all_paths(self) -> list[str]: + return [ + path + for path in super()._all_paths() + if _should_include_source_path(path) + ] + def grep( self, pattern: str, diff --git a/agent_builder/tools.py b/agent_builder/tools.py index cfe336d..7c5f288 100644 --- a/agent_builder/tools.py +++ b/agent_builder/tools.py @@ -12,6 +12,8 @@ import ast import base64 import io import json +import logging +import os import re import tarfile import tempfile @@ -31,7 +33,8 @@ if TYPE_CHECKING: from .config import Settings -A2A_PACK_MIN_VERSION = "0.1.79" +A2A_PACK_MIN_VERSION = "0.1.80" +logger = logging.getLogger(__name__) @dataclass(frozen=True) @@ -213,6 +216,17 @@ _SKILL_NAME_RE = re.compile(r"^[a-z0-9](?:[a-z0-9-]{0,62}[a-z0-9])?$") _BUILDER_STATE_FILE = ".a2a-builder-state.json" _BUILDER_INTERNAL_PREFIX = ".agent-builder/" _DERIVED_PRICING_FIELDS = frozenset({"compute", "total_usd"}) +_EXCLUDED_SOURCE_PARTS = frozenset({ + "__pycache__", + ".git", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + ".venv", + "node_modules", +}) +_EXCLUDED_SOURCE_SUFFIXES = (".pyc", ".pyo", ".pyd") +_EXCLUDED_SOURCE_PART_SUFFIXES = (".egg-info",) def _validate_name(name: str) -> None: @@ -239,6 +253,24 @@ def _builder_state_key(name: str) -> str: return _agent_prefix(name) + _BUILDER_STATE_FILE +def _should_include_source_path(path: str) -> bool: + normalized = path.replace("\\", "/").strip("/") + if not normalized: + return False + parts = normalized.split("/") + if any(part in _EXCLUDED_SOURCE_PARTS for part in parts): + return False + if any(part.endswith(_EXCLUDED_SOURCE_PART_SUFFIXES) for part in parts): + return False + return not normalized.endswith(_EXCLUDED_SOURCE_SUFFIXES) + + +def _should_include_agent_rel_path(rel: str) -> bool: + if rel == _BUILDER_STATE_FILE or rel.startswith(_BUILDER_INTERNAL_PREFIX): + return False + return _should_include_source_path(rel) + + def build_tools(ctx: ToolContext) -> list[Any]: bucket = ctx.bucket settings = ctx.settings @@ -306,9 +338,14 @@ def build_tools(ctx: ToolContext) -> list[Any]: out: list[dict[str, Any]] = [] for key in store.iter_keys(prefix): rel = key[len(prefix):] - if rel == _BUILDER_STATE_FILE or rel.startswith(_BUILDER_INTERNAL_PREFIX): + if not _should_include_agent_rel_path(rel): continue - out.append({"path": rel, "size": len(store.get(key))}) + try: + size = len(store.get(key)) + except FileNotFoundError: + logger.warning("Skipping disappeared source file during list: %s", key) + continue + out.append({"path": rel, "size": size}) return json.dumps({"agent": name, "files": out}) @tool @@ -323,7 +360,7 @@ def build_tools(ctx: ToolContext) -> list[Any]: except ValueError as exc: return json.dumps({"error": str(exc)}) rel = path.lstrip("/") - if rel == _BUILDER_STATE_FILE or rel.startswith(_BUILDER_INTERNAL_PREFIX): + if not _should_include_agent_rel_path(rel): return json.dumps({"error": f"{rel} is managed by agent-builder"}) sanitized_fields: tuple[str, ...] = () if rel == "agent.py": @@ -349,7 +386,7 @@ def build_tools(ctx: ToolContext) -> list[Any]: except ValueError as exc: return json.dumps({"error": str(exc)}) rel = path.lstrip("/") - if rel == _BUILDER_STATE_FILE or rel.startswith(_BUILDER_INTERNAL_PREFIX): + if not _should_include_agent_rel_path(rel): return json.dumps({"error": f"{rel} is managed by agent-builder"}) key = prefix + rel try: @@ -430,6 +467,7 @@ def build_tools(ctx: ToolContext) -> list[Any]: b64 = base64.b64encode(bundle_bytes).decode("ascii") script = ( "set -e\n" + "export PYTHONDONTWRITEBYTECODE=1\n" f"pip install --quiet 'a2a-pack>={A2A_PACK_MIN_VERSION}' >/dev/null\n" "mkdir -p /tmp/agent\n" f"echo '{b64}' | base64 -d | tar -xzf - -C /tmp/agent\n" @@ -949,11 +987,13 @@ def _tarball_workspace_dir( with tarfile.open(fileobj=buf, mode="w:gz") as tf: for key in store.iter_keys(prefix): rel = key[len(prefix):] - if not rel: + if not _should_include_agent_rel_path(rel): continue - if rel == _BUILDER_STATE_FILE or rel.startswith(_BUILDER_INTERNAL_PREFIX): + try: + body = store.get(key) + except FileNotFoundError: + logger.warning("Skipping disappeared source file during tarball: %s", key) continue - body = store.get(key) info = tarfile.TarInfo(name=rel) info.size = len(body) info.mode = 0o644 @@ -1165,7 +1205,7 @@ def _files_from_tarball(bundle: bytes) -> dict[str, bytes]: rel = rel[2:] if not rel or rel.startswith("/") or ".." in rel.split("/"): raise ValueError(f"unsafe member path: {member.name}") - if rel == _BUILDER_STATE_FILE: + if not _should_include_agent_rel_path(rel): continue extracted = tf.extractfile(member) if extracted is None: @@ -1177,6 +1217,7 @@ def _files_from_tarball(bundle: bytes) -> dict[str, bytes]: def _compile_agent_dsl_json(bundle: bytes) -> str: + import sys import yaml from a2a_pack import apply_project_manifest, compile_agent_to_dsl from a2a_pack.cli.loader import load_agent_class @@ -1197,21 +1238,32 @@ def _compile_agent_dsl_json(bundle: bytes) -> str: entrypoint = str(cfg.get("entrypoint") or "").strip() if not entrypoint: raise ValueError("a2a.yaml entrypoint is required") - cls = load_agent_class(entrypoint, project_dir=root) - apply_project_manifest(cls, cfg) - dsl = compile_agent_to_dsl( - cls, - language="python", - entrypoint=entrypoint, - metadata={ - "source": "agent-builder", - "project_manifest": { - key: value - for key, value in cfg.items() - if key in {"name", "version", "entrypoint", "frontend"} + old_dont_write_bytecode = os.environ.get("PYTHONDONTWRITEBYTECODE") + old_sys_dont_write_bytecode = sys.dont_write_bytecode + os.environ["PYTHONDONTWRITEBYTECODE"] = "1" + sys.dont_write_bytecode = True + try: + cls = load_agent_class(entrypoint, project_dir=root) + apply_project_manifest(cls, cfg) + dsl = compile_agent_to_dsl( + cls, + language="python", + entrypoint=entrypoint, + metadata={ + "source": "agent-builder", + "project_manifest": { + key: value + for key, value in cfg.items() + if key in {"name", "version", "entrypoint", "frontend"} + }, }, - }, - ) + ) + finally: + if old_dont_write_bytecode is None: + os.environ.pop("PYTHONDONTWRITEBYTECODE", None) + else: + os.environ["PYTHONDONTWRITEBYTECODE"] = old_dont_write_bytecode + sys.dont_write_bytecode = old_sys_dont_write_bytecode return dsl.model_dump_json() diff --git a/requirements.txt b/requirements.txt index 11a54a9..c38cc50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -a2a-pack>=0.1.79 +a2a-pack>=0.1.80 httpx>=0.27 boto3>=1.34 deepagents>=0.5.0 diff --git a/tests/test_template_init.py b/tests/test_template_init.py index 76c9f8b..6e9d8dd 100644 --- a/tests/test_template_init.py +++ b/tests/test_template_init.py @@ -8,6 +8,7 @@ import unittest from unittest.mock import patch from agent_builder.config import Settings +from agent_builder.builder import _workspace_backend_with_grep_compat from agent_builder.tools import ( _BUILDER_STATE_FILE, _BUILDER_INTERNAL_PREFIX, @@ -160,6 +161,44 @@ class TemplateInitTests(unittest.TestCase): with tarfile.open(fileobj=io.BytesIO(bundle), mode="r:gz") as tf: self.assertEqual(tf.getnames(), ["agent.py"]) + def test_tarball_workspace_excludes_python_cache_artifacts(self) -> None: + prefix = "agents/demo-agent/" + s3 = _FakeS3({ + prefix + "agent.py": b"print('ok')\n", + prefix + "__pycache__/agent.cpython-311.pyc": b"bytecode", + prefix + "helper.pyc": b"bytecode", + prefix + ".pytest_cache/v/cache/nodeids": b"[]", + prefix + "demo_agent.egg-info/PKG-INFO": b"metadata", + }) + + bundle = _tarball_workspace_dir(s3, "bucket", prefix) + + with tarfile.open(fileobj=io.BytesIO(bundle), mode="r:gz") as tf: + self.assertEqual(tf.getnames(), ["agent.py"]) + + def test_tarball_workspace_skips_disappeared_files(self) -> None: + prefix = "agents/demo-agent/" + store = _FakeStaleStore( + objects={prefix + "agent.py": b"print('ok')\n"}, + stale_keys={prefix + "generated.txt"}, + ) + + bundle = _tarball_workspace_dir(store, prefix) + + with tarfile.open(fileobj=io.BytesIO(bundle), mode="r:gz") as tf: + self.assertEqual(tf.getnames(), ["agent.py"]) + + def test_builder_workspace_backend_filters_cache_artifacts(self) -> None: + backend_cls = _workspace_backend_with_grep_compat( + _FakeDeepAgentsBackend, + object(), + ) + + self.assertEqual( + backend_cls._all_paths(), + ["agents/demo-agent/agent.py"], + ) + def test_render_skill_md_creates_valid_frontmatter(self) -> None: text = _render_skill_md( "market-research", @@ -299,6 +338,10 @@ class TemplateInitTests(unittest.TestCase): self.assertEqual(result["exit_code"], 0) self.assertEqual(len(sandbox.calls), 1) self.assertEqual(sandbox.calls[0]["workspace"], "bucket") + self.assertIn( + "export PYTHONDONTWRITEBYTECODE=1", + str(sandbox.calls[0]["script"]), + ) self.assertEqual(_FakeAsyncClient.posts, []) def test_cp_deploy_tarball_posts_agent_dsl(self) -> None: @@ -525,6 +568,36 @@ class _FakeS3: raise AssertionError("batch DeleteObjects should not be used") +class _FakeStaleStore: + def __init__(self, objects: dict[str, bytes], stale_keys: set[str]) -> None: + self.objects = objects + self.stale_keys = stale_keys + + def iter_keys(self, prefix: str) -> list[str]: + return sorted( + key + for key in [*self.objects, *self.stale_keys] + if key.startswith(prefix) + ) + + def get(self, key: str) -> bytes: + if key in self.stale_keys: + raise FileNotFoundError(key) + return self.objects[key] + + +class _FakeDeepAgentsBackend: + def __init__(self, workspace: object, **kwargs: object) -> None: + pass + + def _all_paths(self) -> list[str]: + return [ + "agents/demo-agent/agent.py", + "agents/demo-agent/__pycache__/agent.cpython-311.pyc", + "agents/demo-agent/.pytest_cache/v/cache/nodeids", + ] + + class _FakePaginator: def __init__(self, objects: dict[str, bytes]) -> None: self.objects = objects