diff --git a/examples/tutorials/00_sync/080_lineage/.dockerignore b/examples/tutorials/00_sync/080_lineage/.dockerignore new file mode 100644 index 000000000..c49489471 --- /dev/null +++ b/examples/tutorials/00_sync/080_lineage/.dockerignore @@ -0,0 +1,43 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# Environments +.env** +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# IDE +.idea/ +.vscode/ +*.swp +*.swo + +# Git +.git +.gitignore + +# Misc +.DS_Store diff --git a/examples/tutorials/00_sync/080_lineage/Dockerfile b/examples/tutorials/00_sync/080_lineage/Dockerfile new file mode 100644 index 000000000..a15b01f6b --- /dev/null +++ b/examples/tutorials/00_sync/080_lineage/Dockerfile @@ -0,0 +1,55 @@ +# syntax=docker/dockerfile:1.3 +FROM python:3.12-slim +COPY --from=ghcr.io/astral-sh/uv:0.6.4 /uv /uvx /bin/ + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + htop \ + vim \ + curl \ + tar \ + python3-dev \ + postgresql-client \ + build-essential \ + libpq-dev \ + gcc \ + cmake \ + netcat-openbsd \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN uv pip install --system --upgrade pip setuptools wheel + +ENV UV_HTTP_TIMEOUT=1000 + +# Copy pyproject.toml and README.md to install dependencies +COPY 00_sync/080_lineage/pyproject.toml /app/080_lineage/pyproject.toml +COPY 00_sync/080_lineage/README.md /app/080_lineage/README.md + +WORKDIR /app/080_lineage + +# Copy the project code +COPY 00_sync/080_lineage/project /app/080_lineage/project + +# Copy the test files +COPY 00_sync/080_lineage/tests /app/080_lineage/tests + +# Copy shared test utilities +COPY test_utils /app/test_utils + +# Install the required Python packages with dev dependencies +RUN uv pip install --system .[dev] + +# Set environment variables +ENV PYTHONPATH=/app + +# Set test environment variables +ENV AGENT_NAME=s080-lineage + +# The agent build version stamped onto every span as __agent_version__. +# Pass the image tag or git SHA at build time; unlabeled builds stay "unversioned". +ARG AGENT_VERSION=unversioned +ENV AGENT_VERSION=${AGENT_VERSION} + +# Run the agent using uvicorn +CMD ["uvicorn", "project.acp:acp", "--host", "0.0.0.0", "--port", "8000"] diff --git a/examples/tutorials/00_sync/080_lineage/README.md b/examples/tutorials/00_sync/080_lineage/README.md new file mode 100644 index 000000000..28cf0a092 --- /dev/null +++ b/examples/tutorials/00_sync/080_lineage/README.md @@ -0,0 +1,62 @@ +# Lineage provenance on agent traces + +A sync agent (same harness surface as `050_openai_agents`) whose spans carry +the provenance that SGP Lineage derives graph edges from: which data sources +each tool call read, and which agent build produced the trace. + +## What this demonstrates + +Two span-metadata conventions, both capture-only — nothing here emits to the +lineage service; edges are derived later from materialized traces +([SGP-6513 convention spec](https://github.com/scaleapi/scaleapi/blob/master/packages/sgp-lineage/docs/specs/2026-07-22-sgp-6513-trace-data-source-ref-convention.md)): + +- **`sgp.lineage.refs`** — each tool declares the data sources it reads, via + the three capture forms in `agentex.lib.adk.lineage`: + + ```python + @function_tool + @data_sources(DataSourceRef("elasticsearch://research-cluster", "filings-v1")) + def search_filings(query: str) -> str: ... # static refs + + @function_tool + @data_sources(resolver=_kpi_refs) + def read_kpi(table: str) -> str: ... # refs derived from arguments + + lineage.register_tool_sources( # tools you don't own (MCP) + "company_profile", [DataSourceRef("mcp://research-mcp", "company-profiles")] + ) + ``` + + The harness resolves these on every tool span and merges them into + `span.data["sgp.lineage.refs"]`. + +- **`__agent_version__`** — the SGP tracing processor stamps the + `AGENT_VERSION` env var onto every span. The Dockerfile sets it from a + build arg (pass the image tag or git SHA; unlabeled builds fall back to + the explicit sentinel `unversioned` rather than a fabricated version). + For `agentex agents run`, export it or put it in this directory's `.env`. + +Ref namespaces must use the canonical URI forms from the lineage identifier +conventions (`packages/sgp-lineage/docs/event-contract.md` in `scaleapi`); +malformed namespaces raise at import time. + +## Run it + +```bash +agentex agents run --manifest manifest.yaml +``` + +Ask it to "research ACME Corp" — the instructions route through all three +tools. With `SGP_API_KEY` / `SGP_ACCOUNT_ID` / `SGP_CLIENT_BASE_URL` set, the +resulting trace's tool spans show `sgp.lineage.refs` (and every span +`__agent_version__`) in their metadata, filterable in the SGP traces UI and +spans-search `extra_metadata` DSL. + +## Test it + +The offline test verifies all three capture forms resolve without a server or +API key: + +```bash +pytest tests/test_agent.py -v +``` diff --git a/examples/tutorials/00_sync/080_lineage/manifest.yaml b/examples/tutorials/00_sync/080_lineage/manifest.yaml new file mode 100644 index 000000000..5720eb6e6 --- /dev/null +++ b/examples/tutorials/00_sync/080_lineage/manifest.yaml @@ -0,0 +1,58 @@ +build: + context: + root: ../../ + include_paths: + - 00_sync/080_lineage + - test_utils + dockerfile: 00_sync/080_lineage/Dockerfile + dockerignore: 00_sync/080_lineage/.dockerignore + +local_development: + agent: + port: 8000 + host_address: host.docker.internal + paths: + acp: project/acp.py + +agent: + acp_type: sync + name: s080-lineage + description: A sync agent whose tool calls carry lineage data-source refs and an agent version + + temporal: + enabled: false + + credentials: + - env_var_name: OPENAI_API_KEY + secret_name: openai-api-key + secret_key: api-key + - env_var_name: REDIS_URL + secret_name: redis-url-secret + secret_key: url + - env_var_name: SGP_API_KEY + secret_name: sgp-api-key + secret_key: api-key + - env_var_name: SGP_ACCOUNT_ID + secret_name: sgp-account-id + secret_key: account-id + - env_var_name: SGP_CLIENT_BASE_URL + secret_name: sgp-client-base-url + secret_key: url + +deployment: + image: + repository: "" + tag: "latest" + + global: + agent: + name: "s080-lineage" + description: "A sync agent whose tool calls carry lineage data-source refs and an agent version" + replicaCount: 1 + resources: + requests: + cpu: "500m" + memory: "1Gi" + limits: + cpu: "1000m" + memory: "2Gi" diff --git a/examples/tutorials/00_sync/080_lineage/project/__init__.py b/examples/tutorials/00_sync/080_lineage/project/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/tutorials/00_sync/080_lineage/project/acp.py b/examples/tutorials/00_sync/080_lineage/project/acp.py new file mode 100644 index 000000000..42f475773 --- /dev/null +++ b/examples/tutorials/00_sync/080_lineage/project/acp.py @@ -0,0 +1,86 @@ +"""ACP handler for the lineage tutorial. + +Identical harness wiring to ``00_sync/050_openai_agents``: ``Runner.run_streamed`` +wrapped in an ``OpenAITurn``, delivered through ``UnifiedEmitter.yield_turn``. +Lineage capture rides that surface, and the SGP tracing processor stamps +``__agent_version__`` from the ``AGENT_VERSION`` env var on every span. +""" + +from __future__ import annotations + +import os +from typing import AsyncGenerator + +from dotenv import load_dotenv + +load_dotenv() + +from agents import Runner + +from agentex.lib import adk +from project.agent import MODEL_NAME, create_agent +from agentex.lib.types.acp import SendMessageParams +from agentex.lib.types.tracing import SGPTracingProcessorConfig +from agentex.lib.utils.logging import make_logger +from agentex.lib.sdk.fastacp.fastacp import FastACP +from agentex.lib.core.harness.emitter import UnifiedEmitter +from agentex.types.task_message_update import TaskMessageUpdate +from agentex.types.task_message_content import TaskMessageContent +from agentex.lib.adk.providers._modules.openai_turn import OpenAITurn +from agentex.lib.core.tracing.tracing_processor_manager import add_tracing_processor_config + +logger = make_logger(__name__) + +# LiteLLM proxy auth: copy LITELLM_API_KEY to OPENAI_API_KEY for OpenAI client +# compatibility, so the same example works behind the Scale LiteLLM gateway. +_litellm_key = os.environ.get("LITELLM_API_KEY") +if _litellm_key and not os.environ.get("OPENAI_API_KEY"): + os.environ["OPENAI_API_KEY"] = _litellm_key + +add_tracing_processor_config( + SGPTracingProcessorConfig( + sgp_api_key=os.environ.get("SGP_API_KEY", ""), + sgp_account_id=os.environ.get("SGP_ACCOUNT_ID", ""), + sgp_base_url=os.environ.get("SGP_CLIENT_BASE_URL", ""), + ) +) + +acp = FastACP.create(acp_type="sync") + +_agent = None + + +def get_agent(): + """Get or create the OpenAI Agents SDK agent instance.""" + global _agent + if _agent is None: + _agent = create_agent() + return _agent + + +@acp.on_message_send +async def handle_message_send( + params: SendMessageParams, +) -> TaskMessageContent | list[TaskMessageContent] | AsyncGenerator[TaskMessageUpdate, None]: + """Handle incoming messages, streaming tokens and tool calls via the harness.""" + agent = get_agent() + task_id = params.task.id + user_message = params.content.content + logger.info(f"Processing message for task {task_id}") + + async with adk.tracing.span( + trace_id=task_id, + task_id=task_id, + name="message", + input={"message": user_message}, + data={"__span_type__": "AGENT_WORKFLOW"}, + ) as turn_span: + result = Runner.run_streamed(starting_agent=agent, input=user_message) + turn = OpenAITurn(result=result, model=MODEL_NAME) + emitter = UnifiedEmitter( + task_id=task_id, + trace_id=task_id, + parent_span_id=turn_span.id if turn_span else None, + ) + async for event in emitter.yield_turn(turn): + yield event diff --git a/examples/tutorials/00_sync/080_lineage/project/agent.py b/examples/tutorials/00_sync/080_lineage/project/agent.py new file mode 100644 index 000000000..9306af0e3 --- /dev/null +++ b/examples/tutorials/00_sync/080_lineage/project/agent.py @@ -0,0 +1,73 @@ +"""OpenAI Agents SDK agent whose tool calls carry lineage data-source refs. + +Each tool declares the data sources it reads via one of the three capture +forms in ``agentex.lib.adk.lineage`` (see README.md); the harness resolves +them on every tool span into ``span.data["sgp.lineage.refs"]``. Capture only — +nothing here emits to the lineage service. +""" + +from __future__ import annotations + +from agents import Agent, function_tool, set_tracing_disabled + +from project import tools +from agentex.lib.adk import DataSourceRef, lineage, data_sources + +# Disable the openai-agents SDK's native tracer so it doesn't ship traces to +# api.openai.com (the key may be a gateway/proxy key). Agentex tracing still +# runs via the harness + tracing manager configured in acp.py. +set_tracing_disabled(True) + +MODEL_NAME = "gpt-4o" +INSTRUCTIONS = """You are a market-research assistant with access to tools. + +Guidelines: +- To research a company, use all three tools: search_filings, read_kpi + (table="revenue"), and company_profile. +- Be concise, and always report the real tool output back to the user. +""" + + +@function_tool +@data_sources(DataSourceRef("elasticsearch://research-cluster", "filings-v1")) +def search_filings(query: str) -> str: + """Search the filings index for documents matching a query.""" + return tools.search_filings(query) + + +def _kpi_refs(args: dict) -> list[DataSourceRef]: + table = args.get("table") + if not table: + return [] + return [DataSourceRef("databricks://demo-workspace.cloud.databricks.com", f"main.kpi.{table}")] + + +@function_tool +@data_sources(resolver=_kpi_refs) +def read_kpi(table: str) -> str: + """Read a KPI summary row from a metrics table.""" + return tools.read_kpi(table) + + +@function_tool +def company_profile(name: str) -> str: + """Fetch a company profile.""" + return tools.company_profile(name) + + +# company_profile stands in for a tool this codebase doesn't own (e.g. an MCP +# server's tool), so its refs are registered by name instead of decorating it. +lineage.register_tool_sources( + "company_profile", + [DataSourceRef("mcp://research-mcp", "company-profiles")], +) + + +def create_agent() -> Agent: + """Build and return the agent with the three ref-carrying tools.""" + return Agent( + name="Lineage Research Assistant", + model=MODEL_NAME, + instructions=INSTRUCTIONS, + tools=[search_filings, read_kpi, company_profile], + ) diff --git a/examples/tutorials/00_sync/080_lineage/project/tools.py b/examples/tutorials/00_sync/080_lineage/project/tools.py new file mode 100644 index 000000000..e898554aa --- /dev/null +++ b/examples/tutorials/00_sync/080_lineage/project/tools.py @@ -0,0 +1,23 @@ +"""Tool implementations for the lineage tutorial. + +The bare functions live here so they're easy to unit-test; they're wrapped as +OpenAI Agents SDK ``function_tool``s — and annotated with data-source refs — +in ``project.agent``. Outputs are canned so the tutorial runs offline. +""" + +from __future__ import annotations + + +def search_filings(query: str) -> str: + """Search the filings index for documents matching a query.""" + return f'Top filing for "{query}": ACME Corp 10-K (2025) — revenue up 12%, filed 2026-02-14.' + + +def read_kpi(table: str) -> str: + """Read a KPI summary row from a metrics table.""" + return f"{table}: latest value 4.2M, trailing 12-month growth 8%." + + +def company_profile(name: str) -> str: + """Fetch a company profile (stands in for an MCP-server tool).""" + return f"{name}: industrial supplier, HQ Albuquerque, 1,200 employees, founded 1952." diff --git a/examples/tutorials/00_sync/080_lineage/pyproject.toml b/examples/tutorials/00_sync/080_lineage/pyproject.toml new file mode 100644 index 000000000..5494c7bd6 --- /dev/null +++ b/examples/tutorials/00_sync/080_lineage/pyproject.toml @@ -0,0 +1,36 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "s080-lineage" +version = "0.1.0" +description = "A sync agent whose tool calls carry lineage data-source refs and an agent version" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "agentex-sdk>=0.25.0", + "scale-gp", + "openai-agents", +] + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-asyncio", + "httpx", + "black", + "isort", + "flake8", +] + +[tool.hatch.build.targets.wheel] +packages = ["project"] + +[tool.black] +line-length = 88 +target-version = ['py312'] + +[tool.isort] +profile = "black" +line_length = 88 diff --git a/examples/tutorials/00_sync/080_lineage/tests/test_agent.py b/examples/tutorials/00_sync/080_lineage/tests/test_agent.py new file mode 100644 index 000000000..eb4ba10c2 --- /dev/null +++ b/examples/tutorials/00_sync/080_lineage/tests/test_agent.py @@ -0,0 +1,46 @@ +"""Offline test for the lineage tutorial. + +This test does NOT require a running Agentex server or an OpenAI API key. It +verifies the tutorial's lineage wiring: each of the three capture forms in +``project.agent`` (static decorator, resolver, name-keyed registry) resolves +to the declared data-source refs under the tool name the harness sees. + +To run: ``pytest tests/test_agent.py -v`` +""" + +from __future__ import annotations + +import project.agent # noqa: F401 — registers the tutorial's tool refs on import + +from agentex.lib.adk import lineage + + +def test_static_decorator_refs_resolve_under_function_tool_name(): + refs = lineage.resolve_refs("search_filings", {"query": "acme"}) + assert refs == [{"namespace": "elasticsearch://research-cluster", "name": "filings-v1", "role": "input"}] + + +def test_resolver_derives_ref_from_tool_arguments(): + refs = lineage.resolve_refs("read_kpi", {"table": "revenue"}) + assert refs == [ + { + "namespace": "databricks://demo-workspace.cloud.databricks.com", + "name": "main.kpi.revenue", + "role": "input", + } + ] + + +def test_resolver_yields_nothing_without_the_argument(): + assert lineage.resolve_refs("read_kpi", {}) == [] + + +def test_registry_covers_the_unowned_tool(): + refs = lineage.resolve_refs("company_profile", {"name": "ACME"}) + assert refs == [{"namespace": "mcp://research-mcp", "name": "company-profiles", "role": "input"}] + + +if __name__ == "__main__": + import pytest + + pytest.main([__file__, "-v"])