Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Fixed

- A filtered audit is no longer reused as a full one (#542). The audit
cache records the audit's scope: framework, level, tag filter, and
whether any filter narrowed the control set. A filtered audit is cached
under its own key, and remediation reads only an unfiltered level 3
audit of its framework, through the store the audit wrote to. Before,
`darnit audit --tags level=1` left a cache that remediation treated as
a full audit. Cache files from earlier versions are ignored and rebuilt.
- Installs without the development dependencies work again (`uv tool install`
from a checkout, and any install from a built wheel). Two things broke
them: `tree-sitter-language-pack` 1.6.3, the newest release the old
Expand Down
18 changes: 17 additions & 1 deletion docs/architecture/framework-design.md
Original file line number Diff line number Diff line change
Expand Up @@ -1870,8 +1870,24 @@ Cache files are stored in the system temp directory (`$TMPDIR/darnit/<repo-hash>
- **AND** controls with `status == "WARN"` SHALL NOT be remediated (WARN means automated verification was inconclusive, not that the control is non-compliant)
- **AND** controls with `status == "PASS"` SHALL NOT be remediated

#### Requirement: The cache envelope records the audit's scope
- **WHEN** `write_audit_cache()` is called
- **THEN** the envelope SHALL record `framework`, `level`, `tags` (the tag filter, empty when none) and `filtered` (true when any filter narrowed the control set, including `--include` and `--exclude`)
- **AND** a filtered audit SHALL be written under a distinct cache key, so it does not replace an unfiltered entry
- **AND** `run_sieve_audit()` SHALL record an audit as filtered when it has tags, or when its caller passed `controls` without stating `cache_filtered`; a caller that passes the full control set SHALL state `cache_filtered=False`
- **AND** the cache key SHALL be derived in one place (`audit_cache_key()`), used by the writer and by every reader

#### Requirement: A cache read may constrain the scope it accepts
- **WHEN** `read_audit_cache()` is called with any of `expected_framework`, `expected_level`, `expected_tags` or `expected_filtered`
- **THEN** a mismatch on any supplied expectation SHALL be treated as a cache miss
- **AND** a caller supplying none of them SHALL be unaffected
- **AND** an envelope whose `version` differs from `CACHE_VERSION` SHALL be a miss, since an older envelope cannot express its own scope

#### Requirement: Remediation consumes cached audit results
- **WHEN** `remediate_audit_findings()` is called and `read_audit_cache()` returns valid cached results
- **WHEN** `remediate_audit_findings()` is called
- **THEN** it SHALL read through the store selected by `[stores.cache]`, the same store the audit writes to
- **AND** it SHALL require an unfiltered level-3 `openssf-baseline` cache, matching the audit it would otherwise run
- **WHEN** `read_audit_cache()` returns valid cached results
- **THEN** it SHALL extract failed control IDs from the cached results (entries with `status == "FAIL"`)
- **AND** it SHALL NOT run a redundant audit
- **WHEN** `read_audit_cache()` returns `None` (cache miss)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1117,8 +1117,13 @@ def run_remediation(
error: str | None = None

try:
from darnit.core.audit_cache import read_audit_cache
cache = read_audit_cache(local_path)
# Issue #542: remediation acts on a full level-3 audit of this
# framework, which is what it runs on a miss. A level-1, filtered or
# other-framework cache is a subset, so it must miss. The read goes
# through the store and key the audit writes to.
from darnit.tools.audit import read_full_audit_cache

cache = read_full_audit_cache(local_path, "openssf-baseline", level=3)
except Exception as exc:
logger.warning(f"Audit cache read failed: {exc}")
cache = None
Expand Down
3 changes: 3 additions & 0 deletions packages/darnit-baseline/src/darnit_baseline/tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,9 @@ def audit_openssf_baseline(
level=level,
controls=controls,
tags=tags_list,
# Issue #542: a profile narrows `controls` above. Tags are
# recorded by run_sieve_audit itself.
cache_filtered=bool(profile),
stop_on_llm=True,
framework_name="openssf-baseline",
operator_config=operator_config,
Expand Down
5 changes: 5 additions & 0 deletions packages/darnit/src/darnit/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,11 @@ def cmd_audit(args: argparse.Namespace) -> int:
default_branch=default_branch,
level=3,
controls=controls,
# Issue #542: the CLI narrows `controls` itself, so it says whether
# it did. Without this a `--tags domain=VM` run would be cached as
# if it were a full audit.
cache_tags=",".join(sorted(args.tags)) if args.tags else "",
cache_filtered=bool(args.tags or args.include or args.exclude),
stop_on_llm=True,
# Issue #427: the framework name has to reach the audit driver, not
# just the control loader above. Without it the driver cannot
Expand Down
55 changes: 53 additions & 2 deletions packages/darnit/src/darnit/core/audit_cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
logger = get_logger("core.audit_cache")

CACHE_FILENAME = "audit-cache.json"
CACHE_VERSION = 1
CACHE_VERSION = 2

# Invalidation via write-expired-envelope (feature 035 clarify Q3).
# The AuditCacheStore Protocol has no delete(key) method; instead,
Expand All @@ -68,6 +68,8 @@
"commit_dirty": False,
"level": 0,
"framework": "",
"tags": "",
"filtered": False,
"results": [],
"summary": {},
}
Expand Down Expand Up @@ -179,6 +181,8 @@ def write_audit_cache(
level: int,
framework: str,
*,
tags: str = "",
filtered: bool = False,
store: AuditCacheStore | None = None,
cache_key: str | None = None,
) -> None:
Expand Down Expand Up @@ -214,6 +218,13 @@ def write_audit_cache(
"commit_dirty": _is_working_tree_dirty(local_path),
"level": level,
"framework": framework,
# Issue #542: the scope the audit ran at. A later read that expects a
# different framework, level or tag filter must miss, not silently
# reuse a subset as if it were a full audit.
"tags": tags,
# True when ANY filter narrowed the control set: tags, --include,
# --exclude, or a profile. `tags` alone cannot express the others.
"filtered": filtered,
"results": results,
"summary": summary,
}
Expand All @@ -232,6 +243,10 @@ def read_audit_cache(
local_path: str,
ttl_seconds: int = 3600,
*,
expected_framework: str | None = None,
expected_level: int | None = None,
expected_tags: str | None = None,
expected_filtered: bool | None = None,
store: AuditCacheStore | None = None,
cache_key: str | None = None,
) -> dict[str, Any] | None:
Expand Down Expand Up @@ -266,10 +281,46 @@ def read_audit_cache(

# Version check
version = data.get("version")
if not isinstance(version, int) or version > CACHE_VERSION:
# Issue #542 review: an exact match, not an upper bound. A v1 envelope
# has no `tags`/`filtered` keys, so reading it would present a possibly
# filtered audit as unfiltered. Older caches miss and are re-audited.
if not isinstance(version, int) or version != CACHE_VERSION:
logger.debug("Unknown audit cache version: %s", version)
return None

# Scope: the cached audit must cover what the caller is about to use it
# for (issue #542). A level-1, tag-filtered or other-framework run leaves a
# cache that is a subset of a full Baseline audit; reusing it would make a
# consumer act on results that were never produced.
if expected_framework is not None and data.get("framework") != expected_framework:
logger.debug(
"Audit cache scope mismatch: framework %r != expected %r",
data.get("framework"),
expected_framework,
)
return None
if expected_level is not None and data.get("level") != expected_level:
logger.debug(
"Audit cache scope mismatch: level %r != expected %r",
data.get("level"),
expected_level,
)
return None
if expected_filtered is not None and bool(data.get("filtered")) != expected_filtered:
logger.debug(
"Audit cache scope mismatch: filtered %r != expected %r",
data.get("filtered"),
expected_filtered,
)
return None
if expected_tags is not None and (data.get("tags") or "") != expected_tags:
logger.debug(
"Audit cache scope mismatch: tags %r != expected %r",
data.get("tags"),
expected_tags,
)
return None

# Staleness: TTL expiry
timestamp_str = data.get("timestamp")
if isinstance(timestamp_str, str):
Expand Down
3 changes: 3 additions & 0 deletions packages/darnit/src/darnit/server/tools/builtin_audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,9 @@ async def builtin_audit(
level=level,
controls=all_controls,
tags=tags_list,
# Issue #542: all_controls is the framework's full set. Tags are
# recorded by run_sieve_audit itself.
cache_filtered=False,
stop_on_llm=True,
framework_name=_framework_name,
operator_config=operator_config,
Expand Down
84 changes: 80 additions & 4 deletions packages/darnit/src/darnit/tools/audit.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,78 @@ def _load_merged_stores(framework_name: str | None, operator: "OperatorConfig |
return getattr(effective, "stores", None)


def audit_cache_key(local_path: str, stores_config: Any, *, filtered: bool = False) -> str:
"""Return the key an audit of ``local_path`` is cached under.

The one derivation for the writer (:func:`run_sieve_audit`) and for every
reader. The default store's root already encodes the repository, so the
key is fixed; a configured store may be shared across repositories, so the
key encodes the path. A filtered audit gets its own key, so it never
replaces the full-audit entry (issue #542).
"""
if stores_config is None or stores_config.cache is None:
key = "audit-cache"
else:
key = hashlib.sha256(str(Path(local_path).resolve()).encode()).hexdigest()[:16]
return f"{key}-filtered" if filtered else key


def _cache_scope(
tags: list[str] | None,
controls: list | None,
cache_tags: str | None,
cache_filtered: bool | None,
) -> tuple[str, bool]:
"""Return the (tags, filtered) pair recorded in the audit cache (issue #542).

Tags always mean filtered. Otherwise the caller's word decides. A caller
that passed its own ``controls`` and did not say is treated as filtered:
this function cannot tell a narrowed list from the full set, and a subset
read back as a full audit makes remediation act on results that were never
produced.
"""
if cache_tags is None:
cache_tags = ",".join(sorted(tags)) if tags else ""
if cache_filtered is None:
cache_filtered = controls is not None
return cache_tags, bool(cache_tags) or cache_filtered


def read_full_audit_cache(
local_path: str,
framework_name: str,
level: int,
operator_config: "LoadedOperatorConfig | None" = None,
) -> dict[str, Any] | None:
"""Return the cached unfiltered audit of ``framework_name`` at ``level``, or None.

Reads through the store and key :func:`run_sieve_audit` writes to, so an
operator's ``[stores.cache]`` is honoured on both sides. A cache from
another framework, another level or a filtered run is a miss.
"""
from darnit.core.audit_cache import read_audit_cache
from darnit.stores.selection import resolve_stores

if operator_config is None:
from darnit.config.operator.loader import resolve_operator_config

operator_config = resolve_operator_config(local_path)
stores_config = _load_merged_stores(framework_name, operator_config.config)
stores_bundle = resolve_stores(stores_config, repo_path=Path(local_path))
try:
return read_audit_cache(
local_path,
expected_framework=framework_name,
expected_level=level,
expected_tags="",
expected_filtered=False,
store=stores_bundle.cache,
cache_key=audit_cache_key(local_path, stores_config, filtered=False),
)
finally:
stores_bundle.close_all()


def _load_merged_mcp_servers(framework_name: str | None, operator: "OperatorConfig | None" = None) -> dict[str, Any]:
"""Return the merged ``mcp_servers`` allowlist for this audit run.

Expand Down Expand Up @@ -731,6 +803,8 @@ def run_sieve_audit(
*,
controls: list | None = None,
tags: list[str] | None = None,
cache_tags: str | None = None,
cache_filtered: bool | None = None,
evaluate_claims: bool = True,
stop_on_llm: bool = True,
framework_name: str | None = None,
Expand Down Expand Up @@ -1014,10 +1088,10 @@ def run_sieve_audit(
# already encodes repo identity (uses fixed "audit-cache"); configured
# store's root is operator-picked and may be shared across repos, so
# the key must encode repo identity.
if stores_config is None or stores_config.cache is None:
cache_key = "audit-cache"
else:
cache_key = hashlib.sha256(str(Path(local_path).resolve()).encode()).hexdigest()[:16]
# Issue #542: record the scope this audit ran at, and keep a filtered run
# out of the full-audit entry.
cache_tags, cache_filtered = _cache_scope(tags, controls, cache_tags, cache_filtered)
cache_key = audit_cache_key(local_path, stores_config, filtered=cache_filtered)

from darnit.core.audit_cache import write_audit_cache

Expand All @@ -1033,6 +1107,8 @@ def run_sieve_audit(
summary,
level,
resolved_fw or "",
tags=cache_tags,
filtered=cache_filtered,
store=stores_bundle.cache,
cache_key=cache_key,
)
Expand Down
Loading
Loading