diff --git a/.github/workflows/grafana-alertcheck-release.yml b/.github/workflows/grafana-alertcheck-release.yml deleted file mode 100644 index 4f946b18f..000000000 --- a/.github/workflows/grafana-alertcheck-release.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Grafana Alertcheck Release - -on: - push: - tags: - - grafana-alertcheck/v* - -jobs: - release: - name: Build and Release - runs-on: ubuntu-latest - environment: integration - permissions: - id-token: write - contents: write - steps: - - name: Checkout repo - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v7 - with: - go-version-file: ./grafana-alertcheck/go.mod - cache-dependency-path: ./grafana-alertcheck/go.mod - - name: Goreleaser Release - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 - with: - distribution: goreleaser-pro - version: "~> v2" - args: release --clean -f ./grafana-alertcheck/.goreleaser.yaml - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} diff --git a/grafana-alertcheck/.goreleaser.yaml b/grafana-alertcheck/.goreleaser.yaml deleted file mode 100644 index 09e6032cb..000000000 --- a/grafana-alertcheck/.goreleaser.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# yaml-language-server: $schema=https://goreleaser.com/static/schema-pro.json -version: 2 -project_name: grafana-alertcheck - -dist: grafana-alertcheck/dist - -monorepo: - tag_prefix: grafana-alertcheck/ - dir: grafana-alertcheck - -builds: - - id: grafana-alertcheck - main: ./cmd/grafana-alertcheck/main.go - ldflags: - - -s - - -w - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck.version={{.Version}} - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck.commit={{.ShortCommit}} - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck.date={{.CommitDate}} - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck.builtBy=goreleaser - goos: - - linux - - darwin - goarch: - - amd64 - - arm64 - binary: grafana-alertcheck - env: - - CGO_ENABLED=0 - -before: - hooks: - - sh -c "cd grafana-alertcheck && go mod tidy" diff --git a/grafana-alertcheck/.tool-versions b/grafana-alertcheck/.tool-versions new file mode 100644 index 000000000..ef699cf97 --- /dev/null +++ b/grafana-alertcheck/.tool-versions @@ -0,0 +1,3 @@ +# golangci-lint: keep in sync with devbox.json (used by CI in .github/workflows/linters.yml via `devbox run -- just lint`). +golang 1.26.6 +golangci-lint 2.12.2 diff --git a/grafana-alertcheck/README.md b/grafana-alertcheck/README.md index 43cc03fc4..96aa0e999 100644 --- a/grafana-alertcheck/README.md +++ b/grafana-alertcheck/README.md @@ -1,6 +1,46 @@ # grafana-alertcheck -A CD quality gate for Grafana alerts: bookend a release with `watch` (record) and `check` (classify) to -answer whether any watched alert was in a bad state during the release window. +A CD quality gate for Grafana alerts. It bookends a release with two commands — `watch` (record) and +`check` (classify) — and answers whether any watched alert was in a bad state during the release window. -Under construction. +``` +watch → your work → check +``` + +`watch` starts a background recorder that polls each named alert into a JSONL log. After the work emits a +`from`/`to` pair, `check` proves continuous coverage of that window, classifies each alert's state +timeline, and exits `0`, `1`, or `2`. + +It **fails closed**: if it cannot get an answer, it stops the release — never a pass on an unproven window. + +## Quickstart + +```bash +export GRAFANA_URL=https://grafana.example.com +export GRAFANA_TOKEN=… + +grafana-alertcheck watch --out /tmp/run.jsonl --alerts alerts.txt +./deploy.sh # emits deployed_at= +./verify.sh # emits finished_at= +grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finished_at" +``` + +Requires Grafana >= 13.0.0 and < 14.0.0. Connection details come from the environment only — the token is +never a flag. + +## Documentation + +| Doc | Covers | +| --- | ------ | +| [`docs/index.md`](./docs/index.md) | Overview, quickstarts, exit codes, common surprises | +| [`docs/how-alerts-are-evaluated.md`](./docs/how-alerts-are-evaluated.md) | Verdict model, coverage proof, health/liveness | +| [`docs/advanced.md`](./docs/advanced.md) | Check budget, scheduling, why history isn't queried | +| [`docs/architecture.md`](./docs/architecture.md) | Design invariants, the pure-function seam, recorder lifecycle | +| [`docs/reference/cli.md`](./docs/reference/cli.md) | Full CLI reference — subcommands, flags, naming | +| [`docs/reference/log-format.md`](./docs/reference/log-format.md) | The JSONL log schema, for debugging artifacts | + +## Build + +```bash +go build ./... && go test ./... +``` diff --git a/grafana-alertcheck/cmd/check.go b/grafana-alertcheck/cmd/check.go new file mode 100644 index 000000000..e95fce376 --- /dev/null +++ b/grafana-alertcheck/cmd/check.go @@ -0,0 +1,154 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os/signal" + "syscall" + "time" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +const checkUsage = "usage: grafana-alertcheck check [--in ] [--pidfile F] --from RFC3339 --to RFC3339 " + + "[--alerts ...] [--folder F] [--states ...] [--preexisting ...] [--min-observed N] [--allow-paused] " + + "[--nodata-is-unobservable] [--concurrency N] [--output json]" + +// runCheck is the classify step's CLI surface: parse flags into a gate.Config, +// run gate.Check, and translate its (Result, error) into output and an exit +// code. All of the correctness lives in the gate package — this file's only job +// is presentation and the exit-code mapping, which exitCode below keeps as one +// pure function so it can be tested without a network. +func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("check", flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprintln(stderr, checkUsage) } + + common := registerCommon(fs) + in := fs.String("in", "", "path of a log recorded by watch; empty selects single-step mode") + pidfile := fs.String("pidfile", "", "pidfile of the recorder to stop before reading --in (default .pid)") + from := fs.String("from", "", "the moment the deploy finished, RFC3339 (required with --in)") + to := fs.String("to", "", "the end of the window to classify, RFC3339 (required)") + states := fs.String("states", "", "comma-separated bad states to classify against (default: firing)") + preexisting := fs.String("preexisting", "", "how to judge an instance already bad at `from` (default: fail-unless-recovered)") + minObserved := fs.Int("min-observed", 0, "minimum rules that must be observed (default: every resolved rule)") + allowPaused := fs.Bool("allow-paused", false, "do not count a rule paused before the window against --min-observed") + nodataIsUnobservable := fs.Bool("nodata-is-unobservable", false, "treat a sustained health=nodata as unobservable rather than a note") + output := fs.String("output", "", `"json" writes the machine-readable Result to stdout in addition to the table; default is the table alone`) + + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() != 0 { + fmt.Fprintf(stderr, "check: unexpected arguments %v\n", fs.Args()) + return 2 + } + if *output != "" && *output != "json" { + fmt.Fprintf(stderr, "--output: unknown value %q (only \"json\" is supported)\n", *output) + return 2 + } + + url, token, err := grafanaEnv() + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + alerts, err := readAlerts(stdin, *common.alerts) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + stateList, err := parseStates(*states) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + preexistingPolicy, err := parsePreexisting(*preexisting) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + + cfg := gate.Config{ + URL: url, + Token: token, + Alerts: alerts, + Folder: *common.folder, + States: stateList, + Preexisting: preexistingPolicy, + MinObserved: *minObserved, + AllowPaused: *allowPaused, + NodataIsUnobservable: *nodataIsUnobservable, + Log: *in, + PidFile: *pidfile, + Concurrency: *common.concurrency, + Clock: gate.SystemClock{}, + Notes: newNoteStyler(stderr), + } + if *to == "" { + fmt.Fprintln(stderr, "check: --to is required") + return 2 + } + t, err := time.Parse(time.RFC3339, *to) + if err != nil { + fmt.Fprintf(stderr, "--to: %v\n", err) + return 2 + } + cfg.To = t + if *from != "" { + f, err := time.Parse(time.RFC3339, *from) + if err != nil { + fmt.Fprintf(stderr, "--from: %v\n", err) + return 2 + } + cfg.From = f + } + + // SIGINT/SIGTERM cancel the run cleanly rather than leaving an operator's + // Ctrl-C to kill the process mid-collection: Check's collection loop and + // drain wait both already select on ctx.Done() (check.go), so this makes + // an interrupted run fail the way every other could-not-check path does + // — exit 2, never a silently truncated pass. + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + result, checkErr := gate.Check(ctx, cfg) + + if checkErr != nil { + fmt.Fprintln(stderr, checkErr) + } else if err := renderTable(stderr, result); err != nil { + fmt.Fprintln(stderr, err) + } + if *output == "json" { + enc := json.NewEncoder(stdout) + enc.SetIndent("", " ") + if err := enc.Encode(result); err != nil { + fmt.Fprintf(stderr, "encode --output json: %v\n", err) + return 2 + } + } + return exitCode(result, checkErr) +} + +// exitCode is the whole exit-code mapping, kept as one pure function of +// exactly what Check returns so it is testable without a network: err != nil +// is exit 2 UNCONDITIONALLY — never 0 and never 1, even alongside real +// violations, because an inability to check beats a violation and an error is +// never a pass. Violations without an error is exit 1. Neither is exit 0. +func exitCode(res gate.Result, err error) int { + switch { + case err != nil: + return 2 + case len(res.Violations) > 0: + return 1 + default: + return 0 + } +} diff --git a/grafana-alertcheck/cmd/check_test.go b/grafana-alertcheck/cmd/check_test.go new file mode 100644 index 000000000..c7c797a0b --- /dev/null +++ b/grafana-alertcheck/cmd/check_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "bytes" + "errors" + "os" + "testing" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" + "github.com/stretchr/testify/require" +) + +// The exit-code mapping, pinned directly against exitCode with no network +// involved: err != nil is exit 2 even alongside violations (an inability to +// check beats a violation), violations alone are exit 1, and neither is 0. +func TestExitCode(t *testing.T) { + tests := []struct { + name string + res gate.Result + err error + want int + }{ + {"pass", gate.Result{}, nil, 0}, + {"violation", gate.Result{Violations: []gate.Violation{{}}}, nil, 1}, + {"error alone", gate.Result{}, errors.New("boom"), 2}, + {"error beats violation", gate.Result{Violations: []gate.Violation{{}}}, errors.New("boom"), 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + require.Equal(t, tt.want, exitCode(tt.res, tt.err)) + }) + } +} + +func writeTempAlerts(t *testing.T) string { + t.Helper() + path := t.TempDir() + "/alerts.txt" + require.NoError(t, os.WriteFile(path, []byte("Some Alert\n"), 0o644)) + return path +} + +// The flag-validation matrix: every one of these must fail before any network +// call, because gate.Config.validate() runs first — an unreachable GRAFANA_URL +// succeeding or timing out is a different test than these, which check pure +// input validation. +func TestRunCheck_FlagValidation(t *testing.T) { + tests := []struct { + name string + env bool + args func(t *testing.T) []string + wantErr string + }{ + {"missing env", false, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--alerts", writeTempAlerts(t)} + }, "GRAFANA_URL"}, + {"missing to", true, func(t *testing.T) []string { + return []string{"--alerts", writeTempAlerts(t)} + }, "--to"}, + {"bad to", true, func(t *testing.T) []string { + return []string{"--to", "not-a-time", "--alerts", writeTempAlerts(t)} + }, "--to"}, + {"bad from", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--from", "not-a-time", "--alerts", writeTempAlerts(t)} + }, "--from"}, + {"bad output", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--output", "xml", "--alerts", writeTempAlerts(t)} + }, "--output"}, + {"bad states", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--states", "bogus", "--alerts", writeTempAlerts(t)} + }, "--states"}, + {"states normal is rejected", true, func(t *testing.T) []string { + // normal is the good state, never a state to classify AS bad: + // accepting it would make --states normal fail every healthy + // instance. + return []string{"--to", "2026-01-01T00:00:00Z", "--states", "normal", "--alerts", writeTempAlerts(t)} + }, "--states"}, + {"bad preexisting", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--preexisting", "bogus", "--alerts", writeTempAlerts(t)} + }, "--preexisting"}, + {"alerts with in", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--in", "some.jsonl", "--alerts", writeTempAlerts(t)} + }, "refused"}, + {"no alerts no in", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z"} + }, "no alert names"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env { + t.Setenv("GRAFANA_URL", "http://example.invalid") + t.Setenv("GRAFANA_TOKEN", "test-token") + } else { + t.Setenv("GRAFANA_URL", "") + t.Setenv("GRAFANA_TOKEN", "") + } + var stdout, stderr bytes.Buffer + args := append([]string{"check"}, tt.args(t)...) + code := run(args, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), tt.wantErr) + }) + } +} + +// A `to` already in the past with no recorded log cannot be classified from +// anything, because nothing ever observed the window. +func TestRunCheck_ToInPastNoLog(t *testing.T) { + t.Setenv("GRAFANA_URL", "http://example.invalid") + t.Setenv("GRAFANA_TOKEN", "test-token") + + var stdout, stderr bytes.Buffer + code := run([]string{"check", + "--from", "1999-01-01T00:00:00Z", "--to", "2000-01-01T00:00:00Z", + "--alerts", writeTempAlerts(t), + }, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), "already passed") +} + +// --output json never writes to stdout when Check was never reached, because +// there is no Result to encode — only the table (on stderr) can report a +// configuration failure. +func TestRunCheck_NoResultOnConfigError(t *testing.T) { + t.Setenv("GRAFANA_URL", "") + t.Setenv("GRAFANA_TOKEN", "") + var stdout, stderr bytes.Buffer + code := run([]string{"check", "--to", "2026-01-01T00:00:00Z", "--output", "json"}, &stdout, &stderr) + require.Equal(t, 2, code) + require.Empty(t, stdout.String()) +} diff --git a/grafana-alertcheck/cmd/common.go b/grafana-alertcheck/cmd/common.go new file mode 100644 index 000000000..3c56cd63f --- /dev/null +++ b/grafana-alertcheck/cmd/common.go @@ -0,0 +1,107 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "os" + "strings" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +// commonFlags is registerCommon's result: the exactly three flags watch and +// check share. Connection details are never flags, and states / poll-interval +// are deliberately NOT here — states is check-only because recording is +// unfiltered, and poll-interval is watch-only because check reads the cadence +// from the log header. Putting either here would give both commands an opinion +// about a value only one of them may set. +type commonFlags struct { + folder *string + concurrency *int + alerts *string +} + +func registerCommon(fs *flag.FlagSet) *commonFlags { + return &commonFlags{ + folder: fs.String("folder", "", "default folder to scope an unqualified alert name to"), + concurrency: fs.Int("concurrency", 1, "maximum concurrent requests to Grafana"), + alerts: fs.String("alerts", "", "path to a file of alert names, one per line, or - for stdin"), + } +} + +// readAlerts reads alert names, one per line, from a file or from +// stdin when path is "-". An empty path is not an error here — watch and +// check each decide for themselves whether an empty list is allowed +// (log mode never wants one; single-step / record mode always does). +func readAlerts(stdin io.Reader, path string) ([]string, error) { + if path == "" { + return nil, nil + } + var r io.Reader + if path == "-" { + r = stdin + } else { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("read --alerts %s: %w", path, err) + } + defer f.Close() + r = f + } + var lines []string + sc := bufio.NewScanner(r) + for sc.Scan() { + lines = append(lines, sc.Text()) + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("read --alerts %s: %w", path, err) + } + return lines, nil +} + +// parseStates parses check's --states flag: a comma-separated list of the +// "bad" state vocabulary Config.States matches against (classify.go's +// badStateSet). An empty string is not resolved here — it means "use the +// library default of {firing}" — so this returns nil, nil for "" rather than +// an error. +// +// normal is deliberately NOT accepted. The vocabulary is fixed to +// firing | pending | nodata | error precisely because "normal" is the good +// state, never a bad one to classify against: --states normal would turn every +// healthy instance into a violation and fail every healthy fleet. +func parseStates(s string) ([]gate.State, error) { + if strings.TrimSpace(s) == "" { + return nil, nil + } + var out []gate.State + for _, part := range strings.Split(s, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + switch gate.State(part) { + case gate.StateFiring, gate.StatePending, gate.StateNodata, gate.StateError: + out = append(out, gate.State(part)) + default: + return nil, fmt.Errorf("--states: unknown state %q (want any of: firing, pending, nodata, error)", part) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("--states: %q named no state", s) + } + return out, nil +} + +// parsePreexisting parses check's --preexisting flag. +func parsePreexisting(s string) (gate.PreexistingPolicy, error) { + switch gate.PreexistingPolicy(s) { + case "": + return gate.PreexistingFailUnlessRecovered, nil + case gate.PreexistingFailUnlessRecovered, gate.PreexistingFail, gate.PreexistingIgnore: + return gate.PreexistingPolicy(s), nil + default: + return "", fmt.Errorf("--preexisting: unknown policy %q (want one of: fail-unless-recovered, fail, ignore)", s) + } +} diff --git a/grafana-alertcheck/cmd/env.go b/grafana-alertcheck/cmd/env.go new file mode 100644 index 000000000..02a125f1d --- /dev/null +++ b/grafana-alertcheck/cmd/env.go @@ -0,0 +1,21 @@ +package main + +import ( + "fmt" + "os" +) + +// grafanaEnv reads the connection details from the environment only, never +// from a flag — a flag value lands in the process argv and in CI logs, and +// the token must never be logged or otherwise surface in an error string. +func grafanaEnv() (url, token string, err error) { + url = os.Getenv("GRAFANA_URL") + if url == "" { + return "", "", fmt.Errorf("GRAFANA_URL is not set") + } + token = os.Getenv("GRAFANA_TOKEN") + if token == "" { + return "", "", fmt.Errorf("GRAFANA_TOKEN is not set") + } + return url, token, nil +} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/main.go b/grafana-alertcheck/cmd/grafana-alertcheck/main.go deleted file mode 100644 index 150d617d4..000000000 --- a/grafana-alertcheck/cmd/grafana-alertcheck/main.go +++ /dev/null @@ -1,7 +0,0 @@ -package main - -import "os" - -func main() { - os.Exit(2) -} diff --git a/grafana-alertcheck/cmd/list.go b/grafana-alertcheck/cmd/list.go new file mode 100644 index 000000000..c1d0e8532 --- /dev/null +++ b/grafana-alertcheck/cmd/list.go @@ -0,0 +1,92 @@ +package main + +import ( + "context" + "fmt" + "io" + "sort" + "text/tabwriter" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +// runList reads every rule definition from the ruler endpoint and prints one +// line per rule: its kind, its Folder/Group/Title, and its uid. It validates +// auth, the ruler parse, and the shapes Resolve matches against, all against a +// real Grafana, and it is the surface Resolve's no-match error points operators +// at. +func runList(args []string, stdout, stderr io.Writer) int { + if len(args) != 0 { + fmt.Fprintf(stderr, "list takes no arguments, got %v\n", args) + return 2 + } + + url, token, err := grafanaEnv() + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + + // context.Background(), no outer deadline: httpSource bounds every single + // attempt with its http.Client's 30s Timeout (source.go) and gives up + // after maxSequentialFailures consecutive transport errors, so this call + // always terminates. It can still take minutes end-to-end under repeated + // transient failures (5 retries * up to 30s backoff each, per call) — an + // acceptable wait for an interactive `list`, not for `watch`/`check`, + // which get their own deadlines from `--until`/`--to`. + src := gate.NewHTTPSource(url, token, gate.SystemClock{}) + version, err := src.Version(context.Background()) + if err != nil { + fmt.Fprintf(stderr, "checking grafana version: %v\n", err) + return 2 + } + if err := gate.CheckGrafanaVersion(version); err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + + defs, err := src.Definitions(context.Background()) + if err != nil { + fmt.Fprintf(stderr, "reading rule definitions: %v\n", err) + return 2 + } + + sort.Slice(defs, func(i, j int) bool { + if defs[i].Folder != defs[j].Folder { + return defs[i].Folder < defs[j].Folder + } + if defs[i].Group != defs[j].Group { + return defs[i].Group < defs[j].Group + } + return defs[i].Title < defs[j].Title + }) + + tw := tabwriter.NewWriter(stdout, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "KIND\tFOLDER\tGROUP\tTITLE\tUID") + for _, d := range defs { + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\n", kindLabel(d.Kind), d.Folder, d.Group, d.Title, uidOrDash(d.UID)) + } + if err := tw.Flush(); err != nil { + fmt.Fprintf(stderr, "writing output: %v\n", err) + return 2 + } + return 0 +} + +func kindLabel(k gate.RuleKind) string { + switch k { + case gate.KindDatasourceManaged: + return "datasource-managed" + case gate.KindRecording: + return "recording" + default: + return "grafana-managed" + } +} + +func uidOrDash(uid string) string { + if uid == "" { + return "-" + } + return uid +} diff --git a/grafana-alertcheck/cmd/list_test.go b/grafana-alertcheck/cmd/list_test.go new file mode 100644 index 000000000..62aa200d2 --- /dev/null +++ b/grafana-alertcheck/cmd/list_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "bytes" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" +) + +const rulerBody = `{ + "Example-Zone-A": [ + { + "name": "Gateway", + "rules": [ + { + "for": "5m", + "grafana_alert": { + "title": "Example No Gateways Available", + "uid": "rule0000006a", + "namespace_uid": "folder0000006", + "intervalSeconds": 60, + "no_data_state": "OK", + "exec_err_state": "OK", + "is_paused": false + } + } + ] + } + ] +}` + +func healthBody(version string) string { + return fmt.Sprintf(`{"database":"ok","version":%q,"commit":"abc123"}`, version) +} + +func grafanaTestServer(t *testing.T, version string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/api/health": + _, _ = w.Write([]byte(healthBody(version))) + case "/api/ruler/grafana/api/v1/rules": + _, _ = w.Write([]byte(rulerBody)) + default: + t.Errorf("unexpected path %q", r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func TestRunList_HappyPath(t *testing.T) { + srv := grafanaTestServer(t, "13.1.0") + t.Setenv("GRAFANA_URL", srv.URL) + t.Setenv("GRAFANA_TOKEN", "test-token") + + var stdout, stderr bytes.Buffer + code := run([]string{"list"}, &stdout, &stderr) + require.Equal(t, 0, code) + out := stdout.String() + require.Contains(t, out, "rule0000006a") + require.Contains(t, out, "Example No Gateways Available") + require.Contains(t, out, "grafana-managed") +} + +func TestRunList_UnsupportedVersion(t *testing.T) { + srv := grafanaTestServer(t, "12.5.0") + t.Setenv("GRAFANA_URL", srv.URL) + t.Setenv("GRAFANA_TOKEN", "test-token") + + var stdout, stderr bytes.Buffer + code := run([]string{"list"}, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), "12.5.0") +} + +func TestRunList_RejectsArgs(t *testing.T) { + t.Setenv("GRAFANA_URL", "http://example.invalid") + t.Setenv("GRAFANA_TOKEN", "test-token") + + var stdout, stderr bytes.Buffer + code := run([]string{"list", "extra"}, &stdout, &stderr) + require.Equal(t, 2, code) +} diff --git a/grafana-alertcheck/cmd/main.go b/grafana-alertcheck/cmd/main.go new file mode 100644 index 000000000..7ab5d6397 --- /dev/null +++ b/grafana-alertcheck/cmd/main.go @@ -0,0 +1,47 @@ +// Command grafana-alertcheck is the CLI entry point for the gate: `list`, +// `watch` (record) and `check` (classify). +package main + +import ( + "fmt" + "io" + "os" +) + +func main() { + os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) +} + +const usage = "usage: grafana-alertcheck " + +// run is the whole of main's testable surface: parse the subcommand, dispatch, +// return the process exit code. Exit codes below 2 (pass/violations) belong to +// `check` alone; every failure reachable from here — a missing subcommand, a +// bad flag, a transport or auth failure — is a could-not-check condition and +// maps to 2, never to 0 or 1. +// +// Requested help (-h/--help) is not a failure — it is the one exception to +// that rule. Convention (and every stdlib flag.FlagSet default) is exit 0 to +// stdout for help the caller asked for, reserving 2/stderr for help printed +// *because* something else went wrong (no subcommand, an unknown one). +func run(args []string, stdout, stderr io.Writer) int { + if len(args) == 0 { + fmt.Fprintln(stderr, usage) + return 2 + } + + switch args[0] { + case "list": + return runList(args[1:], stdout, stderr) + case "watch": + return runWatch(args[1:], os.Stdin, stdout, stderr) + case "check": + return runCheck(args[1:], os.Stdin, stdout, stderr) + case "-h", "-help", "--help": + fmt.Fprintln(stdout, usage) + return 0 + default: + fmt.Fprintf(stderr, "unknown subcommand %q\n", args[0]) + return 2 + } +} diff --git a/grafana-alertcheck/cmd/main_test.go b/grafana-alertcheck/cmd/main_test.go new file mode 100644 index 000000000..34dd3b2d2 --- /dev/null +++ b/grafana-alertcheck/cmd/main_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRun_NoArgs(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run(nil, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), "usage") +} + +func TestRun_Help(t *testing.T) { + for _, flag := range []string{"-h", "-help", "--help"} { + t.Run(flag, func(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{flag}, &stdout, &stderr) + require.Equal(t, 0, code, "requested help is not a could-not-check condition") + require.Contains(t, stdout.String(), "usage") + require.Empty(t, stderr.String(), "help goes to stdout") + }) + } +} + +func TestRun_UnknownSubcommand(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{"bogus"}, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), `"bogus"`) +} + +func TestRun_List_MissingEnv(t *testing.T) { + t.Setenv("GRAFANA_URL", "") + t.Setenv("GRAFANA_TOKEN", "") + var stdout, stderr bytes.Buffer + code := run([]string{"list"}, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), "GRAFANA_URL") +} diff --git a/grafana-alertcheck/cmd/style.go b/grafana-alertcheck/cmd/style.go new file mode 100644 index 000000000..bb46fc2ec --- /dev/null +++ b/grafana-alertcheck/cmd/style.go @@ -0,0 +1,125 @@ +package main + +import ( + "bytes" + "io" + "os" + "strings" +) + +// ANSI SGR codes for the human-facing notes and table footer. The colours are +// applied only when the destination is a terminal (see colorEnabled); a pipe, +// file or CI log gets plain text, so stdout stays reserved for --output json +// and no machine reader ever sees escape sequences. +const ( + ansiReset = "\x1b[0m" + ansiRed = "\x1b[31m" + ansiGreen = "\x1b[32m" + ansiYellow = "\x1b[33m" + ansiCyan = "\x1b[36m" + // Orange has no entry in the base-16 palette; 256-colour 208 is a legible + // orange used for warnings, distinct from the yellow used for notes. + ansiOrange = "\x1b[38;5;208m" +) + +// colorEnabled reports whether ANSI colour should be written to w. Colour is +// written only when three things hold: NO_COLOR is unset, w is a real *os.File +// (so text/tabwriter buffers, strings.Builder and bytes.Buffer tests all stay +// plain), and that file is a character device (a terminal, not a redirect). +func colorEnabled(w io.Writer) bool { + if os.Getenv("NO_COLOR") != "" { + return false + } + f, ok := w.(*os.File) + if !ok { + return false + } + fi, err := f.Stat() + if err != nil { + return false + } + return fi.Mode()&os.ModeCharDevice != 0 +} + +// styleLine applies the note vocabulary's colour to one line when enabled. The +// colour wraps the text only; the terminating newline is written uncoloured so +// the terminal's line discipline is never inside the escape sequence. +func styleLine(line string, enabled bool) string { + if !enabled { + return line + } + content := strings.TrimRight(line, "\n") + var color string + switch { + case strings.HasPrefix(content, "warning:"): + color = ansiOrange + case strings.HasPrefix(content, "note:"): + color = ansiYellow + case strings.HasPrefix(content, "drain wait:"): + color = ansiCyan + } + if color == "" { + return line + } + return color + content + ansiReset + "\n" +} + +// noteStyler wraps the gate package's Notes stream — a presentation seam that +// keeps colour out of the library. It colourises each line by its known prefix +// and separates the collection countdown from the setup phase with a single +// blank line before the first "collecting:" line. The gate keeps emitting plain +// prose; only the CLI lays it out. +type noteStyler struct { + w io.Writer + enabled bool + pending []byte + sawCollecting bool +} + +func newNoteStyler(w io.Writer) *noteStyler { + return ¬eStyler{w: w, enabled: colorEnabled(w)} +} + +// startsSection reports whether a line opens a new phase of the stream and so +// deserves a blank line above it. "collecting:" opens the countdown (once — +// later countdown lines follow on from the first), and "drain wait:" opens the +// drain phase. The setup lines (planned run time, warning, min-observed, notes) +// are one contiguous block and are not separated from each other. +func (s *noteStyler) startsSection(line string) bool { + switch { + case strings.HasPrefix(line, "warning:"): + return true + case strings.HasPrefix(line, "drain wait:"): + return true + case strings.HasPrefix(line, "collecting:"): + if s.sawCollecting { + return false + } + s.sawCollecting = true + return true + } + return false +} + +func (s *noteStyler) Write(p []byte) (int, error) { + n := len(p) + s.pending = append(s.pending, p...) + for { + i := bytes.IndexByte(s.pending, '\n') + if i < 0 { + break + } + line := string(s.pending[:i+1]) + s.pending = s.pending[i+1:] + + if s.startsSection(line) { + if _, err := io.WriteString(s.w, "\n"); err != nil { + return n, err + } + } + if _, err := io.WriteString(s.w, styleLine(line, s.enabled)); err != nil { + return n, err + } + } + return n, nil +} diff --git a/grafana-alertcheck/cmd/table.go b/grafana-alertcheck/cmd/table.go new file mode 100644 index 000000000..39ec75cbc --- /dev/null +++ b/grafana-alertcheck/cmd/table.go @@ -0,0 +1,173 @@ +package main + +import ( + "fmt" + "io" + "sort" + "text/tabwriter" + "time" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +// renderTable is the human table. It always writes to the writer it is given, +// which the caller (runCheck) always points at stderr — stdout is reserved for +// the machine-readable --output json. +// +// Three titled tables, in order (the name column is RULE in all of them — one +// row is one resolved alert rule, never a firing instance): +// +// 1. RESULTS, one line per rule: outcome, BadFor, pollEvery, proved-or-not +// with the largest gap; +// 2. VIOLATIONS, one line per Violation (only when any): a rule's worst-of +// outcome does not carry the State/Health of the instance that actually +// caused it — Violation does — so this is also where those two columns +// appear, sorted after the result table rather than folded into it, and it +// is the only place an operator running WITHOUT --output json sees the +// --allow-paused hint that Violation.Note already carries (classify.go); +// 3. THRESHOLDS, the numbers that answer "why" on exit 2: each non-skipped +// rule's maxGap/healthGrace/evalStaleAfter, followed by the global +// transitionGrace and drainTimeout, and the largest measured clock skew +// alongside its own error bound (RTT/2) — SkewHardLimit is a separate, +// fixed input threshold and is reported next to it, never as if it were +// that bound. +func renderTable(w io.Writer, res gate.Result) error { + alertOf := make(map[string]string, len(res.Verdicts)) + for _, v := range res.Verdicts { + alertOf[v.RuleUID] = v.Alert + } + + enabled := colorEnabled(w) + // A blank line separates the result table from the notes the gate streamed + // before it (planned run time, warning, min-observed, collecting, drain + // wait), so the verdict reads as its own section rather than the tail of a + // wall of progress text. + fmt.Fprintln(w) + + fmt.Fprintln(w, "RESULTS") + tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "RULE\tOUTCOME\tBADFOR\tPOLLEVERY\tPROVED\tNOTE") + for _, v := range sortedVerdicts(res.Verdicts) { + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", + v.Alert, v.Outcome, v.BadFor.Round(time.Second), v.PollEvery.Round(time.Second), + provedLabel(res.Coverage[v.RuleUID]), v.Note) + } + if err := tw.Flush(); err != nil { + return fmt.Errorf("render table: %w", err) + } + + if len(res.Violations) > 0 { + fmt.Fprintln(w, "\nVIOLATIONS") + vtw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + fmt.Fprintln(vtw, "RULE\tOUTCOME\tSTATE\tHEALTH\tNOTE") + for _, v := range sortedViolations(res.Violations) { + fmt.Fprintf(vtw, "%s\t%s\t%s\t%s\t%s\n", alertLabel(v, alertOf), v.Outcome, v.State, v.Health, v.Note) + } + if err := vtw.Flush(); err != nil { + return fmt.Errorf("render table: %w", err) + } + } + + // The per-rule thresholds answer "why" on exit 2: a table, not the prose + // "rule NAME: maxGap=... healthGrace=... evalStaleAfter=..." that repeated + // the rule name a fourth time. It is separated from the result above by a + // blank line. + fmt.Fprintln(w) + fmt.Fprintln(w, "THRESHOLDS") + ttw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + fmt.Fprintln(ttw, "RULE\tMAXGAP\tHEALTHGRACE\tEVALSTALEAFTER") + for _, uid := range sortedThresholdUIDs(res.Thresholds, alertOf) { + t := res.Thresholds[uid] + fmt.Fprintf(ttw, "%s\t%s\t%s\t%s\n", + alertOr(uid, alertOf), t.MaxGap, t.HealthGrace, t.EvalStaleAfter) + } + if err := ttw.Flush(); err != nil { + return fmt.Errorf("render table: %w", err) + } + + fmt.Fprintln(w) + fmt.Fprintf(w, "global: transitionGrace=%s (source: %s) drainTimeout=%s\n", + res.Global.TransitionGrace, res.Global.GraceSource, res.Global.DrainTimeout) + fmt.Fprintf(w, "largest measured clock skew: %s (bound ±%s, hard limit %s), grafana %s\n", + res.ClockSkew.Round(time.Millisecond), res.ClockSkewBound.Round(time.Millisecond), + gate.SkewHardLimit, res.GrafanaVersion) + // The verdict — the single number a terminal operator reads last — sits on + // its own line at the very bottom, separated from the diagnostics above and + // from the shell prompt below. + fmt.Fprintf(w, "\n%s\n\n", violationsLabel(len(res.Violations), enabled)) + return nil +} + +// violationsLabel colours the "violations: N" prefix of the footer: green for a +// clean run, red otherwise. The rest of the line is written uncoloured. +func violationsLabel(n int, enabled bool) string { + s := fmt.Sprintf("violations: %d", n) + if !enabled { + return s + } + if n == 0 { + return ansiGreen + s + ansiReset + } + return ansiRed + s + ansiReset +} + +// provedLabel is the table's PROVED column: "yes" for a clean coverage +// proof, "no" with the reason and largest gap for an unobservable rule, and +// "-" for a rule decide never asked proveCoverage about at all (skipped — +// paused before the window opened). +func provedLabel(cov gate.CoverageResult) string { + if cov.Reason == "" && !cov.Unobservable && !cov.Proved { + return "-" + } + if cov.Unobservable { + if cov.LargestGap > 0 { + return fmt.Sprintf("no (%s; largest gap %s at %s)", cov.Reason, + cov.LargestGap.Round(time.Second), cov.LargestGapAt.Format(time.RFC3339)) + } + return fmt.Sprintf("no (%s)", cov.Reason) + } + return "yes" +} + +// alertLabel resolves a Violation's alert name. Most violations already +// carry it directly; the synthetic MinObserved-shortfall entry with no named +// rule (classify.go) has an empty Alert and an empty RuleUID, so alertOf +// cannot resolve it either — "-" says plainly that this row is not about a +// specific rule. +func alertLabel(v gate.Violation, alertOf map[string]string) string { + if v.Alert != "" { + return v.Alert + } + if a, ok := alertOf[v.RuleUID]; ok { + return a + } + return "-" +} + +func alertOr(uid string, alertOf map[string]string) string { + if a, ok := alertOf[uid]; ok { + return a + } + return uid +} + +func sortedVerdicts(in []gate.RuleVerdict) []gate.RuleVerdict { + out := append([]gate.RuleVerdict(nil), in...) + sort.Slice(out, func(i, j int) bool { return out[i].Alert < out[j].Alert }) + return out +} + +func sortedViolations(in []gate.Violation) []gate.Violation { + out := append([]gate.Violation(nil), in...) + sort.SliceStable(out, func(i, j int) bool { return out[i].Alert < out[j].Alert }) + return out +} + +func sortedThresholdUIDs(thresholds map[string]gate.RuleThresholds, alertOf map[string]string) []string { + uids := make([]string, 0, len(thresholds)) + for uid := range thresholds { + uids = append(uids, uid) + } + sort.Slice(uids, func(i, j int) bool { return alertOr(uids[i], alertOf) < alertOr(uids[j], alertOf) }) + return uids +} diff --git a/grafana-alertcheck/cmd/table_test.go b/grafana-alertcheck/cmd/table_test.go new file mode 100644 index 000000000..86c80a19b --- /dev/null +++ b/grafana-alertcheck/cmd/table_test.go @@ -0,0 +1,88 @@ +package main + +import ( + "bytes" + "testing" + "time" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" + "github.com/stretchr/testify/require" +) + +// The golden table test: a fixed Result renders a deterministic, ordered rule +// table, a violations section and a footer carrying the per-rule and global +// thresholds plus the skew and its bound — with no live Check involved. +func TestRenderTable(t *testing.T) { + gapAt := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + res := gate.Result{ + GrafanaVersion: "13.1.0", + ClockSkew: 1500 * time.Millisecond, + ClockSkewBound: 250 * time.Millisecond, + Verdicts: []gate.RuleVerdict{ + {Alert: "Zebra Alert", RuleUID: "uid-z", Outcome: gate.OutcomeClean, PollEvery: 30 * time.Second}, + {Alert: "Ape Alert", RuleUID: "uid-a", Outcome: gate.OutcomeUnobservable, + PollEvery: 30 * time.Second, Note: "gap of 5m0s starting at 2026-01-01T12:00:00Z exceeds maxGap 1m0s"}, + {Alert: "Paused Alert", RuleUID: "uid-p", Outcome: gate.OutcomeSkipped, + Note: "paused before the window opened; counts against --min-observed unless --allow-paused is set"}, + }, + Violations: []gate.Violation{ + {Alert: "Ape Alert", RuleUID: "uid-a", Outcome: gate.OutcomeUnobservable, State: gate.StateFiring, Health: "error", Note: "unobservable"}, + {Alert: "Paused Alert", RuleUID: "uid-p", Outcome: gate.OutcomeSkipped, + Note: "paused before the window opened; counts against --min-observed unless --allow-paused is set"}, + }, + Coverage: map[string]gate.CoverageResult{ + "uid-z": {Proved: true}, + "uid-a": {Unobservable: true, Reason: gate.ReasonHeartbeatGap, LargestGap: 5 * time.Minute, LargestGapAt: gapAt}, + }, + Thresholds: map[string]gate.RuleThresholds{ + "uid-z": {MaxGap: time.Minute, HealthGrace: time.Minute, EvalStaleAfter: time.Minute}, + "uid-a": {MaxGap: time.Minute, HealthGrace: 2 * time.Minute, EvalStaleAfter: time.Minute}, + }, + Global: gate.GlobalThresholds{ + TransitionGrace: 5 * time.Minute, + GraceSource: `Ape Alert (for=5m)`, + DrainTimeout: 2 * time.Minute, + }, + } + + var buf bytes.Buffer + require.NoError(t, renderTable(&buf, res)) + out := buf.String() + + // Rule table: Ape sorts before Zebra sorts before... Paused is skipped and + // carries no coverage entry, so it renders "-" for PROVED. + require.Contains(t, out, "Ape Alert") + require.Contains(t, out, "unobservable") + require.Contains(t, out, "heartbeat_gap") + require.Contains(t, out, "largest gap 5m0s") + require.Contains(t, out, "Zebra Alert") + require.Contains(t, out, "clean") + + // The violations section must show up even without --output json, and must + // carry the --allow-paused hint text verbatim. + require.Contains(t, out, "VIOLATIONS") + require.Contains(t, out, "--allow-paused") + require.Contains(t, out, "STATE") + require.Contains(t, out, "HEALTH") + require.Contains(t, out, string(gate.StateFiring)) + require.Contains(t, out, "error") + + // The footer: per-rule thresholds are a table (RULE/MAXGAP/HEALTHGRACE/ + // EVALSTALEAFTER) rather than prose, followed by the global thresholds and + // the violations count with the skew and its own bound rather than the + // fixed hard limit. + require.Contains(t, out, "MAXGAP") + require.Contains(t, out, "HEALTHGRACE") + require.Contains(t, out, "EVALSTALEAFTER") + require.Contains(t, out, "global: transitionGrace=5m0s (source: Ape Alert (for=5m)) drainTimeout=2m0s") + require.Contains(t, out, "largest measured clock skew: 1.5s (bound ±250ms, hard limit 1m0s)") + require.Contains(t, out, "violations: 2") + require.Contains(t, out, "13.1.0") +} + +// The "-" case: a rule decide never asked proveCoverage about (paused before +// the window opened) has an empty CoverageResult and must not be reported as +// either proved or unobservable. +func TestProvedLabel_Skipped(t *testing.T) { + require.Equal(t, "-", provedLabel(gate.CoverageResult{})) +} diff --git a/grafana-alertcheck/cmd/watch.go b/grafana-alertcheck/cmd/watch.go new file mode 100644 index 000000000..69b585298 --- /dev/null +++ b/grafana-alertcheck/cmd/watch.go @@ -0,0 +1,153 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "time" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +const watchUsage = "usage: grafana-alertcheck watch --out [--pidfile F] [--daemon-log F] " + + "--alerts [--folder F] [--poll-interval D] [--concurrency N] [--until RFC3339]" + +// runWatch is the record step's entire CLI surface, split in two by one flag +// set — gate.DaemonChildFlag ("--daemon-child") and gate.ReadyFDFlag +// ("--ready-fd") select which side of the parent/child split this invocation +// is: +// +// - without them: the record command an operator types. It parses --out, +// --alerts and the rest, builds a gate.WatchConfig and calls gate.Watch, +// which resolves, records the first observation of every rule, and +// detaches the recorder before returning. +// - with them: the detached recorder itself. gate.Watch's own childArgs +// (watch_unix.go) is the only thing that ever sets them — an operator +// never types "--daemon-child" and it does not appear in watchUsage — and +// this dispatches straight to gate.RunDaemonChild. +// +// Both flags live in the SAME flag set as the operator-facing ones rather +// than a second, hidden set: the child is started with childArgs' exact +// argv, e.g. "watch --daemon-child --out log.jsonl --ready-fd 3 +// [--until ...] [--concurrency ...]", and a second parser would have to stay +// byte-for-byte in sync with that slice to accept it. +func runWatch(args []string, stdin io.Reader, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("watch", flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprintln(stderr, watchUsage) } + + common := registerCommon(fs) + out := fs.String("out", "", "JSONL log path to record to") + pidfile := fs.String("pidfile", "", "pidfile path (default .pid)") + daemonLog := fs.String("daemon-log", "", "stdout/stderr sink for the detached recorder (default .daemon.log)") + until := fs.String("until", "", "optional hard stop, RFC3339 (default: run until check stops it)") + pollInterval := fs.String("poll-interval", "", "override every rule's poll cadence (default: half its own evaluation interval)") + + // Hidden: never in watchUsage, never typed by an operator (see doc comment). + daemonChild := fs.Bool(gate.DaemonChildFlag[2:], false, "") + readyFD := fs.Int(gate.ReadyFDFlag[2:], 0, "") + + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() != 0 { + fmt.Fprintf(stderr, "watch: unexpected arguments %v\n", fs.Args()) + return 2 + } + + if *daemonChild { + return runDaemonChild(*out, *until, *common.concurrency, *readyFD, stderr) + } + + url, token, err := grafanaEnv() + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + + alerts, err := readAlerts(stdin, *common.alerts) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + + cfg := gate.WatchConfig{ + URL: url, + Token: token, + Alerts: alerts, + Folder: *common.folder, + Out: *out, + PidFile: *pidfile, + DaemonLog: *daemonLog, + Concurrency: *common.concurrency, + Clock: gate.SystemClock{}, + Notes: newNoteStyler(stderr), + } + if *until != "" { + t, err := time.Parse(time.RFC3339, *until) + if err != nil { + fmt.Fprintf(stderr, "--until: %v\n", err) + return 2 + } + cfg.Until = t + } + if *pollInterval != "" { + d, err := time.ParseDuration(*pollInterval) + if err != nil { + fmt.Fprintf(stderr, "--poll-interval: %v\n", err) + return 2 + } + cfg.PollEvery = d + } + + if err := gate.Watch(context.Background(), cfg); err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + return 0 +} + +// runDaemonChild is the detached recorder's whole entry point. Its stdout and +// stderr are already the daemon log file — spawnChild (watch_unix.go) +// redirects both before Start — so writing to stderr here lands exactly where +// waitForChildReady's failure path quotes from. +func runDaemonChild(out, until string, concurrency, readyFD int, stderr io.Writer) int { + url, token, err := grafanaEnv() + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + cfg := gate.DaemonChildConfig{ + URL: url, + Token: token, + Out: out, + Concurrency: concurrency, + Clock: gate.SystemClock{}, + ReadyFD: readyFD, + } + if until != "" { + t, err := time.Parse(time.RFC3339, until) + if err != nil { + fmt.Fprintf(stderr, "--until: %v\n", err) + return 2 + } + cfg.Until = t + } + if err := gate.RunDaemonChild(context.Background(), cfg); err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + return 0 +} + +// The daemon-child and ready-fd flags registered above must keep matching +// watch_unix.go's childArgs, which names exactly --daemon-child, --out, +// --ready-fd, --until and --concurrency and nothing else: that function +// builds this process's own argv when it re-execs itself as the detached +// recorder, so a flag added to one side without the other means the child +// fails on its very first flag.Parse. diff --git a/grafana-alertcheck/cmd/watch_test.go b/grafana-alertcheck/cmd/watch_test.go new file mode 100644 index 000000000..5c8f53c55 --- /dev/null +++ b/grafana-alertcheck/cmd/watch_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "bytes" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +// The record step's flag-validation matrix. Every case fails inside +// gate.WatchConfig.validate() or before it, so none needs a reachable Grafana. +func TestRunWatch_FlagValidation(t *testing.T) { + tests := []struct { + name string + env bool + args func(t *testing.T) []string + wantErr string + }{ + {"missing env", false, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl", "--alerts", writeTempAlerts(t)} + }, "GRAFANA_URL"}, + {"missing out", true, func(t *testing.T) []string { + return []string{"--alerts", writeTempAlerts(t)} + }, "no log path"}, + {"missing alerts", true, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl"} + }, "no alert names"}, + {"bad until format", true, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl", "--alerts", writeTempAlerts(t), "--until", "not-a-time"} + }, "--until"}, + {"until in the past", true, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl", "--alerts", writeTempAlerts(t), "--until", "2000-01-01T00:00:00Z"} + }, "not in the future"}, + {"bad poll-interval", true, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl", "--alerts", writeTempAlerts(t), "--poll-interval", "not-a-duration"} + }, "--poll-interval"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env { + t.Setenv("GRAFANA_URL", "http://example.invalid") + t.Setenv("GRAFANA_TOKEN", "test-token") + } else { + t.Setenv("GRAFANA_URL", "") + t.Setenv("GRAFANA_TOKEN", "") + } + var stdout, stderr bytes.Buffer + args := append([]string{"watch"}, tt.args(t)...) + code := run(args, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), tt.wantErr) + }) + } +} + +// Seeing gate.DaemonChildFlag must dispatch to gate.RunDaemonChild, and the +// flag must never appear in watchUsage (an operator never types it). +func TestRunWatch_DaemonChildDispatch(t *testing.T) { + t.Setenv("GRAFANA_URL", "http://example.invalid") + t.Setenv("GRAFANA_TOKEN", "test-token") + + var stdout, stderr bytes.Buffer + // No log at this path: RunDaemonChild fails trying to read it, which is + // enough to prove dispatch happened without needing a real recording. + missing := os.DevNull + ".missing" + code := run([]string{"watch", "--daemon-child", "--out", missing, "--ready-fd", "0"}, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), missing) + require.NotContains(t, watchUsage, "daemon-child") + require.NotContains(t, watchUsage, "ready-fd") +} + +func TestRunWatch_DaemonChild_MissingEnv(t *testing.T) { + t.Setenv("GRAFANA_URL", "") + t.Setenv("GRAFANA_TOKEN", "") + + var stdout, stderr bytes.Buffer + code := run([]string{"watch", "--daemon-child", "--out", "log.jsonl"}, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), "GRAFANA_URL") +} diff --git a/grafana-alertcheck/docs/_category_.yaml b/grafana-alertcheck/docs/_category_.yaml new file mode 100644 index 000000000..3cbc431c4 --- /dev/null +++ b/grafana-alertcheck/docs/_category_.yaml @@ -0,0 +1,8 @@ +position: 1 +label: 'Grafana Alertcheck' +collapsible: true +collapsed: false +link: + type: generated-index + slug: /platform-services/devex/cicd/grafana-alertcheck/index + description: 'CD quality gate for Grafana alerts: watch, classify, and gate releases.' diff --git a/grafana-alertcheck/docs/advanced.md b/grafana-alertcheck/docs/advanced.md new file mode 100644 index 000000000..073292cd6 --- /dev/null +++ b/grafana-alertcheck/docs/advanced.md @@ -0,0 +1,41 @@ +--- +id: grafana-alertcheck-advanced +title: Check budget and scheduling +sidebar_label: Budget and scheduling +sidebar_position: 2 +description: Why grafana-alertcheck schedules per rule, how the request budget works, and why it never queries state history. +--- + +# Check budget and scheduling + +## Per-rule schedules, never a global cycle + +Each rule polls at its **own** cadence, `--poll-interval` (default: half the rule's own evaluation interval). There is deliberately no single global minimum-interval cycle. + +One rule at `intervalSeconds=10` beside twenty at `300` keeps a 5 s cadence for itself and 150 s for the other twenty — not a 5 s cycle for all of them, which would be a 60× request bloat at ~1.8 s per request and would fail to start on a reasonable fleet. + +The scheduler staggers each rule's initial next-due time across its cadence, and serves due rules **earliest-due-first**, so a tight rule never queues behind slack ones. + +## The check budget + +The gate records one observation of every rule up front and checks the schedule against those **measured** latencies (payload sizes vary ~230× across rules, so a fixed estimate is meaningless). It errors at start — before waiting — if any of three conditions hold: + +- **Utilization** — total request rate exceeds `--concurrency`. +- **Per-rule** — one rule's request can't fit its own cadence. +- **Burst bound** — the slowest request exceeds the fleet's tightest cadence, which can open a mid-run gap. + +The error names the three levers only: raise `--concurrency`, raise `--poll-interval`, or watch fewer alerts. It never prescribes a single interval. + +## Why the gate never queries state history + +Querying Grafana's alert state history after the fact fails closed *in the wrong direction* — it returns "pass" when the truth is unknown: + +- History stores **transitions**, not states. An alert firing through the whole window has its only record *before* the window. +- The annotations API **does not serve Loki-backed history** at all. +- An empty result is indistinguishable from a healthy one: no alert fired, the backend differs, retention removed data, the token lacked permission — all look identical. +- There is **no coverage signal** — nothing proves the history is complete to time T. +- Artifact transitions (`Paused`, `RuleDeleted`, `Updated`, `MissingSeries`) look like recoveries. + +Instead, `watch` records its own evidence live and the log becomes the source of truth. The trade-off: the gate can miss an episode shorter than a rule's poll interval, though `activeAt` still surfaces sub-interval onsets for instances still active at a poll. + +A corollary of recording fresh: there is no replay. Re-running a failed job is a new deploy with a new `from` and a new recording — never a re-classification of old evidence. diff --git a/grafana-alertcheck/docs/architecture.md b/grafana-alertcheck/docs/architecture.md new file mode 100644 index 000000000..ca493220e --- /dev/null +++ b/grafana-alertcheck/docs/architecture.md @@ -0,0 +1,68 @@ +--- +id: grafana-alertcheck-architecture +title: Architecture +sidebar_label: Architecture +sidebar_position: 3 +description: The design invariants, pure-function seam, and recorder lifecycle of grafana-alertcheck, for maintainers. +--- + +# Architecture + +This page documents the invariants and seams a maintainer must not break. It exists because most of them are the difference between a gate that fails closed and one that silently passes broken windows. + +## Fail-closed invariants + +The gate must stop the release if it cannot get an answer. Every rule below is a specific instance of that: + +- **An error is never a pass.** A pass is exactly `len(Violations) == 0 && err == nil`. Every error path leaves `err` non-nil, and the CLI maps that to exit `2` unconditionally. +- **Inability beats violation.** Any `unobservable` rule is exit `2`, even alongside a real violation found first. +- **Absent never means normal.** An instance that leaves the bad set is looked up in the *same* response: present as `normal` → cleared; absent (or `MissingSeries`) → vanished (a discontinuity, not a recovery). +- **Staleness is absolute.** `grafana_now − lastEvaluation` is compared against a threshold, never "did it increase since the last poll" — a delta check reports stale on ~half the polls of a healthy rule. +- **`grafana_now` is the response `Date` header.** Never the runner clock, in any comparison against a Grafana timestamp. +- **No early exit.** `check` collects to `to + transitionGrace` before classifying once. +- **No replay.** No run-id key, no artifact download, no state between attempts. A retry is a new deploy. + +## The pure-function seam + +All correctness lives in two phases written as **pure functions** over a flat list of polls — no HTTP, no files, no clock, no goroutines: + +``` +HTTP ──> Source ──> []StateRule ──> reduce ──> []Poll ──> proveCoverage ──> decide ──> Result + │ + JSONL log ──> ReadLog ──┘ +``` + +- `proveCoverage` (the nine coverage checks) and `decide` (the instance timelines and outcomes) are pure; tests drive them with `[]Poll` literals and a fake `Clock`, with no sleeping or fixture server. +- `Check`/`Watch` are I/O shells: HTTP, signals, the pidfile, file reads, the countdown print. The only test doubles needed are the `Source` and `Clock` interfaces. +- `Policy` is the narrowed view of `Config` that reaches the pure layer — classification knobs and the window, no URL and no token. The token must never cross that line, which is the cheapest guarantee it never lands in an error string or a result. + +## Strict parsing as the version guard + +Both API responses are parsed strictly: a missing or unparseable **required** field (`health`, `state`, `lastEvaluation`, `interval`) is an error, never a zero value. Optional keys (`alerts`, `totals`, `labels`, `keepFiringFor`) are absent-tolerant, and unknown keys are ignored — so Grafana can add fields without breaking the parser, but removing one fails loudly. + +This, plus the declared supported range (Grafana >= 13.0.0, < 14.0.0), is how a deprecation or schema change is caught instead of silently misread. + +## The recorder lifecycle + +`watch` detaches a background recorder so observation survives the step boundary: + +1. Parent resolves names, writes the header, observes every non-paused rule once, checks the budget. +2. Parent re-execs itself as the child (`--daemon-child`) under a new session/process group, stdout/stderr to the daemon log. +3. Child re-reads the header, reopens the log `O_APPEND`, takes the exclusive `flock`, and writes one readiness byte on `--ready-fd`. +4. Parent writes the pidfile **after** the readiness report, then returns. + +Two authorities, only one of which is evidence: + +- The **pidfile** says a recording ever started (written only after ready, removed on failure). It can go stale — a pid gets reused. +- The **flock** says a writer exists *now*. The kernel drops it on exit, so the lock is always authoritative. + +On a clean stop (SIGTERM/SIGINT/`--until`) the child finishes the in-flight write, appends the `stopped` sentinel, fsyncs, and exits. A hard error writes no sentinel — so a recorder that died reads exactly like a coverage gap, because it is one. + +`check` signals via the pidfile, waits for the **lock** to release (never the pid), and only then reads the log once. Reading while a writer can still append can only produce a shorter window than was recorded. + +## The log is the source of truth + +`watch` records raw evidence, so nothing trusts a state that could become unreachable. Two consequences a maintainer must preserve: + +- The **header is authoritative for recording facts** (the cadence actually used, the URL, the alert set); the ruler API is authoritative for **rule facts** (`for`, `intervalSeconds`, kind). `check` always re-resolves definitions fresh and never reconstructs them from the header — the header duplicates `for`/`interval` only so the uploaded artifact is self-describing. +- The **cadence authority** is the header's `poll_every_seconds`, not the definitions. Re-deriving it would compare gaps recorded at an override cadence against default-cadence thresholds — fail-open in the faster-override direction. diff --git a/grafana-alertcheck/docs/how-alerts-are-evaluated.md b/grafana-alertcheck/docs/how-alerts-are-evaluated.md new file mode 100644 index 000000000..7c9cb3b6a --- /dev/null +++ b/grafana-alertcheck/docs/how-alerts-are-evaluated.md @@ -0,0 +1,85 @@ +--- +id: grafana-alertcheck-evaluation +title: How alerts are evaluated +sidebar_label: How alerts are evaluated +sidebar_position: 1 +description: The verdict model, instance timelines, and coverage proof behind grafana-alertcheck. +--- + +# How alerts are evaluated + +Both `watch`+`check` (recorder mode) and `check` alone (single-step mode) converge on the same input: a flat list of polls. Everything below runs over that list; the mode only changes where the polls came from. + +## Instance states + +Grafana reports instance states in two vocabularies (`Alerting`/`Normal` at instance level, `firing`/`inactive` at rule level). The gate normalizes every instance to one canonical set: + +| Canonical | Meaning | +| --------- | ------- | +| `normal` | Healthy | +| `firing` | The condition is true and `for` has elapsed | +| `pending` | The condition is true, `for` has not elapsed | +| `nodata` | The query returned no series (synthetic instance) | +| `error` | The query failed (synthetic instance) | + +A rule's **rule-level** `state` and `health` are kept verbatim and only reported — they are never classified. The **instance** state is what the classifier reasons about. + +A "bad" instance is one whose canonical state is in `--states` (default `firing`). `pending` and `nodata` are excluded by default. + +## Verdict model + +For each instance the gate builds a timeline of bad spans over `[from, to]`, then takes the worst outcome across a rule's instances as the rule's outcome. + +| Outcome | Shape | Exit | +| ------- | ----- | ---- | +| `clean` | Good throughout, observed throughout | → 0 | +| `newly_bad` | Entered a bad state **inside** the window | → 1 | +| `persistently_bad` | Bad at `from`, still bad at `to` | → 1 | +| `recovered` | Bad at `from`, cleared before `to`, stayed clear | → 0 | +| `flapping` | Cleared, then became bad again | → 1 | +| `skipped` | Paused **before** the window opened | reported, not observable | +| `unobservable` | Coverage gap / sustained `health=error` / stale / absent | → 2 | + +`recovered` has **no deadline** — an alert that clears at minute 58 of a 60-minute window still passes. The total bad time is reported as `BadFor`; the removed deadline is replaced by that measured value rather than a derived limit. + +### Preexisting policy + +For an instance already bad when `from` opened, `--preexisting` decides: + +- `fail-unless-recovered` (default) — clears and stays clear → pass; never clears → fail. +- `fail` — any preexisting instance fails, recovered or not. +- `ignore` — preexisting instances are disregarded; only new episodes fail. + +## Cleared vs vanished + +When an instance leaves the bad set, the gate looks it up **in the same response**: + +- Present as `normal` → `cleared` (a real recovery). +- Absent, or present as `normal (MissingSeries)` → `vanished` (a discontinuity, **not** a recovery). + +A vanished instance that was bad stays `persistently_bad`. A metric that stops being emitted is not evidence of health — this is deliberate and can surprise users whose fix is to remove a metric rather than drive it to a good value. + +## Coverage proof + +Before classifying, `check` must **prove** continuous coverage of `[from, to]` for each alert. Nine checks run; any failure makes the rule `unobservable`: + +1. **Sentinel** — a clean recorder stop, timestamped at or after `to + transitionGrace`. A recorder that died mid-window looks exactly like a coverage gap and is one. +2. **`from` bounds** — `from` earlier than the recording start is unprovable. +3. **Heartbeat gap** — any gap larger than `maxGap` (= 2 × poll cadence) inside the window. Data at both ends with a hole between is not enough. +4. **`health=error`** — a contiguous run longer than `healthGrace` consumes coverage; a short blip is a note. +5. **`health=nodata`** — a note, never fatal (unless `--nodata-is-unobservable`). +6. **Liveness** — `grafana_now − lastEvaluation` must not exceed `evalStaleAfter`. This is an **absolute** check, never a "did it increase since the last poll" delta. +7. **In-window pause** — a poll reporting `isPaused` mid-window is `unobservable` (the primary pause detector). +8. **Rule absent** — an authoritative `2xx` with no matching rule. +9. **`KeepLast`** — a note naming a stale-state blind spot. + +## Health: `error` vs `nodata` + +- `health=error` means the query **failed** — a malfunction. Sustained past `healthGrace`, it makes the rule `unobservable`. +- `health=nodata` means the query **ran and returned no series** — indistinguishable from a quiet system. It is not fatal by default; most of a fleet runs `no_data_state: OK`. + +## The drain wait and `transitionGrace` + +A condition that arises just before `to` becomes `firing` only at the first evaluation after its `for` elapses. `transitionGrace` (derived from the watched rules' `for` values) extends the classification bound past `to` so such a surfacing condition is caught. After collection, a **drain wait** polls until each rule has evaluated through `to + transitionGrace` (bounded by `drainTimeout`); a rule that never does is `unobservable`. + +Run time = `(to − from) + transitionGrace + drainTimeout`. This is printed at start, and the grace is warned about when it exceeds a quarter of the window — the window may be too short for the alert's `for`. diff --git a/grafana-alertcheck/docs/index.md b/grafana-alertcheck/docs/index.md new file mode 100644 index 000000000..95d413772 --- /dev/null +++ b/grafana-alertcheck/docs/index.md @@ -0,0 +1,87 @@ +--- +id: grafana-alertcheck-index +title: Grafana Alertcheck +sidebar_label: Overview +sidebar_position: 0 +description: A CD quality gate that bookends a release with alert-state observation and answers whether any watched Grafana alert was bad during the release window. +--- + +# Grafana Alertcheck + +`grafana-alertcheck` is a CD quality gate for Grafana alerts. It bookends a release with two commands — `watch` (record) and `check` (classify) — and answers one question: + +> A release finished at time T. Was any of these Grafana alerts in a bad state during the next N minutes? + +The contract is `watch → your work → check`. Between the two you run whatever you want (deploy, tests, migration); the gate only observes, then classifies. + +It **fails closed**: if it cannot get an answer, it stops the release. It never passes an unproven window. + +## How it works, in one paragraph + +`watch` starts a background recorder that polls each named alert and appends snapshots to a JSONL log. Your work then emits two RFC3339 timestamps — `from` (when the change landed) and `to` (when the work ended). `check` proves continuous coverage of `[from, to]`, builds a state timeline per alert, classifies it, and exits `0`, `1`, or `2`. + +## Install + +```bash +go install github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd@latest +``` + +Connection details come from the environment — the token is env-only, never a flag: + +```bash +export GRAFANA_URL=https://grafana.example.com +export GRAFANA_TOKEN=… +``` + +Requires Grafana >= 13.0.0 and < 14.0.0. Outside that range the gate exits `2`. + +## Quickstart — recorder mode + +```bash +grafana-alertcheck watch --out /tmp/run.jsonl --alerts alerts.txt +./deploy.sh # emits deployed_at= when the rollout is stable +./verify.sh # emits finished_at= when the work is done +grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finished_at" +``` + +`alerts.txt` holds one alert name per line. See [Naming alerts](./reference/cli#naming-alerts). + +`watch` returns only after the recorder has observed every named, non-paused alert once and reported ready — so auth, name-resolution, and parse failures surface **before** your deploy runs. + +## Quickstart — single-step mode + +Skip the recorder and observe the window inline, from inside `check` itself: + +```bash +grafana-alertcheck check --alerts alerts.txt --to "$finished_at" +``` + +In single-step mode the window starts at `check`'s first observation; if you give no `--from`, the interval before that first observation is declared as a blind spot with a warning (not an error). + +## Exit codes + +| Code | Meaning | +| ---- | ------- | +| `0` | Pass — no violations | +| `1` | Violations (including a paused-rule-only `--min-observed` shortfall) | +| `2` | The gate could not check — config, auth, resolution, a coverage gap, health/staleness, the drain limit, transport, … | + +An error is never a pass: `2` wins over any violation found alongside it. + +## Common surprises + +- A **paused rule fails by default** — even one someone else paused. Use `--allow-paused`. +- A fix that **stops emitting a metric is not a recovery** — the instance vanishes, which is a discontinuity, not health. +- The gate checks alert **state and health**, not notification delivery — a silenced alert that still fires fails. +- `recovered` has **no deadline** — a bad-at-`from` alert that clears by `to` passes; set `--preexisting fail` to forbid it. +- A **retry is a new deploy**, not a replay — re-running the job re-records against a new `from`. +- `watch` and `check` must run in **one job, one runner, one filesystem** — nothing persists across jobs or attempts. +- The gate **never exits early** — a violation at minute 2 still holds the runner to `to + transitionGrace + drainTimeout`; size the job timeout to the planned run time the gate prints at start. + +## More + +- [How alerts are evaluated](./how-alerts-are-evaluated) — the verdict model and coverage proof +- [Check budget and scheduling](./advanced) — why the schedule and budget look the way they do, and why history isn't queried +- [Architecture](./architecture) — design invariants and the recorder lifecycle, for maintainers +- [CLI reference](./reference/cli) — every subcommand and flag +- [Log format](./reference/log-format) — the JSONL log schema, for debugging artifacts diff --git a/grafana-alertcheck/docs/reference/_category_.yaml b/grafana-alertcheck/docs/reference/_category_.yaml new file mode 100644 index 000000000..2e5d50946 --- /dev/null +++ b/grafana-alertcheck/docs/reference/_category_.yaml @@ -0,0 +1,8 @@ +position: 3 +label: Reference +collapsible: true +collapsed: false +link: + type: generated-index + slug: /platform-services/devex/cicd/grafana-alertcheck/reference + description: 'CLI reference for grafana-alertcheck.' diff --git a/grafana-alertcheck/docs/reference/cli.md b/grafana-alertcheck/docs/reference/cli.md new file mode 100644 index 000000000..1530c508d --- /dev/null +++ b/grafana-alertcheck/docs/reference/cli.md @@ -0,0 +1,91 @@ +--- +id: grafana-alertcheck-cli +title: CLI reference +sidebar_label: CLI reference +sidebar_position: 0 +description: Full reference for the grafana-alertcheck CLI: watch, check, list, environment, naming, and output. +--- + +# CLI reference + +``` +grafana-alertcheck +``` + +Connection details are always from the environment: `GRAFANA_URL` and `GRAFANA_TOKEN`. The token is never a flag and never logged. + +## `list` + +Lists every rule from the ruler endpoint — kind, folder, group, title, uid. Useful to check auth and to find `uid:` names. + +```bash +grafana-alertcheck list +``` + +## `watch` — record + +```bash +grafana-alertcheck watch --out [--pidfile F] [--daemon-log F] \ + --alerts [--folder F] [--poll-interval D] [--concurrency N] [--until RFC3339] +``` + +| Flag | Default | Meaning | +| ---- | ------- | ------- | +| `--out` | — | JSONL log path (required) | +| `--pidfile` | `.pid` | Where the recorder's pid is written | +| `--daemon-log` | `.daemon.log` | stdout/stderr sink for the detached recorder | +| `--alerts` | — | File of alert names, one per line, or `-` for stdin (required) | +| `--folder` | — | Default folder to scope unqualified names | +| `--poll-interval` | half the rule's interval | Override every rule's cadence (never clamped) | +| `--concurrency` | `1` | Max concurrent requests to Grafana | +| `--until` | run until signalled | Optional hard stop | + +`watch` writes the header, observes every non-paused rule once, checks the budget, then detaches a background recorder and returns. Recording is **unfiltered** — there is no `--states` here, so the same log can be re-classified later under different `--states` without re-recording. + +## `check` — classify + +```bash +grafana-alertcheck check [--in ] [--pidfile F] --from RFC3339 --to RFC3339 \ + [--alerts ...] [--folder F] [--states ...] [--preexisting ...] [--min-observed N] \ + [--allow-paused] [--nodata-is-unobservable] [--concurrency N] [--output json] +``` + +| Flag | Default | Meaning | +| ---- | ------- | ------- | +| `--in` | — | Log recorded by `watch`; empty selects single-step mode | +| `--pidfile` | `.pid` | Recorder to stop before reading `--in` | +| `--from` | see below | Moment the deploy finished | +| `--to` | — | End of the window (required) | +| `--alerts` | — | Required **without** `--in`; refused **with** `--in` | +| `--states` | `firing` | Comma-separated bad states: `firing,pending,nodata,error` | +| `--preexisting` | `fail-unless-recovered` | `fail-unless-recovered` \| `fail` \| `ignore` | +| `--min-observed` | every resolved rule | Minimum rules that must be observed | +| `--allow-paused` | `false` | Don't count pre-window-paused rules against `--min-observed` | +| `--nodata-is-unobservable` | `false` | Treat sustained `health=nodata` as unobservable | +| `--concurrency` | `1` | Max concurrent requests | +| `--output` | `table` | `json` also writes the machine-readable result to stdout | + +`--from` and `--to` are RFC3339 with an explicit offset and must come from your work — `from` from the deploy step, `to` from the step that finishes. In recorder mode an absent `--from` is a hard error; in single-step mode it falls back (with a warning) to the start of the step. + +## Naming alerts + +Alert names take one of four forms: + +| Form | Meaning | +| ---- | ------- | +| `HighErrorRate` | Title only, scoped by `--folder` | +| `Platform/HighErrorRate` | Folder + title | +| `Platform/api/HighErrorRate` | Folder + group + title (always unique) | +| `uid:abc123` | Exact uid (present on both endpoints) | + +Datasource-managed and recording rules are refused with a specific error. A name matching multiple rules errors listing every candidate with the copyable `Folder/Group/Title` and its `uid:` form. A no-match errors with case-insensitive substring suggestions and points at `list`. Duplicate names that resolve to the same uid collapse to one (a note, not an error). + +## Output and exit codes + +The human table goes to **stderr**: `RESULTS` (one row per rule), `VIOLATIONS` (one per violation), and `THRESHOLDS` (each rule's `maxGap`/`healthGrace`/`evalStaleAfter` plus global `transitionGrace`/`drainTimeout` and the largest measured clock skew). `--output json` writes the result to stdout. + +| Code | Meaning | +| ---- | ------- | +| `0` | Pass | +| `1` | Violations | +| `2` | Could not check — every library error, never a pass | diff --git a/grafana-alertcheck/docs/reference/log-format.md b/grafana-alertcheck/docs/reference/log-format.md new file mode 100644 index 000000000..66e071770 --- /dev/null +++ b/grafana-alertcheck/docs/reference/log-format.md @@ -0,0 +1,98 @@ +--- +id: grafana-alertcheck-log-format +title: Log format +sidebar_label: Log format +sidebar_position: 1 +description: The JSONL log schema written by watch and read by check, for debugging the forensic artifact. +--- + +# Log format + +`watch` records evidence to a JSONL log — one JSON object per line. A poll record *is* the heartbeat; there is no separate heartbeat type. + +## Record types + +Exactly three: + +| `type` | Meaning | +| ------ | ------- | +| `header` | Line 1 — identity and the alert set | +| `poll` | One reduced observation of one rule | +| `stopped` | The sentinel, written on a clean stop only | + +The header must be line 1, appear once, and carry `schema_version` `1` (any other value is a read error). Any unparseable line — including the last, or one after the sentinel — makes the log unreadable: a truncated log is evidence the recorder was killed, and must not pass. + +## Header + +```json +{ + "type": "header", + "schema_version": 1, + "url": "https://grafana.example.com", + "grafana_version": "13.1.0", + "started_at": "2026-09-07T10:00:00Z", + "rules": [ + { + "uid": "rule0000001", + "title": "HighErrorRate", + "folder": "Platform", + "group": "api", + "for_seconds": 300, + "interval_seconds": 60, + "is_paused": false, + "no_data_state": "OK", + "exec_err_state": "OK", + "poll_every_seconds": 30 + } + ] +} +``` + +- `url` and `rules` are the log's identity — `check` validates them against the current environment and a fresh ruler read. +- `is_paused` records the pause state at record start (the moment `skipped` means). +- `poll_every_seconds` is the cadence the recording **actually used** (after any `--poll-interval` override). `check` derives `maxGap` from it, never from `interval_seconds`. +- `for_seconds`, `interval_seconds`, `no_data_state`, `exec_err_state` are forensic only — `check` re-resolves definitions and never reads them back. + +## Poll + +```json +{ + "type": "poll", + "rule_uid": "rule0000001", + "grafana_now": "2026-09-07T10:00:30Z", + "skew_ms": 20, + "skew_bound_ms": 40, + "latency_ms": 123, + "found": true, + "state": "inactive", + "health": "ok", + "last_evaluation": "2026-09-07T10:00:28Z", + "is_paused": false, + "histogram": { "alerting": 0, "normal": 2004 }, + "reasons": { "NoData": 1091 }, + "abnormal": [ { "labels": { "env": "prod" }, "state": "firing", "active_at": "2026-09-07T09:50:00Z", "value": "1.5" } ], + "cleared": [ "env=prod\u0001..." ], + "vanished": [] +} +``` + +Field notes: + +- `grafana_now` is the response's `Date` header — never the runner clock. +- `skew_ms`/`skew_bound_ms` are the per-poll clock-skew estimate and its uncertainty (RTT/2), in milliseconds for compactness only. +- `found: false` is an authoritative `2xx` in which this rule was absent — a transport failure is retried and never becomes a poll. +- `state`, `health`, `last_error` are raw rule-level strings, reporting-only. +- `histogram` is a verbatim copy of the response `totals`; written, never analysed. +- `reasons` counts non-empty instance reasons (`NoData`, `Error`, `KeepLast`, …); composite states stay visible only here. +- `abnormal` holds only instances whose **canonical** state is not `normal`. +- `cleared`/`vanished` are instance keys that left the bad set, resolved against the same response: `cleared` = a real recovery; `vanished` = a discontinuity, never a recovery. + +Instance keys are a sorted `k=v\n` join of labels, so they correlate across polls without hashing. + +## Stopped + +```json +{ "type": "stopped", "at": "2026-09-07T10:10:30Z" } +``` + +`at` is the recorder's own stop time. `check` compares it against `to + transitionGrace`; absent or earlier is `unobservable` — never a pass. diff --git a/grafana-alertcheck/go.mod b/grafana-alertcheck/go.mod index b0c8511ce..d5e0c88be 100644 --- a/grafana-alertcheck/go.mod +++ b/grafana-alertcheck/go.mod @@ -1,3 +1,7 @@ module github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck go 1.26.6 + +require github.com/stretchr/testify v1.12.1 + +require go.yaml.in/yaml/v3 v3.0.5 // indirect diff --git a/grafana-alertcheck/go.sum b/grafana-alertcheck/go.sum new file mode 100644 index 000000000..c2336837e --- /dev/null +++ b/grafana-alertcheck/go.sum @@ -0,0 +1,4 @@ +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= diff --git a/grafana-alertcheck/internal/gate/check.go b/grafana-alertcheck/internal/gate/check.go new file mode 100644 index 000000000..ef6543b71 --- /dev/null +++ b/grafana-alertcheck/internal/gate/check.go @@ -0,0 +1,878 @@ +package gate + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "sort" + "strings" + "time" +) + +// Check returns (Result, error) and no exit code: the code is a presentation +// decision the CLI makes. err != nil is exit 2 unconditionally, even alongside +// real violations; violations with err == nil is exit 1; neither is exit 0. +// Check never reads the environment — the CLI reads URL/token and passes them +// in, and the token must never reach a *flag.FlagSet. + +// countdownEvery is how often the collection loop reports what it is waiting +// for; a silent wait is indistinguishable from a hung process. +const countdownEvery = 30 * time.Second + +// recorderStopTimeout bounds the wait for the recorder's exit after SIGTERM. +// Everything after the signal is local (finish the in-flight write, sentinel, +// fsync), so this is loose; it stays a hard error because a log a writer still +// holds cannot be read. +const recorderStopTimeout = 30 * time.Second + +// recorderStopPoll is how often the wait re-checks the lock. With no wait(2) +// on a detached session leader, its exit is observable only by polling. +const recorderStopPoll = 100 * time.Millisecond + +// Config is check's whole input. It is the CLI's view of a run, and it is +// deliberately wider than Policy: Policy is the narrowed, pure-layer subset +// that reaches decide (classify.go), and the token is the field that must +// never cross that line. +type Config struct { + // URL and Token are the connection details, read from the environment by + // the CLI and never registered as flags. Token never enters the pure layer, + // an error string, or a Result. + URL, Token string + + // Alerts is REQUIRED in single-step mode and must be EMPTY in log mode: + // with a log, the header IS the alert set, and there is nothing to compare + // a second list against. + Alerts []string + Folder string + + States []State + Preexisting PreexistingPolicy + MinObserved int + AllowPaused bool + NodataIsUnobservable bool + + // From is the moment the deploy finished and To is the end of the work. + // They are different moments and both come from the work. In recorder mode + // an absent From is a hard error; in single-step mode it falls back to the + // start of this step, with a blind-interval warning. + From, To time.Time + + // Log is the path of a recording made by watch; "" selects single-step + // mode. PidFile defaults to .pid, the convention watch's parent + // writes and the only way check can reach the recorder it must stop before + // it may read the log. + Log string + PidFile string + + // There is deliberately NO PollEvery here, and `check` has no + // --poll-interval flag. In log mode the cadence comes from the header — + // the cadence the recording actually used — and a second authority would + // let an operator silently widen maxGap over evidence that was recorded at + // a different rate; in single-step mode the same process records and + // classifies, so the default cadence is the only cadence there is. + Concurrency int + Clock Clock + + // Notes is where the shell prints what an operator has to see while the + // run is in progress: the planned run time, the grace and its source, the + // countdown, the blind-interval warning. nil discards them. The library + // renders no table — the CLI owns presentation. + Notes io.Writer +} + +func (cfg Config) withDefaults() Config { + if cfg.Clock == nil { + cfg.Clock = SystemClock{} + } + if cfg.Notes == nil { + cfg.Notes = io.Discard + } + if cfg.Concurrency < 1 { + cfg.Concurrency = 1 + } + if cfg.PidFile == "" && cfg.Log != "" { + cfg.PidFile = cfg.Log + ".pid" + } + return cfg +} + +// namedAlerts returns the alert names that survive Resolve's trim-and-discard, +// so validation counts what Resolve will actually see rather than what the +// caller happened to pass (a file ending in a newline yields an empty line). +func (cfg Config) namedAlerts() []string { + out := make([]string, 0, len(cfg.Alerts)) + for _, a := range cfg.Alerts { + if strings.TrimSpace(a) != "" { + out = append(out, a) + } + } + return out +} + +// Check is the I/O shell: HTTP, signals, the pidfile, file reads, the +// countdown print. Every correctness question it touches is answered elsewhere +// (proveCoverage, decide — both pure), which is the most important seam in the +// project. A pass is exactly len(Violations) == 0 && err == nil; every error +// path leaves err non-nil. +func Check(ctx context.Context, cfg Config) (Result, error) { + cfg = cfg.withDefaults() + if err := cfg.validate(); err != nil { + return Result{}, err + } + // The Source is built here and injected into check() so every behaviour + // below is testable against a scripted fake — the same seam prepareWatch + // uses, and the reason this file needs no test-only setter. + return check(ctx, cfg, NewHTTPSource(cfg.URL, cfg.Token, cfg.Clock)) +} + +// validate runs before any network call, so a configuration mistake costs +// nothing and, more importantly, is never discovered after a ten-minute wait. +func (cfg Config) validate() error { + if cfg.URL == "" { + return errors.New("check: no grafana url") + } + if cfg.To.IsZero() { + return errors.New("check: no `to`: the end of the window is required") + } + + named := cfg.namedAlerts() + if cfg.Log == "" { + // An empty Alerts is an error — but only without a log. + if len(named) == 0 { + return errors.New("check: no alert names given and no recorded log to take them from") + } + } else if len(named) > 0 { + // The other direction: with a log, the alert set comes from the log. + // Accepting both would mean reconciling two sets, which the log being + // the one source removes entirely. + return fmt.Errorf("check: --alerts is refused with a recorded log: %s already names the alert set it recorded", cfg.Log) + } + + now := cfg.Clock.Now() + // from mirrors what check() will use, so the window checks below judge the + // window that will really be classified. + from := cfg.From + switch { + case from.IsZero() && cfg.Log != "": + // Never a warning-and-continue: falling back to the start of the check + // step reinstates exactly the blind interval the recorder exists to + // remove, which is the fail-open shape this design refuses. + return errors.New("check: no `from` in recorder mode: the deploy step must emit a completion timestamp") + case from.IsZero(): + // Single-step only. The caller sees the resulting blind interval named + // exactly, once the first observation has fixed its end. + from = now + } + + if cfg.To.Before(from) { + return fmt.Errorf("check: `to` %s is before `from` %s", cfg.To.Format(time.RFC3339), from.Format(time.RFC3339)) + } + if from.After(now.Add(fromFutureTolerance)) { + return fmt.Errorf("check: `from` %s is more than %s ahead of this runner's clock %s", + from.Format(time.RFC3339), fromFutureTolerance, now.Format(time.RFC3339)) + } + + // A `to` in the past is fine WITH a log (the collection loop is already + // done). Without one it is a request to prove a window nothing observed: + // every heartbeat gap would measure negative, and the run would report a + // proved window it never saw. + if cfg.Log == "" && !cfg.To.After(now) { + return fmt.Errorf("check: `to` %s has already passed and there is no recorded log: a window that ended before check started can only be classified from a recording", + cfg.To.Format(time.RFC3339)) + } + return nil +} + +// check is Check with the Source injected, and its body is one commented block +// per stage of a run, in the order a run performs them. +func check(ctx context.Context, cfg Config, src Source) (Result, error) { + // ---- Validate the configuration. -------------------------------------- + // Done by Check before this function is reached, except for the one part + // that needs a clock reading kept for later: the single-step fallback for + // an absent `from`. + from := cfg.From + if from.IsZero() { + from = cfg.Clock.Now() + fmt.Fprintf(cfg.Notes, "note: no `from` given; the window starts at the start of this step, %s\n", + from.Format(time.RFC3339)) + } + + // ---- Resolve the definitions from the ruler API. ---------------------- + // Unconditional, in BOTH modes. A log's header supplies the alert set as + // UIDs and the recording facts, never the rule facts: `for`, + // intervalSeconds and Kind always come from a fresh ruler read, which is + // why LoggedRule.ForSeconds is never converted back into a Definition. + version, err := src.Version(ctx) + if err != nil { + return Result{}, fmt.Errorf("read grafana version: %w", err) + } + if err := CheckGrafanaVersion(version); err != nil { + return Result{}, err + } + allDefs, err := src.Definitions(ctx) + if err != nil { + return Result{}, fmt.Errorf("read rule definitions: %w", err) + } + + // ---- With a log, validate its identity. ------------------------------- + // The header is read early — line 1 only, the one line a writer can never + // change — so a wrong URL or an unresolvable rule fails closed NOW. It is + // advisory: the authoritative header is re-read once collection ends and + // the writer has exited. + var ( + resolved []Definition + notes []string + earlyHdr Header + logHasHdr bool + rt map[string]ruleTimings + gt globalTimings + timingNote []string + ) + if cfg.Log != "" { + earlyHdr, err = ReadLogHeader(cfg.Log) + if err != nil { + return Result{}, fmt.Errorf("log identity: %w", err) + } + logHasHdr = true + resolved, notes, err = resolveFromLog(allDefs, earlyHdr, cfg) + if err == nil { + // Fail fast on a bound violation that can't change: StartedAt is + // immutable (line 1), so check 2's backstop still catches any bad + // advisory read — fail closed, never false-pass. Recorder mode only; + // single-step warns-and-passes (see below). + if from.Before(earlyHdr.StartedAt) { + return Result{}, fmt.Errorf("check: `from` %s is before recording started at %s", + from.Format(time.RFC3339), earlyHdr.StartedAt.Format(time.RFC3339)) + } + } + } else { + resolved, notes, err = Resolve(allDefs, cfg.namedAlerts(), cfg.Folder) + } + if err != nil { + return Result{}, err + } + if len(resolved) == 0 { + // Reachable only from a header with an empty rule list. Left to run, + // MinObserved would default to zero, no rule would be judged, and the + // gate would return a pass over nothing at all. + return Result{}, fmt.Errorf("check: no rules to classify") + } + for _, n := range notes { + fmt.Fprintf(cfg.Notes, "note: %s\n", n) + } + + // ---- Derive the timings, print them, fit the request budget. ---------- + if logHasHdr { + // The header is the authority for the cadence actually recorded at; + // re-deriving it from defs would compare gaps recorded at an override + // cadence against thresholds computed from the default — fail-open in + // the faster-override direction. + rt, gt, err = DeriveTimingsFromLog(earlyHdr, resolved) + if err != nil { + return Result{}, fmt.Errorf("log identity: %w", err) + } + } else { + rt, gt, timingNote = DeriveTimings(resolved, 0) + for _, n := range timingNote { + fmt.Fprintf(cfg.Notes, "note: %s\n", n) + } + } + summary, warning := StartupSummary(from, cfg.To, gt) + fmt.Fprintln(cfg.Notes, summary) + // MinObserved is printed with the plan, beside "planned run time", rather + // than after it: it is a fact about the run, not a diagnostic. Its default + // is the resolved rule count AFTER duplicate names collapse, which is + // len(resolved) by construction; decide defaults it identically, and it is + // resolved here rather than inferred from the verdict afterwards. + minObserved := cfg.MinObserved + if minObserved == 0 { + minObserved = len(resolved) + } + fmt.Fprintf(cfg.Notes, "min-observed: %d of %d resolved rule(s)\n", minObserved, len(resolved)) + if warning != "" { + fmt.Fprintf(cfg.Notes, "warning: %s\n", warning) + } + + // The measurement pass and budget check are single-step only: in recorder + // mode watch already measured and checked the budget before detaching. + var ( + header Header + initial []Poll + reducer = NewReducer() + ) + if !logHasHdr { + // StartedAt is fixed before the pass rather than after it, so the + // interval it claims to have observed can only be wider than the one + // it really saw — and the first heartbeat's own boundary gap is what + // proves that interval, not this timestamp. + startedAt := cfg.Clock.Now() + active := activeRules(resolved) + var measured map[string]time.Duration + initial, measured, err = firstObservations(ctx, src, active, reducer, cfg.Concurrency, cfg.Notes) + if err != nil { + return Result{}, err + } + if err := CheckBudget(activeTimingsOf(active, rt), measured, cfg.Concurrency); err != nil { + return Result{}, err + } + + // Single-step synthesis — how the pure layer stays unconditional. The + // shell builds the Header and later stamps the sentinel itself, so the + // sentinel and from-bounds coverage checks run exactly as they do over + // a recording and no mode flag ever reaches proveCoverage or decide. + header = Header{ + SchemaVersion: LogSchemaVersion, + URL: cfg.URL, + GrafanaVersion: version, + StartedAt: startedAt, + Rules: loggedRules(resolved, rt), + } + if from.Before(startedAt) { + // The declared blind interval: in single-step mode this is a + // warning and a pass, and ONLY here. Recorder mode keeps the + // from-bounds coverage check strict, because there the recorder + // was supposed to be watching and the gap means it was not. + fmt.Fprintf(cfg.Notes, "warning: cannot see [%s, %s) — %s before the first observation; the window is classified from %s\n", + from.Format(time.RFC3339), startedAt.Format(time.RFC3339), + startedAt.Sub(from).Round(time.Second), startedAt.Format(time.RFC3339)) + from = startedAt + } + } + + // ---- Collect the evidence. -------------------------------------------- + // Collect ONLY. No classification happens here and there is no early exit, + // even once a violation is certain: the loop always runs to + // to + transitionGrace, which is what makes "did the early exit lose the + // coverage proof?" a question that cannot be asked. + windowEnd := cfg.To.Add(gt.transitionGrace) + + var poller *livePoller + if !logHasHdr { + poller = newLivePoller(src, reducer, activeRules(resolved), rt, cfg.Concurrency, cfg.Clock.Now()) + } + collected, err := collectUntil(ctx, cfg, windowEnd, poller) + if err != nil { + // Nothing collected is classified; the count lets an operator tell a + // run that failed at once from one that failed at minute nine. + return Result{}, fmt.Errorf("collect evidence after %d poll(s): %w", len(collected), err) + } + + var ( + polls []Poll + sentinel *time.Time + ) + if logHasHdr { + // In this order and no other: signal the writer, wait for its exit, + // and only THEN read the log once. A log read while a writer can still + // append can only yield a shorter window than the one that was + // actually recorded. + heldLog, err := stopRecorder(ctx, cfg) + if err != nil { + return Result{}, err + } + header, polls, sentinel, err = ReadLog(cfg.Log) + // Held across the read so no writer can appear mid-read, then released + // (everything past here works from memory, and the drain wait is minutes). + _ = heldLog.Close() + if err != nil { + return Result{}, err + } + // The authoritative header wins: the advisory read was only a fail-fast. + resolved, _, err = resolveFromLog(allDefs, header, cfg) + if err != nil { + return Result{}, err + } + // rt is re-derived from the authoritative header. windowEnd is NOT + // recomputed: the loop already stopped at the earlier value, and moving + // it afterwards would prove a window this run did not collect. + if rt, gt, err = DeriveTimingsFromLog(header, resolved); err != nil { + return Result{}, fmt.Errorf("log identity: %w", err) + } + } else { + polls = make([]Poll, 0, len(initial)+len(collected)) + polls = append(polls, initial...) + polls = append(polls, collected...) + // The shell stamps the sentinel itself, when the collection loop + // exits: by construction that is at or after to + transitionGrace, so + // the sentinel check passes for the same reason a clean recorder stop + // does, and for no other. + stoppedAt := cfg.Clock.Now() + sentinel = &stoppedAt + } + + // ---- The drain wait. -------------------------------------------------- + // The last instance of the liveness check: did this rule evaluate through + // the end of the window? It is I/O and it is deliberately NOT part of + // proveCoverage — adding it there would put HTTP inside the pure layer and + // destroy the seam this design depends on. + drained, err := drainWait(ctx, cfg, src, resolved, header.pausedAtStart(), rt, polls, windowEnd, gt.drainTimeout) + if err != nil { + return Result{}, err + } + + // ---- Classify. -------------------------------------------------------- + pol := Policy{ + States: cfg.States, + Preexisting: cfg.Preexisting, + MinObserved: minObserved, + AllowPaused: cfg.AllowPaused, + NodataIsUnobservable: cfg.NodataIsUnobservable, + From: from, + To: cfg.To, + } + result, decideErr := decide(header, polls, sentinel, resolved, rt, gt, pol) + result, drainErr := mergeDrainTimeouts(result, drained) + + // ---- Return the Result. ----------------------------------------------- + // Both errors are joined rather than one shadowing the other: each names + // rules the other does not, and on exit 2 that list IS the answer to + // "why". + return result, errors.Join(decideErr, drainErr) +} + +// resolveFromLog is the log's identity check in practice: the URL must match +// and every header UID must still resolve against a fresh ruler read. The +// alert set is TAKEN from the log, never compared against --alerts (which the +// validator requires empty in log mode). Resolving through Resolve by uid: +// keeps one implementation of the resolution rules. +// +// Only the header-to-defs direction can fail: resolved is BUILT from the +// header, so no resolved definition can be absent from it. +func resolveFromLog(allDefs []Definition, h Header, cfg Config) ([]Definition, []string, error) { + if h.URL != cfg.URL { + return nil, nil, fmt.Errorf("log identity: %s recorded url %q but this run is configured for %q", + cfg.Log, h.URL, cfg.URL) + } + names := make([]string, 0, len(h.Rules)) + for _, lr := range h.Rules { + names = append(names, "uid:"+lr.UID) + } + resolved, notes, err := Resolve(allDefs, names, "") + if err != nil { + return nil, nil, fmt.Errorf("log identity: %s names a rule that no longer resolves: %w", cfg.Log, err) + } + return resolved, notes, nil +} + +// activeRules drops the rules whose DEFINITION says paused. They are skipped: +// never polled, never waited for, and reported from the definitions alone — a +// skipped rule has no poll records at all, so it has no heartbeats to prove +// and no IsPaused poll to detect. +func activeRules(defs []Definition) []Definition { + out := make([]Definition, 0, len(defs)) + for _, d := range defs { + if !d.IsPaused { + out = append(out, d) + } + } + return out +} + +// activeTimingsOf narrows the timings map to the rules that will actually be +// polled, which is what the request budget is spent on: a skipped rule +// consumes none of the capacity, so counting it would refuse schedules that +// fit. +func activeTimingsOf(active []Definition, rt map[string]ruleTimings) map[string]ruleTimings { + out := make(map[string]ruleTimings, len(active)) + for _, d := range active { + out[d.UID] = rt[d.UID] + } + return out +} + +// livePoller is single-step mode's collection engine: the same per-rule +// scheduler and the same Reducer the recorder uses, writing into memory +// instead of a log. Log mode has none — the recorder is doing this work in +// another process — and collectUntil takes a nil poller for it. +type livePoller struct { + src Source + reducer *Reducer + sched *Scheduler + titles map[string]string // uid -> title: poll by title, select by UID + concurrency int +} + +func newLivePoller(src Source, reducer *Reducer, active []Definition, rt map[string]ruleTimings, + concurrency int, now time.Time) *livePoller { + + titles := make(map[string]string, len(active)) + cadence := make(map[string]time.Duration, len(active)) + for _, d := range active { + titles[d.UID] = d.Title + cadence[d.UID] = rt[d.UID].pollEvery + } + return &livePoller{ + src: src, + reducer: reducer, + sched: NewScheduler(cadence, now), + titles: titles, + concurrency: concurrency, + } +} + +// poll runs one round of due rules and returns every poll that succeeded, +// alongside the first failure. +// +// The successes are NOT kept for the reason watchLoopConfig.pollBatch keeps +// its own: those go into a durable log that a later check will read, so +// dropping one would turn a single rule's transport failure into a coverage +// gap for the others. Here there is no later reader. A terminal failure during +// collection is exit 2 and check discards the whole collection, so these come +// back only to let the error say how far the run got before it stopped — +// which is the one part of it an operator can act on. +func (p *livePoller) poll(ctx context.Context, uids []string) ([]Poll, error) { + observed, obsErr := observeAll(ctx, p.src, p.titles, uids, p.concurrency) + out := make([]Poll, 0, len(uids)) + for _, uid := range uids { + obs, ok := observed[uid] + if !ok { + continue + } + out = append(out, p.reducer.Reduce(uid, obs)) + } + return out, obsErr +} + +// collectUntil is the collection loop, shared by both modes. With a poller it +// polls each rule on its own cadence; with nil it only waits, because in +// recorder mode the evidence is being written by another process. Both print +// the same countdown, because both are the same silence to an operator +// watching a job. +// +// It never classifies and never exits early. +func collectUntil(ctx context.Context, cfg Config, deadline time.Time, p *livePoller) ([]Poll, error) { + var ( + polls []Poll + lastPrint time.Time + ) + for { + now := cfg.Clock.Now() + if !now.Before(deadline) { + return polls, nil + } + if lastPrint.IsZero() || now.Sub(lastPrint) >= countdownEvery { + fmt.Fprintf(cfg.Notes, "collecting: %s until the window closes at %s\n", + deadline.Sub(now).Round(time.Second), deadline.Format(time.RFC3339)) + lastPrint = now + } + + wait := min(deadline.Sub(now), countdownEvery) + if p != nil { + // Mark before polling, against the batch's own `now`: the next poll + // is one cadence after this one was DUE, not after it returned, so + // request latency cannot make the heartbeat spacing drift towards + // maxGap (the same rule watchLoop follows). + due := p.sched.Due(now) + for _, uid := range due { + p.sched.Mark(uid, now) + } + if len(due) > 0 { + batch, err := p.poll(ctx, due) + polls = append(polls, batch...) + if err != nil { + return polls, err + } + } + if next, ok := p.sched.earliestDue(); ok { + wait = min(wait, next.Sub(cfg.Clock.Now())) + } + } + + select { + case <-ctx.Done(): + return polls, ctx.Err() + case <-cfg.Clock.After(max(wait, 0)): + } + } +} + +// stopRecorder signals the recorder and waits for it to go; the log may not be +// read until the writer has provably gone, so every failure is a hard error. +// It returns the log held under an exclusive flock, which the caller must keep +// open across ReadLog — the lock is the proof that no writer exists. +// +// Two authorities, only one of which is evidence: +// +// - the PIDFILE says whether a recording ever started (it is written only +// after the child reports ready, and removed on failure). +// - the FLOCK says whether a writer exists right now. A pidfile can go stale +// — nothing removes it on a clean --until stop, so it may name a pid +// somebody else now owns — but the kernel drops a flock when the holder +// exits, so the lock is always authoritative. +// +// So: read the pidfile to learn a recording happened, ask the lock whether it +// is still running, and signal only if it is. +func stopRecorder(ctx context.Context, cfg Config) (*os.File, error) { + pid, err := ReadPidFile(cfg.PidFile) + if err != nil { + return nil, fmt.Errorf("cannot stop the recorder: %w; a pidfile is written only once a recorder reports that it is running, so an unreadable one means the recording never started", err) + } + + log, err := os.Open(cfg.Log) + if err != nil { + return nil, fmt.Errorf("open %s to check for a writer: %w", cfg.Log, err) + } + + held, err := tryLockExclusive(log) + if err != nil { + log.Close() + return nil, err + } + if held { + // No writer. Send no signal, whatever the pidfile says — the pid may + // belong to somebody else entirely by now. Which of --until, a clean + // stop and a death ended the recording is the sentinel's question, + // answered by the coverage proof over the log this unblocks. + fmt.Fprintf(cfg.Notes, "note: no writer holds %s; the recorder has already finished\n", cfg.Log) + return log, nil + } + + // The lock is held, so a writer is alive and the pidfile's pid cannot be + // stale — the recorder that took the lock is the one the parent recorded. + gone, err := signalRecorder(pid) + if err != nil { + log.Close() + return nil, err + } + if gone { + // A live writer holds the log and the pidfile names a process that + // does not exist. That is a broken contract, not a case to reason + // around: signalling the real holder would mean guessing who it is. + log.Close() + return nil, fmt.Errorf("a writer holds %s but pidfile %s names pid %d, which does not exist: the pidfile does not name the process that holds the log", + cfg.Log, cfg.PidFile, pid) + } + + // Wait on the LOCK, not on the pid: its release is the kernel-guaranteed + // writer-is-gone event, and it carries no pid-reuse hazard. + deadline := cfg.Clock.Now().Add(recorderStopTimeout) + for { + select { + case <-ctx.Done(): + log.Close() + return nil, ctx.Err() + case <-cfg.Clock.After(recorderStopPoll): + } + + held, err := tryLockExclusive(log) + if err != nil { + log.Close() + return nil, err + } + if held { + return log, nil + } + if !cfg.Clock.Now().Before(deadline) { + log.Close() + return nil, fmt.Errorf("recorder pid %d still holds %s %s after SIGTERM; refusing to read a log a writer can still append to", + pid, cfg.Log, recorderStopTimeout) + } + } +} + +// drainVerdict is what the drain wait concluded about one rule it could not +// clear. It carries the reason as well as the prose because the two outcomes +// are genuinely different faults: drain_timeout means the rule is still there +// and still behind, rule_absent means it is gone. Collapsing both into +// drain_timeout would name the wait instead of the fault, and Reason is a +// published vocabulary that reaches the JSON output. +type drainVerdict struct { + reason UnobservableReason + note string +} + +// drainWait is the final liveness check: did each rule evaluate through the +// end of the window? A rule that cannot answer within drainTimeout is +// unobservable, never a pass. It returns one verdict per rule it could not +// clear (keyed by UID); an error only for a hard failure of the wait itself. +// +// Two kinds of rule are excluded up front because draining them could not +// change a verdict: a rule the HEADER says was paused at the window open (the +// header, not the late-resolved definitions — see Header.pausedAtStart), and a +// rule whose last poll says Found == false (already unobservable via rule_absent). +func drainWait(ctx context.Context, cfg Config, src Source, defs []Definition, pausedAtStart map[string]bool, + rt map[string]ruleTimings, polls []Poll, windowEnd time.Time, timeout time.Duration) (map[string]drainVerdict, error) { + + pending := make(map[string]string) // uid -> title, the shape observeAll wants + for _, d := range defs { + if pausedAtStart[d.UID] { + continue + } + rulePolls := pollsForRule(polls, d.UID) + if n := len(rulePolls); n > 0 && !rulePolls[n-1].Found { + continue + } + // Evidence already in hand can satisfy the wait outright: a rule whose + // recorded evaluations already reach past the end of the window has + // answered the question, and polling it again asks nothing new. + if !anyPollEvaluatedThrough(rulePolls, windowEnd) { + pending[d.UID] = d.Title + } + } + if len(pending) == 0 { + return nil, nil + } + + fmt.Fprintf(cfg.Notes, "drain wait: %d rule(s) have not yet evaluated through %s (limit %s)\n", + len(pending), windowEnd.Format(time.RFC3339), timeout) + + deadline := cfg.Clock.Now().Add(timeout) + verdicts := make(map[string]drainVerdict) + for { + uids := make([]string, 0, len(pending)) + for uid := range pending { + uids = append(uids, uid) + } + sort.Strings(uids) // deterministic request order and message order + + observed, err := observeAll(ctx, src, pending, uids, cfg.Concurrency) + if err != nil { + return nil, fmt.Errorf("drain wait: %w", err) + } + for _, uid := range uids { + obs, ok := observed[uid] + if !ok { + continue + } + rule := stateRuleByUID(obs.Rules, uid) + if rule == nil { + // A 2xx that parsed and carries no matching rule is an + // authoritative "the rule is gone" — the transport retried + // every transient failure long before this Observation + // existed. It is knowable on the FIRST poll, so waiting the + // rest of drainTimeout would spend two minutes to reach the + // same verdict under a name that describes the wait rather + // than the fault. + verdicts[uid] = drainVerdict{ + reason: ReasonRuleAbsent, + note: fmt.Sprintf("rule %q: absent from the state endpoint during the drain wait; there is no evaluation to wait for", + pending[uid]), + } + delete(pending, uid) + continue + } + if rule.IsPaused { + // A paused rule does not evaluate, so this one can never catch + // up and the rest of drainTimeout would buy nothing. The reason + // stays drain_timeout: UnobservableReason is a published + // vocabulary that reaches the JSON output, and the prose below + // is where the detail belongs. + verdicts[uid] = drainVerdict{ + reason: ReasonDrainTimeout, + note: fmt.Sprintf("rule %q: paused before it evaluated through %s, so it never will", + pending[uid], windowEnd.Format(time.RFC3339)), + } + delete(pending, uid) + continue + } + if evaluatedThrough(rule.LastEvaluation, obs.Skew, obs.SkewBound, windowEnd) { + delete(pending, uid) + } + } + if len(pending) == 0 { + return verdicts, nil + } + + now := cfg.Clock.Now() + if !now.Before(deadline) { + for uid, title := range pending { + verdicts[uid] = drainVerdict{ + reason: ReasonDrainTimeout, + note: fmt.Sprintf("rule %q: did not evaluate through %s within the %s drain limit", + title, windowEnd.Format(time.RFC3339), timeout), + } + } + return verdicts, nil + } + + // Re-ask no faster than the tightest cadence among the rules still + // pending: a rule evaluating every 60s cannot answer differently 200ms + // later, and hammering it would spend the run's request budget on + // nothing. + wait := deadline.Sub(now) + for uid := range pending { + if every := rt[uid].pollEvery; every > 0 { + wait = min(wait, every) + } + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-cfg.Clock.After(max(wait, 0)): + } + } +} + +// anyPollEvaluatedThrough reports whether any recorded poll of a rule already +// proves it evaluated through windowEnd. +func anyPollEvaluatedThrough(polls []Poll, windowEnd time.Time) bool { + for _, p := range polls { + if !p.Found { + continue + } + if evaluatedThrough(p.LastEvaluation, p.Skew(), p.SkewBound(), windowEnd) { + return true + } + } + return false +} + +// evaluatedThrough is the drain wait's cross-domain comparison: a Grafana +// lastEvaluation is translated by its poll's skew, and the bound is SUBTRACTED +// (the pessimistic end) so an evaluation that only *might* have reached the +// window end is not counted as having reached it. A zero lastEvaluation never +// satisfies the wait. +func evaluatedThrough(lastEval time.Time, skew, bound time.Duration, windowEnd time.Time) bool { + if lastEval.IsZero() { + return false + } + return !lastEval.Add(-skew).Add(-bound).Before(windowEnd) +} + +// mergeDrainTimeouts folds the drain wait's I/O verdicts into the pure Result, +// running after decide so that function stays pure of its arguments. It returns +// its own error rather than mutating decide's so neither hides the other: a run +// faulted by both the coverage proof and the drain wait must name both. +func mergeDrainTimeouts(res Result, drained map[string]drainVerdict) (Result, error) { + if len(drained) == 0 { + return res, nil + } + if res.Coverage == nil { + res.Coverage = make(map[string]CoverageResult, len(drained)) + } + + var names []string + for i := range res.Verdicts { + uid := res.Verdicts[i].RuleUID + verdict, ok := drained[uid] + if !ok { + continue + } + cov := res.Coverage[uid] + cov.Unobservable = true + cov.Proved = false + if cov.Reason == "" { + // The FIRST reason wins, as it does inside proveCoverage: a rule + // the coverage proof already faulted keeps the fault it was + // actually caught by. + cov.Reason = verdict.reason + } + cov.Notes = append(cov.Notes, verdict.note) + res.Coverage[uid] = cov + + if res.Verdicts[i].Outcome != OutcomeUnobservable { + names = append(names, fmt.Sprintf("%s (%s)", res.Verdicts[i].Alert, verdict.reason)) + } + res.Verdicts[i].Outcome = OutcomeUnobservable + res.Verdicts[i].Note = strings.Join(cov.Notes, "; ") + } + if len(names) == 0 { + // Every drained rule was already unobservable for an earlier reason, + // so decide's own error already stops the run. Adding a second error + // saying the same thing would only make the message longer. + return res, nil + } + return res, fmt.Errorf("gate: %d rule(s) unobservable at the drain wait: %s", len(names), strings.Join(names, "; ")) +} diff --git a/grafana-alertcheck/internal/gate/check_process.go b/grafana-alertcheck/internal/gate/check_process.go new file mode 100644 index 000000000..86dd73148 --- /dev/null +++ b/grafana-alertcheck/internal/gate/check_process.go @@ -0,0 +1,29 @@ +package gate + +import ( + "errors" + "fmt" + "syscall" +) + +// signalRecorder asks the recorder to stop. +// +// The caller must have established that a writer is alive — by taking the +// log's flock and being refused — before it calls this. Nothing removes the +// pidfile when a recorder exits cleanly, so a pid read without that proof can +// name any same-user process that has since inherited it. +// +// gone reports ESRCH. Given the lock proof, that is a broken contract rather +// than a clean stop, and stopRecorder treats it as one; the value is reported +// instead of raised here because this function knows the errno and not what +// it means. +func signalRecorder(pid int) (gone bool, err error) { + switch err := syscall.Kill(pid, syscall.SIGTERM); { + case err == nil: + return false, nil + case errors.Is(err, syscall.ESRCH): + return true, nil + default: + return false, fmt.Errorf("signal recorder pid %d: %w", pid, err) + } +} diff --git a/grafana-alertcheck/internal/gate/check_test.go b/grafana-alertcheck/internal/gate/check_test.go new file mode 100644 index 000000000..780184bd3 --- /dev/null +++ b/grafana-alertcheck/internal/gate/check_test.go @@ -0,0 +1,1268 @@ +package gate + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// The one rule every test in this file watches, unless it says otherwise: a +// 60s evaluation interval, no `for`, not paused. Every derived value follows +// from those three numbers, and the tests assert against them by name rather +// than by magic constant: +// +// pollEvery 30s (intervalSeconds/2) +// maxGap 60s (2 x pollEvery) +// healthGrace 60s (max(maxGap, interval)) +// evalStaleAfter 120s (2 x interval) +// transitionGrace 60s (for + interval) +// drainTimeout 2m (max(2 x interval, 2m)) +const ( + checkUID = "rule-one" + checkTitle = "Rule One" + + checkPollEvery = 30 * time.Second + checkGrace = 60 * time.Second + checkDrainLimit = 2 * time.Minute +) + +func checkDef() Definition { + return Definition{ + UID: checkUID, Title: checkTitle, Folder: "F", Group: "G", + IntervalSeconds: 60, NoDataState: "OK", ExecErrState: "OK", + Kind: KindGrafanaManaged, + } +} + +// checkSource is a Source whose state answers depend on virtual time and on +// the call count, which is what a collection-loop test needs: fakeSource's +// static script cannot express "healthy for the whole window" without +// scripting every poll, and loopSource (watch_test.go) deliberately refuses +// Version and Definitions because the recorder's child never reads them. +type checkSource struct { + mu sync.Mutex + + version string + versionErr error + defs []Definition + defsErr error + + calls map[string]int + respond func(title string, call int) (Observation, error) +} + +func newCheckSource(respond func(title string, call int) (Observation, error)) *checkSource { + return &checkSource{ + version: "13.1.0", + defs: []Definition{checkDef()}, + calls: map[string]int{}, + respond: respond, + } +} + +func (s *checkSource) Version(context.Context) (string, error) { return s.version, s.versionErr } + +func (s *checkSource) Definitions(context.Context) ([]Definition, error) { return s.defs, s.defsErr } + +// RuleState answers from the responder. A nil responder means the test +// expects no state read at all — it fails with a message rather than a nil +// dereference, because "this path must not poll" is an assertion several tests +// here make on purpose. +func (s *checkSource) RuleState(_ context.Context, title string) (Observation, error) { + s.mu.Lock() + s.calls[title]++ + call := s.calls[title] + s.mu.Unlock() + if s.respond == nil { + return Observation{}, fmt.Errorf("checkSource: this test expects no state read, but %q was polled", title) + } + return s.respond(title, call) +} + +func (s *checkSource) callCount(title string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.calls[title] +} + +var _ Source = (*checkSource)(nil) + +// checkStateRule builds one state-endpoint rule whose totals agree with the +// instances it carries. That agreement is load-bearing: a totals map claiming +// normal instances that the instance list does not contain fails +// VerifyNormalInstancesVisible, which is a different failure from the one most +// of these tests are about. +func checkStateRule(lastEval time.Time, insts ...Instance) StateRule { + totals := map[string]int{} + for _, i := range insts { + totals[string(i.State)]++ + } + return StateRule{ + UID: checkUID, Title: checkTitle, Folder: "F", Group: "G", + Interval: time.Minute, State: "inactive", Health: "ok", + LastEvaluation: lastEval, Totals: totals, Instances: insts, + } +} + +// healthyObservation is a poll of a rule that evaluated at this instant, with +// no skew at all — every test that is not ABOUT skew uses zero so its +// arithmetic reads directly off the timestamps. +func healthyObservation(now time.Time, insts ...Instance) Observation { + return Observation{ + Rules: []StateRule{checkStateRule(now, insts...)}, + GrafanaNow: now, + Latency: 200 * time.Millisecond, + } +} + +// baseConfig is a single-step run over [now, now+5m]: window 5m, grace 60s, so +// the collection loop ends at now+6m. +func baseConfig(t *testing.T, clock Clock) Config { + t.Helper() + now := clock.Now() + return Config{ + URL: "https://grafana.example.com", + Alerts: []string{"uid:" + checkUID}, + From: now, + To: now.Add(5 * time.Minute), + Clock: clock, + Notes: &strings.Builder{}, + }.withDefaults() +} + +func notesOf(cfg Config) string { return cfg.Notes.(*strings.Builder).String() } + +// --------------------------------------------------------------------------- +// Configuration validation +// --------------------------------------------------------------------------- + +func TestCheckValidateRejectsBadConfigurations(t *testing.T) { + clock := newFakeClock(testNow) + base := func() Config { + return Config{ + URL: "https://grafana.example.com", + From: testNow, + To: testNow.Add(5 * time.Minute), + Clock: clock, + } + } + + tests := []struct { + name string + mutate func(*Config) + wantErr string + }{ + { + name: "no url", + mutate: func(c *Config) { c.URL = ""; c.Alerts = []string{"A"} }, + wantErr: "no grafana url", + }, + { + name: "no to", + mutate: func(c *Config) { c.To = time.Time{}; c.Alerts = []string{"A"} }, + wantErr: "no `to`", + }, + { + // An empty Alerts is an error — but only without a log. + name: "single-step without alerts", + mutate: func(c *Config) {}, + wantErr: "no alert names given", + }, + { + // An alerts file ending in a newline must not read as a named alert. + name: "single-step with only blank alert lines", + mutate: func(c *Config) { c.Alerts = []string{"", " "} }, + wantErr: "no alert names given", + }, + { + // The other direction: with a log, the log names the alert set. + name: "log mode with alerts", + mutate: func(c *Config) { c.Log = "log.jsonl"; c.Alerts = []string{"A"} }, + wantErr: "--alerts is refused with a recorded log", + }, + { + // Never a warning-and-continue. + name: "log mode without from", + mutate: func(c *Config) { c.Log = "log.jsonl"; c.From = time.Time{} }, + wantErr: "the deploy step must emit a completion timestamp", + }, + { + name: "from beyond the future tolerance", + mutate: func(c *Config) { + c.Alerts = []string{"A"} + c.From = testNow.Add(2 * time.Minute) + c.To = testNow.Add(10 * time.Minute) + }, + wantErr: "ahead of this runner's clock", + }, + { + name: "to before from", + mutate: func(c *Config) { c.Alerts = []string{"A"}; c.To = testNow.Add(-time.Minute) }, + wantErr: "is before `from`", + }, + { + // A past `to` is only "not a special mode" WITH a log: without one + // the coverage window ends before the first observation exists. + name: "single-step with a to already past", + mutate: func(c *Config) { + c.Alerts = []string{"A"} + c.From = testNow.Add(-10 * time.Minute) + c.To = testNow.Add(-time.Minute) + }, + wantErr: "can only be classified from a recording", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := base() + tc.mutate(&cfg) + err := cfg.withDefaults().validate() + require.Errorf(t, err, "validate()") + require.Contains(t, err.Error(), tc.wantErr) + }) + } +} + +// A past `to` WITH a log is not a special mode: the collection loop's condition +// is already true and the evidence classifies immediately. No branch, and no +// refusal. +func TestCheckValidateAcceptsAPastToWithALog(t *testing.T) { + cfg := Config{ + URL: "https://grafana.example.com", + Log: "log.jsonl", + From: testNow.Add(-10 * time.Minute), + To: testNow.Add(-time.Minute), + Clock: newFakeClock(testNow), + }.withDefaults() + + require.NoError(t, cfg.validate()) + require.Equal(t, "log.jsonl.pid", cfg.PidFile) +} + +// --------------------------------------------------------------------------- +// Single-step mode +// --------------------------------------------------------------------------- + +func TestCheckSingleStepCleanWindowPasses(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + src := newCheckSource(func(_ string, _ int) (Observation, error) { + return healthyObservation(clock.Now()), nil + }) + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err) + // A pass is exactly this shape. + require.Empty(t, res.Violations) + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeClean, res.Verdicts[0].Outcome) + cov := res.Coverage[checkUID] + require.True(t, cov.Proved) + require.False(t, cov.Unobservable) + + // The collection loop ran to to+transitionGrace and no further. + windowEnd := cfg.To.Add(checkGrace) + require.False(t, clock.Now().Before(windowEnd)) + // One measurement-pass poll plus one every 30s across the 6-minute + // collection, plus the drain wait's own polls. The exact count depends on + // the scheduler's random stagger, so assert the order of magnitude a full + // window implies rather than an exact number. + require.GreaterOrEqual(t, src.callCount(checkTitle), 12) + require.Contains(t, notesOf(cfg), "planned run time") +} + +// resolve_test.go proves the collapse-note-plus-satisfied-MinObserved path at +// Resolve() directly; this drives the same shape through check() end to end — +// the two input names must collapse to one verdict, the run must pass, and the +// collapse note must reach the run's own notes, not just Resolve()'s return +// value. +func TestCheckSingleStepDuplicateAlertNamesCollapseWithNote(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + cfg.Alerts = []string{"uid:" + checkUID, checkTitle} // the same rule, named two different ways + src := newCheckSource(func(_ string, _ int) (Observation, error) { + return healthyObservation(clock.Now()), nil + }) + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err) + require.Len(t, res.Verdicts, 1, "the duplicate must collapse to a single rule") + require.Empty(t, res.Violations, "MinObserved must be satisfied by the post-collapse count of 1") + require.Contains(t, notesOf(cfg), "counted once") +} + +// A rule with health=error for the whole window is unobservable, exit 2 — +// driven from the real "[JD] No Job Proposals" capture (testdata/README.md), +// not a synthetic Poll table, so a change in how the real payload shapes +// health/lastError cannot slip past a hand-built fixture that happens to still +// look right. +func TestCheckSingleStepContinuousHealthErrorIsUnobservable(t *testing.T) { + body := readFixture(t, "state_health_error.json") + rules, err := ParseState(body) + require.NoError(t, err) + base := rules[0] + def := Definition{ + UID: base.UID, Title: base.Title, Folder: base.Folder, Group: base.Group, + IntervalSeconds: int(base.Interval / time.Second), NoDataState: "OK", ExecErrState: "OK", + Kind: KindGrafanaManaged, + } + + clock := newVirtualClock(testNow) + cfg := Config{ + URL: "https://grafana.example.com", Alerts: []string{"uid:" + def.UID}, + From: testNow, To: testNow.Add(5 * time.Minute), Clock: clock, Notes: &strings.Builder{}, + }.withDefaults() + + src := newCheckSource(func(_ string, _ int) (Observation, error) { + // Every field but LastEvaluation stays exactly as the real capture + // shaped it (health=error, the real lastError text, the real Error + // instance); LastEvaluation tracks the poll so staleness — a + // different coverage check — never becomes the actual cause. + r := base + r.LastEvaluation = clock.Now() + return Observation{Rules: []StateRule{r}, GrafanaNow: clock.Now(), Latency: 200 * time.Millisecond}, nil + }) + src.defs = []Definition{def} + + res, err := check(context.Background(), cfg, src) + require.Error(t, err, "continuous health=error must be unobservable") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome) + require.Equal(t, ReasonHealthError, res.Coverage[def.UID].Reason) +} + +// A certain violation does not release the runner early, and it does not stop +// the gate reporting exit-1 shape — violations with a nil error. +func TestCheckSingleStepFiringInstanceReportsWithoutExitingEarly(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + firing := Instance{ + Labels: map[string]string{"alertname": "Rule One", "instance": "a"}, + State: StateFiring, + ActiveAt: testNow.Add(-10 * time.Minute), // bad before the window opened + } + src := newCheckSource(func(_ string, _ int) (Observation, error) { + return healthyObservation(clock.Now(), firing), nil + }) + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err, "a violation is exit 1, not an error") + require.Len(t, res.Violations, 1) + require.Equal(t, OutcomePersistentlyBad, res.Violations[0].Outcome) + require.False(t, clock.Now().Before(cfg.To.Add(checkGrace)), "exited early; collection must run to to+grace") +} + +// A newly_bad instance at from+30s gives exit 1, but ONLY after +// to+transitionGrace. The test above covers a rule already bad before the +// window opened (persistently_bad); this covers a fresh onset just inside the +// window, which must not release the runner the instant it is first observed. +func TestCheckSingleStepNewOnsetDoesNotExitEarly(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + onset := testNow.Add(30 * time.Second) + src := newCheckSource(func(_ string, _ int) (Observation, error) { + now := clock.Now() + if now.Before(onset) { + return healthyObservation(now), nil + } + firing := Instance{ + Labels: map[string]string{"alertname": checkTitle, "instance": "a"}, + State: StateFiring, + ActiveAt: onset, + } + return healthyObservation(now, firing), nil + }) + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err) + require.Len(t, res.Violations, 1) + require.Equal(t, OutcomeNewlyBad, res.Violations[0].Outcome) + require.False(t, clock.Now().Before(cfg.To.Add(checkGrace)), + "exited early; collection must run to to+grace even for a fresh onset at from+30s") +} + +// An ABSENT `from` in single-step mode (as opposed to recorder mode, which +// hard-errors — TestCheckValidateRejectsBadConfigurations's "log mode without +// from") falls back to the start of this check step, with the same +// declared-blind-interval warning as an explicit early `from`. +func TestCheckSingleStepAbsentFromFallsBackToStepStart(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + cfg.From = time.Time{} + src := newCheckSource(func(_ string, _ int) (Observation, error) { + return healthyObservation(clock.Now()), nil + }) + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err, "an absent `from` in single-step mode is a fallback, not an error") + require.Contains(t, notesOf(cfg), "no `from` given") + require.True(t, res.From.Equal(testNow)) +} + +// In single-step mode an explicit `from` earlier than the first observation is +// a DECLARED blind interval — a warning and a pass, naming the exact interval +// it cannot see. Recorder mode keeps the from-bounds coverage check strict. +func TestCheckSingleStepFromBeforeFirstObservationWarnsAndPasses(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + cfg.From = testNow.Add(-2 * time.Minute) + src := newCheckSource(func(_ string, _ int) (Observation, error) { + return healthyObservation(clock.Now()), nil + }) + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err) + notes := notesOf(cfg) + require.Contains(t, notes, "cannot see [") + require.Contains(t, notes, testNow.Format(time.RFC3339)) + // The classified window is the clamped one, and Result says so rather than + // reporting a window the run never proved. + require.True(t, res.From.Equal(testNow)) +} + +// The failure limit was exceeded. The measurement pass succeeds and the +// collection loop then hits a terminal failure, so this exercises the path a +// live run really takes. +func TestCheckFailClosedOnExhaustedRetries(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + src := newCheckSource(func(_ string, call int) (Observation, error) { + if call > 1 { + return Observation{}, &RetryExhaustedError{Failures: 6, Cause: errors.New("connection refused")} + } + return healthyObservation(clock.Now()), nil + }) + + res, err := check(context.Background(), cfg, src) + require.Error(t, err, "the collection failure to fail closed") + require.Contains(t, err.Error(), "collect evidence") + require.Empty(t, res.Violations, "an error must never be reported as a verdict") +} + +// The resolution of the definitions failed. Both shapes — the ruler read +// itself failing, and a name that resolves to nothing. +func TestCheckFailClosedOnDefinitionResolution(t *testing.T) { + t.Run("ruler read fails", func(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + src := newCheckSource(nil) + src.defsErr = errors.New("502 bad gateway") + + _, err := check(context.Background(), cfg, src) + require.Error(t, err) + require.Contains(t, err.Error(), "read rule definitions") + }) + + t.Run("unknown alert name", func(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + cfg.Alerts = []string{"No Such Rule"} + src := newCheckSource(nil) + + _, err := check(context.Background(), cfg, src) + require.Error(t, err) + require.Contains(t, err.Error(), "no rule matched") + }) +} + +// The version gate: an unsupported Grafana is exit 2 before anything else is +// attempted. +func TestCheckRefusesUnsupportedGrafanaVersion(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + src := newCheckSource(nil) + src.version = "12.4.0" + + _, err := check(context.Background(), cfg, src) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported grafana version") +} + +// The budget is checked against the latencies the measurement pass actually +// measured, and a schedule that cannot fit errors at START rather than +// producing a gap-riddled recording nobody can classify. +func TestCheckSingleStepRefusesAScheduleThatDoesNotFit(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + src := newCheckSource(func(_ string, _ int) (Observation, error) { + obs := healthyObservation(clock.Now()) + obs.Latency = 45 * time.Second // longer than the rule's own 30s cadence + return obs, nil + }) + + _, err := check(context.Background(), cfg, src) + require.Error(t, err, "the budget check to refuse the schedule") + for _, want := range []string{"raising concurrency", "raising poll-interval", "watching fewer alerts"} { + require.Contains(t, err.Error(), want) + } +} + +// --------------------------------------------------------------------------- +// Recorder mode +// --------------------------------------------------------------------------- + +// recordedLog writes a log the way watch would have: a header, one poll every +// 30s over [start, end], and a stopped sentinel at sentinelAt. lastEvalLag is +// how far behind each poll's own GrafanaNow its lastEvaluation sits, which is +// what the drain-wait tests vary. +func recordedLog(t *testing.T, dir string, url string, startedAt, start, end, sentinelAt time.Time, lastEvalLag time.Duration) string { + t.Helper() + path := filepath.Join(dir, "log.jsonl") + clock := newFakeClock(sentinelAt) + w, err := NewWriter(path, clock) + require.NoError(t, err) + header := Header{ + URL: url, + GrafanaVersion: "13.1.0", + StartedAt: startedAt, + Rules: []LoggedRule{{ + UID: checkUID, Title: checkTitle, Folder: "F", Group: "G", + IntervalSeconds: 60, NoDataState: "OK", ExecErrState: "OK", + PollEverySeconds: checkPollEvery.Seconds(), + }}, + } + require.NoError(t, w.WriteHeader(header)) + for at := start; !at.After(end); at = at.Add(checkPollEvery) { + require.NoError(t, w.WritePoll(Poll{ + RuleUID: checkUID, GrafanaNow: at, Found: true, + State: "inactive", Health: "ok", LastEvaluation: at.Add(-lastEvalLag), + })) + } + require.NoError(t, w.Stop()) + return path +} + +// deadPid returns a pid that is guaranteed to have exited — the normal state +// of a recorder by the time check signals it, since a recorder given --until +// (or one that finished cleanly) is already gone. +func deadPid(t *testing.T) int { + t.Helper() + cmd := exec.Command("/bin/sh", "-c", "exit 0") + require.NoError(t, cmd.Start()) + pid := cmd.Process.Pid + require.NoError(t, cmd.Wait()) + return pid +} + +func writePid(t *testing.T, path, contents string) { + t.Helper() + require.NoError(t, os.WriteFile(path, []byte(contents), 0o644)) +} + +// recorderConfig points check at a recording of [testNow-1m, windowEnd+30s] +// over the window [testNow, testNow+5m]. +func recorderConfig(t *testing.T, clock Clock, logPath string) Config { + t.Helper() + return Config{ + URL: "https://grafana.example.com", + Log: logPath, + From: testNow, + To: testNow.Add(5 * time.Minute), + Clock: clock, + Notes: &strings.Builder{}, + }.withDefaults() +} + +func TestCheckRecorderModeCleanWindowPasses(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(-time.Minute), testNow.Add(-time.Minute), windowEnd.Add(30*time.Second), windowEnd.Add(30*time.Second), 0) + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + clock := newVirtualClock(testNow.Add(time.Minute)) + cfg := recorderConfig(t, clock, logPath) + // The drain wait is satisfied from the log's own evidence, so the source + // must never be asked for a state — asserted by the nil responder. + src := newCheckSource(func(title string, _ int) (Observation, error) { + require.Fail(t, fmt.Sprintf("the drain wait polled %q although the log already proves the evaluations", title)) + return Observation{}, errors.New("unexpected poll") + }) + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err) + require.Empty(t, res.Violations) + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeClean, res.Verdicts[0].Outcome) + require.Equal(t, "13.1.0", res.GrafanaVersion) + // The collection loop still waited out to+transitionGrace even though the + // recorder had already finished. + require.False(t, clock.Now().Before(windowEnd)) +} + +// The identity of the log is not correct. The check runs against the header +// read EARLY, so it fails before the window's wait rather than after it. +func TestCheckFailClosedOnWrongLogIdentity(t *testing.T) { + t.Run("different url", func(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + logPath := recordedLog(t, dir, "https://other.example.com", + testNow.Add(-time.Minute), testNow.Add(-time.Minute), windowEnd, windowEnd.Add(30*time.Second), 0) + + clock := newVirtualClock(testNow) + cfg := recorderConfig(t, clock, logPath) + _, err := check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err) + require.Contains(t, err.Error(), "log identity") + require.True(t, clock.Now().Equal(testNow), "it must fail before the wait") + }) + + t.Run("rule no longer resolves", func(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(-time.Minute), testNow.Add(-time.Minute), windowEnd, windowEnd.Add(30*time.Second), 0) + + cfg := recorderConfig(t, newVirtualClock(testNow), logPath) + src := newCheckSource(nil) + src.defs = []Definition{{UID: "somebody-else", Title: "Other", Kind: KindGrafanaManaged, IntervalSeconds: 60}} + + _, err := check(context.Background(), cfg, src) + require.Error(t, err) + require.Contains(t, err.Error(), "log identity") + }) +} + +// `from` before the recording's StartedAt is statically knowable from the +// header (immutable line 1), so check fails closed on it BEFORE the window's +// wait — exactly like the identity check above — rather than surfacing a +// from_before_record verdict only after the drain. +func TestCheckFailFastWhenFromPrecedesRecordStart(t *testing.T) { + dir := t.TempDir() + startedAt := testNow.Add(time.Minute) // the recording opened a minute AFTER `from` + windowEnd := testNow.Add(5*time.Minute + checkGrace) + // The poll range is irrelevant to the assertion: the fail-fast reads + // StartedAt from the header alone, before any polling would matter. + logPath := recordedLog(t, dir, "https://grafana.example.com", + startedAt, startedAt, windowEnd.Add(30*time.Second), windowEnd.Add(30*time.Second), 0) + + clock := newVirtualClock(testNow) + cfg := recorderConfig(t, clock, logPath) // From = testNow, before StartedAt + _, err := check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err) + require.Contains(t, err.Error(), "before recording started") + require.True(t, clock.Now().Equal(testNow), "it must fail before the wait") +} + +// A whole-second `from` in the same second as the recording's sub-second +// StartedAt is not a blind interval: the whole-second comparison lets the run +// proceed to a clean pass instead of the fail-fast above. +func TestCheckRecorderModeFromSameSecondAsStartedAtPasses(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + // StartedAt is 500ms after `from` (testNow via recorderConfig) — the same + // whole second. Polls still cover the whole window. + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(500*time.Millisecond), testNow.Add(-time.Minute), windowEnd.Add(30*time.Second), windowEnd.Add(30*time.Second), 0) + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + clock := newVirtualClock(testNow.Add(time.Minute)) + cfg := recorderConfig(t, clock, logPath) + src := newCheckSource(func(title string, _ int) (Observation, error) { + require.Fail(t, fmt.Sprintf("the drain wait polled %q although the log already proves the evaluations", title)) + return Observation{}, errors.New("unexpected poll") + }) + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err) + require.Empty(t, res.Violations) + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeClean, res.Verdicts[0].Outcome) +} + +// The coverage proof failed: a hole in the middle of the recording is not +// saved by healthy data at both ends. +func TestCheckFailClosedOnCoverageGap(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + path := filepath.Join(dir, "log.jsonl") + clock := newFakeClock(windowEnd.Add(30 * time.Second)) + w, err := NewWriter(path, clock) + require.NoError(t, err) + require.NoError(t, w.WriteHeader(Header{ + URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), + Rules: []LoggedRule{{UID: checkUID, Title: checkTitle, IntervalSeconds: 60, PollEverySeconds: checkPollEvery.Seconds()}}, + })) + for at := testNow.Add(-time.Minute); !at.After(windowEnd.Add(30 * time.Second)); at = at.Add(checkPollEvery) { + // A three-minute hole in the middle of the window. + if at.After(testNow.Add(time.Minute)) && at.Before(testNow.Add(4*time.Minute)) { + continue + } + require.NoError(t, w.WritePoll(Poll{ + RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at, + })) + } + require.NoError(t, w.Stop()) + writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + cfg := recorderConfig(t, newVirtualClock(testNow), path) + res, err := check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err, "the coverage gap to fail closed") + require.Equal(t, ReasonHeartbeatGap, res.Coverage[checkUID].Reason) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome) +} + +// An episode fully between the deploy and the start of the check: recorder +// mode must find this at the LEADING edge of the window too, right after `from` +// (the deploy's completion), not only in the middle +// (TestCheckFailClosedOnCoverageGap above). No poll exists for [from, from+3m), +// so whatever happened there is invisible to every per-poll check and only the +// coverage gap itself can catch it — the reason the recorder exists at all. +func TestCheckRecorderModeFindsAGapImmediatelyAfterTheDeploy(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + path := filepath.Join(dir, "log.jsonl") + clock := newFakeClock(windowEnd.Add(30 * time.Second)) + w, err := NewWriter(path, clock) + require.NoError(t, err) + require.NoError(t, w.WriteHeader(Header{ + URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), + Rules: []LoggedRule{{UID: checkUID, Title: checkTitle, IntervalSeconds: 60, PollEverySeconds: checkPollEvery.Seconds()}}, + })) + gapEnd := testNow.Add(3 * time.Minute) // nothing recorded from `from` (testNow) to here + for at := gapEnd; !at.After(windowEnd.Add(30 * time.Second)); at = at.Add(checkPollEvery) { + require.NoError(t, w.WritePoll(Poll{ + RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at, + })) + } + require.NoError(t, w.Stop()) + writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + cfg := recorderConfig(t, newVirtualClock(testNow), path) + res, err := check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err, "a hole right after the deploy hides whatever happened there") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome, "never clean") +} + +// The drain limit passed. The recording itself is clean, so this isolates the +// drain wait — the rule simply never evaluates through the end of the window, +// and a rule that cannot answer that question is unobservable. +func TestCheckFailClosedOnDrainTimeout(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + // A 45s lag keeps every poll inside evalStaleAfter (120s), so the liveness + // coverage check is silent and only the drain wait can fail. + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(-time.Minute), testNow.Add(-time.Minute), windowEnd, windowEnd.Add(30*time.Second), 45*time.Second) + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + clock := newVirtualClock(testNow) + cfg := recorderConfig(t, clock, logPath) + frozen := windowEnd.Add(-45 * time.Second) + src := newCheckSource(func(_ string, _ int) (Observation, error) { + now := clock.Now() + return Observation{ + Rules: []StateRule{checkStateRule(frozen)}, + GrafanaNow: now, + }, nil + }) + + res, err := check(context.Background(), cfg, src) + require.Error(t, err, "the drain limit to fail closed") + require.Equal(t, ReasonDrainTimeout, res.Coverage[checkUID].Reason) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome) + require.Contains(t, res.Verdicts[0].Note, "drain limit") + require.GreaterOrEqual(t, clock.Now().Sub(windowEnd), checkDrainLimit, + "the rule never evaluates through the window, so the drain wait must run its full limit") +} + +// A rule the state endpoint no longer serves is knowable on the FIRST drain +// poll, and the answer is rule_absent — the fault — rather than +// drain_timeout, which would only name the wait. It must not spend the whole +// drain limit to reach it. +func TestCheckDrainWaitNamesADeletedRuleAtOnce(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(-time.Minute), testNow.Add(-time.Minute), windowEnd, windowEnd.Add(30*time.Second), 45*time.Second) + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + clock := newVirtualClock(testNow) + cfg := recorderConfig(t, clock, logPath) + // An authoritative 2xx that parsed and carries no matching rule. The + // transport retried every transient failure long before an Observation + // exists, so this is a deletion, not a hiccup. + src := newCheckSource(func(_ string, _ int) (Observation, error) { + return Observation{GrafanaNow: clock.Now()}, nil + }) + + res, err := check(context.Background(), cfg, src) + require.Error(t, err, "a deleted rule to fail closed") + require.Equal(t, ReasonRuleAbsent, res.Coverage[checkUID].Reason, "the fault, not the wait") + require.Equal(t, 1, src.callCount(checkTitle), "the absence is knowable on the first poll") + require.Less(t, clock.Now().Sub(windowEnd), checkDrainLimit) +} + +// --------------------------------------------------------------------------- +// `skipped` comes from the header, not from a definition read after the window +// --------------------------------------------------------------------------- + +// pausedAfterWindowLog records a rule that was ACTIVE at record start and that +// fired inside the window. The caller then tells check that the rule's current +// definition says paused — the state somebody set after the fact. +func pausedAfterWindowLog(t *testing.T, dir string, firesAt time.Time, end, sentinelAt time.Time) string { + t.Helper() + path := filepath.Join(dir, "log.jsonl") + w, err := NewWriter(path, newFakeClock(sentinelAt)) + require.NoError(t, err) + require.NoError(t, w.WriteHeader(Header{ + URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), + Rules: []LoggedRule{{ + UID: checkUID, Title: checkTitle, IntervalSeconds: 60, + IsPaused: false, PollEverySeconds: checkPollEvery.Seconds(), + }}, + })) + firing := Instance{ + Labels: map[string]string{"alertname": checkTitle, "instance": "a"}, + State: StateFiring, + ActiveAt: firesAt, + } + for at := testNow.Add(-time.Minute); !at.After(end); at = at.Add(checkPollEvery) { + p := Poll{RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at} + if !at.Before(firesAt) { + p.State = "firing" + p.Abnormal = []Instance{firing} + } + require.NoError(t, w.WritePoll(p)) + } + require.NoError(t, w.Stop()) + return path +} + +// pausedAfterWindowCheck runs the timeline above. The recording reaches past +// to + transitionGrace, which for this 60s rule is to + 60s: the fresh +// definition says paused, but that no longer shrinks the grace — the header +// does, and the header says the rule was active (deriveGlobalTimings). +func pausedAfterWindowCheck(t *testing.T, allowPaused bool) (Result, error, Config) { + t.Helper() + dir := t.TempDir() + to := testNow.Add(5 * time.Minute) + end := to.Add(checkGrace + 30*time.Second) + logPath := pausedAfterWindowLog(t, dir, testNow.Add(2*time.Minute), end, end) + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + cfg := recorderConfig(t, newVirtualClock(testNow), logPath) + cfg.AllowPaused = allowPaused + + src := newCheckSource(nil) + paused := checkDef() + paused.IsPaused = true // somebody paused it after the alert started paging + src.defs = []Definition{paused} + + res, err := check(context.Background(), cfg, src) + return res, err, cfg +} + +// The window's own evidence outranks a definition read after it closed: a rule +// that was active at record start is classified, whatever its pause state is +// by the time check resolves the definitions. +func TestCheckPausingARuleAfterTheWindowDoesNotMakeItSkipped(t *testing.T) { + res, err, _ := pausedAfterWindowCheck(t, false) + require.NoError(t, err) + require.Equal(t, OutcomeNewlyBad, res.Verdicts[0].Outcome, + "the rule was active for the whole window and fired inside it") + require.Len(t, res.Violations, 1) + require.Equal(t, OutcomeNewlyBad, res.Violations[0].Outcome) + require.NotContains(t, res.Verdicts[0].Note, "paused before the window opened") +} + +// The regression pin for the loophole this fix closed. Reading skipped from +// the post-window definition made the rule skipped; --allow-paused then made +// skipped free; and a window in which the alert fired reported exit 0. The +// default message names --allow-paused, so an operator was led straight to it. +func TestCheckAllowPausedCannotExcuseARulePausedAfterItFired(t *testing.T) { + res, err, _ := pausedAfterWindowCheck(t, true) + require.NoError(t, err) + require.NotEmpty(t, res.Violations, "the run passed over a window in which the alert fired") +} + +// The other direction, unchanged: a rule the HEADER says was paused when the +// recording opened is genuinely skipped. It has no polls, so no coverage is +// attempted for it, and --allow-paused behaves as it always did. +func TestCheckHeaderPausedRuleStaysSkipped(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + path := filepath.Join(dir, "log.jsonl") + w, err := NewWriter(path, newFakeClock(windowEnd.Add(30*time.Second))) + require.NoError(t, err) + // Named in the header, is_paused true, and no poll records at all — the + // shape watch writes for a rule paused before the window opened. + require.NoError(t, w.WriteHeader(Header{ + URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), + Rules: []LoggedRule{{ + UID: checkUID, Title: checkTitle, IntervalSeconds: 60, + IsPaused: true, PollEverySeconds: checkPollEvery.Seconds(), + }}, + })) + require.NoError(t, w.Stop()) + writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + run := func(allowPaused bool) (Result, error) { + cfg := recorderConfig(t, newVirtualClock(testNow), path) + cfg.AllowPaused = allowPaused + // The definition is unpaused now; the header still decides. + src := newCheckSource(func(title string, _ int) (Observation, error) { + require.Fail(t, fmt.Sprintf("the drain wait polled skipped rule %q", title)) + return Observation{}, errors.New("unexpected poll") + }) + return check(context.Background(), cfg, src) + } + + res, err := run(false) + require.NoError(t, err, "a skipped rule is a known condition, not an inability") + require.Equal(t, OutcomeSkipped, res.Verdicts[0].Outcome) + _, ok := res.Coverage[checkUID] + require.False(t, ok, "a skipped rule has no coverage to prove") + require.Len(t, res.Violations, 1, "the MinObserved shortfall") + + res, err = run(true) + require.NoError(t, err) + require.Empty(t, res.Violations, "with --allow-paused: want a pass") +} + +// A paused rule does not evaluate, so it can never catch up: the drain wait +// must conclude on the first poll instead of spending the whole limit. +func TestCheckDrainWaitConcludesAtOnceOnAPausedRule(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(-time.Minute), testNow.Add(-time.Minute), windowEnd, windowEnd.Add(30*time.Second), 45*time.Second) + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + clock := newVirtualClock(testNow) + cfg := recorderConfig(t, clock, logPath) + src := newCheckSource(func(_ string, _ int) (Observation, error) { + now := clock.Now() + rule := checkStateRule(windowEnd.Add(-45 * time.Second)) + rule.IsPaused = true + return Observation{Rules: []StateRule{rule}, GrafanaNow: now}, nil + }) + + res, err := check(context.Background(), cfg, src) + require.Error(t, err, "a rule that stopped evaluating must fail closed") + require.Equal(t, 1, src.callCount(checkTitle), "a paused rule can never catch up") + require.Equal(t, ReasonDrainTimeout, res.Coverage[checkUID].Reason, + "the vocabulary is published, so the detail goes in the note") + require.Contains(t, res.Verdicts[0].Note, "paused before it evaluated through") + require.Less(t, clock.Now().Sub(windowEnd), checkDrainLimit) +} + +// An absent or unparseable pidfile is never "there was nothing to stop". The +// parent writes the pidfile only once the child reports that it is recording, +// so a missing one means the recording never started — and the log must not be +// read at all. +func TestCheckRefusesToReadALogItCannotStop(t *testing.T) { + windowEnd := testNow.Add(5*time.Minute + checkGrace) + + tests := []struct { + name string + pidfile string // "" = do not create one + }{ + {name: "missing pidfile"}, + {name: "unparseable pidfile", pidfile: "not-a-pid\n"}, + {name: "empty pidfile", pidfile: ""}, + } + + for i, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(-time.Minute), testNow.Add(-time.Minute), windowEnd, windowEnd.Add(30*time.Second), 0) + if i != 0 { + writePid(t, logPath+".pid", tc.pidfile) + } + + cfg := recorderConfig(t, newVirtualClock(testNow), logPath) + _, err := check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err) + require.Contains(t, err.Error(), "cannot stop the recorder") + }) + } +} + +// startLockHolder re-execs this test binary as a process that holds the log's +// flock and ignores SIGTERM, and returns its pid once the lock is genuinely +// held. See lockHolderEnv (watch_daemon_test.go) for why it must be a separate +// real process rather than a shell one-liner. +func startLockHolder(t *testing.T, logPath string) int { + t.Helper() + cmd := exec.Command(os.Args[0]) + cmd.Env = append(os.Environ(), lockHolderEnv+"="+logPath) + cmd.Stderr = os.Stderr + stdout, err := cmd.StdoutPipe() + require.NoError(t, err) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err, "the lock holder never reported holding the lock") + return cmd.Process.Pid +} + +// A recorder that will not let go of the log means the log may still be +// appended to, and a log a writer can change cannot be read at all. +func TestCheckFailsWhenTheRecorderWillNotExit(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(-time.Minute), testNow.Add(-time.Minute), windowEnd, windowEnd.Add(30*time.Second), 0) + + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", startLockHolder(t, logPath))) + + cfg := recorderConfig(t, newVirtualClock(testNow), logPath) + _, err := check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err) + require.Contains(t, err.Error(), "still holds") +} + +// The regression pin for a stray SIGTERM. Nothing removes the pidfile when a +// recorder exits cleanly — the parent has returned and the child never learns +// the path — so after a --until run, a supported flow, the pidfile names a pid +// the operating system is free to hand to somebody else. The flock, not the +// pid, is what says whether a writer exists. +func TestCheckDoesNotSignalABystanderHoldingAReusedPid(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(-time.Minute), testNow.Add(-time.Minute), windowEnd.Add(30*time.Second), windowEnd.Add(30*time.Second), 0) + + // An innocent process that happens to hold the pid the finished recorder + // left behind. It does not hold the log's lock, because it is not a + // recorder. + bystander := exec.Command("sleep", "30") + require.NoError(t, bystander.Start()) + t.Cleanup(func() { + _ = bystander.Process.Kill() + _ = bystander.Wait() + }) + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", bystander.Process.Pid)) + + cfg := recorderConfig(t, newVirtualClock(testNow), logPath) + _, err := check(context.Background(), cfg, newCheckSource(nil)) + require.NoError(t, err) + require.NoError(t, syscall.Kill(bystander.Process.Pid, 0), + "check signalled a process that was not the recorder") +} + +// A dead pidfile (the recorder process has already exited, holding no flock) +// with NO sentinel in the log — the shape a killed `watch` leaves behind — +// must not hang the stop wait: the flock is free immediately, so +// check reads the log at once, finds no sentinel, and fails closed. +func TestCheckDeadPidWithNoSentinelIsUnobservable(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + logPath := filepath.Join(dir, "log.jsonl") + w, err := NewWriter(logPath, newFakeClock(testNow)) + require.NoError(t, err) + require.NoError(t, w.WriteHeader(Header{ + URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), + Rules: []LoggedRule{{ + UID: checkUID, Title: checkTitle, Folder: "F", Group: "G", + IntervalSeconds: 60, NoDataState: "OK", ExecErrState: "OK", + PollEverySeconds: checkPollEvery.Seconds(), + }}, + })) + // Healthy heartbeats all the way past windowEnd — evaluatedThrough is + // satisfied, so the drain wait needs no live re-poll — but no sentinel is + // ever written: the recorder died before it could call Stop. + for at := testNow.Add(-time.Minute); !at.After(windowEnd.Add(30 * time.Second)); at = at.Add(checkPollEvery) { + require.NoError(t, w.WritePoll(Poll{RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at})) + } + require.NoError(t, w.Close()) // no sentinel — a clean exit would call Stop + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + cfg := recorderConfig(t, newVirtualClock(testNow), logPath) + res, err := check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err, "no sentinel means the recorder never proved it ran to the end") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome) +} + +// An incomplete last line gives exit 2. log_test.go's TestReadLogRejectsBadLogs +// pins ReadLog's own error and TestExitCode pins that any non-nil error maps to +// exit 2, but only this feeds a genuinely truncated log through check() itself: +// a raw file with a valid header and poll, then a torn JSON tail, exactly what +// a recorder killed mid-write leaves behind. +func TestCheckRecorderModeTruncatedLogFailsClosed(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "log.jsonl") + + h := Header{ + SchemaVersion: LogSchemaVersion, + URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), + Rules: []LoggedRule{{UID: checkUID, Title: checkTitle, IntervalSeconds: 60, PollEverySeconds: checkPollEvery.Seconds()}}, + } + hb, err := json.Marshal(headerRecord{Type: RecordHeader, Header: h}) + require.NoError(t, err) + pb, err := json.Marshal(pollRecord{Type: RecordPoll, Poll: Poll{ + RuleUID: checkUID, GrafanaNow: testNow, Found: true, State: "inactive", Health: "ok", LastEvaluation: testNow, + }}) + require.NoError(t, err) + content := string(hb) + "\n" + string(pb) + "\n" + `{"type":"poll","rule_ui` // torn mid-write + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) + writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + cfg := recorderConfig(t, newVirtualClock(testNow), path) + _, err = check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err) + require.Contains(t, err.Error(), "unparseable") +} + +// One authority for the cadence, from check's side: maxGap comes from the +// cadence the header records, never from a re-derivation off intervalSeconds. +// The fail-open direction is the one asserted — a log recorded at 5s on a 60s +// rule must still fail on a hole a re-derived 30s maxGap would have forgiven. +func TestCheckDerivesMaxGapFromTheRecordedCadence(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + path := filepath.Join(dir, "log.jsonl") + w, err := NewWriter(path, newFakeClock(windowEnd.Add(30*time.Second))) + require.NoError(t, err) + require.NoError(t, w.WriteHeader(Header{ + URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), + Rules: []LoggedRule{{UID: checkUID, Title: checkTitle, IntervalSeconds: 60, PollEverySeconds: 5}}, + })) + for at := testNow.Add(-time.Minute); !at.After(windowEnd.Add(30 * time.Second)); at = at.Add(5 * time.Second) { + // A 20s hole: under the recorded 5s cadence maxGap is 10s and this + // fails; under a cadence re-derived from intervalSeconds it would be + // 60s and the hole would pass unseen. + if at.After(testNow.Add(time.Minute)) && at.Before(testNow.Add(80*time.Second)) { + continue + } + require.NoError(t, w.WritePoll(Poll{ + RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at, + })) + } + require.NoError(t, w.Stop()) + writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + cfg := recorderConfig(t, newVirtualClock(testNow), path) + res, err := check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err, "a 20s hole exceeds the 10s maxGap the recorded 5s cadence implies") + require.Equal(t, ReasonHeartbeatGap, res.Coverage[checkUID].Reason) +} + +// --------------------------------------------------------------------------- +// The pieces, in isolation +// --------------------------------------------------------------------------- + +// The drain wait's one comparison is cross-domain, and its uncertainty is +// spent in the fail-closed direction: an evaluation that only MIGHT have +// reached the end of the window does not count as one that did. +func TestEvaluatedThroughSpendsItsUncertaintyFailingClosed(t *testing.T) { + end := testNow + + tests := []struct { + name string + lastEval time.Time + skew, bound time.Duration + wantSatisfied bool + }{ + {name: "zero lastEvaluation never satisfies", lastEval: time.Time{}, wantSatisfied: false}, + {name: "exactly at the end, no skew", lastEval: end, wantSatisfied: true}, + {name: "one second short", lastEval: end.Add(-time.Second), wantSatisfied: false}, + { + name: "far enough past the end to absorb the bound", + // Grafana runs 10s fast; the reading translates back to end+5s and + // the 1s bound still leaves it past the end. + lastEval: end.Add(16 * time.Second), skew: 10 * time.Second, bound: time.Second, + wantSatisfied: true, + }, + { + name: "inside the bound is not proof", + // Translated it lands exactly on the end, so the bound can put it + // either side — which is not an evaluation THROUGH the end. + lastEval: end.Add(10 * time.Second), skew: 10 * time.Second, bound: time.Second, + wantSatisfied: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.wantSatisfied, evaluatedThrough(tc.lastEval, tc.skew, tc.bound, end)) + }) + } +} + +// A drain timeout on one rule and a coverage failure on another must both +// reach the message. Neither error may shadow the other. +func TestMergeDrainTimeoutsNamesEveryUnobservableRule(t *testing.T) { + res := Result{ + Coverage: map[string]CoverageResult{ + "a": {Proved: true}, + "b": {Unobservable: true, Reason: ReasonHeartbeatGap, Notes: []string{"rule \"B\": gap"}}, + }, + Verdicts: []RuleVerdict{ + {Alert: "A", RuleUID: "a", Outcome: OutcomeClean}, + {Alert: "B", RuleUID: "b", Outcome: OutcomeUnobservable}, + }, + } + + merged, err := mergeDrainTimeouts(res, map[string]drainVerdict{ + "a": {reason: ReasonDrainTimeout, note: "rule \"A\": did not evaluate through the end within the drain limit"}, + "b": {reason: ReasonDrainTimeout, note: "rule \"B\": did not evaluate through the end within the drain limit"}, + }) + require.Error(t, err, "naming the newly unobservable rule") + require.Contains(t, err.Error(), "unobservable at the drain wait") + // Only A is newly unobservable; B was already, so naming it twice would + // only lengthen the message. + require.Contains(t, err.Error(), "A ("+string(ReasonDrainTimeout)+")") + require.NotContains(t, err.Error(), "B (") + require.Equal(t, ReasonDrainTimeout, merged.Coverage["a"].Reason) + // B keeps the reason the coverage proof gave it — the FIRST reason wins, + // as it does inside proveCoverage. + require.Equal(t, ReasonHeartbeatGap, merged.Coverage["b"].Reason) + require.Equal(t, OutcomeUnobservable, merged.Verdicts[0].Outcome) +} + +// ReadLogHeader is the one read of a log a writer may still hold, so its +// refusals matter as much as its successes. +func TestReadLogHeader(t *testing.T) { + dir := t.TempDir() + + t.Run("reads line 1 while the log keeps growing", func(t *testing.T) { + path := filepath.Join(dir, "growing.jsonl") + w, err := NewWriter(path, newFakeClock(testNow)) + require.NoError(t, err) + defer w.Close() + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.WritePoll(Poll{RuleUID: "rule1", GrafanaNow: testNow, Found: true})) + + h, err := ReadLogHeader(path) + require.NoError(t, err) + require.Equal(t, testHeader().URL, h.URL) + require.Len(t, h.Rules, 1) + }) + + t.Run("a half-written header is not a header", func(t *testing.T) { + path := filepath.Join(dir, "torn.jsonl") + require.NoError(t, os.WriteFile(path, []byte(`{"type":"header","url":"htt`), 0o644)) + _, err := ReadLogHeader(path) + require.Error(t, err) + require.Contains(t, err.Error(), "no complete header") + }) + + t.Run("a wrong schema version is refused", func(t *testing.T) { + path := filepath.Join(dir, "old.jsonl") + require.NoError(t, os.WriteFile(path, []byte(`{"type":"header","schema_version":99,"url":"u"}`+"\n"), 0o644)) + _, err := ReadLogHeader(path) + require.Error(t, err) + require.Contains(t, err.Error(), "schema version 99") + }) +} diff --git a/grafana-alertcheck/internal/gate/classify.go b/grafana-alertcheck/internal/gate/classify.go new file mode 100644 index 000000000..d8d058a34 --- /dev/null +++ b/grafana-alertcheck/internal/gate/classify.go @@ -0,0 +1,610 @@ +package gate + +import ( + "fmt" + "slices" + "strings" + "time" +) + +// ReasonNodata is decide's own unobservable reason: proveCoverage never sets it +// — health=nodata is a note there, never fatal — because escalating it needs +// Policy.NodataIsUnobservable, which only decide (the Policy-holding seam) has. +const ReasonNodata UnobservableReason = "nodata" + +// Outcome is the verdict of one instance's timeline, and (after decide takes +// the worst across instances) of the rule. It is a published JSON output: the +// fail values stay distinct even though v1 maps them all to exit 1, so a later +// version can split them without breaking the interface. +type Outcome string + +const ( + OutcomeClean Outcome = "clean" + OutcomeNewlyBad Outcome = "newly_bad" + OutcomeRecovered Outcome = "recovered" + OutcomePersistentlyBad Outcome = "persistently_bad" + OutcomeFlapping Outcome = "flapping" + OutcomeSkipped Outcome = "skipped" + OutcomeUnobservable Outcome = "unobservable" +) + +// PreexistingPolicy governs only the ONE ambiguous case in the outcome table: +// an instance that was already bad when the window opened. A newly_bad or +// flapping instance is a fail under every policy, so this type only ever +// changes how `recovered` and `persistently_bad` are judged (isViolation +// below). +type PreexistingPolicy string + +const ( + // PreexistingFailUnlessRecovered is the default: a preexisting instance + // that clears and stays clear is a pass (`recovered`); one that never + // clears is still a fail (`persistently_bad`). + PreexistingFailUnlessRecovered PreexistingPolicy = "fail-unless-recovered" + // PreexistingFail makes ANY preexisting instance a fail, even one that + // recovers — for a user who wants no benefit of the doubt for a + // condition this release did not cause. + PreexistingFail PreexistingPolicy = "fail" + // PreexistingIgnore disregards a preexisting instance entirely, whether + // it recovers or stays bad for the whole window: only a genuinely NEW + // bad episode (newly_bad or flapping) can fail the rule. + PreexistingIgnore PreexistingPolicy = "ignore" +) + +// Violation is one instance whose timeline outcome counts against the run, +// after the preexisting policy has been applied (isViolation below). +type Violation struct { + Alert, RuleUID string + Outcome Outcome + State State + Health string // raw, reporting-only, like Poll.Health + LastError string + // FirstSeen is the episode's onset in the runner domain (translated by the + // poll's own skew), or `from` when preexisting — never a raw Grafana time. + FirstSeen time.Time + // ClearedAt is zero unless the episode closed via a genuine Cleared event. + ClearedAt time.Time + InstanceLabels map[string]string + // Note explains a Violation with no instance behind it — decide's synthetic + // MinObserved shortfall — and must not double as LastError (reporting-only + // rule state from a real poll). + Note string +} + +// RuleVerdict is one rule's worst-of outcome, present for every resolved rule +// (passes included) so the table shows every alert asked for. +type RuleVerdict struct { + Alert, RuleUID string + Outcome Outcome + BadFor time.Duration // total wall-clock time any instance was bad inside the window, overlaps merged + PollEvery time.Duration + Note string +} + +// Policy is decide's narrowed, pure-layer view of a Config: the classification +// knobs and the window, nothing else. No URL, no token, no I/O handles — those +// never reach the pure layer. +type Policy struct { + States []State + Preexisting PreexistingPolicy + MinObserved int + AllowPaused, NodataIsUnobservable bool + From, To time.Time +} + +// RuleThresholds is one non-skipped rule's resolved coverage thresholds, +// carried on Result so the CLI's table can print the numbers that answer "why" +// on exit 2 without decide exposing the unexported ruleTimings type itself. +type RuleThresholds struct { + MaxGap time.Duration + HealthGrace time.Duration + EvalStaleAfter time.Duration +} + +// GlobalThresholds is the run-wide half of the same information: +// transitionGrace and drainTimeout apply once, across every non-skipped +// watched rule, not per rule (globalTimings). +type GlobalThresholds struct { + TransitionGrace time.Duration + // GraceSource names, and already carries the `for` value of, the rule that + // set TransitionGrace — an operator has to see both. "none" when no rule + // contributed (TransitionGrace is then 0). + GraceSource string + DrainTimeout time.Duration +} + +// Result is decide's whole answer: everything the human table and the JSON +// output need. Coverage carries one CoverageResult per non-skipped rule — +// there is deliberately no separate Interval type anywhere in the project. +type Result struct { + From, To time.Time + GrafanaVersion string + ClockSkew time.Duration // the largest |skew| across every poll decide was given, not only the ones a rule's window actually used + // ClockSkewBound is the skew BOUND (RTT/2) of that SAME poll — not + // the largest bound seen overall, which would pair a wide bound from an + // unrelated slow request with the worst skew and misstate how tightly + // that skew is actually known. SkewHardLimit is a separate, fixed input + // validation threshold (source.go) and is not an error bound on this + // value; the CLI prints both, but must not conflate them. + ClockSkewBound time.Duration + Coverage map[string]CoverageResult + // Thresholds carries one RuleThresholds per rule Coverage also covers — + // every non-skipped rule, keyed by UID. A skipped rule has neither: it was + // never scheduled, so it has no maxGap/healthGrace/evalStaleAfter to + // report. + Thresholds map[string]RuleThresholds + Global GlobalThresholds + Verdicts []RuleVerdict + Violations []Violation +} + +// episode is one contiguous, policy-bad span of one instance's timeline, +// already resolved to the runner domain and clamped to [from, windowEnd]. It +// never crosses a genuine Cleared event: a Vanished marker freezes the state +// instead of closing the episode, which is what keeps a vanish from ever +// reading as a recovery. +type episode struct { + start, end time.Time + closedByRealClear bool +} + +// instanceTimeline accumulates one instance's walk across a rule's in-window +// polls. preexisting is decided once, the first time this key is seen bad: by +// the translated ActiveAt against `from`, never by which poll happened to +// report it first — a poll's own cadence is not evidence of when the condition +// actually began. +type instanceTimeline struct { + labels map[string]string + preexisting bool + seen bool + badOpen bool + episodeStart time.Time + lastState State + lastHealth string + lastError string + episodes []episode +} + +// runnerTime translates a Grafana-domain timestamp into the runner domain by +// undoing poll p's measured skew. The single implementation for the package — +// coverage.go's window-membership and heartbeat-boundary checks use it too. +func runnerTime(p Poll, grafanaDomain time.Time) time.Time { + return grafanaDomain.Add(-p.Skew()) +} + +// classifyRule builds every instance timeline for one rule across +// [from, windowEnd] and reduces them to the rule's worst outcome, merged +// BadFor, and the Violations the preexisting policy charges against the run. +// PURE: no I/O, no clock reads; polls need not be pre-filtered to this rule. +func classifyRule(def Definition, polls []Poll, from, windowEnd time.Time, badStates map[State]bool, pol PreexistingPolicy) (Outcome, time.Duration, []Violation) { + rulePolls := pollsForRule(polls, def.UID) + inWindow := inWindowPolls(rulePolls, from, windowEnd) + + timelines := make(map[string]*instanceTimeline) + order := make([]string, 0) + + // get backfills labels on the first real Instance: a bare Cleared/Vanished + // marker can create the timeline first (with no labels), and a later re-fire + // must not report an empty InstanceLabels. + get := func(key string, labels map[string]string) *instanceTimeline { + tl, ok := timelines[key] + if !ok { + tl = &instanceTimeline{labels: labels} + timelines[key] = tl + order = append(order, key) + return tl + } + if tl.labels == nil && labels != nil { + tl.labels = labels + } + return tl + } + + openEpisode := func(tl *instanceTimeline, start time.Time) { + tl.badOpen = true + tl.episodeStart = start + } + closeEpisode := func(tl *instanceTimeline, end time.Time, real bool) { + // inWindowPolls widens its boundary outward by the skew bound, so a + // translated end can land past windowEnd or before episodeStart; clamp + // both, otherwise mergeDurations gets an inverted span. + if end.After(windowEnd) { + end = windowEnd + } + if end.Before(tl.episodeStart) { + end = tl.episodeStart + } + tl.episodes = append(tl.episodes, episode{start: tl.episodeStart, end: end, closedByRealClear: real}) + tl.badOpen = false + } + // onsetOf resolves a fresh episode's start: the instance's own ActiveAt, + // translated to the runner domain by this poll's skew, clamped to + // [from, windowEnd]. + onsetOf := func(p Poll, inst Instance) time.Time { + start := runnerTime(p, inst.ActiveAt) + if start.Before(from) { + start = from + } + if start.After(windowEnd) { + start = windowEnd + } + return start + } + + for _, p := range inWindow { + byKey := make(map[string]Instance, len(p.Abnormal)) + for _, inst := range p.Abnormal { + byKey[instanceKey(inst.Labels)] = inst + } + + for key, inst := range byKey { + tl := get(key, inst.Labels) + bad := badStates[inst.State] + switch { + case !tl.seen: + tl.seen = true + if bad { + // Fail-closed: "preexisting" only when even the worst-case + // skew error places the onset at or before `from`; an onset + // that might be in-window must classify as a new episode. + activeAtRunner := runnerTime(p, inst.ActiveAt) + tl.preexisting = !activeAtRunner.Add(p.SkewBound()).After(from) + if tl.preexisting { + openEpisode(tl, from) + } else { + openEpisode(tl, onsetOf(p, inst)) + } + } + case bad && !tl.badOpen: + openEpisode(tl, onsetOf(p, inst)) + case !bad && tl.badOpen: + closeEpisode(tl, runnerTime(p, p.GrafanaNow), true) + } + tl.lastState, tl.lastHealth, tl.lastError = inst.State, p.Health, p.LastError + } + + for _, key := range p.Cleared { + tl := get(key, nil) + if !tl.seen { + // Cleared on first mention: the transition happened pre-window, + // with no in-window evidence it was ever bad. + tl.seen = true + continue + } + if tl.badOpen { + closeEpisode(tl, runnerTime(p, p.GrafanaNow), true) + } + tl.lastHealth, tl.lastError = p.Health, p.LastError + } + + // Vanished is a deliberate no-op: freeze badOpen/preexisting as-is, so a + // vanish while bad stays bad (never reading as a recovery). + for _, key := range p.Vanished { + tl := get(key, nil) + tl.seen = true + tl.lastHealth = p.Health + } + } + + // Map iteration order is nondeterministic; sort so Violations/BadFor output + // is stable for a given input (like log.go sorts Cleared/Vanished). + slices.Sort(order) + + var ( + outcome Outcome = OutcomeClean + badFor []episode + viols []Violation + ) + + for _, key := range order { + tl := timelines[key] + if tl.badOpen { + closeEpisode(tl, windowEnd, false) + } + if len(tl.episodes) == 0 { + continue + } + + var instOutcome Outcome + switch { + case len(tl.episodes) > 1: + instOutcome = OutcomeFlapping + case tl.preexisting: + if tl.episodes[0].closedByRealClear { + instOutcome = OutcomeRecovered + } else { + instOutcome = OutcomePersistentlyBad + } + default: + // A genuinely new onset fails whether or not it clears in-window; + // only a preexisting condition earns `recovered`. + instOutcome = OutcomeNewlyBad + } + + if outcomeRank(instOutcome) > outcomeRank(outcome) { + outcome = instOutcome + } + badFor = append(badFor, tl.episodes...) + + if isViolation(instOutcome, pol) { + var clearedAt time.Time + last := tl.episodes[len(tl.episodes)-1] + if last.closedByRealClear { + clearedAt = last.end + } + viols = append(viols, Violation{ + Alert: def.Title, + RuleUID: def.UID, + Outcome: instOutcome, + State: tl.lastState, + Health: tl.lastHealth, + LastError: tl.lastError, + FirstSeen: tl.episodes[0].start, + ClearedAt: clearedAt, + InstanceLabels: tl.labels, + }) + } + } + + return outcome, mergeDurations(badFor), viols +} + +// isViolation decides whether one instance's outcome counts against the run, +// once the preexisting policy is applied. newly_bad and flapping always do: +// both contain a genuinely new bad episode, so no policy forgives them. +// recovered and persistently_bad are, by classifyRule's construction, +// ALWAYS preexisting (a non-preexisting single episode is newly_bad instead, +// regardless of whether it clears) — so these are the only two policy can +// change, and isViolation needs no separate preexisting flag to know that. +func isViolation(o Outcome, pol PreexistingPolicy) bool { + switch o { + case OutcomeNewlyBad, OutcomeFlapping: + return true + case OutcomePersistentlyBad: + return pol != PreexistingIgnore + case OutcomeRecovered: + return pol == PreexistingFail + default: + return false + } +} + +// outcomeRank orders outcomes for classifyRule's worst-of reduction across a +// rule's instances: +// +// unobservable > {flapping, persistently_bad, newly_bad} > recovered > +// skipped > clean +// +// with unobservable and skipped applied outside this function (decide owns +// both: unobservable from CoverageResult, skipped from the log header). The +// three fail values are not ranked against each other by anything that reads +// this, so their relative order here is an arbitrary but fixed tie-break, not +// a claim that one is worse than another. +func outcomeRank(o Outcome) int { + switch o { + case OutcomeFlapping: + return 4 + case OutcomePersistentlyBad: + return 3 + case OutcomeNewlyBad: + return 2 + case OutcomeRecovered: + return 1 + default: // OutcomeClean + return 0 + } +} + +// mergeDurations sums the wall-clock time covered by a set of episodes, +// merging overlaps so a rule with several simultaneously-bad instances is +// not reported as bad for longer than it actually was. +func mergeDurations(eps []episode) time.Duration { + if len(eps) == 0 { + return 0 + } + sorted := slices.Clone(eps) + slices.SortStableFunc(sorted, func(a, b episode) int { return a.start.Compare(b.start) }) + + var total time.Duration + cur := sorted[0] + for _, e := range sorted[1:] { + if e.start.After(cur.end) { + total += cur.end.Sub(cur.start) + cur = e + continue + } + if e.end.After(cur.end) { + cur.end = e.end + } + } + total += cur.end.Sub(cur.start) + return total +} + +// pollsForRule filters polls to one rule and sorts them by GrafanaNow, the +// same selection proveCoverage uses (by UID, never by title) — stable, because +// two polls sharing a coarse Date header must not reorder nondeterministically +// in a pure function. This is the single filter+sort implementation for the +// package: proveCoverage calls it too, rather than keeping its own copy that +// could silently drift from this one's membership test. +func pollsForRule(polls []Poll, uid string) []Poll { + var out []Poll + for _, p := range polls { + if p.RuleUID == uid { + out = append(out, p) + } + } + slices.SortStableFunc(out, func(a, b Poll) int { return a.GrafanaNow.Compare(b.GrafanaNow) }) + return out +} + +// badStateSet turns Policy.States into a lookup set, defaulting to {firing} +// when the caller leaves States empty — decide applies the default itself so a +// test can pass a zero-value Policy and get the real default, rather than +// depending on the CLI to have filled it in. +func badStateSet(states []State) map[State]bool { + if len(states) == 0 { + states = []State{StateFiring} + } + set := make(map[State]bool, len(states)) + for _, s := range states { + set[s] = true + } + return set +} + +// decide is the pure seam between the collected evidence and the CLI's exit +// code, and carries nearly the whole test suite because of it. It combines +// proveCoverage's nine checks with classifyRule's timelines under one Policy, +// and owns the inability-beats-violation rule: any unobservable rule makes +// decide return a non-nil error, which the CLI maps to exit 2 unconditionally +// — never to 0 or 1, and never suppressed by a real violation found alongside +// it. +// +// Result is fully populated even when the returned error is non-nil. A caller +// must not use Violations to second-guess the error, but Result stays useful +// for the human table on exit 2. +func decide(h Header, polls []Poll, sentinel *time.Time, defs []Definition, + rt map[string]ruleTimings, gt globalTimings, pol Policy) (Result, error) { + + badStates := badStateSet(pol.States) + + result := Result{ + From: pol.From, + To: pol.To, + GrafanaVersion: h.GrafanaVersion, + Coverage: make(map[string]CoverageResult), + Thresholds: make(map[string]RuleThresholds), + Global: GlobalThresholds{ + TransitionGrace: gt.transitionGrace, + GraceSource: graceSourceOrNone(gt.graceSource), + DrainTimeout: gt.drainTimeout, + }, + } + skewSeen := false + for _, p := range polls { + s := p.Skew() + if s < 0 { + s = -s + } + // The bound travels with its own poll's skew (see Result.ClockSkewBound), + // overwritten in lockstep. >= rather than > so a bound is still assigned + // when every poll's skew is exactly 0. + if !skewSeen || s > result.ClockSkew { + result.ClockSkew = s + result.ClockSkewBound = p.SkewBound() + skewSeen = true + } + } + + minObserved := pol.MinObserved + if minObserved == 0 { + minObserved = len(defs) + } + + windowEnd := pol.To.Add(gt.transitionGrace) + + var ( + skippedRules []Definition + watchedCount int + anyUnobservable bool + unobservableNames []string + ) + + // `skipped` is decided from the header, never from defs: defs are resolved + // after the window closed, so Definition.IsPaused describes the present, + // while Header.pausedAtStart describes the window open — the only moment + // "paused before the window opened" can mean. + pausedAtStart := h.pausedAtStart() + + for _, def := range defs { + if pausedAtStart[def.UID] { + skippedRules = append(skippedRules, def) + result.Verdicts = append(result.Verdicts, RuleVerdict{ + Alert: def.Title, RuleUID: def.UID, Outcome: OutcomeSkipped, + PollEvery: rt[def.UID].pollEvery, + Note: "paused before the window opened", + }) + continue + } + watchedCount++ + + t := rt[def.UID] + cov := proveCoverage(h, polls, sentinel, t, def, pol.From, pol.To, gt.transitionGrace) + + if pol.NodataIsUnobservable && !cov.Unobservable { + inWindow := inWindowPolls(pollsForRule(polls, def.UID), pol.From, windowEnd) + if runLen, sawAny := longestHealthRun(inWindow, "nodata"); sawAny && runLen > t.healthGrace { + cov.Unobservable = true + cov.Proved = false + if cov.Reason == "" { + cov.Reason = ReasonNodata + } + cov.Notes = append(cov.Notes, fmt.Sprintf( + "rule %q: health=nodata for %s exceeds healthGrace %s and --nodata-is-unobservable is set", + def.Title, runLen, t.healthGrace)) + } + } + result.Coverage[def.UID] = cov + result.Thresholds[def.UID] = RuleThresholds{ + MaxGap: t.maxGap, + HealthGrace: t.healthGrace, + EvalStaleAfter: t.evalStaleAfter, + } + + outcome, badFor, viols := classifyRule(def, polls, pol.From, windowEnd, badStates, pol.Preexisting) + if cov.Unobservable { + outcome = OutcomeUnobservable + anyUnobservable = true + unobservableNames = append(unobservableNames, fmt.Sprintf("%s (%s)", def.Title, cov.Reason)) + } + result.Violations = append(result.Violations, viols...) + result.Verdicts = append(result.Verdicts, RuleVerdict{ + Alert: def.Title, RuleUID: def.UID, Outcome: outcome, BadFor: badFor, + PollEvery: t.pollEvery, Note: strings.Join(cov.Notes, "; "), + }) + } + + // MinObserved defaults to len(defs) (post-collapse). A shortfall counts + // toward exit 1, never exit 2, and surfaces through Violations — so it + // always produces at least one, even when no rule is paused. + counted := watchedCount + var attributable []Definition + if pol.AllowPaused { + counted += len(skippedRules) + } else { + attributable = skippedRules + } + if shortfall := minObserved - counted; shortfall > 0 { + attributed := 0 + for _, def := range attributable { + if attributed >= shortfall { + break + } + // The paused rule and --allow-paused must both be named to the + // user; both live in this one Violation, in Note — the renderer + // prints Note verbatim rather than re-deriving the hint, so the + // exact wording here is what an operator reads. + result.Violations = append(result.Violations, Violation{ + Alert: def.Title, RuleUID: def.UID, Outcome: OutcomeSkipped, + Note: "paused before the window opened; counts against --min-observed unless --allow-paused is set", + }) + attributed++ + } + for ; attributed < shortfall; attributed++ { + // No named rule explains this part of the deficit — e.g. an + // operator-supplied --min-observed above what could ever be + // resolved. Note, not LastError: LastError is reporting-only + // rule state read from a real poll, and this Violation never + // touched one. + result.Violations = append(result.Violations, Violation{ + Outcome: OutcomeSkipped, + Note: fmt.Sprintf("min-observed %d exceeds the %d rule(s) counted as observed", minObserved, counted), + }) + } + } + + if anyUnobservable { + return result, fmt.Errorf("gate: %d rule(s) unobservable: %s", len(unobservableNames), strings.Join(unobservableNames, "; ")) + } + return result, nil +} diff --git a/grafana-alertcheck/internal/gate/classify_test.go b/grafana-alertcheck/internal/gate/classify_test.go new file mode 100644 index 000000000..ad25a6ea0 --- /dev/null +++ b/grafana-alertcheck/internal/gate/classify_test.go @@ -0,0 +1,992 @@ +package gate + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func lbl(name string) map[string]string { return map[string]string{"instance": name} } + +// abnormalPoll builds one Poll carrying a single abnormal instance, with the +// bookkeeping classifyRule needs (RuleUID, GrafanaNow, Health, Abnormal). +func abnormalPoll(uid string, at time.Time, state State, labels map[string]string, activeAt time.Time) Poll { + return Poll{ + RuleUID: uid, + GrafanaNow: at, + Found: true, + Health: "ok", + LastEvaluation: at, + Abnormal: []Instance{{Labels: labels, State: state, ActiveAt: activeAt}}, + } +} + +func clearedPoll(uid string, at time.Time, cleared ...string) Poll { + return Poll{RuleUID: uid, GrafanaNow: at, Found: true, Health: "ok", LastEvaluation: at, Cleared: cleared} +} + +func vanishedPoll(uid string, at time.Time, vanished ...string) Poll { + return Poll{RuleUID: uid, GrafanaNow: at, Found: true, Health: "ok", LastEvaluation: at, Vanished: vanished} +} + +func quietPoll(uid string, at time.Time) Poll { + return Poll{RuleUID: uid, GrafanaNow: at, Found: true, Health: "ok", LastEvaluation: at} +} + +var defaultBad = badStateSet(nil) // {firing} + +// pausedHeader builds the header decide reads `skipped` from: the pause state +// as of record start. Definition.IsPaused is deliberately NOT that authority +// — it comes from a ruler read taken after the window closed — so a test that +// wants a rule treated as skipped must say so HERE (Header.pausedAtStart). +func pausedHeader(startedAt time.Time, pausedUIDs ...string) Header { + h := Header{SchemaVersion: LogSchemaVersion, StartedAt: startedAt} + for _, uid := range pausedUIDs { + h.Rules = append(h.Rules, LoggedRule{UID: uid, IsPaused: true}) + } + return h +} + +// --- clean / newly_bad --- + +func TestClassifyRule_NoEvidenceIsClean(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{quietPoll("r1", from), quietPoll("r1", to)} + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeClean, outcome) + require.Zero(t, badFor) + require.Empty(t, viols) +} + +func TestClassifyRule_NewOnsetInsideWindowIsNewlyBad(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(5 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + quietPoll("r1", from), + abnormalPoll("r1", onset, StateFiring, lbl("a"), onset), + abnormalPoll("r1", to, StateFiring, lbl("a"), onset), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeNewlyBad, outcome) + require.Equal(t, to.Sub(onset), badFor) + require.Len(t, viols, 1) + require.Equal(t, OutcomeNewlyBad, viols[0].Outcome) +} + +// A genuinely new bad episode fails even if it clears again before the window +// ends — only a PREEXISTING condition earns the benefit of `recovered`. +func TestClassifyRule_NewOnsetThatClearsStillFails(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(2 * time.Minute) + clearAt := from.Add(3 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + quietPoll("r1", from), + abnormalPoll("r1", onset, StateFiring, lbl("a"), onset), + clearedPoll("r1", clearAt, instanceKey(lbl("a"))), + quietPoll("r1", to), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeNewlyBad, outcome, "even though it cleared") + require.Len(t, viols, 1) +} + +// --- recovered / persistently_bad (preexisting) --- + +func TestClassifyRule_PreexistingThatRecoversIsRecoveredAndNotAViolation(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + clearAt := from.Add(8 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + clearedPoll("r1", clearAt, key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeRecovered, outcome) + require.Equal(t, clearAt.Sub(from), badFor) + require.Empty(t, viols, "default policy passes a recovered preexisting instance") +} + +// The late condition: bad for 58 of a 60-minute window, clear at minute 58, +// still passes with a large BadFor — never a fail against some derived +// deadline (e.g. "must clear before 90% of the window"). +func TestClassifyRule_LateRecoveryPassesRegardlessOfHowLateItIs(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(60 * time.Minute) + clearAt := from.Add(58 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + clearedPoll("r1", clearAt, key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeRecovered, outcome, "even 58 minutes into a 60-minute window") + require.Equal(t, clearAt.Sub(from), badFor, "not a value clamped against a deadline") + require.Empty(t, viols, "there is no deadline a preexisting recovery must beat") +} + +func TestClassifyRule_PreexistingStillBadAtWindowEndIsPersistentlyBad(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + abnormalPoll("r1", to, StateFiring, lbl("a"), from.Add(-time.Hour)), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomePersistentlyBad, outcome) + require.Equal(t, to.Sub(from), badFor) + require.Len(t, viols, 1) + require.Equal(t, OutcomePersistentlyBad, viols[0].Outcome) +} + +// --- flapping --- + +func TestClassifyRule_ClearThenBadAgainIsFlapping(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from), + clearedPoll("r1", from.Add(2*time.Minute), key), + abnormalPoll("r1", from.Add(5*time.Minute), StateFiring, lbl("a"), from.Add(5*time.Minute)), + quietPoll("r1", to), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeFlapping, outcome) + require.Len(t, viols, 1) + require.Equal(t, OutcomeFlapping, viols[0].Outcome, "always a fail regardless of policy") +} + +// A clear and then a second bad state gives flapping, wherever the second bad +// state lands. A table over where the second onset falls — immediately after +// the clear, mid-window, and right at the +// last instant before windowEnd — closes the boundary this single fixed +// timing above cannot. +func TestClassifyRule_FlappingAtEveryTimingOfTheSecondOnset(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + clearAt := from.Add(2 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + tests := []struct { + name string + secondOnset time.Time + }{ + {"immediately after the clear", clearAt.Add(time.Second)}, + {"mid-window", from.Add(5 * time.Minute)}, + {"the last instant before windowEnd", to.Add(-time.Second)}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from), + clearedPoll("r1", clearAt, key), + abnormalPoll("r1", tc.secondOnset, StateFiring, lbl("a"), tc.secondOnset), + quietPoll("r1", to), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equalf(t, OutcomeFlapping, outcome, "second onset at %s", tc.secondOnset) + require.Len(t, viols, 1) + require.Equal(t, OutcomeFlapping, viols[0].Outcome) + }) + } +} + +// --- vanished is a discontinuity, never a clear --- + +func TestClassifyRule_VanishedWhileBadStaysPersistentlyBad(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Minute)), + vanishedPoll("r1", from.Add(5*time.Minute), key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomePersistentlyBad, outcome, "a vanish must never read as a recovery") + require.Equal(t, to.Sub(from), badFor, "the freeze must hold the episode open to windowEnd") + require.Len(t, viols, 1) +} + +func TestClassifyRule_VanishedWhileNeverBadIsUninteresting(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + // Pending is abnormal (non-normal) but not in the default {firing} bad + // set, so its vanish must stay uninteresting too. + polls := []Poll{ + abnormalPoll("r1", from, StatePending, lbl("a"), from), + vanishedPoll("r1", from.Add(5*time.Minute), key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeClean, outcome) + require.Zero(t, badFor) + require.Empty(t, viols) +} + +// --- preexisting policy --- + +func TestClassifyRule_PreexistingPolicyFailFailsARecoveredInstance(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + clearedPoll("r1", from.Add(2*time.Minute), key), + quietPoll("r1", to), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFail) + require.Equal(t, OutcomeRecovered, outcome, "the descriptive outcome does not change under policy=fail") + require.Len(t, viols, 1) + require.Equal(t, OutcomeRecovered, viols[0].Outcome, + "policy=fail gives no benefit of the doubt to a preexisting instance") +} + +func TestClassifyRule_PreexistingPolicyIgnoreForgivesPersistentlyBad(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + abnormalPoll("r1", to, StateFiring, lbl("a"), from.Add(-time.Hour)), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingIgnore) + require.Equal(t, OutcomePersistentlyBad, outcome, "the descriptive outcome does not change under policy=ignore") + require.Empty(t, viols, "policy=ignore disregards a preexisting instance even if it never recovers") +} + +func TestClassifyRule_PreexistingPolicyIgnoreStillFailsANewOnset(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(5 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + quietPoll("r1", from), + abnormalPoll("r1", onset, StateFiring, lbl("a"), onset), + abnormalPoll("r1", to, StateFiring, lbl("a"), onset), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingIgnore) + require.Equal(t, OutcomeNewlyBad, outcome) + require.Len(t, viols, 1, "ignore only forgives PREEXISTING badness") +} + +// --- worst-of across instances --- + +func TestClassifyRule_WorstOfMultipleInstancesWins(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + { + RuleUID: "r1", GrafanaNow: from, Found: true, Health: "ok", + Abnormal: []Instance{ + {Labels: lbl("a"), State: StateFiring, ActiveAt: from.Add(-time.Hour)}, // preexisting, will recover + {Labels: lbl("b"), State: StateFiring, ActiveAt: from}, // preexisting, will stay bad + }, + }, + clearedPoll("r1", from.Add(2*time.Minute), instanceKey(lbl("a"))), + { + RuleUID: "r1", GrafanaNow: to, Found: true, Health: "ok", + Abnormal: []Instance{{Labels: lbl("b"), State: StateFiring, ActiveAt: from}}, + }, + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomePersistentlyBad, outcome, "the worse of {recovered, persistently_bad}") + require.Len(t, viols, 1) + require.Equal(t, OutcomePersistentlyBad, viols[0].Outcome) +} + +// --- decide(): skipped rules, unobservable, MinObserved, exit mapping --- + +func TestDecide_SkippedRuleNeverReachesProveCoverage(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1", IsPaused: true} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to, AllowPaused: true} + + // No polls, no sentinel at all: a heartbeat_gap/no_sentinel misclassification + // here would mean proveCoverage ran for a skipped rule. + // The HEADER is what says paused — decide reads skipped from there, not + // from def.IsPaused, which is a post-window reading (Header.pausedAtStart). + res, err := decide(pausedHeader(from.Add(-time.Hour), "r1"), nil, nil, defs, rt, gt, pol) + require.NoError(t, err, "a rule paused before the window is skipped, not unobservable") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeSkipped, res.Verdicts[0].Outcome) + _, ok := res.Coverage["r1"] + require.False(t, ok, "a skipped rule has no coverage to prove") +} + +func TestDecide_UnobservableRuleAlwaysReturnsAnError(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to} + + // No sentinel at all: check 1 fails, so the rule is unobservable + // regardless of anything else. + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, nil, nil, defs, rt, gt, pol) + require.Error(t, err, "an unobservable rule must always fail the run") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome) +} + +// Any unobservable rule means exit 2, with no exception — even alongside a +// real newly_bad. +func TestDecide_UnobservableWinsEvenAlongsideARealViolation(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(5 * time.Minute) + + defBroken := Definition{UID: "broken", Title: "Broken"} + defBad := Definition{UID: "bad", Title: "Bad"} + defs := []Definition{defBroken, defBad} + rt := map[string]ruleTimings{ + "broken": newRuleTimings(30*time.Second, 60), + "bad": newRuleTimings(30*time.Second, 60), + } + gt := globalTimings{} + pol := Policy{From: from, To: to} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + if ts.Equal(onset) || ts.After(onset) { + polls = append(polls, abnormalPoll("bad", ts, StateFiring, lbl("a"), onset)) + } else { + polls = append(polls, quietPoll("bad", ts)) + } + } + // "broken" gets no polls at all: no sentinel, no heartbeats -> unobservable. + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + require.Error(t, err, "one rule is unobservable") + var gotBroken, gotBad Outcome + for _, v := range res.Verdicts { + switch v.RuleUID { + case "broken": + gotBroken = v.Outcome + case "bad": + gotBad = v.Outcome + } + } + require.Equal(t, OutcomeUnobservable, gotBroken) + require.Equal(t, OutcomeNewlyBad, gotBad, + "classification still runs and is still visible in Verdicts") + require.NotEmpty(t, res.Violations, + "the newly_bad instance still reported even though the run fails on the unobservable rule") +} + +// A clean verdict with a coverage gap must never give exit 0, and recovered +// and skipped verdicts need proved coverage of the full window just as much. +// One genuinely unobservable rule ("broken", zero polls) alongside a rule with +// each of the three favorable outcomes — none of them may waive the run. +func TestDecide_UnobservableRuleWinsOverEveryFavorableOutcome(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + + tests := []struct { + name string + goodPolls []Poll + pausedAtStart bool + wantOutcome Outcome + }{ + { + name: "clean", + goodPolls: func() []Poll { + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("good", ts)) + } + return polls + }(), + wantOutcome: OutcomeClean, + }, + { + // Dense 30s-spaced polls throughout, so "good"'s own coverage + // proves clean on its own — a sparse abnormal/cleared/quiet + // triple (enough for classifyRule alone) would leave a + // heartbeat gap that muddies which rule made the run fail. + name: "recovered", + goodPolls: func() []Poll { + var polls []Poll + clearAt := from.Add(3 * time.Minute) + key := instanceKey(lbl("a")) + cleared := false + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + switch { + case ts.Equal(clearAt): + polls = append(polls, clearedPoll("good", ts, key)) + cleared = true + case !cleared: + polls = append(polls, abnormalPoll("good", ts, StateFiring, lbl("a"), from.Add(-time.Hour))) + default: + polls = append(polls, quietPoll("good", ts)) + } + } + return polls + }(), + wantOutcome: OutcomeRecovered, + }, + { + name: "skipped", + pausedAtStart: true, + wantOutcome: OutcomeSkipped, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + defs := []Definition{{UID: "good", Title: "Good"}, {UID: "broken", Title: "Broken"}} + rt := map[string]ruleTimings{ + "good": newRuleTimings(30*time.Second, 60), + "broken": newRuleTimings(30*time.Second, 60), + } + gt := globalTimings{} + pol := Policy{From: from, To: to} + + h := Header{StartedAt: from.Add(-time.Hour)} + if tc.pausedAtStart { + h.Rules = []LoggedRule{{UID: "good", IsPaused: true}} + } + + // "broken" gets no polls at all: no sentinel-worthy heartbeats, + // so it is unobservable regardless of "good". + sentinel := to + res, err := decide(h, tc.goodPolls, &sentinel, defs, rt, gt, pol) + require.Errorf(t, err, "'broken' is unobservable regardless of 'good' being %s", tc.name) + var gotGood, gotBroken Outcome + for _, v := range res.Verdicts { + switch v.RuleUID { + case "good": + gotGood = v.Outcome + case "broken": + gotBroken = v.Outcome + } + } + require.Equal(t, tc.wantOutcome, gotGood) + require.Equal(t, OutcomeUnobservable, gotBroken) + }) + } +} + +// The table above puts the coverage gap on a DIFFERENT rule from the one with +// the favorable outcome. This pins the tighter claim: a rule that +// itself recovers, but ALSO itself has a coverage gap, is still overridden to +// unobservable — the favorable classification of a rule is never a reason to +// skip that same rule's own coverage check. +func TestDecide_RecoveredOutcomeOverriddenByItsOwnCoverageGap(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} // maxGap = 60s + gt := globalTimings{} + pol := Policy{From: from, To: to} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + clearedPoll("r1", from.Add(30*time.Second), key), + } + for ts := from.Add(time.Minute); !ts.After(to); ts = ts.Add(30 * time.Second) { + // A gap from from+1.5m to from+4m — well past the 60s maxGap — + // sitting entirely AFTER the clear, so classifyRule alone would + // still call this rule `recovered`. + if ts.After(from.Add(90*time.Second)) && ts.Before(from.Add(4*time.Minute)) { + continue + } + polls = append(polls, quietPoll("r1", ts)) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + require.Error(t, err, "r1's own coverage gap must fail the run even though it recovered") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome, "never recovered") + require.False(t, res.Coverage["r1"].Proved) +} + +func TestDecide_CleanWindowIsAPass(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("r1", ts)) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + require.NoError(t, err) + require.Empty(t, res.Violations, "a pass is exactly len(Violations)==0 && err==nil") + require.Equal(t, OutcomeClean, res.Verdicts[0].Outcome) +} + +// A pause and then an unpause inside the window, with an episode that would +// fire and resolve entirely inside the blind interval. A drain wait alone — +// "did the rule eventually evaluate through windowEnd?" — would see +// lastEvaluation catch up after the unpause and answer yes, a pass. decide() +// never runs a drain wait (that is check.go's I/O concern); this pins that +// proveCoverage's own per-poll checks already refuse the window without one, +// so a live drain wait is not what is saving this case. +func TestDecide_PauseThenUnpauseWithHiddenEpisodeGivesUnobservableNotClean(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(20 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to} + + pauseStart := from.Add(5 * time.Minute) + pauseEnd := from.Add(10 * time.Minute) + + var polls []Poll + for ts := from; !ts.After(pauseStart.Add(-30 * time.Second)); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("r1", ts)) + } + for ts := pauseStart; !ts.After(pauseEnd); ts = ts.Add(30 * time.Second) { + // No fire/resolve is ever observed here: the rule was not + // evaluating, so any real episode inside this stretch is invisible + // to every poll. + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", IsPaused: true, LastEvaluation: pauseStart}) + } + for ts := pauseEnd.Add(30 * time.Second); !ts.After(to); ts = ts.Add(30 * time.Second) { + // Evaluations resume and catch straight up — a drain wait's final + // "did it reach windowEnd" question would answer yes. + polls = append(polls, quietPoll("r1", ts)) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + require.Error(t, err, "the pause-then-unpause blind interval must fail closed") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome, "never clean") + require.False(t, res.Coverage["r1"].Proved) +} + +// --- MinObserved shortfall --- + +func TestDecide_SkippedOnlyShortfallProducesAViolationWithoutAnError(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + + watched := Definition{UID: "watched", Title: "Watched"} + paused := Definition{UID: "paused", Title: "Paused", IsPaused: true} + defs := []Definition{watched, paused} + rt := map[string]ruleTimings{ + "watched": newRuleTimings(30*time.Second, 60), + "paused": newRuleTimings(30*time.Second, 60), + } + gt := globalTimings{} + // MinObserved defaults to len(defs) = 2, but only "watched" is observable. + pol := Policy{From: from, To: to} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("watched", ts)) + } + sentinel := to + + res, err := decide(pausedHeader(from.Add(-time.Hour), "paused"), polls, &sentinel, defs, rt, gt, pol) + require.NoError(t, err, "a shortfall caused only by a skipped rule is exit 1, not exit 2") + require.Len(t, res.Violations, 1) + v := res.Violations[0] + require.Equal(t, OutcomeSkipped, v.Outcome) + require.Equal(t, "paused", v.RuleUID) + require.Equal(t, "Paused", v.Alert) + require.NotEmpty(t, v.Note, + "the shortfall reason must not be smuggled into LastError") +} + +// An operator-supplied MinObserved that exceeds what could ever be resolved is +// still a shortfall, even with zero paused rules to blame it on — it must not +// silently read as a pass. +func TestDecide_ExplicitMinObservedShortfallWithNoPausedRuleStillProducesAViolation(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to, MinObserved: 3} // only one rule will ever be resolved + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("r1", ts)) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + require.NoError(t, err, "an unmet MinObserved is exit 1, never exit 2") + require.Len(t, res.Violations, 2, "the shortfall (3-1=2) must surface directly rather than pass silently") + for _, v := range res.Violations { + require.Equal(t, OutcomeSkipped, v.Outcome) + } +} + +func TestDecide_AllowPausedSuppressesTheShortfall(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + + watched := Definition{UID: "watched", Title: "Watched"} + paused := Definition{UID: "paused", Title: "Paused", IsPaused: true} + defs := []Definition{watched, paused} + rt := map[string]ruleTimings{ + "watched": newRuleTimings(30*time.Second, 60), + "paused": newRuleTimings(30*time.Second, 60), + } + gt := globalTimings{} + pol := Policy{From: from, To: to, AllowPaused: true} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("watched", ts)) + } + sentinel := to + + res, err := decide(pausedHeader(from.Add(-time.Hour), "paused"), polls, &sentinel, defs, rt, gt, pol) + require.NoError(t, err) + require.Empty(t, res.Violations, "--allow-paused must suppress the shortfall entirely") +} + +// --- nodata escalation (decide's own Policy-driven check) --- + +func TestDecide_NodataIsUnobservableEscalatesASustainedRun(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} // healthGrace = max(60s,60s) = 60s + gt := globalTimings{} + pol := Policy{From: from, To: to, NodataIsUnobservable: true} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "nodata", LastEvaluation: ts}) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + require.Error(t, err, "a sustained nodata run must be unobservable under --nodata-is-unobservable") + require.Equal(t, ReasonNodata, res.Coverage["r1"].Reason) +} + +func TestDecide_NodataIsANoteByDefault(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to} // NodataIsUnobservable defaults to false + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "nodata", LastEvaluation: ts}) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + require.NoError(t, err, "96%% of the fleet runs no_data_state:OK and must not fail by default") + require.False(t, res.Coverage["r1"].Unobservable) +} + +// --- preexisting is decided by ActiveAt, not poll timing --- + +// An instance whose true onset (ActiveAt) falls strictly inside the window — +// even though the first poll that happens to observe it already shows it bad — +// must never be treated as preexisting. If it then clears, that is newly_bad +// (exit 1), not recovered (exit 0). +func TestClassifyRule_OnsetBetweenFromAndFirstPollIsNewlyBadNotRecovered(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(1 * time.Minute) // the true onset, strictly after `from` + firstPoll := from.Add(2 * time.Minute) // the first poll that happens to observe it + clearAt := from.Add(5 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", firstPoll, StateFiring, lbl("a"), onset), + clearedPoll("r1", clearAt, key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeNewlyBad, outcome, + "the onset is after `from`, so it is not preexisting even though the FIRST in-window poll already observes it bad") + require.Len(t, viols, 1) + require.Equal(t, OutcomeNewlyBad, viols[0].Outcome) + require.Equal(t, clearAt.Sub(onset), badFor, "BadFor must count from the true onset, not from `from`") +} + +// TestClassifyRule_OnsetJustBeforeFromIsPreexisting is the mirror check: an +// onset at or before `from` (even if the first poll is later) is genuinely +// preexisting and, if it clears, is `recovered`. +func TestClassifyRule_OnsetJustBeforeFromIsPreexisting(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(-time.Minute) + firstPoll := from.Add(2 * time.Minute) + clearAt := from.Add(5 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", firstPoll, StateFiring, lbl("a"), onset), + clearedPoll("r1", clearAt, key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeRecovered, outcome, "the onset is at/before `from`, genuinely preexisting") + require.Empty(t, viols, "default policy passes a recovered preexisting instance") + require.Equal(t, clearAt.Sub(from), badFor, + "a preexisting episode's BadFor is clamped to window-open, not backdated past it") +} + +// A poll carrying a nonzero skew must have its ActiveAt (and GrafanaNow) +// translated to the runner domain before comparing against `from` — a raw, +// untranslated comparison would land on the wrong side of that boundary. +func TestClassifyRule_SkewTranslatesActiveAtAcrossTheWindowBoundary(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + // Grafana's clock reads 90s ahead of the runner's (skew = +90s). The + // poll's raw GrafanaNow/ActiveAt both sit 90s past `from` in Grafana's + // domain, but translate to exactly `from` in the runner domain — genuinely + // preexisting once translated, and wrongly "newly_bad" if the skew is + // ignored. + skew := 90 * time.Second + rawActiveAt := from.Add(skew) + poll := Poll{ + RuleUID: "r1", GrafanaNow: from.Add(skew), Found: true, Health: "ok", + LastEvaluation: from.Add(skew), SkewMS: skew.Milliseconds(), + Abnormal: []Instance{{Labels: lbl("a"), State: StateFiring, ActiveAt: rawActiveAt}}, + } + stillBad := poll + stillBad.GrafanaNow = to.Add(skew) + stillBad.LastEvaluation = to.Add(skew) + + outcome, badFor, _ := classifyRule(def, []Poll{poll, stillBad}, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomePersistentlyBad, outcome, + "a +90s skew must translate ActiveAt back to exactly `from`") + require.Equal(t, to.Sub(from), badFor) +} + +// --- InstanceLabels must survive a timeline first created by a bare marker --- + +func TestClassifyRule_LabelsSurviveWhenTimelineStartsFromAClearedMarker(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + newOnset := from.Add(5 * time.Minute) + + polls := []Poll{ + // The very first mention of this key is a bare Cleared marker (its + // prior bad episode, if any, started before the window) — no labels + // travel with a Cleared/Vanished event. + clearedPoll("r1", from.Add(1*time.Minute), key), + abnormalPoll("r1", newOnset, StateFiring, lbl("a"), newOnset), + quietPoll("r1", to), + } + _, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Len(t, viols, 1) + require.NotNil(t, viols[0].InstanceLabels) + require.Equal(t, "a", viols[0].InstanceLabels["instance"], + "labels must backfill even though the timeline was first created by a label-less Cleared marker") +} + +// FirstSeen/ClearedAt are pinned exactly, not just that a violation exists. +func TestClassifyRule_ViolationFieldsArePrecise(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(2 * time.Minute) + clearAt := from.Add(3 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + quietPoll("r1", from), + abnormalPoll("r1", onset, StateFiring, lbl("a"), onset), + clearedPoll("r1", clearAt, instanceKey(lbl("a"))), + quietPoll("r1", to), + } + _, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Len(t, viols, 1) + v := viols[0] + require.True(t, v.FirstSeen.Equal(onset)) + require.True(t, v.ClearedAt.Equal(clearAt)) + require.Equal(t, "a", v.InstanceLabels["instance"]) +} + +// The episode.end clamp: inWindowPolls admits a poll up to its own skew bound +// past windowEnd, so a genuine Cleared event on such a poll must not leave the +// episode extending beyond windowEnd. +func TestClassifyRule_ClearedEventPastWindowEndClampsToWindowEnd(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + bound := 30 * time.Second + clearedAt := to.Add(20 * time.Second) // past windowEnd, but within the skew bound + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + { + RuleUID: "r1", GrafanaNow: clearedAt, Found: true, Health: "ok", + SkewBoundMS: bound.Milliseconds(), Cleared: []string{key}, + }, + } + outcome, badFor, _ := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeRecovered, outcome) + require.Equal(t, to.Sub(from), badFor, + "the episode end must clamp to windowEnd, not extend to the late Cleared event's raw time") +} + +// TestClassifyRule_OnsetJustPastWindowEndIsNewlyBadNotClean pins the fail-closed +// reading of the upper boundary: an instance whose runner-domain onset lands +// only slightly past windowEnd (to + transitionGrace) is reachable at all only +// because inWindowPolls widens the boundary outward by the skew bound, so the +// gate cannot PROVE it belongs to the next window. It is charged as newly_bad — +// with BadFor truncated to zero — rather than silently forgiven as clean. +func TestClassifyRule_OnsetJustPastWindowEndIsNewlyBadNotClean(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + grace := time.Minute + windowEnd := to.Add(grace) + def := Definition{UID: "r1", Title: "R1"} + + // A poll admitted only by its own skew bound: its GrafanaNow sits 20s past + // windowEnd, inside the 30s tolerance. It carries an instance whose onset + // is 10s past windowEnd — still "after the grace", but only by less than + // the measurement's own uncertainty. + bound := 30 * time.Second + poll := Poll{ + RuleUID: "r1", GrafanaNow: windowEnd.Add(20 * time.Second), Found: true, Health: "ok", + LastEvaluation: windowEnd.Add(20 * time.Second), SkewBoundMS: bound.Milliseconds(), + Abnormal: []Instance{{Labels: lbl("a"), State: StateFiring, ActiveAt: windowEnd.Add(10 * time.Second)}}, + } + + outcome, badFor, viols := classifyRule(def, []Poll{poll}, from, windowEnd, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeNewlyBad, outcome, + "an onset past windowEnd seen only via the skew bound must fail closed") + require.Zero(t, badFor, "the zero-length episode must truncate to the window end") + require.Len(t, viols, 1) +} + +// A clear after `to` gives persistently_bad. classifyRule filters +// its input to [from, windowEnd] itself (inWindowPolls), so a Cleared event +// GENUINELY past windowEnd — well beyond any skew bound, unlike the clamp +// case above — never reaches the timeline at all: the instance is still bad +// at windowEnd as far as this window is concerned. +func TestClassifyRule_ClearAfterWindowEndIsPersistentlyBad(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + clearedPoll("r1", to.Add(time.Hour), key), // far past `to`, not a boundary case + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomePersistentlyBad, outcome, "a clear outside the window must not read as a recovery") + require.Equal(t, to.Sub(from), badFor) + require.Len(t, viols, 1) + require.Equal(t, OutcomePersistentlyBad, viols[0].Outcome) +} + +// TestClassifyRule_CloseBeforeOpenClampsToZeroNotNegative pins the +// end-before-start clamp: two polls with different measured skews can +// translate so that a closing poll's runner-domain time lands before the +// opening poll's, which — unclamped — would feed mergeDurations a negative +// span. +func TestClassifyRule_CloseBeforeOpenClampsToZeroNotNegative(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onsetPoll := from.Add(5 * time.Minute) + closePoll := from.Add(6 * time.Minute) + closeSkew := 2 * time.Minute // translates closePoll back to from+4min, before onsetPoll's from+5min + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + quietPoll("r1", from), + abnormalPoll("r1", onsetPoll, StateFiring, lbl("a"), onsetPoll), // skew 0 + { + RuleUID: "r1", GrafanaNow: closePoll, Found: true, Health: "ok", + SkewMS: closeSkew.Milliseconds(), Cleared: []string{key}, + }, + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + require.Equal(t, OutcomeNewlyBad, outcome) + require.GreaterOrEqual(t, badFor, time.Duration(0), + "a non-negative duration even though the closing poll's translated time landed before the opening poll's") + require.Zero(t, badFor, "the clamp collapses the inverted span to a zero-length episode") + require.Len(t, viols, 1) +} + +// --- mergeDurations --- + +func TestMergeDurations_OverlappingEpisodesCountOnce(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + eps := []episode{ + {start: from, end: from.Add(5 * time.Minute)}, + {start: from.Add(2 * time.Minute), end: from.Add(8 * time.Minute)}, // overlaps the first + {start: from.Add(20 * time.Minute), end: from.Add(21 * time.Minute)}, // disjoint + } + got := mergeDurations(eps) + want := 8*time.Minute + 1*time.Minute // [0,8) merged = 8m, plus the disjoint 1m + require.Equal(t, want, got, "two simultaneously-bad instances must not double-count their overlap") +} + +func TestMergeDurations_Empty(t *testing.T) { + require.Zero(t, mergeDurations(nil)) +} diff --git a/grafana-alertcheck/internal/gate/coverage.go b/grafana-alertcheck/internal/gate/coverage.go new file mode 100644 index 000000000..603a7f7d8 --- /dev/null +++ b/grafana-alertcheck/internal/gate/coverage.go @@ -0,0 +1,314 @@ +package gate + +import ( + "fmt" + "time" +) + +// keepLastReason is the instance Reason that check 9 watches for. +const keepLastReason = "KeepLast" + +// Two things this file leaves to its callers: the "from too far ahead" bound is +// Config.validate's once-per-run input validation (check 2 owns only the +// "from < StartedAt" half), and a rule paused at the window open never reaches +// proveCoverage — decide reads `skipped` from Header.pausedAtStart first, so a +// paused rule's zero polls read as skipped, not as one large heartbeat gap. + +// UnobservableReason names why proveCoverage could not prove a rule's window. +// It reaches the JSON output, so it is a published vocabulary like Outcome; +// prose belongs in Notes. +type UnobservableReason string + +const ( + ReasonNoSentinel UnobservableReason = "no_sentinel" + ReasonSentinelEarly UnobservableReason = "sentinel_early" + ReasonFromBeforeRecord UnobservableReason = "from_before_record" + ReasonHeartbeatGap UnobservableReason = "heartbeat_gap" + ReasonHealthError UnobservableReason = "health_error" + ReasonStaleEvaluation UnobservableReason = "stale_evaluation" + ReasonFutureEvaluation UnobservableReason = "future_evaluation" + ReasonPausedInWindow UnobservableReason = "paused_in_window" + ReasonRuleAbsent UnobservableReason = "rule_absent" + // ReasonDrainTimeout is set by check.go's drain wait, never by + // proveCoverage: the wait is I/O and must not be added to this pure + // function — that would put HTTP inside the pure layer and destroy the + // seam this design depends on. + ReasonDrainTimeout UnobservableReason = "drain_timeout" +) + +// CoverageResult is proveCoverage's whole answer for one rule. No interval +// list: proved-or-not plus the largest gap and where is everything a human +// reads on exit 2, and everything the rendered table needs. +type CoverageResult struct { + Proved bool + LargestGap time.Duration + LargestGapAt time.Time + Unobservable bool + Reason UnobservableReason + Notes []string + // BlindFor is the worst staleness (GrafanaNow - LastEvaluation) that + // tripped check 6; zero when check 6 never fired. + BlindFor time.Duration +} + +// proveCoverage applies the nine coverage checks to one rule's polls. PURE: no +// HTTP, no files, no clock reads — everything arrives as an argument. polls need +// not be pre-filtered to this rule (selection is by def.UID). Every check runs +// even after Unobservable is set, so LargestGap and the notes are complete on +// exit 2; Reason names only the FIRST check that failed. +func proveCoverage(h Header, polls []Poll, sentinel *time.Time, t ruleTimings, def Definition, + from, to time.Time, grace time.Duration) CoverageResult { + + windowEnd := to.Add(grace) + + // pollsForRule (classify.go) is the single filter+sort implementation; this + // and classifyRule must not carry two independent copies. + rulePolls := pollsForRule(polls, def.UID) + + var res CoverageResult + fail := func(reason UnobservableReason, note string) { + res.Unobservable = true + if res.Reason == "" { + res.Reason = reason + } + res.Notes = append(res.Notes, fmt.Sprintf("rule %q: %s", def.Title, note)) + } + + // Check 1 — sentinel. Present and At >= to+grace -> coverage provable; + // absent, or short of it, is never a pass. A recorder that died early must + // look exactly like a coverage gap, because it is one. + switch { + case sentinel == nil: + fail(ReasonNoSentinel, "no stopped sentinel: the recorder never reported finishing") + case sentinel.Before(windowEnd): + fail(ReasonSentinelEarly, fmt.Sprintf("stopped sentinel at %s is before the required %s (to+grace)", + sentinel.Format(time.RFC3339), windowEnd.Format(time.RFC3339))) + } + + // Check 2 — from bounds: from < StartedAt makes coverage unprovable, no + // matter how healthy the polls that DO exist look. Both are runner-domain + // clock reads (the recorder's own Clock.Now()), so no cross-domain + // translation applies here. The comparison is at whole-second granularity: + // `from` is supplied at second precision (--from RFC3339) while StartedAt + // carries the recorder's sub-second clock stamp, so an operator naming the + // exact second the recording opened must not be judged early for the + // sub-second sliver inside that same second. The other half of the bound — + // from too far ahead of the runner's clock — is Check's input validation, + // once per run rather than per rule. + if from.Truncate(time.Second).Before(h.StartedAt.Truncate(time.Second)) { + fail(ReasonFromBeforeRecord, fmt.Sprintf( + "requested from %s is before recording started at %s", from.Format(time.RFC3339Nano), h.StartedAt.Format(time.RFC3339Nano))) + } + + // Filtered once and threaded through every remaining check. + inWindow := inWindowPolls(rulePolls, from, windowEnd) + + // Check 3 — heartbeat continuity. Data at both ends with a hole in between + // is not enough: this scans every gap inside the window, not just its + // edges. + res.LargestGap, res.LargestGapAt = ruleHeartbeatGap(inWindow, from, windowEnd) + if res.LargestGap > t.maxGap { + fail(ReasonHeartbeatGap, fmt.Sprintf( + "gap of %s starting at %s exceeds maxGap %s", res.LargestGap, res.LargestGapAt.Format(time.RFC3339), t.maxGap)) + } + + // Check 4 — health=="error". A short blip is a note only — one failed + // evaluation must not exit 2 over an otherwise clean window; only a run + // longer than healthGrace consumes coverage. + if runLen, sawAny := longestHealthRun(inWindow, "error"); sawAny { + res.Notes = append(res.Notes, fmt.Sprintf("rule %q: health=error observed (longest run %s)", def.Title, runLen)) + if runLen > t.healthGrace { + fail(ReasonHealthError, fmt.Sprintf("health=error for %s exceeds healthGrace %s", runLen, t.healthGrace)) + } + } + + // Check 5 — health=="nodata". Never fatal here (most of the fleet runs + // no_data_state:OK, so it would block healthy idle deploys). Escalating + // under Policy.NodataIsUnobservable is decide's job, since this pure + // function has no Policy to consult. + if _, sawAny := longestHealthRun(inWindow, "nodata"); sawAny { + res.Notes = append(res.Notes, fmt.Sprintf("rule %q: health=nodata observed (not fatal; see --nodata-is-unobservable)", def.Title)) + } + + // Check 6 — liveness. Same-domain (GrafanaNow and LastEvaluation are from + // the SAME response), so raw values — never a delta against a previous + // poll, which reports stale ~half the polls of a healthy rule. Skipped only + // for IsPaused (check 7) or !Found (check 8); a zero LastEvaluation on a + // found, unpaused poll is treated as maximally stale, not waved through. + var staleCount int + var worstStale time.Duration + var worstStaleAt time.Time + for _, p := range inWindow { + if p.IsPaused || !p.Found { + continue + } + // lastEvaluation in the future of its own poll's grafana_now is + // corrupted or hand-edited data (ReadLog does no field validation); + // GrafanaNow-LastEvaluation would go negative and silently read as + // fresh — fail-open. Treat it as unobservable instead. + if p.LastEvaluation.Truncate(time.Second).After(p.GrafanaNow) { + fail(ReasonFutureEvaluation, fmt.Sprintf( + "lastEvaluation %s is after grafana_now %s (corrupted poll)", + p.LastEvaluation.Format(time.RFC3339), p.GrafanaNow.Format(time.RFC3339))) + continue + } + if stale := p.GrafanaNow.Sub(p.LastEvaluation); stale > t.evalStaleAfter { + staleCount++ + if stale > worstStale { + worstStale, worstStaleAt = stale, p.GrafanaNow + } + } + } + if staleCount > 0 { + res.BlindFor = worstStale + fail(ReasonStaleEvaluation, fmt.Sprintf( + "lastEvaluation stale on %d poll(s); worst %s (> evalStaleAfter %s) as of grafana_now %s", + staleCount, worstStale, t.evalStaleAfter, worstStaleAt.Format(time.RFC3339))) + } + + // Check 7 — isPaused in-window. The PRIMARY pause detector: liveness + // (check 6) is only the backup for what IsPaused cannot show (a deleted + // rule, a stopped scheduler, a blocked evaluation). This is what catches + // pause-then-unpause, which the drain wait alone passes. + var pausedCount int + var pausedAt time.Time + for _, p := range inWindow { + if p.IsPaused { + pausedCount++ + if pausedAt.IsZero() { + pausedAt = p.GrafanaNow + } + } + } + if pausedCount > 0 { + fail(ReasonPausedInWindow, fmt.Sprintf("observed paused on %d poll(s), first at %s", pausedCount, pausedAt.Format(time.RFC3339))) + } + + // Check 8 — rule absent. Found==false is authoritative (the transport + // already retried every transient failure before a Poll record ever + // exists): the rule resolved at resolve time but the state endpoint + // stopped serving it. Never drop a watched rule from the verdict set + // silently. + var absentCount int + var absentAt time.Time + for _, p := range inWindow { + if !p.Found { + absentCount++ + if absentAt.IsZero() { + absentAt = p.GrafanaNow + } + } + } + if absentCount > 0 { + fail(ReasonRuleAbsent, fmt.Sprintf("state endpoint returned no rule on %d poll(s), first at %s", absentCount, absentAt.Format(time.RFC3339))) + } + + // Check 9 — KeepLast. Two non-fatal notes: DECLARED (the rule is configured + // with no_data_state/exec_err_state=KeepLast, read from def so it fires once), + // and OBSERVED (an instance reported KeepLast in-window; Reasons keys can be + // comma-joined, so membership via reasonsContain, never a literal index). + nds, ees := def.NoDataState, def.ExecErrState + for _, lr := range h.Rules { + if lr.UID == def.UID { + nds, ees = lr.NoDataState, lr.ExecErrState + break + } + } + if nds == keepLastReason || ees == keepLastReason { + res.Notes = append(res.Notes, fmt.Sprintf( + "rule %q: configured with no_data_state/exec_err_state=KeepLast — a stale state can continue past a real fault", def.Title)) + } + for _, p := range inWindow { + if reasonsContain(p.Reasons, keepLastReason) { + res.Notes = append(res.Notes, fmt.Sprintf( + "rule %q: KeepLast observed at %s: a held-over state may hide a real blind spot", def.Title, p.GrafanaNow.Format(time.RFC3339))) + break + } + } + + res.Proved = !res.Unobservable + return res +} + +// inWindowPolls filters to polls inside [from, windowEnd] via the cross-domain +// membership test: each GrafanaNow is translated to the runner domain by its +// own skew, widened by its skew bound, so clock imprecision never excludes a +// genuinely in-window poll. Everything downstream reads same-domain raw fields; +// only this filter and check 3's boundary segments cross domains. +func inWindowPolls(polls []Poll, from, windowEnd time.Time) []Poll { + var out []Poll + for _, p := range polls { + bound := p.SkewBound() + runner := runnerTime(p, p.GrafanaNow) + if runner.Before(from.Add(-bound)) || runner.After(windowEnd.Add(bound)) { + continue + } + out = append(out, p) + } + return out +} + +// ruleHeartbeatGap finds the largest unobserved span inside [from, windowEnd], +// including the two boundary segments — which is why data at both ends with a +// hole in the middle still fails. in must be filtered (inWindowPolls) and +// sorted by GrafanaNow. +// +// Boundary segments are cross-domain, so each translated poll time is widened +// by its skew bound on the side that makes the gap LARGER (never smaller — an +// uncertain boundary must read as at least as big a gap as it might be). +// Consecutive-poll spacing is same-domain and uses the raw GrafanaNow diff. +func ruleHeartbeatGap(in []Poll, from, windowEnd time.Time) (largestGap time.Duration, largestGapAt time.Time) { + if len(in) == 0 { + return windowEnd.Sub(from), from + } + + runnerOf := func(p Poll) time.Time { return runnerTime(p, p.GrafanaNow) } + + first := in[0] + if gap := runnerOf(first).Sub(from) + first.SkewBound(); gap > largestGap { + largestGap, largestGapAt = gap, from + } + for i := 1; i < len(in); i++ { + if gap := in[i].GrafanaNow.Sub(in[i-1].GrafanaNow); gap > largestGap { + largestGap, largestGapAt = gap, runnerOf(in[i-1]) + } + } + last := in[len(in)-1] + if gap := windowEnd.Sub(runnerOf(last)) + last.SkewBound(); gap > largestGap { + largestGap, largestGapAt = gap, runnerOf(last) + } + return largestGap, largestGapAt +} + +// longestHealthRun returns the longest contiguous span of polls reading the +// given rule-level Health, measured incrementally so a run still failing at the +// last in-window poll is measured correctly without assuming anything past the +// window. +func longestHealthRun(polls []Poll, health string) (longest time.Duration, sawAny bool) { + var runStart time.Time + for _, p := range polls { + if p.Health != health { + runStart = time.Time{} + continue + } + sawAny = true + if runStart.IsZero() { + runStart = p.GrafanaNow + } + if span := p.GrafanaNow.Sub(runStart); span > longest { + longest = span + } + } + return longest, sawAny +} + +// reasonsContain reports whether any key of reasons names want, honoring +// Grafana's comma-joined composite reason strings via reasonNames (log.go). +func reasonsContain(reasons map[string]int, want string) bool { + for reason := range reasons { + if reasonNames(reason, want) { + return true + } + } + return false +} diff --git a/grafana-alertcheck/internal/gate/coverage_test.go b/grafana-alertcheck/internal/gate/coverage_test.go new file mode 100644 index 000000000..3507ee4fc --- /dev/null +++ b/grafana-alertcheck/internal/gate/coverage_test.go @@ -0,0 +1,711 @@ +package gate + +import ( + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestProveCoverage_CleanWindowIsProved(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", State: "inactive", LastEvaluation: ts}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.True(t, res.Proved) + require.False(t, res.Unobservable) + require.Empty(t, res.Reason) +} + +func TestProveCoverage_FiltersPollsByUID(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + // A different rule's polls, deliberately broken, must never + // contaminate r1's verdict: proveCoverage selects by UID itself. + polls = append(polls, Poll{RuleUID: "other", GrafanaNow: ts, Found: false}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.True(t, res.Proved, "a different rule's broken polls must not affect this rule's verdict") +} + +// --- Check 1: sentinel --- + +func TestProveCoverage_NoSentinelIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, nil, nil, rt, def, from, to, 0) + require.False(t, res.Proved) + require.Equal(t, ReasonNoSentinel, res.Reason, "an absent sentinel must never be a pass") +} + +func TestProveCoverage_SentinelBeforeGraceIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + grace := 2 * time.Minute + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + sentinel := to.Add(grace).Add(-time.Second) // one second short of to+grace + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, nil, &sentinel, rt, def, from, to, grace) + require.Equal(t, ReasonSentinelEarly, res.Reason) + require.True(t, res.Unobservable) + require.False(t, res.Proved, "a reason string with no consequence is not a coverage failure") + + // The consequence: decide() must turn this into exit 2, never a pass. + defs := []Definition{def} + drt := map[string]ruleTimings{def.UID: rt} + gt := globalTimings{transitionGrace: grace} + pol := Policy{From: from, To: to} + dres, err := decide(Header{StartedAt: from.Add(-time.Hour)}, nil, &sentinel, defs, drt, gt, pol) + require.Error(t, err, "a sentinel short of to+grace must fail the run") + require.Len(t, dres.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, dres.Verdicts[0].Outcome) +} + +func TestProveCoverage_SentinelExactlyAtGraceIsFine(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + grace := 2 * time.Minute + windowEnd := to.Add(grace) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(windowEnd); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + } + sentinel := windowEnd + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, grace) + require.True(t, res.Proved, "sentinel exactly at to+grace must satisfy check 1") +} + +// --- Check 2: from bounds --- + +func TestProveCoverage_FromBeforeRecordIsUnobservable(t *testing.T) { + started := time.Date(2026, 1, 1, 1, 0, 0, 0, time.UTC) + from := started.Add(-time.Minute) // the requested window opens before recording started + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + sentinel := to + res := proveCoverage(Header{StartedAt: started}, nil, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonFromBeforeRecord, res.Reason) + require.True(t, res.Unobservable) + require.False(t, res.Proved) + + // The consequence: decide() must turn this into exit 2, never a pass. + defs := []Definition{def} + drt := map[string]ruleTimings{def.UID: rt} + gt := globalTimings{} + pol := Policy{From: from, To: to} + dres, err := decide(Header{StartedAt: started}, nil, &sentinel, defs, drt, gt, pol) + require.Error(t, err, "`from` before the recording started must fail the run") + require.Len(t, dres.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, dres.Verdicts[0].Outcome) +} + +// The from-bounds check compares at whole-second granularity: a whole-second +// `from` may precede the recorder's sub-second StartedAt INSIDE the same second +// without being judged early. That one sliver is the --from truncation, not a +// blind interval, so the window is still proved. +func TestProveCoverage_FromSameSecondAsStartedAtIsProved(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + started := from.Add(500 * time.Millisecond) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", State: "inactive", LastEvaluation: ts}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: started}, polls, &sentinel, rt, def, from, to, 0) + require.True(t, res.Proved) + require.False(t, res.Unobservable) + require.Empty(t, res.Reason) +} + +// Exactly one whole second later is a different second: even at the boundary, +// the whole-second comparison reads it as before, however healthy the polls. +func TestProveCoverage_FromExactlyOneSecondBeforeStartedAtIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + started := from.Add(time.Second) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + sentinel := to + res := proveCoverage(Header{StartedAt: started}, nil, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonFromBeforeRecord, res.Reason) + require.True(t, res.Unobservable) + require.False(t, res.Proved) +} + +// A sub-second sliver that straddles the second boundary is still "before": +// 900ms into one second vs 100ms into the next are distinct seconds, so the +// 200ms gap is a from_before_record, not rounding noise. +func TestProveCoverage_FromSubSecondEarlierAcrossSecondBoundaryIsUnobservable(t *testing.T) { + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + from := base.Add(900 * time.Millisecond) + started := base.Add(time.Second + 100*time.Millisecond) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + sentinel := to + res := proveCoverage(Header{StartedAt: started}, nil, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonFromBeforeRecord, res.Reason) + require.True(t, res.Unobservable) + require.False(t, res.Proved) +} + +// --- Check 3: heartbeat continuity --- + +// The core heartbeat regression: data at both ends with a hole between is not +// enough. +func TestProveCoverage_HeartbeatGapBetweenBoundariesIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) // maxGap = 60s + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + {RuleUID: "r1", GrafanaNow: from.Add(time.Second), Found: true, Health: "ok", LastEvaluation: from}, + {RuleUID: "r1", GrafanaNow: to.Add(-time.Second), Found: true, Health: "ok", LastEvaluation: to}, + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonHeartbeatGap, res.Reason, "healthy edges with a hole in the middle must still fail") + // The gap is the SPACING between the two polls (598s), not either + // boundary segment (1s each) — pin the actual values, not just the verdict. + require.Equal(t, 598*time.Second, res.LargestGap) + require.True(t, res.LargestGapAt.Equal(from.Add(time.Second))) +} + +// --- Check 4/5: health --- + +func TestProveCoverage_HealthErrorShortBlipPassesWithNote(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) // healthGrace = 60s + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + blip := from.Add(2 * time.Minute) + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + health := "ok" + if ts.Equal(blip) { + health = "error" // one isolated failed evaluation + } + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: health, LastEvaluation: ts}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.True(t, res.Proved, "one failed evaluation must not fail an otherwise clean window") + require.True(t, anyContains(res.Notes, "health=error"), "want a health=error note even though it did not fail the window") +} + +func TestProveCoverage_HealthErrorSustainedIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) // healthGrace = 60s + def := Definition{UID: "r1", Title: "R1"} + + runStart, runEnd := from.Add(2*time.Minute), from.Add(5*time.Minute) // a 3-minute run, well past healthGrace + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + health := "ok" + if !ts.Before(runStart) && !ts.After(runEnd) { + health = "error" + } + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: health, LastEvaluation: ts}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonHealthError, res.Reason, "a run that outlasts healthGrace") +} + +func TestProveCoverage_HealthNodataNeverFatalHere(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "nodata", LastEvaluation: ts}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.True(t, res.Proved, "health=nodata for the WHOLE window must still not be fatal by itself") + require.True(t, anyContains(res.Notes, "health=nodata")) +} + +// --- Check 6: liveness --- + +// A healthy rule polled at intervalSeconds/2, across the full window, must +// show zero staleness violations. lastEvaluation only advances once per full +// evaluation interval here — the realistic shape a delta check misreads as +// stale on roughly half of all polls. +func TestProveCoverage_LivenessAbsoluteNeverFalseStale(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + pollEvery := 30 * time.Second + intervalSeconds := 60 + windowEnd := from.Add(10 * time.Minute) + rt := newRuleTimings(pollEvery, intervalSeconds) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + lastEval := from + for ts := from; !ts.After(windowEnd); ts = ts.Add(pollEvery) { + if ts.Sub(lastEval) >= time.Duration(intervalSeconds)*time.Second { + lastEval = ts + } + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: lastEval}) + } + sentinel := windowEnd + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, windowEnd, 0) + require.NotEqual(t, ReasonStaleEvaluation, res.Reason, "liveness must be absolute, never a delta against a previous poll") + require.Zero(t, res.BlindFor) + require.True(t, res.Proved) +} + +func TestProveCoverage_StaleEvaluationIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) // evalStaleAfter = 120s + def := Definition{UID: "r1", Title: "R1"} + + // Dense, otherwise-healthy polling so heartbeat continuity (check 3) + // stays intact — only check 6 should be able to fire. + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + staleAt := from.Add(5 * time.Minute) + for i := range polls { + if polls[i].GrafanaNow.Equal(staleAt) { + polls[i].LastEvaluation = staleAt.Add(-3 * time.Minute) + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonStaleEvaluation, res.Reason) + require.Equal(t, 3*time.Minute, res.BlindFor) +} + +func TestProveCoverage_ZeroLastEvaluationNeverFalseStale(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + // A paused rule legitimately reports the zero time; check 6 must not read + // that as an enormous staleness violation. Check 7 is its detector. + polls := []Poll{ + {RuleUID: "r1", GrafanaNow: from.Add(time.Minute), Found: true, IsPaused: true}, + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.NotEqual(t, ReasonStaleEvaluation, res.Reason, "a zero lastEvaluation on a paused poll must not trigger check 6") +} + +// --- Check 7: isPaused in-window --- + +func TestProveCoverage_PausedInWindowIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + // Dense, otherwise-healthy polling so heartbeat continuity (check 3) + // stays intact — only check 7 should be able to fire. + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + pausedAt := from.Add(5 * time.Minute) + for i := range polls { + if polls[i].GrafanaNow.Equal(pausedAt) { + polls[i].IsPaused = true + polls[i].LastEvaluation = time.Time{} // legal only while paused + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonPausedInWindow, res.Reason) +} + +// TestProveCoverage_PausedAfterWindowIsFine pins check 7's respect for the +// window boundary: a poll that reports paused but lands beyond windowEnd (a +// rule paused only after THIS release window closed) is filtered out by +// inWindowPolls and must not fail the window. Without that filter, a pause in +// the next release's window would wrongly fail this one. +func TestProveCoverage_PausedAfterWindowIsFine(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + polls = append(polls, Poll{ + RuleUID: "r1", GrafanaNow: to.Add(2 * time.Minute), + Found: true, Health: "ok", IsPaused: true, + }) + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.NotEqual(t, ReasonPausedInWindow, res.Reason, "a paused poll after windowEnd tripped check 7") + require.True(t, res.Proved) +} + +// --- Check 8: rule absent --- + +func TestProveCoverage_RuleAbsentIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + // Dense, otherwise-healthy polling so heartbeat continuity (check 3) + // stays intact — only check 8 should be able to fire. + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + absentAt := from.Add(5 * time.Minute) + for i := range polls { + if polls[i].GrafanaNow.Equal(absentAt) { + polls[i].Found = false + polls[i].Health = "" + polls[i].LastEvaluation = time.Time{} + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonRuleAbsent, res.Reason) +} + +// denseHealthyPolls builds a clean poll sequence at a fixed cadence, with +// zero staleness and nothing abnormal — the baseline the single-check tests +// mutate exactly one poll of, so heartbeat continuity (check 3) never +// confounds the check under test. +func denseHealthyPolls(uid string, from, to time.Time, every time.Duration) []Poll { + var out []Poll + for ts := from; !ts.After(to); ts = ts.Add(every) { + out = append(out, Poll{RuleUID: uid, GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + } + return out +} + +// --- Check 9: KeepLast --- + +func TestProveCoverage_KeepLastObservedIsNoteOnly(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{ + RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts, + // A comma-joined composite — reasonsContain must match by + // membership, never by an exact key. + Reasons: map[string]int{"KeepLast, MissingSeries": 1}, + }) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.True(t, res.Proved, "KeepLast is a note, never fatal") + require.True(t, anyContains(res.Notes, "KeepLast"), "comma-joined membership, not a literal-key match") +} + +// KeepLast in the CONFIGURATION gives a note — a different claim from the +// observed-reason test above. A rule DECLARED with +// no_data_state or exec_err_state = KeepLast is a standing blind spot +// whether or not any poll ever actually reports the reason, so the note +// must fire off the definition alone, over an otherwise perfectly healthy +// window with zero KeepLast reasons anywhere in it. +func TestProveCoverage_KeepLastConfiguredIsNoteOnly(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + + tests := []struct { + name string + def Definition + }{ + {"no_data_state", Definition{UID: "r1", Title: "R1", NoDataState: "KeepLast"}}, + {"exec_err_state", Definition{UID: "r1", Title: "R1", ExecErrState: "KeepLast"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, tc.def, from, to, 0) + require.True(t, res.Proved, "a declared KeepLast is a note, never fatal") + require.True(t, anyContains(res.Notes, "KeepLast"), + "from the definition alone, with zero KeepLast reasons observed") + }) + } +} + +// --- Clock domains --- + +// A constant clock skew on every poll must not itself read as a coverage gap +// or a from-before-record violation, because every cross-domain comparison +// translates by that poll's own skew first. +func TestProveCoverage_SkewTranslationAtWindowBoundary(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + const skew = 45 * time.Second // Grafana's clock reads 45s ahead of the runner's + const bound = 5 * time.Second + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + grafanaTime := ts.Add(skew) + polls = append(polls, Poll{ + RuleUID: "r1", GrafanaNow: grafanaTime, SkewMS: skew.Milliseconds(), SkewBoundMS: bound.Milliseconds(), + Found: true, Health: "ok", LastEvaluation: grafanaTime, + }) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.True(t, res.Proved, "a constant clock skew must not itself read as a coverage gap") +} + +// --- Override round-trip: one authority for the cadence --- + +// This exercises DeriveTimingsFromLog and proveCoverage together, exactly as +// check does, to prove maxGap tracks the RECORDED cadence and never a +// re-derivation from the rule's own evaluation interval. +func TestProveCoverage_OverrideRoundTrip(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + t.Run("slower override on a tighter rule classifies clean", func(t *testing.T) { + windowEnd := from.Add(10 * time.Minute) + h := Header{ + StartedAt: from.Add(-time.Hour), + Rules: []LoggedRule{{UID: "r1", Title: "R1", IntervalSeconds: 60, PollEverySeconds: 120}}, + } + defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 60}} + rt, _, err := DeriveTimingsFromLog(h, defs) + require.NoError(t, err) + + var polls []Poll + for ts := from; !ts.After(windowEnd); ts = ts.Add(120 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + } + sentinel := windowEnd + + res := proveCoverage(h, polls, &sentinel, rt["r1"], defs[0], from, windowEnd, 0) + require.True(t, res.Proved, + "maxGap must come from the recorded 120s cadence, not the 30s default") + }) + + t.Run("faster override still catches a real recorder gap", func(t *testing.T) { + windowEnd := from.Add(20 * time.Minute) + h := Header{ + StartedAt: from.Add(-time.Hour), + Rules: []LoggedRule{{UID: "r1", Title: "R1", IntervalSeconds: 300, PollEverySeconds: 5}}, + } + defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 300}} + rt, _, err := DeriveTimingsFromLog(h, defs) + require.NoError(t, err) + + var polls []Poll + ts := from + for range 20 { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + ts = ts.Add(5 * time.Second) + } + // The one real gap: 250s, nowhere near this recording's actual 5s + // cadence. Resume 5s polling afterward all the way to windowEnd, so + // this hole is the ONLY gap in the window — otherwise an uncovered + // tail would exceed even the WRONG (definition-derived) 300s maxGap + // on its own, and the test could not tell the two derivations apart. + ts = ts.Add(250 * time.Second) + for !ts.After(windowEnd) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + ts = ts.Add(5 * time.Second) + } + sentinel := windowEnd + + res := proveCoverage(h, polls, &sentinel, rt["r1"], defs[0], from, windowEnd, 0) + require.Equal(t, ReasonHeartbeatGap, res.Reason, + "if maxGap had been re-derived from the 300s definition, this 250s gap would pass silently") + }) +} + +func anyContains(notes []string, substr string) bool { + for _, n := range notes { + if strings.Contains(n, substr) { + return true + } + } + return false +} + +// --- Check 6, tightened: a corrupted log must not silently disable liveness --- + +// TestProveCoverage_ZeroLastEvaluationWithoutPauseIsStale guards check 6's +// skip condition. ReadLog does no field validation, so a log line can claim +// found:true, is_paused:false and still carry a zero LastEvaluation (a +// corrupted write, a hand-edited fixture, a future log format bug). That +// combination must read as maximally stale, not be waved through the way a +// legitimately paused poll's zero time is — the skip must key off +// IsPaused/Found, never off LastEvaluation being zero. +func TestProveCoverage_ZeroLastEvaluationWithoutPauseIsStale(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + corruptAt := from.Add(5 * time.Minute) + for i := range polls { + if polls[i].GrafanaNow.Equal(corruptAt) { + polls[i].LastEvaluation = time.Time{} // found:true, is_paused:false, yet zero + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonStaleEvaluation, res.Reason, + "a zero lastEvaluation on a found, non-paused poll must fail closed") +} + +// A lastEvaluation in the future of grafana_now (corrupted log) must fail closed. +func TestProveCoverage_FutureLastEvaluationIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + corruptAt := from.Add(5 * time.Minute) + for i := range polls { + if polls[i].GrafanaNow.Equal(corruptAt) { + polls[i].LastEvaluation = corruptAt.Add(2 * time.Minute) // in the future of its own grafana_now + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonFutureEvaluation, res.Reason, + "a lastEvaluation in the future of grafana_now must fail closed rather than read its negative staleness as fresh") +} + +// --- Check 3, tightened: the boundary segments must widen by the skew bound --- + +// The two boundary segments take their own poll's bound as the tolerance: a +// boundary gap that lands EXACTLY at maxGap must still fail once the poll's +// own skew bound is added, because the translation is only a best +// estimate and understating the gap by up to the bound would be fail-open. +func TestProveCoverage_BoundaryGapWidensBySkewBound(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) // maxGap = 60s + def := Definition{UID: "r1", Title: "R1"} + + const bound = 5 * time.Second + first := Poll{ + RuleUID: "r1", GrafanaNow: from.Add(rt.maxGap), Found: true, Health: "ok", + LastEvaluation: from.Add(rt.maxGap), SkewBoundMS: bound.Milliseconds(), + } + rest := denseHealthyPolls("r1", from.Add(rt.maxGap+30*time.Second), to, 30*time.Second) + polls := append([]Poll{first}, rest...) + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonHeartbeatGap, res.Reason, + "the poll's own %s skew bound must push it past the threshold", bound) +} + +// --- Multi-failure contract --- + +// Two checks failing in the same rule: check 7 (paused in-window) runs before +// check 8 (rule absent), so Reason must name the pause even though the rule +// also goes absent later — and the later failure must still add its own Note +// rather than being swallowed once Reason is set. +func TestProveCoverage_MultipleFailuresReasonIsFirstButAllNoted(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + pausedAt := from.Add(3 * time.Minute) + absentAt := from.Add(6 * time.Minute) + for i := range polls { + switch { + case polls[i].GrafanaNow.Equal(pausedAt): + polls[i].IsPaused = true + polls[i].LastEvaluation = time.Time{} + case polls[i].GrafanaNow.Equal(absentAt): + polls[i].Found = false + polls[i].Health = "" + polls[i].LastEvaluation = time.Time{} + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonPausedInWindow, res.Reason, "the FIRST check to fail names the reason") + require.True(t, anyContains(res.Notes, "paused")) + require.True(t, anyContains(res.Notes, "no rule"), + "a later failure must still be recorded, not swallowed once Reason is already set") +} + +// --- Skipped rules --- + +// A known limit of this function's contract, not a bug in it: a rule paused +// BEFORE the window opened is never scheduled or polled (watch.go), so it +// reaches proveCoverage with zero polls at all. proveCoverage has no notion of +// "skipped" — that classification belongs to the definitions +// (LoggedRule.IsPaused / Definition.IsPaused), never to the polls — so it +// reports the whole window as one big heartbeat_gap instead. decide is what +// reads skipped status from the header and never calls this function for such +// a rule; this pins the behavior it relies on not reaching. +func TestProveCoverage_SkippedRuleWithZeroPollsPinnedAsHeartbeatGap(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1", IsPaused: true} + + sentinel := to + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, nil, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonHeartbeatGap, res.Reason, + "proveCoverage has no 'skipped' concept, so decide must handle a skipped rule's classification itself") +} diff --git a/grafana-alertcheck/internal/gate/duration.go b/grafana-alertcheck/internal/gate/duration.go index 011c27f14..3690a2549 100644 --- a/grafana-alertcheck/internal/gate/duration.go +++ b/grafana-alertcheck/internal/gate/duration.go @@ -29,7 +29,7 @@ var promDurationUnits = []promDurationUnit{ } // ParsePromDuration parses a Grafana/Prometheus-style duration ("1h30m", "1d", "1w"). -// Unlike time.ParseDuration, it accepts "d" and "w" (§11.8). "" and "0" are 0. +// Unlike time.ParseDuration, it accepts "d" and "w". "" and "0" are 0. func ParsePromDuration(s string) (time.Duration, error) { if s == "" || s == "0" { return 0, nil diff --git a/grafana-alertcheck/internal/gate/duration_test.go b/grafana-alertcheck/internal/gate/duration_test.go index 73a0c00e1..6262c713e 100644 --- a/grafana-alertcheck/internal/gate/duration_test.go +++ b/grafana-alertcheck/internal/gate/duration_test.go @@ -3,6 +3,8 @@ package gate import ( "testing" "time" + + "github.com/stretchr/testify/require" ) func TestParsePromDuration(t *testing.T) { @@ -35,31 +37,27 @@ func TestParsePromDuration(t *testing.T) { } for _, c := range cases { got, err := ParsePromDuration(c.in) - if err != nil { - t.Errorf("ParsePromDuration(%q): unexpected error: %v", c.in, err) - continue - } - if got != c.want { - t.Errorf("ParsePromDuration(%q) = %v, want %v", c.in, got, c.want) - } + require.NoErrorf(t, err, "ParsePromDuration(%q)", c.in) + require.Equalf(t, c.want, got, "ParsePromDuration(%q)", c.in) } } func TestParsePromDuration_Errors(t *testing.T) { cases := []string{ - "5", // bare number, no unit - "-5m", // negative - "5x", // unknown unit - "30m1h", // ascending order (must be descending) - "1h1h", // duplicate unit - "m", // unit with no number - "1.5h", // fractional number not supported by this grammar - "1 h", // whitespace - "300y", // overflows time.Duration (int64 nanoseconds) — must error, not wrap negative + "5", // bare number, no unit + "-5m", // negative + "5x", // unknown unit + "30m1h", // ascending order (must be descending) + "1h1h", // duplicate unit + "m", // unit with no number + "1.5h", // fractional number not supported by this grammar + "1 h", // whitespace + "300y", // overflows time.Duration (int64 nanoseconds) — must error, not wrap negative + "1w2d3h4m5s6ms7us8ns", // too many units + "carrot", // completely invalid } for _, in := range cases { - if _, err := ParsePromDuration(in); err == nil { - t.Errorf("ParsePromDuration(%q): expected an error, got none", in) - } + _, err := ParsePromDuration(in) + require.Errorf(t, err, "ParsePromDuration(%q): expected an error, got none", in) } } diff --git a/grafana-alertcheck/internal/gate/flock.go b/grafana-alertcheck/internal/gate/flock.go new file mode 100644 index 000000000..ae21f943b --- /dev/null +++ b/grafana-alertcheck/internal/gate/flock.go @@ -0,0 +1,45 @@ +package gate + +import ( + "errors" + "fmt" + "os" + "syscall" +) + +// lockExclusive takes a non-blocking exclusive lock on f. Non-blocking is the +// point: a second writer must fail immediately with an error the operator +// sees, not queue behind the first and start appending to a log somebody else +// already finished. +func lockExclusive(f *os.File) error { + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { + return fmt.Errorf("flock: %w", err) + } + return nil +} + +// isLockContention reports whether a flock failure means another writer holds +// the lock, as opposed to an unrelated failure. +func isLockContention(err error) bool { + return errors.Is(err, syscall.EWOULDBLOCK) || errors.Is(err, syscall.EAGAIN) +} + +// tryLockExclusive is the same call read as a question rather than as a +// demand: held is false when another process holds the lock, and err is +// non-nil only for a failure that is not contention. +// +// check needs that distinction where NewWriter does not. NewWriter is entitled +// to treat any refusal as "another writer has it", because it wants the lock; +// check only wants to know whether a writer EXISTS. The lock answers +// that directly, where a pid can only infer it — the kernel releases a flock +// when the holder exits, crash included, and pids get reused. +func tryLockExclusive(f *os.File) (held bool, err error) { + switch err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); { + case err == nil: + return true, nil + case errors.Is(err, syscall.EWOULDBLOCK): + return false, nil + default: + return false, fmt.Errorf("flock %s: %w", f.Name(), err) + } +} diff --git a/grafana-alertcheck/internal/gate/flock_test.go b/grafana-alertcheck/internal/gate/flock_test.go new file mode 100644 index 000000000..3dfdd8b73 --- /dev/null +++ b/grafana-alertcheck/internal/gate/flock_test.go @@ -0,0 +1,32 @@ +package gate + +import ( + "errors" + "fmt" + "syscall" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsLockContention(t *testing.T) { + contended := []error{syscall.EWOULDBLOCK, syscall.EAGAIN} + for _, e := range contended { + require.Truef(t, isLockContention(e), "isLockContention(%v)", e) + // lockExclusive wraps the raw error via fmt.Errorf("flock: %w", ...). + require.Truef(t, isLockContention(fmt.Errorf("flock: %w", e)), "isLockContention(wrapped %v)", e) + } + + notContended := []error{ + syscall.EROFS, + syscall.ENOTSUP, + syscall.ENOLCK, + syscall.EBADF, + syscall.EIO, + errors.New("something else"), + } + for _, e := range notContended { + require.Falsef(t, isLockContention(e), "isLockContention(%v)", e) + require.Falsef(t, isLockContention(fmt.Errorf("flock: %w", e)), "isLockContention(wrapped %v)", e) + } +} diff --git a/grafana-alertcheck/internal/gate/jsonreq.go b/grafana-alertcheck/internal/gate/jsonreq.go index bfddca382..5b0799a55 100644 --- a/grafana-alertcheck/internal/gate/jsonreq.go +++ b/grafana-alertcheck/internal/gate/jsonreq.go @@ -8,7 +8,7 @@ import ( // req decodes m[key] into *dst. It returns an error when key is absent from m // or explicitly JSON null, so a caller can never mistake absence for a zero -// value (H1) — json.Unmarshal treats "null" as a documented no-op for +// value — json.Unmarshal treats "null" as a documented no-op for // non-pointer targets (string, bool, int, ...), so without this check a // required field sent as null would silently pass through as its zero value. func req[T any](m map[string]json.RawMessage, key string, dst *T) error { diff --git a/grafana-alertcheck/internal/gate/jsonreq_test.go b/grafana-alertcheck/internal/gate/jsonreq_test.go index bf3881e4b..72b446503 100644 --- a/grafana-alertcheck/internal/gate/jsonreq_test.go +++ b/grafana-alertcheck/internal/gate/jsonreq_test.go @@ -3,14 +3,14 @@ package gate import ( "encoding/json" "testing" + + "github.com/stretchr/testify/require" ) func rawMap(t *testing.T, jsonObj string) map[string]json.RawMessage { t.Helper() var m map[string]json.RawMessage - if err := json.Unmarshal([]byte(jsonObj), &m); err != nil { - t.Fatalf("rawMap: %v", err) - } + require.NoError(t, json.Unmarshal([]byte(jsonObj), &m)) return m } @@ -19,34 +19,24 @@ func TestReq(t *testing.T) { t.Run("present key decodes", func(t *testing.T) { var s string - if err := req(m, "present", &s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if s != "hello" { - t.Errorf("got %q, want hello", s) - } + require.NoError(t, req(m, "present", &s)) + require.Equal(t, "hello", s) }) t.Run("absent key errors", func(t *testing.T) { var s string - if err := req(m, "missing", &s); err == nil { - t.Fatalf("expected an error, got none") - } + require.Error(t, req(m, "missing", &s)) }) t.Run("wrong type errors", func(t *testing.T) { var s string - if err := req(m, "wrongtype", &s); err == nil { - t.Fatalf("expected an error, got none") - } + require.Error(t, req(m, "wrongtype", &s)) }) t.Run("explicit JSON null errors, never a zero value", func(t *testing.T) { var s string err := req(m, "nullval", &s) - if err == nil { - t.Fatalf("expected an error, got none (s=%q) — a null required field must not silently become a zero value", s) - } + require.Error(t, err, "a null required field must not silently become a zero value") }) } @@ -55,38 +45,24 @@ func TestOpt(t *testing.T) { t.Run("present key decodes", func(t *testing.T) { var s string - if err := opt(m, "present", &s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if s != "hello" { - t.Errorf("got %q, want hello", s) - } + require.NoError(t, opt(m, "present", &s)) + require.Equal(t, "hello", s) }) t.Run("absent key leaves dst untouched", func(t *testing.T) { s := "unchanged" - if err := opt(m, "missing", &s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if s != "unchanged" { - t.Errorf("got %q, want unchanged", s) - } + require.NoError(t, opt(m, "missing", &s)) + require.Equal(t, "unchanged", s) }) t.Run("wrong type errors", func(t *testing.T) { var s string - if err := opt(m, "wrongtype", &s); err == nil { - t.Fatalf("expected an error, got none") - } + require.Error(t, opt(m, "wrongtype", &s)) }) t.Run("explicit JSON null leaves dst at its zero value", func(t *testing.T) { var s string - if err := opt(m, "nullval", &s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if s != "" { - t.Errorf("got %q, want empty string", s) - } + require.NoError(t, opt(m, "nullval", &s)) + require.Equal(t, "", s) }) } diff --git a/grafana-alertcheck/internal/gate/log.go b/grafana-alertcheck/internal/gate/log.go new file mode 100644 index 000000000..f8f52d01f --- /dev/null +++ b/grafana-alertcheck/internal/gate/log.go @@ -0,0 +1,597 @@ +package gate + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "sync" + "time" +) + +// LogSchemaVersion is the version stamped into every log header. A log with +// any other value is a read error, never a best-effort read: the log is the +// gate's only evidence, and misreading a stale shape is a fail-open. +const LogSchemaVersion = 1 + +// RecordType tags each JSONL line. There are exactly three, and a poll record +// IS the heartbeat — there is deliberately no separate heartbeat type. +type RecordType string + +const ( + RecordHeader RecordType = "header" + RecordPoll RecordType = "poll" + RecordStopped RecordType = "stopped" +) + +// missingSeriesReason is the reason Grafana parks a disappearing series at +// ("Normal (MissingSeries)") for a couple of evaluations before deleting the +// instance. Reading that as a recovery would turn a disappearing series into a +// fake recovery, so the markers below route it to Vanished. +const missingSeriesReason = "MissingSeries" + +// LoggedRule is the per-rule identity written into the header. Together with +// the header URL it IS the log's identity, which check validates, and it +// supplies the alert set in check mode. +type LoggedRule struct { + UID string `json:"uid"` + Title string `json:"title"` + Folder string `json:"folder"` + Group string `json:"group"` + // ForSeconds, IntervalSeconds, NoDataState and ExecErrState are purely + // forensic: a resolve-time snapshot that makes the uploaded artifact + // self-describing to a human reading it after the runner is gone. check + // never converts them back into a Definition — it always re-resolves + // definitions from the ruler API. + ForSeconds float64 `json:"for_seconds"` + IntervalSeconds int `json:"interval_seconds"` + // IsPaused is load-bearing (beside PollEverySeconds): the pause state at + // record start, the only moment `skipped` can honestly mean. decide reads + // it via Header.pausedAtStart, never a ruler read taken after the window. + IsPaused bool `json:"is_paused"` + NoDataState string `json:"no_data_state"` + ExecErrState string `json:"exec_err_state"` + // PollEverySeconds is the cadence this recording ACTUALLY used. Load-bearing: + // check derives maxGap from it, never from the definitions — getting that + // wrong is fail-open in the faster-override direction. + PollEverySeconds float64 `json:"poll_every_seconds"` +} + +// Header is the log's first line: what was recorded, from where, and when the +// recording started. It carries no States field — recording is deliberately +// unfiltered, so the same log can be re-classified under different --states +// without re-recording. +type Header struct { + SchemaVersion int `json:"schema_version"` + URL string `json:"url"` // the log's identity + GrafanaVersion string `json:"grafana_version"` + StartedAt time.Time `json:"started_at"` // the record start + Rules []LoggedRule `json:"rules"` // THE alert set +} + +// pausedAtStart reports, per rule UID, whether the rule was paused when the +// recording opened. That instant — and no other — is what `skipped` means: a +// rule nobody was watching on purpose. +// +// It is the authority for `skipped` in BOTH modes, and the reason is that no +// other source knows the right moment. `check` re-resolves the definitions +// AFTER the window closed, so Definition.IsPaused there describes the present, +// not the window: a rule that fired and was then paused would read as skipped, +// its firing would never be classified, and under --allow-paused the run would +// pass. The header cannot drift that way, because watch stamps it before the +// deploy step runs and single-step check stamps it from definitions resolved at +// the start of its own step. +// +// A UID the header does not name is reported NOT paused, which is the safe +// direction: it then reaches proveCoverage with no polls and fails closed as +// a heartbeat gap, rather than being waved through as legitimately unwatched. +func (h Header) pausedAtStart() map[string]bool { + paused := make(map[string]bool, len(h.Rules)) + for _, lr := range h.Rules { + paused[lr.UID] = lr.IsPaused + } + return paused +} + +// Poll is one reduced observation of one rule — the log's heartbeat and the +// only input the pure coverage and classification layers ever see. +type Poll struct { + RuleUID string `json:"rule_uid"` + GrafanaNow time.Time `json:"grafana_now"` // the response's Date header + // SkewMS, SkewBoundMS and LatencyMS are milliseconds for JSONL + // compactness ONLY. The pure layer never touches raw ms: it reads + // Skew(), SkewBound() and Latency() below, which convert at the + // (de)serialization boundary. + SkewMS int64 `json:"skew_ms"` + SkewBoundMS int64 `json:"skew_bound_ms"` + LatencyMS int64 `json:"latency_ms"` + // Found false means an authoritative 2xx in which this rule was absent — + // never a transport failure, which the transport retries and never turns + // into a Poll. The coverage proof turns it into unobservable. + Found bool `json:"found"` + // State, Health and LastError are the raw rule-level strings, reporting + // only and never classified. + State string `json:"state,omitempty"` + Health string `json:"health,omitempty"` + LastError string `json:"last_error,omitempty"` + // omitzero, not omitempty: a not-found poll (and a paused rule) has no + // evaluation time, and writing "0001-01-01T00:00:00Z" into an artifact + // humans and jq read invites reading it as a real timestamp. + LastEvaluation time.Time `json:"last_evaluation,omitzero"` + IsPaused bool `json:"is_paused"` + Histogram map[string]int `json:"histogram,omitempty"` // written, never analysed + // Reasons counts this poll's non-empty instance reasons, e.g. + // {"NoData":1091,"Error":14}; nil when none. Reporting-only, and the only + // place composite states stay visible (they are canonical normal, dropped + // from Abnormal). Keys are raw reason strings and can be comma-joined + // composites ("KeepLast, MissingSeries"), so consumers must test membership + // via reasonNames and never index a literal key. + Reasons map[string]int `json:"reasons,omitempty"` + // Abnormal holds the instances whose CANONICAL state is not normal. + // "Normal (NoData)" and "Normal (Error)" are canonical normal and are + // deliberately not retained here. + Abnormal []Instance `json:"abnormal,omitempty"` + // Cleared and Vanished are instance keys that left the abnormal set, + // resolved against the SAME response — a clear and a discontinuity are not + // the same fact. + Cleared []string `json:"cleared,omitempty"` + Vanished []string `json:"vanished,omitempty"` +} + +// Skew is the signed clock skew of this poll. +func (p Poll) Skew() time.Duration { return time.Duration(p.SkewMS) * time.Millisecond } + +// SkewBound is the uncertainty on Skew — the tolerance every cross-domain +// comparison applies alongside it. +func (p Poll) SkewBound() time.Duration { return time.Duration(p.SkewBoundMS) * time.Millisecond } + +// Latency is the wall time this poll's request took, feeding the budget check. +func (p Poll) Latency() time.Duration { return time.Duration(p.LatencyMS) * time.Millisecond } + +// Reducer turns each Observation into the single Poll record that goes into +// the log. It holds the previous poll's abnormal instance keys per rule, which +// is all the state the transition markers need. +// +// A Reducer is safe for concurrent use: watch polls a fleet of rules +// concurrently and every one of those goroutines reduces through the same +// instance, because the per-rule marker state has to live in one place. The +// lock is per-Reducer rather than per-rule — Reduce only touches maps and +// slices, so it never blocks on I/O while holding it. +type Reducer struct { + mu sync.Mutex + prevAbnormal map[string]map[string]struct{} +} + +func NewReducer() *Reducer { + return &Reducer{prevAbnormal: make(map[string]map[string]struct{})} +} + +// Reduce selects the rule identified by uid out of obs and reduces it to a +// Poll. Selection is BY UID, never by title: a filtered response can carry +// several rules sharing one title, and picking the first would silently watch +// the wrong rule. +// +// The reduction keeps the rule-level fields, the raw totals histogram, the +// reason counts, and only the instances whose canonical state is not normal. +// That makes per-poll size independent of NORMAL cardinality — not of +// cardinality outright: a rule with 449 firing instances still stores all 449. +func (r *Reducer) Reduce(uid string, obs Observation) Poll { + r.mu.Lock() + defer r.mu.Unlock() + + p := Poll{ + RuleUID: uid, + GrafanaNow: obs.GrafanaNow, + SkewMS: obs.Skew.Milliseconds(), + SkewBoundMS: obs.SkewBound.Milliseconds(), + LatencyMS: obs.Latency.Milliseconds(), + } + + rule := stateRuleByUID(obs.Rules, uid) + if rule == nil { + // An authoritative "the rule is absent". No markers are computed and + // the previous abnormal set is kept untouched: if the rule comes back + // with an instance missing, the next poll still reports that instance + // as vanished rather than losing the transition entirely. + return p + } + + p.Found = true + p.State = rule.State + p.Health = rule.Health + p.LastError = rule.LastError + p.LastEvaluation = rule.LastEvaluation + p.IsPaused = rule.IsPaused + p.Histogram = rule.Totals + + // present indexes every instance in THIS response, normal ones included — + // the markers below must resolve a departed key against the same response, + // which is impossible from the abnormal subset alone. + present := make(map[string]Instance, len(rule.Instances)) + curAbnormal := make(map[string]struct{}) + for _, inst := range rule.Instances { + key := instanceKey(inst.Labels) + present[key] = inst + if inst.Reason != "" { + if p.Reasons == nil { + p.Reasons = make(map[string]int) + } + p.Reasons[inst.Reason]++ + } + if inst.State != StateNormal { + p.Abnormal = append(p.Abnormal, inst) + curAbnormal[key] = struct{}{} + } + } + + for key := range r.prevAbnormal[uid] { + if _, still := curAbnormal[key]; still { + continue + } + inst, found := present[key] + switch { + case !found: + // Fully absent from the response: a discontinuity, not a recovery. + p.Vanished = append(p.Vanished, key) + case reasonNames(inst.Reason, missingSeriesReason): + // The vanish in disguise, caught one poll earlier than the fully + // absent case. + p.Vanished = append(p.Vanished, key) + default: + // Present as canonical normal without a MissingSeries reason. + p.Cleared = append(p.Cleared, key) + } + } + // Map iteration is unordered; sort so a log line is byte-stable for a + // given poll and a golden fixture stays meaningful. + sort.Strings(p.Cleared) + sort.Strings(p.Vanished) + + r.prevAbnormal[uid] = curAbnormal + return p +} + +// seedFrom restores the marker state above from polls that are already in the +// log, so the first poll a NEW Reducer produces compares against the last poll +// the previous one wrote instead of against an empty set. +// +// It exists for the one place a recording changes hands: watch's parent takes +// the first observation of every rule and its detached child continues from +// there. Without the seed, an instance that is abnormal in the parent's +// observation and gone by the child's first poll produces no marker at all — +// it leaves the record as though it had never been bad, the same fail-open a +// misread MissingSeries causes, reached through the handoff instead. +// +// Not-found polls are skipped, mirroring Reduce: an absent rule leaves the +// previous abnormal set untouched rather than emptying it. +func (r *Reducer) seedFrom(polls []Poll) { + r.mu.Lock() + defer r.mu.Unlock() + for _, p := range polls { + if !p.Found { + continue + } + keys := make(map[string]struct{}, len(p.Abnormal)) + for _, inst := range p.Abnormal { + keys[instanceKey(inst.Labels)] = struct{}{} + } + r.prevAbnormal[p.RuleUID] = keys + } +} + +// stateRuleByUID picks one rule out of a state response BY UID (nil = the +// authoritative "rule absent"). Never by title: the ?rule_name= filter is a +// title filter and can return several rules sharing a title. The single +// selection for the package — Reduce and the drain wait both use it. +func stateRuleByUID(rules []StateRule, uid string) *StateRule { + for i := range rules { + if rules[i].UID == uid { + return &rules[i] + } + } + return nil +} + +// reasonNames reports whether reason names want. Newer Grafana versions +// comma-join several reasons into one string, so this tests membership rather +// than equality. +func reasonNames(reason, want string) bool { + for part := range strings.SplitSeq(reason, ",") { + if strings.TrimSpace(part) == want { + return true + } + } + return false +} + +// VerifyNormalInstancesVisible checks, on a first observation, that the state +// endpoint really returns normal instances: if it ever stops, the reduction's +// "keep the non-normal" becomes "keep everything the API sent", a silent +// fail-open in the transition markers. Counts are summed over every totals key +// whose lowercased name is "normal" or "inactive" — never a literal key, since +// the vocabulary is mixed and its case has drifted. +func VerifyNormalInstancesVisible(rules []StateRule) error { + for _, r := range rules { + var claimed int + for k, v := range r.Totals { + switch strings.ToLower(k) { + case "normal", "inactive": + claimed += v + } + } + if claimed == 0 { + continue + } + if hasNormalInstance(r.Instances) { + continue + } + return fmt.Errorf( + "rule %q (%s): totals claim %d normal instances but the response returned none — "+ + "the state endpoint no longer returns normal instances, which the reduction depends on", + r.Title, r.UID, claimed) + } + return nil +} + +func hasNormalInstance(instances []Instance) bool { + for _, inst := range instances { + if inst.State == StateNormal { + return true + } + } + return false +} + +// headerRecord, pollRecord and stoppedRecord are the three wire shapes. The +// type tag is a real field on each line rather than an envelope, so a human +// (or jq) reading an uploaded log sees flat records. +type headerRecord struct { + Type RecordType `json:"type"` + Header +} + +type pollRecord struct { + Type RecordType `json:"type"` + Poll +} + +type stoppedRecord struct { + Type RecordType `json:"type"` + At time.Time `json:"at"` +} + +// Writer appends records to the JSONL log. It is append-only by construction — +// O_APPEND|O_CREATE|O_WRONLY, never O_TRUNC — so no writer can ever destroy +// evidence a previous one recorded. An exclusive non-blocking flock makes a +// second writer fail immediately rather than interleave. +type Writer struct { + mu sync.Mutex + f *os.File + enc *json.Encoder + clock Clock + stopped bool +} + +// NewWriter opens path for appending and takes the exclusive lock. A second +// writer on the same path fails here, immediately — it never blocks and never +// waits, because two recorders on one log means one of them is recording a +// window nobody will classify. +func NewWriter(path string, clock Clock) (*Writer, error) { + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return nil, fmt.Errorf("open log %s: %w", path, err) + } + if err := lockExclusive(f); err != nil { + f.Close() + if isLockContention(err) { + return nil, fmt.Errorf("lock log %s: %w (another writer holds it)", path, err) + } + return nil, fmt.Errorf("lock log %s: %w", path, err) + } + return &Writer{f: f, enc: json.NewEncoder(f), clock: clock}, nil +} + +// WriteHeader writes line 1 and stamps the current schema version, so no +// caller can leave it at zero. It refuses a non-empty file: the log already +// has a header, and a second one would make ReadLog's "header is line 1" +// contract a lie. In watch's handoff the parent writes the header and the +// detached child only appends polls. +func (w *Writer) WriteHeader(h Header) error { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped { + return fmt.Errorf("log writer already stopped") + } + info, err := w.f.Stat() + if err != nil { + return fmt.Errorf("stat log: %w", err) + } + if info.Size() != 0 { + return fmt.Errorf("log %s is not empty: it already has a header", w.f.Name()) + } + h.SchemaVersion = LogSchemaVersion + if err := w.enc.Encode(headerRecord{Type: RecordHeader, Header: h}); err != nil { + return fmt.Errorf("write log header: %w", err) + } + return nil +} + +// WritePoll appends one poll record — the heartbeat. +func (w *Writer) WritePoll(p Poll) error { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped { + return fmt.Errorf("log writer already stopped") + } + if err := w.enc.Encode(pollRecord{Type: RecordPoll, Poll: p}); err != nil { + return fmt.Errorf("write poll for rule %s: %w", p.RuleUID, err) + } + return nil +} + +// Stop finishes recording in a fixed order that must not be rearranged: let the +// in-flight write finish, append the sentinel, fsync, release — any other order +// can leave a sentinel that was never preceded by the polls it vouches for. +// The sentinel uses the writer's OWN stop time; check does the `to` comparison +// after this has exited. Calling Stop twice is a no-op (watch reaches it from a +// signal handler and a defer). +func (w *Writer) Stop() error { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped { + return nil + } + w.stopped = true + + encErr := w.enc.Encode(stoppedRecord{Type: RecordStopped, At: w.clock.Now()}) + syncErr := w.f.Sync() + closeErr := w.f.Close() + + switch { + case encErr != nil: + return fmt.Errorf("write stopped sentinel: %w", encErr) + case syncErr != nil: + return fmt.Errorf("fsync log: %w", syncErr) + case closeErr != nil: + return fmt.Errorf("close log: %w", closeErr) + } + return nil +} + +// Close releases the file and the lock WITHOUT writing a sentinel. It exists +// for exactly one caller: watch's parent, which writes the header and then +// hands the log to the detached child that will finish it. A sentinel here +// would tell check the recording ended before the child had even started. +func (w *Writer) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped { + return nil + } + w.stopped = true + if err := w.f.Close(); err != nil { + return fmt.Errorf("close log: %w", err) + } + return nil +} + +// ReadLogHeader reads ONLY line 1 — the one read safe while a writer may still +// hold the log. The header is written once by watch's parent before any child +// appends a byte, so line 1 is immutable. It lets check fail closed EARLY on a +// wrong URL or unresolvable rule; it is advisory only, and the authoritative +// identity read is still ReadLog after the writer exits. +func ReadLogHeader(path string) (Header, error) { + f, err := os.Open(path) + if err != nil { + return Header{}, fmt.Errorf("read log header %s: %w", path, err) + } + defer f.Close() + + line, err := bufio.NewReader(f).ReadString('\n') + if err != nil { + // io.EOF included: a log whose first line has no terminating newline is + // a log whose header was never fully written, which is not a header. + return Header{}, fmt.Errorf("log %s: no complete header on line 1: %w", path, err) + } + + var rec headerRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + return Header{}, fmt.Errorf("log %s line 1: unparseable header: %w", path, err) + } + if rec.Type != RecordHeader { + return Header{}, fmt.Errorf("log %s line 1: got record type %q; the header must be line 1", path, rec.Type) + } + if rec.SchemaVersion != LogSchemaVersion { + return Header{}, fmt.Errorf( + "log %s: schema version %d is not %d — this log was written by a different version of the gate", + path, rec.SchemaVersion, LogSchemaVersion) + } + return rec.Header, nil +} + +// ReadLog reads the whole log once and returns its header, its polls in +// recorded order, and the sentinel time when one is present (nil when the +// recording never finished — check turns that into unobservable, never a +// pass). +// +// Call this only after the writer has exited. Reading a log a writer can still +// append to can only produce a shorter window than the one that was recorded. +// +// The parse rules are deliberately the crudest possible: the header must be +// line 1 with a matching schema version, and ANY unparseable line — including +// the last, or one after a sentinel — is an error, full stop. A truncated log +// is evidence something killed the recorder, which must not pass. +func ReadLog(path string) (Header, []Poll, *time.Time, error) { + b, err := os.ReadFile(path) + if err != nil { + return Header{}, nil, nil, fmt.Errorf("read log %s: %w", path, err) + } + + lines := strings.Split(string(b), "\n") + // A complete record always ends with the encoder's newline, so the split + // leaves one trailing empty element. Drop exactly that one; any other + // empty line stays and fails below as unparseable. + if len(lines) > 0 && lines[len(lines)-1] == "" { + lines = lines[:len(lines)-1] + } + if len(lines) == 0 { + return Header{}, nil, nil, fmt.Errorf("log %s is empty: the header must be line 1", path) + } + + var ( + header Header + polls []Poll + sentinel *time.Time + ) + for i, line := range lines { + var probe struct { + Type RecordType `json:"type"` + } + if err := json.Unmarshal([]byte(line), &probe); err != nil { + return Header{}, nil, nil, fmt.Errorf("log %s line %d: unparseable record: %w", path, i+1, err) + } + if sentinel != nil { + return Header{}, nil, nil, fmt.Errorf( + "log %s line %d: a %q record follows the stopped sentinel — the log had a second writer", + path, i+1, probe.Type) + } + if (i == 0) != (probe.Type == RecordHeader) { + return Header{}, nil, nil, fmt.Errorf( + "log %s line %d: got record type %q; the header must be line 1 and appear only once", + path, i+1, probe.Type) + } + + switch probe.Type { + case RecordHeader: + var rec headerRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + return Header{}, nil, nil, fmt.Errorf("log %s line 1: unparseable header: %w", path, err) + } + if rec.SchemaVersion != LogSchemaVersion { + return Header{}, nil, nil, fmt.Errorf( + "log %s: schema version %d is not %d — this log was written by a different version of the gate", + path, rec.SchemaVersion, LogSchemaVersion) + } + header = rec.Header + case RecordPoll: + var rec pollRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + return Header{}, nil, nil, fmt.Errorf("log %s line %d: unparseable poll: %w", path, i+1, err) + } + polls = append(polls, rec.Poll) + case RecordStopped: + var rec stoppedRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + return Header{}, nil, nil, fmt.Errorf("log %s line %d: unparseable sentinel: %w", path, i+1, err) + } + at := rec.At + sentinel = &at + default: + return Header{}, nil, nil, fmt.Errorf("log %s line %d: unknown record type %q", path, i+1, probe.Type) + } + } + + return header, polls, sentinel, nil +} diff --git a/grafana-alertcheck/internal/gate/log_test.go b/grafana-alertcheck/internal/gate/log_test.go new file mode 100644 index 000000000..8ef3863e1 --- /dev/null +++ b/grafana-alertcheck/internal/gate/log_test.go @@ -0,0 +1,694 @@ +package gate + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// Every time literal in this file is UTC and built with time.Date, so it +// carries no monotonic reading and survives a JSON round trip byte-identical — +// which is what lets the round-trip tests below compare whole Poll values +// instead of comparing field by field. +var testNow = time.Date(2026, 8, 31, 9, 0, 0, 0, time.UTC) + +func testInstance(state State, reason, instanceLabel string) Instance { + return Instance{ + Labels: map[string]string{"alertname": "Example", "instance": instanceLabel}, + State: state, + Reason: reason, + ActiveAt: testNow, + } +} + +// observation wraps rules into an Observation with plausible timing numbers, +// deliberately not round millisecond values so a lost conversion at the ms +// boundary shows up as a wrong number rather than a coincidentally equal one. +func observation(grafanaNow time.Time, rules ...StateRule) Observation { + return Observation{ + Rules: rules, + GrafanaNow: grafanaNow, + Skew: 1500 * time.Millisecond, + SkewBound: 40 * time.Millisecond, + Latency: 1800 * time.Millisecond, + } +} + +func TestLogReduceKeepsOnlyAbnormalInstances(t *testing.T) { + rule := StateRule{ + UID: "rule1", Title: "Example", Folder: "F", Group: "G", + Interval: time.Minute, State: "firing", Health: "ok", + LastEvaluation: testNow, Totals: map[string]int{"alerting": 1, "normal": 2}, + Instances: []Instance{ + testInstance(StateNormal, "", "a"), + testInstance(StateFiring, "", "b"), + // Both composites are canonical normal: they must NOT be + // retained as abnormal, and their reasons must still be counted. + testInstance(StateNormal, "NoData", "c"), + testInstance(StateNormal, "Error", "d"), + }, + } + + p := NewReducer().Reduce("rule1", observation(testNow, rule)) + + require.True(t, p.Found) + require.Len(t, p.Abnormal, 1) + require.Equal(t, "b", p.Abnormal[0].Labels["instance"]) + require.Equal(t, map[string]int{"NoData": 1, "Error": 1}, p.Reasons) + // The histogram is a verbatim copy of the response totals — raw keys, no + // normalization. + require.Equal(t, map[string]int{"alerting": 1, "normal": 2}, p.Histogram) + // Rule-level state and health stay raw and unnormalized. + require.Equal(t, "firing", p.State) + require.Equal(t, "ok", p.Health) + require.Equal(t, 1500*time.Millisecond, p.Skew()) + require.Equal(t, 40*time.Millisecond, p.SkewBound()) + require.Equal(t, 1800*time.Millisecond, p.Latency()) + require.Zero(t, p.Reasons["MissingSeries"]) +} + +// A filtered response can hold several rules sharing one title, so the reducer +// must select by UID. +func TestLogReduceSelectsRuleByUID(t *testing.T) { + first := StateRule{UID: "ruleA", Title: "Same Title", Health: "ok", State: "inactive", LastEvaluation: testNow} + second := StateRule{ + UID: "ruleB", Title: "Same Title", Health: "error", State: "firing", LastEvaluation: testNow, + Instances: []Instance{testInstance(StateFiring, "", "x")}, + } + + p := NewReducer().Reduce("ruleB", observation(testNow, first, second)) + + require.Equal(t, "error", p.Health) + require.Len(t, p.Abnormal, 1) +} + +func TestLogReduceRuleAbsentIsAuthoritative(t *testing.T) { + other := StateRule{UID: "other", Title: "Other", Health: "ok", State: "inactive", LastEvaluation: testNow} + + p := NewReducer().Reduce("rule1", observation(testNow, other)) + + require.False(t, p.Found, "a rule absent from an authoritative 2xx") + require.Equal(t, "rule1", p.RuleUID, "an absent rule is still attributed") + // The heartbeat still exists: a not-found poll is evidence that Grafana + // answered at this time, which the coverage proof reads. + require.True(t, p.GrafanaNow.Equal(testNow)) + require.NotZero(t, p.Latency()) + require.Empty(t, p.Health) + require.Nil(t, p.Abnormal) +} + +// An instance that leaves the abnormal set is resolved against the SAME +// response, and MissingSeries is a vanish, never a recovery. +func TestTransitionMarkersClearedVersusVanished(t *testing.T) { + badKey := instanceKey(testInstance(StateFiring, "", "b").Labels) + + cases := []struct { + name string + second []Instance + wantCleared []string + wantVanished []string + }{ + { + name: "present as canonical normal is a clear", + second: []Instance{testInstance(StateNormal, "", "b")}, + wantCleared: []string{badKey}, + }, + { + name: "fully absent is a discontinuity", + second: nil, + wantVanished: []string{badKey}, + }, + { + name: "Normal (MissingSeries) is the vanish in disguise", + second: []Instance{testInstance(StateNormal, "MissingSeries", "b")}, + wantVanished: []string{badKey}, + }, + { + name: "a comma-joined reason naming MissingSeries still vanishes", + second: []Instance{testInstance(StateNormal, "KeepLast, MissingSeries", "b")}, + wantVanished: []string{badKey}, + }, + { + name: "an unrelated comma-joined reason still clears", + second: []Instance{testInstance(StateNormal, "KeepLast, Updated", "b")}, + wantCleared: []string{badKey}, + }, + { + name: "still abnormal is neither", + second: []Instance{testInstance(StateFiring, "", "b")}, + }, + { + name: "abnormal under a different state, then gone, still vanishes", + second: []Instance{testInstance(StateNormal, "", "unrelated")}, + wantVanished: []string{badKey}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + r := NewReducer() + firing := StateRule{ + UID: "rule1", Health: "ok", State: "firing", LastEvaluation: testNow, + Instances: []Instance{testInstance(StateFiring, "", "b")}, + } + first := r.Reduce("rule1", observation(testNow, firing)) + require.Nil(t, first.Cleared, "first poll produced cleared markers with no previous poll") + require.Nil(t, first.Vanished, "first poll produced vanished markers with no previous poll") + + next := StateRule{UID: "rule1", Health: "ok", State: "inactive", LastEvaluation: testNow, Instances: c.second} + p := r.Reduce("rule1", observation(testNow.Add(30*time.Second), next)) + + require.Equal(t, c.wantCleared, p.Cleared) + require.Equal(t, c.wantVanished, p.Vanished) + }) + } +} + +// A departed key must be resolved against the response the reducer is holding, +// not against a later one — so a rule that goes absent and comes back with the +// instance missing still reports the vanish rather than losing it. +func TestTransitionMarkersSurviveAnAbsentPoll(t *testing.T) { + r := NewReducer() + firing := StateRule{ + UID: "rule1", Health: "ok", State: "firing", LastEvaluation: testNow, + Instances: []Instance{testInstance(StateFiring, "", "b")}, + } + r.Reduce("rule1", observation(testNow, firing)) + + absent := r.Reduce("rule1", observation(testNow.Add(30*time.Second))) + require.Nil(t, absent.Vanished, "an absent rule produced vanished markers") + require.Nil(t, absent.Cleared, "an absent rule produced cleared markers") + + back := StateRule{UID: "rule1", Health: "ok", State: "inactive", LastEvaluation: testNow} + p := r.Reduce("rule1", observation(testNow.Add(60*time.Second), back)) + require.Len(t, p.Vanished, 1, "want the instance that disappeared across the absent poll") +} + +func TestTransitionMarkersAreSortedAndPerRule(t *testing.T) { + r := NewReducer() + ruleOne := StateRule{ + UID: "rule1", Health: "ok", State: "firing", LastEvaluation: testNow, + Instances: []Instance{ + testInstance(StateFiring, "", "z"), + testInstance(StateFiring, "", "a"), + testInstance(StateFiring, "", "m"), + }, + } + ruleTwo := StateRule{ + UID: "rule2", Health: "ok", State: "firing", LastEvaluation: testNow, + Instances: []Instance{testInstance(StateFiring, "", "q")}, + } + r.Reduce("rule1", observation(testNow, ruleOne, ruleTwo)) + r.Reduce("rule2", observation(testNow, ruleOne, ruleTwo)) + + clearedOne := StateRule{UID: "rule1", Health: "ok", State: "inactive", LastEvaluation: testNow} + p := r.Reduce("rule1", observation(testNow.Add(time.Minute), clearedOne, ruleTwo)) + require.Len(t, p.Vanished, 3) + for i := 1; i < len(p.Vanished); i++ { + require.Less(t, p.Vanished[i-1], p.Vanished[i], "Vanished is not sorted: %q", p.Vanished) + } + // rule2's own abnormal set is untouched by rule1's transitions. + q := r.Reduce("rule2", observation(testNow.Add(time.Minute), clearedOne, ruleTwo)) + require.Nil(t, q.Cleared, "rule2 picked up rule1's transitions") + require.Nil(t, q.Vanished, "rule2 picked up rule1's transitions") +} + +// The reduction depends on the state endpoint returning normal instances. If it +// ever stops, that must fail loudly at start, never be assumed. +func TestLogVerifyNormalInstancesVisible(t *testing.T) { + cases := []struct { + fixture string + wantError bool + }{ + {"state_one_instance.json", false}, + {"state_reason_composite.json", false}, // composites plus one plain Normal + {"state_paused.json", false}, // no totals, no instances + {"state_missing_optional.json", false}, // no totals key at all + {"state_health_error.json", false}, // totals {"error":1} claims no normal + {"state_only_active_instances.json", true}, + } + + for _, c := range cases { + t.Run(c.fixture, func(t *testing.T) { + rules, err := ParseState(readFixture(t, c.fixture)) + require.NoError(t, err) + err = VerifyNormalInstancesVisible(rules) + if c.wantError { + require.Error(t, err) + require.Contains(t, err.Error(), "no longer returns normal instances") + return + } + require.NoError(t, err) + }) + } +} + +// The totals vocabulary is mixed and its case has drifted, so the check sums +// every key that lowercases to normal or inactive rather than indexing one +// literal key. +func TestLogVerifyNormalInstancesVisibleVocabularies(t *testing.T) { + cases := []struct { + name string + totals map[string]int + wantError bool + }{ + {"lowercase normal", map[string]int{"alerting": 1, "normal": 4}, true}, + {"capitalized Normal", map[string]int{"Alerting": 1, "Normal": 4}, true}, + {"rule vocabulary inactive", map[string]int{"firing": 2, "inactive": 363}, true}, + {"no normal claimed", map[string]int{"alerting": 1}, false}, + {"nil totals", nil, false}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + rules := []StateRule{{ + UID: "rule1", Title: "Example", Totals: c.totals, + Instances: []Instance{testInstance(StateFiring, "", "b")}, + }} + err := VerifyNormalInstancesVisible(rules) + require.Equal(t, c.wantError, err != nil) + }) + } +} + +// The two authorities: the header owns the recording facts (the cadence +// actually used), the ruler API owns the rule facts. Mixing them up is +// fail-open in the faster-override direction, so this pins both. +func TestLogModeCadenceComesFromTheHeader(t *testing.T) { + defs := []Definition{{UID: "rule1", Title: "Example", IntervalSeconds: 300, For: time.Minute}} + + h := testHeader() + h.Rules[0].IntervalSeconds = 300 + h.Rules[0].PollEverySeconds = 5 // an operator override far tighter than the default 150s + + rt, _, err := DeriveTimingsFromLog(h, defs) + require.NoError(t, err) + got := rt["rule1"] + require.Equal(t, 5*time.Second, got.pollEvery, "the header's 5s, not the default 150s") + // maxGap and healthGrace follow the recorded cadence; without this a 250s + // hole in a log recorded at 5s would pass silently. + require.Equal(t, 10*time.Second, got.maxGap) + require.Equal(t, 300*time.Second, got.healthGrace) + // evalStaleAfter is a rule fact, so it stays 2 x intervalSeconds from the + // definitions regardless of how often the gate polled. + require.Equal(t, 600*time.Second, got.evalStaleAfter) + + // A log that cannot say how often it was written cannot have its coverage + // proved, and neither can one naming a rule that no longer resolves. + missingCadence := testHeader() + missingCadence.Rules[0].PollEverySeconds = 0 + require.Error(t, func() error { _, _, err := DeriveTimingsFromLog(missingCadence, defs); return err }(), + "a header with no recorded cadence was accepted") + require.Error(t, func() error { _, _, err := DeriveTimingsFromLog(testHeader(), nil); return err }(), + "a header naming an unresolvable rule was accepted") + + // A duplicated UID must not resolve last-one-wins: the slower duplicate + // would widen maxGap, which is fail-open through log corruption alone. + duplicated := testHeader() + slower := duplicated.Rules[0] + slower.PollEverySeconds = 600 + duplicated.Rules = append(duplicated.Rules, slower) + require.Error(t, func() error { _, _, err := DeriveTimingsFromLog(duplicated, defs); return err }(), + "a header naming one rule twice was accepted") +} + +// watch polls a fleet concurrently through one Reducer, so the marker +// state it holds per rule must be safe under -race — a latent data race here +// surfaces as a wrong transition, which is the one thing markers exist to get +// right. +func TestLogReduceIsSafeForConcurrentUse(t *testing.T) { + r := NewReducer() + rules := make([]StateRule, 0, 8) + for i := range 8 { + rules = append(rules, StateRule{ + UID: fmt.Sprintf("rule%d", i), Health: "ok", State: "firing", LastEvaluation: testNow, + Instances: []Instance{testInstance(StateFiring, "", "b")}, + }) + } + obs := observation(testNow, rules...) + + var wg sync.WaitGroup + for range 3 { + for _, rule := range rules { + wg.Go(func() { r.Reduce(rule.UID, obs) }) + } + } + wg.Wait() + + // Each rule's abnormal instance never left, so no round may invent a + // transition — the concurrency must not corrupt the per-rule state either. + for _, rule := range rules { + p := r.Reduce(rule.UID, obs) + require.Nilf(t, p.Cleared, "rule %s: cleared markers after concurrent reduction", rule.UID) + require.Nilf(t, p.Vanished, "rule %s: vanished markers after concurrent reduction", rule.UID) + } +} + +// A not-found poll has no evaluation time, and the artifact is read by humans +// and jq — the zero time must not appear as though it were real. +func TestLogPollOmitsTheZeroEvaluationTime(t *testing.T) { + absent := NewReducer().Reduce("rule1", observation(testNow)) + b, err := json.Marshal(pollRecord{Type: RecordPoll, Poll: absent}) + require.NoError(t, err) + require.NotContains(t, string(b), "0001-01-01") + require.NotContains(t, string(b), "last_evaluation") + + // A real evaluation time still round-trips. + found := StateRule{UID: "rule1", Health: "ok", State: "inactive", LastEvaluation: testNow} + p := NewReducer().Reduce("rule1", observation(testNow, found)) + b, err = json.Marshal(pollRecord{Type: RecordPoll, Poll: p}) + require.NoError(t, err) + var back pollRecord + require.NoError(t, json.Unmarshal(b, &back)) + require.True(t, back.LastEvaluation.Equal(testNow)) +} + +func testHeader() Header { + return Header{ + URL: "https://grafana.example.com", + GrafanaVersion: "13.1.0", + StartedAt: testNow, + Rules: []LoggedRule{{ + UID: "rule1", Title: "Example", Folder: "F", Group: "G", + ForSeconds: 300, IntervalSeconds: 60, NoDataState: "OK", ExecErrState: "OK", + PollEverySeconds: 30, + }}, + } +} + +func newTestWriter(t *testing.T, path string) (*Writer, *fakeClock) { + t.Helper() + clock := newFakeClock(testNow) + w, err := NewWriter(path, clock) + require.NoError(t, err) + return w, clock +} + +func TestWriterReadLogRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + w, clock := newTestWriter(t, path) + + h := testHeader() + require.NoError(t, w.WriteHeader(h)) + + r := NewReducer() + firing := StateRule{ + UID: "rule1", Health: "ok", State: "firing", LastError: "", LastEvaluation: testNow, + Totals: map[string]int{"alerting": 1}, + Instances: []Instance{testInstance(StateFiring, "", "b")}, + } + cleared := StateRule{UID: "rule1", Health: "ok", State: "inactive", LastEvaluation: testNow.Add(time.Minute)} + want := []Poll{ + r.Reduce("rule1", observation(testNow, firing)), + r.Reduce("rule1", observation(testNow.Add(time.Minute), cleared)), + } + for _, p := range want { + require.NoError(t, w.WritePoll(p)) + } + + clock.Advance(2 * time.Minute) + require.NoError(t, w.Stop()) + + gotHeader, gotPolls, sentinel, err := ReadLog(path) + require.NoError(t, err) + h.SchemaVersion = LogSchemaVersion // WriteHeader stamps it + require.Equal(t, h, gotHeader, "header round trip") + require.Equal(t, want, gotPolls, "poll round trip") + require.NotNil(t, sentinel, "sentinel is nil after Stop") + // Stop stamps the recorder's own stop time and makes no comparison + // against `to` — watch never knows it. + require.True(t, sentinel.Equal(testNow.Add(2*time.Minute))) +} + +// The log is append-only. A second run against the same path must never +// destroy the evidence the first one recorded. +func TestWriterAppendsAndNeverTruncates(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + w, _ := newTestWriter(t, path) + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow})) + require.NoError(t, w.Close()) + before, err := os.ReadFile(path) + require.NoError(t, err) + + // The handoff: the parent wrote the header and closed; the child + // reopens the same path and appends without a second header. + child, _ := newTestWriter(t, path) + require.NoError(t, child.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow.Add(time.Minute)})) + require.NoError(t, child.Stop()) + + after, err := os.ReadFile(path) + require.NoError(t, err) + require.True(t, strings.HasPrefix(string(after), string(before)), "reopening the log rewrote earlier records") + _, polls, sentinel, err := ReadLog(path) + require.NoError(t, err) + require.Len(t, polls, 2) + require.NotNil(t, sentinel) +} + +// Two recorders on one log means one of them is recording a window nobody +// will classify, so the second writer fails immediately — it never blocks. +func TestWriterSecondWriterFails(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + first, _ := newTestWriter(t, path) + defer first.Close() + + done := make(chan error, 1) + go func() { + _, err := NewWriter(path, newFakeClock(testNow)) + done <- err + }() + + select { + case err := <-done: + require.Error(t, err, "a second writer took the lock") + require.Contains(t, err.Error(), "another writer") + case <-time.After(5 * time.Second): + require.Fail(t, "the second NewWriter blocked instead of failing immediately") + } +} + +func TestWriterHeaderRefusesANonEmptyLog(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + w, _ := newTestWriter(t, path) + require.NoError(t, w.WriteHeader(testHeader())) + require.Error(t, w.WriteHeader(testHeader()), "a second header was accepted") + require.NoError(t, w.Close()) + + reopened, _ := newTestWriter(t, path) + defer reopened.Close() + require.Error(t, reopened.WriteHeader(testHeader()), "a header was accepted on a non-empty log") +} + +func TestSentinelStopIsIdempotentAndLast(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + w, _ := newTestWriter(t, path) + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.Stop()) + // watch reaches Stop from both a signal handler and a defer; a second + // sentinel would be indistinguishable from a second writer. + require.NoError(t, w.Stop()) + // Nothing may be appended after the sentinel — not even by the same writer. + require.Error(t, w.WritePoll(Poll{RuleUID: "rule1"}), "WritePoll after Stop was accepted") + + b, err := os.ReadFile(path) + require.NoError(t, err) + lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n") + require.Len(t, lines, 2, "header + one sentinel") + require.Contains(t, lines[1], `"type":"stopped"`) +} + +// Close is the parent's handoff path: a sentinel there would tell check the +// recording ended before the child had even started. +func TestSentinelCloseWritesNone(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + w, _ := newTestWriter(t, path) + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.Close()) + + _, polls, sentinel, err := ReadLog(path) + require.NoError(t, err) + require.Nil(t, sentinel, "Close wrote a sentinel") + require.Nil(t, polls) +} + +// An unfinished recording reads cleanly with a nil sentinel — ReadLog reports +// the absence and the coverage proof turns it into unobservable. It is never +// ReadLog's job to call that a failure, and never anyone's job to call it a +// pass. +func TestReadLogWithoutASentinel(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + w, _ := newTestWriter(t, path) + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow})) + require.NoError(t, w.Close()) + + _, polls, sentinel, err := ReadLog(path) + require.NoError(t, err) + require.Nil(t, sentinel) + require.Len(t, polls, 1) +} + +// The read rules are deliberately the crudest possible: any unparseable +// line is an error, full stop — including the last one, and including a last +// line that follows a sentinel. +func TestReadLogRejectsBadLogs(t *testing.T) { + header := func(version int) string { + h := testHeader() + h.SchemaVersion = version + b, err := json.Marshal(headerRecord{Type: RecordHeader, Header: h}) + require.NoError(t, err, "marshal header") + return string(b) + } + poll := func() string { + b, err := json.Marshal(pollRecord{Type: RecordPoll, Poll: Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow}}) + require.NoError(t, err, "marshal poll") + return string(b) + } + sentinel := func() string { + b, err := json.Marshal(stoppedRecord{Type: RecordStopped, At: testNow}) + require.NoError(t, err, "marshal sentinel") + return string(b) + } + + cases := []struct { + name string + content string + wantIn string + }{ + {"empty file", "", "empty"}, + {"no header", poll() + "\n", "header must be line 1"}, + {"header not first", poll() + "\n" + header(LogSchemaVersion) + "\n", "header must be line 1"}, + {"second header", header(LogSchemaVersion) + "\n" + header(LogSchemaVersion) + "\n", "appear only once"}, + {"wrong schema version", header(2) + "\n" + poll() + "\n", "schema version"}, + { + name: "unparseable last line", + content: header(LogSchemaVersion) + "\n" + poll() + "\n" + `{"type":"poll","rule_ui`, + wantIn: "unparseable", + }, + { + // A preceding sentinel makes no difference: a truncated tail is + // evidence that something killed the recorder. + name: "unparseable line after the sentinel", + content: header(LogSchemaVersion) + "\n" + sentinel() + "\n" + `{"type":"pol`, + wantIn: "unparseable", + }, + { + name: "unparseable middle line", + content: header(LogSchemaVersion) + "\n" + `{"type":` + "\n" + poll() + "\n", + wantIn: "unparseable", + }, + { + name: "empty middle line", + content: header(LogSchemaVersion) + "\n\n" + poll() + "\n", + wantIn: "unparseable", + }, + { + name: "a record after the sentinel", + content: header(LogSchemaVersion) + "\n" + sentinel() + "\n" + poll() + "\n", + wantIn: "second writer", + }, + { + name: "unknown record type", + content: header(LogSchemaVersion) + "\n" + `{"type":"heartbeat"}` + "\n", + wantIn: "unknown record type", + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + require.NoError(t, os.WriteFile(path, []byte(c.content), 0o600)) + _, _, _, err := ReadLog(path) + require.Error(t, err) + require.Contains(t, err.Error(), c.wantIn) + }) + } +} + +func TestReadLogMissingFile(t *testing.T) { + _, _, _, err := ReadLog(filepath.Join(t.TempDir(), "absent.jsonl")) + require.Error(t, err) +} + +// Per-poll log size must not grow across polls on a high-cardinality +// rule, and the one firing instance among 2446 must still be attributed by its +// labels. The reduction makes size independent of NORMAL cardinality — the +// firing instances are still stored, which is why a clear shrinks the record. +func TestLogSizeIsFlatAcrossPollsOnAHighCardinalityRule(t *testing.T) { + body := synthesizeHighCardinalityState(t, 1, 2445) + rules, err := ParseState(body) + require.NoError(t, err) + require.Len(t, rules[0].Instances, 2446) + uid := rules[0].UID + + path := filepath.Join(t.TempDir(), "log.jsonl") + w, _ := newTestWriter(t, path) + require.NoError(t, w.WriteHeader(testHeader())) + + r := NewReducer() + var sizes []int64 + // Measure from the end of the header line, so sizes[0] is the first poll + // record alone rather than the header plus it. + info, err := os.Stat(path) + require.NoError(t, err) + previous := info.Size() + for i := range 5 { + p := r.Reduce(uid, observation(testNow.Add(time.Duration(i)*30*time.Second), rules[0])) + require.Lenf(t, p.Abnormal, 1, "poll %d", i) + require.Equalf(t, "alerting-0", p.Abnormal[0].Labels["instance"], "poll %d: the firing instance lost its identity", i) + require.NoError(t, w.WritePoll(p)) + info, err := os.Stat(path) + require.NoError(t, err) + sizes = append(sizes, info.Size()-previous) + previous = info.Size() + } + + for i := 1; i < len(sizes); i++ { + require.Equal(t, sizes[0], sizes[i], "per-poll size grew across polls: %v", sizes) + } + // One firing instance among 2446 costs a few hundred bytes, against the + // ~600 KB the unreduced response carries. + require.LessOrEqual(t, sizes[0], int64(2048)) + + // When the firing instance clears, the record collapses further and the + // transition is still attributed. + rules[0].Instances[0].State = StateNormal + p := r.Reduce(uid, observation(testNow.Add(5*30*time.Second), rules[0])) + require.Len(t, p.Cleared, 1) + require.Empty(t, p.Abnormal) + require.NoError(t, w.Stop()) +} + +// The log must stay readable by anything that reads JSONL, one flat object per +// line with its type tag — an uploaded artifact is read by humans and by jq, +// not only by ReadLog. +func TestLogRecordsAreFlatOneLineObjects(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + w, _ := newTestWriter(t, path) + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow})) + require.NoError(t, w.Stop()) + + b, err := os.ReadFile(path) + require.NoError(t, err) + lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n") + wantTypes := []RecordType{RecordHeader, RecordPoll, RecordStopped} + require.Len(t, lines, len(wantTypes)) + for i, line := range lines { + var m map[string]json.RawMessage + require.NoErrorf(t, json.Unmarshal([]byte(line), &m), "line %d is not one JSON object", i+1) + var gotType RecordType + require.NoErrorf(t, json.Unmarshal(m["type"], &gotType), "line %d has no type tag", i+1) + require.Equalf(t, wantTypes[i], gotType, "line %d type", i+1) + _, nested := m["header"] + require.Falsef(t, nested, "line %d wraps its payload instead of being flat", i+1) + } +} diff --git a/grafana-alertcheck/internal/gate/parse_ruler.go b/grafana-alertcheck/internal/gate/parse_ruler.go index c27e5b9c4..5517fcce2 100644 --- a/grafana-alertcheck/internal/gate/parse_ruler.go +++ b/grafana-alertcheck/internal/gate/parse_ruler.go @@ -7,9 +7,9 @@ import ( "time" ) -// RuleKind classifies a ruler-endpoint rule by shape, not by name (P1.3). -// P3 rejects KindDatasourceManaged and KindRecording, but only for rules a -// user actually named — ParseDefinitions itself never rejects. +// RuleKind classifies a ruler-endpoint rule by shape, not by name. Resolve +// rejects KindDatasourceManaged and KindRecording, but only for rules a user +// actually named — ParseDefinitions itself never rejects. type RuleKind int const ( @@ -22,8 +22,8 @@ const ( // (/api/ruler/grafana/api/v1/rules). IntervalSeconds, NoDataState and // ExecErrState live inside the grafana_alert block and are only populated for // KindGrafanaManaged — a datasource-managed rule has no such block by -// definition (§11.6 drops relativeTimeRange/keep_firing_for entirely; neither -// is parsed here). +// definition. relativeTimeRange and keep_firing_for are deliberately not +// parsed: nothing in the gate reads them. type Definition struct { UID, Title, Folder, FolderUID, Group string For time.Duration @@ -43,8 +43,8 @@ func ParseDefinitions(body []byte) ([]Definition, error) { } // Map iteration order is nondeterministic; sort namespace names so - // ParseDefinitions' output order is stable across calls (P3's candidate - // listings and any golden test depend on that). + // ParseDefinitions' output order is stable across calls — Resolve's + // candidate listings and the golden tests depend on that. names := make([]string, 0, len(namespaces)) for name := range namespaces { names = append(names, name) @@ -84,7 +84,7 @@ func ParseDefinitions(body []byte) ([]Definition, error) { func parseDefinition(raw json.RawMessage, folder, group string) (Definition, error) { var m map[string]json.RawMessage if err := json.Unmarshal(raw, &m); err != nil { - return Definition{}, fmt.Errorf("%w", err) + return Definition{}, err } var forStr string @@ -137,11 +137,11 @@ func parseDefinition(raw json.RawMessage, folder, group string) (Definition, err // Classify by the presence of "record" before requiring anything else. // no_data_state/exec_err_state/is_paused/intervalSeconds are alerting-only // concepts a recording rule may not carry at all — its real shape is - // unverified (none exist in the fleet capture) — and P3 refuses this - // Kind categorically before any of this would gate a release. Strict- - // parsing a recording rule into a hard error over fields it was never - // going to use would brick `list` and every resolve for rules nobody - // named (§11.6, "do not reject here"). + // unverified, none exist in the fleet capture — and Resolve refuses this + // Kind categorically before any of this would gate a release. + // Strict-parsing a recording rule into a hard error over fields it was + // never going to use would brick `list` and every resolve for rules nobody + // named. var record json.RawMessage if err := opt(ga, "record", &record); err != nil { return Definition{}, fmt.Errorf("rule %q: grafana_alert: %w", uid, err) diff --git a/grafana-alertcheck/internal/gate/parse_ruler_test.go b/grafana-alertcheck/internal/gate/parse_ruler_test.go index dda66b77e..280dc9910 100644 --- a/grafana-alertcheck/internal/gate/parse_ruler_test.go +++ b/grafana-alertcheck/internal/gate/parse_ruler_test.go @@ -3,125 +3,88 @@ package gate import ( "testing" "time" + + "github.com/stretchr/testify/require" ) func TestParseDefinitions_RulerRules(t *testing.T) { defs, err := ParseDefinitions(readFixture(t, "ruler_rules.json")) - if err != nil { - t.Fatalf("ParseDefinitions: unexpected error: %v", err) - } + require.NoError(t, err) byUID := map[string]Definition{} for _, d := range defs { - if d.Kind != KindGrafanaManaged { - t.Errorf("rule %q: Kind = %v, want KindGrafanaManaged", d.UID, d.Kind) - } + require.Equalf(t, KindGrafanaManaged, d.Kind, "rule %q: Kind", d.UID) byUID[d.UID] = d } // The real 2-way duplicate title: same folder, same group, same title, - // distinct UIDs (§17, §22.2). + // distinct UIDs — only uid: can tell them apart. a, ok := byUID["rule0000006a"] - if !ok { - t.Fatalf("missing rule0000006a") - } + require.True(t, ok, "missing rule0000006a") b, ok := byUID["rule0000006b"] - if !ok { - t.Fatalf("missing rule0000006b") - } - if a.Title != b.Title || a.Folder != b.Folder || a.Group != b.Group { - t.Errorf("duplicate-title pair should share Title/Folder/Group: a=%+v b=%+v", a, b) - } - if a.UID == b.UID { - t.Errorf("duplicate-title pair should have distinct UIDs") - } + require.True(t, ok, "missing rule0000006b") + require.Equal(t, a.Title, b.Title, "duplicate-title pair should share Title") + require.Equal(t, a.Folder, b.Folder, "duplicate-title pair should share Folder") + require.Equal(t, a.Group, b.Group, "duplicate-title pair should share Group") + require.NotEqual(t, a.UID, b.UID, "duplicate-title pair should have distinct UIDs") // The 3 real paused rules. pausedUIDs := []string{"rule0000002", "rule0000007", "rule0000008"} for _, uid := range pausedUIDs { d, ok := byUID[uid] - if !ok { - t.Fatalf("missing paused rule %q", uid) - } - if !d.IsPaused { - t.Errorf("rule %q: IsPaused = false, want true", uid) - } + require.True(t, ok, "missing paused rule %q", uid) + require.Truef(t, d.IsPaused, "rule %q: IsPaused = false, want true", uid) } // for:1d and the derived for:1w rule. dayRule, ok := byUID["rule0000009"] - if !ok || dayRule.For != 24*time.Hour { - t.Fatalf("rule0000009: For = %v, want 24h (ok=%v)", dayRule.For, ok) - } + require.Truef(t, ok, "missing rule0000009") + require.Equal(t, 24*time.Hour, dayRule.For) weekRule, ok := byUID["rule0000010"] - if !ok || weekRule.For != 7*24*time.Hour { - t.Fatalf("rule0000010: For = %v, want 168h (ok=%v)", weekRule.For, ok) - } + require.Truef(t, ok, "missing rule0000010") + require.Equal(t, 7*24*time.Hour, weekRule.For) // Identity shared with testdata/state_paused.json. shared := byUID["rule0000002"] - if shared.FolderUID != "folder0000002" { - t.Errorf("rule0000002: FolderUID = %q, want folder0000002", shared.FolderUID) - } + require.Equal(t, "folder0000002", shared.FolderUID) } func TestParseDefinitions_DatasourceManaged(t *testing.T) { defs, err := ParseDefinitions(readFixture(t, "ruler_datasource_managed.json")) - if err != nil { - t.Fatalf("ParseDefinitions: unexpected error: %v", err) - } - if len(defs) != 1 { - t.Fatalf("got %d definitions, want 1", len(defs)) - } - if defs[0].Kind != KindDatasourceManaged { - t.Errorf("Kind = %v, want KindDatasourceManaged", defs[0].Kind) - } - if defs[0].For != 5*time.Minute { - t.Errorf("For = %v, want 5m", defs[0].For) - } + require.NoError(t, err) + require.Len(t, defs, 1) + require.Equal(t, KindDatasourceManaged, defs[0].Kind) + require.Equal(t, 5*time.Minute, defs[0].For) // A datasource-managed rule has no uid in this shape; its only identity // is the Prometheus "alert" name — a synthetic UID would be invented - // shape, and an empty Title would make P3's refusal-by-name unreachable. - if defs[0].Title != "ExampleTargetDown" { - t.Errorf("Title = %q, want ExampleTargetDown", defs[0].Title) - } - if defs[0].UID != "" { - t.Errorf("UID = %q, want empty (this shape has no uid)", defs[0].UID) - } + // shape, and an empty Title would make Resolve's refusal-by-name + // unreachable. + require.Equal(t, "ExampleTargetDown", defs[0].Title) + require.Empty(t, defs[0].UID, "this shape has no uid") } func TestParseDefinitions_Recording(t *testing.T) { defs, err := ParseDefinitions(readFixture(t, "ruler_recording.json")) - if err != nil { - t.Fatalf("ParseDefinitions: unexpected error: %v", err) - } - if len(defs) != 1 { - t.Fatalf("got %d definitions, want 1", len(defs)) - } + require.NoError(t, err) + require.Len(t, defs, 1) d := defs[0] - if d.Kind != KindRecording { - t.Errorf("Kind = %v, want KindRecording", d.Kind) - } - if d.UID != "rule0000011" { - t.Errorf("UID = %q, want rule0000011", d.UID) - } + require.Equal(t, KindRecording, d.Kind) + require.Equal(t, "rule0000011", d.UID) // The fixture deliberately omits no_data_state/exec_err_state/is_paused/ // intervalSeconds/namespace_uid — alerting-only concepts a recording // rule may not carry. Requiring them would brick ParseDefinitions for // every named rule in the same response over one recording rule // elsewhere in the fleet; they must come back as zero values, not errors. - if d.NoDataState != "" || d.ExecErrState != "" || d.IsPaused || d.IntervalSeconds != 0 || d.FolderUID != "" { - t.Errorf("expected zero-valued alert-only fields for a recording rule, got %+v", d) - } + require.Empty(t, d.NoDataState) + require.Empty(t, d.ExecErrState) + require.False(t, d.IsPaused) + require.Zero(t, d.IntervalSeconds) + require.Empty(t, d.FolderUID) } -// A datasource-managed rule with neither an "alert" nor a "record" name has no -// identity (its only name is the Prometheus rule name), and an empty Title -// would make P3's refusal-by-name unreachable. It must fail parsing, not hand -// back a silently unusable Definition. +// A datasource-managed rule with no alert/record name must fail parsing. func TestParseDefinitions_DatasourceManagedNoName(t *testing.T) { body := []byte(`{"ExampleMetrics":[{"name":"g","rules":[{"expr":"up == 0","for":"5m"}]}]}`) - if _, err := ParseDefinitions(body); err == nil { - t.Fatalf("ParseDefinitions: expected error for datasource-managed rule with no alert/record, got nil") - } + _, err := ParseDefinitions(body) + require.Error(t, err, "a datasource-managed rule with no alert/record must fail") } diff --git a/grafana-alertcheck/internal/gate/parse_state.go b/grafana-alertcheck/internal/gate/parse_state.go index d49dca0fa..7c35c7e7d 100644 --- a/grafana-alertcheck/internal/gate/parse_state.go +++ b/grafana-alertcheck/internal/gate/parse_state.go @@ -7,7 +7,7 @@ import ( "time" ) -// State is the canonical instance state (P1.2a). It is distinct from the raw, +// State is the canonical instance state. It is distinct from the raw, // unnormalized vocabularies the API uses at the rule level and at the instance // level — see normalizeInstanceState. type State string @@ -22,23 +22,27 @@ const ( // Instance is one entry of a rule's alerts[]. State is always canonical; Reason // is the opaque suffix of a "State (Reason)" composite ("" when the API gave a -// bare state). Reason is reporting-only except for the H2 MissingSeries routing +// bare state). Reason is reporting-only except for the MissingSeries routing // done downstream in the log markers. +// +// The json tags are for the JSONL log's abnormal-instance list only — parsing +// an API response never goes through them, because parseInstance decodes field +// by field through req/opt to keep the presence checks explicit. type Instance struct { - Labels map[string]string - State State - Reason string - ActiveAt time.Time - Value string + Labels map[string]string `json:"labels"` + State State `json:"state"` + Reason string `json:"reason,omitempty"` + ActiveAt time.Time `json:"active_at"` + Value string `json:"value,omitempty"` } // StateRule is one rule from the state endpoint -// (/api/prometheus/grafana/api/v1/rules), fully and strictly parsed (H1). +// (/api/prometheus/grafana/api/v1/rules), fully and strictly parsed. type StateRule struct { UID, Title, Folder, Group string Interval time.Duration // State and Health are raw, lowercase, and reporting-only — never - // classified (P1.2a). State in particular is never normalized. + // classified. State in particular is never normalized. State, Health string LastError string LastEvaluation time.Time @@ -49,7 +53,7 @@ type StateRule struct { // ParseState strictly parses a state-endpoint response body into its rules. // A missing or unparseable required field (health, state, lastEvaluation on -// each rule; interval on each group) is an error, never a zero value (H1). +// each rule; interval on each group) is an error, never a zero value. func ParseState(body []byte) ([]StateRule, error) { var top map[string]json.RawMessage if err := json.Unmarshal(body, &top); err != nil { @@ -129,11 +133,10 @@ func parseStateRule(raw json.RawMessage, folder, group string, interval time.Dur if err := req(m, "health", &r.Health); err != nil { return StateRule{}, fmt.Errorf("rule %q: %w", uid, err) } - // isPaused is not one of H1's four named required fields, but this parser - // extends that contract to it: the zero-time rule below can't tell a - // paused rule from a broken one without it, and it's the primary - // in-window pause detector (H2/§12.2) — a silent false default would be - // exactly the fail-open bug H1 exists to kill. + // isPaused is required rather than optional: the zero-time rule below + // can't tell a paused rule from a broken one without it, and it's the + // primary in-window pause detector — a silent false default would be + // exactly the fail-open this parser's strictness exists to kill. if err := req(m, "isPaused", &r.IsPaused); err != nil { return StateRule{}, fmt.Errorf("rule %q: %w", uid, err) } @@ -146,7 +149,7 @@ func parseStateRule(raw json.RawMessage, folder, group string, interval time.Dur if err != nil { return StateRule{}, fmt.Errorf("rule %q: lastEvaluation: %w", uid, err) } - // The zero-time rule (§2.3): only a paused rule may report the zero time. + // Only a paused rule may report the zero time. if lastEval.IsZero() && !r.IsPaused { return StateRule{}, fmt.Errorf("rule %q: lastEvaluation is the zero time but isPaused is false", uid) } @@ -194,10 +197,10 @@ func parseInstance(raw json.RawMessage) (Instance, error) { return Instance{}, err } - // activeAt is also not in H1's named list, extended here for the same - // reason as StateRule.IsPaused: it's the onset time BadFor (P8) measures - // from, so a silently zeroed one would misclassify how long an instance - // has been bad rather than failing loudly. + // activeAt is required for the same reason as StateRule.IsPaused: it's the + // onset time BadFor measures from, so a silently zeroed one would + // misclassify how long an instance has been bad rather than failing + // loudly. var activeAtStr string if err := req(m, "activeAt", &activeAtStr); err != nil { return Instance{}, err @@ -220,8 +223,8 @@ func parseInstance(raw json.RawMessage) (Instance, error) { } // baseInstanceStates is the strict 5-value allowlist for the base of an -// instance state (P1.2a). Anything else — including an unrecognized base -// inside a "Base (Reason)" composite — is a parse error (H1, §2.7 control 3). +// instance state. Anything else — including an unrecognized base inside a +// "Base (Reason)" composite — is a parse error. var baseInstanceStates = map[string]State{ "Normal": StateNormal, "Alerting": StateFiring, diff --git a/grafana-alertcheck/internal/gate/parse_state_test.go b/grafana-alertcheck/internal/gate/parse_state_test.go index e370c0f5e..f55c10185 100644 --- a/grafana-alertcheck/internal/gate/parse_state_test.go +++ b/grafana-alertcheck/internal/gate/parse_state_test.go @@ -1,23 +1,20 @@ package gate import ( - "bytes" "encoding/json" "fmt" - "maps" "os" "path/filepath" - "strings" "testing" "time" + + "github.com/stretchr/testify/require" ) func readFixture(t *testing.T, name string) []byte { t.Helper() b, err := os.ReadFile(filepath.Join("testdata", name)) - if err != nil { - t.Fatalf("reading fixture %s: %v", name, err) - } + require.NoErrorf(t, err, "reading fixture %s", name) return b } @@ -33,24 +30,15 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 1, checkFirst: func(t *testing.T, r StateRule) { - if r.UID != "rule0000001" { - t.Errorf("UID = %q, want rule0000001", r.UID) - } - if r.Folder != "ExampleTeam" || r.Group != "Example Service - Prod" { - t.Errorf("Folder/Group = %q/%q, want ExampleTeam/Example Service - Prod", r.Folder, r.Group) - } - if r.Health != "ok" || r.State != "inactive" { - t.Errorf("Health/State = %q/%q, want ok/inactive", r.Health, r.State) - } - if r.Interval.Seconds() != 60 { - t.Errorf("Interval = %v, want 60s", r.Interval) - } - if r.IsPaused { - t.Errorf("IsPaused = true, want false") - } - if len(r.Instances) != 1 || r.Instances[0].State != StateNormal { - t.Fatalf("Instances = %+v, want one normal instance", r.Instances) - } + require.Equal(t, "rule0000001", r.UID) + require.Equal(t, "ExampleTeam", r.Folder) + require.Equal(t, "Example Service - Prod", r.Group) + require.Equal(t, "ok", r.Health) + require.Equal(t, "inactive", r.State) + require.Equal(t, float64(60), r.Interval.Seconds()) + require.False(t, r.IsPaused) + require.Len(t, r.Instances, 1) + require.Equal(t, StateNormal, r.Instances[0].State) inst := r.Instances[0] wantLabels := map[string]string{ @@ -62,19 +50,11 @@ func TestParseState_HappyPaths(t *testing.T) { "severity": "critical", "team": "example-team", } - if !maps.Equal(inst.Labels, wantLabels) { - t.Errorf("Labels = %+v, want %+v", inst.Labels, wantLabels) - } + require.Equal(t, wantLabels, inst.Labels) wantActiveAt, err := time.Parse(time.RFC3339, "2026-08-31T08:02:50Z") - if err != nil { - t.Fatalf("test setup: %v", err) - } - if !inst.ActiveAt.Equal(wantActiveAt) { - t.Errorf("ActiveAt = %v, want %v", inst.ActiveAt, wantActiveAt) - } - if inst.Value != "" { - t.Errorf("Value = %q, want empty string", inst.Value) - } + require.NoError(t, err, "test setup") + require.True(t, inst.ActiveAt.Equal(wantActiveAt)) + require.Empty(t, inst.Value) }, }, { @@ -82,15 +62,10 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 0, checkFirst: func(t *testing.T, r StateRule) { - if !r.IsPaused { - t.Errorf("IsPaused = false, want true") - } - if !r.LastEvaluation.IsZero() { - t.Errorf("LastEvaluation = %v, want zero time", r.LastEvaluation) - } - if r.Health != "ok" || r.State != "inactive" { - t.Errorf("Health/State = %q/%q, want ok/inactive", r.Health, r.State) - } + require.True(t, r.IsPaused) + require.True(t, r.LastEvaluation.IsZero()) + require.Equal(t, "ok", r.Health) + require.Equal(t, "inactive", r.State) }, }, { @@ -98,15 +73,10 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 1, checkFirst: func(t *testing.T, r StateRule) { - if r.Health != "error" { - t.Errorf("Health = %q, want error", r.Health) - } - if r.LastError == "" { - t.Errorf("LastError is empty, want a message") - } - if len(r.Instances) != 1 || r.Instances[0].State != StateError { - t.Fatalf("Instances = %+v, want one error instance", r.Instances) - } + require.Equal(t, "error", r.Health) + require.NotEmpty(t, r.LastError) + require.Len(t, r.Instances, 1) + require.Equal(t, StateError, r.Instances[0].State) }, }, { @@ -114,12 +84,9 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 1, checkFirst: func(t *testing.T, r StateRule) { - if r.Health != "nodata" { - t.Errorf("Health = %q, want nodata", r.Health) - } - if len(r.Instances) != 1 || r.Instances[0].State != StateNodata { - t.Fatalf("Instances = %+v, want one nodata instance", r.Instances) - } + require.Equal(t, "nodata", r.Health) + require.Len(t, r.Instances, 1) + require.Equal(t, StateNodata, r.Instances[0].State) }, }, { @@ -132,17 +99,14 @@ func TestParseState_HappyPaths(t *testing.T) { byReason[inst.Reason] = inst } errInst, ok := byReason["Error"] - if !ok || errInst.State != StateNormal { - t.Errorf(`want an instance with State=normal Reason="Error", got %+v`, byReason["Error"]) - } + require.True(t, ok, `want an instance with Reason="Error"`) + require.Equal(t, StateNormal, errInst.State) nodataInst, ok := byReason["NoData"] - if !ok || nodataInst.State != StateNormal { - t.Errorf(`want an instance with State=normal Reason="NoData", got %+v`, byReason["NoData"]) - } + require.True(t, ok, `want an instance with Reason="NoData"`) + require.Equal(t, StateNormal, nodataInst.State) plain, ok := byReason[""] - if !ok || plain.State != StateNormal { - t.Errorf(`want a plain State=normal Reason="" instance, got %+v`, byReason[""]) - } + require.True(t, ok, `want a plain Reason="" instance`) + require.Equal(t, StateNormal, plain.State) }, }, { @@ -150,12 +114,8 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 0, checkFirst: func(t *testing.T, r StateRule) { - if r.Instances != nil { - t.Errorf("Instances = %+v, want nil", r.Instances) - } - if r.Totals != nil { - t.Errorf("Totals = %+v, want nil", r.Totals) - } + require.Nil(t, r.Instances) + require.Nil(t, r.Totals) }, }, { @@ -163,12 +123,10 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 1, checkFirst: func(t *testing.T, r StateRule) { - if len(r.Instances) != 1 || r.Instances[0].State != StateFiring { - t.Fatalf("Instances = %+v, want one firing instance", r.Instances) - } - if r.Totals["normal"] == 0 { - t.Errorf(`Totals["normal"] = 0, want >0 (this is the §3.2 mismatch the fixture exists to capture)`) - } + require.Len(t, r.Instances, 1) + require.Equal(t, StateFiring, r.Instances[0].State) + require.NotZero(t, r.Totals["normal"], + "the totals/instances mismatch this fixture exists to capture") }, }, } @@ -176,15 +134,9 @@ func TestParseState_HappyPaths(t *testing.T) { for _, c := range cases { t.Run(c.fixture, func(t *testing.T) { rules, err := ParseState(readFixture(t, c.fixture)) - if err != nil { - t.Fatalf("ParseState(%s): unexpected error: %v", c.fixture, err) - } - if len(rules) != c.wantRules { - t.Fatalf("ParseState(%s): got %d rules, want %d", c.fixture, len(rules), c.wantRules) - } - if got := len(rules[0].Instances); got != c.wantInstances { - t.Fatalf("ParseState(%s): got %d instances, want %d", c.fixture, got, c.wantInstances) - } + require.NoErrorf(t, err, "ParseState(%s)", c.fixture) + require.Lenf(t, rules, c.wantRules, "ParseState(%s)", c.fixture) + require.Lenf(t, rules[0].Instances, c.wantInstances, "ParseState(%s)", c.fixture) if c.checkFirst != nil { c.checkFirst(t, rules[0]) } @@ -192,11 +144,11 @@ func TestParseState_HappyPaths(t *testing.T) { } } -// TestParseState_MustError is the H1 regression suite: it doesn't just check -// err != nil (a stray comma in a fixture would keep that green forever while -// the actual check regressed) — it asserts the error names the specific -// offending field or value, so a real H1 check going missing fails loudly -// here instead of surviving unnoticed. +// The strict-parsing regression suite: it doesn't just check err != nil (a +// stray comma in a fixture would keep that green forever while the actual +// check regressed) — it asserts the error names the specific offending field +// or value, so a check going missing fails loudly here instead of surviving +// unnoticed. func TestParseState_MustError(t *testing.T) { cases := []struct { fixture string @@ -214,13 +166,9 @@ func TestParseState_MustError(t *testing.T) { for _, c := range cases { t.Run(c.fixture, func(t *testing.T) { _, err := ParseState(readFixture(t, c.fixture)) - if err == nil { - t.Fatalf("ParseState(%s): expected an error, got none", c.fixture) - } + require.Errorf(t, err, "ParseState(%s): expected an error, got none", c.fixture) for _, want := range c.wantContains { - if !strings.Contains(err.Error(), want) { - t.Errorf("ParseState(%s): error %q does not mention %q", c.fixture, err.Error(), want) - } + require.Containsf(t, err.Error(), want, "ParseState(%s): error", c.fixture) } }) } @@ -248,84 +196,97 @@ func TestParseNormalizeInstanceState(t *testing.T) { for _, c := range cases { state, reason, err := normalizeInstanceState(c.in) if c.wantErr { - if err == nil { - t.Errorf("normalizeInstanceState(%q): expected an error, got none", c.in) - } + require.Errorf(t, err, "normalizeInstanceState(%q)", c.in) continue } - if err != nil { - t.Errorf("normalizeInstanceState(%q): unexpected error: %v", c.in, err) - continue - } - if state != c.wantState || reason != c.wantReason { - t.Errorf("normalizeInstanceState(%q) = (%q, %q), want (%q, %q)", c.in, state, reason, c.wantState, c.wantReason) - } + require.NoErrorf(t, err, "normalizeInstanceState(%q)", c.in) + require.Equalf(t, c.wantState, state, "normalizeInstanceState(%q)", c.in) + require.Equalf(t, c.wantReason, reason, "normalizeInstanceState(%q)", c.in) } } func TestInstanceKey(t *testing.T) { a := instanceKey(map[string]string{"b": "2", "a": "1"}) b := instanceKey(map[string]string{"a": "1", "b": "2"}) - if a != b { - t.Errorf("instanceKey order-independence: %q != %q", a, b) - } - if a != `{"a":"1","b":"2"}` { - t.Errorf("instanceKey = %q, want %q", a, `{"a":"1","b":"2"}`) - } + require.Equal(t, b, a, "instanceKey order-independence") + require.Equal(t, `{"a":"1","b":"2"}`, a) diff := instanceKey(map[string]string{"a": "1", "b": "3"}) - if a == diff { - t.Errorf("instanceKey should differ when a label value differs") - } + require.NotEqual(t, a, diff, "instanceKey should differ when a label value differs") - if instanceKey(nil) != "null" { - t.Errorf("instanceKey(nil) = %q, want \"null\"", instanceKey(nil)) - } + require.Equal(t, "null", instanceKey(nil), "instanceKey(nil) should be \"null\"") } -// TestInstanceKey_NoCollision guards against ambiguous identities: label -// values may legally contain "\n" or "=", and a naive "k=v\n" join would -// collide e.g. {a:"1\nb=2"} with {a:"1",b:"2"}. The JSON encoding must keep -// such sets distinct. func TestInstanceKey_NoCollision(t *testing.T) { - if instanceKey(map[string]string{"a": "1\nb=2"}) == instanceKey(map[string]string{"a": "1", "b": "2"}) { - t.Errorf("instanceKey collided for sets {a:1\\nb=2} and {a:1,b:2}") + require.NotEqual(t, instanceKey(map[string]string{"a": "1\nb=2"}), instanceKey(map[string]string{"a": "1", "b": "2"}), "instanceKey should not collide for sets {a:1\\nb=2} and {a:1,b:2}") + + require.NotEqual(t, instanceKey(map[string]string{"a": "1=b"}), instanceKey(map[string]string{"a": "1", "b": ""}), "instanceKey should not collide for sets {a:1=b} and {a:1,b:}") +} + +// minimalStateBody is the smallest legal state response: one group, one +// rule, no optional keys at all, plus whatever extra is spliced in verbatim +// before the rule's closing brace — for isolating one optional key at a time +// rather than relying on a fixture that removes several together. +func minimalStateBody(extraRuleJSON string) []byte { + return fmt.Appendf(nil, + `{"status":"success","data":{"groups":[{"file":"F","name":"G","interval":60,`+ + `"rules":[{"uid":"r1","name":"R1","state":"inactive","health":"ok","isPaused":false,`+ + `"lastEvaluation":"2026-01-01T00:00:00Z"%s}]}]}}`, extraRuleJSON) +} + +// keepFiringFor is optional alongside alerts/totals/labels, but +// state_missing_optional.json removes it together with everything else — never +// in isolation, so a regression that made it required specifically would not +// be caught by that fixture alone. +func TestParseState_KeepFiringForIsOptional(t *testing.T) { + tests := []struct { + name string + extra string + }{ + {"present", `,"keepFiringFor":300`}, + {"absent", ""}, } - if instanceKey(map[string]string{"a": "1=b"}) == instanceKey(map[string]string{"a": "1", "b": ""}) { - t.Errorf("instanceKey collided for a value containing '='") + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rules, err := ParseState(minimalStateBody(tc.extra)) + require.NoError(t, err) + require.Len(t, rules, 1) + }) } } +// labels is optional at the INSTANCE level (opt(m, "labels", ...) in +// parseInstance), distinct from the rule-level labels state_missing_optional.json +// already covers — an instance can exist with no labels of its own. +func TestParseState_InstanceWithoutLabelsParses(t *testing.T) { + body := minimalStateBody(`,"alerts":[{"state":"Normal","activeAt":"2026-01-01T00:00:00Z"}]`) + rules, err := ParseState(body) + require.NoError(t, err) + require.Len(t, rules, 1) + require.Len(t, rules[0].Instances, 1) + require.Empty(t, rules[0].Instances[0].Labels) +} + // synthesizeHighCardinalityState builds a state response with a single rule // holding `alerting` Alerting instances and `normal` Normal instances, by // cloning the one real instance in state_one_instance.json. It is never -// committed (§3.2, §22.3, §22.6) — the 2446-instance rule this stands in for -// is ~600 KB and exists only to prove the parser and (in later phases) the -// reducer don't choke on real fleet cardinality. +// committed — the 2446-instance rule this stands in for is ~600 KB and exists +// only to prove the parser and the reducer don't choke on real fleet +// cardinality. func synthesizeHighCardinalityState(t *testing.T, alerting, normal int) []byte { t.Helper() base := readFixture(t, "state_one_instance.json") var top map[string]json.RawMessage - if err := json.Unmarshal(base, &top); err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, json.Unmarshal(base, &top)) var data map[string]json.RawMessage - if err := json.Unmarshal(top["data"], &data); err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, json.Unmarshal(top["data"], &data)) var groups []map[string]json.RawMessage - if err := json.Unmarshal(data["groups"], &groups); err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, json.Unmarshal(data["groups"], &groups)) var rules []map[string]json.RawMessage - if err := json.Unmarshal(groups[0]["rules"], &rules); err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, json.Unmarshal(groups[0]["rules"], &rules)) var alerts []map[string]json.RawMessage - if err := json.Unmarshal(rules[0]["alerts"], &alerts); err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, json.Unmarshal(rules[0]["alerts"], &alerts)) template := alerts[0] newAlerts := make([]map[string]json.RawMessage, 0, alerting+normal) @@ -350,9 +311,7 @@ func synthesizeHighCardinalityState(t *testing.T, alerting, normal int) []byte { top["data"] = mustRaw(t, data) out, err := json.Marshal(top) - if err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, err) return out } @@ -369,29 +328,19 @@ func cloneRawMap(m map[string]json.RawMessage) map[string]json.RawMessage { func mustRaw(t *testing.T, v any) json.RawMessage { t.Helper() b, err := json.Marshal(v) - if err != nil { - t.Fatalf("marshal: %v", err) - } + require.NoError(t, err) return json.RawMessage(b) } func TestParseState_HighCardinality(t *testing.T) { body := synthesizeHighCardinalityState(t, 445, 2004) - if !bytes.Contains(body, []byte("alerting-0")) { - t.Fatalf("synthesized body missing expected content") - } + require.Contains(t, string(body), "alerting-0") rules, err := ParseState(body) - if err != nil { - t.Fatalf("ParseState: unexpected error: %v", err) - } - if len(rules) != 1 { - t.Fatalf("got %d rules, want 1", len(rules)) - } + require.NoError(t, err) + require.Len(t, rules, 1) r := rules[0] - if len(r.Instances) != 445+2004 { - t.Fatalf("got %d instances, want %d", len(r.Instances), 445+2004) - } + require.Len(t, r.Instances, 445+2004) var firing, normal int for _, inst := range r.Instances { @@ -401,28 +350,21 @@ func TestParseState_HighCardinality(t *testing.T) { case StateNormal: normal++ default: - t.Fatalf("unexpected instance state %q", inst.State) + require.Fail(t, fmt.Sprintf("unexpected instance state %q", inst.State)) } } - if firing != 445 || normal != 2004 { - t.Fatalf("got firing=%d normal=%d, want firing=445 normal=2004", firing, normal) - } + require.Equal(t, 445, firing) + require.Equal(t, 2004, normal) // Each synthesized instance carries a distinct "instance" label; confirm // Labels actually made it through parsing (not just State) by checking // instanceKey produces one unique key per instance, with no collisions. seen := make(map[string]bool, len(r.Instances)) for _, inst := range r.Instances { - if inst.Labels == nil { - t.Fatalf("instance has nil Labels") - } + require.NotNil(t, inst.Labels) k := instanceKey(inst.Labels) - if seen[k] { - t.Fatalf("duplicate instance key %q", k) - } + require.Falsef(t, seen[k], "duplicate instance key %q", k) seen[k] = true } - if len(seen) != 445+2004 { - t.Fatalf("got %d unique instance keys, want %d", len(seen), 445+2004) - } + require.Len(t, seen, 445+2004) } diff --git a/grafana-alertcheck/internal/gate/resolve.go b/grafana-alertcheck/internal/gate/resolve.go new file mode 100644 index 000000000..329644f95 --- /dev/null +++ b/grafana-alertcheck/internal/gate/resolve.go @@ -0,0 +1,208 @@ +package gate + +import ( + "fmt" + "slices" + "sort" + "strings" +) + +// Resolve turns the operator-supplied alert names into resolved Definitions. +// Order is load-bearing: +// +// 1. Trim each name. +// 2. Discard empty lines. +// 3. Resolve each name to a UID (this is what resolveOne does). +// 4. Collapse the result by UID — two names hitting the same rule is a note, +// never an error (almost always a copy mistake, and a message costs the +// user less than a failure). +// +// The caller-visible consequence: len(resolved) is the count *after* the +// collapse, and MinObserved must default from that length, never from +// len(names) — using the input line count would make one rule named twice turn +// an achievable default into an unsatisfiable one. +func Resolve(defs []Definition, names []string, folder string) (resolved []Definition, notes []string, err error) { + seenUID := map[string]string{} // uid -> the first input name that resolved to it + for _, raw := range names { + name := strings.TrimSpace(raw) + if name == "" { + continue + } + + def, rerr := resolveOne(defs, name, folder) + if rerr != nil { + return nil, nil, rerr + } + + if firstName, ok := seenUID[def.UID]; ok { + notes = append(notes, fmt.Sprintf( + "%q and %q both resolve to %s (uid:%s); counted once", firstName, name, def.Title, def.UID)) + continue + } + seenUID[def.UID] = name + resolved = append(resolved, def) + } + return resolved, notes, nil +} + +// resolveOne resolves a single trimmed, non-empty name against defs: one match +// wins outright, zero is an error with suggestions, two or more is an error +// listing every candidate. folder scopes a bare title (no "/" in the name) to +// one folder; it is ignored for the "Folder/Title" and "Folder/Group/Title" +// forms, which already name their own folder. +// +// Unsupported kinds (datasource-managed, recording) are refused, and how that +// interacts with the no-match/ambiguous surfaces is decided here: a name can +// still match an unsupported rule (so +// naming one by title still gets the specific, named refusal, not a bare "no +// match"), but only *supported* candidates count for ambiguity — an +// unsupported rule sharing a title with a supported one is resolved silently +// in the supported rule's favor rather than reported as ambiguous — and the +// "%d rules available" count and substring suggestions in a genuine no-match +// are scoped to supported rules only, so an unsupported rule never inflates +// or pollutes either. uid: is always exact regardless of kind (typically +// copy-pasted from `list`, which already shows Kind). +func resolveOne(defs []Definition, name, folder string) (Definition, error) { + if uid, ok := strings.CutPrefix(name, "uid:"); ok { + if uid != "" { + for _, d := range defs { + if d.UID == uid { + return refuseUnsupportedKind(name, d) + } + } + } + // uid == "" falls through to the same message as "not found": several + // Definition kinds legitimately carry UID == "" (datasource-managed + // rules have no uid at all), so matching on an empty suffix + // would silently hit one of those and report a misleading + // kind-specific refusal for what is really an empty/typo'd uid. This + // deliberately does not go through noMatchError: that function's + // substring suggestion would degenerate to an empty needle, which + // strings.Contains matches against every title — printing the whole + // fleet instead of a real suggestion. + return Definition{}, fmt.Errorf("no rule matched %q: no rule has this uid (run 'grafana-alertcheck list' to see uids)", name) + } + + wantFolder, wantGroup, wantTitle, err := classifyForm(name, folder) + if err != nil { + return Definition{}, err + } + + var supportedCandidates, unsupportedCandidates []Definition + for _, d := range defs { + if wantFolder != "" && d.Folder != wantFolder { + continue + } + if wantGroup != "" && d.Group != wantGroup { + continue + } + if d.Title != wantTitle { + continue + } + if d.Kind == KindGrafanaManaged { + supportedCandidates = append(supportedCandidates, d) + } else { + unsupportedCandidates = append(unsupportedCandidates, d) + } + } + + switch { + case len(supportedCandidates) == 1: + return supportedCandidates[0], nil + case len(supportedCandidates) > 1: + return Definition{}, ambiguousError(name, supportedCandidates) + case len(unsupportedCandidates) > 0: + return refuseUnsupportedKind(name, unsupportedCandidates[0]) + default: + return Definition{}, noMatchError(supportedDefs(defs), name, wantTitle) + } +} + +// supportedDefs filters out the two refused kinds. Only these participate in +// name-based matching, the no-match rule count, and substring suggestions (see +// the policy note on resolveOne). +func supportedDefs(defs []Definition) []Definition { + out := make([]Definition, 0, len(defs)) + for _, d := range defs { + if d.Kind == KindGrafanaManaged { + out = append(out, d) + } + } + return out +} + +// classifyForm splits name into the Title | Folder/Title | Folder/Group/Title +// forms. A bare title is scoped by folder when the caller supplied one; +// the two- and three-segment forms already carry their own folder and ignore +// it. +// +// Every segment must be non-empty. Without this, "/Title" would parse as an +// empty wantFolder — silently dropping the folder filter and matching +// unscoped, a fail-open — and "Folder/" would parse as an empty wantTitle, +// which would then feed noMatchError's substring search an empty needle that +// matches every title. +func classifyForm(name, folder string) (wantFolder, wantGroup, wantTitle string, err error) { + parts := strings.Split(name, "/") + if slices.Contains(parts, "") { + return "", "", "", fmt.Errorf("no rule matched %q: empty /-separated segment (want Title, Folder/Title, or Folder/Group/Title)", name) + } + switch len(parts) { + case 1: + return folder, "", parts[0], nil + case 2: + return parts[0], "", parts[1], nil + case 3: + return parts[0], parts[1], parts[2], nil + default: + return "", "", "", fmt.Errorf("no rule matched %q: too many /-separated segments (want Title, Folder/Title, or Folder/Group/Title)", name) + } +} + +// refuseUnsupportedKind rejects the two unsupported kinds with a clear, +// specific error — distinct from "no match" and from "ambiguous" — so +// an operator who names a recording or datasource-managed rule learns why, +// not just that nothing matched. +func refuseUnsupportedKind(name string, d Definition) (Definition, error) { + switch d.Kind { + case KindDatasourceManaged: + return Definition{}, fmt.Errorf("%q resolves to %s, a datasource-managed rule, which is not supported", name, d.Title) + case KindRecording: + return Definition{}, fmt.Errorf("%q resolves to %s, a recording rule, which is not supported", name, d.Title) + default: + return d, nil + } +} + +// noMatchError reports a no-match with the count of rules the gate could see +// and case-insensitive substring matches as suggestions. +func noMatchError(defs []Definition, name, wantTitle string) error { + msg := fmt.Sprintf("no rule matched %q (%d rules available; run 'grafana-alertcheck list' to see titles)", name, len(defs)) + + needle := strings.ToLower(wantTitle) + var subs []string + for _, d := range defs { + if strings.Contains(strings.ToLower(d.Title), needle) { + subs = append(subs, fmt.Sprintf("%s/%s/%s", d.Folder, d.Group, d.Title)) + } + } + if len(subs) > 0 { + sort.Strings(subs) + msg += fmt.Sprintf("; did you mean: %s", strings.Join(subs, ", ")) + } + return fmt.Errorf("%s", msg) +} + +// ambiguousError lists every candidate with its folder, its group, and the +// full copyable Folder/Group/Title — including the uid: form, which resolves +// unambiguously on the next attempt. +func ambiguousError(name string, candidates []Definition) error { + sorted := append([]Definition(nil), candidates...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i].UID < sorted[j].UID }) + + var b strings.Builder + fmt.Fprintf(&b, "%q matches %d rules; use uid: or the full Folder/Group/Title:", name, len(sorted)) + for _, d := range sorted { + fmt.Fprintf(&b, "\n %s/%s/%s (uid:%s)", d.Folder, d.Group, d.Title, d.UID) + } + return fmt.Errorf("%s", b.String()) +} diff --git a/grafana-alertcheck/internal/gate/resolve_test.go b/grafana-alertcheck/internal/gate/resolve_test.go new file mode 100644 index 000000000..33214516e --- /dev/null +++ b/grafana-alertcheck/internal/gate/resolve_test.go @@ -0,0 +1,210 @@ +package gate + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func rulerDefs(t *testing.T) []Definition { + t.Helper() + defs, err := ParseDefinitions(readFixture(t, "ruler_rules.json")) + require.NoError(t, err) + return defs +} + +func TestResolve_SingleMatch(t *testing.T) { + defs := rulerDefs(t) + resolved, notes, err := Resolve(defs, []string{"example_workflow_paused_rule"}, "") + require.NoError(t, err) + require.Empty(t, notes) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000007", resolved[0].UID) +} + +func TestResolve_UIDForm(t *testing.T) { + defs := rulerDefs(t) + resolved, _, err := Resolve(defs, []string{"uid:rule0000006a"}, "") + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000006a", resolved[0].UID) +} + +func TestResolve_FolderTitleForm(t *testing.T) { + defs := rulerDefs(t) + resolved, _, err := Resolve(defs, []string{"ExampleFeeds/TEMP - Example depeg alert"}, "") + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000008", resolved[0].UID) +} + +func TestResolve_FolderGroupTitleForm(t *testing.T) { + defs := rulerDefs(t) + resolved, _, err := Resolve(defs, []string{"Example-Zone-A/Gateway/Example No Gateways Available"}, "") + require.Error(t, err, "want ambiguous error (real 2-way collision), got resolved=%+v", resolved) + require.Contains(t, err.Error(), "matches 2 rules") + require.Contains(t, err.Error(), "uid:rule0000006a") + require.Contains(t, err.Error(), "uid:rule0000006b") +} + +func TestResolve_TrueCollisionResolvesByUID(t *testing.T) { + defs := rulerDefs(t) + resolved, _, err := Resolve(defs, []string{"uid:rule0000006a", "uid:rule0000006b"}, "") + require.NoError(t, err) + require.Len(t, resolved, 2) +} + +func TestResolve_NoMatch(t *testing.T) { + defs := rulerDefs(t) + _, _, err := Resolve(defs, []string{"Does Not Exist"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "no rule matched") + require.Contains(t, err.Error(), "list") +} + +func TestResolve_NoMatchSubstringSuggestion(t *testing.T) { + defs := rulerDefs(t) + _, _, err := Resolve(defs, []string{"paused rule"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "did you mean") + require.Contains(t, err.Error(), "Example Paused Rule") +} + +func TestResolve_RefusesDatasourceManaged(t *testing.T) { + defs, err := ParseDefinitions(readFixture(t, "ruler_datasource_managed.json")) + require.NoError(t, err) + _, _, err = Resolve(defs, []string{"ExampleTargetDown"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "datasource-managed") +} + +func TestResolve_RefusesRecording(t *testing.T) { + defs, err := ParseDefinitions(readFixture(t, "ruler_recording.json")) + require.NoError(t, err) + _, _, err = Resolve(defs, []string{"uid:rule0000011"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "recording rule") +} + +func TestResolve_RejectsEmptySegments(t *testing.T) { + defs := rulerDefs(t) + cases := []string{"/Title", "Folder/", "a//b", "//", "/"} + for _, name := range cases { + t.Run(name, func(t *testing.T) { + _, _, err := Resolve(defs, []string{name}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "empty") + }) + } +} + +func TestResolve_UIDEmptySuffix(t *testing.T) { + // ruler_datasource_managed.json's only rule has UID == "" — that shape has + // no uid at all. "uid:" with an empty suffix must not match it + // — that would report the misleading "datasource-managed rule, not + // supported" for what is really a typo'd/empty uid. + defs, err := ParseDefinitions(readFixture(t, "ruler_datasource_managed.json")) + require.NoError(t, err) + _, _, err = Resolve(defs, []string{"uid:"}, "") + require.Error(t, err) + require.Contains(t, err.Error(), "no rule has this uid") + require.NotContains(t, err.Error(), "datasource-managed") +} + +func TestResolve_UnsupportedKindsExcludedFromNoMatchSurfaces(t *testing.T) { + dsDefs, err := ParseDefinitions(readFixture(t, "ruler_datasource_managed.json")) + require.NoError(t, err) + recDefs, err := ParseDefinitions(readFixture(t, "ruler_recording.json")) + require.NoError(t, err) + supported := rulerDefs(t) + combined := append(append(append([]Definition{}, supported...), dsDefs...), recDefs...) + + _, _, err = Resolve(combined, []string{"Example"}, "") + require.Error(t, err, "want a no-match error for a name matching no title exactly") + + wantCount := fmt.Sprintf("(%d rules available", len(supported)) + require.Contains(t, err.Error(), wantCount) + require.NotContains(t, err.Error(), "ExampleTargetDown") + require.NotContains(t, err.Error(), "example:recorded_metric:rate5m") + require.Contains(t, err.Error(), "Example Paused Rule") +} + +func TestResolve_UnsupportedHomonymResolvesSupportedSilently(t *testing.T) { + // Synthetic: a supported and an unsupported rule sharing an identical + // Folder/Group/Title. Real Grafana data has no such case in the capture, + // but the policy must not treat this as ambiguous — the unsupported rule + // is invisible next to a same-named supported one. + defs := []Definition{ + {UID: "supported-1", Folder: "F", Group: "G", Title: "Shared Title", Kind: KindGrafanaManaged}, + {UID: "", Folder: "F", Group: "G", Title: "Shared Title", Kind: KindDatasourceManaged}, + } + resolved, _, err := Resolve(defs, []string{"F/G/Shared Title"}, "") + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "supported-1", resolved[0].UID) +} + +func TestResolve_CollapseByUIDGivesNoteNotError(t *testing.T) { + defs := rulerDefs(t) + // The bare title and its Folder/Group/Title spelling both name the same + // rule (rule0000007) — a duplicate-name copy mistake, not an error. + resolved, notes, err := Resolve(defs, []string{ + "example_workflow_paused_rule", + "ExampleObservability/Example Auth Production/example_workflow_paused_rule", + }, "") + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000007", resolved[0].UID) + require.Len(t, notes, 1) +} + +// The same rule named twice with the identical string must collapse to one +// rule — distinct from the different-spellings case above. +func TestResolve_IdenticalDuplicateNameCollapsesWithNote(t *testing.T) { + defs := rulerDefs(t) + resolved, notes, err := Resolve(defs, []string{ + "example_workflow_paused_rule", + "example_workflow_paused_rule", + }, "") + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000007", resolved[0].UID) + require.Len(t, notes, 1) +} + +func TestResolve_MinObservedCountIsPostCollapse(t *testing.T) { + defs := rulerDefs(t) + names := []string{ + "example_workflow_paused_rule", + "ExampleObservability/Example Auth Production/example_workflow_paused_rule", // duplicate of the same rule + "Example Paused Rule", + } + resolved, notes, err := Resolve(defs, names, "") + require.NoError(t, err) + // The default MinObserved must come from len(resolved) (2 distinct + // rules) — never len(names) (3 input lines), which would be unsatisfiable. + require.Len(t, resolved, 2) + require.Len(t, notes, 1) +} + +func TestResolve_EmptyAndBlankLinesDiscarded(t *testing.T) { + defs := rulerDefs(t) + resolved, _, err := Resolve(defs, []string{"", " ", "example_workflow_paused_rule", " \t "}, "") + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000007", resolved[0].UID) +} + +func TestResolve_FolderScopesBareTitle(t *testing.T) { + defs := rulerDefs(t) + // Bare title, scoped to the wrong folder — must not match. + _, _, err := Resolve(defs, []string{"example_workflow_paused_rule"}, "Example-Zone-A") + require.Error(t, err, "want no-match when folder scope excludes the only candidate") + + // Scoped to the right folder — must match. + resolved, _, err := Resolve(defs, []string{"example_workflow_paused_rule"}, "ExampleObservability") + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000007", resolved[0].UID) +} diff --git a/grafana-alertcheck/internal/gate/schedule.go b/grafana-alertcheck/internal/gate/schedule.go new file mode 100644 index 000000000..c2fe6e1c5 --- /dev/null +++ b/grafana-alertcheck/internal/gate/schedule.go @@ -0,0 +1,371 @@ +package gate + +import ( + "fmt" + "math/rand/v2" + "sort" + "strings" + "time" +) + +// SkewHardLimit is the largest runner↔Grafana clock skew a run tolerates. +// Exported so the CLI reports it verbatim next to a measured skew. +const SkewHardLimit = 60 * time.Second + +// fromFutureTolerance is how far ahead of the runner's clock a `from` may sit +// before check refuses it — the same 60s as SkewHardLimit, since a future `from` +// can only be clock disagreement. Once-per-run input validation, not a coverage +// check, so Check applies it and proveCoverage does not. +const fromFutureTolerance = 60 * time.Second + +// minDrainTimeout floors drainTimeout (otherwise 2 × max intervalSeconds) so a +// fleet of tight rules still lets a healthy in-flight poll land. +const minDrainTimeout = 2 * time.Minute + +// graceWarnFraction is the share of the window above which transitionGrace is +// worth warning about. +const graceWarnFraction = 0.25 + +// ruleTimings groups the per-rule thresholds derived from a rule's poll +// cadence and its own evaluation interval. +type ruleTimings struct { + pollEvery time.Duration + maxGap time.Duration + healthGrace time.Duration + evalStaleAfter time.Duration +} + +// globalTimings groups the values that apply to the whole run rather than to +// one rule: transitionGrace and drainTimeout are each derived once, across +// every non-skipped watched rule, not per rule. +type globalTimings struct { + transitionGrace time.Duration + // graceSource names, and already carries the `for` value of, the rule that + // set transitionGrace — one string field rather than a second + // (rule, duration) pair, matching this struct's fixed shape. "none" when no + // rule contributed (transitionGrace is then 0). + graceSource string + drainTimeout time.Duration +} + +// newRuleTimings derives one rule's thresholds from its resolved cadence and +// evaluation interval. pollEvery is an input — already resolved for the +// caller's mode — so no caller can compute maxGap against the wrong authority. +func newRuleTimings(pollEvery time.Duration, intervalSeconds int) ruleTimings { + interval := time.Duration(intervalSeconds) * time.Second + maxGap := 2 * pollEvery + healthGrace := max(maxGap, interval) + return ruleTimings{ + pollEvery: pollEvery, + maxGap: maxGap, + healthGrace: healthGrace, + evalStaleAfter: 2 * interval, + } +} + +// defaultPollEvery is the default per-rule cadence: half the rule's own +// evaluation interval. +func defaultPollEvery(intervalSeconds int) time.Duration { + return time.Duration(intervalSeconds) * time.Second / 2 +} + +// DeriveTimings computes every resolved rule's ruleTimings (keyed by UID) plus +// the shared globalTimings. A non-zero override is used verbatim for every rule +// and never clamped to the default — an override above intervalSeconds/2 widens +// maxGap and is reported as a note, not corrected. +func DeriveTimings(defs []Definition, override time.Duration) (rules map[string]ruleTimings, global globalTimings, notes []string) { + rules = make(map[string]ruleTimings, len(defs)) + for _, d := range defs { + def := defaultPollEvery(d.IntervalSeconds) + pollEvery := def + if override > 0 { + pollEvery = override + if override > def { + notes = append(notes, fmt.Sprintf( + "rule %s: --poll-interval %s exceeds half its %ds evaluation interval (%s); maxGap widens accordingly", + d.Title, override, d.IntervalSeconds, def)) + } + } + rules[d.UID] = newRuleTimings(pollEvery, d.IntervalSeconds) + } + // In this mode defs ARE the start-of-step snapshot, so they answer what was + // paused at the window open; only the log-mode counterpart uses the header. + return rules, deriveGlobalTimings(defs, pausedSet(defs)), notes +} + +// pausedSet is Header.pausedAtStart's counterpart for a set of definitions +// resolved at the start of the step, which is the one moment a definition can +// answer "was this paused when the window opened". +func pausedSet(defs []Definition) map[string]bool { + paused := make(map[string]bool, len(defs)) + for _, d := range defs { + paused[d.UID] = d.IsPaused + } + return paused +} + +// DeriveTimingsFromLog is DeriveTimings' log-mode counterpart: pollEvery comes +// from the header (the cadence actually used), not the definitions — re-deriving +// it here would compare recorded gaps against default-cadence thresholds, an +// exit 2 on a clean window (slower override) or a silently passing recorder gap +// (faster override). evalStaleAfter still comes from defs (2 × intervalSeconds). +// +// Three header shapes are hard errors rather than a best-effort derivation, +// because each would silently widen a threshold: a rule with no matching +// definition, a non-positive recorded cadence, and a UID listed twice +// (last-one-wins would widen maxGap on a corrupt log). +// +// It checks only the header-to-defs direction. A definition absent from the +// header is Check's log-identity validation to judge, not this function's. +func DeriveTimingsFromLog(h Header, defs []Definition) (rules map[string]ruleTimings, global globalTimings, err error) { + byUID := make(map[string]Definition, len(defs)) + for _, d := range defs { + byUID[d.UID] = d + } + + rules = make(map[string]ruleTimings, len(h.Rules)) + for _, lr := range h.Rules { + def, ok := byUID[lr.UID] + if !ok { + return nil, globalTimings{}, fmt.Errorf( + "log header names rule %s (%q), which no current definition matches", lr.UID, lr.Title) + } + if _, duplicate := rules[lr.UID]; duplicate { + return nil, globalTimings{}, fmt.Errorf( + "log header names rule %s (%q) twice; its recorded cadence is ambiguous", lr.UID, lr.Title) + } + if lr.PollEverySeconds <= 0 { + return nil, globalTimings{}, fmt.Errorf( + "log header records poll_every_seconds=%v for rule %s (%q); the recorded cadence is required to derive maxGap", + lr.PollEverySeconds, lr.UID, lr.Title) + } + pollEvery := time.Duration(lr.PollEverySeconds * float64(time.Second)) + rules[lr.UID] = newRuleTimings(pollEvery, def.IntervalSeconds) + } + // The header, not defs, decides which rules are excluded from the grace: + // defs were resolved after the window closed. See deriveGlobalTimings. + return rules, deriveGlobalTimings(defs, h.pausedAtStart()), nil +} + +// deriveGlobalTimings computes transitionGrace and drainTimeout over defs. A +// rule skipped at the window open is excluded from the transitionGrace max (its +// `for` can never fire in-window); drainTimeout runs over every resolved rule. +// +// The exclusion authority is pausedAtStart, never Definition.IsPaused: log-mode +// defs are re-resolved after the window closed. Reading the late definitions +// was a quiet fail-open — a rule paused after `to` would drop out of the max, +// collapse the grace past windowEnd (the classification bound AND collection +// deadline), and pass a window the surfacing poll was never recorded for. +func deriveGlobalTimings(defs []Definition, pausedAtStart map[string]bool) globalTimings { + var g globalTimings + var maxInterval time.Duration + for _, d := range defs { + interval := time.Duration(d.IntervalSeconds) * time.Second + if interval > maxInterval { + maxInterval = interval + } + if pausedAtStart[d.UID] { + continue + } + if candidate := d.For + interval; candidate > g.transitionGrace { + g.transitionGrace = candidate + g.graceSource = fmt.Sprintf("%s (for=%s, interval=%s)", d.Title, d.For, interval) + } + } + g.drainTimeout = max(2*maxInterval, minDrainTimeout) + return g +} + +// Scheduler drives one per-rule schedule, never a global cycle: a rule at +// intervalSeconds=10 alongside twenty at 300 keeps its own 5s cadence without +// forcing the same cadence onto the other twenty. +type Scheduler struct { + next map[string]time.Time + every map[string]time.Duration +} + +// NewScheduler builds a Scheduler over per-rule cadences, staggering each +// rule's initial next-due time across [0, pollEvery) so a phase-aligned fleet +// (which would void CheckBudget's burst bound) never arises by construction. +// It takes cadences, not ruleTimings: a scheduler only decides when to poll and +// must not be handed coverage thresholds it never applies. +func NewScheduler(every map[string]time.Duration, now time.Time) *Scheduler { + s := &Scheduler{ + next: make(map[string]time.Time, len(every)), + every: make(map[string]time.Duration, len(every)), + } + for uid, pollEvery := range every { + s.every[uid] = pollEvery + var offset time.Duration + if pollEvery > 0 { + offset = rand.N(pollEvery) + } + s.next[uid] = now.Add(offset) + } + return s +} + +// Due returns the due UIDs, earliest-due-first. Ties break by tightest cadence +// first: the burst bound assumes a newly-due tight rule waits at most one +// in-flight request, which only holds if a simultaneous batch serves the +// tightest rule first. A map-order tie-break would silently void that. +func (s *Scheduler) Due(now time.Time) []string { + var due []string + for uid, t := range s.next { + if !t.After(now) { + due = append(due, uid) + } + } + sort.Slice(due, func(i, j int) bool { + a, b := due[i], due[j] + if !s.next[a].Equal(s.next[b]) { + return s.next[a].Before(s.next[b]) + } + if s.every[a] != s.every[b] { + return s.every[a] < s.every[b] + } + return a < b // stable, deterministic fallback for an exact tie + }) + return due +} + +// Mark records that uid was just polled at now, scheduling its next poll one +// cadence later. It fails on an unknown uid rather than silently treating the +// missing cadence as zero: a zero cadence would schedule an immediate re-due +// (next = now.Add(0)) and insert a bogus next-due entry, hiding a caller that +// is polling a rule the scheduler never owns. +func (s *Scheduler) Mark(uid string, now time.Time) error { + every, ok := s.every[uid] + if !ok { + return fmt.Errorf("Mark: unknown rule uid %q", uid) + } + s.next[uid] = now.Add(every) + return nil +} + +// earliestDue returns the earliest next-due time (false when empty). The loop +// waits exactly that long instead of a fixed tick, which would poll slack rules +// early (wasting budget) or wake late for the tightest rule (opening a gap). +func (s *Scheduler) earliestDue() (time.Time, bool) { + var earliest time.Time + ok := false + for _, t := range s.next { + if !ok || t.Before(earliest) { + earliest = t + ok = true + } + } + return earliest, ok +} + +// CheckBudget proves at start that a fully resolved schedule can be served. t +// and measured are keyed by UID, and measured must carry every UID in t (a rule +// never measured cannot have its budget proved). It fails on any of three +// conditions: +// +// - utilization — the long-run request rate exceeds the concurrency; +// - a single rule's request cannot fit inside its own cadence; +// - the burst bound — the slowest measured request is slower than the fleet's +// tightest cadence, which can open a mid-run gap beyond that rule's maxGap. +// +// The message names only the three operator controls: concurrency, +// poll-interval, and the alert list. +func CheckBudget(t map[string]ruleTimings, measured map[string]time.Duration, concurrency int) error { + if len(t) == 0 { + return nil + } + + uids := make([]string, 0, len(t)) + for uid := range t { + uids = append(uids, uid) + } + sort.Strings(uids) // deterministic message order + + for _, uid := range uids { + if _, ok := measured[uid]; !ok { + return fmt.Errorf("schedule budget: rule %s was never measured", uid) + } + if t[uid].pollEvery <= 0 { + return fmt.Errorf("schedule budget: rule %s has a non-positive poll-interval %s", uid, t[uid].pollEvery) + } + } + + var utilization float64 + tightestUID := uids[0] // the rule with the smallest pollEvery seen so far — a UID, not a duration + var maxMeasuredUID string + var maxMeasured time.Duration + var overCadence []string + for _, uid := range uids { + rt, m := t[uid], measured[uid] + utilization += float64(m) / float64(rt.pollEvery) + if t[tightestUID].pollEvery > rt.pollEvery { + tightestUID = uid + } + if m > maxMeasured { + maxMeasured, maxMeasuredUID = m, uid + } + if m > rt.pollEvery { + overCadence = append(overCadence, uid) + } + } + + var problems []string + if utilization > float64(concurrency) { + problems = append(problems, fmt.Sprintf("utilization %.2f exceeds concurrency %d", utilization, concurrency)) + } + for _, uid := range overCadence { + problems = append(problems, fmt.Sprintf( + "rule %s: measured %s exceeds its own poll-interval %s", uid, measured[uid], t[uid].pollEvery)) + } + if maxMeasured > t[tightestUID].pollEvery { + problems = append(problems, fmt.Sprintf( + "burst bound: rule %s's measured %s exceeds the fleet's tightest poll-interval %s (rule %s)", + maxMeasuredUID, maxMeasured, t[tightestUID].pollEvery, tightestUID)) + } + + if len(problems) == 0 { + return nil + } + + var b strings.Builder + fmt.Fprintf(&b, "schedule does not fit at concurrency %d:\n", concurrency) + for _, uid := range uids { + fmt.Fprintf(&b, " rule %s: measured %s, poll-interval %s\n", uid, measured[uid], t[uid].pollEvery) + } + for _, p := range problems { + fmt.Fprintf(&b, " - %s\n", p) + } + b.WriteString("fix by: raising concurrency, raising poll-interval, or watching fewer alerts") + return fmt.Errorf("%s", b.String()) +} + +// graceSourceOrNone is the single "none" default for the grace-source field: +// an empty source means no rule contributed a transitionGrace. Applied here so +// StartupSummary and the human table print the same thing. +func graceSourceOrNone(source string) string { + if source == "" { + return "none" + } + return source +} + +// StartupSummary formats the pre-run print an operator sees before the wait: +// the total planned run time and the rule (with its `for` value) that set +// transitionGrace, plus a warning when the grace eats more than +// graceWarnFraction of the requested window. from/to are the requested +// classification window. +func StartupSummary(from, to time.Time, global globalTimings) (summary, warning string) { + window := to.Sub(from) + total := window + global.transitionGrace + global.drainTimeout + source := graceSourceOrNone(global.graceSource) + summary = fmt.Sprintf( + "planned run time: %s\n window %s + transitionGrace %s + drainTimeout %s\n transitionGrace source: %s", + total, window, global.transitionGrace, global.drainTimeout, source) + + if window > 0 && float64(global.transitionGrace) > float64(window)*graceWarnFraction { + warning = fmt.Sprintf( + "transitionGrace %s is more than %.0f%% of the window %s — the window may be too short for this alert's `for`\n source: %s", + global.transitionGrace, graceWarnFraction*100, window, source) + } + return summary, warning +} diff --git a/grafana-alertcheck/internal/gate/schedule_test.go b/grafana-alertcheck/internal/gate/schedule_test.go new file mode 100644 index 000000000..ace598650 --- /dev/null +++ b/grafana-alertcheck/internal/gate/schedule_test.go @@ -0,0 +1,418 @@ +package gate + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDeriveTimings_Default(t *testing.T) { + defs := []Definition{ + {UID: "r1", Title: "R1", IntervalSeconds: 60}, + } + rules, _, notes := DeriveTimings(defs, 0) + require.Empty(t, notes) + rt := rules["r1"] + require.Equal(t, 30*time.Second, rt.pollEvery) + require.Equal(t, 60*time.Second, rt.maxGap) + require.Equal(t, 60*time.Second, rt.healthGrace) + require.Equal(t, 120*time.Second, rt.evalStaleAfter) +} + +func TestDeriveTimings_OverrideVerbatimNoClamp(t *testing.T) { + defs := []Definition{ + {UID: "r1", Title: "R1", IntervalSeconds: 10}, // default pollEvery = 5s + } + rules, _, notes := DeriveTimings(defs, 20*time.Second) + rt := rules["r1"] + require.Equal(t, 20*time.Second, rt.pollEvery, "the override verbatim (20s), never clamped down to the 5s default") + require.Equal(t, 40*time.Second, rt.maxGap) + require.Len(t, notes, 1) + require.Contains(t, notes[0], "R1") +} + +func TestDeriveTimings_OverrideBelowDefaultNoNote(t *testing.T) { + defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 60}} // default pollEvery = 30s + _, _, notes := DeriveTimings(defs, 5*time.Second) + require.Empty(t, notes) +} + +func TestDeriveTimings_TransitionGraceExcludesSkippedRule(t *testing.T) { + defs := []Definition{ + {UID: "r1", Title: "Tight", IntervalSeconds: 60, For: time.Minute}, + {UID: "r2", Title: "PausedLongFor", IntervalSeconds: 60, For: time.Hour, IsPaused: true}, + } + _, global, _ := DeriveTimings(defs, 0) + want := time.Minute + 60*time.Second // r1's for+interval; r2 (skipped) must not win despite its huge `for` + require.Equal(t, want, global.transitionGrace) + require.Contains(t, global.graceSource, "Tight") +} + +// `for: 1d` and `for: 1w` parse correctly (parse_ruler_test.go), +// but that alone never proves they flow into transitionGrace — a Prometheus +// duration parser that silently truncated to time.Duration's other units, or +// a transitionGrace derivation that only ever saw hand-built values, could +// each pass every existing test and still be wrong together. This drives the +// real ruler_rules.json fixture (rule0000010, for:1w, DERIVED to exercise the +// w unit — testdata/README.md) through ParseDefinitions and DeriveTimings. +func TestDeriveTimings_RealForOneWeekRuleSetsTransitionGrace(t *testing.T) { + defs := rulerDefs(t) + _, global, notes := DeriveTimings(defs, 0) + require.Empty(t, notes, "no --poll-interval override is given, so no override note should fire") + + want := 7*24*time.Hour + 60*time.Second // rule0000010: for=1w, intervalSeconds=60 + require.Equal(t, want, global.transitionGrace) + require.Contains(t, global.graceSource, "Example Failure Ratio Above 10 Percent Weekly") +} + +func TestDeriveTimings_TransitionGraceZeroWhenAllSkipped(t *testing.T) { + defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 60, For: time.Hour, IsPaused: true}} + _, global, _ := DeriveTimings(defs, 0) + require.Zero(t, global.transitionGrace) +} + +// TestDeriveTimingsFromLog_TransitionGraceFollowsTheHeaderNotTheDefinition +// pins the log-mode authority for the grace exclusion. Definitions are +// re-resolved AFTER the window closed, so "paused" in a definition says +// nothing about whether the rule was watched during it. +// +// The fail-open direction is the first case. transitionGrace exists so a +// condition arising just before `to` is still seen when it surfaces at +// to + `for`, and windowEnd is both the classification bound and the +// collection deadline — so a rule somebody paused after `to` dropping out of +// the max collapses the grace, the surfacing poll is never recorded, and the +// run reports clean. +func TestDeriveTimingsFromLog_TransitionGraceFollowsTheHeaderNotTheDefinition(t *testing.T) { + loggedRule := func(uid string, pausedAtStart bool) LoggedRule { + return LoggedRule{UID: uid, Title: uid, IntervalSeconds: 60, PollEverySeconds: 30, IsPaused: pausedAtStart} + } + // The definition says paused in BOTH cases: it is the post-window reading, + // and it must change nothing. + defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 60, For: 5 * time.Minute, IsPaused: true}} + want := 5*time.Minute + 60*time.Second + + t.Run("header says active: the rule stays in the max", func(t *testing.T) { + h := Header{Rules: []LoggedRule{loggedRule("r1", false)}} + _, global, err := DeriveTimingsFromLog(h, defs) + require.NoError(t, err) + require.Equal(t, want, global.transitionGrace, + "the rule was active when the recording opened, so a pause applied afterwards must not shrink the window") + require.Contains(t, global.graceSource, "R1") + }) + + t.Run("header says paused: the rule stays out", func(t *testing.T) { + h := Header{Rules: []LoggedRule{loggedRule("r1", true)}} + _, global, err := DeriveTimingsFromLog(h, defs) + require.NoError(t, err) + require.Zero(t, global.transitionGrace, + "a rule paused before the window opened can never fire during it") + }) + + t.Run("drainTimeout counts every rule either way", func(t *testing.T) { + // drainTimeout carries no pause exclusion, so both headers give the + // same floor-bound value. + for _, pausedAtStart := range []bool{false, true} { + h := Header{Rules: []LoggedRule{loggedRule("r1", pausedAtStart)}} + _, global, err := DeriveTimingsFromLog(h, defs) + require.NoError(t, err) + require.Equalf(t, minDrainTimeout, global.drainTimeout, "pausedAtStart=%v", pausedAtStart) + } + }) +} + +func TestDeriveTimings_DrainTimeoutIncludesPaused(t *testing.T) { + defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 10}} + _, global, _ := DeriveTimings(defs, 0) + require.Equal(t, minDrainTimeout, global.drainTimeout) +} + +func TestDeriveTimings_DrainTimeoutFloor(t *testing.T) { + defs := []Definition{ + {UID: "r1", Title: "Tight", IntervalSeconds: 60, For: time.Minute}, + {UID: "r2", Title: "PausedLongFor", IntervalSeconds: 180, For: time.Hour, IsPaused: true}, + } + _, global, _ := DeriveTimings(defs, 0) + // double the longest interval (2 * 180s) should be the drain timeout + require.Equal(t, 2*180*time.Second, global.drainTimeout) +} + +func TestDeriveTimings_DrainTimeoutAboveFloor(t *testing.T) { + defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 300}} // 2x300s = 600s > 2m floor + _, global, _ := DeriveTimings(defs, 0) + require.Equal(t, 600*time.Second, global.drainTimeout) +} + +// The ordering invariant the burst bound depends on: when several rules become +// due at the exact same instant, Due must serve the tightest cadence first, not +// whatever order the underlying map happens to iterate in. A refactor that +// loses this ordering must fail here, not in a production phase-aligned gap. +func TestScheduler_DueOrderingTiesBreakByTightestCadence(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + s := &Scheduler{ + next: map[string]time.Time{ + "slack1": now, "slack2": now, "tight": now, "slack3": now, + }, + every: map[string]time.Duration{ + "slack1": 300 * time.Second, + "slack2": 300 * time.Second, + "tight": 10 * time.Second, + "slack3": 300 * time.Second, + }, + } + due := s.Due(now) + require.Len(t, due, 4) + require.Equal(t, "tight", due[0], "the tightest-cadence rule must be first when all are simultaneously due") +} + +func TestScheduler_DueExcludesNotYetDue(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + s := &Scheduler{ + next: map[string]time.Time{"soon": now.Add(-time.Second), "later": now.Add(time.Minute)}, + every: map[string]time.Duration{"soon": 10 * time.Second, "later": 10 * time.Second}, + } + due := s.Due(now) + require.Len(t, due, 1) + require.Equal(t, "soon", due[0]) +} + +func TestScheduler_MarkAdvancesNextDue(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + s := &Scheduler{ + next: map[string]time.Time{"r1": now}, + every: map[string]time.Duration{"r1": 30 * time.Second}, + } + require.NoError(t, s.Mark("r1", now), "Mark must succeed for a known rule") + require.Empty(t, s.Due(now), "next due is 30s out") + require.Len(t, s.Due(now.Add(30*time.Second)), 1) +} + +func TestScheduler_MarkUnknownUIDFails(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + s := &Scheduler{ + next: map[string]time.Time{"r1": now}, + every: map[string]time.Duration{"r1": 30 * time.Second}, + } + err := s.Mark("not-a-rule", now) + require.Error(t, err, "a missing cadence must not read as zero and loop") + // The failed Mark must not have inserted a bogus next-due entry. + _, ok := s.next["not-a-rule"] + require.False(t, ok, "a failed Mark must not insert a next-due entry") +} + +// TestScheduler_PerRuleCadenceOverTime simulates a run and counts how often +// each rule comes due: schedules are per rule, never a global cycle. A tight +// rule must be polled at its own cadence regardless +// of what slower rules in the same fleet need, and a slack rule must never be +// forced onto the tight rule's cadence. +func TestScheduler_PerRuleCadenceOverTime(t *testing.T) { + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + rules := map[string]time.Duration{ + "tight": 10 * time.Second, + "slack": 300 * time.Second, + } + s := NewScheduler(rules, start) + + const runFor = 900 * time.Second + const step = time.Second + counts := map[string]int{} + for elapsed := time.Duration(0); elapsed <= runFor; elapsed += step { + now := start.Add(elapsed) + for _, uid := range s.Due(now) { + counts[uid]++ + require.NoErrorf(t, s.Mark(uid, now), "Mark(%q)", uid) + } + } + + // 900s of runtime: "tight" (10s cadence) polls ~90 times, "slack" (300s + // cadence) ~3 times. Assert the ratio holds rather than an exact count, + // since the staggered initial offset shifts each by up to one cadence. + require.GreaterOrEqual(t, counts["tight"], 85) + require.LessOrEqual(t, counts["tight"], 91) + require.GreaterOrEqual(t, counts["slack"], 2) + require.LessOrEqual(t, counts["slack"], 4) + require.Less(t, counts["slack"], counts["tight"], + "schedules must be per rule, not a shared global cycle") +} + +func TestNewScheduler_StaggersWithinPollEvery(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + rules := map[string]time.Duration{"r1": 100 * time.Second} + s := NewScheduler(rules, now) + offset := s.next["r1"].Sub(now) + require.GreaterOrEqual(t, offset, time.Duration(0)) + require.Less(t, offset, 100*time.Second) +} + +func TestScheduler_EarliestDueEmpty(t *testing.T) { + s := &Scheduler{next: map[string]time.Time{}, every: map[string]time.Duration{}} + _, ok := s.earliestDue() + require.False(t, ok) +} + +// A zero next-due time is real, not an empty scheduler. +func TestScheduler_EarliestDueZeroTime(t *testing.T) { + s := &Scheduler{ + next: map[string]time.Time{"r1": {}}, + every: map[string]time.Duration{"r1": time.Second}, + } + earliest, ok := s.earliestDue() + require.True(t, ok, "the zero time is a real next-due, not an empty scheduler") + require.Truef(t, earliest.IsZero(), "earliestDue = %v, want the zero time", earliest) +} + +func TestScheduler_EarliestDuePicksMinimum(t *testing.T) { + now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + s := &Scheduler{ + next: map[string]time.Time{ + "later": now.Add(2 * time.Minute), + "soon": now.Add(time.Minute), + }, + every: map[string]time.Duration{"later": time.Minute, "soon": time.Minute}, + } + earliest, ok := s.earliestDue() + require.True(t, ok) + require.Truef(t, earliest.Equal(now.Add(time.Minute)), "earliestDue = %v, want the earliest next-due time", earliest) +} + +// One rule at 10s beside twenty at 300s, all measured ~1.8s, must not error at +// any reasonable concurrency — the exact case a naive worst-case-slot +func TestCheckBudget_MixedIntervalRegression(t *testing.T) { + timings := map[string]ruleTimings{"tight": {pollEvery: 5 * time.Second}} + measured := map[string]time.Duration{"tight": 1800 * time.Millisecond} + for i := range 20 { + uid := uidN(i) + timings[uid] = ruleTimings{pollEvery: 150 * time.Second} + measured[uid] = 1800 * time.Millisecond + } + require.NoError(t, CheckBudget(timings, measured, 1)) +} + +func TestCheckBudget_UtilizationExceeded(t *testing.T) { + timings := map[string]ruleTimings{ + "a": {pollEvery: 10 * time.Second}, + "b": {pollEvery: 10 * time.Second}, + } + measured := map[string]time.Duration{"a": 9 * time.Second, "b": 9 * time.Second} + err := CheckBudget(timings, measured, 1) + require.Error(t, err) + assertBudgetMessage(t, err.Error()) +} + +func TestCheckBudget_SingleRuleExceedsOwnCadence(t *testing.T) { + timings := map[string]ruleTimings{"slow": {pollEvery: 5 * time.Second}} + measured := map[string]time.Duration{"slow": 6 * time.Second} + err := CheckBudget(timings, measured, 10) + require.Error(t, err, "measured 6s exceeds its own 5s poll-interval") + assertBudgetMessage(t, err.Error()) +} + +func TestCheckBudget_BurstBoundViolation(t *testing.T) { + // Utilization is trivially fine, but the slower rule's request time (3s) + // exceeds the tighter rule's cadence (2s) — a mid-run gap risk even + // though no single rule breaches its own cadence and utilization is low. + timings := map[string]ruleTimings{ + "tight": {pollEvery: 2 * time.Second}, + "slow": {pollEvery: 100 * time.Second}, + } + measured := map[string]time.Duration{"tight": 100 * time.Millisecond, "slow": 3 * time.Second} + err := CheckBudget(timings, measured, 10) + require.Error(t, err, "slow's 3s measured exceeds tight's 2s cadence") + require.Contains(t, err.Error(), "burst bound") + assertBudgetMessage(t, err.Error()) +} + +func TestCheckBudget_BurstBoundOKWhenNotExceeded(t *testing.T) { + timings := map[string]ruleTimings{ + "tight": {pollEvery: 5 * time.Second}, + "slow": {pollEvery: 100 * time.Second}, + } + measured := map[string]time.Duration{"tight": 100 * time.Millisecond, "slow": 1800 * time.Millisecond} + require.NoError(t, CheckBudget(timings, measured, 10)) +} + +func TestCheckBudget_MissingMeasurementIsAnError(t *testing.T) { + timings := map[string]ruleTimings{"r1": {pollEvery: 30 * time.Second}} + err := CheckBudget(timings, map[string]time.Duration{}, 10) + require.Error(t, err, "r1 was never measured (fail closed, not a silent zero)") +} + +func TestCheckBudget_MissingMixedMeasurementIsAnError(t *testing.T) { + timings := map[string]ruleTimings{ + "tight": {pollEvery: 5 * time.Second}, + "slow": {pollEvery: 100 * time.Second}, + } + measured := map[string]time.Duration{"tight": 100 * time.Millisecond} + require.Error(t, CheckBudget(timings, measured, 10), + "slow was never measured (fail closed, not a silent zero)") +} + +func TestCheckBudget_EmptyScheduleIsFine(t *testing.T) { + require.NoError(t, CheckBudget(nil, nil, 1)) +} + +func TestCheckBudget_NonPositivePollIntervalIsAnError(t *testing.T) { + for _, pe := range []time.Duration{0, -time.Second} { + timings := map[string]ruleTimings{"r1": {pollEvery: pe}} + measured := map[string]time.Duration{"r1": time.Second} + err := CheckBudget(timings, measured, 1) + require.Errorf(t, err, "pollEvery=%s would divide by zero", pe) + require.Contains(t, err.Error(), "non-positive", "the error must name the non-positive poll-interval") + } +} + +// assertBudgetMessage checks the message contents: a measured duration is +// present, and all three controls are named — never a single suggested +// interval. +func assertBudgetMessage(t *testing.T, msg string) { + t.Helper() + for _, want := range []string{"measured", "concurrency", "poll-interval", "fewer"} { + require.Contains(t, msg, want) + } +} + +func uidN(i int) string { + return "slack" + string(rune('a'+i)) +} + +func TestStartupSummary_WarningWhenGraceTooLarge(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + global := globalTimings{transitionGrace: 5 * time.Minute, graceSource: "R (for=4m30s, interval=30s)", drainTimeout: time.Minute} + summary, warning := StartupSummary(from, to, global) + require.Contains(t, summary, "planned run time") + require.NotEmpty(t, warning, "transitionGrace (5m) > 1/4 of the 10m window") + require.Contains(t, warning, "R (for=4m30s, interval=30s)") +} + +// The test above pins the warning formula with a hand-built globalTimings. +// This drives the same warning off the real ruler_rules.json fixture's for:1w +// rule instead, tying ParseDefinitions and DeriveTimings into the warning end +// to end. +func TestStartupSummary_RealForOneWeekRuleTriggersWarning(t *testing.T) { + defs := rulerDefs(t) + _, global, notes := DeriveTimings(defs, 0) + require.Empty(t, notes) + + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) // transitionGrace (>1w) dwarfs 1/4 of this window + summary, warning := StartupSummary(from, to, global) + require.Contains(t, summary, "planned run time") + require.NotEmpty(t, warning) + require.Contains(t, warning, "Example Failure Ratio Above 10 Percent Weekly") +} + +func TestStartupSummary_NoWarningWhenGraceSmall(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(time.Hour) + global := globalTimings{transitionGrace: time.Minute, graceSource: "R (for=30s, interval=30s)", drainTimeout: time.Minute} + _, warning := StartupSummary(from, to, global) + require.Empty(t, warning) +} + +func TestStartupSummary_NoGraceSourceReadsNone(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(time.Hour) + summary, _ := StartupSummary(from, to, globalTimings{}) + require.Contains(t, summary, "none") +} diff --git a/grafana-alertcheck/internal/gate/source.go b/grafana-alertcheck/internal/gate/source.go new file mode 100644 index 000000000..721151e21 --- /dev/null +++ b/grafana-alertcheck/internal/gate/source.go @@ -0,0 +1,362 @@ +package gate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand/v2" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +// maxResponseBytes caps how much doRequest will read from a response body. +// It is far above the largest real payload the gate retrieves (the ~600 KB +// high-cardinality state fetch documented in parse_state_test.go), so a +// legitimate response never trips it, but a misbehaving server or proxy +// streaming an unbounded body is cut off loudly instead of OOMing the process +// across retries. +const maxResponseBytes = 25 << 20 // 25 MiB + +// Clock is the seam that lets tests advance time without sleeping — the only +// two operations the gate ever needs from a clock. +type Clock interface { + Now() time.Time + After(d time.Duration) <-chan time.Time +} + +// SystemClock is the production Clock: the real wall clock. +type SystemClock struct{} + +func (SystemClock) Now() time.Time { return time.Now() } +func (SystemClock) After(d time.Duration) <-chan time.Time { return time.After(d) } + +// Observation is one successful poll of the state endpoint for a single rule. +type Observation struct { + Rules []StateRule // may be empty — an authoritative 2xx saying the rule is absent + GrafanaNow time.Time // the response's Date header + Skew time.Duration // serverDate - (t_send+t_headers)/2, signed + SkewBound time.Duration // (t_headers-t_send)/2 — RTT/2 to the response headers + Latency time.Duration // t_send through the full body read — see requestResult.Latency +} + +// TransportError marks a failure worth retrying: a 5xx/429 response, a network +// failure, or a body that failed to parse. Not a 4xx (wrong auth, missing +// resource), not a deleted rule (an authoritative 2xx) and not a clock problem +// (a hard error — see doRequest). +type TransportError struct { + Err error + Status int // 0 when the failure never got a status (network/transport failure) +} + +func (e *TransportError) Error() string { + if e.Status != 0 { + return fmt.Sprintf("transport error: status %d: %v", e.Status, e.Err) + } + return fmt.Sprintf("transport error: %v", e.Err) +} + +func (e *TransportError) Unwrap() error { return e.Err } + +// RetryExhaustedError is the hard, terminal failure retryTransport returns once +// it gives up. It deliberately omits Unwrap into *TransportError so +// errors.AsType can never re-classify it as retryable; Cause stays a plain +// field for logging only. +type RetryExhaustedError struct { + Failures int + Cause error +} + +func (e *RetryExhaustedError) Error() string { + return fmt.Sprintf("gave up after %d sequential failures: %v", e.Failures, e.Cause) +} + +// Source is everything the gate reads from Grafana. httpSource is the one +// production implementation; the tests use a scripted fake +// (source_fake_test.go) instead of real HTTP. +type Source interface { + Version(ctx context.Context) (string, error) + Definitions(ctx context.Context) ([]Definition, error) + RuleState(ctx context.Context, title string) (Observation, error) +} + +// grafanaVersion is a parsed major.minor.patch triple. +type grafanaVersion struct{ major, minor, patch int } + +func (v grafanaVersion) String() string { return fmt.Sprintf("%d.%d.%d", v.major, v.minor, v.patch) } + +// ord encodes the triple as a single comparable integer. Safe as long as +// minor and patch stay under 1000, true of every real Grafana version. +func (v grafanaVersion) ord() int64 { + return int64(v.major)*1_000_000 + int64(v.minor)*1_000 + int64(v.patch) +} + +func parseGrafanaVersion(s string) (grafanaVersion, error) { + s = strings.TrimSpace(s) + if s == "" { + return grafanaVersion{}, errors.New("empty version string") + } + // Grafana's /api/health always reports exactly three components + // (health.json: "13.1.0"). Require all three explicitly rather than + // defaulting missing ones to zero or silently dropping extras — either + // would accept a value ("13", "13.1.0.5") that was never actually seen + // and never verified against. + parts := strings.Split(s, ".") + if len(parts) != 3 { + return grafanaVersion{}, fmt.Errorf("unparseable version %q: want exactly 3 dot-separated components, got %d", s, len(parts)) + } + var v grafanaVersion + fields := [3]*int{&v.major, &v.minor, &v.patch} + for i, field := range fields { + n, err := strconv.Atoi(parts[i]) + if err != nil { + return grafanaVersion{}, fmt.Errorf("unparseable version %q: %w", s, err) + } + *field = n + } + return v, nil +} + +// supportedGrafanaMin and supportedGrafanaMax bound the platform this gate is +// verified against: >= 13.0.0, < 14.0.0. +var ( + supportedGrafanaMin = grafanaVersion{13, 0, 0} + supportedGrafanaMax = grafanaVersion{14, 0, 0} // exclusive +) + +// CheckGrafanaVersion enforces the supported range. An unparseable or +// out-of-range version is a hard error naming both what was found and what is +// supported: the response schemas this gate parses are only verified against +// that range, and trusting an unverified one is how a deprecation turns into a +// silent misread. +func CheckGrafanaVersion(version string) error { + v, err := parseGrafanaVersion(version) + if err != nil { + return fmt.Errorf("grafana version %q: %w (supported: >=%s, <%s)", + version, err, supportedGrafanaMin, supportedGrafanaMax) + } + if v.ord() < supportedGrafanaMin.ord() || v.ord() >= supportedGrafanaMax.ord() { + return fmt.Errorf("unsupported grafana version %q (supported: >=%s, <%s)", + version, supportedGrafanaMin, supportedGrafanaMax) + } + return nil +} + +// httpSource is the production Source: stdlib net/http only, bearer auth from +// a token supplied at construction (the caller reads it from the environment; +// this type never touches env itself), and manual strict decoding via +// ParseState/ParseDefinitions. The retry limit and backoff parameters are +// struct fields with production defaults set here, not package constants, so a +// test can shrink them without a hook. +type httpSource struct { + baseURL string + token string + client *http.Client + clock Clock + + maxSequentialFailures int + backoffBase time.Duration + backoffCap time.Duration +} + +// NewHTTPSource builds the production Source. token is never logged and never +// enters an error string — it is used only to set the Authorization header. +func NewHTTPSource(baseURL, token string, clock Clock) Source { + return &httpSource{ + baseURL: strings.TrimSuffix(baseURL, "/"), + token: token, + client: &http.Client{Timeout: 30 * time.Second}, + clock: clock, + maxSequentialFailures: 5, + backoffBase: time.Second, + backoffCap: 30 * time.Second, + } +} + +func (s *httpSource) Version(ctx context.Context) (string, error) { + return retryTransport(ctx, s.clock, s.maxSequentialFailures, s.backoffBase, s.backoffCap, func() (string, error) { + r, err := s.doRequest(ctx, "/api/health") + if err != nil { + return "", err + } + var health struct { + Version string `json:"version"` + } + if err := json.Unmarshal(r.Body, &health); err != nil { + return "", &TransportError{Err: fmt.Errorf("parse /api/health: %w", err)} + } + if health.Version == "" { + return "", &TransportError{Err: errors.New("/api/health: empty version")} + } + return health.Version, nil + }) +} + +func (s *httpSource) Definitions(ctx context.Context) ([]Definition, error) { + return retryTransport(ctx, s.clock, s.maxSequentialFailures, s.backoffBase, s.backoffCap, func() ([]Definition, error) { + r, err := s.doRequest(ctx, "/api/ruler/grafana/api/v1/rules") + if err != nil { + return nil, err + } + defs, parseErr := ParseDefinitions(r.Body) + if parseErr != nil { + return nil, &TransportError{Err: fmt.Errorf("parse ruler definitions: %w", parseErr)} + } + return defs, nil + }) +} + +func (s *httpSource) RuleState(ctx context.Context, title string) (Observation, error) { + path := "/api/prometheus/grafana/api/v1/rules?rule_name=" + url.QueryEscape(title) + return retryTransport(ctx, s.clock, s.maxSequentialFailures, s.backoffBase, s.backoffCap, func() (Observation, error) { + r, err := s.doRequest(ctx, path) + if err != nil { + return Observation{}, err + } + rules, parseErr := ParseState(r.Body) + if parseErr != nil { + // Treated as transient, not a schema break: an unparseable 2xx + // is far more likely a mid-stream hiccup than a permanent shape + // change, and the strict parser already turns a real shape change + // into a loud per-field error the moment it's visible. + return Observation{}, &TransportError{Err: fmt.Errorf("parse rule state: %w", parseErr)} + } + return Observation{ + Rules: rules, + GrafanaNow: r.ServerDate, + Skew: r.Skew, + SkewBound: r.SkewBound, + Latency: r.Latency, + }, nil + }) +} + +// requestResult is the outcome of one successful HTTP attempt in doRequest: +// the raw body plus everything derived from timing the round trip against +// the response's own clock. +type requestResult struct { + Body []byte + ServerDate time.Time // the response's Date header + Skew time.Duration // serverDate - (t_send+t_headers)/2, signed + SkewBound time.Duration // (t_headers-t_send)/2 — RTT/2 to the response headers + // Latency spans t_send through the full body read: the budget check needs + // the whole poll's wall time (header-only latency would be fail-open). The + // caller's JSON parse runs outside doRequest; extend here if that ever must + // be folded in. + Latency time.Duration +} + +// doRequest performs one HTTP GET and classifies the outcome: a 5xx/429, a +// network failure, or a body-read failure is retryable (*TransportError); a +// 4xx (wrong auth, missing resource — retrying cannot fix it), a missing or +// unparseable Date header, or a skew beyond SkewHardLimit is a hard error, so +// none of those enters the backoff loop. +// +// The Date/skew check runs on every endpoint (even /api/health): a skew only +// noticed once RuleState starts polling has already masked earlier reads, so it +// fails closed on the first response. +func (s *httpSource) doRequest(ctx context.Context, path string) (requestResult, error) { + req, buildErr := http.NewRequestWithContext(ctx, http.MethodGet, s.baseURL+path, nil) + if buildErr != nil { + return requestResult{}, fmt.Errorf("build request for %s: %w", path, buildErr) + } + if s.token != "" { + req.Header.Set("Authorization", "Bearer "+s.token) + } + + tSend := s.clock.Now() + resp, doErr := s.client.Do(req) + tHeaders := s.clock.Now() + if doErr != nil { + return requestResult{}, &TransportError{Err: doErr} + } + defer resp.Body.Close() + + b, readErr := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes)) + tBodyRead := s.clock.Now() + if readErr != nil { + return requestResult{}, &TransportError{Err: fmt.Errorf("read response body (status %d): %w", resp.StatusCode, readErr)} + } + if len(b) >= maxResponseBytes { + // Hard error, never retried: a response this large is a stable + // property of the server's reply, not a transient network hiccup, so + // retrying would just reallocate the same bounded-but-pointless body. + return requestResult{}, fmt.Errorf("response body exceeded %d bytes", maxResponseBytes) + } + latency := tBodyRead.Sub(tSend) + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + err := fmt.Errorf("unexpected status %d", resp.StatusCode) + if resp.StatusCode >= 400 && resp.StatusCode < 500 && resp.StatusCode != http.StatusTooManyRequests { + return requestResult{}, err + } + return requestResult{}, &TransportError{Err: err, Status: resp.StatusCode} + } + + dateHeader := resp.Header.Get("Date") + if dateHeader == "" { + return requestResult{}, fmt.Errorf("%s: response has no Date header", path) + } + serverDate, parseErr := http.ParseTime(dateHeader) + if parseErr != nil { + return requestResult{}, fmt.Errorf("%s: unparseable Date header %q: %w", path, dateHeader, parseErr) + } + + bound := tHeaders.Sub(tSend) / 2 + mid := tSend.Add(bound) + signedSkew := serverDate.Sub(mid) + absSkew := signedSkew + if absSkew < 0 { + absSkew = -absSkew + } + if absSkew > SkewHardLimit { + return requestResult{}, fmt.Errorf("%s: clock skew %s exceeds hard limit %s", path, absSkew, SkewHardLimit) + } + + return requestResult{Body: b, ServerDate: serverDate, Skew: signedSkew, SkewBound: bound, Latency: latency}, nil +} + +// retryTransport runs fn, retrying with backoff only on *TransportError — any +// other error returns immediately. failures counts consecutive *TransportError +// results; exceeding maxFailures gives up with a wrapped hard error. Waits go +// through clock.After so a fake Clock never sleeps real time. +func retryTransport[T any](ctx context.Context, clock Clock, maxFailures int, backoffBase, backoffCap time.Duration, fn func() (T, error)) (T, error) { + var zero T + failures := 0 + for { + v, err := fn() + if err == nil { + return v, nil + } + if _, ok := errors.AsType[*TransportError](err); !ok { + return zero, err + } + failures++ + if failures > maxFailures { + return zero, &RetryExhaustedError{Failures: failures, Cause: err} + } + select { + case <-ctx.Done(): + return zero, ctx.Err() + case <-clock.After(backoffDelay(backoffBase, backoffCap, failures)): + } + } +} + +// backoffDelay is 1s base, doubling per failure, capped at maxDelay, with +// ±20% jitter. +func backoffDelay(base, maxDelay time.Duration, failureCount int) time.Duration { + d := base + for i := 1; i < failureCount && d < maxDelay; i++ { + d *= 2 + } + if d > maxDelay { + d = maxDelay + } + jitter := 0.8 + rand.Float64()*0.4 // [0.8, 1.2] + return time.Duration(float64(d) * jitter) +} diff --git a/grafana-alertcheck/internal/gate/source_fake_test.go b/grafana-alertcheck/internal/gate/source_fake_test.go new file mode 100644 index 000000000..97de92f02 --- /dev/null +++ b/grafana-alertcheck/internal/gate/source_fake_test.go @@ -0,0 +1,180 @@ +package gate + +import ( + "context" + "fmt" + "slices" + "sync" + "time" +) + +// fakeClock is a manually-advanced Clock — no test in this package sleeps on +// real time. It is goroutine-safe (a concurrent fleet under -race must not trip +// on the double itself), but After always fires immediately, regardless of the +// requested duration or whether Advance was ever called. That is enough for the +// retry/backoff tests, which only need to avoid a real sleep. It is NOT enough +// for a test that must prove a wait did not fire early — e.g. asserting Due() +// does not return a rule before its next-due time. Use virtualClock below for +// that: it makes a wait and the passage of time the same event. +type fakeClock struct { + mu sync.Mutex + now time.Time +} + +func newFakeClock(now time.Time) *fakeClock { return &fakeClock{now: now} } + +func (c *fakeClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *fakeClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +func (c *fakeClock) After(d time.Duration) <-chan time.Time { + c.mu.Lock() + fireAt := c.now.Add(d) + c.mu.Unlock() + ch := make(chan time.Time, 1) + ch <- fireAt + return ch +} + +// virtualClock is a Clock in which time moves only when something waits for +// it: After(d) jumps Now() forward by d and fires at once. That makes a +// recorder-loop test both instant and exact — a loop that waits for its next +// scheduled poll gets that poll's time, never an early or a late wake — and it +// terminates, which a clock whose After fires without advancing Now does not +// (the loop would spin forever on a rule that never comes due). +// +// It is goroutine-safe, but a test that advances time from two goroutines gets +// what it deserves: use it from the loop under test only. +type virtualClock struct { + mu sync.Mutex + now time.Time +} + +func newVirtualClock(now time.Time) *virtualClock { return &virtualClock{now: now} } + +func (c *virtualClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.now +} + +func (c *virtualClock) After(d time.Duration) <-chan time.Time { + c.mu.Lock() + if d > 0 { + c.now = c.now.Add(d) + } + fireAt := c.now + c.mu.Unlock() + ch := make(chan time.Time, 1) + ch <- fireAt + return ch +} + +// steppingClock advances by a fixed step on every Now() call, so a test can +// assert exact latency/skew-bound arithmetic (doRequest's three clock reads +// per attempt) without depending on real wall-clock timing. +type steppingClock struct { + mu sync.Mutex + now time.Time + step time.Duration +} + +func (c *steppingClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + t := c.now + c.now = c.now.Add(c.step) + return t +} + +func (c *steppingClock) After(d time.Duration) <-chan time.Time { + c.mu.Lock() + fireAt := c.now.Add(d) + c.mu.Unlock() + ch := make(chan time.Time, 1) + ch <- fireAt + return ch +} + +// scriptedObservation is one canned (Observation, error) pair a fakeSource +// returns from RuleState, in the order scripted. +type scriptedObservation struct { + obs Observation + err error +} + +// fakeSource is a scripted Source with no HTTP, goroutine-safe so a test that +// polls several rules concurrently can share one instance across goroutines +// without tripping -race. A test that needs it to behave like a live server +// under concurrent load beyond simple locking should verify that assumption +// rather than take this comment's word for it. +type fakeSource struct { + mu sync.Mutex + + version string + versionErr error + + defs []Definition + defsErr error + + // states maps a rule title to a queue of scripted results, popped one + // per call to RuleState. Once the queue is down to its last entry, that + // entry repeats — so a test can script the interesting transitions and + // let a long collection loop settle into steady state without scripting + // every single poll. + states map[string][]scriptedObservation +} + +func newFakeSource() *fakeSource { + return &fakeSource{states: make(map[string][]scriptedObservation)} +} + +func (f *fakeSource) Version(_ context.Context) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.version, f.versionErr +} + +func (f *fakeSource) Definitions(_ context.Context) ([]Definition, error) { + f.mu.Lock() + defer f.mu.Unlock() + // Defensive copy: Definition is a value type, so Clone copies the + // full slice contents, not just the header — callers are free to mutate + // what they got back without racing or corrupting later reads. + return slices.Clone(f.defs), f.defsErr +} + +func (f *fakeSource) RuleState(_ context.Context, title string) (Observation, error) { + f.mu.Lock() + defer f.mu.Unlock() + q := f.states[title] + if len(q) == 0 { + return Observation{}, fmt.Errorf("fakeSource: no scripted response for %q", title) + } + next := q[0] + if len(q) > 1 { + f.states[title] = q[1:] + } + // Defensive copy of the shared Rules slice so a caller mutating the + // returned Observation can't corrupt the scripted state other calls read. + next.obs.Rules = slices.Clone(next.obs.Rules) + return next.obs, next.err +} + +// script appends one scripted (Observation, error) pair to be returned, in +// order, by RuleState(ctx, title). +func (f *fakeSource) script(title string, obs Observation, err error) { + f.mu.Lock() + defer f.mu.Unlock() + f.states[title] = append(f.states[title], scriptedObservation{obs: obs, err: err}) +} + +var _ Source = (*fakeSource)(nil) diff --git a/grafana-alertcheck/internal/gate/source_test.go b/grafana-alertcheck/internal/gate/source_test.go new file mode 100644 index 000000000..e5f24f46f --- /dev/null +++ b/grafana-alertcheck/internal/gate/source_test.go @@ -0,0 +1,538 @@ +package gate + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func healthBody(version string) string { + return fmt.Sprintf(`{"database":"ok","version":%q,"commit":"abc123"}`, version) +} + +func emptyStateBody() string { + return `{"status":"success","data":{"groups":[]}}` +} + +// rawHTTPServer starts an httptest server whose handler hijacks the +// connection and writes exactly the bytes respond returns, bypassing +// net/http's automatic Date-header insertion — the only way to test a +// response with no Date header at all, or a deliberately garbled one. +func rawHTTPServer(t *testing.T, respond func(r *http.Request) []byte) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hj, ok := w.(http.Hijacker) + if !ok { + t.Errorf("ResponseWriter does not support Hijacker") + return + } + conn, buf, err := hj.Hijack() + if err != nil { + t.Errorf("hijack: %v", err) + return + } + defer conn.Close() + if _, err := buf.Write(respond(r)); err != nil { + t.Errorf("write raw response: %v", err) + return + } + _ = buf.Flush() + })) + t.Cleanup(srv.Close) + return srv +} + +// rawResponse builds a minimal, fully-controlled HTTP/1.1 response: no +// header net/http would add unasked, in particular no automatic Date. +func rawResponse(status int, statusText string, headers map[string]string, body string) []byte { + out := fmt.Sprintf("HTTP/1.1 %d %s\r\n", status, statusText) + for k, v := range headers { + out += fmt.Sprintf("%s: %s\r\n", k, v) + } + out += fmt.Sprintf("Content-Length: %d\r\n", len(body)) + out += "Connection: close\r\n\r\n" + out += body + return []byte(out) +} + +func TestCheckGrafanaVersion(t *testing.T) { + cases := []struct { + version string + wantErr bool + wantContains []string // required substrings of the error message, per case, when wantErr + }{ + {"13.1.0", false, nil}, + {"13.0.0", false, nil}, + {"13.99.99", false, nil}, + {"12.9.9", true, []string{`"12.9.9"`, "13.0.0", "14.0.0"}}, + {"14.0.0", true, []string{`"14.0.0"`, "13.0.0", "14.0.0"}}, + {"14.1.0", true, []string{`"14.1.0"`, "13.0.0", "14.0.0"}}, + {"not-a-version", true, []string{`"not-a-version"`}}, + {"", true, nil}, + } + for _, c := range cases { + t.Run(c.version, func(t *testing.T) { + err := CheckGrafanaVersion(c.version) + if c.wantErr { + require.Errorf(t, err, "CheckGrafanaVersion(%q)", c.version) + } else { + require.NoErrorf(t, err, "CheckGrafanaVersion(%q)", c.version) + } + for _, want := range c.wantContains { + require.Contains(t, err.Error(), want, + "must name both what was found and what is supported") + } + }) + } +} + +func TestBackoffDelay(t *testing.T) { + base := time.Second + maxDelay := 30 * time.Second + maxWithJitter := maxDelay + maxDelay/5 + time.Millisecond + for n := 1; n <= 10; n++ { + d := backoffDelay(base, maxDelay, n) + require.Positive(t, d) + require.LessOrEqual(t, d, maxWithJitter) + } +} + +func TestHTTPSource_Version_HappyPath(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "/api/health", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(healthBody("13.1.0"))) + })) + defer srv.Close() + + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + v, err := src.Version(context.Background()) + require.NoError(t, err) + require.Equal(t, "13.1.0", v) +} + +func TestHTTPSource_Version_NeverLogsToken(t *testing.T) { + const secret = "super-secret-token" + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "Bearer "+secret, r.Header.Get("Authorization")) + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, secret, clock) + _, err := src.Version(context.Background()) + require.Error(t, err) + require.NotContains(t, err.Error(), secret) +} + +func TestHTTPSource_RuleState_EmptyIsNotAnError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(emptyStateBody())) + })) + defer srv.Close() + + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + obs, err := src.RuleState(context.Background(), "Anything") + require.NoError(t, err) + require.Empty(t, obs.Rules, "an authoritative 2xx is not a transport error") + require.False(t, obs.GrafanaNow.IsZero(), "want the response's Date header value") +} + +func TestHTTPSource_RuleState_EscapesRuleName(t *testing.T) { + var gotQuery string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotQuery = r.URL.RawQuery + require.Equal(t, "/api/prometheus/grafana/api/v1/rules", r.URL.Path) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(emptyStateBody())) + })) + defer srv.Close() + + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + title := "[JD] No Job Proposals & More" + _, err := src.RuleState(context.Background(), title) + require.NoError(t, err) + require.Equal(t, "rule_name="+url.QueryEscape(title), gotQuery) +} + +func TestHTTPSource_Definitions_HappyPath(t *testing.T) { + body := readFixture(t, "ruler_rules.json") + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, "/api/ruler/grafana/api/v1/rules", r.URL.Path) + require.Empty(t, r.URL.RawQuery, "Definitions reads the ruler API unfiltered") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(body) + })) + defer srv.Close() + + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + defs, err := src.Definitions(context.Background()) + require.NoError(t, err) + require.NotEmpty(t, defs) +} + +func TestHTTPSource_Skew(t *testing.T) { + cases := []struct { + name string + drift time.Duration + wantErr bool + }{ + {"0s skew is fine", 0 * time.Second, false}, + {"30s skew is fine", 30 * time.Second, false}, + {"60s skew is fine", 60 * time.Second, false}, + {"61s skew is a hard error", 61 * time.Second, true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + anchor := time.Now() + clock := newFakeClock(anchor) + var calls atomic.Int32 + srv := rawHTTPServer(t, func(r *http.Request) []byte { + calls.Add(1) + date := anchor.Add(c.drift).UTC().Format(http.TimeFormat) + return rawResponse(200, "OK", map[string]string{ + "Content-Type": "application/json", + "Date": date, + }, healthBody("13.1.0")) + }) + src := NewHTTPSource(srv.URL, "", clock) + _, err := src.Version(context.Background()) + if c.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + if c.wantErr { + require.Equal(t, int32(1), calls.Load(), "a skew hard error must never be retried") + } + }) + } +} + +func TestHTTPSource_MissingDateHeader(t *testing.T) { + var calls atomic.Int32 + srv := rawHTTPServer(t, func(r *http.Request) []byte { + calls.Add(1) + return rawResponse(200, "OK", map[string]string{ + "Content-Type": "application/json", + }, healthBody("13.1.0")) + }) + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + _, err := src.Version(context.Background()) + require.Error(t, err, "a missing Date header is a hard error") + require.Equal(t, int32(1), calls.Load(), "a missing Date header must never be retried") +} + +func TestHTTPSource_UnparseableDateHeader(t *testing.T) { + var calls atomic.Int32 + srv := rawHTTPServer(t, func(r *http.Request) []byte { + calls.Add(1) + return rawResponse(200, "OK", map[string]string{ + "Content-Type": "application/json", + "Date": "definitely not a date", + }, healthBody("13.1.0")) + }) + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + _, err := src.Version(context.Background()) + require.Error(t, err, "an unparseable Date header is a hard error") + require.Equal(t, int32(1), calls.Load(), "an unparseable Date header must never be retried") +} + +// TestHTTPSource_ObservationTiming pins the arithmetic behind Observation's +// three derived fields, not just the hard-limit behavior TestHTTPSource_Skew +// already covers: the sign of Skew, SkewBound as exactly RTT/2 to the +// response headers, and Latency as the full send-through-body-read span +// (not just the header round trip). steppingClock advances by a fixed 2s on +// every clock.Now() call, and doRequest calls Now() exactly three times per +// attempt (before send, after headers, after the body read), so the +// arithmetic is exact rather than a real-time approximation. +func TestHTTPSource_ObservationTiming(t *testing.T) { + cases := []struct { + name string + drift time.Duration + }{ + {"positive skew", 5 * time.Second}, + {"negative skew", -5 * time.Second}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + anchor := time.Now().Truncate(time.Second) + clock := &steppingClock{now: anchor, step: 2 * time.Second} + // With step=2s: t_send=anchor, t_headers=anchor+2s, t_bodyRead=anchor+4s. + // mid = t_send + (t_headers-t_send)/2 = anchor+1s, so serverDate = + // anchor+1s+drift makes Skew land on exactly `drift`. + srv := rawHTTPServer(t, func(r *http.Request) []byte { + date := anchor.Add(time.Second + c.drift).UTC().Format(http.TimeFormat) + return rawResponse(200, "OK", map[string]string{ + "Content-Type": "application/json", + "Date": date, + }, emptyStateBody()) + }) + src := NewHTTPSource(srv.URL, "", clock) + obs, err := src.RuleState(context.Background(), "Anything") + require.NoError(t, err) + require.Equal(t, c.drift, obs.Skew) + require.Equal(t, time.Second, obs.SkewBound, "RTT/2 with a 2s round trip to headers") + require.Equal(t, 4*time.Second, obs.Latency, "send through full body read — not just the 2s header round trip") + }) + } +} + +// A discriminating regression for "the gate compares staleness against the +// Date header, never the runner's clock". lastEvaluation +// sits 100s behind Grafana's TRUE now (obs.GrafanaNow, from the Date header) +// — under the 120s evalStaleAfter limit — but 130s behind the RUNNER's clock. +// An implementation that leaked the runner's clock into the staleness +// comparison, instead of the Date header, would report a false violation +// here; coverage_test.go's TestProveCoverage_SkewTranslationAtWindowBoundary +// cannot catch that, because it sets LastEvaluation equal to GrafanaNow on +// every poll, making staleness zero regardless of which clock is used. +func TestHTTPSourceStalenessNeverFalsePositiveUnderSkew(t *testing.T) { + runnerNow := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newFakeClock(runnerNow) + const skew = 30 * time.Second // the runner's clock reads 30s ahead of Grafana's + serverDate := runnerNow.Add(-skew) + lastEval := serverDate.Add(-100 * time.Second) + + def := Definition{UID: "r1", Title: "Rule One"} + srv := rawHTTPServer(t, func(r *http.Request) []byte { + body := fmt.Sprintf(`{"status":"success","data":{"groups":[{"file":"F","name":"G","interval":60,"rules":[`+ + `{"uid":%q,"name":%q,"state":"inactive","health":"ok","isPaused":false,"lastEvaluation":%q}`+ + `]}]}}`, def.UID, def.Title, lastEval.UTC().Format(time.RFC3339)) + return rawResponse(200, "OK", map[string]string{ + "Content-Type": "application/json", + "Date": serverDate.UTC().Format(http.TimeFormat), + }, body) + }) + + src := NewHTTPSource(srv.URL, "", clock) + obs, err := src.RuleState(context.Background(), def.Title) + require.NoError(t, err) + require.True(t, obs.GrafanaNow.Equal(serverDate), + "want the Date header, never the runner's clock") + require.Len(t, obs.Rules, 1) + + rt := newRuleTimings(30*time.Second, 60) // evalStaleAfter = 120s + from := serverDate.Add(-10 * time.Minute) + to := serverDate + polls := denseHealthyPolls(def.UID, from, to, 30*time.Second) + polls[len(polls)-1].LastEvaluation = obs.Rules[0].LastEvaluation // the real, HTTP-sourced value + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + require.False(t, res.Unobservable, + "100s behind Grafana's TRUE now is under the 120s limit — only a runner-clock leak (skewed +30s here) would push this over") +} + +func TestHTTPSource_Retry_TransientRecovers(t *testing.T) { + var mu sync.Mutex + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + calls++ + n := calls + mu.Unlock() + if n <= 2 { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(healthBody("13.1.0"))) + })) + defer srv.Close() + + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + v, err := src.Version(context.Background()) + require.NoError(t, err) + require.Equal(t, "13.1.0", v) + mu.Lock() + n := calls + mu.Unlock() + require.Equal(t, 3, n) +} + +func TestHTTPSource_Retry_ExceedsLimit(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer srv.Close() + + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + _, err := src.Version(context.Background()) + require.Error(t, err) + require.Equal(t, int32(6), calls.Load(), + "maxSequentialFailures=5 tolerates 5, gives up on the 6th") + assertRetryExhausted(t, err, 6) +} + +func TestHTTPSource_RuleState_GarbageBodyRetries(t *testing.T) { + var mu sync.Mutex + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + calls++ + n := calls + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + if n <= 2 { + // A 2xx with a body that fails ParseState — classified as a + // transient *TransportError (source.go), not a hard schema + // break, so it must retry rather than fail immediately. + _, _ = w.Write([]byte("{not valid json")) + return + } + _, _ = w.Write([]byte(emptyStateBody())) + })) + defer srv.Close() + + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + obs, err := src.RuleState(context.Background(), "Anything") + require.NoError(t, err) + require.Empty(t, obs.Rules) + mu.Lock() + n := calls + mu.Unlock() + require.Equal(t, 3, n) +} + +func TestHTTPSource_Definitions_GarbageBodyGivesUp(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte("{not valid json")) + })) + defer srv.Close() + + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + _, err := src.Definitions(context.Background()) + require.Error(t, err) + require.Equal(t, int32(6), calls.Load(), + "a persistently unparseable 2xx body retries like any other transport failure") + assertRetryExhausted(t, err, 6) +} + +func TestHTTPSource_ResponseBodyTooLarge(t *testing.T) { + var calls atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + w.Header().Set("Content-Type", "application/json") + // Stream more than maxResponseBytes in 1 MiB chunks so the test never + // materializes the whole oversized body in its own memory — doRequest + // must cut it off, not buffer it. + chunk := strings.Repeat("x", 1<<20) + for i := 0; i < (maxResponseBytes/(1<<20))+2; i++ { + if _, err := w.Write([]byte(chunk)); err != nil { + return + } + } + })) + defer srv.Close() + + clock := newFakeClock(time.Now()) + src := NewHTTPSource(srv.URL, "", clock) + _, err := src.Version(context.Background()) + require.Error(t, err, "an oversized body must fail loudly") + require.Contains(t, err.Error(), "exceeded", "the error must name the size limit") + // An oversized body is a stable condition, not a transient one: it must + // fail hard on the first attempt, never burning retries re-reading it. + require.Equal(t, int32(1), calls.Load(), "an oversized body must never be retried") +} + +func TestHTTPSource_NetworkFailureRetries(t *testing.T) { + // A server that is never listening: every attempt is a network failure, + // classified as *TransportError, so this exercises the same retry path + // as a 5xx without needing a real listening socket per failure. + clock := newFakeClock(time.Now()) + src := NewHTTPSource("http://127.0.0.1:1", "", clock) + _, err := src.Version(context.Background()) + require.Error(t, err) + assertRetryExhausted(t, err, 6) +} + +// assertRetryExhausted checks the two properties a retry give-up must have: +// it names how many failures it gave up after, and — the regression this +// pins — it is never itself classified as a *TransportError. If it were, +// something one layer up that also retries on *TransportError would treat an +// already-exhausted give-up as retryable again. +func assertRetryExhausted(t *testing.T, err error, wantFailures int) { + t.Helper() + var reErr *RetryExhaustedError + require.ErrorAs(t, err, &reErr) + require.Equal(t, wantFailures, reErr.Failures) + require.Contains(t, err.Error(), fmt.Sprintf("gave up after %d", wantFailures)) + _, ok := errors.AsType[*TransportError](err) + require.False(t, ok, "an exhausted retry must be a terminal, non-retryable error") +} + +func TestFakeClock(t *testing.T) { + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + c := newFakeClock(start) + require.True(t, c.Now().Equal(start)) + c.Advance(5 * time.Minute) + want := start.Add(5 * time.Minute) + require.True(t, c.Now().Equal(want)) + + select { + case fired := <-c.After(time.Hour): + require.True(t, fired.Equal(want.Add(time.Hour))) + default: + require.Fail(t, "After(1h) did not fire immediately") + } +} + +func TestFakeSource(t *testing.T) { + f := newFakeSource() + f.version = "13.1.0" + f.defs = []Definition{{UID: "u1", Title: "Rule One"}} + + ctx := context.Background() + v, err := f.Version(ctx) + require.NoError(t, err) + require.Equal(t, "13.1.0", v) + defs, err := f.Definitions(ctx) + require.NoError(t, err) + require.Len(t, defs, 1) + + f.script("Rule One", Observation{Rules: []StateRule{{UID: "u1"}}}, nil) + f.script("Rule One", Observation{}, fmt.Errorf("boom")) + f.script("Rule One", Observation{Rules: nil}, nil) + + obs, err := f.RuleState(ctx, "Rule One") + require.NoError(t, err) + require.Len(t, obs.Rules, 1) + _, err = f.RuleState(ctx, "Rule One") + require.Error(t, err, "RuleState() call 2: want the scripted error, got nil") + obs, err = f.RuleState(ctx, "Rule One") + require.NoError(t, err) + require.Nil(t, obs.Rules, "last script entry, then repeats") + obs, err = f.RuleState(ctx, "Rule One") + require.NoError(t, err) + require.Nil(t, obs.Rules) + + _, err = f.RuleState(ctx, "Unscripted Rule") + require.Error(t, err) +} diff --git a/grafana-alertcheck/internal/gate/testdata/README.md b/grafana-alertcheck/internal/gate/testdata/README.md index fb437b3ae..1a6bf5877 100644 --- a/grafana-alertcheck/internal/gate/testdata/README.md +++ b/grafana-alertcheck/internal/gate/testdata/README.md @@ -1,6 +1,6 @@ # Fixture provenance -All fixtures are sanitized slices of the real Grafana 13.1.0 payloads captured next to the plan in +All fixtures are sanitized slices of real Grafana 13.1.0 payloads captured into `tmp/` (`tmp/state_all.json`, `tmp/ruler_all.json`, `tmp/health.json` — gitignored, never committed). Renames are consistent across files: the same real folder/rule keeps the same fake identity everywhere it appears (e.g. `folder0000002`/`rule0000002` is the same real paused rule in both @@ -23,45 +23,38 @@ here instead, for every fixture, for consistency. `rule0000002`/"Example Paused Rule". Unmodified: `isPaused:true`, zero `lastEvaluation`, `health:ok`, `state:inactive`, absent `alerts`/`labels`. - **state_health_error.json** — real `health:error` rule ("[JD] No Job Proposals", folder - `job-distributor`), highest priority per §22.1. Renamed to folder `ExampleService`/`folder0000003`, + `job-distributor`). Renamed to folder `ExampleService`/`folder0000003`, rule `rule0000003`/"Example No Data Source". Unmodified: `health:error`, `lastError` text, the single `Error` instance. - **state_health_nodata.json** — real `health:nodata` rule ("ARE test", folder `diegos_playground`). Renamed to folder `ExamplePlayground`/`folder0000004`, rule `rule0000004`/"Example NoData Rule". Unmodified: `health:nodata`, the single `NoData` instance. -- **state_reason_composite.json** — composite of two real instances combined under one rule for P1.2a - coverage: a real `"Normal (Error)"` instance (from a Flux-reconciliation rule; 14 of that state exist +- **state_reason_composite.json** — two real instances combined under one rule to cover composite + state parsing: a real `"Normal (Error)"` instance (from a Flux-reconciliation rule; 14 of that state exist in the capture) and a real `"Normal (NoData)"` instance (from a pod-liveness rule; 1091 of that state exist), plus one plain `"Normal"` instance for contrast. Renamed to folder `ExampleInfra`/`folder0000005`, rule `rule0000005`/"Example Composite Reasons". - **state_missing_optional.json** — derived from `state_one_instance.json`: `alerts`, `totals`, `totalsFiltered` and `labels` all removed. Must parse with `Instances=nil`, `Totals=nil`. -- **state_missing_health.json** — derived from `state_one_instance.json`: the required `health` key - removed. Must be a parse error (H1). -- **state_missing_lasteval.json** — derived from `state_one_instance.json`: the required - `lastEvaluation` key removed. Must be a parse error (H1). -- **state_missing_state.json** — derived from `state_one_instance.json`: the required rule-level - `state` key removed. Must be a parse error (H1). Closes must-error coverage for H1's four required - fields — a review pass found `health`/`lastEvaluation` covered but `state`/`interval` weren't, even - though the code already `req`'d them correctly. -- **state_missing_interval.json** — derived from `state_one_instance.json`: the required group-level - `interval` key removed. Must be a parse error (H1); same review-pass gap as above. +- **state_missing_health.json**, **state_missing_lasteval.json**, **state_missing_state.json**, + **state_missing_interval.json** — derived from `state_one_instance.json`, each with one of the four + required keys removed (`health`, `lastEvaluation`, rule-level `state`, group-level `interval`). Each + must be a parse error. - **state_missing_file.json** / **state_missing_name.json** — derived from `state_one_instance.json`: - the group-level `file`/`name` keys removed respectively. Not part of H1's four (those are `health`, - `state`, `lastEvaluation`, `interval`), but the code treats group identity as strict too, and the same - review pass flagged the gap — closed rather than deferred to a later §22 sweep since the fixture is - the same 10-line edit. + the group-level `file`/`name` keys removed respectively. Not among the four required fields above, + but the parser treats group identity as strict too. - **state_zerotime_unpaused.json** — derived from `state_one_instance.json`: `lastEvaluation` set to - the zero time while `isPaused` stays `false`. Must be a parse error (§2.3). + the zero time while `isPaused` stays `false`. Must be a parse error — only a paused rule may report + the zero time. - **state_unknown_state.json** — derived from `state_one_instance.json`: the instance state hand-edited to `"Weird (NoData)"`, a syntactically valid composite whose base isn't in the 5-value allowlist. Must - be a parse error (P1.2a). + be a parse error. - **state_only_active_instances.json** — derived from a real rule that genuinely had 1 `Alerting` + 22 `Normal` instances (`totals: {alerting:1, normal:22}`, rule `dfhp1t5pkosu8f`, folder `BCM`). `alerts[]` - trimmed to the single `Alerting` instance only, while `totals` is left **unchanged** — reproducing the - §3.2 violation shape (instance list says "only active" while totals disagrees). Renamed to folder - `ExampleTeam`/`folder0000001`, rule `rule0000006`. `ParseState` itself parses this fine; the §3.2 - verification lives in a later phase (P5/P9). + trimmed to the single `Alerting` instance only, while `totals` is left **unchanged** — the shape a + state endpoint that stopped returning normal instances would produce (the instance list says "only + active" while totals disagrees). Renamed to folder `ExampleTeam`/`folder0000001`, rule `rule0000006`. + `ParseState` itself parses this fine; `VerifyNormalInstancesVisible` is what rejects it. ## Ruler endpoint (`/api/ruler/grafana/api/v1/rules`) @@ -69,7 +62,7 @@ here instead, for every fixture, for consistency. - The real true 2-way title collision: namespace `CRE-BCM-Prod-Zone-A`, group `Gateway`, identical folder+group+title, distinct UIDs (`ffvabtvvbozcwf`/`efvabtwbxlvk0b`) — renamed to namespace `Example-Zone-A`, rules `rule0000006a`/`rule0000006b`, both titled "Example No Gateways Available". - Folder/Group/Title alone does **not** disambiguate this pair (§17, §22.2). + Folder/Group/Title alone does **not** disambiguate this pair; only `uid:` does. - The 3 real `is_paused:true` rules, renamed to `rule0000002`/`rule0000007`/`rule0000008`. `rule0000002` intentionally shares its identity (`folder0000002`) with `state_paused.json`. - A real `for:1d` rule (`afs438kjd4v7kd` → `rule0000009`). @@ -81,7 +74,7 @@ here instead, for every fixture, for consistency. Grafana represents a datasource-managed (native Prometheus-format) alerting rule. `ParseDefinitions` must classify it as `KindDatasourceManaged`, parse `Title` from `alert`, and leave `UID` empty (this shape has no uid at all — inventing one would be inventing shape) without rejecting the rule - (rejection is P3's job, only for rules a user actually named). + (rejection is `Resolve`'s job, and only for rules a user actually named). - **ruler_recording.json** — **DERIVED**, no recording rule exists in the capture (verified: 0 rules carry `grafana_alert.record`). Hand-built: a `grafana_alert` block with a `record` sub-object but deliberately *without* `no_data_state`/`exec_err_state`/`is_paused`/`intervalSeconds`/`namespace_uid` diff --git a/grafana-alertcheck/internal/gate/watch.go b/grafana-alertcheck/internal/gate/watch.go new file mode 100644 index 000000000..74dd3945b --- /dev/null +++ b/grafana-alertcheck/internal/gate/watch.go @@ -0,0 +1,765 @@ +package gate + +import ( + "context" + "fmt" + "io" + "os" + "os/signal" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +// DaemonChildFlag is the hidden flag the parent passes when it re-execs itself +// as the detached recorder. It is deliberately absent from the CLI's +// usage text: an operator never types it, and a child started by hand against +// a log no parent prepared fails immediately on the header read. +const DaemonChildFlag = "--daemon-child" + +// ReadyFDFlag names the inherited descriptor the child reports readiness on. +// "Ready" is a POSITIVE byte from the child (header read, flock taken, in its +// poll loop), never a timer heuristic — a timer can't tell a healthy child from +// one about to die on a slow runner. +const ReadyFDFlag = "--ready-fd" + +// childReadyTimeout bounds that wait. Everything before the signal is local, so +// it is loose enough for an overloaded runner yet still fails closed. +const childReadyTimeout = 30 * time.Second + +// daemonLogTailBytes bounds how much of a dead child's output the parent +// quotes back. A child dies in its first few lines or not at all. +const daemonLogTailBytes = 4096 + +// WatchConfig is the record step's whole input. +// +// It has no To field (and must never gain one): watch writes the sentinel with +// its OWN stop time and makes no `to` comparison — only check knows `to`, and +// the recorder exits before the grace it would have to wait for. +// +// It has no States field either: recording is unfiltered, so the same raw log +// can be re-classified under different --states without re-recording. +type WatchConfig struct { + // URL and Token are the connection details. The CLI reads both from the + // environment and never from a flag; Token is never logged and never + // enters an error string. + URL, Token string + + // Alerts are the operator-supplied names, one per line, in any of the forms + // Resolve accepts. Empty lines are discarded by Resolve. + Alerts []string + Folder string + + // Out is the JSONL log path. PidFile and DaemonLog default to + // .pid and .daemon.log — the same convention check uses to find + // the recorder it must stop, so nothing has to be wired by hand. + Out string + PidFile string + DaemonLog string + + // Until is an optional hard stop for the child. Zero means "record until + // signalled", which is the normal case: check sends SIGTERM when its + // collection loop ends. + Until time.Time + + // PollEvery is the --poll-interval override, used verbatim for every rule + // and never clamped. Zero means each rule polls at half its own evaluation + // interval. Whatever this resolves to is written into the header as the + // cadence actually used, and that header value — never a re-derivation from + // the definitions — is what check derives maxGap from. + PollEvery time.Duration + + Concurrency int + Clock Clock + + // Notes is where the parent prints what an operator has to see before the + // deploy step runs: resolve notes, the cadence per rule, the rules it will + // not wait for. nil discards them. The library prints nothing else — the + // CLI owns presentation. + Notes io.Writer +} + +func (cfg WatchConfig) withDefaults() WatchConfig { + if cfg.Clock == nil { + cfg.Clock = SystemClock{} + } + if cfg.Notes == nil { + cfg.Notes = io.Discard + } + if cfg.Concurrency < 1 { + cfg.Concurrency = 1 + } + if cfg.PidFile == "" && cfg.Out != "" { + cfg.PidFile = cfg.Out + ".pid" + } + if cfg.DaemonLog == "" && cfg.Out != "" { + cfg.DaemonLog = cfg.Out + ".daemon.log" + } + return cfg +} + +func (cfg WatchConfig) validate() error { + if cfg.URL == "" { + return fmt.Errorf("watch: no grafana url (it is the log's identity, which check validates)") + } + if cfg.Out == "" { + return fmt.Errorf("watch: no log path") + } + named := 0 + for _, a := range cfg.Alerts { + if strings.TrimSpace(a) != "" { + named++ + } + } + if named == 0 { + return fmt.Errorf("watch: no alert names given; there is nothing to record") + } + // An --until already in the past would make the child stop before it ever + // polled, and the parent would then report a child that never reported + // ready — a true statement about a config mistake, but a confusing one. + if !cfg.Until.IsZero() && !cfg.Until.After(cfg.Clock.Now()) { + return fmt.Errorf("watch: --until %s is not in the future", cfg.Until.Format(time.RFC3339)) + } + return nil +} + +// Watch is the record step's parent process, returning only once the window is +// genuinely being recorded (version gate, resolve, write header, one +// observation per rule, budget check, then detach and await the child's +// readiness). The first-observation wait is what surfaces auth, name-resolution +// and parse failures before deploy.sh runs, rather than ten minutes later. +func Watch(ctx context.Context, cfg WatchConfig) error { + cfg = cfg.withDefaults() + if err := cfg.validate(); err != nil { + return err + } + + src := NewHTTPSource(cfg.URL, cfg.Token, cfg.Clock) + prep, err := prepareWatch(ctx, cfg, src) + if err != nil { + return err + } + + // Hand the log over with Close, never Stop: a sentinel here would tell + // check the recording ended before the child had even started. Closing + // also releases the flock the child is about to take. + if err := prep.writer.Close(); err != nil { + return err + } + + child, err := spawnChild(cfg) + if err != nil { + return fmt.Errorf("detach recorder: %w", err) + } + if err := waitForChildReady(cfg, child); err != nil { + return err + } + + // The PARENT writes the pidfile, not the child: check must find the pid the + // instant Watch returns, and a child writing its own would race the very + // next step of the pipeline. That is why the child is never given + // --pidfile at all. + // + // It is written only once the child has reported ready, so no path through + // this function leaves a pidfile naming a process that is not recording. + // Pids are reused: a stale pidfile is a live process somewhere else, and a + // pipeline that ignored this function's error would SIGTERM it. + if err := writePidFile(cfg.PidFile, child.cmd.Process.Pid); err != nil { + _ = child.cmd.Process.Kill() + _ = os.Remove(cfg.PidFile) + return err + } + fmt.Fprintf(cfg.Notes, "recording to %s (pid %d, pidfile %s, output %s)\n", + cfg.Out, child.cmd.Process.Pid, cfg.PidFile, cfg.DaemonLog) + return nil +} + +// waitForChildReady waits for the child's own readiness byte, and treats every +// other outcome as a failure to record: the pipe closing without a byte (the +// child died on its way to the loop), the process exiting, or the timeout. +// Each of the three quotes the daemon log, which is the only place a detached +// process can explain itself. +func waitForChildReady(cfg WatchConfig, child detachedChild) error { + defer child.ready.Close() + + // This goroutine outlives the function when the child keeps running. It + // stays behind only to reap a child that dies while this short-lived parent + // is still alive, and costs nothing. + exited := make(chan error, 1) + go func() { exited <- child.cmd.Wait() }() + + signalled := make(chan error, 1) + go func() { + _, err := child.ready.Read(make([]byte, 1)) + signalled <- err + }() + + fail := func(format string, args ...any) error { + _ = child.cmd.Process.Kill() + return fmt.Errorf("%s; its output was:\n%s", + fmt.Sprintf(format, args...), daemonLogTail(cfg.DaemonLog, child.logOffset)) + } + + select { + case err := <-signalled: + if err == nil { + return nil + } + // The child closed the pipe — by exiting — without ever reporting that + // it had the log and was polling. + return fail("the detached recorder never reported ready (%v)", err) + case waitErr := <-exited: + status := "exit status 0" + if waitErr != nil { + status = waitErr.Error() + } + return fail("the detached recorder exited before it started recording (%s)", status) + case <-cfg.Clock.After(childReadyTimeout): + return fail("the detached recorder did not report ready within %s", childReadyTimeout) + } +} + +// daemonLogTail quotes the end of the daemon log, starting at from — the size +// the file had when THIS run opened it. The offset is what keeps the quote +// honest when several runs share one --daemon-log path: without it the tail can +// name a previous run's failure as the current one's cause. +func daemonLogTail(path string, from int64) string { + f, err := os.Open(path) + if err != nil { + return fmt.Sprintf("(daemon log %s is unreadable: %v)", path, err) + } + defer f.Close() + + info, err := f.Stat() + if err != nil { + return fmt.Sprintf("(daemon log %s is unreadable: %v)", path, err) + } + size := info.Size() + + // Only the tail is ever quoted, so read at most daemonLogTailBytes from + // disk rather than the whole (possibly unbounded, shared-across-runs) file. + start := int64(0) + if from > 0 && from <= size { + start = from + } + if tailStart := size - daemonLogTailBytes; tailStart > start { + start = tailStart + } + if _, err := f.Seek(start, io.SeekStart); err != nil { + return fmt.Sprintf("(daemon log %s is unreadable: %v)", path, err) + } + + b, err := io.ReadAll(io.LimitReader(f, daemonLogTailBytes)) + if err != nil { + return fmt.Sprintf("(daemon log %s is unreadable: %v)", path, err) + } + if len(b) == 0 { + return fmt.Sprintf("(this run wrote nothing to the daemon log %s)", path) + } + return strings.TrimRight(string(b), "\n") +} + +// preparedWatch is what the parent has established by the time it is willing +// to detach: an open log with a header and one poll per non-skipped rule, the +// timings that produced them, and the latencies it measured doing so. +type preparedWatch struct { + writer *Writer + header Header + timings map[string]ruleTimings + measured map[string]time.Duration +} + +// prepareWatch is everything the parent does before it detaches. It takes a +// Source rather than building one so the paused-rule, first-observation, +// instance-visibility and budget behaviours are all testable with a scripted +// fake — only the process spawning needs a real binary. +func prepareWatch(ctx context.Context, cfg WatchConfig, src Source) (*preparedWatch, error) { + version, err := src.Version(ctx) + if err != nil { + return nil, fmt.Errorf("read grafana version: %w", err) + } + if err := CheckGrafanaVersion(version); err != nil { + return nil, err + } + + defs, err := src.Definitions(ctx) + if err != nil { + return nil, fmt.Errorf("read rule definitions: %w", err) + } + resolved, notes, err := Resolve(defs, cfg.Alerts, cfg.Folder) + if err != nil { + return nil, err + } + for _, n := range notes { + fmt.Fprintf(cfg.Notes, "note: %s\n", n) + } + + rt, _, timingNotes := DeriveTimings(resolved, cfg.PollEvery) + for _, n := range timingNotes { + fmt.Fprintf(cfg.Notes, "note: %s\n", n) + } + for _, d := range resolved { + // A cadence of zero would make the child spin: every rule is due the + // instant it was marked. It also cannot be written into the header, + // where check requires a positive value to derive maxGap from. + if rt[d.UID].pollEvery <= 0 { + return nil, fmt.Errorf("rule %q (%s) reports intervalSeconds=%d: there is no poll cadence to record at", + d.Title, d.UID, d.IntervalSeconds) + } + } + + writer, err := NewWriter(cfg.Out, cfg.Clock) + if err != nil { + return nil, err + } + prep, err := openRecording(ctx, cfg, src, writer, version, resolved, rt) + if err != nil { + // Close, never Stop. The log keeps whatever was written and gets no + // sentinel, so nothing can later mistake it for a finished recording. + _ = writer.Close() + return nil, err + } + return prep, nil +} + +// openRecording writes the header, takes the first observation of every rule +// the recorder will actually watch, appends those observations as the log's +// first heartbeats, and only then decides whether the schedule is feasible. +func openRecording(ctx context.Context, cfg WatchConfig, src Source, writer *Writer, + version string, resolved []Definition, rt map[string]ruleTimings) (*preparedWatch, error) { + + header := Header{ + SchemaVersion: LogSchemaVersion, + URL: cfg.URL, + GrafanaVersion: version, + StartedAt: cfg.Clock.Now(), + Rules: loggedRules(resolved, rt), + } + if err := writer.WriteHeader(header); err != nil { + return nil, err + } + + // A rule whose definition says is_paused is skipped (never waited for or + // polled); polling it would report an in-window pause (check 7) for a rule + // already paused at the open. The header still names it, so check reports + // it skipped from the definitions. + var active []Definition + activeTimings := make(map[string]ruleTimings, len(resolved)) + for _, d := range resolved { + if d.IsPaused { + fmt.Fprintf(cfg.Notes, "note: rule %q (%s) is paused: recorded as skipped, not waited for\n", d.Title, d.UID) + continue + } + active = append(active, d) + activeTimings[d.UID] = rt[d.UID] + fmt.Fprintf(cfg.Notes, "recording %q (%s) every %s (maxGap %s)\n", d.Title, d.UID, rt[d.UID].pollEvery, rt[d.UID].maxGap) + } + + polls, measured, err := firstObservations(ctx, src, active, NewReducer(), cfg.Concurrency, cfg.Notes) + if err != nil { + return nil, err + } + for _, p := range polls { + if err := writer.WritePoll(p); err != nil { + return nil, err + } + } + + // Budget last, on the latencies just measured — never on a fixed estimate. + // Only the active rules count: a skipped rule is never polled and consumes + // none of the capacity. + if err := CheckBudget(activeTimings, measured, cfg.Concurrency); err != nil { + return nil, err + } + + return &preparedWatch{writer: writer, header: header, timings: rt, measured: measured}, nil +} + +// loggedRules snapshots the resolved definitions into the header's rule list. +// Every field but PollEverySeconds is forensic — a resolve-time snapshot that +// makes an uploaded log self-describing — while PollEverySeconds is +// load-bearing: it is the cadence this recording actually used, and check +// derives maxGap from it rather than from the definitions. +func loggedRules(defs []Definition, rt map[string]ruleTimings) []LoggedRule { + out := make([]LoggedRule, 0, len(defs)) + for _, d := range defs { + out = append(out, LoggedRule{ + UID: d.UID, + Title: d.Title, + Folder: d.Folder, + Group: d.Group, + ForSeconds: d.For.Seconds(), + IntervalSeconds: d.IntervalSeconds, + IsPaused: d.IsPaused, + NoDataState: d.NoDataState, + ExecErrState: d.ExecErrState, + PollEverySeconds: rt[d.UID].pollEvery.Seconds(), + }) + } + return out +} + +// firstObservations takes one observation of every active rule, verifies normal +// instances are visible in those very responses, and reduces each into the +// window's first heartbeat, plus measured latency (the only honest budget +// input). Watches parent and single-step check both share it. Polls return in +// `active` order, so a log written from them is byte-stable. +func firstObservations(ctx context.Context, src Source, active []Definition, reducer *Reducer, + concurrency int, notes io.Writer) ([]Poll, map[string]time.Duration, error) { + + titles := make(map[string]string, len(active)) + uids := make([]string, 0, len(active)) + for _, d := range active { + titles[d.UID] = d.Title + uids = append(uids, d.UID) + } + + observed, err := observeAll(ctx, src, titles, uids, concurrency) + if err != nil { + return nil, nil, err + } + + // Verify this before anything downstream relies on it: if the state + // endpoint ever stops returning normal instances, the reduction's "keep + // the non-normal ones" silently becomes "keep everything it happened to + // send" and the transition markers lose their ground truth. + for _, d := range active { + if err := VerifyNormalInstancesVisible(observed[d.UID].Rules); err != nil { + return nil, nil, err + } + } + + polls := make([]Poll, 0, len(active)) + measured := make(map[string]time.Duration, len(active)) + for _, d := range active { + obs := observed[d.UID] + measured[d.UID] = obs.Latency + poll := reducer.Reduce(d.UID, obs) + if !poll.Found { + // Authoritative, not transient (the transport already retried + // every transient failure): the rule resolved in the ruler API but + // the state endpoint does not serve it. Recorded as Found=false, + // which the coverage proof turns into unobservable — a note rather + // than an error here, because the state endpoint can lag a freshly + // created rule and the coverage proof fails closed either way. + fmt.Fprintf(notes, "warning: rule %q (%s) is absent from the state endpoint; recorded as not found\n", d.Title, d.UID) + } + polls = append(polls, poll) + } + return polls, measured, nil +} + +// observeAll polls every rule in uids concurrently, bounded by concurrency, +// returning one Observation per rule that answered. Each rule is polled by +// TITLE (the ?rule_name= filter is a title filter) and selected by UID — a +// filtered response can carry several rules sharing a title. Returns the +// successes alongside the first error in UID order, so a caller can keep the +// good heartbeats. +func observeAll(ctx context.Context, src Source, titles map[string]string, uids []string, concurrency int) (map[string]Observation, error) { + if concurrency < 1 { + concurrency = 1 + } + var ( + mu sync.Mutex + out = make(map[string]Observation, len(uids)) + firstErr error + firstErrUID string + ) + sem := make(chan struct{}, concurrency) + var wg sync.WaitGroup + for _, uid := range uids { + wg.Go(func() { + sem <- struct{}{} + defer func() { <-sem }() + + obs, err := src.RuleState(ctx, titles[uid]) + + mu.Lock() + defer mu.Unlock() + if err != nil { + if firstErr == nil || uid < firstErrUID { + firstErr, firstErrUID = err, uid + } + return + } + out[uid] = obs + }) + } + wg.Wait() + + if firstErr != nil { + return out, fmt.Errorf("poll rule %q (%s): %w", titles[firstErrUID], firstErrUID, firstErr) + } + return out, nil +} + +// DaemonChildConfig is the detached recorder's whole input, and it is +// deliberately tiny. The rule set and every cadence come from the header the +// parent already wrote — one source of truth, no parent/child drift, and it +// exercises ReadLog's header path — and the connection details come from the +// inherited environment. Only the run facts the header does not carry travel +// in argv. +type DaemonChildConfig struct { + URL, Token string // from the inherited environment, never from argv + Out string + Until time.Time + Concurrency int + Clock Clock + // ReadyFD is the inherited descriptor to report readiness on (ReadyFDFlag). + // Zero means nobody is waiting — a hand-started child — and the report is + // then skipped rather than written to stdin. + ReadyFD int +} + +// RunDaemonChild is the detached recorder. The CLI dispatches to it when it +// sees DaemonChildFlag; nothing else ever calls it. +// +// It re-reads the log the parent wrote, restores the transition-marker state +// from the polls already in it, reopens the log for appending, takes the flock +// the parent released, and then polls until it is signalled or reaches Until. +func RunDaemonChild(ctx context.Context, cfg DaemonChildConfig) error { + if cfg.Clock == nil { + cfg.Clock = SystemClock{} + } + if cfg.Concurrency < 1 { + cfg.Concurrency = 1 + } + if cfg.Out == "" { + return fmt.Errorf("recorder: no log path") + } + + // Safe to read: the parent closed its writer before spawning this process, + // and no other writer can hold the log's flock. + header, polls, sentinel, err := ReadLog(cfg.Out) + if err != nil { + return err + } + if sentinel != nil { + return fmt.Errorf("log %s already carries a stopped sentinel: another recorder finished it", cfg.Out) + } + // The header's URL is the log's identity. Checking it here + // catches a child that inherited an environment pointing somewhere else, + // before it appends a single poll from the wrong Grafana. + if header.URL != cfg.URL { + return fmt.Errorf("log %s records url %q but this recorder is configured for %q", cfg.Out, header.URL, cfg.URL) + } + + titles, cadence, err := childSchedule(header) + if err != nil { + return err + } + + writer, err := NewWriter(cfg.Out, cfg.Clock) + if err != nil { + return err + } + + reducer := NewReducer() + reducer.seedFrom(polls) + + // SIGTERM is how check stops the recorder; SIGINT is the + // same request from a human at a terminal. Both are clean stops, so both + // end with a sentinel. Registered before the readiness report, so a signal + // arriving the moment the parent unblocks is already handled. + sigCtx, stop := signal.NotifyContext(ctx, syscall.SIGTERM, syscall.SIGINT) + defer stop() + + // Everything that can fail before a single poll has now succeeded: the + // header parsed, the identity matched, the flock is held. That — and not + // the mere fact of having been started — is what the parent waits for. + if err := reportReady(cfg.ReadyFD); err != nil { + return err + } + + return watchLoop(sigCtx, watchLoopConfig{ + Src: NewHTTPSource(cfg.URL, cfg.Token, cfg.Clock), + Writer: writer, + Reducer: reducer, + Titles: titles, + Cadence: cadence, + Until: cfg.Until, + Concurrency: cfg.Concurrency, + Clock: cfg.Clock, + }) +} + +// reportReady writes one byte to the inherited readiness descriptor and closes +// it. fd 0 means no parent is waiting: descriptor 0 is stdin, so it can never +// be a readiness pipe, which makes the zero value safe as "absent". +func reportReady(fd int) error { + if fd == 0 { + return nil + } + pipe := os.NewFile(uintptr(fd), "ready") + if pipe == nil { + return fmt.Errorf("readiness descriptor %d is not open", fd) + } + defer pipe.Close() + if _, err := pipe.Write([]byte{'1'}); err != nil { + return fmt.Errorf("report ready on descriptor %d: %w", fd, err) + } + return nil +} + +// childSchedule derives what the child polls, and how often, from the header +// alone. Cadence comes from PollEverySeconds (the cadence actually used), never +// re-derived from the evaluation interval; paused rules are excluded. It +// returns cadences only — the recorder must not carry coverage thresholds it +// has no business applying. +func childSchedule(h Header) (titles map[string]string, cadence map[string]time.Duration, err error) { + titles = make(map[string]string, len(h.Rules)) + cadence = make(map[string]time.Duration, len(h.Rules)) + for _, lr := range h.Rules { + if lr.IsPaused { + continue + } + if lr.PollEverySeconds <= 0 { + return nil, nil, fmt.Errorf("log header records poll_every_seconds=%v for rule %s (%q): there is no cadence to record at", + lr.PollEverySeconds, lr.UID, lr.Title) + } + if _, duplicate := titles[lr.UID]; duplicate { + return nil, nil, fmt.Errorf("log header names rule %s (%q) twice; its recorded cadence is ambiguous", lr.UID, lr.Title) + } + titles[lr.UID] = lr.Title + cadence[lr.UID] = time.Duration(lr.PollEverySeconds * float64(time.Second)) + } + return titles, cadence, nil +} + +// watchLoopConfig is the child's working state: what to poll, how often, and +// where to append it. There is no threshold in here and no policy — the child +// records and classifies nothing. +type watchLoopConfig struct { + Src Source + Writer *Writer + Reducer *Reducer + Titles map[string]string // uid -> title: poll by title, select by UID + Cadence map[string]time.Duration // uid -> pollEvery, as recorded in the header + Until time.Time + Concurrency int + Clock Clock +} + +// watchLoop is the child's whole working life: poll due rules, reduce each +// observation to a poll record, append it, and — on a clean stop only — finish +// the log with the stopped sentinel. +// +// The sentinel policy is load-bearing: a clean stop (signal or Until) writes +// it; a hard error does NOT. A recorder that died must look exactly like a +// coverage gap to check, because it is one. +func watchLoop(ctx context.Context, cfg watchLoopConfig) error { + sched := NewScheduler(cfg.Cadence, cfg.Clock.Now()) + + for { + if ctx.Err() != nil { + return cfg.Writer.Stop() + } + now := cfg.Clock.Now() + if !cfg.Until.IsZero() && !now.Before(cfg.Until) { + return cfg.Writer.Stop() + } + + due := sched.Due(now) + if len(due) == 0 { + wait, ok := untilNextPoll(sched, cfg.Until, now) + if !ok { + // Nothing will ever come due: every watched rule is paused and + // there is no hard stop. Wait for the signal — and still write + // a sentinel, because "the recorder ran and finished" is + // exactly what check needs to prove about the window. + <-ctx.Done() + return cfg.Writer.Stop() + } + select { + case <-ctx.Done(): + return cfg.Writer.Stop() + case <-cfg.Clock.After(wait): + } + continue + } + + // Mark before polling, against the batch's own now: the next poll is + // one cadence after this one was DUE, not one cadence after it + // returned, so request latency cannot make the heartbeat spacing drift + // towards maxGap. + for _, uid := range due { + sched.Mark(uid, now) + } + + pollErr := cfg.pollBatch(ctx, due) + if ctx.Err() != nil { + // Signalled while a poll was in flight. The aborted poll's error is + // not a recorder failure, and a clean stop wins over it: finish the + // in-flight write, then the sentinel. + return cfg.Writer.Stop() + } + if pollErr != nil { + return pollErr + } + } +} + +// pollBatch polls one round of due rules and appends every poll that +// succeeded, in due order, before returning the first failure. Writing the +// successes first is deliberate: a heartbeat that was genuinely observed is +// evidence, and dropping it because a different rule failed would turn one +// rule's transport failure into a coverage gap for the others. +func (cfg watchLoopConfig) pollBatch(ctx context.Context, uids []string) error { + observed, obsErr := observeAll(ctx, cfg.Src, cfg.Titles, uids, cfg.Concurrency) + for _, uid := range uids { + obs, ok := observed[uid] + if !ok { + continue + } + if err := cfg.Writer.WritePoll(cfg.Reducer.Reduce(uid, obs)); err != nil { + return err + } + } + return obsErr +} + +// untilNextPoll returns how long to wait for the next scheduled poll, cut +// short by Until when that comes first. ok is false when nothing will ever +// come due: no rules to poll and no hard stop. +func untilNextPoll(sched *Scheduler, until, now time.Time) (time.Duration, bool) { + next, hasNext := sched.earliestDue() + switch { + case hasNext && (until.IsZero() || next.Before(until)): + // keep next + case !until.IsZero(): + next = until + default: + return 0, false + } + return max(next.Sub(now), 0), true +} + +// writePidFile records the child's pid where check looks for it (its +// --pidfile, default .pid). The format is the decimal pid and a newline, +// so `kill $(cat log.jsonl.pid)` works and ReadPidFile stays trivial. +func writePidFile(path string, pid int) error { + if err := os.WriteFile(path, []byte(strconv.Itoa(pid)+"\n"), 0o644); err != nil { + return fmt.Errorf("write pidfile %s: %w", path, err) + } + return nil +} + +// ReadPidFile is the other side of that contract: the pid of the recorder +// check must stop before it may read the log. +func ReadPidFile(path string) (int, error) { + b, err := os.ReadFile(path) + if err != nil { + return 0, fmt.Errorf("read pidfile %s: %w", path, err) + } + text := strings.TrimSpace(string(b)) + pid, err := strconv.Atoi(text) + if err != nil { + return 0, fmt.Errorf("pidfile %s: unparseable pid %q", path, text) + } + if pid <= 0 { + return 0, fmt.Errorf("pidfile %s: %d is not a pid", path, pid) + } + return pid, nil +} diff --git a/grafana-alertcheck/internal/gate/watch_daemon_test.go b/grafana-alertcheck/internal/gate/watch_daemon_test.go new file mode 100644 index 000000000..73a6f3d7a --- /dev/null +++ b/grafana-alertcheck/internal/gate/watch_daemon_test.go @@ -0,0 +1,334 @@ +package gate + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/signal" + "path/filepath" + "slices" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// TestMain doubles this test binary as the detached recorder. Watch spawns +// os.Executable(), which under `go test` is this binary, so the one integration +// test below exercises the real thing — a real fork/exec, a real setsid, a real +// inherited environment, a real SIGTERM — with this function standing in for +// the CLI's `watch --daemon-child` dispatch. +func TestMain(m *testing.M) { + if path := os.Getenv(lockHolderEnv); path != "" { + os.Exit(runTestLockHolder(path)) + } + if slices.Contains(os.Args, DaemonChildFlag) { + os.Exit(runTestDaemonChild(os.Args[1:])) + } + os.Exit(m.Run()) +} + +// lockHolderEnv turns this test binary into a stand-in recorder that holds the +// log's flock and refuses to die: a process check's stop protocol must wait +// for and, on a timeout, refuse to read around. +// +// It has to be a real second process. flock is what stopRecorder probes, and +// there is no flock(1) on darwin, so a shell one-liner cannot take the lock — +// while a lock taken in the test process itself would be granted to the probe +// on some platforms and prove nothing. +const lockHolderEnv = "GRAFANA_ALERTCHECK_TEST_LOCK_HOLDER" + +// runTestLockHolder takes the log's exclusive lock, reports that it has it on +// stdout, ignores SIGTERM, and waits to be killed. The report is what lets the +// test start only once the lock is genuinely held, rather than racing it. +func runTestLockHolder(path string) int { + signal.Ignore(syscall.SIGTERM) + + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + if err := lockExclusive(f); err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + fmt.Println("locked") + + // Long enough to outlive any test that starts it; the test kills it, and + // SIGKILL is not ignorable. + time.Sleep(5 * time.Minute) + return 0 +} + +// runTestDaemonChild parses the child argv childArgs() writes, and reads the +// connection details from the environment — never from argv. The CLI's `watch` +// FlagSet does the same four flags. +func runTestDaemonChild(args []string) int { + cfg := DaemonChildConfig{ + URL: os.Getenv("GRAFANA_URL"), + Token: os.Getenv("GRAFANA_TOKEN"), + } + for i := 0; i < len(args); i++ { + value := func() string { + if i+1 >= len(args) { + fmt.Fprintf(os.Stderr, "flag %s wants a value\n", args[i]) + os.Exit(2) + } + i++ + return args[i] + } + switch args[i] { + case "--out": + cfg.Out = value() + case "--until": + until, err := time.Parse(time.RFC3339, value()) + if err != nil { + fmt.Fprintf(os.Stderr, "--until: %v\n", err) + return 2 + } + cfg.Until = until + case "--concurrency": + n, err := strconv.Atoi(value()) + if err != nil { + fmt.Fprintf(os.Stderr, "--concurrency: %v\n", err) + return 2 + } + cfg.Concurrency = n + case ReadyFDFlag: + fd, err := strconv.Atoi(value()) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", ReadyFDFlag, err) + return 2 + } + cfg.ReadyFD = fd + } + } + if err := RunDaemonChild(context.Background(), cfg); err != nil { + fmt.Fprintln(os.Stderr, err) + return 2 + } + return 0 +} + +// testBearerToken is what every request to grafanaTestServer must carry. The +// child never receives it in argv, so a request that arrives +// authenticated is proof that the token reached the detached process through +// the inherited environment — and a 401 is what a test sees if that ever +// breaks. +const testBearerToken = "test-token" + +// grafanaTestServer serves the three endpoints the record step reads, from the +// real captured fixtures: /api/health, the ruler definitions, and the +// rule_name-filtered state response. The state body is the one-instance +// fixture with its uid and name patched to the ruler fixture's live rule, so +// the reducer's select-by-UID finds it. +func grafanaTestServer(t *testing.T) *httptest.Server { + t.Helper() + ruler := readFixture(t, "ruler_rules.json") + state := patchedStateBody(t) + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get("Authorization"); got != "Bearer "+testBearerToken { + // Not t.Errorf: this must reach the client as a real 401, so the + // parent fails its version gate and the child fails its polls. + http.Error(w, "unauthorized: Authorization = "+got, http.StatusUnauthorized) + return + } + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/api/health": + fmt.Fprint(w, healthBody("13.1.0")) + case strings.HasPrefix(r.URL.Path, "/api/ruler/"): + _, _ = w.Write(ruler) + case strings.HasPrefix(r.URL.Path, "/api/prometheus/"): + if r.URL.Query().Get("rule_name") == "" { + // The gate must never read the state endpoint unfiltered. + http.Error(w, "unfiltered state read", http.StatusBadRequest) + return + } + _, _ = w.Write(state) + default: + http.Error(w, "unexpected path "+r.URL.Path, http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + return srv +} + +func patchedStateBody(t *testing.T) []byte { + t.Helper() + var body map[string]any + require.NoError(t, json.Unmarshal(readFixture(t, "state_one_instance.json"), &body)) + data, ok := body["data"].(map[string]any) + require.True(t, ok, "state fixture: no data object") + groups, ok := data["groups"].([]any) + require.True(t, ok, "state fixture: no groups") + require.NotEmpty(t, groups, "state fixture: no groups") + group, ok := groups[0].(map[string]any) + require.True(t, ok, "state fixture: group 0 is not an object") + rules, ok := group["rules"].([]any) + require.True(t, ok, "state fixture: group 0 has no rules") + require.NotEmpty(t, rules, "state fixture: group 0 has no rules") + rule, ok := rules[0].(map[string]any) + require.True(t, ok, "state fixture: rule 0 is not an object") + rule["uid"] = watchActiveUID + rule["name"] = watchActiveTitle + rule["lastEvaluation"] = time.Now().UTC().Format(time.RFC3339Nano) + + b, err := json.Marshal(body) + require.NoError(t, err) + return b +} + +// waitFor polls cond until it holds. This is the one tier of the project where +// a test waits on real time: it drives real processes over real HTTP, so there +// is no clock to fake. +func waitFor(t *testing.T, what string, timeout time.Duration, cond func() bool) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(20 * time.Millisecond) + } + require.Fail(t, fmt.Sprintf("timed out after %s waiting for %s", timeout, what)) +} + +// The one watch integration test: everything from the version gate to the +// sentinel, through a real detached process. +// +// It asserts the four things only a real spawn can show — the pidfile points +// at a live process, that process is in its own session (setsid, not a bare +// `&`), it keeps appending after Watch returned, and SIGTERM makes it finish +// the log in the stop order — and it uses a 200ms --poll-interval to do it in +// about a second, which also exercises the unclamped-override path. +func TestWatchSpawnsADetachedRecorder(t *testing.T) { + srv := grafanaTestServer(t) + t.Setenv("GRAFANA_URL", srv.URL) + t.Setenv("GRAFANA_TOKEN", testBearerToken) + + out := filepath.Join(t.TempDir(), "log.jsonl") + var notes strings.Builder + cfg := WatchConfig{ + URL: srv.URL, + Token: testBearerToken, + Alerts: []string{"uid:" + watchActiveUID}, + Out: out, + PollEvery: 200 * time.Millisecond, + Concurrency: 2, + Notes: ¬es, + } + + require.NoError(t, Watch(context.Background(), cfg)) + t.Cleanup(func() { + if t.Failed() { + t.Logf("notes:\n%s", notes.String()) + t.Logf("daemon log:\n%s", daemonLogTail(out+".daemon.log", 0)) + } + }) + + pid, err := ReadPidFile(out + ".pid") + require.NoError(t, err) + require.NoError(t, syscall.Kill(pid, 0), "recorder pid %d is not running right after Watch returned", pid) + // Setsid, not a bare `&`: a session leader's process group id is its own + // pid. Without this the child would still share the parent's process group + // and die with the step that started it. + pgid, err := syscall.Getpgid(pid) + require.NoError(t, err) + require.Equal(t, pid, pgid, "it did not get its own session") + + // The parent already wrote the first heartbeat before it returned; + // these later ones prove the detached child is the one appending now. + waitFor(t, "the detached recorder to append its own polls", 10*time.Second, func() bool { + _, polls, _, err := ReadLog(out) + return err == nil && len(polls) >= 3 + }) + + // Stop it exactly the way check does. + require.NoError(t, syscall.Kill(pid, syscall.SIGTERM)) + waitFor(t, "the stopped sentinel", 10*time.Second, func() bool { + _, _, sentinel, err := ReadLog(out) + return err == nil && sentinel != nil + }) + + header, polls, sentinel, err := ReadLog(out) + require.NoError(t, err) + require.Equal(t, srv.URL, header.URL) + require.Equal(t, "13.1.0", header.GrafanaVersion) + require.Len(t, header.Rules, 1) + require.Equal(t, float64(0.2), header.Rules[0].PollEverySeconds) + for i, p := range polls { + require.Equalf(t, watchActiveUID, p.RuleUID, "poll %d", i) + require.Truef(t, p.Found, "poll %d", i) + require.Falsef(t, p.GrafanaNow.IsZero(), "poll %d has no grafana_now; every poll needs the Date header of its own response", i) + } + require.False(t, sentinel.Before(header.StartedAt), "sentinel precedes the record start") + + waitFor(t, "the recorder to exit", 10*time.Second, func() bool { + return syscall.Kill(pid, 0) != nil + }) +} + +// TestDaemonChildRejectsAnAlreadyFinishedLog covers the RunDaemonChild guard +// against a reused --out path: a log that already carries a stopped sentinel is +// a finished recording, and a child starting against it would either append to +// a window already declared over, or take a flock over evidence that is about +// to be classified — so it must refuse before polling once. This is the +// fail-closed counterpart of §4.5 on the recorder's own startup path. +func TestDaemonChildRejectsAnAlreadyFinishedLog(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + clock := newFakeClock(testNow) + + w, err := NewWriter(path, clock) + require.NoError(t, err) + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.Stop()) + + err = RunDaemonChild(context.Background(), DaemonChildConfig{ + URL: testHeader().URL, + Out: path, + Clock: clock, + }) + require.Error(t, err, "no error against a log that already carries a stopped sentinel") + require.Contains(t, err.Error(), "sentinel") +} + +// TestWatchFailsWhenTheChildCannotStartRecording is the other half of the +// readiness contract. The child dies on its identity check, so it never reports +// ready — and Watch must say so instead of returning success over a window +// nothing is recording, and must leave no pidfile naming a dead process for the +// next step to signal. +// +// The child is made to fail through the environment, which is the only channel +// it takes its connection details from: the header says one URL and the +// inherited GRAFANA_URL says another. +func TestWatchFailsWhenTheChildCannotStartRecording(t *testing.T) { + srv := grafanaTestServer(t) + t.Setenv("GRAFANA_URL", srv.URL+"/somewhere-else") + t.Setenv("GRAFANA_TOKEN", testBearerToken) + + out := filepath.Join(t.TempDir(), "log.jsonl") + var notes strings.Builder + err := Watch(context.Background(), WatchConfig{ + URL: srv.URL, // what the parent uses, and what the header records + Token: testBearerToken, + Alerts: []string{"uid:" + watchActiveUID}, + Out: out, + PollEvery: 200 * time.Millisecond, + Concurrency: 2, + Notes: ¬es, + }) + require.Error(t, err, "the child could never have started recording") + require.Contains(t, err.Error(), "records url") + _, statErr := os.Stat(out + ".pid") + require.True(t, os.IsNotExist(statErr), + "a pidfile survived a failed detach; pids are reused, so the next step would signal a stranger") +} diff --git a/grafana-alertcheck/internal/gate/watch_process.go b/grafana-alertcheck/internal/gate/watch_process.go new file mode 100644 index 000000000..169a4f65a --- /dev/null +++ b/grafana-alertcheck/internal/gate/watch_process.go @@ -0,0 +1,106 @@ +package gate + +import ( + "fmt" + "os" + "os/exec" + "strconv" + "syscall" + "time" +) + +// readyFD is where ExtraFiles[0] lands in the child: exec.Cmd starts extra +// descriptors at 3, after stdin, stdout and stderr. +const readyFD = 3 + +// detachedChild is a started recorder: the process, the read end of its +// readiness pipe, and the size the daemon log had before it wrote anything. +type detachedChild struct { + cmd *exec.Cmd + ready *os.File + // logOffset is where this run's output starts in the daemon log, so a + // failure quotes this child and never a previous run's. + logOffset int64 +} + +// spawnChild re-execs this binary as the detached recorder. A trailing +// `&` is NOT sufficient: the child would keep the parent's session and process +// group, so it would still take the terminal's signals and, on a runner, die +// with the step that started it. Setsid gives it a new session AND a new +// process group, which is what makes it survive to the end of the window. +func spawnChild(cfg WatchConfig) (detachedChild, error) { + exe, err := os.Executable() + if err != nil { + return detachedChild{}, fmt.Errorf("find own executable: %w", err) + } + + // The child is detached, so its output has nowhere to go but a file, and + // that file is the only place it can ever explain a failure. Opened + // O_APPEND, never O_TRUNC: --daemon-log is an operator-supplied path and + // this process does not get to destroy what is already in it. The offset + // below is what keeps a shared path from misattributing a failure. + logFile, err := os.OpenFile(cfg.DaemonLog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return detachedChild{}, fmt.Errorf("open daemon log %s: %w", cfg.DaemonLog, err) + } + // The child inherits the descriptor at Start; this process does not need + // its own copy afterwards. + defer logFile.Close() + + var logOffset int64 + if info, err := logFile.Stat(); err == nil { + logOffset = info.Size() + } + + // The readiness pipe: the child gets the write end as + // descriptor 3 and reports on it once it holds the log and is polling. + readyRead, readyWrite, err := os.Pipe() + if err != nil { + return detachedChild{}, fmt.Errorf("open readiness pipe: %w", err) + } + + cmd := exec.Command(exe, childArgs(cfg)...) + cmd.Stdin = nil // /dev/null + cmd.Stdout = logFile + cmd.Stderr = logFile + cmd.ExtraFiles = []*os.File{readyWrite} // descriptor 3 in the child + // The environment is how the connection details reach the child: the token + // must never appear in argv, where it would land in the process table and + // in CI logs. + cmd.Env = os.Environ() + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + + if err := cmd.Start(); err != nil { + readyRead.Close() + readyWrite.Close() + return detachedChild{}, fmt.Errorf("start recorder %s: %w", exe, err) + } + // Drop the parent's copy of the write end at once: with only the child + // holding it, a child that dies before signalling closes the pipe and the + // parent reads EOF instead of waiting out the whole timeout. + readyWrite.Close() + + return detachedChild{cmd: cmd, ready: readyRead, logOffset: logOffset}, nil +} + +// childArgs builds the child's command line. The rule set, every cadence and +// the recording's identity all come from the header the parent already wrote, +// and the connection details come from the environment — so what is left here +// is the log path plus the two run facts the header does not carry: the +// optional hard stop and the concurrency limit. +// +// Notably absent: --pidfile (the parent writes it, so check can find the pid +// the instant Watch returns), --alerts, --folder, --poll-interval, and +// anything derived from them. The CLI's `watch` FlagSet needs one flag of its +// own for this path — ReadyFDFlag — and dispatches to RunDaemonChild when it +// sees DaemonChildFlag. +func childArgs(cfg WatchConfig) []string { + args := []string{"watch", DaemonChildFlag, "--out", cfg.Out, ReadyFDFlag, strconv.Itoa(readyFD)} + if !cfg.Until.IsZero() { + args = append(args, "--until", cfg.Until.Format(time.RFC3339)) + } + if cfg.Concurrency > 0 { + args = append(args, "--concurrency", strconv.Itoa(cfg.Concurrency)) + } + return args +} diff --git a/grafana-alertcheck/internal/gate/watch_test.go b/grafana-alertcheck/internal/gate/watch_test.go new file mode 100644 index 000000000..41ee3b09b --- /dev/null +++ b/grafana-alertcheck/internal/gate/watch_test.go @@ -0,0 +1,620 @@ +package gate + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// The two fixture rules every prepareWatch test below uses: one live, one +// paused in its definition. Both are addressed by uid:, because the ruler +// fixture deliberately contains a 2-way title collision and a title would make +// the tests depend on which side of it they hit. +const ( + watchActiveUID = "rule0000009" + watchActiveTitle = "Example Failure Ratio Above 10 Percent" + watchPausedUID = "rule0000007" + watchPausedTitle = "example_workflow_paused_rule" +) + +// loopSource answers every RuleState call from a responder that also sees the +// call count, so a recorder-loop test can make the answer depend on virtual +// time or fail on the Nth poll. The loop never reads Version or Definitions — +// the parent did that before detaching — so both fail loudly here. +type loopSource struct { + mu sync.Mutex + calls map[string]int + respond func(title string, call int) (Observation, error) +} + +func newLoopSource(respond func(title string, call int) (Observation, error)) *loopSource { + return &loopSource{calls: map[string]int{}, respond: respond} +} + +func (s *loopSource) Version(context.Context) (string, error) { + return "", errors.New("loopSource: the recorder loop must not read the version") +} + +func (s *loopSource) Definitions(context.Context) ([]Definition, error) { + return nil, errors.New("loopSource: the recorder loop must not read the definitions") +} + +func (s *loopSource) RuleState(_ context.Context, title string) (Observation, error) { + s.mu.Lock() + s.calls[title]++ + call := s.calls[title] + s.mu.Unlock() + return s.respond(title, call) +} + +var _ Source = (*loopSource)(nil) + +// testStateRule is one rule as the state endpoint would return it, healthy and +// evaluated at grafanaNow. +func testStateRule(uid, title string, interval time.Duration, grafanaNow time.Time, instances ...Instance) StateRule { + totals := map[string]int{"normal": len(instances)} + return StateRule{ + UID: uid, Title: title, Folder: "F", Group: "G", + Interval: interval, State: "inactive", Health: "ok", + LastEvaluation: grafanaNow, Totals: totals, Instances: instances, + } +} + +// newLoopWriter opens a log with a header already written, exactly as the +// parent hands it to the child. +func newLoopWriter(t *testing.T, path string, clock Clock) *Writer { + t.Helper() + w, err := NewWriter(path, clock) + require.NoError(t, err) + require.NoError(t, w.WriteHeader(testHeader())) + return w +} + +func countPolls(polls []Poll, uid string) int { + n := 0 + for _, p := range polls { + if p.RuleUID == uid { + n++ + } + } + return n +} + +// The per-rule schedule seen from the recorder: a 10s rule beside a 300s one +// keeps its own 5s cadence instead of dragging the slack rule along with it or +// being slowed to its pace. +func TestWatchLoopPollsEachRuleAtItsOwnCadence(t *testing.T) { + const tightUID, slackUID = "tight", "slack" + path := filepath.Join(t.TempDir(), "log.jsonl") + clock := newVirtualClock(testNow) + w := newLoopWriter(t, path, clock) + + src := newLoopSource(func(title string, _ int) (Observation, error) { + uid := tightUID + if title == "Slack Rule" { + uid = slackUID + } + now := clock.Now() + return observation(now, testStateRule(uid, title, time.Minute, now)), nil + }) + + err := watchLoop(context.Background(), watchLoopConfig{ + Src: src, + Writer: w, + Reducer: NewReducer(), + Titles: map[string]string{tightUID: "Tight Rule", slackUID: "Slack Rule"}, + Cadence: map[string]time.Duration{ + tightUID: 5 * time.Second, + slackUID: 150 * time.Second, + }, + Until: testNow.Add(300 * time.Second), + Concurrency: 2, + Clock: clock, + }) + require.NoError(t, err) + + _, polls, sentinel, readErr := ReadLog(path) + require.NoError(t, readErr) + require.NotNil(t, sentinel, "no stopped sentinel after a clean stop") + require.False(t, sentinel.Before(testNow.Add(300*time.Second))) + // 300s of window at 5s and 150s, minus the initial stagger offset of up to + // one cadence: 59-60 and 1-2. The assertion is the ratio, not the exact + // count — a single global cycle would give both rules the same number. + got := countPolls(polls, tightUID) + require.GreaterOrEqual(t, got, 59) + require.LessOrEqual(t, got, 61) + got = countPolls(polls, slackUID) + require.GreaterOrEqual(t, got, 1) + require.LessOrEqual(t, got, 3) +} + +// Fail-closed from the recorder's side: a recorder that dies must look exactly +// like a coverage gap, so it must not sign off the log on its way out. +func TestWatchLoopHardErrorLeavesNoSentinel(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + clock := newVirtualClock(testNow) + w := newLoopWriter(t, path, clock) + + boom := errors.New("grafana went away for good") + src := newLoopSource(func(title string, call int) (Observation, error) { + if call >= 2 { + return Observation{}, boom + } + now := clock.Now() + return observation(now, testStateRule("r1", title, time.Minute, now)), nil + }) + + err := watchLoop(context.Background(), watchLoopConfig{ + Src: src, + Writer: w, + Reducer: NewReducer(), + Titles: map[string]string{"r1": "Example"}, + Cadence: map[string]time.Duration{"r1": 30 * time.Second}, + Until: testNow.Add(time.Hour), + Concurrency: 1, + Clock: clock, + }) + require.ErrorIs(t, err, boom) + + _, polls, sentinel, readErr := ReadLog(path) + require.NoError(t, readErr) + require.Nil(t, sentinel, "check would read that as a finished window") + require.Len(t, polls, 1, "want the 1 that succeeded before the failure") +} + +// SIGTERM arriving while a poll is in flight is a clean stop, so the aborted +// poll's error must not suppress the sentinel — otherwise every normal check +// run, which stops the recorder exactly this way, would end unobservable. +func TestWatchLoopSignalDuringPollIsACleanStop(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + clock := newVirtualClock(testNow) + w := newLoopWriter(t, path, clock) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + src := newLoopSource(func(title string, call int) (Observation, error) { + if call >= 2 { + // The signal lands while this request is out. + cancel() + return Observation{}, ctx.Err() + } + now := clock.Now() + return observation(now, testStateRule("r1", title, time.Minute, now)), nil + }) + + require.NoError(t, watchLoop(ctx, watchLoopConfig{ + Src: src, + Writer: w, + Reducer: NewReducer(), + Titles: map[string]string{"r1": "Example"}, + Cadence: map[string]time.Duration{"r1": 30 * time.Second}, + Concurrency: 1, + Clock: clock, + })) + + _, _, sentinel, err := ReadLog(path) + require.NoError(t, err) + require.NotNil(t, sentinel, "no sentinel after a signalled stop; check would call a fully observed window unobservable") +} + +// TestWatchLoopWithNothingToPollStillFinishesTheLog covers the every-rule-is- +// paused case: there is nothing to record, but "the recorder ran and finished" +// is still what check has to prove about the window. +func TestWatchLoopWithNothingToPollStillFinishesTheLog(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + clock := newVirtualClock(testNow) + w := newLoopWriter(t, path, clock) + + src := newLoopSource(func(title string, _ int) (Observation, error) { + return Observation{}, fmt.Errorf("nothing should be polled, got %q", title) + }) + + require.NoError(t, watchLoop(context.Background(), watchLoopConfig{ + Src: src, + Writer: w, + Reducer: NewReducer(), + Titles: map[string]string{}, + Cadence: map[string]time.Duration{}, + Until: testNow.Add(time.Minute), + Concurrency: 1, + Clock: clock, + })) + + _, polls, sentinel, err := ReadLog(path) + require.NoError(t, err) + require.Empty(t, polls) + require.NotNil(t, sentinel, "no sentinel: check cannot tell this recording from one that died") +} + +// TestWatchLoopPollBatchKeepsTheHeartbeatsItGot: one rule's failure must not +// discard another rule's observed heartbeat, or a single transport failure +// turns into a coverage gap for every rule that answered. +func TestWatchLoopPollBatchKeepsTheHeartbeatsItGot(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl") + clock := newVirtualClock(testNow) + w := newLoopWriter(t, path, clock) + + boom := errors.New("one rule is unreachable") + src := newLoopSource(func(title string, _ int) (Observation, error) { + if title == "Broken" { + return Observation{}, boom + } + now := clock.Now() + return observation(now, testStateRule("ok", title, time.Minute, now)), nil + }) + + cfg := watchLoopConfig{ + Src: src, + Writer: w, + Reducer: NewReducer(), + Titles: map[string]string{"ok": "Healthy", "bad": "Broken"}, + Cadence: map[string]time.Duration{"ok": 30 * time.Second, "bad": 30 * time.Second}, + Concurrency: 2, + Clock: clock, + } + require.ErrorIs(t, cfg.pollBatch(context.Background(), []string{"ok", "bad"}), boom) + require.NoError(t, w.Close()) + + _, polls, _, err := ReadLog(path) + require.NoError(t, err) + require.Len(t, polls, 1) + require.Equal(t, "ok", polls[0].RuleUID) +} + +// The vanish-versus-clear distinction at the one seam the parent/child handoff +// introduces. The parent observes a firing instance; the child starts with a +// fresh Reducer and sees the instance gone. Seeded, that is a vanish — a +// discontinuity. Unseeded, it is nothing at all, and the instance silently +// leaves the record as if it had never been bad. +func TestReducerSeedFromKeepsMarkersAcrossTheHandoff(t *testing.T) { + firing := testInstance(StateFiring, "", "b") + key := instanceKey(firing.Labels) + parentPoll := Poll{RuleUID: "r1", Found: true, Abnormal: []Instance{firing}} + // The child's first response: the instance is gone from the response + // entirely, which is a vanish and never a clear. + childObs := observation(testNow, testStateRule("r1", "Example", time.Minute, testNow)) + + t.Run("seeded", func(t *testing.T) { + r := NewReducer() + r.seedFrom([]Poll{parentPoll}) + p := r.Reduce("r1", childObs) + require.Contains(t, p.Vanished, key) + require.Empty(t, p.Cleared, "a vanish is not a recovery") + }) + + t.Run("unseeded loses the transition", func(t *testing.T) { + p := NewReducer().Reduce("r1", childObs) + require.Empty(t, p.Vanished, "this subtest exists to show the seed is what produces the marker") + }) + + t.Run("a not-found poll does not clear the seed", func(t *testing.T) { + r := NewReducer() + r.seedFrom([]Poll{parentPoll, {RuleUID: "r1", Found: false}}) + p := r.Reduce("r1", childObs) + require.Contains(t, p.Vanished, key, "an absent rule leaves the abnormal set untouched") + }) +} + +// watchTestConfig is a prepareWatch config over a temp log, with the notes +// captured so the tests can assert on what an operator is told. +func watchTestConfig(t *testing.T, notes *strings.Builder, alerts ...string) WatchConfig { + t.Helper() + return WatchConfig{ + URL: "https://grafana.example.com", + Token: "secret-token", + Alerts: alerts, + Out: filepath.Join(t.TempDir(), "log.jsonl"), + Concurrency: 2, + Clock: newFakeClock(testNow), + Notes: notes, + }.withDefaults() +} + +// watchTestSource is a fakeSource with the real ruler fixture and a scripted +// state response for the live rule only. The paused rule is deliberately +// unscripted: fakeSource errors on an unscripted title, so any attempt to poll +// it fails the test rather than passing silently. +func watchTestSource(t *testing.T, obs Observation) *fakeSource { + t.Helper() + src := newFakeSource() + src.version = "13.1.0" + src.defs = rulerDefs(t) + src.script(watchActiveTitle, obs, nil) + return src +} + +func liveObservation(grafanaNow time.Time) Observation { + return observation(grafanaNow, testStateRule(watchActiveUID, watchActiveTitle, time.Minute, grafanaNow, + testInstance(StateNormal, "", "a"))) +} + +// A rule paused in its definition is skipped, never waited for. Waiting for one +// either hangs forever or errors before the deploy — and the header must still +// name it, so check can report it as skipped rather than lose it. +func TestPrepareWatchDoesNotWaitForPausedRules(t *testing.T) { + var notes strings.Builder + cfg := watchTestConfig(t, ¬es, "uid:"+watchActiveUID, "uid:"+watchPausedUID) + src := watchTestSource(t, liveObservation(testNow)) + + prep, err := prepareWatch(context.Background(), cfg, src) + require.NoError(t, err) + require.NoError(t, prep.writer.Close()) + + header, polls, sentinel, err := ReadLog(cfg.Out) + require.NoError(t, err) + require.Nil(t, sentinel, "the parent wrote a sentinel; that would tell check the recording ended before the child started") + + require.Len(t, header.Rules, 2) + for _, lr := range header.Rules { + require.Positive(t, lr.PollEverySeconds, + "check needs a positive cadence to derive maxGap from") + if lr.UID == watchPausedUID { + require.True(t, lr.IsPaused, "want the resolve-time snapshot to say true") + } + } + + // One poll, for the live rule only — and it is already in the log before + // prepareWatch returned, which is the whole point of the record step. + require.Len(t, polls, 1) + require.Equal(t, watchActiveUID, polls[0].RuleUID) + require.True(t, polls[0].Found) + require.True(t, polls[0].GrafanaNow.Equal(testNow)) + // The poll record holds the state histogram, asserted through a real + // prepareWatch()/Reducer call rather than log_test.go's hand-built + // Writer/ReadLog round trip. + require.Equal(t, map[string]int{"normal": 1}, polls[0].Histogram) + require.Contains(t, notes.String(), watchPausedTitle) + require.Contains(t, notes.String(), "paused") +} + +// One authority for the cadence, from the writing side: whatever +// --poll-interval resolves to is what the header records, because that is the +// only value check may derive maxGap from. +func TestPrepareWatchHeaderRecordsTheOverriddenCadence(t *testing.T) { + var notes strings.Builder + cfg := watchTestConfig(t, ¬es, "uid:"+watchActiveUID) + cfg.PollEvery = 120 * time.Second // the rule evaluates every 60s + src := watchTestSource(t, liveObservation(testNow)) + + prep, err := prepareWatch(context.Background(), cfg, src) + require.NoError(t, err) + defer prep.writer.Close() + + require.Equal(t, float64(120), prep.header.Rules[0].PollEverySeconds, + "the override, used verbatim and never clamped") + require.Equal(t, 240*time.Second, prep.timings[watchActiveUID].maxGap) + require.Contains(t, notes.String(), "--poll-interval") +} + +// The budget check runs on the latencies the parent just measured, before the +// deploy runs. +func TestPrepareWatchFailsWhenTheScheduleDoesNotFit(t *testing.T) { + var notes strings.Builder + cfg := watchTestConfig(t, ¬es, "uid:"+watchActiveUID) + + obs := liveObservation(testNow) + obs.Latency = 60 * time.Second // against a 30s cadence + src := watchTestSource(t, obs) + + _, err := prepareWatch(context.Background(), cfg, src) + require.Error(t, err, "a schedule cannot hold its own cadence") + assertBudgetMessage(t, err.Error()) +} + +// Normal instances are verified visible at the one place it is still cheap: +// the first observation. If the state endpoint stops returning them, the +// reduction's predicate quietly inverts. +func TestPrepareWatchVerifiesNormalInstancesAreVisible(t *testing.T) { + var notes strings.Builder + cfg := watchTestConfig(t, ¬es, "uid:"+watchActiveUID) + + rule := testStateRule(watchActiveUID, watchActiveTitle, time.Minute, testNow, testInstance(StateFiring, "", "b")) + rule.Totals = map[string]int{"alerting": 1, "normal": 4} // claims normals it did not return + src := watchTestSource(t, observation(testNow, rule)) + + _, err := prepareWatch(context.Background(), cfg, src) + require.Error(t, err, "totals claim normal instances the response omitted") + require.Contains(t, err.Error(), "no longer returns normal instances") + + // The failure happens before any poll is appended, so the log holds a + // header and nothing else. + _, polls, _, readErr := ReadLog(cfg.Out) + require.NoError(t, readErr) + require.Empty(t, polls) +} + +func TestPrepareWatchRejectsAnUnsupportedGrafana(t *testing.T) { + var notes strings.Builder + cfg := watchTestConfig(t, ¬es, "uid:"+watchActiveUID) + src := watchTestSource(t, liveObservation(testNow)) + src.version = "12.4.0" + + _, err := prepareWatch(context.Background(), cfg, src) + require.Error(t, err) + require.Contains(t, err.Error(), "12.4.0") + require.Contains(t, err.Error(), "13.0.0") +} + +// A rule that resolved in the ruler API but is absent from the state endpoint +// is recorded as Found=false — authoritative evidence the coverage proof turns +// into unobservable — not silently dropped. +func TestPrepareWatchNotesAnAbsentRule(t *testing.T) { + var notes strings.Builder + cfg := watchTestConfig(t, ¬es, "uid:"+watchActiveUID) + src := watchTestSource(t, observation(testNow)) // an authoritative, empty 2xx + + prep, err := prepareWatch(context.Background(), cfg, src) + require.NoError(t, err) + require.NoError(t, prep.writer.Close()) + + _, polls, _, err := ReadLog(cfg.Out) + require.NoError(t, err) + require.Len(t, polls, 1) + require.False(t, polls[0].Found, "want one poll recorded as not found") + require.Contains(t, notes.String(), "absent from the state endpoint") +} + +func TestWatchConfigValidation(t *testing.T) { + base := func() WatchConfig { + return WatchConfig{ + URL: "https://grafana.example.com", + Alerts: []string{"Example"}, + Out: filepath.Join(t.TempDir(), "log.jsonl"), + Clock: newFakeClock(testNow), + } + } + + for _, tc := range []struct { + name string + mutate func(*WatchConfig) + want string + }{ + {"no url", func(c *WatchConfig) { c.URL = "" }, "url"}, + {"no log path", func(c *WatchConfig) { c.Out = "" }, "no log path"}, + {"no alerts", func(c *WatchConfig) { c.Alerts = nil }, "no alert names"}, + {"blank alerts only", func(c *WatchConfig) { c.Alerts = []string{"", " "} }, "no alert names"}, + {"until in the past", func(c *WatchConfig) { c.Until = testNow.Add(-time.Second) }, "not in the future"}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := base() + tc.mutate(&cfg) + err := cfg.withDefaults().validate() + require.Errorf(t, err, "validate: no error, want one naming %q", tc.want) + require.Containsf(t, err.Error(), tc.want, "validate error") + }) + } + + t.Run("defaults derive the pidfile and daemon log from the log path", func(t *testing.T) { + cfg := base().withDefaults() + require.Equal(t, cfg.Out+".pid", cfg.PidFile) + require.NotEmpty(t, cfg.DaemonLog, "a detached child would have nowhere to explain a failure") + require.NoError(t, cfg.validate()) + }) +} + +// The fail-open direction, checked on the child's side: a log recorded at 5s on +// a 300s rule must schedule at 5s. +// Re-deriving from the interval would give 150s — and every real 250s hole in +// that recording would pass. +func TestChildScheduleUsesTheRecordedCadence(t *testing.T) { + h := Header{Rules: []LoggedRule{ + {UID: "fast", Title: "Fast", IntervalSeconds: 300, PollEverySeconds: 5}, + {UID: "paused", Title: "Paused", IntervalSeconds: 60, PollEverySeconds: 30, IsPaused: true}, + }} + + titles, cadence, err := childSchedule(h) + require.NoError(t, err) + _, ok := titles["paused"] + require.False(t, ok, "the child scheduled a rule that was paused when the window opened") + require.Equal(t, 5*time.Second, cadence["fast"]) +} + +func TestChildScheduleRejectsAnUnusableHeader(t *testing.T) { + for _, tc := range []struct { + name string + h Header + want string + }{ + { + "no recorded cadence", + Header{Rules: []LoggedRule{{UID: "r1", Title: "Example", IntervalSeconds: 60}}}, + "no cadence", + }, + { + "the same rule twice", + Header{Rules: []LoggedRule{ + {UID: "r1", Title: "Example", IntervalSeconds: 60, PollEverySeconds: 30}, + {UID: "r1", Title: "Example", IntervalSeconds: 60, PollEverySeconds: 300}, + }}, + "twice", + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, _, err := childSchedule(tc.h) + require.Errorf(t, err, "childSchedule: no error, want one naming %q", tc.want) + require.Contains(t, err.Error(), tc.want) + }) + } +} + +// TestChildArgsCarryNoSecretsAndNoRuleSet: the child's command line lands in +// the process table and in CI logs. Everything it needs about the rules comes +// from the header, and everything about the connection comes from the +// environment — so argv holds the log path and the two run facts only. +func TestChildArgsCarryNoSecretsAndNoRuleSet(t *testing.T) { + cfg := WatchConfig{ + URL: "https://grafana.example.com", + Token: "secret-token", + Alerts: []string{"Example"}, + Folder: "F", + Out: "/tmp/log.jsonl", + PidFile: "/tmp/log.jsonl.pid", + Until: testNow.Add(time.Hour), + PollEvery: 17 * time.Second, + Concurrency: 3, + } + args := childArgs(cfg) + joined := strings.Join(args, " ") + + for _, want := range []string{DaemonChildFlag, "--out /tmp/log.jsonl", "--concurrency 3", "--until ", ReadyFDFlag + " 3"} { + require.Contains(t, joined, want) + } + for _, forbidden := range []string{"secret-token", "Example", "--folder", "--poll-interval", "--pidfile"} { + require.NotContains(t, joined, forbidden) + } +} + +func TestPidFileRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), "log.jsonl.pid") + require.NoError(t, writePidFile(path, 4242)) + pid, err := ReadPidFile(path) + require.NoError(t, err) + require.Equal(t, 4242, pid) + + t.Run("garbage is an error, never a pid", func(t *testing.T) { + bad := filepath.Join(t.TempDir(), "bad.pid") + require.NoError(t, os.WriteFile(bad, []byte("not-a-pid\n"), 0o644)) + _, err := ReadPidFile(bad) + require.Error(t, err, "no error on an unparseable pidfile") + }) +} + +func TestDaemonLogTail(t *testing.T) { + t.Run("missing file is unreadable", func(t *testing.T) { + out := daemonLogTail(filepath.Join(t.TempDir(), "nope.daemon.log"), 0) + require.Contains(t, out, "unreadable") + }) + + t.Run("small file returns its content", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "small.daemon.log") + require.NoError(t, os.WriteFile(path, []byte("line one\nline two\n"), 0o644)) + require.Equal(t, "line one\nline two", daemonLogTail(path, 0)) + }) + + t.Run("large file keeps only the tail", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "large.daemon.log") + prefix := strings.Repeat("P", 1000) + suffix := strings.Repeat("S", daemonLogTailBytes) + require.NoError(t, os.WriteFile(path, []byte(prefix+suffix), 0o644)) + require.Equal(t, suffix, daemonLogTail(path, 0)) + }) + + t.Run("offset skips a previous run's content", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "shared.daemon.log") + prior := strings.Repeat("p", 2000) + require.NoError(t, os.WriteFile(path, []byte(prior), 0o644)) + from := int64(len(prior)) + thisRun := "this run's output\n" + require.NoError(t, os.WriteFile(path, []byte(prior+thisRun), 0o644)) + require.Equal(t, "this run's output", daemonLogTail(path, from)) + }) +}