diff --git a/.github/workflows/grafana-alertcheck-release.yml b/.github/workflows/grafana-alertcheck-release.yml deleted file mode 100644 index 4f946b18f..000000000 --- a/.github/workflows/grafana-alertcheck-release.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Grafana Alertcheck Release - -on: - push: - tags: - - grafana-alertcheck/v* - -jobs: - release: - name: Build and Release - runs-on: ubuntu-latest - environment: integration - permissions: - id-token: write - contents: write - steps: - - name: Checkout repo - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@v7 - with: - go-version-file: ./grafana-alertcheck/go.mod - cache-dependency-path: ./grafana-alertcheck/go.mod - - name: Goreleaser Release - uses: goreleaser/goreleaser-action@f06c13b6b1a9625abc9e6e439d9c05a8f2190e94 # v7.2.3 - with: - distribution: goreleaser-pro - version: "~> v2" - args: release --clean -f ./grafana-alertcheck/.goreleaser.yaml - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} diff --git a/grafana-alertcheck/.goreleaser.yaml b/grafana-alertcheck/.goreleaser.yaml deleted file mode 100644 index 0890d9f82..000000000 --- a/grafana-alertcheck/.goreleaser.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# yaml-language-server: $schema=https://goreleaser.com/static/schema-pro.json -version: 2 -project_name: grafana-alertcheck - -dist: grafana-alertcheck/dist - -monorepo: - tag_prefix: grafana-alertcheck/ - dir: grafana-alertcheck - -builds: - - id: grafana-alertcheck - main: ./cmd/grafana-alertcheck/main.go - ldflags: - - -s - - -w - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd/grafana-alertcheck.version={{.Version}} - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd/grafana-alertcheck.commit={{.ShortCommit}} - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd/grafana-alertcheck.date={{.CommitDate}} - - -X github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd/grafana-alertcheck.builtBy=goreleaser - goos: - - linux - - darwin - goarch: - - amd64 - - arm64 - binary: grafana-alertcheck - env: - - CGO_ENABLED=0 - -before: - hooks: - - sh -c "cd grafana-alertcheck && go mod tidy" diff --git a/grafana-alertcheck/README.md b/grafana-alertcheck/README.md index 43cc03fc4..96aa0e999 100644 --- a/grafana-alertcheck/README.md +++ b/grafana-alertcheck/README.md @@ -1,6 +1,46 @@ # grafana-alertcheck -A CD quality gate for Grafana alerts: bookend a release with `watch` (record) and `check` (classify) to -answer whether any watched alert was in a bad state during the release window. +A CD quality gate for Grafana alerts. It bookends a release with two commands — `watch` (record) and +`check` (classify) — and answers whether any watched alert was in a bad state during the release window. -Under construction. +``` +watch → your work → check +``` + +`watch` starts a background recorder that polls each named alert into a JSONL log. After the work emits a +`from`/`to` pair, `check` proves continuous coverage of that window, classifies each alert's state +timeline, and exits `0`, `1`, or `2`. + +It **fails closed**: if it cannot get an answer, it stops the release — never a pass on an unproven window. + +## Quickstart + +```bash +export GRAFANA_URL=https://grafana.example.com +export GRAFANA_TOKEN=… + +grafana-alertcheck watch --out /tmp/run.jsonl --alerts alerts.txt +./deploy.sh # emits deployed_at= +./verify.sh # emits finished_at= +grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finished_at" +``` + +Requires Grafana >= 13.0.0 and < 14.0.0. Connection details come from the environment only — the token is +never a flag. + +## Documentation + +| Doc | Covers | +| --- | ------ | +| [`docs/index.md`](./docs/index.md) | Overview, quickstarts, exit codes, common surprises | +| [`docs/how-alerts-are-evaluated.md`](./docs/how-alerts-are-evaluated.md) | Verdict model, coverage proof, health/liveness | +| [`docs/advanced.md`](./docs/advanced.md) | Check budget, scheduling, why history isn't queried | +| [`docs/architecture.md`](./docs/architecture.md) | Design invariants, the pure-function seam, recorder lifecycle | +| [`docs/reference/cli.md`](./docs/reference/cli.md) | Full CLI reference — subcommands, flags, naming | +| [`docs/reference/log-format.md`](./docs/reference/log-format.md) | The JSONL log schema, for debugging artifacts | + +## Build + +```bash +go build ./... && go test ./... +``` diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/check.go b/grafana-alertcheck/cmd/check.go similarity index 94% rename from grafana-alertcheck/cmd/grafana-alertcheck/check.go rename to grafana-alertcheck/cmd/check.go index 3f2d295d6..e95fce376 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/check.go +++ b/grafana-alertcheck/cmd/check.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "flag" "fmt" "io" @@ -40,6 +41,13 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { output := fs.String("output", "", `"json" writes the machine-readable Result to stdout in addition to the table; default is the table alone`) if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() != 0 { + fmt.Fprintf(stderr, "check: unexpected arguments %v\n", fs.Args()) return 2 } if *output != "" && *output != "json" { @@ -82,7 +90,7 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { PidFile: *pidfile, Concurrency: *common.concurrency, Clock: gate.SystemClock{}, - Notes: stderr, + Notes: newNoteStyler(stderr), } if *to == "" { fmt.Fprintln(stderr, "check: --to is required") @@ -113,11 +121,10 @@ func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { result, checkErr := gate.Check(ctx, cfg) - if err := renderTable(stderr, result); err != nil { - fmt.Fprintln(stderr, err) - } if checkErr != nil { fmt.Fprintln(stderr, checkErr) + } else if err := renderTable(stderr, result); err != nil { + fmt.Fprintln(stderr, err) } if *output == "json" { enc := json.NewEncoder(stdout) diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/check_test.go b/grafana-alertcheck/cmd/check_test.go similarity index 84% rename from grafana-alertcheck/cmd/grafana-alertcheck/check_test.go rename to grafana-alertcheck/cmd/check_test.go index 99e8ebc3a..c7c797a0b 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/check_test.go +++ b/grafana-alertcheck/cmd/check_test.go @@ -4,10 +4,10 @@ import ( "bytes" "errors" "os" - "strings" "testing" "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" + "github.com/stretchr/testify/require" ) // The exit-code mapping, pinned directly against exitCode with no network @@ -27,9 +27,7 @@ func TestExitCode(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := exitCode(tt.res, tt.err); got != tt.want { - t.Fatalf("exitCode(...) = %d, want %d", got, tt.want) - } + require.Equal(t, tt.want, exitCode(tt.res, tt.err)) }) } } @@ -37,9 +35,7 @@ func TestExitCode(t *testing.T) { func writeTempAlerts(t *testing.T) string { t.Helper() path := t.TempDir() + "/alerts.txt" - if err := os.WriteFile(path, []byte("Some Alert\n"), 0o644); err != nil { - t.Fatal(err) - } + require.NoError(t, os.WriteFile(path, []byte("Some Alert\n"), 0o644)) return path } @@ -100,12 +96,8 @@ func TestRunCheck_FlagValidation(t *testing.T) { var stdout, stderr bytes.Buffer args := append([]string{"check"}, tt.args(t)...) code := run(args, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2; stderr = %q", code, stderr.String()) - } - if !strings.Contains(stderr.String(), tt.wantErr) { - t.Fatalf("stderr = %q, want it to contain %q", stderr.String(), tt.wantErr) - } + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), tt.wantErr) }) } } @@ -121,12 +113,8 @@ func TestRunCheck_ToInPastNoLog(t *testing.T) { "--from", "1999-01-01T00:00:00Z", "--to", "2000-01-01T00:00:00Z", "--alerts", writeTempAlerts(t), }, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2; stderr = %q", code, stderr.String()) - } - if !strings.Contains(stderr.String(), "already passed") { - t.Fatalf("stderr = %q, want the past-`to` refusal", stderr.String()) - } + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), "already passed") } // --output json never writes to stdout when Check was never reached, because @@ -137,10 +125,6 @@ func TestRunCheck_NoResultOnConfigError(t *testing.T) { t.Setenv("GRAFANA_TOKEN", "") var stdout, stderr bytes.Buffer code := run([]string{"check", "--to", "2026-01-01T00:00:00Z", "--output", "json"}, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2", code) - } - if stdout.Len() != 0 { - t.Fatalf("stdout = %q, want empty", stdout.String()) - } + require.Equal(t, 2, code) + require.Empty(t, stdout.String()) } diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/common.go b/grafana-alertcheck/cmd/common.go similarity index 100% rename from grafana-alertcheck/cmd/grafana-alertcheck/common.go rename to grafana-alertcheck/cmd/common.go diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/env.go b/grafana-alertcheck/cmd/env.go similarity index 100% rename from grafana-alertcheck/cmd/grafana-alertcheck/env.go rename to grafana-alertcheck/cmd/env.go diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/main_test.go b/grafana-alertcheck/cmd/grafana-alertcheck/main_test.go deleted file mode 100644 index 1d7906674..000000000 --- a/grafana-alertcheck/cmd/grafana-alertcheck/main_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package main - -import ( - "bytes" - "strings" - "testing" -) - -func TestRun_NoArgs(t *testing.T) { - var stdout, stderr bytes.Buffer - code := run(nil, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), "usage") { - t.Fatalf("stderr = %q, want a usage message", stderr.String()) - } -} - -func TestRun_Help(t *testing.T) { - for _, flag := range []string{"-h", "-help", "--help"} { - t.Run(flag, func(t *testing.T) { - var stdout, stderr bytes.Buffer - code := run([]string{flag}, &stdout, &stderr) - if code != 0 { - t.Fatalf("code = %d, want 0 (requested help is not a could-not-check condition)", code) - } - if !strings.Contains(stdout.String(), "usage") { - t.Fatalf("stdout = %q, want a usage message", stdout.String()) - } - if stderr.String() != "" { - t.Fatalf("stderr = %q, want empty — help goes to stdout", stderr.String()) - } - }) - } -} - -func TestRun_UnknownSubcommand(t *testing.T) { - var stdout, stderr bytes.Buffer - code := run([]string{"bogus"}, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), `"bogus"`) { - t.Fatalf("stderr = %q, want it to name the unknown subcommand", stderr.String()) - } -} - -func TestRun_List_MissingEnv(t *testing.T) { - t.Setenv("GRAFANA_URL", "") - t.Setenv("GRAFANA_TOKEN", "") - var stdout, stderr bytes.Buffer - code := run([]string{"list"}, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), "GRAFANA_URL") { - t.Fatalf("stderr = %q, want it to name the missing env var", stderr.String()) - } -} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/version.go b/grafana-alertcheck/cmd/grafana-alertcheck/version.go deleted file mode 100644 index 5cf68b6af..000000000 --- a/grafana-alertcheck/cmd/grafana-alertcheck/version.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "fmt" - "io" -) - -// Build metadata. These are package-level variables so goreleaser's ldflags -// (-X) can stamp them at build time; left unstamped they fall back to the -// "dev" defaults below, which is what a plain `go build` produces. -var ( - version = "dev" - commit = "unknown" - date = "unknown" - builtBy = "unknown" -) - -// runVersion prints the build metadata to stdout. Unlike list/watch/check it -// needs no Grafana connection, so it never touches the environment or the -// network; it exists purely so operators can answer "what am I running?" -// against a deployed binary. -func runVersion(args []string, stdout, stderr io.Writer) int { - if len(args) != 0 { - fmt.Fprintf(stderr, "version takes no arguments, got %v\n", args) - return 2 - } - fmt.Fprintf(stdout, "version: %s\ncommit: %s\ndate: %s\nbuiltBy: %s\n", version, commit, date, builtBy) - return 0 -} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/version_test.go b/grafana-alertcheck/cmd/grafana-alertcheck/version_test.go deleted file mode 100644 index 4122c2f13..000000000 --- a/grafana-alertcheck/cmd/grafana-alertcheck/version_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "bytes" - "strings" - "testing" -) - -func TestRunVersion(t *testing.T) { - var stdout, stderr bytes.Buffer - code := run([]string{"version"}, &stdout, &stderr) - if code != 0 { - t.Fatalf("code = %d, want 0", code) - } - out := stdout.String() - for _, want := range []string{"version:", "commit:", "date:", "builtBy:"} { - if !strings.Contains(out, want) { - t.Errorf("stdout = %q, want it to contain %q", out, want) - } - } - if stderr.String() != "" { - t.Errorf("stderr = %q, want empty", stderr.String()) - } -} - -func TestRunVersion_RejectsArgs(t *testing.T) { - var stdout, stderr bytes.Buffer - code := run([]string{"version", "extra"}, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2", code) - } -} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/list.go b/grafana-alertcheck/cmd/list.go similarity index 100% rename from grafana-alertcheck/cmd/grafana-alertcheck/list.go rename to grafana-alertcheck/cmd/list.go diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/list_test.go b/grafana-alertcheck/cmd/list_test.go similarity index 73% rename from grafana-alertcheck/cmd/grafana-alertcheck/list_test.go rename to grafana-alertcheck/cmd/list_test.go index 119326c2c..62aa200d2 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/list_test.go +++ b/grafana-alertcheck/cmd/list_test.go @@ -5,8 +5,9 @@ import ( "fmt" "net/http" "net/http/httptest" - "strings" "testing" + + "github.com/stretchr/testify/require" ) const rulerBody = `{ @@ -60,19 +61,11 @@ func TestRunList_HappyPath(t *testing.T) { var stdout, stderr bytes.Buffer code := run([]string{"list"}, &stdout, &stderr) - if code != 0 { - t.Fatalf("code = %d, want 0; stderr = %q", code, stderr.String()) - } + require.Equal(t, 0, code) out := stdout.String() - if !strings.Contains(out, "rule0000006a") { - t.Errorf("stdout = %q, want it to list rule0000006a", out) - } - if !strings.Contains(out, "Example No Gateways Available") { - t.Errorf("stdout = %q, want it to list the rule title", out) - } - if !strings.Contains(out, "grafana-managed") { - t.Errorf("stdout = %q, want it to name the rule kind", out) - } + require.Contains(t, out, "rule0000006a") + require.Contains(t, out, "Example No Gateways Available") + require.Contains(t, out, "grafana-managed") } func TestRunList_UnsupportedVersion(t *testing.T) { @@ -82,12 +75,8 @@ func TestRunList_UnsupportedVersion(t *testing.T) { var stdout, stderr bytes.Buffer code := run([]string{"list"}, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2", code) - } - if !strings.Contains(stderr.String(), "12.5.0") { - t.Fatalf("stderr = %q, want it to name the unsupported version", stderr.String()) - } + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), "12.5.0") } func TestRunList_RejectsArgs(t *testing.T) { @@ -96,7 +85,5 @@ func TestRunList_RejectsArgs(t *testing.T) { var stdout, stderr bytes.Buffer code := run([]string{"list", "extra"}, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2", code) - } + require.Equal(t, 2, code) } diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/main.go b/grafana-alertcheck/cmd/main.go similarity index 91% rename from grafana-alertcheck/cmd/grafana-alertcheck/main.go rename to grafana-alertcheck/cmd/main.go index 2672de1a3..7ab5d6397 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/main.go +++ b/grafana-alertcheck/cmd/main.go @@ -12,7 +12,7 @@ func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } -const usage = "usage: grafana-alertcheck " +const usage = "usage: grafana-alertcheck " // run is the whole of main's testable surface: parse the subcommand, dispatch, // return the process exit code. Exit codes below 2 (pass/violations) belong to @@ -37,8 +37,6 @@ func run(args []string, stdout, stderr io.Writer) int { return runWatch(args[1:], os.Stdin, stdout, stderr) case "check": return runCheck(args[1:], os.Stdin, stdout, stderr) - case "version": - return runVersion(args[1:], stdout, stderr) case "-h", "-help", "--help": fmt.Fprintln(stdout, usage) return 0 diff --git a/grafana-alertcheck/cmd/main_test.go b/grafana-alertcheck/cmd/main_test.go new file mode 100644 index 000000000..34dd3b2d2 --- /dev/null +++ b/grafana-alertcheck/cmd/main_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRun_NoArgs(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run(nil, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), "usage") +} + +func TestRun_Help(t *testing.T) { + for _, flag := range []string{"-h", "-help", "--help"} { + t.Run(flag, func(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{flag}, &stdout, &stderr) + require.Equal(t, 0, code, "requested help is not a could-not-check condition") + require.Contains(t, stdout.String(), "usage") + require.Empty(t, stderr.String(), "help goes to stdout") + }) + } +} + +func TestRun_UnknownSubcommand(t *testing.T) { + var stdout, stderr bytes.Buffer + code := run([]string{"bogus"}, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), `"bogus"`) +} + +func TestRun_List_MissingEnv(t *testing.T) { + t.Setenv("GRAFANA_URL", "") + t.Setenv("GRAFANA_TOKEN", "") + var stdout, stderr bytes.Buffer + code := run([]string{"list"}, &stdout, &stderr) + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), "GRAFANA_URL") +} diff --git a/grafana-alertcheck/cmd/style.go b/grafana-alertcheck/cmd/style.go new file mode 100644 index 000000000..bb46fc2ec --- /dev/null +++ b/grafana-alertcheck/cmd/style.go @@ -0,0 +1,125 @@ +package main + +import ( + "bytes" + "io" + "os" + "strings" +) + +// ANSI SGR codes for the human-facing notes and table footer. The colours are +// applied only when the destination is a terminal (see colorEnabled); a pipe, +// file or CI log gets plain text, so stdout stays reserved for --output json +// and no machine reader ever sees escape sequences. +const ( + ansiReset = "\x1b[0m" + ansiRed = "\x1b[31m" + ansiGreen = "\x1b[32m" + ansiYellow = "\x1b[33m" + ansiCyan = "\x1b[36m" + // Orange has no entry in the base-16 palette; 256-colour 208 is a legible + // orange used for warnings, distinct from the yellow used for notes. + ansiOrange = "\x1b[38;5;208m" +) + +// colorEnabled reports whether ANSI colour should be written to w. Colour is +// written only when three things hold: NO_COLOR is unset, w is a real *os.File +// (so text/tabwriter buffers, strings.Builder and bytes.Buffer tests all stay +// plain), and that file is a character device (a terminal, not a redirect). +func colorEnabled(w io.Writer) bool { + if os.Getenv("NO_COLOR") != "" { + return false + } + f, ok := w.(*os.File) + if !ok { + return false + } + fi, err := f.Stat() + if err != nil { + return false + } + return fi.Mode()&os.ModeCharDevice != 0 +} + +// styleLine applies the note vocabulary's colour to one line when enabled. The +// colour wraps the text only; the terminating newline is written uncoloured so +// the terminal's line discipline is never inside the escape sequence. +func styleLine(line string, enabled bool) string { + if !enabled { + return line + } + content := strings.TrimRight(line, "\n") + var color string + switch { + case strings.HasPrefix(content, "warning:"): + color = ansiOrange + case strings.HasPrefix(content, "note:"): + color = ansiYellow + case strings.HasPrefix(content, "drain wait:"): + color = ansiCyan + } + if color == "" { + return line + } + return color + content + ansiReset + "\n" +} + +// noteStyler wraps the gate package's Notes stream — a presentation seam that +// keeps colour out of the library. It colourises each line by its known prefix +// and separates the collection countdown from the setup phase with a single +// blank line before the first "collecting:" line. The gate keeps emitting plain +// prose; only the CLI lays it out. +type noteStyler struct { + w io.Writer + enabled bool + pending []byte + sawCollecting bool +} + +func newNoteStyler(w io.Writer) *noteStyler { + return ¬eStyler{w: w, enabled: colorEnabled(w)} +} + +// startsSection reports whether a line opens a new phase of the stream and so +// deserves a blank line above it. "collecting:" opens the countdown (once — +// later countdown lines follow on from the first), and "drain wait:" opens the +// drain phase. The setup lines (planned run time, warning, min-observed, notes) +// are one contiguous block and are not separated from each other. +func (s *noteStyler) startsSection(line string) bool { + switch { + case strings.HasPrefix(line, "warning:"): + return true + case strings.HasPrefix(line, "drain wait:"): + return true + case strings.HasPrefix(line, "collecting:"): + if s.sawCollecting { + return false + } + s.sawCollecting = true + return true + } + return false +} + +func (s *noteStyler) Write(p []byte) (int, error) { + n := len(p) + s.pending = append(s.pending, p...) + for { + i := bytes.IndexByte(s.pending, '\n') + if i < 0 { + break + } + line := string(s.pending[:i+1]) + s.pending = s.pending[i+1:] + + if s.startsSection(line) { + if _, err := io.WriteString(s.w, "\n"); err != nil { + return n, err + } + } + if _, err := io.WriteString(s.w, styleLine(line, s.enabled)); err != nil { + return n, err + } + } + return n, nil +} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/table.go b/grafana-alertcheck/cmd/table.go similarity index 55% rename from grafana-alertcheck/cmd/grafana-alertcheck/table.go rename to grafana-alertcheck/cmd/table.go index 11b72a074..39ec75cbc 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/table.go +++ b/grafana-alertcheck/cmd/table.go @@ -14,29 +14,39 @@ import ( // which the caller (runCheck) always points at stderr — stdout is reserved for // the machine-readable --output json. // -// Three sections, in order: +// Three titled tables, in order (the name column is RULE in all of them — one +// row is one resolved alert rule, never a firing instance): // -// 1. one line per rule: outcome, BadFor, pollEvery, proved-or-not with the -// largest gap; -// 2. one line per Violation: a rule's worst-of outcome does not carry the -// State/Health of the instance that actually caused it — Violation does — -// so this is also where those two columns appear, sorted after the rule -// table rather than folded into it, and it is the only place an operator -// running WITHOUT --output json sees the --allow-paused hint that -// Violation.Note already carries (classify.go); -// 3. a footer with the numbers that answer "why" on exit 2: each non-skipped -// rule's maxGap/healthGrace/evalStaleAfter, the global transitionGrace and -// drainTimeout, and the largest measured clock skew alongside its own -// error bound (RTT/2) — SkewHardLimit is a separate, fixed input threshold -// and is reported next to it, never as if it were that bound. +// 1. RESULTS, one line per rule: outcome, BadFor, pollEvery, proved-or-not +// with the largest gap; +// 2. VIOLATIONS, one line per Violation (only when any): a rule's worst-of +// outcome does not carry the State/Health of the instance that actually +// caused it — Violation does — so this is also where those two columns +// appear, sorted after the result table rather than folded into it, and it +// is the only place an operator running WITHOUT --output json sees the +// --allow-paused hint that Violation.Note already carries (classify.go); +// 3. THRESHOLDS, the numbers that answer "why" on exit 2: each non-skipped +// rule's maxGap/healthGrace/evalStaleAfter, followed by the global +// transitionGrace and drainTimeout, and the largest measured clock skew +// alongside its own error bound (RTT/2) — SkewHardLimit is a separate, +// fixed input threshold and is reported next to it, never as if it were +// that bound. func renderTable(w io.Writer, res gate.Result) error { alertOf := make(map[string]string, len(res.Verdicts)) for _, v := range res.Verdicts { alertOf[v.RuleUID] = v.Alert } + enabled := colorEnabled(w) + // A blank line separates the result table from the notes the gate streamed + // before it (planned run time, warning, min-observed, collecting, drain + // wait), so the verdict reads as its own section rather than the tail of a + // wall of progress text. + fmt.Fprintln(w) + + fmt.Fprintln(w, "RESULTS") tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - fmt.Fprintln(tw, "ALERT\tOUTCOME\tBADFOR\tPOLLEVERY\tPROVED\tNOTE") + fmt.Fprintln(tw, "RULE\tOUTCOME\tBADFOR\tPOLLEVERY\tPROVED\tNOTE") for _, v := range sortedVerdicts(res.Verdicts) { fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", v.Alert, v.Outcome, v.BadFor.Round(time.Second), v.PollEvery.Round(time.Second), @@ -49,7 +59,7 @@ func renderTable(w io.Writer, res gate.Result) error { if len(res.Violations) > 0 { fmt.Fprintln(w, "\nVIOLATIONS") vtw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) - fmt.Fprintln(vtw, "ALERT\tOUTCOME\tSTATE\tHEALTH\tNOTE") + fmt.Fprintln(vtw, "RULE\tOUTCOME\tSTATE\tHEALTH\tNOTE") for _, v := range sortedViolations(res.Violations) { fmt.Fprintf(vtw, "%s\t%s\t%s\t%s\t%s\n", alertLabel(v, alertOf), v.Outcome, v.State, v.Health, v.Note) } @@ -58,20 +68,49 @@ func renderTable(w io.Writer, res gate.Result) error { } } + // The per-rule thresholds answer "why" on exit 2: a table, not the prose + // "rule NAME: maxGap=... healthGrace=... evalStaleAfter=..." that repeated + // the rule name a fourth time. It is separated from the result above by a + // blank line. fmt.Fprintln(w) + fmt.Fprintln(w, "THRESHOLDS") + ttw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + fmt.Fprintln(ttw, "RULE\tMAXGAP\tHEALTHGRACE\tEVALSTALEAFTER") for _, uid := range sortedThresholdUIDs(res.Thresholds, alertOf) { t := res.Thresholds[uid] - fmt.Fprintf(w, "rule %s: maxGap=%s healthGrace=%s evalStaleAfter=%s\n", + fmt.Fprintf(ttw, "%s\t%s\t%s\t%s\n", alertOr(uid, alertOf), t.MaxGap, t.HealthGrace, t.EvalStaleAfter) } + if err := ttw.Flush(); err != nil { + return fmt.Errorf("render table: %w", err) + } + + fmt.Fprintln(w) fmt.Fprintf(w, "global: transitionGrace=%s (source: %s) drainTimeout=%s\n", res.Global.TransitionGrace, res.Global.GraceSource, res.Global.DrainTimeout) - fmt.Fprintf(w, "violations: %d, largest measured clock skew: %s (bound ±%s, hard limit %s), grafana %s\n", - len(res.Violations), res.ClockSkew.Round(time.Millisecond), res.ClockSkewBound.Round(time.Millisecond), + fmt.Fprintf(w, "largest measured clock skew: %s (bound ±%s, hard limit %s), grafana %s\n", + res.ClockSkew.Round(time.Millisecond), res.ClockSkewBound.Round(time.Millisecond), gate.SkewHardLimit, res.GrafanaVersion) + // The verdict — the single number a terminal operator reads last — sits on + // its own line at the very bottom, separated from the diagnostics above and + // from the shell prompt below. + fmt.Fprintf(w, "\n%s\n\n", violationsLabel(len(res.Violations), enabled)) return nil } +// violationsLabel colours the "violations: N" prefix of the footer: green for a +// clean run, red otherwise. The rest of the line is written uncoloured. +func violationsLabel(n int, enabled bool) string { + s := fmt.Sprintf("violations: %d", n) + if !enabled { + return s + } + if n == 0 { + return ansiGreen + s + ansiReset + } + return ansiRed + s + ansiReset +} + // provedLabel is the table's PROVED column: "yes" for a clean coverage // proof, "no" with the reason and largest gap for an unobservable rule, and // "-" for a rule decide never asked proveCoverage about at all (skipped — diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/table_test.go b/grafana-alertcheck/cmd/table_test.go similarity index 52% rename from grafana-alertcheck/cmd/grafana-alertcheck/table_test.go rename to grafana-alertcheck/cmd/table_test.go index e585e9b61..86c80a19b 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/table_test.go +++ b/grafana-alertcheck/cmd/table_test.go @@ -2,11 +2,11 @@ package main import ( "bytes" - "strings" "testing" "time" "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" + "github.com/stretchr/testify/require" ) // The golden table test: a fixed Result renders a deterministic, ordered rule @@ -46,68 +46,43 @@ func TestRenderTable(t *testing.T) { } var buf bytes.Buffer - if err := renderTable(&buf, res); err != nil { - t.Fatalf("renderTable: %v", err) - } + require.NoError(t, renderTable(&buf, res)) out := buf.String() // Rule table: Ape sorts before Zebra sorts before... Paused is skipped and // carries no coverage entry, so it renders "-" for PROVED. - if !strings.Contains(out, "Ape Alert") || !strings.Contains(out, "unobservable") { - t.Fatalf("out = %q, want Ape's unobservable row", out) - } - if !strings.Contains(out, "heartbeat_gap") || !strings.Contains(out, "largest gap 5m0s") { - t.Fatalf("out = %q, want the coverage reason and largest gap", out) - } - if !strings.Contains(out, "Zebra Alert") || !strings.Contains(out, "clean") { - t.Fatalf("out = %q, want Zebra's clean row", out) - } + require.Contains(t, out, "Ape Alert") + require.Contains(t, out, "unobservable") + require.Contains(t, out, "heartbeat_gap") + require.Contains(t, out, "largest gap 5m0s") + require.Contains(t, out, "Zebra Alert") + require.Contains(t, out, "clean") // The violations section must show up even without --output json, and must // carry the --allow-paused hint text verbatim. - if !strings.Contains(out, "VIOLATIONS") { - t.Fatalf("out = %q, want a VIOLATIONS section", out) - } - if !strings.Contains(out, "--allow-paused") { - t.Fatalf("out = %q, want the --allow-paused hint in the human table", out) - } - if !strings.Contains(out, "STATE") || !strings.Contains(out, "HEALTH") { - t.Fatalf("out = %q, want the violations table to have STATE and HEALTH columns", out) - } - if !strings.Contains(out, string(gate.StateFiring)) || !strings.Contains(out, "error") { - t.Fatalf("out = %q, want Ape's violation State/Health", out) - } + require.Contains(t, out, "VIOLATIONS") + require.Contains(t, out, "--allow-paused") + require.Contains(t, out, "STATE") + require.Contains(t, out, "HEALTH") + require.Contains(t, out, string(gate.StateFiring)) + require.Contains(t, out, "error") - // The footer: per-rule thresholds, global thresholds, and skew with its own - // bound rather than the fixed hard limit. - if !strings.Contains(out, "Ape Alert: maxGap=1m0s healthGrace=2m0s evalStaleAfter=1m0s") { - t.Fatalf("out = %q, want Ape's per-rule thresholds", out) - } - if !strings.Contains(out, "Zebra Alert: maxGap=1m0s healthGrace=1m0s evalStaleAfter=1m0s") { - t.Fatalf("out = %q, want Zebra's per-rule thresholds", out) - } - if strings.Contains(out, "Paused Alert: maxGap") { - t.Fatalf("out = %q, a skipped rule must not report thresholds it never had", out) - } - if !strings.Contains(out, "global: transitionGrace=5m0s (source: Ape Alert (for=5m)) drainTimeout=2m0s") { - t.Fatalf("out = %q, want the global thresholds line", out) - } - if !strings.Contains(out, "largest measured clock skew: 1.5s (bound ±250ms, hard limit 1m0s)") { - t.Fatalf("out = %q, want the skew and its own bound, not the hard limit misused as one", out) - } - if !strings.Contains(out, "violations: 2") { - t.Fatalf("out = %q, want the violation count", out) - } - if !strings.Contains(out, "13.1.0") { - t.Fatalf("out = %q, want the grafana version", out) - } + // The footer: per-rule thresholds are a table (RULE/MAXGAP/HEALTHGRACE/ + // EVALSTALEAFTER) rather than prose, followed by the global thresholds and + // the violations count with the skew and its own bound rather than the + // fixed hard limit. + require.Contains(t, out, "MAXGAP") + require.Contains(t, out, "HEALTHGRACE") + require.Contains(t, out, "EVALSTALEAFTER") + require.Contains(t, out, "global: transitionGrace=5m0s (source: Ape Alert (for=5m)) drainTimeout=2m0s") + require.Contains(t, out, "largest measured clock skew: 1.5s (bound ±250ms, hard limit 1m0s)") + require.Contains(t, out, "violations: 2") + require.Contains(t, out, "13.1.0") } // The "-" case: a rule decide never asked proveCoverage about (paused before // the window opened) has an empty CoverageResult and must not be reported as // either proved or unobservable. func TestProvedLabel_Skipped(t *testing.T) { - if got := provedLabel(gate.CoverageResult{}); got != "-" { - t.Fatalf("provedLabel(zero value) = %q, want \"-\"", got) - } + require.Equal(t, "-", provedLabel(gate.CoverageResult{})) } diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/watch.go b/grafana-alertcheck/cmd/watch.go similarity index 95% rename from grafana-alertcheck/cmd/grafana-alertcheck/watch.go rename to grafana-alertcheck/cmd/watch.go index df2c50029..69b585298 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/watch.go +++ b/grafana-alertcheck/cmd/watch.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "flag" "fmt" "io" @@ -49,6 +50,13 @@ func runWatch(args []string, stdin io.Reader, stdout, stderr io.Writer) int { readyFD := fs.Int(gate.ReadyFDFlag[2:], 0, "") if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + return 0 + } + return 2 + } + if fs.NArg() != 0 { + fmt.Fprintf(stderr, "watch: unexpected arguments %v\n", fs.Args()) return 2 } @@ -78,7 +86,7 @@ func runWatch(args []string, stdin io.Reader, stdout, stderr io.Writer) int { DaemonLog: *daemonLog, Concurrency: *common.concurrency, Clock: gate.SystemClock{}, - Notes: stderr, + Notes: newNoteStyler(stderr), } if *until != "" { t, err := time.Parse(time.RFC3339, *until) diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/watch_test.go b/grafana-alertcheck/cmd/watch_test.go similarity index 73% rename from grafana-alertcheck/cmd/grafana-alertcheck/watch_test.go rename to grafana-alertcheck/cmd/watch_test.go index 6d8c8c1a6..5c8f53c55 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/watch_test.go +++ b/grafana-alertcheck/cmd/watch_test.go @@ -3,8 +3,9 @@ package main import ( "bytes" "os" - "strings" "testing" + + "github.com/stretchr/testify/require" ) // The record step's flag-validation matrix. Every case fails inside @@ -47,12 +48,8 @@ func TestRunWatch_FlagValidation(t *testing.T) { var stdout, stderr bytes.Buffer args := append([]string{"watch"}, tt.args(t)...) code := run(args, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2; stderr = %q", code, stderr.String()) - } - if !strings.Contains(stderr.String(), tt.wantErr) { - t.Fatalf("stderr = %q, want it to contain %q", stderr.String(), tt.wantErr) - } + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), tt.wantErr) }) } } @@ -68,18 +65,10 @@ func TestRunWatch_DaemonChildDispatch(t *testing.T) { // enough to prove dispatch happened without needing a real recording. missing := os.DevNull + ".missing" code := run([]string{"watch", "--daemon-child", "--out", missing, "--ready-fd", "0"}, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2; stderr = %q", code, stderr.String()) - } - if !strings.Contains(stderr.String(), missing) { - t.Fatalf("stderr = %q, want RunDaemonChild's read failure naming %q", stderr.String(), missing) - } - if strings.Contains(watchUsage, "daemon-child") { - t.Fatalf("watchUsage = %q, must never name --daemon-child", watchUsage) - } - if strings.Contains(watchUsage, "ready-fd") { - t.Fatalf("watchUsage = %q, must never name --ready-fd", watchUsage) - } + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), missing) + require.NotContains(t, watchUsage, "daemon-child") + require.NotContains(t, watchUsage, "ready-fd") } func TestRunWatch_DaemonChild_MissingEnv(t *testing.T) { @@ -88,10 +77,6 @@ func TestRunWatch_DaemonChild_MissingEnv(t *testing.T) { var stdout, stderr bytes.Buffer code := run([]string{"watch", "--daemon-child", "--out", "log.jsonl"}, &stdout, &stderr) - if code != 2 { - t.Fatalf("code = %d, want 2; stderr = %q", code, stderr.String()) - } - if !strings.Contains(stderr.String(), "GRAFANA_URL") { - t.Fatalf("stderr = %q, want it to name the missing env var", stderr.String()) - } + require.Equal(t, 2, code) + require.Contains(t, stderr.String(), "GRAFANA_URL") } diff --git a/grafana-alertcheck/docs/_category_.yaml b/grafana-alertcheck/docs/_category_.yaml new file mode 100644 index 000000000..3cbc431c4 --- /dev/null +++ b/grafana-alertcheck/docs/_category_.yaml @@ -0,0 +1,8 @@ +position: 1 +label: 'Grafana Alertcheck' +collapsible: true +collapsed: false +link: + type: generated-index + slug: /platform-services/devex/cicd/grafana-alertcheck/index + description: 'CD quality gate for Grafana alerts: watch, classify, and gate releases.' diff --git a/grafana-alertcheck/docs/advanced.md b/grafana-alertcheck/docs/advanced.md new file mode 100644 index 000000000..073292cd6 --- /dev/null +++ b/grafana-alertcheck/docs/advanced.md @@ -0,0 +1,41 @@ +--- +id: grafana-alertcheck-advanced +title: Check budget and scheduling +sidebar_label: Budget and scheduling +sidebar_position: 2 +description: Why grafana-alertcheck schedules per rule, how the request budget works, and why it never queries state history. +--- + +# Check budget and scheduling + +## Per-rule schedules, never a global cycle + +Each rule polls at its **own** cadence, `--poll-interval` (default: half the rule's own evaluation interval). There is deliberately no single global minimum-interval cycle. + +One rule at `intervalSeconds=10` beside twenty at `300` keeps a 5 s cadence for itself and 150 s for the other twenty — not a 5 s cycle for all of them, which would be a 60× request bloat at ~1.8 s per request and would fail to start on a reasonable fleet. + +The scheduler staggers each rule's initial next-due time across its cadence, and serves due rules **earliest-due-first**, so a tight rule never queues behind slack ones. + +## The check budget + +The gate records one observation of every rule up front and checks the schedule against those **measured** latencies (payload sizes vary ~230× across rules, so a fixed estimate is meaningless). It errors at start — before waiting — if any of three conditions hold: + +- **Utilization** — total request rate exceeds `--concurrency`. +- **Per-rule** — one rule's request can't fit its own cadence. +- **Burst bound** — the slowest request exceeds the fleet's tightest cadence, which can open a mid-run gap. + +The error names the three levers only: raise `--concurrency`, raise `--poll-interval`, or watch fewer alerts. It never prescribes a single interval. + +## Why the gate never queries state history + +Querying Grafana's alert state history after the fact fails closed *in the wrong direction* — it returns "pass" when the truth is unknown: + +- History stores **transitions**, not states. An alert firing through the whole window has its only record *before* the window. +- The annotations API **does not serve Loki-backed history** at all. +- An empty result is indistinguishable from a healthy one: no alert fired, the backend differs, retention removed data, the token lacked permission — all look identical. +- There is **no coverage signal** — nothing proves the history is complete to time T. +- Artifact transitions (`Paused`, `RuleDeleted`, `Updated`, `MissingSeries`) look like recoveries. + +Instead, `watch` records its own evidence live and the log becomes the source of truth. The trade-off: the gate can miss an episode shorter than a rule's poll interval, though `activeAt` still surfaces sub-interval onsets for instances still active at a poll. + +A corollary of recording fresh: there is no replay. Re-running a failed job is a new deploy with a new `from` and a new recording — never a re-classification of old evidence. diff --git a/grafana-alertcheck/docs/architecture.md b/grafana-alertcheck/docs/architecture.md new file mode 100644 index 000000000..ca493220e --- /dev/null +++ b/grafana-alertcheck/docs/architecture.md @@ -0,0 +1,68 @@ +--- +id: grafana-alertcheck-architecture +title: Architecture +sidebar_label: Architecture +sidebar_position: 3 +description: The design invariants, pure-function seam, and recorder lifecycle of grafana-alertcheck, for maintainers. +--- + +# Architecture + +This page documents the invariants and seams a maintainer must not break. It exists because most of them are the difference between a gate that fails closed and one that silently passes broken windows. + +## Fail-closed invariants + +The gate must stop the release if it cannot get an answer. Every rule below is a specific instance of that: + +- **An error is never a pass.** A pass is exactly `len(Violations) == 0 && err == nil`. Every error path leaves `err` non-nil, and the CLI maps that to exit `2` unconditionally. +- **Inability beats violation.** Any `unobservable` rule is exit `2`, even alongside a real violation found first. +- **Absent never means normal.** An instance that leaves the bad set is looked up in the *same* response: present as `normal` → cleared; absent (or `MissingSeries`) → vanished (a discontinuity, not a recovery). +- **Staleness is absolute.** `grafana_now − lastEvaluation` is compared against a threshold, never "did it increase since the last poll" — a delta check reports stale on ~half the polls of a healthy rule. +- **`grafana_now` is the response `Date` header.** Never the runner clock, in any comparison against a Grafana timestamp. +- **No early exit.** `check` collects to `to + transitionGrace` before classifying once. +- **No replay.** No run-id key, no artifact download, no state between attempts. A retry is a new deploy. + +## The pure-function seam + +All correctness lives in two phases written as **pure functions** over a flat list of polls — no HTTP, no files, no clock, no goroutines: + +``` +HTTP ──> Source ──> []StateRule ──> reduce ──> []Poll ──> proveCoverage ──> decide ──> Result + │ + JSONL log ──> ReadLog ──┘ +``` + +- `proveCoverage` (the nine coverage checks) and `decide` (the instance timelines and outcomes) are pure; tests drive them with `[]Poll` literals and a fake `Clock`, with no sleeping or fixture server. +- `Check`/`Watch` are I/O shells: HTTP, signals, the pidfile, file reads, the countdown print. The only test doubles needed are the `Source` and `Clock` interfaces. +- `Policy` is the narrowed view of `Config` that reaches the pure layer — classification knobs and the window, no URL and no token. The token must never cross that line, which is the cheapest guarantee it never lands in an error string or a result. + +## Strict parsing as the version guard + +Both API responses are parsed strictly: a missing or unparseable **required** field (`health`, `state`, `lastEvaluation`, `interval`) is an error, never a zero value. Optional keys (`alerts`, `totals`, `labels`, `keepFiringFor`) are absent-tolerant, and unknown keys are ignored — so Grafana can add fields without breaking the parser, but removing one fails loudly. + +This, plus the declared supported range (Grafana >= 13.0.0, < 14.0.0), is how a deprecation or schema change is caught instead of silently misread. + +## The recorder lifecycle + +`watch` detaches a background recorder so observation survives the step boundary: + +1. Parent resolves names, writes the header, observes every non-paused rule once, checks the budget. +2. Parent re-execs itself as the child (`--daemon-child`) under a new session/process group, stdout/stderr to the daemon log. +3. Child re-reads the header, reopens the log `O_APPEND`, takes the exclusive `flock`, and writes one readiness byte on `--ready-fd`. +4. Parent writes the pidfile **after** the readiness report, then returns. + +Two authorities, only one of which is evidence: + +- The **pidfile** says a recording ever started (written only after ready, removed on failure). It can go stale — a pid gets reused. +- The **flock** says a writer exists *now*. The kernel drops it on exit, so the lock is always authoritative. + +On a clean stop (SIGTERM/SIGINT/`--until`) the child finishes the in-flight write, appends the `stopped` sentinel, fsyncs, and exits. A hard error writes no sentinel — so a recorder that died reads exactly like a coverage gap, because it is one. + +`check` signals via the pidfile, waits for the **lock** to release (never the pid), and only then reads the log once. Reading while a writer can still append can only produce a shorter window than was recorded. + +## The log is the source of truth + +`watch` records raw evidence, so nothing trusts a state that could become unreachable. Two consequences a maintainer must preserve: + +- The **header is authoritative for recording facts** (the cadence actually used, the URL, the alert set); the ruler API is authoritative for **rule facts** (`for`, `intervalSeconds`, kind). `check` always re-resolves definitions fresh and never reconstructs them from the header — the header duplicates `for`/`interval` only so the uploaded artifact is self-describing. +- The **cadence authority** is the header's `poll_every_seconds`, not the definitions. Re-deriving it would compare gaps recorded at an override cadence against default-cadence thresholds — fail-open in the faster-override direction. diff --git a/grafana-alertcheck/docs/how-alerts-are-evaluated.md b/grafana-alertcheck/docs/how-alerts-are-evaluated.md new file mode 100644 index 000000000..7c9cb3b6a --- /dev/null +++ b/grafana-alertcheck/docs/how-alerts-are-evaluated.md @@ -0,0 +1,85 @@ +--- +id: grafana-alertcheck-evaluation +title: How alerts are evaluated +sidebar_label: How alerts are evaluated +sidebar_position: 1 +description: The verdict model, instance timelines, and coverage proof behind grafana-alertcheck. +--- + +# How alerts are evaluated + +Both `watch`+`check` (recorder mode) and `check` alone (single-step mode) converge on the same input: a flat list of polls. Everything below runs over that list; the mode only changes where the polls came from. + +## Instance states + +Grafana reports instance states in two vocabularies (`Alerting`/`Normal` at instance level, `firing`/`inactive` at rule level). The gate normalizes every instance to one canonical set: + +| Canonical | Meaning | +| --------- | ------- | +| `normal` | Healthy | +| `firing` | The condition is true and `for` has elapsed | +| `pending` | The condition is true, `for` has not elapsed | +| `nodata` | The query returned no series (synthetic instance) | +| `error` | The query failed (synthetic instance) | + +A rule's **rule-level** `state` and `health` are kept verbatim and only reported — they are never classified. The **instance** state is what the classifier reasons about. + +A "bad" instance is one whose canonical state is in `--states` (default `firing`). `pending` and `nodata` are excluded by default. + +## Verdict model + +For each instance the gate builds a timeline of bad spans over `[from, to]`, then takes the worst outcome across a rule's instances as the rule's outcome. + +| Outcome | Shape | Exit | +| ------- | ----- | ---- | +| `clean` | Good throughout, observed throughout | → 0 | +| `newly_bad` | Entered a bad state **inside** the window | → 1 | +| `persistently_bad` | Bad at `from`, still bad at `to` | → 1 | +| `recovered` | Bad at `from`, cleared before `to`, stayed clear | → 0 | +| `flapping` | Cleared, then became bad again | → 1 | +| `skipped` | Paused **before** the window opened | reported, not observable | +| `unobservable` | Coverage gap / sustained `health=error` / stale / absent | → 2 | + +`recovered` has **no deadline** — an alert that clears at minute 58 of a 60-minute window still passes. The total bad time is reported as `BadFor`; the removed deadline is replaced by that measured value rather than a derived limit. + +### Preexisting policy + +For an instance already bad when `from` opened, `--preexisting` decides: + +- `fail-unless-recovered` (default) — clears and stays clear → pass; never clears → fail. +- `fail` — any preexisting instance fails, recovered or not. +- `ignore` — preexisting instances are disregarded; only new episodes fail. + +## Cleared vs vanished + +When an instance leaves the bad set, the gate looks it up **in the same response**: + +- Present as `normal` → `cleared` (a real recovery). +- Absent, or present as `normal (MissingSeries)` → `vanished` (a discontinuity, **not** a recovery). + +A vanished instance that was bad stays `persistently_bad`. A metric that stops being emitted is not evidence of health — this is deliberate and can surprise users whose fix is to remove a metric rather than drive it to a good value. + +## Coverage proof + +Before classifying, `check` must **prove** continuous coverage of `[from, to]` for each alert. Nine checks run; any failure makes the rule `unobservable`: + +1. **Sentinel** — a clean recorder stop, timestamped at or after `to + transitionGrace`. A recorder that died mid-window looks exactly like a coverage gap and is one. +2. **`from` bounds** — `from` earlier than the recording start is unprovable. +3. **Heartbeat gap** — any gap larger than `maxGap` (= 2 × poll cadence) inside the window. Data at both ends with a hole between is not enough. +4. **`health=error`** — a contiguous run longer than `healthGrace` consumes coverage; a short blip is a note. +5. **`health=nodata`** — a note, never fatal (unless `--nodata-is-unobservable`). +6. **Liveness** — `grafana_now − lastEvaluation` must not exceed `evalStaleAfter`. This is an **absolute** check, never a "did it increase since the last poll" delta. +7. **In-window pause** — a poll reporting `isPaused` mid-window is `unobservable` (the primary pause detector). +8. **Rule absent** — an authoritative `2xx` with no matching rule. +9. **`KeepLast`** — a note naming a stale-state blind spot. + +## Health: `error` vs `nodata` + +- `health=error` means the query **failed** — a malfunction. Sustained past `healthGrace`, it makes the rule `unobservable`. +- `health=nodata` means the query **ran and returned no series** — indistinguishable from a quiet system. It is not fatal by default; most of a fleet runs `no_data_state: OK`. + +## The drain wait and `transitionGrace` + +A condition that arises just before `to` becomes `firing` only at the first evaluation after its `for` elapses. `transitionGrace` (derived from the watched rules' `for` values) extends the classification bound past `to` so such a surfacing condition is caught. After collection, a **drain wait** polls until each rule has evaluated through `to + transitionGrace` (bounded by `drainTimeout`); a rule that never does is `unobservable`. + +Run time = `(to − from) + transitionGrace + drainTimeout`. This is printed at start, and the grace is warned about when it exceeds a quarter of the window — the window may be too short for the alert's `for`. diff --git a/grafana-alertcheck/docs/index.md b/grafana-alertcheck/docs/index.md new file mode 100644 index 000000000..95d413772 --- /dev/null +++ b/grafana-alertcheck/docs/index.md @@ -0,0 +1,87 @@ +--- +id: grafana-alertcheck-index +title: Grafana Alertcheck +sidebar_label: Overview +sidebar_position: 0 +description: A CD quality gate that bookends a release with alert-state observation and answers whether any watched Grafana alert was bad during the release window. +--- + +# Grafana Alertcheck + +`grafana-alertcheck` is a CD quality gate for Grafana alerts. It bookends a release with two commands — `watch` (record) and `check` (classify) — and answers one question: + +> A release finished at time T. Was any of these Grafana alerts in a bad state during the next N minutes? + +The contract is `watch → your work → check`. Between the two you run whatever you want (deploy, tests, migration); the gate only observes, then classifies. + +It **fails closed**: if it cannot get an answer, it stops the release. It never passes an unproven window. + +## How it works, in one paragraph + +`watch` starts a background recorder that polls each named alert and appends snapshots to a JSONL log. Your work then emits two RFC3339 timestamps — `from` (when the change landed) and `to` (when the work ended). `check` proves continuous coverage of `[from, to]`, builds a state timeline per alert, classifies it, and exits `0`, `1`, or `2`. + +## Install + +```bash +go install github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/cmd@latest +``` + +Connection details come from the environment — the token is env-only, never a flag: + +```bash +export GRAFANA_URL=https://grafana.example.com +export GRAFANA_TOKEN=… +``` + +Requires Grafana >= 13.0.0 and < 14.0.0. Outside that range the gate exits `2`. + +## Quickstart — recorder mode + +```bash +grafana-alertcheck watch --out /tmp/run.jsonl --alerts alerts.txt +./deploy.sh # emits deployed_at= when the rollout is stable +./verify.sh # emits finished_at= when the work is done +grafana-alertcheck check --in /tmp/run.jsonl --from "$deployed_at" --to "$finished_at" +``` + +`alerts.txt` holds one alert name per line. See [Naming alerts](./reference/cli#naming-alerts). + +`watch` returns only after the recorder has observed every named, non-paused alert once and reported ready — so auth, name-resolution, and parse failures surface **before** your deploy runs. + +## Quickstart — single-step mode + +Skip the recorder and observe the window inline, from inside `check` itself: + +```bash +grafana-alertcheck check --alerts alerts.txt --to "$finished_at" +``` + +In single-step mode the window starts at `check`'s first observation; if you give no `--from`, the interval before that first observation is declared as a blind spot with a warning (not an error). + +## Exit codes + +| Code | Meaning | +| ---- | ------- | +| `0` | Pass — no violations | +| `1` | Violations (including a paused-rule-only `--min-observed` shortfall) | +| `2` | The gate could not check — config, auth, resolution, a coverage gap, health/staleness, the drain limit, transport, … | + +An error is never a pass: `2` wins over any violation found alongside it. + +## Common surprises + +- A **paused rule fails by default** — even one someone else paused. Use `--allow-paused`. +- A fix that **stops emitting a metric is not a recovery** — the instance vanishes, which is a discontinuity, not health. +- The gate checks alert **state and health**, not notification delivery — a silenced alert that still fires fails. +- `recovered` has **no deadline** — a bad-at-`from` alert that clears by `to` passes; set `--preexisting fail` to forbid it. +- A **retry is a new deploy**, not a replay — re-running the job re-records against a new `from`. +- `watch` and `check` must run in **one job, one runner, one filesystem** — nothing persists across jobs or attempts. +- The gate **never exits early** — a violation at minute 2 still holds the runner to `to + transitionGrace + drainTimeout`; size the job timeout to the planned run time the gate prints at start. + +## More + +- [How alerts are evaluated](./how-alerts-are-evaluated) — the verdict model and coverage proof +- [Check budget and scheduling](./advanced) — why the schedule and budget look the way they do, and why history isn't queried +- [Architecture](./architecture) — design invariants and the recorder lifecycle, for maintainers +- [CLI reference](./reference/cli) — every subcommand and flag +- [Log format](./reference/log-format) — the JSONL log schema, for debugging artifacts diff --git a/grafana-alertcheck/docs/reference/_category_.yaml b/grafana-alertcheck/docs/reference/_category_.yaml new file mode 100644 index 000000000..2e5d50946 --- /dev/null +++ b/grafana-alertcheck/docs/reference/_category_.yaml @@ -0,0 +1,8 @@ +position: 3 +label: Reference +collapsible: true +collapsed: false +link: + type: generated-index + slug: /platform-services/devex/cicd/grafana-alertcheck/reference + description: 'CLI reference for grafana-alertcheck.' diff --git a/grafana-alertcheck/docs/reference/cli.md b/grafana-alertcheck/docs/reference/cli.md new file mode 100644 index 000000000..1530c508d --- /dev/null +++ b/grafana-alertcheck/docs/reference/cli.md @@ -0,0 +1,91 @@ +--- +id: grafana-alertcheck-cli +title: CLI reference +sidebar_label: CLI reference +sidebar_position: 0 +description: Full reference for the grafana-alertcheck CLI: watch, check, list, environment, naming, and output. +--- + +# CLI reference + +``` +grafana-alertcheck +``` + +Connection details are always from the environment: `GRAFANA_URL` and `GRAFANA_TOKEN`. The token is never a flag and never logged. + +## `list` + +Lists every rule from the ruler endpoint — kind, folder, group, title, uid. Useful to check auth and to find `uid:` names. + +```bash +grafana-alertcheck list +``` + +## `watch` — record + +```bash +grafana-alertcheck watch --out [--pidfile F] [--daemon-log F] \ + --alerts [--folder F] [--poll-interval D] [--concurrency N] [--until RFC3339] +``` + +| Flag | Default | Meaning | +| ---- | ------- | ------- | +| `--out` | — | JSONL log path (required) | +| `--pidfile` | `.pid` | Where the recorder's pid is written | +| `--daemon-log` | `.daemon.log` | stdout/stderr sink for the detached recorder | +| `--alerts` | — | File of alert names, one per line, or `-` for stdin (required) | +| `--folder` | — | Default folder to scope unqualified names | +| `--poll-interval` | half the rule's interval | Override every rule's cadence (never clamped) | +| `--concurrency` | `1` | Max concurrent requests to Grafana | +| `--until` | run until signalled | Optional hard stop | + +`watch` writes the header, observes every non-paused rule once, checks the budget, then detaches a background recorder and returns. Recording is **unfiltered** — there is no `--states` here, so the same log can be re-classified later under different `--states` without re-recording. + +## `check` — classify + +```bash +grafana-alertcheck check [--in ] [--pidfile F] --from RFC3339 --to RFC3339 \ + [--alerts ...] [--folder F] [--states ...] [--preexisting ...] [--min-observed N] \ + [--allow-paused] [--nodata-is-unobservable] [--concurrency N] [--output json] +``` + +| Flag | Default | Meaning | +| ---- | ------- | ------- | +| `--in` | — | Log recorded by `watch`; empty selects single-step mode | +| `--pidfile` | `.pid` | Recorder to stop before reading `--in` | +| `--from` | see below | Moment the deploy finished | +| `--to` | — | End of the window (required) | +| `--alerts` | — | Required **without** `--in`; refused **with** `--in` | +| `--states` | `firing` | Comma-separated bad states: `firing,pending,nodata,error` | +| `--preexisting` | `fail-unless-recovered` | `fail-unless-recovered` \| `fail` \| `ignore` | +| `--min-observed` | every resolved rule | Minimum rules that must be observed | +| `--allow-paused` | `false` | Don't count pre-window-paused rules against `--min-observed` | +| `--nodata-is-unobservable` | `false` | Treat sustained `health=nodata` as unobservable | +| `--concurrency` | `1` | Max concurrent requests | +| `--output` | `table` | `json` also writes the machine-readable result to stdout | + +`--from` and `--to` are RFC3339 with an explicit offset and must come from your work — `from` from the deploy step, `to` from the step that finishes. In recorder mode an absent `--from` is a hard error; in single-step mode it falls back (with a warning) to the start of the step. + +## Naming alerts + +Alert names take one of four forms: + +| Form | Meaning | +| ---- | ------- | +| `HighErrorRate` | Title only, scoped by `--folder` | +| `Platform/HighErrorRate` | Folder + title | +| `Platform/api/HighErrorRate` | Folder + group + title (always unique) | +| `uid:abc123` | Exact uid (present on both endpoints) | + +Datasource-managed and recording rules are refused with a specific error. A name matching multiple rules errors listing every candidate with the copyable `Folder/Group/Title` and its `uid:` form. A no-match errors with case-insensitive substring suggestions and points at `list`. Duplicate names that resolve to the same uid collapse to one (a note, not an error). + +## Output and exit codes + +The human table goes to **stderr**: `RESULTS` (one row per rule), `VIOLATIONS` (one per violation), and `THRESHOLDS` (each rule's `maxGap`/`healthGrace`/`evalStaleAfter` plus global `transitionGrace`/`drainTimeout` and the largest measured clock skew). `--output json` writes the result to stdout. + +| Code | Meaning | +| ---- | ------- | +| `0` | Pass | +| `1` | Violations | +| `2` | Could not check — every library error, never a pass | diff --git a/grafana-alertcheck/docs/reference/log-format.md b/grafana-alertcheck/docs/reference/log-format.md new file mode 100644 index 000000000..66e071770 --- /dev/null +++ b/grafana-alertcheck/docs/reference/log-format.md @@ -0,0 +1,98 @@ +--- +id: grafana-alertcheck-log-format +title: Log format +sidebar_label: Log format +sidebar_position: 1 +description: The JSONL log schema written by watch and read by check, for debugging the forensic artifact. +--- + +# Log format + +`watch` records evidence to a JSONL log — one JSON object per line. A poll record *is* the heartbeat; there is no separate heartbeat type. + +## Record types + +Exactly three: + +| `type` | Meaning | +| ------ | ------- | +| `header` | Line 1 — identity and the alert set | +| `poll` | One reduced observation of one rule | +| `stopped` | The sentinel, written on a clean stop only | + +The header must be line 1, appear once, and carry `schema_version` `1` (any other value is a read error). Any unparseable line — including the last, or one after the sentinel — makes the log unreadable: a truncated log is evidence the recorder was killed, and must not pass. + +## Header + +```json +{ + "type": "header", + "schema_version": 1, + "url": "https://grafana.example.com", + "grafana_version": "13.1.0", + "started_at": "2026-09-07T10:00:00Z", + "rules": [ + { + "uid": "rule0000001", + "title": "HighErrorRate", + "folder": "Platform", + "group": "api", + "for_seconds": 300, + "interval_seconds": 60, + "is_paused": false, + "no_data_state": "OK", + "exec_err_state": "OK", + "poll_every_seconds": 30 + } + ] +} +``` + +- `url` and `rules` are the log's identity — `check` validates them against the current environment and a fresh ruler read. +- `is_paused` records the pause state at record start (the moment `skipped` means). +- `poll_every_seconds` is the cadence the recording **actually used** (after any `--poll-interval` override). `check` derives `maxGap` from it, never from `interval_seconds`. +- `for_seconds`, `interval_seconds`, `no_data_state`, `exec_err_state` are forensic only — `check` re-resolves definitions and never reads them back. + +## Poll + +```json +{ + "type": "poll", + "rule_uid": "rule0000001", + "grafana_now": "2026-09-07T10:00:30Z", + "skew_ms": 20, + "skew_bound_ms": 40, + "latency_ms": 123, + "found": true, + "state": "inactive", + "health": "ok", + "last_evaluation": "2026-09-07T10:00:28Z", + "is_paused": false, + "histogram": { "alerting": 0, "normal": 2004 }, + "reasons": { "NoData": 1091 }, + "abnormal": [ { "labels": { "env": "prod" }, "state": "firing", "active_at": "2026-09-07T09:50:00Z", "value": "1.5" } ], + "cleared": [ "env=prod\u0001..." ], + "vanished": [] +} +``` + +Field notes: + +- `grafana_now` is the response's `Date` header — never the runner clock. +- `skew_ms`/`skew_bound_ms` are the per-poll clock-skew estimate and its uncertainty (RTT/2), in milliseconds for compactness only. +- `found: false` is an authoritative `2xx` in which this rule was absent — a transport failure is retried and never becomes a poll. +- `state`, `health`, `last_error` are raw rule-level strings, reporting-only. +- `histogram` is a verbatim copy of the response `totals`; written, never analysed. +- `reasons` counts non-empty instance reasons (`NoData`, `Error`, `KeepLast`, …); composite states stay visible only here. +- `abnormal` holds only instances whose **canonical** state is not `normal`. +- `cleared`/`vanished` are instance keys that left the bad set, resolved against the same response: `cleared` = a real recovery; `vanished` = a discontinuity, never a recovery. + +Instance keys are a sorted `k=v\n` join of labels, so they correlate across polls without hashing. + +## Stopped + +```json +{ "type": "stopped", "at": "2026-09-07T10:10:30Z" } +``` + +`at` is the recorder's own stop time. `check` compares it against `to + transitionGrace`; absent or earlier is `unobservable` — never a pass. diff --git a/grafana-alertcheck/go.mod b/grafana-alertcheck/go.mod index b0c8511ce..d5e0c88be 100644 --- a/grafana-alertcheck/go.mod +++ b/grafana-alertcheck/go.mod @@ -1,3 +1,7 @@ module github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck go 1.26.6 + +require github.com/stretchr/testify v1.12.1 + +require go.yaml.in/yaml/v3 v3.0.5 // indirect diff --git a/grafana-alertcheck/go.sum b/grafana-alertcheck/go.sum new file mode 100644 index 000000000..c2336837e --- /dev/null +++ b/grafana-alertcheck/go.sum @@ -0,0 +1,4 @@ +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= diff --git a/grafana-alertcheck/internal/gate/check.go b/grafana-alertcheck/internal/gate/check.go index eab5d931b..ef6543b71 100644 --- a/grafana-alertcheck/internal/gate/check.go +++ b/grafana-alertcheck/internal/gate/check.go @@ -237,6 +237,16 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { } logHasHdr = true resolved, notes, err = resolveFromLog(allDefs, earlyHdr, cfg) + if err == nil { + // Fail fast on a bound violation that can't change: StartedAt is + // immutable (line 1), so check 2's backstop still catches any bad + // advisory read — fail closed, never false-pass. Recorder mode only; + // single-step warns-and-passes (see below). + if from.Before(earlyHdr.StartedAt) { + return Result{}, fmt.Errorf("check: `from` %s is before recording started at %s", + from.Format(time.RFC3339), earlyHdr.StartedAt.Format(time.RFC3339)) + } + } } else { resolved, notes, err = Resolve(allDefs, cfg.namedAlerts(), cfg.Folder) } @@ -271,6 +281,16 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { } summary, warning := StartupSummary(from, cfg.To, gt) fmt.Fprintln(cfg.Notes, summary) + // MinObserved is printed with the plan, beside "planned run time", rather + // than after it: it is a fact about the run, not a diagnostic. Its default + // is the resolved rule count AFTER duplicate names collapse, which is + // len(resolved) by construction; decide defaults it identically, and it is + // resolved here rather than inferred from the verdict afterwards. + minObserved := cfg.MinObserved + if minObserved == 0 { + minObserved = len(resolved) + } + fmt.Fprintf(cfg.Notes, "min-observed: %d of %d resolved rule(s)\n", minObserved, len(resolved)) if warning != "" { fmt.Fprintf(cfg.Notes, "warning: %s\n", warning) } @@ -321,17 +341,6 @@ func check(ctx context.Context, cfg Config, src Source) (Result, error) { } } - // ---- Apply MinObserved. ----------------------------------------------- - // Its default is the resolved rule count AFTER duplicate names collapse, - // which is len(resolved) by construction. decide defaults it identically; - // it is resolved here as well so the value the run will judge against is - // printed before the wait rather than inferred from the verdict afterwards. - minObserved := cfg.MinObserved - if minObserved == 0 { - minObserved = len(resolved) - } - fmt.Fprintf(cfg.Notes, "min-observed: %d of %d resolved rule(s)\n", minObserved, len(resolved)) - // ---- Collect the evidence. -------------------------------------------- // Collect ONLY. No classification happens here and there is no early exit, // even once a violation is certain: the loop always runs to diff --git a/grafana-alertcheck/internal/gate/check_test.go b/grafana-alertcheck/internal/gate/check_test.go index d13914a8e..780184bd3 100644 --- a/grafana-alertcheck/internal/gate/check_test.go +++ b/grafana-alertcheck/internal/gate/check_test.go @@ -14,6 +14,8 @@ import ( "syscall" "testing" "time" + + "github.com/stretchr/testify/require" ) // The one rule every test in this file watches, unless it says otherwise: a @@ -228,12 +230,8 @@ func TestCheckValidateRejectsBadConfigurations(t *testing.T) { cfg := base() tc.mutate(&cfg) err := cfg.withDefaults().validate() - if err == nil { - t.Fatalf("validate() = nil, want an error containing %q", tc.wantErr) - } - if !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("validate() = %q, want it to contain %q", err, tc.wantErr) - } + require.Errorf(t, err, "validate()") + require.Contains(t, err.Error(), tc.wantErr) }) } } @@ -250,12 +248,8 @@ func TestCheckValidateAcceptsAPastToWithALog(t *testing.T) { Clock: newFakeClock(testNow), }.withDefaults() - if err := cfg.validate(); err != nil { - t.Fatalf("validate() = %v, want nil", err) - } - if cfg.PidFile != "log.jsonl.pid" { - t.Errorf("PidFile = %q, want the .pid default", cfg.PidFile) - } + require.NoError(t, cfg.validate()) + require.Equal(t, "log.jsonl.pid", cfg.PidFile) } // --------------------------------------------------------------------------- @@ -270,35 +264,24 @@ func TestCheckSingleStepCleanWindowPasses(t *testing.T) { }) res, err := check(context.Background(), cfg, src) - if err != nil { - t.Fatalf("check() = %v, want nil\nnotes:\n%s", err, notesOf(cfg)) - } + require.NoError(t, err) // A pass is exactly this shape. - if len(res.Violations) != 0 { - t.Fatalf("Violations = %+v, want none", res.Violations) - } - if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeClean { - t.Fatalf("Verdicts = %+v, want one clean verdict", res.Verdicts) - } - if cov := res.Coverage[checkUID]; !cov.Proved || cov.Unobservable { - t.Fatalf("Coverage = %+v, want proved", cov) - } + require.Empty(t, res.Violations) + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeClean, res.Verdicts[0].Outcome) + cov := res.Coverage[checkUID] + require.True(t, cov.Proved) + require.False(t, cov.Unobservable) // The collection loop ran to to+transitionGrace and no further. windowEnd := cfg.To.Add(checkGrace) - if clock.Now().Before(windowEnd) { - t.Errorf("stopped collecting at %s, before to+grace %s", clock.Now(), windowEnd) - } + require.False(t, clock.Now().Before(windowEnd)) // One measurement-pass poll plus one every 30s across the 6-minute // collection, plus the drain wait's own polls. The exact count depends on // the scheduler's random stagger, so assert the order of magnitude a full // window implies rather than an exact number. - if got := src.callCount(checkTitle); got < 12 { - t.Errorf("polled %d times, want at least the ~13 a full 6-minute window at 30s implies", got) - } - if notes := notesOf(cfg); !strings.Contains(notes, "planned run time") { - t.Errorf("the planned run time must be printed at start; notes were:\n%s", notes) - } + require.GreaterOrEqual(t, src.callCount(checkTitle), 12) + require.Contains(t, notesOf(cfg), "planned run time") } // resolve_test.go proves the collapse-note-plus-satisfied-MinObserved path at @@ -315,18 +298,10 @@ func TestCheckSingleStepDuplicateAlertNamesCollapseWithNote(t *testing.T) { }) res, err := check(context.Background(), cfg, src) - if err != nil { - t.Fatalf("check() = %v, want nil\nnotes:\n%s", err, notesOf(cfg)) - } - if len(res.Verdicts) != 1 { - t.Fatalf("Verdicts = %+v, want exactly one — the duplicate must collapse to a single rule", res.Verdicts) - } - if len(res.Violations) != 0 { - t.Fatalf("Violations = %+v, want none: MinObserved must be satisfied by the post-collapse count of 1", res.Violations) - } - if notes := notesOf(cfg); !strings.Contains(notes, "counted once") { - t.Errorf("want the collapse note in the run's own notes; got:\n%s", notes) - } + require.NoError(t, err) + require.Len(t, res.Verdicts, 1, "the duplicate must collapse to a single rule") + require.Empty(t, res.Violations, "MinObserved must be satisfied by the post-collapse count of 1") + require.Contains(t, notesOf(cfg), "counted once") } // A rule with health=error for the whole window is unobservable, exit 2 — @@ -337,9 +312,7 @@ func TestCheckSingleStepDuplicateAlertNamesCollapseWithNote(t *testing.T) { func TestCheckSingleStepContinuousHealthErrorIsUnobservable(t *testing.T) { body := readFixture(t, "state_health_error.json") rules, err := ParseState(body) - if err != nil { - t.Fatalf("ParseState: %v", err) - } + require.NoError(t, err) base := rules[0] def := Definition{ UID: base.UID, Title: base.Title, Folder: base.Folder, Group: base.Group, @@ -365,15 +338,10 @@ func TestCheckSingleStepContinuousHealthErrorIsUnobservable(t *testing.T) { src.defs = []Definition{def} res, err := check(context.Background(), cfg, src) - if err == nil { - t.Fatalf("check() = nil, want an error: continuous health=error must be unobservable\nnotes:\n%s", notesOf(cfg)) - } - if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { - t.Fatalf("Verdicts = %+v, want one unobservable verdict", res.Verdicts) - } - if cov := res.Coverage[def.UID]; cov.Reason != ReasonHealthError { - t.Fatalf("Coverage[%s].Reason = %q, want %q", def.UID, cov.Reason, ReasonHealthError) - } + require.Error(t, err, "continuous health=error must be unobservable") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome) + require.Equal(t, ReasonHealthError, res.Coverage[def.UID].Reason) } // A certain violation does not release the runner early, and it does not stop @@ -391,18 +359,10 @@ func TestCheckSingleStepFiringInstanceReportsWithoutExitingEarly(t *testing.T) { }) res, err := check(context.Background(), cfg, src) - if err != nil { - t.Fatalf("check() = %v, want nil (a violation is exit 1, not an error)", err) - } - if len(res.Violations) != 1 { - t.Fatalf("Violations = %+v, want exactly one", res.Violations) - } - if got := res.Violations[0].Outcome; got != OutcomePersistentlyBad { - t.Errorf("Outcome = %q, want %q", got, OutcomePersistentlyBad) - } - if windowEnd := cfg.To.Add(checkGrace); clock.Now().Before(windowEnd) { - t.Errorf("exited early at %s; collection must run to %s", clock.Now(), windowEnd) - } + require.NoError(t, err, "a violation is exit 1, not an error") + require.Len(t, res.Violations, 1) + require.Equal(t, OutcomePersistentlyBad, res.Violations[0].Outcome) + require.False(t, clock.Now().Before(cfg.To.Add(checkGrace)), "exited early; collection must run to to+grace") } // A newly_bad instance at from+30s gives exit 1, but ONLY after @@ -427,15 +387,11 @@ func TestCheckSingleStepNewOnsetDoesNotExitEarly(t *testing.T) { }) res, err := check(context.Background(), cfg, src) - if err != nil { - t.Fatalf("check() = %v, want nil (a violation is exit 1, not an error)", err) - } - if len(res.Violations) != 1 || res.Violations[0].Outcome != OutcomeNewlyBad { - t.Fatalf("Violations = %+v, want exactly one newly_bad", res.Violations) - } - if windowEnd := cfg.To.Add(checkGrace); clock.Now().Before(windowEnd) { - t.Errorf("exited early at %s; collection must run to %s even for a fresh onset at from+30s", clock.Now(), windowEnd) - } + require.NoError(t, err) + require.Len(t, res.Violations, 1) + require.Equal(t, OutcomeNewlyBad, res.Violations[0].Outcome) + require.False(t, clock.Now().Before(cfg.To.Add(checkGrace)), + "exited early; collection must run to to+grace even for a fresh onset at from+30s") } // An ABSENT `from` in single-step mode (as opposed to recorder mode, which @@ -451,16 +407,9 @@ func TestCheckSingleStepAbsentFromFallsBackToStepStart(t *testing.T) { }) res, err := check(context.Background(), cfg, src) - if err != nil { - t.Fatalf("check() = %v, want nil: an absent `from` in single-step mode is a fallback, not an error\nnotes:\n%s", err, notesOf(cfg)) - } - notes := notesOf(cfg) - if !strings.Contains(notes, "no `from` given") { - t.Errorf("want the step-start fallback note; notes were:\n%s", notes) - } - if !res.From.Equal(testNow) { - t.Errorf("Result.From = %s, want the step-start fallback %s", res.From, testNow) - } + require.NoError(t, err, "an absent `from` in single-step mode is a fallback, not an error") + require.Contains(t, notesOf(cfg), "no `from` given") + require.True(t, res.From.Equal(testNow)) } // In single-step mode an explicit `from` earlier than the first observation is @@ -475,18 +424,13 @@ func TestCheckSingleStepFromBeforeFirstObservationWarnsAndPasses(t *testing.T) { }) res, err := check(context.Background(), cfg, src) - if err != nil { - t.Fatalf("check() = %v, want a pass with a warning\nnotes:\n%s", err, notesOf(cfg)) - } + require.NoError(t, err) notes := notesOf(cfg) - if !strings.Contains(notes, "cannot see [") || !strings.Contains(notes, testNow.Format(time.RFC3339)) { - t.Errorf("want a warning naming the unseen interval; notes were:\n%s", notes) - } + require.Contains(t, notes, "cannot see [") + require.Contains(t, notes, testNow.Format(time.RFC3339)) // The classified window is the clamped one, and Result says so rather than // reporting a window the run never proved. - if !res.From.Equal(testNow) { - t.Errorf("Result.From = %s, want the clamped %s", res.From, testNow) - } + require.True(t, res.From.Equal(testNow)) } // The failure limit was exceeded. The measurement pass succeeds and the @@ -503,15 +447,9 @@ func TestCheckFailClosedOnExhaustedRetries(t *testing.T) { }) res, err := check(context.Background(), cfg, src) - if err == nil { - t.Fatalf("check() = nil, want the collection failure to fail closed") - } - if !strings.Contains(err.Error(), "collect evidence") { - t.Errorf("err = %q, want it to name the collection step", err) - } - if len(res.Violations) != 0 { - t.Errorf("Violations = %+v; an error must never be reported as a verdict", res.Violations) - } + require.Error(t, err, "the collection failure to fail closed") + require.Contains(t, err.Error(), "collect evidence") + require.Empty(t, res.Violations, "an error must never be reported as a verdict") } // The resolution of the definitions failed. Both shapes — the ruler read @@ -523,10 +461,9 @@ func TestCheckFailClosedOnDefinitionResolution(t *testing.T) { src := newCheckSource(nil) src.defsErr = errors.New("502 bad gateway") - if _, err := check(context.Background(), cfg, src); err == nil || - !strings.Contains(err.Error(), "read rule definitions") { - t.Fatalf("check() = %v, want a definitions-read failure", err) - } + _, err := check(context.Background(), cfg, src) + require.Error(t, err) + require.Contains(t, err.Error(), "read rule definitions") }) t.Run("unknown alert name", func(t *testing.T) { @@ -535,10 +472,9 @@ func TestCheckFailClosedOnDefinitionResolution(t *testing.T) { cfg.Alerts = []string{"No Such Rule"} src := newCheckSource(nil) - if _, err := check(context.Background(), cfg, src); err == nil || - !strings.Contains(err.Error(), "no rule matched") { - t.Fatalf("check() = %v, want a no-match failure", err) - } + _, err := check(context.Background(), cfg, src) + require.Error(t, err) + require.Contains(t, err.Error(), "no rule matched") }) } @@ -550,10 +486,9 @@ func TestCheckRefusesUnsupportedGrafanaVersion(t *testing.T) { src := newCheckSource(nil) src.version = "12.4.0" - if _, err := check(context.Background(), cfg, src); err == nil || - !strings.Contains(err.Error(), "unsupported grafana version") { - t.Fatalf("check() = %v, want the version gate to refuse 12.4.0", err) - } + _, err := check(context.Background(), cfg, src) + require.Error(t, err) + require.Contains(t, err.Error(), "unsupported grafana version") } // The budget is checked against the latencies the measurement pass actually @@ -569,13 +504,9 @@ func TestCheckSingleStepRefusesAScheduleThatDoesNotFit(t *testing.T) { }) _, err := check(context.Background(), cfg, src) - if err == nil { - t.Fatalf("check() = nil, want the budget check to refuse the schedule") - } + require.Error(t, err, "the budget check to refuse the schedule") for _, want := range []string{"raising concurrency", "raising poll-interval", "watching fewer alerts"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("err = %q, want it to name the control %q", err, want) - } + require.Contains(t, err.Error(), want) } } @@ -592,9 +523,7 @@ func recordedLog(t *testing.T, dir string, url string, startedAt, start, end, se path := filepath.Join(dir, "log.jsonl") clock := newFakeClock(sentinelAt) w, err := NewWriter(path, clock) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } + require.NoError(t, err) header := Header{ URL: url, GrafanaVersion: "13.1.0", @@ -605,20 +534,14 @@ func recordedLog(t *testing.T, dir string, url string, startedAt, start, end, se PollEverySeconds: checkPollEvery.Seconds(), }}, } - if err := w.WriteHeader(header); err != nil { - t.Fatalf("WriteHeader: %v", err) - } + require.NoError(t, w.WriteHeader(header)) for at := start; !at.After(end); at = at.Add(checkPollEvery) { - if err := w.WritePoll(Poll{ + require.NoError(t, w.WritePoll(Poll{ RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at.Add(-lastEvalLag), - }); err != nil { - t.Fatalf("WritePoll: %v", err) - } - } - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) + })) } + require.NoError(t, w.Stop()) return path } @@ -628,21 +551,15 @@ func recordedLog(t *testing.T, dir string, url string, startedAt, start, end, se func deadPid(t *testing.T) int { t.Helper() cmd := exec.Command("/bin/sh", "-c", "exit 0") - if err := cmd.Start(); err != nil { - t.Fatalf("start a throwaway process: %v", err) - } + require.NoError(t, cmd.Start()) pid := cmd.Process.Pid - if err := cmd.Wait(); err != nil { - t.Fatalf("wait for the throwaway process: %v", err) - } + require.NoError(t, cmd.Wait()) return pid } func writePid(t *testing.T, path, contents string) { t.Helper() - if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { - t.Fatalf("write pidfile: %v", err) - } + require.NoError(t, os.WriteFile(path, []byte(contents), 0o644)) } // recorderConfig points check at a recording of [testNow-1m, windowEnd+30s] @@ -671,28 +588,19 @@ func TestCheckRecorderModeCleanWindowPasses(t *testing.T) { // The drain wait is satisfied from the log's own evidence, so the source // must never be asked for a state — asserted by the nil responder. src := newCheckSource(func(title string, _ int) (Observation, error) { - t.Errorf("the drain wait polled %q although the log already proves the evaluations", title) + require.Fail(t, fmt.Sprintf("the drain wait polled %q although the log already proves the evaluations", title)) return Observation{}, errors.New("unexpected poll") }) res, err := check(context.Background(), cfg, src) - if err != nil { - t.Fatalf("check() = %v, want nil\nnotes:\n%s", err, notesOf(cfg)) - } - if len(res.Violations) != 0 { - t.Fatalf("Violations = %+v, want none", res.Violations) - } - if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeClean { - t.Fatalf("Verdicts = %+v, want one clean verdict", res.Verdicts) - } - if res.GrafanaVersion != "13.1.0" { - t.Errorf("GrafanaVersion = %q, want the recorded one", res.GrafanaVersion) - } + require.NoError(t, err) + require.Empty(t, res.Violations) + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeClean, res.Verdicts[0].Outcome) + require.Equal(t, "13.1.0", res.GrafanaVersion) // The collection loop still waited out to+transitionGrace even though the // recorder had already finished. - if clock.Now().Before(windowEnd) { - t.Errorf("returned at %s, before to+grace %s", clock.Now(), windowEnd) - } + require.False(t, clock.Now().Before(windowEnd)) } // The identity of the log is not correct. The check runs against the header @@ -707,12 +615,9 @@ func TestCheckFailClosedOnWrongLogIdentity(t *testing.T) { clock := newVirtualClock(testNow) cfg := recorderConfig(t, clock, logPath) _, err := check(context.Background(), cfg, newCheckSource(nil)) - if err == nil || !strings.Contains(err.Error(), "log identity") { - t.Fatalf("check() = %v, want a log-identity failure", err) - } - if !clock.Now().Equal(testNow) { - t.Errorf("the identity check waited out the window (now %s); it must fail before the wait", clock.Now()) - } + require.Error(t, err) + require.Contains(t, err.Error(), "log identity") + require.True(t, clock.Now().Equal(testNow), "it must fail before the wait") }) t.Run("rule no longer resolves", func(t *testing.T) { @@ -726,10 +631,56 @@ func TestCheckFailClosedOnWrongLogIdentity(t *testing.T) { src.defs = []Definition{{UID: "somebody-else", Title: "Other", Kind: KindGrafanaManaged, IntervalSeconds: 60}} _, err := check(context.Background(), cfg, src) - if err == nil || !strings.Contains(err.Error(), "log identity") { - t.Fatalf("check() = %v, want a log-identity failure", err) - } + require.Error(t, err) + require.Contains(t, err.Error(), "log identity") + }) +} + +// `from` before the recording's StartedAt is statically knowable from the +// header (immutable line 1), so check fails closed on it BEFORE the window's +// wait — exactly like the identity check above — rather than surfacing a +// from_before_record verdict only after the drain. +func TestCheckFailFastWhenFromPrecedesRecordStart(t *testing.T) { + dir := t.TempDir() + startedAt := testNow.Add(time.Minute) // the recording opened a minute AFTER `from` + windowEnd := testNow.Add(5*time.Minute + checkGrace) + // The poll range is irrelevant to the assertion: the fail-fast reads + // StartedAt from the header alone, before any polling would matter. + logPath := recordedLog(t, dir, "https://grafana.example.com", + startedAt, startedAt, windowEnd.Add(30*time.Second), windowEnd.Add(30*time.Second), 0) + + clock := newVirtualClock(testNow) + cfg := recorderConfig(t, clock, logPath) // From = testNow, before StartedAt + _, err := check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err) + require.Contains(t, err.Error(), "before recording started") + require.True(t, clock.Now().Equal(testNow), "it must fail before the wait") +} + +// A whole-second `from` in the same second as the recording's sub-second +// StartedAt is not a blind interval: the whole-second comparison lets the run +// proceed to a clean pass instead of the fail-fast above. +func TestCheckRecorderModeFromSameSecondAsStartedAtPasses(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + // StartedAt is 500ms after `from` (testNow via recorderConfig) — the same + // whole second. Polls still cover the whole window. + logPath := recordedLog(t, dir, "https://grafana.example.com", + testNow.Add(500*time.Millisecond), testNow.Add(-time.Minute), windowEnd.Add(30*time.Second), windowEnd.Add(30*time.Second), 0) + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + clock := newVirtualClock(testNow.Add(time.Minute)) + cfg := recorderConfig(t, clock, logPath) + src := newCheckSource(func(title string, _ int) (Observation, error) { + require.Fail(t, fmt.Sprintf("the drain wait polled %q although the log already proves the evaluations", title)) + return Observation{}, errors.New("unexpected poll") }) + + res, err := check(context.Background(), cfg, src) + require.NoError(t, err) + require.Empty(t, res.Violations) + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeClean, res.Verdicts[0].Outcome) } // The coverage proof failed: a hole in the middle of the recording is not @@ -740,42 +691,28 @@ func TestCheckFailClosedOnCoverageGap(t *testing.T) { path := filepath.Join(dir, "log.jsonl") clock := newFakeClock(windowEnd.Add(30 * time.Second)) w, err := NewWriter(path, clock) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - if err := w.WriteHeader(Header{ + require.NoError(t, err) + require.NoError(t, w.WriteHeader(Header{ URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), Rules: []LoggedRule{{UID: checkUID, Title: checkTitle, IntervalSeconds: 60, PollEverySeconds: checkPollEvery.Seconds()}}, - }); err != nil { - t.Fatalf("WriteHeader: %v", err) - } + })) for at := testNow.Add(-time.Minute); !at.After(windowEnd.Add(30 * time.Second)); at = at.Add(checkPollEvery) { // A three-minute hole in the middle of the window. if at.After(testNow.Add(time.Minute)) && at.Before(testNow.Add(4*time.Minute)) { continue } - if err := w.WritePoll(Poll{ + require.NoError(t, w.WritePoll(Poll{ RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at, - }); err != nil { - t.Fatalf("WritePoll: %v", err) - } - } - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) + })) } + require.NoError(t, w.Stop()) writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) cfg := recorderConfig(t, newVirtualClock(testNow), path) res, err := check(context.Background(), cfg, newCheckSource(nil)) - if err == nil { - t.Fatalf("check() = nil, want the coverage gap to fail closed") - } - if got := res.Coverage[checkUID].Reason; got != ReasonHeartbeatGap { - t.Errorf("Reason = %q, want %q", got, ReasonHeartbeatGap) - } - if got := res.Verdicts[0].Outcome; got != OutcomeUnobservable { - t.Errorf("Outcome = %q, want %q", got, OutcomeUnobservable) - } + require.Error(t, err, "the coverage gap to fail closed") + require.Equal(t, ReasonHeartbeatGap, res.Coverage[checkUID].Reason) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome) } // An episode fully between the deploy and the start of the check: recorder @@ -790,36 +727,25 @@ func TestCheckRecorderModeFindsAGapImmediatelyAfterTheDeploy(t *testing.T) { path := filepath.Join(dir, "log.jsonl") clock := newFakeClock(windowEnd.Add(30 * time.Second)) w, err := NewWriter(path, clock) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - if err := w.WriteHeader(Header{ + require.NoError(t, err) + require.NoError(t, w.WriteHeader(Header{ URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), Rules: []LoggedRule{{UID: checkUID, Title: checkTitle, IntervalSeconds: 60, PollEverySeconds: checkPollEvery.Seconds()}}, - }); err != nil { - t.Fatalf("WriteHeader: %v", err) - } + })) gapEnd := testNow.Add(3 * time.Minute) // nothing recorded from `from` (testNow) to here for at := gapEnd; !at.After(windowEnd.Add(30 * time.Second)); at = at.Add(checkPollEvery) { - if err := w.WritePoll(Poll{ + require.NoError(t, w.WritePoll(Poll{ RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at, - }); err != nil { - t.Fatalf("WritePoll: %v", err) - } - } - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) + })) } + require.NoError(t, w.Stop()) writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) cfg := recorderConfig(t, newVirtualClock(testNow), path) res, err := check(context.Background(), cfg, newCheckSource(nil)) - if err == nil { - t.Fatalf("check() = nil, want exit 2: a hole right after the deploy hides whatever happened there just as much as one in the middle") - } - if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { - t.Fatalf("Verdicts = %+v, want one unobservable verdict, never clean", res.Verdicts) - } + require.Error(t, err, "a hole right after the deploy hides whatever happened there") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome, "never clean") } // The drain limit passed. The recording itself is clean, so this isolates the @@ -846,21 +772,12 @@ func TestCheckFailClosedOnDrainTimeout(t *testing.T) { }) res, err := check(context.Background(), cfg, src) - if err == nil { - t.Fatalf("check() = nil, want the drain limit to fail closed") - } - if got := res.Coverage[checkUID].Reason; got != ReasonDrainTimeout { - t.Errorf("Reason = %q, want %q", got, ReasonDrainTimeout) - } - if got := res.Verdicts[0].Outcome; got != OutcomeUnobservable { - t.Errorf("Outcome = %q, want %q", got, OutcomeUnobservable) - } - if !strings.Contains(res.Verdicts[0].Note, "drain limit") { - t.Errorf("Note = %q, want it to explain the drain limit", res.Verdicts[0].Note) - } - if waited := clock.Now().Sub(windowEnd); waited < checkDrainLimit { - t.Errorf("gave up after %s of drain wait, want the full %s", waited, checkDrainLimit) - } + require.Error(t, err, "the drain limit to fail closed") + require.Equal(t, ReasonDrainTimeout, res.Coverage[checkUID].Reason) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome) + require.Contains(t, res.Verdicts[0].Note, "drain limit") + require.GreaterOrEqual(t, clock.Now().Sub(windowEnd), checkDrainLimit, + "the rule never evaluates through the window, so the drain wait must run its full limit") } // A rule the state endpoint no longer serves is knowable on the FIRST drain @@ -884,18 +801,10 @@ func TestCheckDrainWaitNamesADeletedRuleAtOnce(t *testing.T) { }) res, err := check(context.Background(), cfg, src) - if err == nil { - t.Fatalf("check() = nil, want a deleted rule to fail closed") - } - if got := res.Coverage[checkUID].Reason; got != ReasonRuleAbsent { - t.Errorf("Reason = %q, want %q — the fault, not the wait", got, ReasonRuleAbsent) - } - if got := src.callCount(checkTitle); got != 1 { - t.Errorf("polled %d times, want exactly 1: the absence is knowable on the first poll", got) - } - if waited := clock.Now().Sub(windowEnd); waited >= checkDrainLimit { - t.Errorf("spent %s in the drain wait, want it to conclude at once", waited) - } + require.Error(t, err, "a deleted rule to fail closed") + require.Equal(t, ReasonRuleAbsent, res.Coverage[checkUID].Reason, "the fault, not the wait") + require.Equal(t, 1, src.callCount(checkTitle), "the absence is knowable on the first poll") + require.Less(t, clock.Now().Sub(windowEnd), checkDrainLimit) } // --------------------------------------------------------------------------- @@ -909,18 +818,14 @@ func pausedAfterWindowLog(t *testing.T, dir string, firesAt time.Time, end, sent t.Helper() path := filepath.Join(dir, "log.jsonl") w, err := NewWriter(path, newFakeClock(sentinelAt)) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - if err := w.WriteHeader(Header{ + require.NoError(t, err) + require.NoError(t, w.WriteHeader(Header{ URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), Rules: []LoggedRule{{ UID: checkUID, Title: checkTitle, IntervalSeconds: 60, IsPaused: false, PollEverySeconds: checkPollEvery.Seconds(), }}, - }); err != nil { - t.Fatalf("WriteHeader: %v", err) - } + })) firing := Instance{ Labels: map[string]string{"alertname": checkTitle, "instance": "a"}, State: StateFiring, @@ -932,13 +837,9 @@ func pausedAfterWindowLog(t *testing.T, dir string, firesAt time.Time, end, sent p.State = "firing" p.Abnormal = []Instance{firing} } - if err := w.WritePoll(p); err != nil { - t.Fatalf("WritePoll: %v", err) - } - } - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) + require.NoError(t, w.WritePoll(p)) } + require.NoError(t, w.Stop()) return path } @@ -970,19 +871,13 @@ func pausedAfterWindowCheck(t *testing.T, allowPaused bool) (Result, error, Conf // that was active at record start is classified, whatever its pause state is // by the time check resolves the definitions. func TestCheckPausingARuleAfterTheWindowDoesNotMakeItSkipped(t *testing.T) { - res, err, cfg := pausedAfterWindowCheck(t, false) - if err != nil { - t.Fatalf("check() = %v, want a classified verdict\nnotes:\n%s", err, notesOf(cfg)) - } - if got := res.Verdicts[0].Outcome; got != OutcomeNewlyBad { - t.Fatalf("Outcome = %q, want %q: the rule was active for the whole window and fired inside it", got, OutcomeNewlyBad) - } - if len(res.Violations) != 1 || res.Violations[0].Outcome != OutcomeNewlyBad { - t.Fatalf("Violations = %+v, want the firing reported", res.Violations) - } - if strings.Contains(res.Verdicts[0].Note, "paused before the window opened") { - t.Errorf("Note = %q, which the log's own polls contradict", res.Verdicts[0].Note) - } + res, err, _ := pausedAfterWindowCheck(t, false) + require.NoError(t, err) + require.Equal(t, OutcomeNewlyBad, res.Verdicts[0].Outcome, + "the rule was active for the whole window and fired inside it") + require.Len(t, res.Violations, 1) + require.Equal(t, OutcomeNewlyBad, res.Violations[0].Outcome) + require.NotContains(t, res.Verdicts[0].Note, "paused before the window opened") } // The regression pin for the loophole this fix closed. Reading skipped from @@ -990,13 +885,9 @@ func TestCheckPausingARuleAfterTheWindowDoesNotMakeItSkipped(t *testing.T) { // skipped free; and a window in which the alert fired reported exit 0. The // default message names --allow-paused, so an operator was led straight to it. func TestCheckAllowPausedCannotExcuseARulePausedAfterItFired(t *testing.T) { - res, err, cfg := pausedAfterWindowCheck(t, true) - if err != nil { - t.Fatalf("check() = %v, want a classified verdict\nnotes:\n%s", err, notesOf(cfg)) - } - if len(res.Violations) == 0 { - t.Fatalf("Violations = none with --allow-paused: the run passed over a window in which the alert fired") - } + res, err, _ := pausedAfterWindowCheck(t, true) + require.NoError(t, err) + require.NotEmpty(t, res.Violations, "the run passed over a window in which the alert fired") } // The other direction, unchanged: a rule the HEADER says was paused when the @@ -1007,23 +898,17 @@ func TestCheckHeaderPausedRuleStaysSkipped(t *testing.T) { windowEnd := testNow.Add(5*time.Minute + checkGrace) path := filepath.Join(dir, "log.jsonl") w, err := NewWriter(path, newFakeClock(windowEnd.Add(30*time.Second))) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } + require.NoError(t, err) // Named in the header, is_paused true, and no poll records at all — the // shape watch writes for a rule paused before the window opened. - if err := w.WriteHeader(Header{ + require.NoError(t, w.WriteHeader(Header{ URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), Rules: []LoggedRule{{ UID: checkUID, Title: checkTitle, IntervalSeconds: 60, IsPaused: true, PollEverySeconds: checkPollEvery.Seconds(), }}, - }); err != nil { - t.Fatalf("WriteHeader: %v", err) - } - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) - } + })) + require.NoError(t, w.Stop()) writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) run := func(allowPaused bool) (Result, error) { @@ -1031,30 +916,22 @@ func TestCheckHeaderPausedRuleStaysSkipped(t *testing.T) { cfg.AllowPaused = allowPaused // The definition is unpaused now; the header still decides. src := newCheckSource(func(title string, _ int) (Observation, error) { - t.Errorf("the drain wait polled skipped rule %q", title) + require.Fail(t, fmt.Sprintf("the drain wait polled skipped rule %q", title)) return Observation{}, errors.New("unexpected poll") }) return check(context.Background(), cfg, src) } res, err := run(false) - if err != nil { - t.Fatalf("check() = %v, want exit-1 shape: a skipped rule is a known condition, not an inability", err) - } - if got := res.Verdicts[0].Outcome; got != OutcomeSkipped { - t.Fatalf("Outcome = %q, want %q", got, OutcomeSkipped) - } - if _, ok := res.Coverage[checkUID]; ok { - t.Errorf("Coverage[%s] present, want absent: a skipped rule has no coverage to prove", checkUID) - } - if len(res.Violations) != 1 { - t.Errorf("Violations = %+v, want the MinObserved shortfall", res.Violations) - } + require.NoError(t, err, "a skipped rule is a known condition, not an inability") + require.Equal(t, OutcomeSkipped, res.Verdicts[0].Outcome) + _, ok := res.Coverage[checkUID] + require.False(t, ok, "a skipped rule has no coverage to prove") + require.Len(t, res.Violations, 1, "the MinObserved shortfall") res, err = run(true) - if err != nil || len(res.Violations) != 0 { - t.Errorf("with --allow-paused: err = %v, Violations = %+v, want a pass", err, res.Violations) - } + require.NoError(t, err) + require.Empty(t, res.Violations, "with --allow-paused: want a pass") } // A paused rule does not evaluate, so it can never catch up: the drain wait @@ -1076,21 +953,12 @@ func TestCheckDrainWaitConcludesAtOnceOnAPausedRule(t *testing.T) { }) res, err := check(context.Background(), cfg, src) - if err == nil { - t.Fatalf("check() = nil, want a rule that stopped evaluating to fail closed") - } - if got := src.callCount(checkTitle); got != 1 { - t.Errorf("polled %d times, want exactly 1: a paused rule can never catch up", got) - } - if got := res.Coverage[checkUID].Reason; got != ReasonDrainTimeout { - t.Errorf("Reason = %q, want %q — the vocabulary is published, so the detail goes in the note", got, ReasonDrainTimeout) - } - if !strings.Contains(res.Verdicts[0].Note, "paused before it evaluated through") { - t.Errorf("Note = %q, want it to say the rule was paused", res.Verdicts[0].Note) - } - if waited := clock.Now().Sub(windowEnd); waited >= checkDrainLimit { - t.Errorf("spent %s in the drain wait, want it to conclude at once", waited) - } + require.Error(t, err, "a rule that stopped evaluating must fail closed") + require.Equal(t, 1, src.callCount(checkTitle), "a paused rule can never catch up") + require.Equal(t, ReasonDrainTimeout, res.Coverage[checkUID].Reason, + "the vocabulary is published, so the detail goes in the note") + require.Contains(t, res.Verdicts[0].Note, "paused before it evaluated through") + require.Less(t, clock.Now().Sub(windowEnd), checkDrainLimit) } // An absent or unparseable pidfile is never "there was nothing to stop". The @@ -1120,9 +988,8 @@ func TestCheckRefusesToReadALogItCannotStop(t *testing.T) { cfg := recorderConfig(t, newVirtualClock(testNow), logPath) _, err := check(context.Background(), cfg, newCheckSource(nil)) - if err == nil || !strings.Contains(err.Error(), "cannot stop the recorder") { - t.Fatalf("check() = %v, want a refusal to stop the recorder", err) - } + require.Error(t, err) + require.Contains(t, err.Error(), "cannot stop the recorder") }) } } @@ -1137,19 +1004,14 @@ func startLockHolder(t *testing.T, logPath string) int { cmd.Env = append(os.Environ(), lockHolderEnv+"="+logPath) cmd.Stderr = os.Stderr stdout, err := cmd.StdoutPipe() - if err != nil { - t.Fatalf("pipe: %v", err) - } - if err := cmd.Start(); err != nil { - t.Fatalf("start the lock holder: %v", err) - } + require.NoError(t, err) + require.NoError(t, cmd.Start()) t.Cleanup(func() { _ = cmd.Process.Kill() _ = cmd.Wait() }) - if _, err := bufio.NewReader(stdout).ReadString('\n'); err != nil { - t.Fatalf("the lock holder never reported holding the lock: %v", err) - } + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err, "the lock holder never reported holding the lock") return cmd.Process.Pid } @@ -1165,9 +1027,8 @@ func TestCheckFailsWhenTheRecorderWillNotExit(t *testing.T) { cfg := recorderConfig(t, newVirtualClock(testNow), logPath) _, err := check(context.Background(), cfg, newCheckSource(nil)) - if err == nil || !strings.Contains(err.Error(), "still holds") { - t.Fatalf("check() = %v, want the stop wait to time out on the lock", err) - } + require.Error(t, err) + require.Contains(t, err.Error(), "still holds") } // The regression pin for a stray SIGTERM. Nothing removes the pidfile when a @@ -1185,9 +1046,7 @@ func TestCheckDoesNotSignalABystanderHoldingAReusedPid(t *testing.T) { // left behind. It does not hold the log's lock, because it is not a // recorder. bystander := exec.Command("sleep", "30") - if err := bystander.Start(); err != nil { - t.Fatalf("start the bystander: %v", err) - } + require.NoError(t, bystander.Start()) t.Cleanup(func() { _ = bystander.Process.Kill() _ = bystander.Wait() @@ -1195,12 +1054,10 @@ func TestCheckDoesNotSignalABystanderHoldingAReusedPid(t *testing.T) { writePid(t, logPath+".pid", fmt.Sprintf("%d\n", bystander.Process.Pid)) cfg := recorderConfig(t, newVirtualClock(testNow), logPath) - if _, err := check(context.Background(), cfg, newCheckSource(nil)); err != nil { - t.Fatalf("check() = %v, want nil\nnotes:\n%s", err, notesOf(cfg)) - } - if err := syscall.Kill(bystander.Process.Pid, 0); err != nil { - t.Fatalf("the bystander is gone (%v): check signalled a process that was not the recorder", err) - } + _, err := check(context.Background(), cfg, newCheckSource(nil)) + require.NoError(t, err) + require.NoError(t, syscall.Kill(bystander.Process.Pid, 0), + "check signalled a process that was not the recorder") } // A dead pidfile (the recorder process has already exited, holding no flock) @@ -1212,40 +1069,29 @@ func TestCheckDeadPidWithNoSentinelIsUnobservable(t *testing.T) { windowEnd := testNow.Add(5*time.Minute + checkGrace) logPath := filepath.Join(dir, "log.jsonl") w, err := NewWriter(logPath, newFakeClock(testNow)) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - if err := w.WriteHeader(Header{ + require.NoError(t, err) + require.NoError(t, w.WriteHeader(Header{ URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), Rules: []LoggedRule{{ UID: checkUID, Title: checkTitle, Folder: "F", Group: "G", IntervalSeconds: 60, NoDataState: "OK", ExecErrState: "OK", PollEverySeconds: checkPollEvery.Seconds(), }}, - }); err != nil { - t.Fatalf("WriteHeader: %v", err) - } + })) // Healthy heartbeats all the way past windowEnd — evaluatedThrough is // satisfied, so the drain wait needs no live re-poll — but no sentinel is // ever written: the recorder died before it could call Stop. for at := testNow.Add(-time.Minute); !at.After(windowEnd.Add(30 * time.Second)); at = at.Add(checkPollEvery) { - if err := w.WritePoll(Poll{RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at}); err != nil { - t.Fatalf("WritePoll: %v", err) - } - } - if err := w.Close(); err != nil { // no sentinel — a clean exit would call Stop - t.Fatalf("Close: %v", err) + require.NoError(t, w.WritePoll(Poll{RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at})) } + require.NoError(t, w.Close()) // no sentinel — a clean exit would call Stop writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) cfg := recorderConfig(t, newVirtualClock(testNow), logPath) res, err := check(context.Background(), cfg, newCheckSource(nil)) - if err == nil { - t.Fatalf("check() = nil, want an error: no sentinel means the recorder never proved it ran to the end") - } - if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { - t.Fatalf("Verdicts = %+v, want one unobservable verdict", res.Verdicts) - } + require.Error(t, err, "no sentinel means the recorder never proved it ran to the end") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome) } // An incomplete last line gives exit 2. log_test.go's TestReadLogRejectsBadLogs @@ -1263,25 +1109,19 @@ func TestCheckRecorderModeTruncatedLogFailsClosed(t *testing.T) { Rules: []LoggedRule{{UID: checkUID, Title: checkTitle, IntervalSeconds: 60, PollEverySeconds: checkPollEvery.Seconds()}}, } hb, err := json.Marshal(headerRecord{Type: RecordHeader, Header: h}) - if err != nil { - t.Fatalf("marshal header: %v", err) - } + require.NoError(t, err) pb, err := json.Marshal(pollRecord{Type: RecordPoll, Poll: Poll{ RuleUID: checkUID, GrafanaNow: testNow, Found: true, State: "inactive", Health: "ok", LastEvaluation: testNow, }}) - if err != nil { - t.Fatalf("marshal poll: %v", err) - } + require.NoError(t, err) content := string(hb) + "\n" + string(pb) + "\n" + `{"type":"poll","rule_ui` // torn mid-write - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatalf("write log: %v", err) - } + require.NoError(t, os.WriteFile(path, []byte(content), 0o644)) writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) cfg := recorderConfig(t, newVirtualClock(testNow), path) - if _, err := check(context.Background(), cfg, newCheckSource(nil)); err == nil || !strings.Contains(err.Error(), "unparseable") { - t.Fatalf("check() = %v, want a refusal naming the unparseable tail", err) - } + _, err = check(context.Background(), cfg, newCheckSource(nil)) + require.Error(t, err) + require.Contains(t, err.Error(), "unparseable") } // One authority for the cadence, from check's side: maxGap comes from the @@ -1293,15 +1133,11 @@ func TestCheckDerivesMaxGapFromTheRecordedCadence(t *testing.T) { windowEnd := testNow.Add(5*time.Minute + checkGrace) path := filepath.Join(dir, "log.jsonl") w, err := NewWriter(path, newFakeClock(windowEnd.Add(30*time.Second))) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - if err := w.WriteHeader(Header{ + require.NoError(t, err) + require.NoError(t, w.WriteHeader(Header{ URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), Rules: []LoggedRule{{UID: checkUID, Title: checkTitle, IntervalSeconds: 60, PollEverySeconds: 5}}, - }); err != nil { - t.Fatalf("WriteHeader: %v", err) - } + })) for at := testNow.Add(-time.Minute); !at.After(windowEnd.Add(30 * time.Second)); at = at.Add(5 * time.Second) { // A 20s hole: under the recorded 5s cadence maxGap is 10s and this // fails; under a cadence re-derived from intervalSeconds it would be @@ -1309,25 +1145,17 @@ func TestCheckDerivesMaxGapFromTheRecordedCadence(t *testing.T) { if at.After(testNow.Add(time.Minute)) && at.Before(testNow.Add(80*time.Second)) { continue } - if err := w.WritePoll(Poll{ + require.NoError(t, w.WritePoll(Poll{ RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at, - }); err != nil { - t.Fatalf("WritePoll: %v", err) - } - } - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) + })) } + require.NoError(t, w.Stop()) writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) cfg := recorderConfig(t, newVirtualClock(testNow), path) res, err := check(context.Background(), cfg, newCheckSource(nil)) - if err == nil { - t.Fatalf("check() = nil; a 20s hole exceeds the 10s maxGap the recorded 5s cadence implies") - } - if got := res.Coverage[checkUID].Reason; got != ReasonHeartbeatGap { - t.Errorf("Reason = %q, want %q", got, ReasonHeartbeatGap) - } + require.Error(t, err, "a 20s hole exceeds the 10s maxGap the recorded 5s cadence implies") + require.Equal(t, ReasonHeartbeatGap, res.Coverage[checkUID].Reason) } // --------------------------------------------------------------------------- @@ -1367,9 +1195,7 @@ func TestEvaluatedThroughSpendsItsUncertaintyFailingClosed(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - if got := evaluatedThrough(tc.lastEval, tc.skew, tc.bound, end); got != tc.wantSatisfied { - t.Errorf("evaluatedThrough() = %v, want %v", got, tc.wantSatisfied) - } + require.Equal(t, tc.wantSatisfied, evaluatedThrough(tc.lastEval, tc.skew, tc.bound, end)) }) } } @@ -1392,33 +1218,17 @@ func TestMergeDrainTimeoutsNamesEveryUnobservableRule(t *testing.T) { "a": {reason: ReasonDrainTimeout, note: "rule \"A\": did not evaluate through the end within the drain limit"}, "b": {reason: ReasonDrainTimeout, note: "rule \"B\": did not evaluate through the end within the drain limit"}, }) - if err == nil { - t.Fatal("mergeDrainTimeouts() = nil, want an error naming the newly unobservable rule") - } - // Its own shape: joined with decide's, two counts under one identical - // phrase would read as a contradiction rather than as two findings. - if !strings.Contains(err.Error(), "unobservable at the drain wait") { - t.Errorf("err = %q, want the drain wait's own error shape", err) - } + require.Error(t, err, "naming the newly unobservable rule") + require.Contains(t, err.Error(), "unobservable at the drain wait") // Only A is newly unobservable; B was already, so naming it twice would // only lengthen the message. - if !strings.Contains(err.Error(), "A ("+string(ReasonDrainTimeout)+")") { - t.Errorf("err = %q, want it to name A's drain timeout", err) - } - if strings.Contains(err.Error(), "B (") { - t.Errorf("err = %q, want it not to re-report B, which decide already reported", err) - } - if got := merged.Coverage["a"].Reason; got != ReasonDrainTimeout { - t.Errorf("Coverage[a].Reason = %q, want %q", got, ReasonDrainTimeout) - } + require.Contains(t, err.Error(), "A ("+string(ReasonDrainTimeout)+")") + require.NotContains(t, err.Error(), "B (") + require.Equal(t, ReasonDrainTimeout, merged.Coverage["a"].Reason) // B keeps the reason the coverage proof gave it — the FIRST reason wins, // as it does inside proveCoverage. - if got := merged.Coverage["b"].Reason; got != ReasonHeartbeatGap { - t.Errorf("Coverage[b].Reason = %q, want the earlier %q", got, ReasonHeartbeatGap) - } - if merged.Verdicts[0].Outcome != OutcomeUnobservable { - t.Errorf("Verdicts[0].Outcome = %q, want %q", merged.Verdicts[0].Outcome, OutcomeUnobservable) - } + require.Equal(t, ReasonHeartbeatGap, merged.Coverage["b"].Reason) + require.Equal(t, OutcomeUnobservable, merged.Verdicts[0].Outcome) } // ReadLogHeader is the one read of a log a writer may still hold, so its @@ -1429,45 +1239,30 @@ func TestReadLogHeader(t *testing.T) { t.Run("reads line 1 while the log keeps growing", func(t *testing.T) { path := filepath.Join(dir, "growing.jsonl") w, err := NewWriter(path, newFakeClock(testNow)) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } + require.NoError(t, err) defer w.Close() - if err := w.WriteHeader(testHeader()); err != nil { - t.Fatalf("WriteHeader: %v", err) - } - if err := w.WritePoll(Poll{RuleUID: "rule1", GrafanaNow: testNow, Found: true}); err != nil { - t.Fatalf("WritePoll: %v", err) - } + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.WritePoll(Poll{RuleUID: "rule1", GrafanaNow: testNow, Found: true})) h, err := ReadLogHeader(path) - if err != nil { - t.Fatalf("ReadLogHeader: %v", err) - } - if h.URL != testHeader().URL || len(h.Rules) != 1 { - t.Errorf("header = %+v, want the written one", h) - } + require.NoError(t, err) + require.Equal(t, testHeader().URL, h.URL) + require.Len(t, h.Rules, 1) }) t.Run("a half-written header is not a header", func(t *testing.T) { path := filepath.Join(dir, "torn.jsonl") - if err := os.WriteFile(path, []byte(`{"type":"header","url":"htt`), 0o644); err != nil { - t.Fatalf("write: %v", err) - } - if _, err := ReadLogHeader(path); err == nil || - !strings.Contains(err.Error(), "no complete header") { - t.Fatalf("ReadLogHeader() = %v, want a refusal", err) - } + require.NoError(t, os.WriteFile(path, []byte(`{"type":"header","url":"htt`), 0o644)) + _, err := ReadLogHeader(path) + require.Error(t, err) + require.Contains(t, err.Error(), "no complete header") }) t.Run("a wrong schema version is refused", func(t *testing.T) { path := filepath.Join(dir, "old.jsonl") - if err := os.WriteFile(path, []byte(`{"type":"header","schema_version":99,"url":"u"}`+"\n"), 0o644); err != nil { - t.Fatalf("write: %v", err) - } - if _, err := ReadLogHeader(path); err == nil || - !strings.Contains(err.Error(), "schema version 99") { - t.Fatalf("ReadLogHeader() = %v, want a schema refusal", err) - } + require.NoError(t, os.WriteFile(path, []byte(`{"type":"header","schema_version":99,"url":"u"}`+"\n"), 0o644)) + _, err := ReadLogHeader(path) + require.Error(t, err) + require.Contains(t, err.Error(), "schema version 99") }) } diff --git a/grafana-alertcheck/internal/gate/classify.go b/grafana-alertcheck/internal/gate/classify.go index d5d9c8e12..d8d058a34 100644 --- a/grafana-alertcheck/internal/gate/classify.go +++ b/grafana-alertcheck/internal/gate/classify.go @@ -476,7 +476,7 @@ func decide(h Header, polls []Poll, sentinel *time.Time, defs []Definition, Thresholds: make(map[string]RuleThresholds), Global: GlobalThresholds{ TransitionGrace: gt.transitionGrace, - GraceSource: gt.graceSource, + GraceSource: graceSourceOrNone(gt.graceSource), DrainTimeout: gt.drainTimeout, }, } diff --git a/grafana-alertcheck/internal/gate/classify_test.go b/grafana-alertcheck/internal/gate/classify_test.go index 017d7ae6b..ad25a6ea0 100644 --- a/grafana-alertcheck/internal/gate/classify_test.go +++ b/grafana-alertcheck/internal/gate/classify_test.go @@ -3,6 +3,8 @@ package gate import ( "testing" "time" + + "github.com/stretchr/testify/require" ) func lbl(name string) map[string]string { return map[string]string{"instance": name} } @@ -55,9 +57,9 @@ func TestClassifyRule_NoEvidenceIsClean(t *testing.T) { polls := []Poll{quietPoll("r1", from), quietPoll("r1", to)} outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeClean || badFor != 0 || len(viols) != 0 { - t.Fatalf("outcome=%v badFor=%v viols=%v, want clean/0/none", outcome, badFor, viols) - } + require.Equal(t, OutcomeClean, outcome) + require.Zero(t, badFor) + require.Empty(t, viols) } func TestClassifyRule_NewOnsetInsideWindowIsNewlyBad(t *testing.T) { @@ -72,15 +74,10 @@ func TestClassifyRule_NewOnsetInsideWindowIsNewlyBad(t *testing.T) { abnormalPoll("r1", to, StateFiring, lbl("a"), onset), } outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeNewlyBad { - t.Fatalf("outcome = %v, want newly_bad", outcome) - } - if want := to.Sub(onset); badFor != want { - t.Fatalf("badFor = %v, want %v", badFor, want) - } - if len(viols) != 1 || viols[0].Outcome != OutcomeNewlyBad { - t.Fatalf("viols = %+v, want exactly one newly_bad violation", viols) - } + require.Equal(t, OutcomeNewlyBad, outcome) + require.Equal(t, to.Sub(onset), badFor) + require.Len(t, viols, 1) + require.Equal(t, OutcomeNewlyBad, viols[0].Outcome) } // A genuinely new bad episode fails even if it clears again before the window @@ -99,12 +96,8 @@ func TestClassifyRule_NewOnsetThatClearsStillFails(t *testing.T) { quietPoll("r1", to), } outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeNewlyBad { - t.Fatalf("outcome = %v, want newly_bad even though it cleared", outcome) - } - if len(viols) != 1 { - t.Fatalf("viols = %+v, want one violation", viols) - } + require.Equal(t, OutcomeNewlyBad, outcome, "even though it cleared") + require.Len(t, viols, 1) } // --- recovered / persistently_bad (preexisting) --- @@ -122,15 +115,9 @@ func TestClassifyRule_PreexistingThatRecoversIsRecoveredAndNotAViolation(t *test quietPoll("r1", to), } outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeRecovered { - t.Fatalf("outcome = %v, want recovered", outcome) - } - if want := clearAt.Sub(from); badFor != want { - t.Fatalf("badFor = %v, want %v", badFor, want) - } - if len(viols) != 0 { - t.Fatalf("viols = %+v, want none: default policy passes a recovered preexisting instance", viols) - } + require.Equal(t, OutcomeRecovered, outcome) + require.Equal(t, clearAt.Sub(from), badFor) + require.Empty(t, viols, "default policy passes a recovered preexisting instance") } // The late condition: bad for 58 of a 60-minute window, clear at minute 58, @@ -149,15 +136,9 @@ func TestClassifyRule_LateRecoveryPassesRegardlessOfHowLateItIs(t *testing.T) { quietPoll("r1", to), } outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeRecovered { - t.Fatalf("outcome = %v, want recovered even 58 minutes into a 60-minute window", outcome) - } - if want := clearAt.Sub(from); badFor != want { - t.Fatalf("badFor = %v, want the full %v bad duration, not a value clamped against a deadline", badFor, want) - } - if len(viols) != 0 { - t.Fatalf("viols = %+v, want none: there is no deadline a preexisting recovery must beat", viols) - } + require.Equal(t, OutcomeRecovered, outcome, "even 58 minutes into a 60-minute window") + require.Equal(t, clearAt.Sub(from), badFor, "not a value clamped against a deadline") + require.Empty(t, viols, "there is no deadline a preexisting recovery must beat") } func TestClassifyRule_PreexistingStillBadAtWindowEndIsPersistentlyBad(t *testing.T) { @@ -170,15 +151,10 @@ func TestClassifyRule_PreexistingStillBadAtWindowEndIsPersistentlyBad(t *testing abnormalPoll("r1", to, StateFiring, lbl("a"), from.Add(-time.Hour)), } outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomePersistentlyBad { - t.Fatalf("outcome = %v, want persistently_bad", outcome) - } - if badFor != to.Sub(from) { - t.Fatalf("badFor = %v, want the full window %v", badFor, to.Sub(from)) - } - if len(viols) != 1 || viols[0].Outcome != OutcomePersistentlyBad { - t.Fatalf("viols = %+v, want one persistently_bad violation", viols) - } + require.Equal(t, OutcomePersistentlyBad, outcome) + require.Equal(t, to.Sub(from), badFor) + require.Len(t, viols, 1) + require.Equal(t, OutcomePersistentlyBad, viols[0].Outcome) } // --- flapping --- @@ -196,12 +172,9 @@ func TestClassifyRule_ClearThenBadAgainIsFlapping(t *testing.T) { quietPoll("r1", to), } outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeFlapping { - t.Fatalf("outcome = %v, want flapping", outcome) - } - if len(viols) != 1 || viols[0].Outcome != OutcomeFlapping { - t.Fatalf("viols = %+v, want one flapping violation, always a fail regardless of policy", viols) - } + require.Equal(t, OutcomeFlapping, outcome) + require.Len(t, viols, 1) + require.Equal(t, OutcomeFlapping, viols[0].Outcome, "always a fail regardless of policy") } // A clear and then a second bad state gives flapping, wherever the second bad @@ -233,12 +206,9 @@ func TestClassifyRule_FlappingAtEveryTimingOfTheSecondOnset(t *testing.T) { quietPoll("r1", to), } outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeFlapping { - t.Fatalf("outcome = %v, want flapping for a second onset at %s", outcome, tc.secondOnset) - } - if len(viols) != 1 || viols[0].Outcome != OutcomeFlapping { - t.Fatalf("viols = %+v, want one flapping violation", viols) - } + require.Equalf(t, OutcomeFlapping, outcome, "second onset at %s", tc.secondOnset) + require.Len(t, viols, 1) + require.Equal(t, OutcomeFlapping, viols[0].Outcome) }) } } @@ -257,15 +227,9 @@ func TestClassifyRule_VanishedWhileBadStaysPersistentlyBad(t *testing.T) { quietPoll("r1", to), } outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomePersistentlyBad { - t.Fatalf("outcome = %v, want persistently_bad: a vanish must never read as a recovery", outcome) - } - if badFor != to.Sub(from) { - t.Fatalf("badFor = %v, want the full window %v: the freeze must hold the episode open to windowEnd", badFor, to.Sub(from)) - } - if len(viols) != 1 { - t.Fatalf("viols = %+v, want one violation", viols) - } + require.Equal(t, OutcomePersistentlyBad, outcome, "a vanish must never read as a recovery") + require.Equal(t, to.Sub(from), badFor, "the freeze must hold the episode open to windowEnd") + require.Len(t, viols, 1) } func TestClassifyRule_VanishedWhileNeverBadIsUninteresting(t *testing.T) { @@ -282,9 +246,9 @@ func TestClassifyRule_VanishedWhileNeverBadIsUninteresting(t *testing.T) { quietPoll("r1", to), } outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeClean || badFor != 0 || len(viols) != 0 { - t.Fatalf("outcome=%v badFor=%v viols=%v, want clean/0/none", outcome, badFor, viols) - } + require.Equal(t, OutcomeClean, outcome) + require.Zero(t, badFor) + require.Empty(t, viols) } // --- preexisting policy --- @@ -301,12 +265,10 @@ func TestClassifyRule_PreexistingPolicyFailFailsARecoveredInstance(t *testing.T) quietPoll("r1", to), } outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFail) - if outcome != OutcomeRecovered { - t.Fatalf("outcome = %v, want recovered — the descriptive outcome does not change under policy=fail", outcome) - } - if len(viols) != 1 || viols[0].Outcome != OutcomeRecovered { - t.Fatalf("viols = %+v, want one violation: policy=fail gives no benefit of the doubt to a preexisting instance", viols) - } + require.Equal(t, OutcomeRecovered, outcome, "the descriptive outcome does not change under policy=fail") + require.Len(t, viols, 1) + require.Equal(t, OutcomeRecovered, viols[0].Outcome, + "policy=fail gives no benefit of the doubt to a preexisting instance") } func TestClassifyRule_PreexistingPolicyIgnoreForgivesPersistentlyBad(t *testing.T) { @@ -319,12 +281,8 @@ func TestClassifyRule_PreexistingPolicyIgnoreForgivesPersistentlyBad(t *testing. abnormalPoll("r1", to, StateFiring, lbl("a"), from.Add(-time.Hour)), } outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingIgnore) - if outcome != OutcomePersistentlyBad { - t.Fatalf("outcome = %v, want persistently_bad — the descriptive outcome does not change under policy=ignore", outcome) - } - if len(viols) != 0 { - t.Fatalf("viols = %+v, want none: policy=ignore disregards a preexisting instance even if it never recovers", viols) - } + require.Equal(t, OutcomePersistentlyBad, outcome, "the descriptive outcome does not change under policy=ignore") + require.Empty(t, viols, "policy=ignore disregards a preexisting instance even if it never recovers") } func TestClassifyRule_PreexistingPolicyIgnoreStillFailsANewOnset(t *testing.T) { @@ -339,9 +297,8 @@ func TestClassifyRule_PreexistingPolicyIgnoreStillFailsANewOnset(t *testing.T) { abnormalPoll("r1", to, StateFiring, lbl("a"), onset), } outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingIgnore) - if outcome != OutcomeNewlyBad || len(viols) != 1 { - t.Fatalf("outcome=%v viols=%v, want newly_bad/1: ignore only forgives PREEXISTING badness", outcome, viols) - } + require.Equal(t, OutcomeNewlyBad, outcome) + require.Len(t, viols, 1, "ignore only forgives PREEXISTING badness") } // --- worst-of across instances --- @@ -366,12 +323,9 @@ func TestClassifyRule_WorstOfMultipleInstancesWins(t *testing.T) { }, } outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomePersistentlyBad { - t.Fatalf("outcome = %v, want persistently_bad: the worse of {recovered, persistently_bad}", outcome) - } - if len(viols) != 1 || viols[0].Outcome != OutcomePersistentlyBad { - t.Fatalf("viols = %+v, want exactly the persistently_bad instance's violation", viols) - } + require.Equal(t, OutcomePersistentlyBad, outcome, "the worse of {recovered, persistently_bad}") + require.Len(t, viols, 1) + require.Equal(t, OutcomePersistentlyBad, viols[0].Outcome) } // --- decide(): skipped rules, unobservable, MinObserved, exit mapping --- @@ -390,15 +344,11 @@ func TestDecide_SkippedRuleNeverReachesProveCoverage(t *testing.T) { // The HEADER is what says paused — decide reads skipped from there, not // from def.IsPaused, which is a post-window reading (Header.pausedAtStart). res, err := decide(pausedHeader(from.Add(-time.Hour), "r1"), nil, nil, defs, rt, gt, pol) - if err != nil { - t.Fatalf("err = %v, want nil: a rule paused before the window is skipped, not unobservable", err) - } - if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeSkipped { - t.Fatalf("Verdicts = %+v, want exactly one skipped verdict", res.Verdicts) - } - if _, ok := res.Coverage["r1"]; ok { - t.Fatalf("Coverage[r1] present, want absent: a skipped rule has no coverage to prove") - } + require.NoError(t, err, "a rule paused before the window is skipped, not unobservable") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeSkipped, res.Verdicts[0].Outcome) + _, ok := res.Coverage["r1"] + require.False(t, ok, "a skipped rule has no coverage to prove") } func TestDecide_UnobservableRuleAlwaysReturnsAnError(t *testing.T) { @@ -413,12 +363,9 @@ func TestDecide_UnobservableRuleAlwaysReturnsAnError(t *testing.T) { // No sentinel at all: check 1 fails, so the rule is unobservable // regardless of anything else. res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, nil, nil, defs, rt, gt, pol) - if err == nil { - t.Fatalf("err = nil, want non-nil: an unobservable rule must always fail the run") - } - if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { - t.Fatalf("Verdicts = %+v, want exactly one unobservable verdict", res.Verdicts) - } + require.Error(t, err, "an unobservable rule must always fail the run") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome) } // Any unobservable rule means exit 2, with no exception — even alongside a @@ -450,9 +397,7 @@ func TestDecide_UnobservableWinsEvenAlongsideARealViolation(t *testing.T) { sentinel := to res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) - if err == nil { - t.Fatalf("err = nil, want non-nil: one rule is unobservable") - } + require.Error(t, err, "one rule is unobservable") var gotBroken, gotBad Outcome for _, v := range res.Verdicts { switch v.RuleUID { @@ -462,15 +407,11 @@ func TestDecide_UnobservableWinsEvenAlongsideARealViolation(t *testing.T) { gotBad = v.Outcome } } - if gotBroken != OutcomeUnobservable { - t.Fatalf("broken.Outcome = %v, want unobservable", gotBroken) - } - if gotBad != OutcomeNewlyBad { - t.Fatalf("bad.Outcome = %v, want newly_bad: classification still runs and is still visible in Verdicts", gotBad) - } - if len(res.Violations) == 0 { - t.Fatalf("Violations empty, want the newly_bad instance still reported even though the run fails on the unobservable rule") - } + require.Equal(t, OutcomeUnobservable, gotBroken) + require.Equal(t, OutcomeNewlyBad, gotBad, + "classification still runs and is still visible in Verdicts") + require.NotEmpty(t, res.Violations, + "the newly_bad instance still reported even though the run fails on the unobservable rule") } // A clean verdict with a coverage gap must never give exit 0, and recovered @@ -550,9 +491,7 @@ func TestDecide_UnobservableRuleWinsOverEveryFavorableOutcome(t *testing.T) { // so it is unobservable regardless of "good". sentinel := to res, err := decide(h, tc.goodPolls, &sentinel, defs, rt, gt, pol) - if err == nil { - t.Fatalf("err = nil, want non-nil: 'broken' is unobservable regardless of 'good' being %s", tc.name) - } + require.Errorf(t, err, "'broken' is unobservable regardless of 'good' being %s", tc.name) var gotGood, gotBroken Outcome for _, v := range res.Verdicts { switch v.RuleUID { @@ -562,12 +501,8 @@ func TestDecide_UnobservableRuleWinsOverEveryFavorableOutcome(t *testing.T) { gotBroken = v.Outcome } } - if gotGood != tc.wantOutcome { - t.Errorf("good.Outcome = %v, want %v", gotGood, tc.wantOutcome) - } - if gotBroken != OutcomeUnobservable { - t.Errorf("broken.Outcome = %v, want unobservable", gotBroken) - } + require.Equal(t, tc.wantOutcome, gotGood) + require.Equal(t, OutcomeUnobservable, gotBroken) }) } } @@ -603,15 +538,10 @@ func TestDecide_RecoveredOutcomeOverriddenByItsOwnCoverageGap(t *testing.T) { sentinel := to res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) - if err == nil { - t.Fatalf("err = nil, want non-nil: r1's own coverage gap must fail the run even though it recovered") - } - if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { - t.Fatalf("Verdicts = %+v, want unobservable, never recovered", res.Verdicts) - } - if cov := res.Coverage["r1"]; cov.Proved { - t.Fatalf("Coverage = %+v, want not proved", cov) - } + require.Error(t, err, "r1's own coverage gap must fail the run even though it recovered") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome, "never recovered") + require.False(t, res.Coverage["r1"].Proved) } func TestDecide_CleanWindowIsAPass(t *testing.T) { @@ -630,15 +560,9 @@ func TestDecide_CleanWindowIsAPass(t *testing.T) { sentinel := to res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) - if err != nil { - t.Fatalf("err = %v, want nil", err) - } - if len(res.Violations) != 0 { - t.Fatalf("Violations = %+v, want none: a pass is exactly len(Violations)==0 && err==nil", res.Violations) - } - if res.Verdicts[0].Outcome != OutcomeClean { - t.Fatalf("Outcome = %v, want clean", res.Verdicts[0].Outcome) - } + require.NoError(t, err) + require.Empty(t, res.Violations, "a pass is exactly len(Violations)==0 && err==nil") + require.Equal(t, OutcomeClean, res.Verdicts[0].Outcome) } // A pause and then an unpause inside the window, with an episode that would @@ -678,15 +602,10 @@ func TestDecide_PauseThenUnpauseWithHiddenEpisodeGivesUnobservableNotClean(t *te sentinel := to res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) - if err == nil { - t.Fatalf("err = nil, want the pause-then-unpause blind interval to fail closed") - } - if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { - t.Fatalf("Verdicts = %+v, want unobservable, never clean", res.Verdicts) - } - if cov := res.Coverage["r1"]; cov.Proved { - t.Fatalf("Coverage = %+v, want not proved", cov) - } + require.Error(t, err, "the pause-then-unpause blind interval must fail closed") + require.Len(t, res.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, res.Verdicts[0].Outcome, "never clean") + require.False(t, res.Coverage["r1"].Proved) } // --- MinObserved shortfall --- @@ -713,19 +632,14 @@ func TestDecide_SkippedOnlyShortfallProducesAViolationWithoutAnError(t *testing. sentinel := to res, err := decide(pausedHeader(from.Add(-time.Hour), "paused"), polls, &sentinel, defs, rt, gt, pol) - if err != nil { - t.Fatalf("err = %v, want nil: a shortfall caused only by a skipped rule is exit 1, not exit 2", err) - } - if len(res.Violations) != 1 { - t.Fatalf("Violations = %+v, want exactly one: a shortfall must be visible through Violations like any other fail reason", res.Violations) - } - if v := res.Violations[0]; v.Outcome != OutcomeSkipped || v.RuleUID != "paused" || v.Alert != "Paused" { - t.Fatalf("Violations[0] = %+v, want Outcome=skipped naming the paused rule", v) - } - if res.Violations[0].Note == "" { - t.Fatalf("Violations[0].Note is empty, want an explanation: the shortfall reason must not be smuggled into LastError, " + - "which is reporting-only rule state from a real poll this synthetic Violation never touched") - } + require.NoError(t, err, "a shortfall caused only by a skipped rule is exit 1, not exit 2") + require.Len(t, res.Violations, 1) + v := res.Violations[0] + require.Equal(t, OutcomeSkipped, v.Outcome) + require.Equal(t, "paused", v.RuleUID) + require.Equal(t, "Paused", v.Alert) + require.NotEmpty(t, v.Note, + "the shortfall reason must not be smuggled into LastError") } // An operator-supplied MinObserved that exceeds what could ever be resolved is @@ -748,17 +662,10 @@ func TestDecide_ExplicitMinObservedShortfallWithNoPausedRuleStillProducesAViolat sentinel := to res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) - if err != nil { - t.Fatalf("err = %v, want nil: an unmet MinObserved is exit 1, never exit 2", err) - } - if len(res.Violations) != 2 { - t.Fatalf("Violations = %+v, want two: the shortfall (3-1=2) is not explained by any paused rule, "+ - "so it must surface directly rather than pass silently", res.Violations) - } + require.NoError(t, err, "an unmet MinObserved is exit 1, never exit 2") + require.Len(t, res.Violations, 2, "the shortfall (3-1=2) must surface directly rather than pass silently") for _, v := range res.Violations { - if v.Outcome != OutcomeSkipped { - t.Fatalf("Violations = %+v, want Outcome=skipped on the synthetic shortfall entries", res.Violations) - } + require.Equal(t, OutcomeSkipped, v.Outcome) } } @@ -783,12 +690,8 @@ func TestDecide_AllowPausedSuppressesTheShortfall(t *testing.T) { sentinel := to res, err := decide(pausedHeader(from.Add(-time.Hour), "paused"), polls, &sentinel, defs, rt, gt, pol) - if err != nil { - t.Fatalf("err = %v, want nil", err) - } - if len(res.Violations) != 0 { - t.Fatalf("Violations = %+v, want none: --allow-paused must suppress the shortfall entirely", res.Violations) - } + require.NoError(t, err) + require.Empty(t, res.Violations, "--allow-paused must suppress the shortfall entirely") } // --- nodata escalation (decide's own Policy-driven check) --- @@ -809,12 +712,8 @@ func TestDecide_NodataIsUnobservableEscalatesASustainedRun(t *testing.T) { sentinel := to res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) - if err == nil { - t.Fatalf("err = nil, want non-nil: a sustained nodata run must be unobservable under --nodata-is-unobservable") - } - if res.Coverage["r1"].Reason != ReasonNodata { - t.Fatalf("Reason = %q, want %q", res.Coverage["r1"].Reason, ReasonNodata) - } + require.Error(t, err, "a sustained nodata run must be unobservable under --nodata-is-unobservable") + require.Equal(t, ReasonNodata, res.Coverage["r1"].Reason) } func TestDecide_NodataIsANoteByDefault(t *testing.T) { @@ -833,12 +732,8 @@ func TestDecide_NodataIsANoteByDefault(t *testing.T) { sentinel := to res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) - if err != nil { - t.Fatalf("err = %v, want nil: 96%% of the fleet runs no_data_state:OK and must not fail by default", err) - } - if res.Coverage["r1"].Unobservable { - t.Fatalf("Coverage[r1].Unobservable = true, want false by default") - } + require.NoError(t, err, "96%% of the fleet runs no_data_state:OK and must not fail by default") + require.False(t, res.Coverage["r1"].Unobservable) } // --- preexisting is decided by ActiveAt, not poll timing --- @@ -862,16 +757,11 @@ func TestClassifyRule_OnsetBetweenFromAndFirstPollIsNewlyBadNotRecovered(t *test quietPoll("r1", to), } outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeNewlyBad { - t.Fatalf("outcome = %v, want newly_bad: the onset is after `from`, so it is not preexisting even though "+ - "the FIRST in-window poll already observes it bad", outcome) - } - if len(viols) != 1 || viols[0].Outcome != OutcomeNewlyBad { - t.Fatalf("viols = %+v, want one newly_bad violation: a policy=fail-unless-recovered default must still fail this", viols) - } - if want := clearAt.Sub(onset); badFor != want { - t.Fatalf("badFor = %v, want %v: BadFor must count from the true onset, not from `from`", badFor, want) - } + require.Equal(t, OutcomeNewlyBad, outcome, + "the onset is after `from`, so it is not preexisting even though the FIRST in-window poll already observes it bad") + require.Len(t, viols, 1) + require.Equal(t, OutcomeNewlyBad, viols[0].Outcome) + require.Equal(t, clearAt.Sub(onset), badFor, "BadFor must count from the true onset, not from `from`") } // TestClassifyRule_OnsetJustBeforeFromIsPreexisting is the mirror check: an @@ -892,15 +782,10 @@ func TestClassifyRule_OnsetJustBeforeFromIsPreexisting(t *testing.T) { quietPoll("r1", to), } outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeRecovered { - t.Fatalf("outcome = %v, want recovered: the onset is at/before `from`, genuinely preexisting", outcome) - } - if len(viols) != 0 { - t.Fatalf("viols = %+v, want none: default policy passes a recovered preexisting instance", viols) - } - if want := clearAt.Sub(from); badFor != want { - t.Fatalf("badFor = %v, want %v: a preexisting episode's BadFor is clamped to window-open, not backdated past it", badFor, want) - } + require.Equal(t, OutcomeRecovered, outcome, "the onset is at/before `from`, genuinely preexisting") + require.Empty(t, viols, "default policy passes a recovered preexisting instance") + require.Equal(t, clearAt.Sub(from), badFor, + "a preexisting episode's BadFor is clamped to window-open, not backdated past it") } // A poll carrying a nonzero skew must have its ActiveAt (and GrafanaNow) @@ -928,12 +813,9 @@ func TestClassifyRule_SkewTranslatesActiveAtAcrossTheWindowBoundary(t *testing.T stillBad.LastEvaluation = to.Add(skew) outcome, badFor, _ := classifyRule(def, []Poll{poll, stillBad}, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomePersistentlyBad { - t.Fatalf("outcome = %v, want persistently_bad: a +90s skew must translate ActiveAt back to exactly `from`", outcome) - } - if badFor != to.Sub(from) { - t.Fatalf("badFor = %v, want the full window %v", badFor, to.Sub(from)) - } + require.Equal(t, OutcomePersistentlyBad, outcome, + "a +90s skew must translate ActiveAt back to exactly `from`") + require.Equal(t, to.Sub(from), badFor) } // --- InstanceLabels must survive a timeline first created by a bare marker --- @@ -954,13 +836,10 @@ func TestClassifyRule_LabelsSurviveWhenTimelineStartsFromAClearedMarker(t *testi quietPoll("r1", to), } _, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if len(viols) != 1 { - t.Fatalf("viols = %+v, want exactly one newly_bad violation", viols) - } - if viols[0].InstanceLabels == nil || viols[0].InstanceLabels["instance"] != "a" { - t.Fatalf("InstanceLabels = %+v, want {instance: a}: labels must backfill even though the "+ - "timeline was first created by a label-less Cleared marker", viols[0].InstanceLabels) - } + require.Len(t, viols, 1) + require.NotNil(t, viols[0].InstanceLabels) + require.Equal(t, "a", viols[0].InstanceLabels["instance"], + "labels must backfill even though the timeline was first created by a label-less Cleared marker") } // FirstSeen/ClearedAt are pinned exactly, not just that a violation exists. @@ -978,19 +857,11 @@ func TestClassifyRule_ViolationFieldsArePrecise(t *testing.T) { quietPoll("r1", to), } _, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if len(viols) != 1 { - t.Fatalf("viols = %+v, want exactly one violation", viols) - } + require.Len(t, viols, 1) v := viols[0] - if !v.FirstSeen.Equal(onset) { - t.Fatalf("FirstSeen = %v, want %v", v.FirstSeen, onset) - } - if !v.ClearedAt.Equal(clearAt) { - t.Fatalf("ClearedAt = %v, want %v", v.ClearedAt, clearAt) - } - if v.InstanceLabels["instance"] != "a" { - t.Fatalf("InstanceLabels = %+v, want {instance: a}", v.InstanceLabels) - } + require.True(t, v.FirstSeen.Equal(onset)) + require.True(t, v.ClearedAt.Equal(clearAt)) + require.Equal(t, "a", v.InstanceLabels["instance"]) } // The episode.end clamp: inWindowPolls admits a poll up to its own skew bound @@ -1013,13 +884,9 @@ func TestClassifyRule_ClearedEventPastWindowEndClampsToWindowEnd(t *testing.T) { }, } outcome, badFor, _ := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeRecovered { - t.Fatalf("outcome = %v, want recovered", outcome) - } - if badFor != to.Sub(from) { - t.Fatalf("badFor = %v, want the window %v exactly: the episode end must clamp to windowEnd, "+ - "not extend to the late Cleared event's raw time", badFor, to.Sub(from)) - } + require.Equal(t, OutcomeRecovered, outcome) + require.Equal(t, to.Sub(from), badFor, + "the episode end must clamp to windowEnd, not extend to the late Cleared event's raw time") } // TestClassifyRule_OnsetJustPastWindowEndIsNewlyBadNotClean pins the fail-closed @@ -1047,16 +914,10 @@ func TestClassifyRule_OnsetJustPastWindowEndIsNewlyBadNotClean(t *testing.T) { } outcome, badFor, viols := classifyRule(def, []Poll{poll}, from, windowEnd, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeNewlyBad { - t.Fatalf("outcome = %v, want newly_bad: an onset past windowEnd seen only via the skew bound must fail closed", outcome) - } - if badFor != 0 { - t.Fatalf("badFor = %v, want 0: the zero-length episode must truncate to the window end", badFor) - } - if len(viols) != 1 { - t.Fatalf("viols = %+v, want exactly one newly_bad violation", viols) - - } + require.Equal(t, OutcomeNewlyBad, outcome, + "an onset past windowEnd seen only via the skew bound must fail closed") + require.Zero(t, badFor, "the zero-length episode must truncate to the window end") + require.Len(t, viols, 1) } // A clear after `to` gives persistently_bad. classifyRule filters @@ -1075,15 +936,10 @@ func TestClassifyRule_ClearAfterWindowEndIsPersistentlyBad(t *testing.T) { clearedPoll("r1", to.Add(time.Hour), key), // far past `to`, not a boundary case } outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomePersistentlyBad { - t.Fatalf("outcome = %v, want persistently_bad: a clear outside the window must not read as a recovery", outcome) - } - if badFor != to.Sub(from) { - t.Fatalf("badFor = %v, want the full window %v", badFor, to.Sub(from)) - } - if len(viols) != 1 || viols[0].Outcome != OutcomePersistentlyBad { - t.Fatalf("viols = %+v, want one persistently_bad violation", viols) - } + require.Equal(t, OutcomePersistentlyBad, outcome, "a clear outside the window must not read as a recovery") + require.Equal(t, to.Sub(from), badFor) + require.Len(t, viols, 1) + require.Equal(t, OutcomePersistentlyBad, viols[0].Outcome) } // TestClassifyRule_CloseBeforeOpenClampsToZeroNotNegative pins the @@ -1110,19 +966,11 @@ func TestClassifyRule_CloseBeforeOpenClampsToZeroNotNegative(t *testing.T) { quietPoll("r1", to), } outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) - if outcome != OutcomeNewlyBad { - t.Fatalf("outcome = %v, want newly_bad", outcome) - } - if badFor < 0 { - t.Fatalf("badFor = %v, want a non-negative duration even though the closing poll's translated "+ - "time landed before the opening poll's", badFor) - } - if badFor != 0 { - t.Fatalf("badFor = %v, want 0: the clamp collapses the inverted span to a zero-length episode", badFor) - } - if len(viols) != 1 { - t.Fatalf("viols = %+v, want one violation", viols) - } + require.Equal(t, OutcomeNewlyBad, outcome) + require.GreaterOrEqual(t, badFor, time.Duration(0), + "a non-negative duration even though the closing poll's translated time landed before the opening poll's") + require.Zero(t, badFor, "the clamp collapses the inverted span to a zero-length episode") + require.Len(t, viols, 1) } // --- mergeDurations --- @@ -1136,13 +984,9 @@ func TestMergeDurations_OverlappingEpisodesCountOnce(t *testing.T) { } got := mergeDurations(eps) want := 8*time.Minute + 1*time.Minute // [0,8) merged = 8m, plus the disjoint 1m - if got != want { - t.Fatalf("mergeDurations = %v, want %v: two simultaneously-bad instances must not double-count their overlap", got, want) - } + require.Equal(t, want, got, "two simultaneously-bad instances must not double-count their overlap") } func TestMergeDurations_Empty(t *testing.T) { - if got := mergeDurations(nil); got != 0 { - t.Fatalf("mergeDurations(nil) = %v, want 0", got) - } + require.Zero(t, mergeDurations(nil)) } diff --git a/grafana-alertcheck/internal/gate/coverage.go b/grafana-alertcheck/internal/gate/coverage.go index 527720bfe..603a7f7d8 100644 --- a/grafana-alertcheck/internal/gate/coverage.go +++ b/grafana-alertcheck/internal/gate/coverage.go @@ -88,12 +88,16 @@ func proveCoverage(h Header, polls []Poll, sentinel *time.Time, t ruleTimings, d // Check 2 — from bounds: from < StartedAt makes coverage unprovable, no // matter how healthy the polls that DO exist look. Both are runner-domain // clock reads (the recorder's own Clock.Now()), so no cross-domain - // translation applies here. The other half of the bound — from too far - // ahead of the runner's clock — is Check's input validation, once per run - // rather than per rule. - if from.Before(h.StartedAt) { + // translation applies here. The comparison is at whole-second granularity: + // `from` is supplied at second precision (--from RFC3339) while StartedAt + // carries the recorder's sub-second clock stamp, so an operator naming the + // exact second the recording opened must not be judged early for the + // sub-second sliver inside that same second. The other half of the bound — + // from too far ahead of the runner's clock — is Check's input validation, + // once per run rather than per rule. + if from.Truncate(time.Second).Before(h.StartedAt.Truncate(time.Second)) { fail(ReasonFromBeforeRecord, fmt.Sprintf( - "requested from %s is before recording started at %s", from.Format(time.RFC3339), h.StartedAt.Format(time.RFC3339))) + "requested from %s is before recording started at %s", from.Format(time.RFC3339Nano), h.StartedAt.Format(time.RFC3339Nano))) } // Filtered once and threaded through every remaining check. @@ -142,7 +146,7 @@ func proveCoverage(h Header, polls []Poll, sentinel *time.Time, t ruleTimings, d // corrupted or hand-edited data (ReadLog does no field validation); // GrafanaNow-LastEvaluation would go negative and silently read as // fresh — fail-open. Treat it as unobservable instead. - if p.LastEvaluation.After(p.GrafanaNow) { + if p.LastEvaluation.Truncate(time.Second).After(p.GrafanaNow) { fail(ReasonFutureEvaluation, fmt.Sprintf( "lastEvaluation %s is after grafana_now %s (corrupted poll)", p.LastEvaluation.Format(time.RFC3339), p.GrafanaNow.Format(time.RFC3339))) diff --git a/grafana-alertcheck/internal/gate/coverage_test.go b/grafana-alertcheck/internal/gate/coverage_test.go index 720386d65..3507ee4fc 100644 --- a/grafana-alertcheck/internal/gate/coverage_test.go +++ b/grafana-alertcheck/internal/gate/coverage_test.go @@ -4,6 +4,8 @@ import ( "strings" "testing" "time" + + "github.com/stretchr/testify/require" ) func TestProveCoverage_CleanWindowIsProved(t *testing.T) { @@ -19,9 +21,9 @@ func TestProveCoverage_CleanWindowIsProved(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if !res.Proved || res.Unobservable || res.Reason != "" { - t.Fatalf("res = %+v, want a clean proved window", res) - } + require.True(t, res.Proved) + require.False(t, res.Unobservable) + require.Empty(t, res.Reason) } func TestProveCoverage_FiltersPollsByUID(t *testing.T) { @@ -40,9 +42,7 @@ func TestProveCoverage_FiltersPollsByUID(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if !res.Proved { - t.Fatalf("res = %+v, want proved: a different rule's broken polls must not affect this rule's verdict", res) - } + require.True(t, res.Proved, "a different rule's broken polls must not affect this rule's verdict") } // --- Check 1: sentinel --- @@ -54,9 +54,8 @@ func TestProveCoverage_NoSentinelIsUnobservable(t *testing.T) { def := Definition{UID: "r1", Title: "R1"} res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, nil, nil, rt, def, from, to, 0) - if res.Proved || res.Reason != ReasonNoSentinel { - t.Fatalf("res = %+v, want unobservable/no_sentinel: an absent sentinel must never be a pass", res) - } + require.False(t, res.Proved) + require.Equal(t, ReasonNoSentinel, res.Reason, "an absent sentinel must never be a pass") } func TestProveCoverage_SentinelBeforeGraceIsUnobservable(t *testing.T) { @@ -68,12 +67,9 @@ func TestProveCoverage_SentinelBeforeGraceIsUnobservable(t *testing.T) { sentinel := to.Add(grace).Add(-time.Second) // one second short of to+grace res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, nil, &sentinel, rt, def, from, to, grace) - if res.Reason != ReasonSentinelEarly { - t.Fatalf("Reason = %q, want sentinel_early", res.Reason) - } - if !res.Unobservable || res.Proved { - t.Fatalf("res = %+v, want Unobservable and not Proved — a reason string with no consequence is not a coverage failure", res) - } + require.Equal(t, ReasonSentinelEarly, res.Reason) + require.True(t, res.Unobservable) + require.False(t, res.Proved, "a reason string with no consequence is not a coverage failure") // The consequence: decide() must turn this into exit 2, never a pass. defs := []Definition{def} @@ -81,12 +77,9 @@ func TestProveCoverage_SentinelBeforeGraceIsUnobservable(t *testing.T) { gt := globalTimings{transitionGrace: grace} pol := Policy{From: from, To: to} dres, err := decide(Header{StartedAt: from.Add(-time.Hour)}, nil, &sentinel, defs, drt, gt, pol) - if err == nil { - t.Fatalf("decide() err = nil, want non-nil: a sentinel short of to+grace must fail the run") - } - if len(dres.Verdicts) != 1 || dres.Verdicts[0].Outcome != OutcomeUnobservable { - t.Fatalf("Verdicts = %+v, want one unobservable verdict", dres.Verdicts) - } + require.Error(t, err, "a sentinel short of to+grace must fail the run") + require.Len(t, dres.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, dres.Verdicts[0].Outcome) } func TestProveCoverage_SentinelExactlyAtGraceIsFine(t *testing.T) { @@ -104,9 +97,7 @@ func TestProveCoverage_SentinelExactlyAtGraceIsFine(t *testing.T) { sentinel := windowEnd res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, grace) - if !res.Proved { - t.Fatalf("Proved = false, want true: sentinel exactly at to+grace must satisfy check 1: %+v", res) - } + require.True(t, res.Proved, "sentinel exactly at to+grace must satisfy check 1") } // --- Check 2: from bounds --- @@ -120,12 +111,9 @@ func TestProveCoverage_FromBeforeRecordIsUnobservable(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: started}, nil, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonFromBeforeRecord { - t.Fatalf("Reason = %q, want from_before_record", res.Reason) - } - if !res.Unobservable || res.Proved { - t.Fatalf("res = %+v, want Unobservable and not Proved — a reason string with no consequence is not a coverage failure", res) - } + require.Equal(t, ReasonFromBeforeRecord, res.Reason) + require.True(t, res.Unobservable) + require.False(t, res.Proved) // The consequence: decide() must turn this into exit 2, never a pass. defs := []Definition{def} @@ -133,12 +121,66 @@ func TestProveCoverage_FromBeforeRecordIsUnobservable(t *testing.T) { gt := globalTimings{} pol := Policy{From: from, To: to} dres, err := decide(Header{StartedAt: started}, nil, &sentinel, defs, drt, gt, pol) - if err == nil { - t.Fatalf("decide() err = nil, want non-nil: `from` before the recording started must fail the run") - } - if len(dres.Verdicts) != 1 || dres.Verdicts[0].Outcome != OutcomeUnobservable { - t.Fatalf("Verdicts = %+v, want one unobservable verdict", dres.Verdicts) + require.Error(t, err, "`from` before the recording started must fail the run") + require.Len(t, dres.Verdicts, 1) + require.Equal(t, OutcomeUnobservable, dres.Verdicts[0].Outcome) +} + +// The from-bounds check compares at whole-second granularity: a whole-second +// `from` may precede the recorder's sub-second StartedAt INSIDE the same second +// without being judged early. That one sliver is the --from truncation, not a +// blind interval, so the window is still proved. +func TestProveCoverage_FromSameSecondAsStartedAtIsProved(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + started := from.Add(500 * time.Millisecond) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", State: "inactive", LastEvaluation: ts}) } + sentinel := to + + res := proveCoverage(Header{StartedAt: started}, polls, &sentinel, rt, def, from, to, 0) + require.True(t, res.Proved) + require.False(t, res.Unobservable) + require.Empty(t, res.Reason) +} + +// Exactly one whole second later is a different second: even at the boundary, +// the whole-second comparison reads it as before, however healthy the polls. +func TestProveCoverage_FromExactlyOneSecondBeforeStartedAtIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + started := from.Add(time.Second) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + sentinel := to + res := proveCoverage(Header{StartedAt: started}, nil, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonFromBeforeRecord, res.Reason) + require.True(t, res.Unobservable) + require.False(t, res.Proved) +} + +// A sub-second sliver that straddles the second boundary is still "before": +// 900ms into one second vs 100ms into the next are distinct seconds, so the +// 200ms gap is a from_before_record, not rounding noise. +func TestProveCoverage_FromSubSecondEarlierAcrossSecondBoundaryIsUnobservable(t *testing.T) { + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + from := base.Add(900 * time.Millisecond) + started := base.Add(time.Second + 100*time.Millisecond) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + sentinel := to + res := proveCoverage(Header{StartedAt: started}, nil, &sentinel, rt, def, from, to, 0) + require.Equal(t, ReasonFromBeforeRecord, res.Reason) + require.True(t, res.Unobservable) + require.False(t, res.Proved) } // --- Check 3: heartbeat continuity --- @@ -158,18 +200,11 @@ func TestProveCoverage_HeartbeatGapBetweenBoundariesIsUnobservable(t *testing.T) sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonHeartbeatGap { - t.Fatalf("Reason = %q, want heartbeat_gap: healthy edges with a hole in the middle must still fail", res.Reason) - } + require.Equal(t, ReasonHeartbeatGap, res.Reason, "healthy edges with a hole in the middle must still fail") // The gap is the SPACING between the two polls (598s), not either // boundary segment (1s each) — pin the actual values, not just the verdict. - if res.LargestGap != 598*time.Second { - t.Fatalf("LargestGap = %s, want 598s (the spacing between the two polls, not a boundary segment)", res.LargestGap) - } - wantAt := from.Add(time.Second) - if !res.LargestGapAt.Equal(wantAt) { - t.Fatalf("LargestGapAt = %s, want %s (where the gap starts, at the first poll)", res.LargestGapAt, wantAt) - } + require.Equal(t, 598*time.Second, res.LargestGap) + require.True(t, res.LargestGapAt.Equal(from.Add(time.Second))) } // --- Check 4/5: health --- @@ -192,12 +227,8 @@ func TestProveCoverage_HealthErrorShortBlipPassesWithNote(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if !res.Proved { - t.Fatalf("Proved = false, want true: one failed evaluation must not fail an otherwise clean window: %+v", res) - } - if !anyContains(res.Notes, "health=error") { - t.Fatalf("Notes = %v, want a health=error note even though it did not fail the window", res.Notes) - } + require.True(t, res.Proved, "one failed evaluation must not fail an otherwise clean window") + require.True(t, anyContains(res.Notes, "health=error"), "want a health=error note even though it did not fail the window") } func TestProveCoverage_HealthErrorSustainedIsUnobservable(t *testing.T) { @@ -218,9 +249,7 @@ func TestProveCoverage_HealthErrorSustainedIsUnobservable(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonHealthError { - t.Fatalf("Reason = %q, want health_error for a run that outlasts healthGrace", res.Reason) - } + require.Equal(t, ReasonHealthError, res.Reason, "a run that outlasts healthGrace") } func TestProveCoverage_HealthNodataNeverFatalHere(t *testing.T) { @@ -236,13 +265,8 @@ func TestProveCoverage_HealthNodataNeverFatalHere(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if !res.Proved { - t.Fatalf("Proved = false, want true: health=nodata for the WHOLE window must still not be fatal by itself "+ - "(escalating it is Policy.NodataIsUnobservable's job, applied by decide): %+v", res) - } - if !anyContains(res.Notes, "health=nodata") { - t.Fatalf("Notes = %v, want a health=nodata note", res.Notes) - } + require.True(t, res.Proved, "health=nodata for the WHOLE window must still not be fatal by itself") + require.True(t, anyContains(res.Notes, "health=nodata")) } // --- Check 6: liveness --- @@ -270,13 +294,9 @@ func TestProveCoverage_LivenessAbsoluteNeverFalseStale(t *testing.T) { sentinel := windowEnd res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, windowEnd, 0) - if res.Reason == ReasonStaleEvaluation || res.BlindFor != 0 { - t.Fatalf("proveCoverage flagged staleness on a healthy rule polled at intervalSeconds/2 — liveness must be absolute, "+ - "never a delta against a previous poll: %+v", res) - } - if !res.Proved { - t.Fatalf("Proved = false, want true: %+v (notes: %v)", res, res.Notes) - } + require.NotEqual(t, ReasonStaleEvaluation, res.Reason, "liveness must be absolute, never a delta against a previous poll") + require.Zero(t, res.BlindFor) + require.True(t, res.Proved) } func TestProveCoverage_StaleEvaluationIsUnobservable(t *testing.T) { @@ -297,12 +317,8 @@ func TestProveCoverage_StaleEvaluationIsUnobservable(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonStaleEvaluation { - t.Fatalf("Reason = %q, want stale_evaluation", res.Reason) - } - if res.BlindFor != 3*time.Minute { - t.Fatalf("BlindFor = %s, want 3m", res.BlindFor) - } + require.Equal(t, ReasonStaleEvaluation, res.Reason) + require.Equal(t, 3*time.Minute, res.BlindFor) } func TestProveCoverage_ZeroLastEvaluationNeverFalseStale(t *testing.T) { @@ -319,9 +335,7 @@ func TestProveCoverage_ZeroLastEvaluationNeverFalseStale(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason == ReasonStaleEvaluation { - t.Fatalf("a zero lastEvaluation on a paused poll must not trigger check 6: %+v", res) - } + require.NotEqual(t, ReasonStaleEvaluation, res.Reason, "a zero lastEvaluation on a paused poll must not trigger check 6") } // --- Check 7: isPaused in-window --- @@ -345,9 +359,7 @@ func TestProveCoverage_PausedInWindowIsUnobservable(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonPausedInWindow { - t.Fatalf("Reason = %q, want paused_in_window", res.Reason) - } + require.Equal(t, ReasonPausedInWindow, res.Reason) } // TestProveCoverage_PausedAfterWindowIsFine pins check 7's respect for the @@ -369,12 +381,8 @@ func TestProveCoverage_PausedAfterWindowIsFine(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason == ReasonPausedInWindow { - t.Fatalf("a paused poll after windowEnd tripped check 7: %+v", res.Notes) - } - if !res.Proved { - t.Fatalf("Proved = false, want a clean window: %+v", res.Notes) - } + require.NotEqual(t, ReasonPausedInWindow, res.Reason, "a paused poll after windowEnd tripped check 7") + require.True(t, res.Proved) } // --- Check 8: rule absent --- @@ -399,9 +407,7 @@ func TestProveCoverage_RuleAbsentIsUnobservable(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonRuleAbsent { - t.Fatalf("Reason = %q, want rule_absent", res.Reason) - } + require.Equal(t, ReasonRuleAbsent, res.Reason) } // denseHealthyPolls builds a clean poll sequence at a fixed cadence, with @@ -436,12 +442,8 @@ func TestProveCoverage_KeepLastObservedIsNoteOnly(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if !res.Proved { - t.Fatalf("Proved = false, want true: KeepLast is a note, never fatal: %+v", res) - } - if !anyContains(res.Notes, "KeepLast") { - t.Fatalf("Notes = %v, want a KeepLast note (comma-joined membership, not a literal-key match)", res.Notes) - } + require.True(t, res.Proved, "KeepLast is a note, never fatal") + require.True(t, anyContains(res.Notes, "KeepLast"), "comma-joined membership, not a literal-key match") } // KeepLast in the CONFIGURATION gives a note — a different claim from the @@ -468,12 +470,9 @@ func TestProveCoverage_KeepLastConfiguredIsNoteOnly(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, tc.def, from, to, 0) - if !res.Proved { - t.Fatalf("Proved = false, want true: a declared KeepLast is a note, never fatal: %+v", res) - } - if !anyContains(res.Notes, "KeepLast") { - t.Fatalf("Notes = %v, want a KeepLast note from the definition alone, with zero KeepLast reasons observed", res.Notes) - } + require.True(t, res.Proved, "a declared KeepLast is a note, never fatal") + require.True(t, anyContains(res.Notes, "KeepLast"), + "from the definition alone, with zero KeepLast reasons observed") }) } } @@ -503,9 +502,7 @@ func TestProveCoverage_SkewTranslationAtWindowBoundary(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if !res.Proved { - t.Fatalf("res = %+v, want proved: a constant clock skew must not itself read as a coverage gap", res) - } + require.True(t, res.Proved, "a constant clock skew must not itself read as a coverage gap") } // --- Override round-trip: one authority for the cadence --- @@ -524,9 +521,7 @@ func TestProveCoverage_OverrideRoundTrip(t *testing.T) { } defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 60}} rt, _, err := DeriveTimingsFromLog(h, defs) - if err != nil { - t.Fatalf("DeriveTimingsFromLog: %v", err) - } + require.NoError(t, err) var polls []Poll for ts := from; !ts.After(windowEnd); ts = ts.Add(120 * time.Second) { @@ -535,9 +530,8 @@ func TestProveCoverage_OverrideRoundTrip(t *testing.T) { sentinel := windowEnd res := proveCoverage(h, polls, &sentinel, rt["r1"], defs[0], from, windowEnd, 0) - if !res.Proved { - t.Fatalf("Proved = false, want true (maxGap must come from the recorded 120s cadence, not the 30s default): %+v", res) - } + require.True(t, res.Proved, + "maxGap must come from the recorded 120s cadence, not the 30s default") }) t.Run("faster override still catches a real recorder gap", func(t *testing.T) { @@ -548,9 +542,7 @@ func TestProveCoverage_OverrideRoundTrip(t *testing.T) { } defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 300}} rt, _, err := DeriveTimingsFromLog(h, defs) - if err != nil { - t.Fatalf("DeriveTimingsFromLog: %v", err) - } + require.NoError(t, err) var polls []Poll ts := from @@ -571,10 +563,8 @@ func TestProveCoverage_OverrideRoundTrip(t *testing.T) { sentinel := windowEnd res := proveCoverage(h, polls, &sentinel, rt["r1"], defs[0], from, windowEnd, 0) - if res.Reason != ReasonHeartbeatGap { - t.Fatalf("Reason = %q, want heartbeat_gap: if maxGap had been re-derived from the 300s definition instead of "+ - "the recorded 5s cadence, this 250s gap would pass silently — the fail-open direction", res.Reason) - } + require.Equal(t, ReasonHeartbeatGap, res.Reason, + "if maxGap had been re-derived from the 300s definition, this 250s gap would pass silently") }) } @@ -612,10 +602,8 @@ func TestProveCoverage_ZeroLastEvaluationWithoutPauseIsStale(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonStaleEvaluation { - t.Fatalf("Reason = %q, want stale_evaluation: a zero lastEvaluation on a found, non-paused poll must fail "+ - "closed, not be silently skipped as if it were a legitimately paused observation", res.Reason) - } + require.Equal(t, ReasonStaleEvaluation, res.Reason, + "a zero lastEvaluation on a found, non-paused poll must fail closed") } // A lastEvaluation in the future of grafana_now (corrupted log) must fail closed. @@ -635,10 +623,8 @@ func TestProveCoverage_FutureLastEvaluationIsUnobservable(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonFutureEvaluation { - t.Fatalf("Reason = %q, want future_evaluation: a lastEvaluation in the future of grafana_now must fail "+ - "closed rather than read its negative staleness as fresh", res.Reason) - } + require.Equal(t, ReasonFutureEvaluation, res.Reason, + "a lastEvaluation in the future of grafana_now must fail closed rather than read its negative staleness as fresh") } // --- Check 3, tightened: the boundary segments must widen by the skew bound --- @@ -663,10 +649,8 @@ func TestProveCoverage_BoundaryGapWidensBySkewBound(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonHeartbeatGap { - t.Fatalf("Reason = %q, want heartbeat_gap: the leading boundary segment sits at EXACTLY maxGap (60s) before "+ - "widening; the poll's own %s skew bound must push it past the threshold, not just the skew translation", res.Reason, bound) - } + require.Equal(t, ReasonHeartbeatGap, res.Reason, + "the poll's own %s skew bound must push it past the threshold", bound) } // --- Multi-failure contract --- @@ -698,16 +682,10 @@ func TestProveCoverage_MultipleFailuresReasonIsFirstButAllNoted(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonPausedInWindow { - t.Fatalf("Reason = %q, want paused_in_window: the FIRST check to fail names the reason", res.Reason) - } - if !anyContains(res.Notes, "paused") { - t.Fatalf("Notes = %v, want a note about the pause", res.Notes) - } - if !anyContains(res.Notes, "no rule") { - t.Fatalf("Notes = %v, want a note about the absence too — a later failure must still be recorded, "+ - "not swallowed once Reason is already set", res.Notes) - } + require.Equal(t, ReasonPausedInWindow, res.Reason, "the FIRST check to fail names the reason") + require.True(t, anyContains(res.Notes, "paused")) + require.True(t, anyContains(res.Notes, "no rule"), + "a later failure must still be recorded, not swallowed once Reason is already set") } // --- Skipped rules --- @@ -728,9 +706,6 @@ func TestProveCoverage_SkippedRuleWithZeroPollsPinnedAsHeartbeatGap(t *testing.T sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, nil, &sentinel, rt, def, from, to, 0) - if res.Reason != ReasonHeartbeatGap { - t.Fatalf("Reason = %q, want heartbeat_gap (pinned, not the desired end state): proveCoverage has no "+ - "'skipped' concept, so decide must handle a skipped rule's classification itself, before or "+ - "instead of calling this function", res.Reason) - } + require.Equal(t, ReasonHeartbeatGap, res.Reason, + "proveCoverage has no 'skipped' concept, so decide must handle a skipped rule's classification itself") } diff --git a/grafana-alertcheck/internal/gate/duration_test.go b/grafana-alertcheck/internal/gate/duration_test.go index ba3a1629a..6262c713e 100644 --- a/grafana-alertcheck/internal/gate/duration_test.go +++ b/grafana-alertcheck/internal/gate/duration_test.go @@ -3,6 +3,8 @@ package gate import ( "testing" "time" + + "github.com/stretchr/testify/require" ) func TestParsePromDuration(t *testing.T) { @@ -35,13 +37,8 @@ func TestParsePromDuration(t *testing.T) { } for _, c := range cases { got, err := ParsePromDuration(c.in) - if err != nil { - t.Errorf("ParsePromDuration(%q): unexpected error: %v", c.in, err) - continue - } - if got != c.want { - t.Errorf("ParsePromDuration(%q) = %v, want %v", c.in, got, c.want) - } + require.NoErrorf(t, err, "ParsePromDuration(%q)", c.in) + require.Equalf(t, c.want, got, "ParsePromDuration(%q)", c.in) } } @@ -60,8 +57,7 @@ func TestParsePromDuration_Errors(t *testing.T) { "carrot", // completely invalid } for _, in := range cases { - if _, err := ParsePromDuration(in); err == nil { - t.Errorf("ParsePromDuration(%q): expected an error, got none", in) - } + _, err := ParsePromDuration(in) + require.Errorf(t, err, "ParsePromDuration(%q): expected an error, got none", in) } } diff --git a/grafana-alertcheck/internal/gate/flock_test.go b/grafana-alertcheck/internal/gate/flock_test.go index 3e9ef268c..3dfdd8b73 100644 --- a/grafana-alertcheck/internal/gate/flock_test.go +++ b/grafana-alertcheck/internal/gate/flock_test.go @@ -5,18 +5,16 @@ import ( "fmt" "syscall" "testing" + + "github.com/stretchr/testify/require" ) func TestIsLockContention(t *testing.T) { contended := []error{syscall.EWOULDBLOCK, syscall.EAGAIN} for _, e := range contended { - if !isLockContention(e) { - t.Errorf("isLockContention(%v) = false, want true", e) - } + require.Truef(t, isLockContention(e), "isLockContention(%v)", e) // lockExclusive wraps the raw error via fmt.Errorf("flock: %w", ...). - if !isLockContention(fmt.Errorf("flock: %w", e)) { - t.Errorf("isLockContention(wrapped %v) = false, want true", e) - } + require.Truef(t, isLockContention(fmt.Errorf("flock: %w", e)), "isLockContention(wrapped %v)", e) } notContended := []error{ @@ -28,11 +26,7 @@ func TestIsLockContention(t *testing.T) { errors.New("something else"), } for _, e := range notContended { - if isLockContention(e) { - t.Errorf("isLockContention(%v) = true, want false (not a contender)", e) - } - if isLockContention(fmt.Errorf("flock: %w", e)) { - t.Errorf("isLockContention(wrapped %v) = true, want false", e) - } + require.Falsef(t, isLockContention(e), "isLockContention(%v)", e) + require.Falsef(t, isLockContention(fmt.Errorf("flock: %w", e)), "isLockContention(wrapped %v)", e) } } diff --git a/grafana-alertcheck/internal/gate/jsonreq_test.go b/grafana-alertcheck/internal/gate/jsonreq_test.go index bf3881e4b..72b446503 100644 --- a/grafana-alertcheck/internal/gate/jsonreq_test.go +++ b/grafana-alertcheck/internal/gate/jsonreq_test.go @@ -3,14 +3,14 @@ package gate import ( "encoding/json" "testing" + + "github.com/stretchr/testify/require" ) func rawMap(t *testing.T, jsonObj string) map[string]json.RawMessage { t.Helper() var m map[string]json.RawMessage - if err := json.Unmarshal([]byte(jsonObj), &m); err != nil { - t.Fatalf("rawMap: %v", err) - } + require.NoError(t, json.Unmarshal([]byte(jsonObj), &m)) return m } @@ -19,34 +19,24 @@ func TestReq(t *testing.T) { t.Run("present key decodes", func(t *testing.T) { var s string - if err := req(m, "present", &s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if s != "hello" { - t.Errorf("got %q, want hello", s) - } + require.NoError(t, req(m, "present", &s)) + require.Equal(t, "hello", s) }) t.Run("absent key errors", func(t *testing.T) { var s string - if err := req(m, "missing", &s); err == nil { - t.Fatalf("expected an error, got none") - } + require.Error(t, req(m, "missing", &s)) }) t.Run("wrong type errors", func(t *testing.T) { var s string - if err := req(m, "wrongtype", &s); err == nil { - t.Fatalf("expected an error, got none") - } + require.Error(t, req(m, "wrongtype", &s)) }) t.Run("explicit JSON null errors, never a zero value", func(t *testing.T) { var s string err := req(m, "nullval", &s) - if err == nil { - t.Fatalf("expected an error, got none (s=%q) — a null required field must not silently become a zero value", s) - } + require.Error(t, err, "a null required field must not silently become a zero value") }) } @@ -55,38 +45,24 @@ func TestOpt(t *testing.T) { t.Run("present key decodes", func(t *testing.T) { var s string - if err := opt(m, "present", &s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if s != "hello" { - t.Errorf("got %q, want hello", s) - } + require.NoError(t, opt(m, "present", &s)) + require.Equal(t, "hello", s) }) t.Run("absent key leaves dst untouched", func(t *testing.T) { s := "unchanged" - if err := opt(m, "missing", &s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if s != "unchanged" { - t.Errorf("got %q, want unchanged", s) - } + require.NoError(t, opt(m, "missing", &s)) + require.Equal(t, "unchanged", s) }) t.Run("wrong type errors", func(t *testing.T) { var s string - if err := opt(m, "wrongtype", &s); err == nil { - t.Fatalf("expected an error, got none") - } + require.Error(t, opt(m, "wrongtype", &s)) }) t.Run("explicit JSON null leaves dst at its zero value", func(t *testing.T) { var s string - if err := opt(m, "nullval", &s); err != nil { - t.Fatalf("unexpected error: %v", err) - } - if s != "" { - t.Errorf("got %q, want empty string", s) - } + require.NoError(t, opt(m, "nullval", &s)) + require.Equal(t, "", s) }) } diff --git a/grafana-alertcheck/internal/gate/log_test.go b/grafana-alertcheck/internal/gate/log_test.go index 179adb5fe..8ef3863e1 100644 --- a/grafana-alertcheck/internal/gate/log_test.go +++ b/grafana-alertcheck/internal/gate/log_test.go @@ -5,17 +5,18 @@ import ( "fmt" "os" "path/filepath" - "reflect" "strings" "sync" "testing" "time" + + "github.com/stretchr/testify/require" ) // Every time literal in this file is UTC and built with time.Date, so it // carries no monotonic reading and survives a JSON round trip byte-identical — -// which is what lets the round-trip tests below use reflect.DeepEqual on whole -// Poll values instead of comparing field by field. +// which is what lets the round-trip tests below compare whole Poll values +// instead of comparing field by field. var testNow = time.Date(2026, 8, 31, 9, 0, 0, 0, time.UTC) func testInstance(state State, reason, instanceLabel string) Instance { @@ -57,30 +58,20 @@ func TestLogReduceKeepsOnlyAbnormalInstances(t *testing.T) { p := NewReducer().Reduce("rule1", observation(testNow, rule)) - if !p.Found { - t.Fatalf("Found = false, want true") - } - if len(p.Abnormal) != 1 || p.Abnormal[0].Labels["instance"] != "b" { - t.Errorf("Abnormal = %+v, want only the firing instance b", p.Abnormal) - } - if want := map[string]int{"NoData": 1, "Error": 1}; !reflect.DeepEqual(p.Reasons, want) { - t.Errorf("Reasons = %v, want %v", p.Reasons, want) - } + require.True(t, p.Found) + require.Len(t, p.Abnormal, 1) + require.Equal(t, "b", p.Abnormal[0].Labels["instance"]) + require.Equal(t, map[string]int{"NoData": 1, "Error": 1}, p.Reasons) // The histogram is a verbatim copy of the response totals — raw keys, no // normalization. - if want := map[string]int{"alerting": 1, "normal": 2}; !reflect.DeepEqual(p.Histogram, want) { - t.Errorf("Histogram = %v, want %v", p.Histogram, want) - } + require.Equal(t, map[string]int{"alerting": 1, "normal": 2}, p.Histogram) // Rule-level state and health stay raw and unnormalized. - if p.State != "firing" || p.Health != "ok" { - t.Errorf("State/Health = %q/%q, want firing/ok", p.State, p.Health) - } - if p.Skew() != 1500*time.Millisecond || p.SkewBound() != 40*time.Millisecond || p.Latency() != 1800*time.Millisecond { - t.Errorf("durations = %s/%s/%s, want 1.5s/40ms/1.8s", p.Skew(), p.SkewBound(), p.Latency()) - } - if p.Reasons["MissingSeries"] != 0 { - t.Errorf("unexpected MissingSeries count") - } + require.Equal(t, "firing", p.State) + require.Equal(t, "ok", p.Health) + require.Equal(t, 1500*time.Millisecond, p.Skew()) + require.Equal(t, 40*time.Millisecond, p.SkewBound()) + require.Equal(t, 1800*time.Millisecond, p.Latency()) + require.Zero(t, p.Reasons["MissingSeries"]) } // A filtered response can hold several rules sharing one title, so the reducer @@ -94,9 +85,8 @@ func TestLogReduceSelectsRuleByUID(t *testing.T) { p := NewReducer().Reduce("ruleB", observation(testNow, first, second)) - if p.Health != "error" || len(p.Abnormal) != 1 { - t.Errorf("reduced the wrong rule: %+v", p) - } + require.Equal(t, "error", p.Health) + require.Len(t, p.Abnormal, 1) } func TestLogReduceRuleAbsentIsAuthoritative(t *testing.T) { @@ -104,20 +94,14 @@ func TestLogReduceRuleAbsentIsAuthoritative(t *testing.T) { p := NewReducer().Reduce("rule1", observation(testNow, other)) - if p.Found { - t.Errorf("Found = true, want false for a rule absent from an authoritative 2xx") - } - if p.RuleUID != "rule1" { - t.Errorf("RuleUID = %q, want rule1 — an absent rule is still attributed", p.RuleUID) - } + require.False(t, p.Found, "a rule absent from an authoritative 2xx") + require.Equal(t, "rule1", p.RuleUID, "an absent rule is still attributed") // The heartbeat still exists: a not-found poll is evidence that Grafana // answered at this time, which the coverage proof reads. - if !p.GrafanaNow.Equal(testNow) || p.Latency() == 0 { - t.Errorf("absent-rule poll lost its timing evidence: %+v", p) - } - if p.Health != "" || p.Abnormal != nil { - t.Errorf("absent-rule poll carries rule fields: %+v", p) - } + require.True(t, p.GrafanaNow.Equal(testNow)) + require.NotZero(t, p.Latency()) + require.Empty(t, p.Health) + require.Nil(t, p.Abnormal) } // An instance that leaves the abnormal set is resolved against the SAME @@ -175,19 +159,14 @@ func TestTransitionMarkersClearedVersusVanished(t *testing.T) { Instances: []Instance{testInstance(StateFiring, "", "b")}, } first := r.Reduce("rule1", observation(testNow, firing)) - if first.Cleared != nil || first.Vanished != nil { - t.Fatalf("first poll produced markers with no previous poll: %+v", first) - } + require.Nil(t, first.Cleared, "first poll produced cleared markers with no previous poll") + require.Nil(t, first.Vanished, "first poll produced vanished markers with no previous poll") next := StateRule{UID: "rule1", Health: "ok", State: "inactive", LastEvaluation: testNow, Instances: c.second} p := r.Reduce("rule1", observation(testNow.Add(30*time.Second), next)) - if !reflect.DeepEqual(p.Cleared, c.wantCleared) { - t.Errorf("Cleared = %q, want %q", p.Cleared, c.wantCleared) - } - if !reflect.DeepEqual(p.Vanished, c.wantVanished) { - t.Errorf("Vanished = %q, want %q", p.Vanished, c.wantVanished) - } + require.Equal(t, c.wantCleared, p.Cleared) + require.Equal(t, c.wantVanished, p.Vanished) }) } } @@ -204,15 +183,12 @@ func TestTransitionMarkersSurviveAnAbsentPoll(t *testing.T) { r.Reduce("rule1", observation(testNow, firing)) absent := r.Reduce("rule1", observation(testNow.Add(30*time.Second))) - if absent.Vanished != nil || absent.Cleared != nil { - t.Fatalf("an absent rule produced markers: %+v", absent) - } + require.Nil(t, absent.Vanished, "an absent rule produced vanished markers") + require.Nil(t, absent.Cleared, "an absent rule produced cleared markers") back := StateRule{UID: "rule1", Health: "ok", State: "inactive", LastEvaluation: testNow} p := r.Reduce("rule1", observation(testNow.Add(60*time.Second), back)) - if len(p.Vanished) != 1 { - t.Errorf("Vanished = %q, want the instance that disappeared across the absent poll", p.Vanished) - } + require.Len(t, p.Vanished, 1, "want the instance that disappeared across the absent poll") } func TestTransitionMarkersAreSortedAndPerRule(t *testing.T) { @@ -234,19 +210,14 @@ func TestTransitionMarkersAreSortedAndPerRule(t *testing.T) { clearedOne := StateRule{UID: "rule1", Health: "ok", State: "inactive", LastEvaluation: testNow} p := r.Reduce("rule1", observation(testNow.Add(time.Minute), clearedOne, ruleTwo)) - if len(p.Vanished) != 3 { - t.Fatalf("Vanished = %q, want 3 keys", p.Vanished) - } + require.Len(t, p.Vanished, 3) for i := 1; i < len(p.Vanished); i++ { - if p.Vanished[i-1] > p.Vanished[i] { - t.Errorf("Vanished is not sorted: %q", p.Vanished) - } + require.Less(t, p.Vanished[i-1], p.Vanished[i], "Vanished is not sorted: %q", p.Vanished) } // rule2's own abnormal set is untouched by rule1's transitions. q := r.Reduce("rule2", observation(testNow.Add(time.Minute), clearedOne, ruleTwo)) - if q.Cleared != nil || q.Vanished != nil { - t.Errorf("rule2 picked up rule1's transitions: %+v", q) - } + require.Nil(t, q.Cleared, "rule2 picked up rule1's transitions") + require.Nil(t, q.Vanished, "rule2 picked up rule1's transitions") } // The reduction depends on the state endpoint returning normal instances. If it @@ -267,22 +238,14 @@ func TestLogVerifyNormalInstancesVisible(t *testing.T) { for _, c := range cases { t.Run(c.fixture, func(t *testing.T) { rules, err := ParseState(readFixture(t, c.fixture)) - if err != nil { - t.Fatalf("ParseState: %v", err) - } + require.NoError(t, err) err = VerifyNormalInstancesVisible(rules) if c.wantError { - if err == nil { - t.Fatalf("VerifyNormalInstancesVisible: want an error, got nil") - } - if !strings.Contains(err.Error(), "no longer returns normal instances") { - t.Errorf("error does not say the endpoint stopped returning normal instances: %v", err) - } + require.Error(t, err) + require.Contains(t, err.Error(), "no longer returns normal instances") return } - if err != nil { - t.Fatalf("VerifyNormalInstancesVisible: unexpected error: %v", err) - } + require.NoError(t, err) }) } } @@ -310,9 +273,7 @@ func TestLogVerifyNormalInstancesVisibleVocabularies(t *testing.T) { Instances: []Instance{testInstance(StateFiring, "", "b")}, }} err := VerifyNormalInstancesVisible(rules) - if (err != nil) != c.wantError { - t.Errorf("VerifyNormalInstancesVisible: error = %v, want error = %v", err, c.wantError) - } + require.Equal(t, c.wantError, err != nil) }) } } @@ -328,37 +289,25 @@ func TestLogModeCadenceComesFromTheHeader(t *testing.T) { h.Rules[0].PollEverySeconds = 5 // an operator override far tighter than the default 150s rt, _, err := DeriveTimingsFromLog(h, defs) - if err != nil { - t.Fatalf("DeriveTimingsFromLog: %v", err) - } + require.NoError(t, err) got := rt["rule1"] - if got.pollEvery != 5*time.Second { - t.Errorf("pollEvery = %s, want the header's 5s, not the default 150s", got.pollEvery) - } + require.Equal(t, 5*time.Second, got.pollEvery, "the header's 5s, not the default 150s") // maxGap and healthGrace follow the recorded cadence; without this a 250s // hole in a log recorded at 5s would pass silently. - if got.maxGap != 10*time.Second { - t.Errorf("maxGap = %s, want 10s (2 x the recorded cadence)", got.maxGap) - } - if got.healthGrace != 300*time.Second { - t.Errorf("healthGrace = %s, want 300s (max(maxGap, interval))", got.healthGrace) - } + require.Equal(t, 10*time.Second, got.maxGap) + require.Equal(t, 300*time.Second, got.healthGrace) // evalStaleAfter is a rule fact, so it stays 2 x intervalSeconds from the // definitions regardless of how often the gate polled. - if got.evalStaleAfter != 600*time.Second { - t.Errorf("evalStaleAfter = %s, want 600s from the definition's interval", got.evalStaleAfter) - } + require.Equal(t, 600*time.Second, got.evalStaleAfter) // A log that cannot say how often it was written cannot have its coverage // proved, and neither can one naming a rule that no longer resolves. missingCadence := testHeader() missingCadence.Rules[0].PollEverySeconds = 0 - if _, _, err := DeriveTimingsFromLog(missingCadence, defs); err == nil { - t.Errorf("a header with no recorded cadence was accepted") - } - if _, _, err := DeriveTimingsFromLog(testHeader(), nil); err == nil { - t.Errorf("a header naming an unresolvable rule was accepted") - } + require.Error(t, func() error { _, _, err := DeriveTimingsFromLog(missingCadence, defs); return err }(), + "a header with no recorded cadence was accepted") + require.Error(t, func() error { _, _, err := DeriveTimingsFromLog(testHeader(), nil); return err }(), + "a header naming an unresolvable rule was accepted") // A duplicated UID must not resolve last-one-wins: the slower duplicate // would widen maxGap, which is fail-open through log corruption alone. @@ -366,9 +315,8 @@ func TestLogModeCadenceComesFromTheHeader(t *testing.T) { slower := duplicated.Rules[0] slower.PollEverySeconds = 600 duplicated.Rules = append(duplicated.Rules, slower) - if _, _, err := DeriveTimingsFromLog(duplicated, defs); err == nil { - t.Errorf("a header naming one rule twice was accepted") - } + require.Error(t, func() error { _, _, err := DeriveTimingsFromLog(duplicated, defs); return err }(), + "a header naming one rule twice was accepted") } // watch polls a fleet concurrently through one Reducer, so the marker @@ -398,9 +346,8 @@ func TestLogReduceIsSafeForConcurrentUse(t *testing.T) { // transition — the concurrency must not corrupt the per-rule state either. for _, rule := range rules { p := r.Reduce(rule.UID, obs) - if p.Cleared != nil || p.Vanished != nil { - t.Errorf("rule %s: markers after concurrent reduction: %+v", rule.UID, p) - } + require.Nilf(t, p.Cleared, "rule %s: cleared markers after concurrent reduction", rule.UID) + require.Nilf(t, p.Vanished, "rule %s: vanished markers after concurrent reduction", rule.UID) } } @@ -409,30 +356,18 @@ func TestLogReduceIsSafeForConcurrentUse(t *testing.T) { func TestLogPollOmitsTheZeroEvaluationTime(t *testing.T) { absent := NewReducer().Reduce("rule1", observation(testNow)) b, err := json.Marshal(pollRecord{Type: RecordPoll, Poll: absent}) - if err != nil { - t.Fatalf("marshal: %v", err) - } - if strings.Contains(string(b), "0001-01-01") { - t.Errorf("a not-found poll wrote the zero time: %s", b) - } - if strings.Contains(string(b), "last_evaluation") { - t.Errorf("a not-found poll wrote last_evaluation at all: %s", b) - } + require.NoError(t, err) + require.NotContains(t, string(b), "0001-01-01") + require.NotContains(t, string(b), "last_evaluation") // A real evaluation time still round-trips. found := StateRule{UID: "rule1", Health: "ok", State: "inactive", LastEvaluation: testNow} p := NewReducer().Reduce("rule1", observation(testNow, found)) b, err = json.Marshal(pollRecord{Type: RecordPoll, Poll: p}) - if err != nil { - t.Fatalf("marshal: %v", err) - } + require.NoError(t, err) var back pollRecord - if err := json.Unmarshal(b, &back); err != nil { - t.Fatalf("unmarshal: %v", err) - } - if !back.LastEvaluation.Equal(testNow) { - t.Errorf("last_evaluation = %s, want %s", back.LastEvaluation, testNow) - } + require.NoError(t, json.Unmarshal(b, &back)) + require.True(t, back.LastEvaluation.Equal(testNow)) } func testHeader() Header { @@ -452,9 +387,7 @@ func newTestWriter(t *testing.T, path string) (*Writer, *fakeClock) { t.Helper() clock := newFakeClock(testNow) w, err := NewWriter(path, clock) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } + require.NoError(t, err) return w, clock } @@ -463,9 +396,7 @@ func TestWriterReadLogRoundTrip(t *testing.T) { w, clock := newTestWriter(t, path) h := testHeader() - if err := w.WriteHeader(h); err != nil { - t.Fatalf("WriteHeader: %v", err) - } + require.NoError(t, w.WriteHeader(h)) r := NewReducer() firing := StateRule{ @@ -479,35 +410,21 @@ func TestWriterReadLogRoundTrip(t *testing.T) { r.Reduce("rule1", observation(testNow.Add(time.Minute), cleared)), } for _, p := range want { - if err := w.WritePoll(p); err != nil { - t.Fatalf("WritePoll: %v", err) - } + require.NoError(t, w.WritePoll(p)) } clock.Advance(2 * time.Minute) - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) - } + require.NoError(t, w.Stop()) gotHeader, gotPolls, sentinel, err := ReadLog(path) - if err != nil { - t.Fatalf("ReadLog: %v", err) - } + require.NoError(t, err) h.SchemaVersion = LogSchemaVersion // WriteHeader stamps it - if !reflect.DeepEqual(gotHeader, h) { - t.Errorf("header round trip:\n got %+v\nwant %+v", gotHeader, h) - } - if !reflect.DeepEqual(gotPolls, want) { - t.Errorf("poll round trip:\n got %+v\nwant %+v", gotPolls, want) - } - if sentinel == nil { - t.Fatalf("sentinel is nil after Stop") - } + require.Equal(t, h, gotHeader, "header round trip") + require.Equal(t, want, gotPolls, "poll round trip") + require.NotNil(t, sentinel, "sentinel is nil after Stop") // Stop stamps the recorder's own stop time and makes no comparison // against `to` — watch never knows it. - if !sentinel.Equal(testNow.Add(2 * time.Minute)) { - t.Errorf("sentinel = %s, want the writer's stop time %s", sentinel, testNow.Add(2*time.Minute)) - } + require.True(t, sentinel.Equal(testNow.Add(2*time.Minute))) } // The log is append-only. A second run against the same path must never @@ -515,44 +432,25 @@ func TestWriterReadLogRoundTrip(t *testing.T) { func TestWriterAppendsAndNeverTruncates(t *testing.T) { path := filepath.Join(t.TempDir(), "log.jsonl") w, _ := newTestWriter(t, path) - if err := w.WriteHeader(testHeader()); err != nil { - t.Fatalf("WriteHeader: %v", err) - } - if err := w.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow}); err != nil { - t.Fatalf("WritePoll: %v", err) - } - if err := w.Close(); err != nil { - t.Fatalf("Close: %v", err) - } + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow})) + require.NoError(t, w.Close()) before, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read: %v", err) - } + require.NoError(t, err) // The handoff: the parent wrote the header and closed; the child // reopens the same path and appends without a second header. child, _ := newTestWriter(t, path) - if err := child.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow.Add(time.Minute)}); err != nil { - t.Fatalf("child WritePoll: %v", err) - } - if err := child.Stop(); err != nil { - t.Fatalf("child Stop: %v", err) - } + require.NoError(t, child.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow.Add(time.Minute)})) + require.NoError(t, child.Stop()) after, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read: %v", err) - } - if !strings.HasPrefix(string(after), string(before)) { - t.Fatalf("reopening the log rewrote earlier records:\n%s", after) - } + require.NoError(t, err) + require.True(t, strings.HasPrefix(string(after), string(before)), "reopening the log rewrote earlier records") _, polls, sentinel, err := ReadLog(path) - if err != nil { - t.Fatalf("ReadLog: %v", err) - } - if len(polls) != 2 || sentinel == nil { - t.Errorf("got %d polls, sentinel %v; want 2 polls and a sentinel", len(polls), sentinel) - } + require.NoError(t, err) + require.Len(t, polls, 2) + require.NotNil(t, sentinel) } // Two recorders on one log means one of them is recording a window nobody @@ -570,67 +468,41 @@ func TestWriterSecondWriterFails(t *testing.T) { select { case err := <-done: - if err == nil { - t.Fatalf("a second writer took the lock") - } - if !strings.Contains(err.Error(), "another writer") { - t.Errorf("error does not name the conflict: %v", err) - } + require.Error(t, err, "a second writer took the lock") + require.Contains(t, err.Error(), "another writer") case <-time.After(5 * time.Second): - t.Fatalf("the second NewWriter blocked instead of failing immediately") + require.Fail(t, "the second NewWriter blocked instead of failing immediately") } } func TestWriterHeaderRefusesANonEmptyLog(t *testing.T) { path := filepath.Join(t.TempDir(), "log.jsonl") w, _ := newTestWriter(t, path) - if err := w.WriteHeader(testHeader()); err != nil { - t.Fatalf("WriteHeader: %v", err) - } - if err := w.WriteHeader(testHeader()); err == nil { - t.Fatalf("a second header was accepted") - } - if err := w.Close(); err != nil { - t.Fatalf("Close: %v", err) - } + require.NoError(t, w.WriteHeader(testHeader())) + require.Error(t, w.WriteHeader(testHeader()), "a second header was accepted") + require.NoError(t, w.Close()) reopened, _ := newTestWriter(t, path) defer reopened.Close() - if err := reopened.WriteHeader(testHeader()); err == nil { - t.Fatalf("a header was accepted on a non-empty log") - } + require.Error(t, reopened.WriteHeader(testHeader()), "a header was accepted on a non-empty log") } func TestSentinelStopIsIdempotentAndLast(t *testing.T) { path := filepath.Join(t.TempDir(), "log.jsonl") w, _ := newTestWriter(t, path) - if err := w.WriteHeader(testHeader()); err != nil { - t.Fatalf("WriteHeader: %v", err) - } - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) - } + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.Stop()) // watch reaches Stop from both a signal handler and a defer; a second // sentinel would be indistinguishable from a second writer. - if err := w.Stop(); err != nil { - t.Errorf("second Stop: %v", err) - } + require.NoError(t, w.Stop()) // Nothing may be appended after the sentinel — not even by the same writer. - if err := w.WritePoll(Poll{RuleUID: "rule1"}); err == nil { - t.Errorf("WritePoll after Stop was accepted") - } + require.Error(t, w.WritePoll(Poll{RuleUID: "rule1"}), "WritePoll after Stop was accepted") b, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read: %v", err) - } + require.NoError(t, err) lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n") - if len(lines) != 2 { - t.Fatalf("got %d lines, want header + one sentinel:\n%s", len(lines), b) - } - if !strings.Contains(lines[1], `"type":"stopped"`) { - t.Errorf("last line is not the sentinel: %s", lines[1]) - } + require.Len(t, lines, 2, "header + one sentinel") + require.Contains(t, lines[1], `"type":"stopped"`) } // Close is the parent's handoff path: a sentinel there would tell check the @@ -638,23 +510,13 @@ func TestSentinelStopIsIdempotentAndLast(t *testing.T) { func TestSentinelCloseWritesNone(t *testing.T) { path := filepath.Join(t.TempDir(), "log.jsonl") w, _ := newTestWriter(t, path) - if err := w.WriteHeader(testHeader()); err != nil { - t.Fatalf("WriteHeader: %v", err) - } - if err := w.Close(); err != nil { - t.Fatalf("Close: %v", err) - } + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.Close()) _, polls, sentinel, err := ReadLog(path) - if err != nil { - t.Fatalf("ReadLog: %v", err) - } - if sentinel != nil { - t.Errorf("Close wrote a sentinel: %s", sentinel) - } - if polls != nil { - t.Errorf("polls = %+v, want none", polls) - } + require.NoError(t, err) + require.Nil(t, sentinel, "Close wrote a sentinel") + require.Nil(t, polls) } // An unfinished recording reads cleanly with a nil sentinel — ReadLog reports @@ -664,23 +526,14 @@ func TestSentinelCloseWritesNone(t *testing.T) { func TestReadLogWithoutASentinel(t *testing.T) { path := filepath.Join(t.TempDir(), "log.jsonl") w, _ := newTestWriter(t, path) - if err := w.WriteHeader(testHeader()); err != nil { - t.Fatalf("WriteHeader: %v", err) - } - if err := w.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow}); err != nil { - t.Fatalf("WritePoll: %v", err) - } - if err := w.Close(); err != nil { - t.Fatalf("Close: %v", err) - } + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow})) + require.NoError(t, w.Close()) _, polls, sentinel, err := ReadLog(path) - if err != nil { - t.Fatalf("ReadLog: %v", err) - } - if sentinel != nil || len(polls) != 1 { - t.Errorf("got %d polls, sentinel %v; want 1 poll and no sentinel", len(polls), sentinel) - } + require.NoError(t, err) + require.Nil(t, sentinel) + require.Len(t, polls, 1) } // The read rules are deliberately the crudest possible: any unparseable @@ -691,23 +544,17 @@ func TestReadLogRejectsBadLogs(t *testing.T) { h := testHeader() h.SchemaVersion = version b, err := json.Marshal(headerRecord{Type: RecordHeader, Header: h}) - if err != nil { - t.Fatalf("marshal header: %v", err) - } + require.NoError(t, err, "marshal header") return string(b) } poll := func() string { b, err := json.Marshal(pollRecord{Type: RecordPoll, Poll: Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow}}) - if err != nil { - t.Fatalf("marshal poll: %v", err) - } + require.NoError(t, err, "marshal poll") return string(b) } sentinel := func() string { b, err := json.Marshal(stoppedRecord{Type: RecordStopped, At: testNow}) - if err != nil { - t.Fatalf("marshal sentinel: %v", err) - } + require.NoError(t, err, "marshal sentinel") return string(b) } @@ -758,25 +605,17 @@ func TestReadLogRejectsBadLogs(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { path := filepath.Join(t.TempDir(), "log.jsonl") - if err := os.WriteFile(path, []byte(c.content), 0o600); err != nil { - t.Fatalf("write: %v", err) - } + require.NoError(t, os.WriteFile(path, []byte(c.content), 0o600)) _, _, _, err := ReadLog(path) - if err == nil { - t.Fatalf("ReadLog: want an error, got nil") - } - if !strings.Contains(err.Error(), c.wantIn) { - t.Errorf("error %q does not contain %q", err, c.wantIn) - } + require.Error(t, err) + require.Contains(t, err.Error(), c.wantIn) }) } } func TestReadLogMissingFile(t *testing.T) { _, _, _, err := ReadLog(filepath.Join(t.TempDir(), "absent.jsonl")) - if err == nil { - t.Fatalf("ReadLog on a missing log: want an error, got nil") - } + require.Error(t, err) } // Per-poll log size must not grow across polls on a high-cardinality @@ -786,69 +625,46 @@ func TestReadLogMissingFile(t *testing.T) { func TestLogSizeIsFlatAcrossPollsOnAHighCardinalityRule(t *testing.T) { body := synthesizeHighCardinalityState(t, 1, 2445) rules, err := ParseState(body) - if err != nil { - t.Fatalf("ParseState: %v", err) - } - if len(rules[0].Instances) != 2446 { - t.Fatalf("got %d instances, want 2446", len(rules[0].Instances)) - } + require.NoError(t, err) + require.Len(t, rules[0].Instances, 2446) uid := rules[0].UID path := filepath.Join(t.TempDir(), "log.jsonl") w, _ := newTestWriter(t, path) - if err := w.WriteHeader(testHeader()); err != nil { - t.Fatalf("WriteHeader: %v", err) - } + require.NoError(t, w.WriteHeader(testHeader())) r := NewReducer() var sizes []int64 // Measure from the end of the header line, so sizes[0] is the first poll // record alone rather than the header plus it. info, err := os.Stat(path) - if err != nil { - t.Fatalf("stat: %v", err) - } + require.NoError(t, err) previous := info.Size() for i := range 5 { p := r.Reduce(uid, observation(testNow.Add(time.Duration(i)*30*time.Second), rules[0])) - if len(p.Abnormal) != 1 { - t.Fatalf("poll %d: Abnormal = %d instances, want the single firing one", i, len(p.Abnormal)) - } - if got := p.Abnormal[0].Labels["instance"]; got != "alerting-0" { - t.Fatalf("poll %d: the firing instance lost its identity: %q", i, got) - } - if err := w.WritePoll(p); err != nil { - t.Fatalf("WritePoll: %v", err) - } + require.Lenf(t, p.Abnormal, 1, "poll %d", i) + require.Equalf(t, "alerting-0", p.Abnormal[0].Labels["instance"], "poll %d: the firing instance lost its identity", i) + require.NoError(t, w.WritePoll(p)) info, err := os.Stat(path) - if err != nil { - t.Fatalf("stat: %v", err) - } + require.NoError(t, err) sizes = append(sizes, info.Size()-previous) previous = info.Size() } for i := 1; i < len(sizes); i++ { - if sizes[i] != sizes[0] { - t.Errorf("per-poll size grew across polls: %v", sizes) - } + require.Equal(t, sizes[0], sizes[i], "per-poll size grew across polls: %v", sizes) } // One firing instance among 2446 costs a few hundred bytes, against the // ~600 KB the unreduced response carries. - if sizes[0] > 2048 { - t.Errorf("per-poll size %d bytes is not a reduction of a %d-byte response", sizes[0], len(body)) - } + require.LessOrEqual(t, sizes[0], int64(2048)) // When the firing instance clears, the record collapses further and the // transition is still attributed. rules[0].Instances[0].State = StateNormal p := r.Reduce(uid, observation(testNow.Add(5*30*time.Second), rules[0])) - if len(p.Cleared) != 1 || len(p.Abnormal) != 0 { - t.Errorf("cleared poll = %+v, want exactly one cleared key and no abnormal instances", p) - } - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) - } + require.Len(t, p.Cleared, 1) + require.Empty(t, p.Abnormal) + require.NoError(t, w.Stop()) } // The log must stay readable by anything that reads JSONL, one flat object per @@ -857,39 +673,22 @@ func TestLogSizeIsFlatAcrossPollsOnAHighCardinalityRule(t *testing.T) { func TestLogRecordsAreFlatOneLineObjects(t *testing.T) { path := filepath.Join(t.TempDir(), "log.jsonl") w, _ := newTestWriter(t, path) - if err := w.WriteHeader(testHeader()); err != nil { - t.Fatalf("WriteHeader: %v", err) - } - if err := w.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow}); err != nil { - t.Fatalf("WritePoll: %v", err) - } - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) - } + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.WritePoll(Poll{RuleUID: "rule1", Found: true, GrafanaNow: testNow})) + require.NoError(t, w.Stop()) b, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read: %v", err) - } + require.NoError(t, err) lines := strings.Split(strings.TrimSuffix(string(b), "\n"), "\n") wantTypes := []RecordType{RecordHeader, RecordPoll, RecordStopped} - if len(lines) != len(wantTypes) { - t.Fatalf("got %d lines, want %d:\n%s", len(lines), len(wantTypes), b) - } + require.Len(t, lines, len(wantTypes)) for i, line := range lines { var m map[string]json.RawMessage - if err := json.Unmarshal([]byte(line), &m); err != nil { - t.Fatalf("line %d is not one JSON object: %v", i+1, err) - } + require.NoErrorf(t, json.Unmarshal([]byte(line), &m), "line %d is not one JSON object", i+1) var gotType RecordType - if err := json.Unmarshal(m["type"], &gotType); err != nil { - t.Fatalf("line %d has no type tag: %v", i+1, err) - } - if gotType != wantTypes[i] { - t.Errorf("line %d type = %q, want %q", i+1, gotType, wantTypes[i]) - } - if _, nested := m["header"]; nested { - t.Errorf("line %d wraps its payload instead of being flat: %s", i+1, line) - } + require.NoErrorf(t, json.Unmarshal(m["type"], &gotType), "line %d has no type tag", i+1) + require.Equalf(t, wantTypes[i], gotType, "line %d type", i+1) + _, nested := m["header"] + require.Falsef(t, nested, "line %d wraps its payload instead of being flat", i+1) } } diff --git a/grafana-alertcheck/internal/gate/parse_ruler.go b/grafana-alertcheck/internal/gate/parse_ruler.go index ae878d4c9..5517fcce2 100644 --- a/grafana-alertcheck/internal/gate/parse_ruler.go +++ b/grafana-alertcheck/internal/gate/parse_ruler.go @@ -84,7 +84,7 @@ func ParseDefinitions(body []byte) ([]Definition, error) { func parseDefinition(raw json.RawMessage, folder, group string) (Definition, error) { var m map[string]json.RawMessage if err := json.Unmarshal(raw, &m); err != nil { - return Definition{}, fmt.Errorf("%w", err) + return Definition{}, err } var forStr string diff --git a/grafana-alertcheck/internal/gate/parse_ruler_test.go b/grafana-alertcheck/internal/gate/parse_ruler_test.go index b225ffc2c..280dc9910 100644 --- a/grafana-alertcheck/internal/gate/parse_ruler_test.go +++ b/grafana-alertcheck/internal/gate/parse_ruler_test.go @@ -3,123 +3,88 @@ package gate import ( "testing" "time" + + "github.com/stretchr/testify/require" ) func TestParseDefinitions_RulerRules(t *testing.T) { defs, err := ParseDefinitions(readFixture(t, "ruler_rules.json")) - if err != nil { - t.Fatalf("ParseDefinitions: unexpected error: %v", err) - } + require.NoError(t, err) byUID := map[string]Definition{} for _, d := range defs { - if d.Kind != KindGrafanaManaged { - t.Errorf("rule %q: Kind = %v, want KindGrafanaManaged", d.UID, d.Kind) - } + require.Equalf(t, KindGrafanaManaged, d.Kind, "rule %q: Kind", d.UID) byUID[d.UID] = d } // The real 2-way duplicate title: same folder, same group, same title, // distinct UIDs — only uid: can tell them apart. a, ok := byUID["rule0000006a"] - if !ok { - t.Fatalf("missing rule0000006a") - } + require.True(t, ok, "missing rule0000006a") b, ok := byUID["rule0000006b"] - if !ok { - t.Fatalf("missing rule0000006b") - } - if a.Title != b.Title || a.Folder != b.Folder || a.Group != b.Group { - t.Errorf("duplicate-title pair should share Title/Folder/Group: a=%+v b=%+v", a, b) - } - if a.UID == b.UID { - t.Errorf("duplicate-title pair should have distinct UIDs") - } + require.True(t, ok, "missing rule0000006b") + require.Equal(t, a.Title, b.Title, "duplicate-title pair should share Title") + require.Equal(t, a.Folder, b.Folder, "duplicate-title pair should share Folder") + require.Equal(t, a.Group, b.Group, "duplicate-title pair should share Group") + require.NotEqual(t, a.UID, b.UID, "duplicate-title pair should have distinct UIDs") // The 3 real paused rules. pausedUIDs := []string{"rule0000002", "rule0000007", "rule0000008"} for _, uid := range pausedUIDs { d, ok := byUID[uid] - if !ok { - t.Fatalf("missing paused rule %q", uid) - } - if !d.IsPaused { - t.Errorf("rule %q: IsPaused = false, want true", uid) - } + require.True(t, ok, "missing paused rule %q", uid) + require.Truef(t, d.IsPaused, "rule %q: IsPaused = false, want true", uid) } // for:1d and the derived for:1w rule. dayRule, ok := byUID["rule0000009"] - if !ok || dayRule.For != 24*time.Hour { - t.Fatalf("rule0000009: For = %v, want 24h (ok=%v)", dayRule.For, ok) - } + require.Truef(t, ok, "missing rule0000009") + require.Equal(t, 24*time.Hour, dayRule.For) weekRule, ok := byUID["rule0000010"] - if !ok || weekRule.For != 7*24*time.Hour { - t.Fatalf("rule0000010: For = %v, want 168h (ok=%v)", weekRule.For, ok) - } + require.Truef(t, ok, "missing rule0000010") + require.Equal(t, 7*24*time.Hour, weekRule.For) // Identity shared with testdata/state_paused.json. shared := byUID["rule0000002"] - if shared.FolderUID != "folder0000002" { - t.Errorf("rule0000002: FolderUID = %q, want folder0000002", shared.FolderUID) - } + require.Equal(t, "folder0000002", shared.FolderUID) } func TestParseDefinitions_DatasourceManaged(t *testing.T) { defs, err := ParseDefinitions(readFixture(t, "ruler_datasource_managed.json")) - if err != nil { - t.Fatalf("ParseDefinitions: unexpected error: %v", err) - } - if len(defs) != 1 { - t.Fatalf("got %d definitions, want 1", len(defs)) - } - if defs[0].Kind != KindDatasourceManaged { - t.Errorf("Kind = %v, want KindDatasourceManaged", defs[0].Kind) - } - if defs[0].For != 5*time.Minute { - t.Errorf("For = %v, want 5m", defs[0].For) - } + require.NoError(t, err) + require.Len(t, defs, 1) + require.Equal(t, KindDatasourceManaged, defs[0].Kind) + require.Equal(t, 5*time.Minute, defs[0].For) // A datasource-managed rule has no uid in this shape; its only identity // is the Prometheus "alert" name — a synthetic UID would be invented // shape, and an empty Title would make Resolve's refusal-by-name // unreachable. - if defs[0].Title != "ExampleTargetDown" { - t.Errorf("Title = %q, want ExampleTargetDown", defs[0].Title) - } - if defs[0].UID != "" { - t.Errorf("UID = %q, want empty (this shape has no uid)", defs[0].UID) - } + require.Equal(t, "ExampleTargetDown", defs[0].Title) + require.Empty(t, defs[0].UID, "this shape has no uid") } func TestParseDefinitions_Recording(t *testing.T) { defs, err := ParseDefinitions(readFixture(t, "ruler_recording.json")) - if err != nil { - t.Fatalf("ParseDefinitions: unexpected error: %v", err) - } - if len(defs) != 1 { - t.Fatalf("got %d definitions, want 1", len(defs)) - } + require.NoError(t, err) + require.Len(t, defs, 1) d := defs[0] - if d.Kind != KindRecording { - t.Errorf("Kind = %v, want KindRecording", d.Kind) - } - if d.UID != "rule0000011" { - t.Errorf("UID = %q, want rule0000011", d.UID) - } + require.Equal(t, KindRecording, d.Kind) + require.Equal(t, "rule0000011", d.UID) // The fixture deliberately omits no_data_state/exec_err_state/is_paused/ // intervalSeconds/namespace_uid — alerting-only concepts a recording // rule may not carry. Requiring them would brick ParseDefinitions for // every named rule in the same response over one recording rule // elsewhere in the fleet; they must come back as zero values, not errors. - if d.NoDataState != "" || d.ExecErrState != "" || d.IsPaused || d.IntervalSeconds != 0 || d.FolderUID != "" { - t.Errorf("expected zero-valued alert-only fields for a recording rule, got %+v", d) - } + require.Empty(t, d.NoDataState) + require.Empty(t, d.ExecErrState) + require.False(t, d.IsPaused) + require.Zero(t, d.IntervalSeconds) + require.Empty(t, d.FolderUID) } // A datasource-managed rule with no alert/record name must fail parsing. func TestParseDefinitions_DatasourceManagedNoName(t *testing.T) { body := []byte(`{"ExampleMetrics":[{"name":"g","rules":[{"expr":"up == 0","for":"5m"}]}]}`) - if _, err := ParseDefinitions(body); err == nil { - t.Fatalf("ParseDefinitions: expected error for datasource-managed rule with no alert/record, got nil") - } + _, err := ParseDefinitions(body) + require.Error(t, err, "a datasource-managed rule with no alert/record must fail") } diff --git a/grafana-alertcheck/internal/gate/parse_state_test.go b/grafana-alertcheck/internal/gate/parse_state_test.go index 4ccca1391..f55c10185 100644 --- a/grafana-alertcheck/internal/gate/parse_state_test.go +++ b/grafana-alertcheck/internal/gate/parse_state_test.go @@ -1,23 +1,20 @@ package gate import ( - "bytes" "encoding/json" "fmt" - "maps" "os" "path/filepath" - "strings" "testing" "time" + + "github.com/stretchr/testify/require" ) func readFixture(t *testing.T, name string) []byte { t.Helper() b, err := os.ReadFile(filepath.Join("testdata", name)) - if err != nil { - t.Fatalf("reading fixture %s: %v", name, err) - } + require.NoErrorf(t, err, "reading fixture %s", name) return b } @@ -33,24 +30,15 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 1, checkFirst: func(t *testing.T, r StateRule) { - if r.UID != "rule0000001" { - t.Errorf("UID = %q, want rule0000001", r.UID) - } - if r.Folder != "ExampleTeam" || r.Group != "Example Service - Prod" { - t.Errorf("Folder/Group = %q/%q, want ExampleTeam/Example Service - Prod", r.Folder, r.Group) - } - if r.Health != "ok" || r.State != "inactive" { - t.Errorf("Health/State = %q/%q, want ok/inactive", r.Health, r.State) - } - if r.Interval.Seconds() != 60 { - t.Errorf("Interval = %v, want 60s", r.Interval) - } - if r.IsPaused { - t.Errorf("IsPaused = true, want false") - } - if len(r.Instances) != 1 || r.Instances[0].State != StateNormal { - t.Fatalf("Instances = %+v, want one normal instance", r.Instances) - } + require.Equal(t, "rule0000001", r.UID) + require.Equal(t, "ExampleTeam", r.Folder) + require.Equal(t, "Example Service - Prod", r.Group) + require.Equal(t, "ok", r.Health) + require.Equal(t, "inactive", r.State) + require.Equal(t, float64(60), r.Interval.Seconds()) + require.False(t, r.IsPaused) + require.Len(t, r.Instances, 1) + require.Equal(t, StateNormal, r.Instances[0].State) inst := r.Instances[0] wantLabels := map[string]string{ @@ -62,19 +50,11 @@ func TestParseState_HappyPaths(t *testing.T) { "severity": "critical", "team": "example-team", } - if !maps.Equal(inst.Labels, wantLabels) { - t.Errorf("Labels = %+v, want %+v", inst.Labels, wantLabels) - } + require.Equal(t, wantLabels, inst.Labels) wantActiveAt, err := time.Parse(time.RFC3339, "2026-08-31T08:02:50Z") - if err != nil { - t.Fatalf("test setup: %v", err) - } - if !inst.ActiveAt.Equal(wantActiveAt) { - t.Errorf("ActiveAt = %v, want %v", inst.ActiveAt, wantActiveAt) - } - if inst.Value != "" { - t.Errorf("Value = %q, want empty string", inst.Value) - } + require.NoError(t, err, "test setup") + require.True(t, inst.ActiveAt.Equal(wantActiveAt)) + require.Empty(t, inst.Value) }, }, { @@ -82,15 +62,10 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 0, checkFirst: func(t *testing.T, r StateRule) { - if !r.IsPaused { - t.Errorf("IsPaused = false, want true") - } - if !r.LastEvaluation.IsZero() { - t.Errorf("LastEvaluation = %v, want zero time", r.LastEvaluation) - } - if r.Health != "ok" || r.State != "inactive" { - t.Errorf("Health/State = %q/%q, want ok/inactive", r.Health, r.State) - } + require.True(t, r.IsPaused) + require.True(t, r.LastEvaluation.IsZero()) + require.Equal(t, "ok", r.Health) + require.Equal(t, "inactive", r.State) }, }, { @@ -98,15 +73,10 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 1, checkFirst: func(t *testing.T, r StateRule) { - if r.Health != "error" { - t.Errorf("Health = %q, want error", r.Health) - } - if r.LastError == "" { - t.Errorf("LastError is empty, want a message") - } - if len(r.Instances) != 1 || r.Instances[0].State != StateError { - t.Fatalf("Instances = %+v, want one error instance", r.Instances) - } + require.Equal(t, "error", r.Health) + require.NotEmpty(t, r.LastError) + require.Len(t, r.Instances, 1) + require.Equal(t, StateError, r.Instances[0].State) }, }, { @@ -114,12 +84,9 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 1, checkFirst: func(t *testing.T, r StateRule) { - if r.Health != "nodata" { - t.Errorf("Health = %q, want nodata", r.Health) - } - if len(r.Instances) != 1 || r.Instances[0].State != StateNodata { - t.Fatalf("Instances = %+v, want one nodata instance", r.Instances) - } + require.Equal(t, "nodata", r.Health) + require.Len(t, r.Instances, 1) + require.Equal(t, StateNodata, r.Instances[0].State) }, }, { @@ -132,17 +99,14 @@ func TestParseState_HappyPaths(t *testing.T) { byReason[inst.Reason] = inst } errInst, ok := byReason["Error"] - if !ok || errInst.State != StateNormal { - t.Errorf(`want an instance with State=normal Reason="Error", got %+v`, byReason["Error"]) - } + require.True(t, ok, `want an instance with Reason="Error"`) + require.Equal(t, StateNormal, errInst.State) nodataInst, ok := byReason["NoData"] - if !ok || nodataInst.State != StateNormal { - t.Errorf(`want an instance with State=normal Reason="NoData", got %+v`, byReason["NoData"]) - } + require.True(t, ok, `want an instance with Reason="NoData"`) + require.Equal(t, StateNormal, nodataInst.State) plain, ok := byReason[""] - if !ok || plain.State != StateNormal { - t.Errorf(`want a plain State=normal Reason="" instance, got %+v`, byReason[""]) - } + require.True(t, ok, `want a plain Reason="" instance`) + require.Equal(t, StateNormal, plain.State) }, }, { @@ -150,12 +114,8 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 0, checkFirst: func(t *testing.T, r StateRule) { - if r.Instances != nil { - t.Errorf("Instances = %+v, want nil", r.Instances) - } - if r.Totals != nil { - t.Errorf("Totals = %+v, want nil", r.Totals) - } + require.Nil(t, r.Instances) + require.Nil(t, r.Totals) }, }, { @@ -163,12 +123,10 @@ func TestParseState_HappyPaths(t *testing.T) { wantRules: 1, wantInstances: 1, checkFirst: func(t *testing.T, r StateRule) { - if len(r.Instances) != 1 || r.Instances[0].State != StateFiring { - t.Fatalf("Instances = %+v, want one firing instance", r.Instances) - } - if r.Totals["normal"] == 0 { - t.Errorf(`Totals["normal"] = 0, want >0 (the totals/instances mismatch this fixture exists to capture)`) - } + require.Len(t, r.Instances, 1) + require.Equal(t, StateFiring, r.Instances[0].State) + require.NotZero(t, r.Totals["normal"], + "the totals/instances mismatch this fixture exists to capture") }, }, } @@ -176,15 +134,9 @@ func TestParseState_HappyPaths(t *testing.T) { for _, c := range cases { t.Run(c.fixture, func(t *testing.T) { rules, err := ParseState(readFixture(t, c.fixture)) - if err != nil { - t.Fatalf("ParseState(%s): unexpected error: %v", c.fixture, err) - } - if len(rules) != c.wantRules { - t.Fatalf("ParseState(%s): got %d rules, want %d", c.fixture, len(rules), c.wantRules) - } - if got := len(rules[0].Instances); got != c.wantInstances { - t.Fatalf("ParseState(%s): got %d instances, want %d", c.fixture, got, c.wantInstances) - } + require.NoErrorf(t, err, "ParseState(%s)", c.fixture) + require.Lenf(t, rules, c.wantRules, "ParseState(%s)", c.fixture) + require.Lenf(t, rules[0].Instances, c.wantInstances, "ParseState(%s)", c.fixture) if c.checkFirst != nil { c.checkFirst(t, rules[0]) } @@ -214,13 +166,9 @@ func TestParseState_MustError(t *testing.T) { for _, c := range cases { t.Run(c.fixture, func(t *testing.T) { _, err := ParseState(readFixture(t, c.fixture)) - if err == nil { - t.Fatalf("ParseState(%s): expected an error, got none", c.fixture) - } + require.Errorf(t, err, "ParseState(%s): expected an error, got none", c.fixture) for _, want := range c.wantContains { - if !strings.Contains(err.Error(), want) { - t.Errorf("ParseState(%s): error %q does not mention %q", c.fixture, err.Error(), want) - } + require.Containsf(t, err.Error(), want, "ParseState(%s): error", c.fixture) } }) } @@ -248,49 +196,31 @@ func TestParseNormalizeInstanceState(t *testing.T) { for _, c := range cases { state, reason, err := normalizeInstanceState(c.in) if c.wantErr { - if err == nil { - t.Errorf("normalizeInstanceState(%q): expected an error, got none", c.in) - } - continue - } - if err != nil { - t.Errorf("normalizeInstanceState(%q): unexpected error: %v", c.in, err) + require.Errorf(t, err, "normalizeInstanceState(%q)", c.in) continue } - if state != c.wantState || reason != c.wantReason { - t.Errorf("normalizeInstanceState(%q) = (%q, %q), want (%q, %q)", c.in, state, reason, c.wantState, c.wantReason) - } + require.NoErrorf(t, err, "normalizeInstanceState(%q)", c.in) + require.Equalf(t, c.wantState, state, "normalizeInstanceState(%q)", c.in) + require.Equalf(t, c.wantReason, reason, "normalizeInstanceState(%q)", c.in) } } func TestInstanceKey(t *testing.T) { a := instanceKey(map[string]string{"b": "2", "a": "1"}) b := instanceKey(map[string]string{"a": "1", "b": "2"}) - if a != b { - t.Errorf("instanceKey order-independence: %q != %q", a, b) - } - if a != `{"a":"1","b":"2"}` { - t.Errorf("instanceKey = %q, want %q", a, `{"a":"1","b":"2"}`) - } + require.Equal(t, b, a, "instanceKey order-independence") + require.Equal(t, `{"a":"1","b":"2"}`, a) diff := instanceKey(map[string]string{"a": "1", "b": "3"}) - if a == diff { - t.Errorf("instanceKey should differ when a label value differs") - } + require.NotEqual(t, a, diff, "instanceKey should differ when a label value differs") - if instanceKey(nil) != "null" { - t.Errorf("instanceKey(nil) = %q, want \"null\"", instanceKey(nil)) - } + require.Equal(t, "null", instanceKey(nil), "instanceKey(nil) should be \"null\"") } -// Label values may contain "\n" or "="; the JSON encoding must keep them distinct. func TestInstanceKey_NoCollision(t *testing.T) { - if instanceKey(map[string]string{"a": "1\nb=2"}) == instanceKey(map[string]string{"a": "1", "b": "2"}) { - t.Errorf("instanceKey collided for sets {a:1\\nb=2} and {a:1,b:2}") - } - if instanceKey(map[string]string{"a": "1=b"}) == instanceKey(map[string]string{"a": "1", "b": ""}) { - t.Errorf("instanceKey collided for a value containing '='") - } + require.NotEqual(t, instanceKey(map[string]string{"a": "1\nb=2"}), instanceKey(map[string]string{"a": "1", "b": "2"}), "instanceKey should not collide for sets {a:1\\nb=2} and {a:1,b:2}") + + require.NotEqual(t, instanceKey(map[string]string{"a": "1=b"}), instanceKey(map[string]string{"a": "1", "b": ""}), "instanceKey should not collide for sets {a:1=b} and {a:1,b:}") } // minimalStateBody is the smallest legal state response: one group, one @@ -319,12 +249,8 @@ func TestParseState_KeepFiringForIsOptional(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { rules, err := ParseState(minimalStateBody(tc.extra)) - if err != nil { - t.Fatalf("ParseState: %v", err) - } - if len(rules) != 1 { - t.Fatalf("rules = %+v, want one", rules) - } + require.NoError(t, err) + require.Len(t, rules, 1) }) } } @@ -335,15 +261,10 @@ func TestParseState_KeepFiringForIsOptional(t *testing.T) { func TestParseState_InstanceWithoutLabelsParses(t *testing.T) { body := minimalStateBody(`,"alerts":[{"state":"Normal","activeAt":"2026-01-01T00:00:00Z"}]`) rules, err := ParseState(body) - if err != nil { - t.Fatalf("ParseState: %v", err) - } - if len(rules) != 1 || len(rules[0].Instances) != 1 { - t.Fatalf("rules = %+v, want one rule with one instance", rules) - } - if got := rules[0].Instances[0].Labels; len(got) != 0 { - t.Errorf("Instance.Labels = %v, want empty/nil", got) - } + require.NoError(t, err) + require.Len(t, rules, 1) + require.Len(t, rules[0].Instances, 1) + require.Empty(t, rules[0].Instances[0].Labels) } // synthesizeHighCardinalityState builds a state response with a single rule @@ -357,25 +278,15 @@ func synthesizeHighCardinalityState(t *testing.T, alerting, normal int) []byte { base := readFixture(t, "state_one_instance.json") var top map[string]json.RawMessage - if err := json.Unmarshal(base, &top); err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, json.Unmarshal(base, &top)) var data map[string]json.RawMessage - if err := json.Unmarshal(top["data"], &data); err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, json.Unmarshal(top["data"], &data)) var groups []map[string]json.RawMessage - if err := json.Unmarshal(data["groups"], &groups); err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, json.Unmarshal(data["groups"], &groups)) var rules []map[string]json.RawMessage - if err := json.Unmarshal(groups[0]["rules"], &rules); err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, json.Unmarshal(groups[0]["rules"], &rules)) var alerts []map[string]json.RawMessage - if err := json.Unmarshal(rules[0]["alerts"], &alerts); err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, json.Unmarshal(rules[0]["alerts"], &alerts)) template := alerts[0] newAlerts := make([]map[string]json.RawMessage, 0, alerting+normal) @@ -400,9 +311,7 @@ func synthesizeHighCardinalityState(t *testing.T, alerting, normal int) []byte { top["data"] = mustRaw(t, data) out, err := json.Marshal(top) - if err != nil { - t.Fatalf("synthesize: %v", err) - } + require.NoError(t, err) return out } @@ -419,29 +328,19 @@ func cloneRawMap(m map[string]json.RawMessage) map[string]json.RawMessage { func mustRaw(t *testing.T, v any) json.RawMessage { t.Helper() b, err := json.Marshal(v) - if err != nil { - t.Fatalf("marshal: %v", err) - } + require.NoError(t, err) return json.RawMessage(b) } func TestParseState_HighCardinality(t *testing.T) { body := synthesizeHighCardinalityState(t, 445, 2004) - if !bytes.Contains(body, []byte("alerting-0")) { - t.Fatalf("synthesized body missing expected content") - } + require.Contains(t, string(body), "alerting-0") rules, err := ParseState(body) - if err != nil { - t.Fatalf("ParseState: unexpected error: %v", err) - } - if len(rules) != 1 { - t.Fatalf("got %d rules, want 1", len(rules)) - } + require.NoError(t, err) + require.Len(t, rules, 1) r := rules[0] - if len(r.Instances) != 445+2004 { - t.Fatalf("got %d instances, want %d", len(r.Instances), 445+2004) - } + require.Len(t, r.Instances, 445+2004) var firing, normal int for _, inst := range r.Instances { @@ -451,28 +350,21 @@ func TestParseState_HighCardinality(t *testing.T) { case StateNormal: normal++ default: - t.Fatalf("unexpected instance state %q", inst.State) + require.Fail(t, fmt.Sprintf("unexpected instance state %q", inst.State)) } } - if firing != 445 || normal != 2004 { - t.Fatalf("got firing=%d normal=%d, want firing=445 normal=2004", firing, normal) - } + require.Equal(t, 445, firing) + require.Equal(t, 2004, normal) // Each synthesized instance carries a distinct "instance" label; confirm // Labels actually made it through parsing (not just State) by checking // instanceKey produces one unique key per instance, with no collisions. seen := make(map[string]bool, len(r.Instances)) for _, inst := range r.Instances { - if inst.Labels == nil { - t.Fatalf("instance has nil Labels") - } + require.NotNil(t, inst.Labels) k := instanceKey(inst.Labels) - if seen[k] { - t.Fatalf("duplicate instance key %q", k) - } + require.Falsef(t, seen[k], "duplicate instance key %q", k) seen[k] = true } - if len(seen) != 445+2004 { - t.Fatalf("got %d unique instance keys, want %d", len(seen), 445+2004) - } + require.Len(t, seen, 445+2004) } diff --git a/grafana-alertcheck/internal/gate/resolve_test.go b/grafana-alertcheck/internal/gate/resolve_test.go index d58ec73bd..33214516e 100644 --- a/grafana-alertcheck/internal/gate/resolve_test.go +++ b/grafana-alertcheck/internal/gate/resolve_test.go @@ -2,128 +2,89 @@ package gate import ( "fmt" - "strings" "testing" + + "github.com/stretchr/testify/require" ) func rulerDefs(t *testing.T) []Definition { t.Helper() defs, err := ParseDefinitions(readFixture(t, "ruler_rules.json")) - if err != nil { - t.Fatalf("ParseDefinitions: unexpected error: %v", err) - } + require.NoError(t, err) return defs } func TestResolve_SingleMatch(t *testing.T) { defs := rulerDefs(t) resolved, notes, err := Resolve(defs, []string{"example_workflow_paused_rule"}, "") - if err != nil { - t.Fatalf("Resolve: unexpected error: %v", err) - } - if len(notes) != 0 { - t.Errorf("notes = %v, want none", notes) - } - if len(resolved) != 1 || resolved[0].UID != "rule0000007" { - t.Fatalf("resolved = %+v, want [rule0000007]", resolved) - } + require.NoError(t, err) + require.Empty(t, notes) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000007", resolved[0].UID) } func TestResolve_UIDForm(t *testing.T) { defs := rulerDefs(t) resolved, _, err := Resolve(defs, []string{"uid:rule0000006a"}, "") - if err != nil { - t.Fatalf("Resolve: unexpected error: %v", err) - } - if len(resolved) != 1 || resolved[0].UID != "rule0000006a" { - t.Fatalf("resolved = %+v, want [rule0000006a]", resolved) - } + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000006a", resolved[0].UID) } func TestResolve_FolderTitleForm(t *testing.T) { defs := rulerDefs(t) resolved, _, err := Resolve(defs, []string{"ExampleFeeds/TEMP - Example depeg alert"}, "") - if err != nil { - t.Fatalf("Resolve: unexpected error: %v", err) - } - if len(resolved) != 1 || resolved[0].UID != "rule0000008" { - t.Fatalf("resolved = %+v, want [rule0000008]", resolved) - } + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000008", resolved[0].UID) } func TestResolve_FolderGroupTitleForm(t *testing.T) { defs := rulerDefs(t) resolved, _, err := Resolve(defs, []string{"Example-Zone-A/Gateway/Example No Gateways Available"}, "") - if err == nil { - t.Fatalf("Resolve: want ambiguous error (real 2-way collision), got resolved=%+v", resolved) - } - if !strings.Contains(err.Error(), "matches 2 rules") { - t.Fatalf("Resolve: error = %q, want it to report 2 matches", err) - } - if !strings.Contains(err.Error(), "uid:rule0000006a") || !strings.Contains(err.Error(), "uid:rule0000006b") { - t.Fatalf("Resolve: error = %q, want both candidate uids listed", err) - } + require.Error(t, err, "want ambiguous error (real 2-way collision), got resolved=%+v", resolved) + require.Contains(t, err.Error(), "matches 2 rules") + require.Contains(t, err.Error(), "uid:rule0000006a") + require.Contains(t, err.Error(), "uid:rule0000006b") } func TestResolve_TrueCollisionResolvesByUID(t *testing.T) { defs := rulerDefs(t) resolved, _, err := Resolve(defs, []string{"uid:rule0000006a", "uid:rule0000006b"}, "") - if err != nil { - t.Fatalf("Resolve: unexpected error: %v", err) - } - if len(resolved) != 2 { - t.Fatalf("resolved = %+v, want 2 distinct rules", resolved) - } + require.NoError(t, err) + require.Len(t, resolved, 2) } func TestResolve_NoMatch(t *testing.T) { defs := rulerDefs(t) _, _, err := Resolve(defs, []string{"Does Not Exist"}, "") - if err == nil { - t.Fatal("Resolve: want error for unknown name") - } - if !strings.Contains(err.Error(), "no rule matched") || !strings.Contains(err.Error(), "list") { - t.Errorf("Resolve: error = %q, want it to name 'no rule matched' and point at 'list'", err) - } + require.Error(t, err) + require.Contains(t, err.Error(), "no rule matched") + require.Contains(t, err.Error(), "list") } func TestResolve_NoMatchSubstringSuggestion(t *testing.T) { defs := rulerDefs(t) _, _, err := Resolve(defs, []string{"paused rule"}, "") - if err == nil { - t.Fatal("Resolve: want error for unknown name") - } - if !strings.Contains(err.Error(), "did you mean") || !strings.Contains(err.Error(), "Example Paused Rule") { - t.Errorf("Resolve: error = %q, want a case-insensitive substring suggestion", err) - } + require.Error(t, err) + require.Contains(t, err.Error(), "did you mean") + require.Contains(t, err.Error(), "Example Paused Rule") } func TestResolve_RefusesDatasourceManaged(t *testing.T) { defs, err := ParseDefinitions(readFixture(t, "ruler_datasource_managed.json")) - if err != nil { - t.Fatalf("ParseDefinitions: unexpected error: %v", err) - } + require.NoError(t, err) _, _, err = Resolve(defs, []string{"ExampleTargetDown"}, "") - if err == nil { - t.Fatal("Resolve: want refusal for a datasource-managed rule") - } - if !strings.Contains(err.Error(), "datasource-managed") { - t.Errorf("Resolve: error = %q, want it to name the datasource-managed kind", err) - } + require.Error(t, err) + require.Contains(t, err.Error(), "datasource-managed") } func TestResolve_RefusesRecording(t *testing.T) { defs, err := ParseDefinitions(readFixture(t, "ruler_recording.json")) - if err != nil { - t.Fatalf("ParseDefinitions: unexpected error: %v", err) - } + require.NoError(t, err) _, _, err = Resolve(defs, []string{"uid:rule0000011"}, "") - if err == nil { - t.Fatal("Resolve: want refusal for a recording rule") - } - if !strings.Contains(err.Error(), "recording rule") { - t.Errorf("Resolve: error = %q, want it to name the recording kind", err) - } + require.Error(t, err) + require.Contains(t, err.Error(), "recording rule") } func TestResolve_RejectsEmptySegments(t *testing.T) { @@ -132,12 +93,8 @@ func TestResolve_RejectsEmptySegments(t *testing.T) { for _, name := range cases { t.Run(name, func(t *testing.T) { _, _, err := Resolve(defs, []string{name}, "") - if err == nil { - t.Fatalf("Resolve(%q): want error for an empty /-separated segment", name) - } - if !strings.Contains(err.Error(), "empty") { - t.Errorf("Resolve(%q): error = %q, want it to name the empty segment", name, err) - } + require.Error(t, err) + require.Contains(t, err.Error(), "empty") }) } } @@ -148,51 +105,29 @@ func TestResolve_UIDEmptySuffix(t *testing.T) { // — that would report the misleading "datasource-managed rule, not // supported" for what is really a typo'd/empty uid. defs, err := ParseDefinitions(readFixture(t, "ruler_datasource_managed.json")) - if err != nil { - t.Fatalf("ParseDefinitions: unexpected error: %v", err) - } + require.NoError(t, err) _, _, err = Resolve(defs, []string{"uid:"}, "") - if err == nil { - t.Fatal("Resolve: want error for an empty uid: suffix") - } - if !strings.Contains(err.Error(), "no rule has this uid") { - t.Errorf("Resolve: error = %q, want it to say no rule has this uid", err) - } - if strings.Contains(err.Error(), "datasource-managed") { - t.Errorf("Resolve: error = %q, must not misreport this as a datasource-managed refusal", err) - } + require.Error(t, err) + require.Contains(t, err.Error(), "no rule has this uid") + require.NotContains(t, err.Error(), "datasource-managed") } func TestResolve_UnsupportedKindsExcludedFromNoMatchSurfaces(t *testing.T) { dsDefs, err := ParseDefinitions(readFixture(t, "ruler_datasource_managed.json")) - if err != nil { - t.Fatalf("ParseDefinitions(datasource_managed): unexpected error: %v", err) - } + require.NoError(t, err) recDefs, err := ParseDefinitions(readFixture(t, "ruler_recording.json")) - if err != nil { - t.Fatalf("ParseDefinitions(recording): unexpected error: %v", err) - } + require.NoError(t, err) supported := rulerDefs(t) combined := append(append(append([]Definition{}, supported...), dsDefs...), recDefs...) _, _, err = Resolve(combined, []string{"Example"}, "") - if err == nil { - t.Fatal("Resolve: want a no-match error for a name matching no title exactly") - } + require.Error(t, err, "want a no-match error for a name matching no title exactly") wantCount := fmt.Sprintf("(%d rules available", len(supported)) - if !strings.Contains(err.Error(), wantCount) { - t.Errorf("Resolve: error = %q, want the available count scoped to the %d supported rules, not the %d combined", err, len(supported), len(combined)) - } - if strings.Contains(err.Error(), "ExampleTargetDown") { - t.Errorf("Resolve: error = %q, must not suggest the datasource-managed rule", err) - } - if strings.Contains(err.Error(), "example:recorded_metric:rate5m") { - t.Errorf("Resolve: error = %q, must not suggest the recording rule", err) - } - if !strings.Contains(err.Error(), "Example Paused Rule") { - t.Errorf("Resolve: error = %q, want it to still suggest a matching supported rule", err) - } + require.Contains(t, err.Error(), wantCount) + require.NotContains(t, err.Error(), "ExampleTargetDown") + require.NotContains(t, err.Error(), "example:recorded_metric:rate5m") + require.Contains(t, err.Error(), "Example Paused Rule") } func TestResolve_UnsupportedHomonymResolvesSupportedSilently(t *testing.T) { @@ -205,12 +140,9 @@ func TestResolve_UnsupportedHomonymResolvesSupportedSilently(t *testing.T) { {UID: "", Folder: "F", Group: "G", Title: "Shared Title", Kind: KindDatasourceManaged}, } resolved, _, err := Resolve(defs, []string{"F/G/Shared Title"}, "") - if err != nil { - t.Fatalf("Resolve: unexpected error: %v", err) - } - if len(resolved) != 1 || resolved[0].UID != "supported-1" { - t.Fatalf("resolved = %+v, want the supported rule alone, no ambiguity", resolved) - } + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "supported-1", resolved[0].UID) } func TestResolve_CollapseByUIDGivesNoteNotError(t *testing.T) { @@ -221,15 +153,10 @@ func TestResolve_CollapseByUIDGivesNoteNotError(t *testing.T) { "example_workflow_paused_rule", "ExampleObservability/Example Auth Production/example_workflow_paused_rule", }, "") - if err != nil { - t.Fatalf("Resolve: unexpected error: %v", err) - } - if len(resolved) != 1 || resolved[0].UID != "rule0000007" { - t.Fatalf("resolved = %+v, want exactly one rule0000007", resolved) - } - if len(notes) != 1 { - t.Fatalf("notes = %v, want exactly one collapse note", notes) - } + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000007", resolved[0].UID) + require.Len(t, notes, 1) } // The same rule named twice with the identical string must collapse to one @@ -240,15 +167,10 @@ func TestResolve_IdenticalDuplicateNameCollapsesWithNote(t *testing.T) { "example_workflow_paused_rule", "example_workflow_paused_rule", }, "") - if err != nil { - t.Fatalf("Resolve: unexpected error: %v", err) - } - if len(resolved) != 1 || resolved[0].UID != "rule0000007" { - t.Fatalf("resolved = %+v, want exactly one rule0000007", resolved) - } - if len(notes) != 1 { - t.Fatalf("notes = %v, want exactly one collapse note", notes) - } + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000007", resolved[0].UID) + require.Len(t, notes, 1) } func TestResolve_MinObservedCountIsPostCollapse(t *testing.T) { @@ -259,44 +181,30 @@ func TestResolve_MinObservedCountIsPostCollapse(t *testing.T) { "Example Paused Rule", } resolved, notes, err := Resolve(defs, names, "") - if err != nil { - t.Fatalf("Resolve: unexpected error: %v", err) - } + require.NoError(t, err) // The default MinObserved must come from len(resolved) (2 distinct // rules) — never len(names) (3 input lines), which would be unsatisfiable. - if len(resolved) != 2 { - t.Fatalf("resolved = %+v, want 2 distinct rules after collapse", resolved) - } - if len(notes) != 1 { - t.Fatalf("notes = %v, want exactly one collapse note", notes) - } + require.Len(t, resolved, 2) + require.Len(t, notes, 1) } func TestResolve_EmptyAndBlankLinesDiscarded(t *testing.T) { defs := rulerDefs(t) resolved, _, err := Resolve(defs, []string{"", " ", "example_workflow_paused_rule", " \t "}, "") - if err != nil { - t.Fatalf("Resolve: unexpected error: %v", err) - } - if len(resolved) != 1 || resolved[0].UID != "rule0000007" { - t.Fatalf("resolved = %+v, want [rule0000007]", resolved) - } + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000007", resolved[0].UID) } func TestResolve_FolderScopesBareTitle(t *testing.T) { defs := rulerDefs(t) // Bare title, scoped to the wrong folder — must not match. _, _, err := Resolve(defs, []string{"example_workflow_paused_rule"}, "Example-Zone-A") - if err == nil { - t.Fatal("Resolve: want no-match when folder scope excludes the only candidate") - } + require.Error(t, err, "want no-match when folder scope excludes the only candidate") // Scoped to the right folder — must match. resolved, _, err := Resolve(defs, []string{"example_workflow_paused_rule"}, "ExampleObservability") - if err != nil { - t.Fatalf("Resolve: unexpected error: %v", err) - } - if len(resolved) != 1 || resolved[0].UID != "rule0000007" { - t.Fatalf("resolved = %+v, want [rule0000007]", resolved) - } + require.NoError(t, err) + require.Len(t, resolved, 1) + require.Equal(t, "rule0000007", resolved[0].UID) } diff --git a/grafana-alertcheck/internal/gate/schedule.go b/grafana-alertcheck/internal/gate/schedule.go index f2b9bd759..c2fe6e1c5 100644 --- a/grafana-alertcheck/internal/gate/schedule.go +++ b/grafana-alertcheck/internal/gate/schedule.go @@ -339,6 +339,16 @@ func CheckBudget(t map[string]ruleTimings, measured map[string]time.Duration, co return fmt.Errorf("%s", b.String()) } +// graceSourceOrNone is the single "none" default for the grace-source field: +// an empty source means no rule contributed a transitionGrace. Applied here so +// StartupSummary and the human table print the same thing. +func graceSourceOrNone(source string) string { + if source == "" { + return "none" + } + return source +} + // StartupSummary formats the pre-run print an operator sees before the wait: // the total planned run time and the rule (with its `for` value) that set // transitionGrace, plus a warning when the grace eats more than @@ -347,17 +357,14 @@ func CheckBudget(t map[string]ruleTimings, measured map[string]time.Duration, co func StartupSummary(from, to time.Time, global globalTimings) (summary, warning string) { window := to.Sub(from) total := window + global.transitionGrace + global.drainTimeout - source := global.graceSource - if source == "" { - source = "none" - } + source := graceSourceOrNone(global.graceSource) summary = fmt.Sprintf( - "planned run time: %s (window %s + transitionGrace %s [source: %s] + drainTimeout %s)", - total, window, global.transitionGrace, source, global.drainTimeout) + "planned run time: %s\n window %s + transitionGrace %s + drainTimeout %s\n transitionGrace source: %s", + total, window, global.transitionGrace, global.drainTimeout, source) if window > 0 && float64(global.transitionGrace) > float64(window)*graceWarnFraction { warning = fmt.Sprintf( - "transitionGrace %s is more than %.0f%% of the window %s (source: %s) — the window may be too short for this alert's `for`", + "transitionGrace %s is more than %.0f%% of the window %s — the window may be too short for this alert's `for`\n source: %s", global.transitionGrace, graceWarnFraction*100, window, source) } return summary, warning diff --git a/grafana-alertcheck/internal/gate/schedule_test.go b/grafana-alertcheck/internal/gate/schedule_test.go index 70cc8817c..ace598650 100644 --- a/grafana-alertcheck/internal/gate/schedule_test.go +++ b/grafana-alertcheck/internal/gate/schedule_test.go @@ -1,9 +1,10 @@ package gate import ( - "strings" "testing" "time" + + "github.com/stretchr/testify/require" ) func TestDeriveTimings_Default(t *testing.T) { @@ -11,22 +12,12 @@ func TestDeriveTimings_Default(t *testing.T) { {UID: "r1", Title: "R1", IntervalSeconds: 60}, } rules, _, notes := DeriveTimings(defs, 0) - if len(notes) != 0 { - t.Fatalf("notes = %v, want none", notes) - } + require.Empty(t, notes) rt := rules["r1"] - if rt.pollEvery != 30*time.Second { - t.Errorf("pollEvery = %s, want 30s", rt.pollEvery) - } - if rt.maxGap != 60*time.Second { - t.Errorf("maxGap = %s, want 60s", rt.maxGap) - } - if rt.healthGrace != 60*time.Second { - t.Errorf("healthGrace = %s, want 60s", rt.healthGrace) - } - if rt.evalStaleAfter != 120*time.Second { - t.Errorf("evalStaleAfter = %s, want 120s", rt.evalStaleAfter) - } + require.Equal(t, 30*time.Second, rt.pollEvery) + require.Equal(t, 60*time.Second, rt.maxGap) + require.Equal(t, 60*time.Second, rt.healthGrace) + require.Equal(t, 120*time.Second, rt.evalStaleAfter) } func TestDeriveTimings_OverrideVerbatimNoClamp(t *testing.T) { @@ -35,23 +26,16 @@ func TestDeriveTimings_OverrideVerbatimNoClamp(t *testing.T) { } rules, _, notes := DeriveTimings(defs, 20*time.Second) rt := rules["r1"] - if rt.pollEvery != 20*time.Second { - t.Fatalf("pollEvery = %s, want the override verbatim (20s), never clamped down to the 5s default", rt.pollEvery) - } - if rt.maxGap != 40*time.Second { - t.Errorf("maxGap = %s, want 2x the override (40s)", rt.maxGap) - } - if len(notes) != 1 || !strings.Contains(notes[0], "R1") { - t.Fatalf("notes = %v, want one note naming R1's exceeded default", notes) - } + require.Equal(t, 20*time.Second, rt.pollEvery, "the override verbatim (20s), never clamped down to the 5s default") + require.Equal(t, 40*time.Second, rt.maxGap) + require.Len(t, notes, 1) + require.Contains(t, notes[0], "R1") } func TestDeriveTimings_OverrideBelowDefaultNoNote(t *testing.T) { defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 60}} // default pollEvery = 30s _, _, notes := DeriveTimings(defs, 5*time.Second) - if len(notes) != 0 { - t.Fatalf("notes = %v, want none when the override tightens rather than exceeds the default", notes) - } + require.Empty(t, notes) } func TestDeriveTimings_TransitionGraceExcludesSkippedRule(t *testing.T) { @@ -61,12 +45,8 @@ func TestDeriveTimings_TransitionGraceExcludesSkippedRule(t *testing.T) { } _, global, _ := DeriveTimings(defs, 0) want := time.Minute + 60*time.Second // r1's for+interval; r2 (skipped) must not win despite its huge `for` - if global.transitionGrace != want { - t.Fatalf("transitionGrace = %s, want %s (paused rule r2 must be excluded from the max)", global.transitionGrace, want) - } - if !strings.Contains(global.graceSource, "Tight") { - t.Errorf("graceSource = %q, want it to name the contributing rule Tight", global.graceSource) - } + require.Equal(t, want, global.transitionGrace) + require.Contains(t, global.graceSource, "Tight") } // `for: 1d` and `for: 1w` parse correctly (parse_ruler_test.go), @@ -79,25 +59,17 @@ func TestDeriveTimings_TransitionGraceExcludesSkippedRule(t *testing.T) { func TestDeriveTimings_RealForOneWeekRuleSetsTransitionGrace(t *testing.T) { defs := rulerDefs(t) _, global, notes := DeriveTimings(defs, 0) - if len(notes) != 0 { - t.Fatalf("notes = %v, want none: no --poll-interval override is given, so no override note should fire", notes) - } + require.Empty(t, notes, "no --poll-interval override is given, so no override note should fire") want := 7*24*time.Hour + 60*time.Second // rule0000010: for=1w, intervalSeconds=60 - if global.transitionGrace != want { - t.Fatalf("transitionGrace = %s, want %s (rule0000010's for:1w plus its interval)", global.transitionGrace, want) - } - if !strings.Contains(global.graceSource, "Example Failure Ratio Above 10 Percent Weekly") { - t.Errorf("graceSource = %q, want it to name rule0000010", global.graceSource) - } + require.Equal(t, want, global.transitionGrace) + require.Contains(t, global.graceSource, "Example Failure Ratio Above 10 Percent Weekly") } func TestDeriveTimings_TransitionGraceZeroWhenAllSkipped(t *testing.T) { defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 60, For: time.Hour, IsPaused: true}} _, global, _ := DeriveTimings(defs, 0) - if global.transitionGrace != 0 { - t.Fatalf("transitionGrace = %s, want 0 when every rule is skipped", global.transitionGrace) - } + require.Zero(t, global.transitionGrace) } // TestDeriveTimingsFromLog_TransitionGraceFollowsTheHeaderNotTheDefinition @@ -123,28 +95,18 @@ func TestDeriveTimingsFromLog_TransitionGraceFollowsTheHeaderNotTheDefinition(t t.Run("header says active: the rule stays in the max", func(t *testing.T) { h := Header{Rules: []LoggedRule{loggedRule("r1", false)}} _, global, err := DeriveTimingsFromLog(h, defs) - if err != nil { - t.Fatalf("DeriveTimingsFromLog: %v", err) - } - if global.transitionGrace != want { - t.Fatalf("transitionGrace = %s, want %s: the rule was active when the recording opened, "+ - "so a pause applied afterwards must not shrink the window", global.transitionGrace, want) - } - if !strings.Contains(global.graceSource, "R1") { - t.Errorf("graceSource = %q, want it to name R1", global.graceSource) - } + require.NoError(t, err) + require.Equal(t, want, global.transitionGrace, + "the rule was active when the recording opened, so a pause applied afterwards must not shrink the window") + require.Contains(t, global.graceSource, "R1") }) t.Run("header says paused: the rule stays out", func(t *testing.T) { h := Header{Rules: []LoggedRule{loggedRule("r1", true)}} _, global, err := DeriveTimingsFromLog(h, defs) - if err != nil { - t.Fatalf("DeriveTimingsFromLog: %v", err) - } - if global.transitionGrace != 0 { - t.Fatalf("transitionGrace = %s, want 0: a rule paused before the window opened can never fire during it", - global.transitionGrace) - } + require.NoError(t, err) + require.Zero(t, global.transitionGrace, + "a rule paused before the window opened can never fire during it") }) t.Run("drainTimeout counts every rule either way", func(t *testing.T) { @@ -153,13 +115,8 @@ func TestDeriveTimingsFromLog_TransitionGraceFollowsTheHeaderNotTheDefinition(t for _, pausedAtStart := range []bool{false, true} { h := Header{Rules: []LoggedRule{loggedRule("r1", pausedAtStart)}} _, global, err := DeriveTimingsFromLog(h, defs) - if err != nil { - t.Fatalf("DeriveTimingsFromLog: %v", err) - } - if global.drainTimeout != minDrainTimeout { - t.Fatalf("drainTimeout = %s with pausedAtStart=%v, want the %s floor", - global.drainTimeout, pausedAtStart, minDrainTimeout) - } + require.NoError(t, err) + require.Equalf(t, minDrainTimeout, global.drainTimeout, "pausedAtStart=%v", pausedAtStart) } }) } @@ -167,9 +124,7 @@ func TestDeriveTimingsFromLog_TransitionGraceFollowsTheHeaderNotTheDefinition(t func TestDeriveTimings_DrainTimeoutIncludesPaused(t *testing.T) { defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 10}} _, global, _ := DeriveTimings(defs, 0) - if global.drainTimeout != minDrainTimeout { - t.Fatalf("drainTimeout = %s, want the %s floor", global.drainTimeout, minDrainTimeout) - } + require.Equal(t, minDrainTimeout, global.drainTimeout) } func TestDeriveTimings_DrainTimeoutFloor(t *testing.T) { @@ -179,17 +134,13 @@ func TestDeriveTimings_DrainTimeoutFloor(t *testing.T) { } _, global, _ := DeriveTimings(defs, 0) // double the longest interval (2 * 180s) should be the drain timeout - if global.drainTimeout != 2*180*time.Second { - t.Fatalf("drainTimeout = %s, want %s", global.drainTimeout, 2*180*time.Second) - } + require.Equal(t, 2*180*time.Second, global.drainTimeout) } func TestDeriveTimings_DrainTimeoutAboveFloor(t *testing.T) { defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 300}} // 2x300s = 600s > 2m floor _, global, _ := DeriveTimings(defs, 0) - if global.drainTimeout != 600*time.Second { - t.Fatalf("drainTimeout = %s, want 600s", global.drainTimeout) - } + require.Equal(t, 600*time.Second, global.drainTimeout) } // The ordering invariant the burst bound depends on: when several rules become @@ -210,9 +161,8 @@ func TestScheduler_DueOrderingTiesBreakByTightestCadence(t *testing.T) { }, } due := s.Due(now) - if len(due) != 4 || due[0] != "tight" { - t.Fatalf("Due = %v, want the tightest-cadence rule (tight) first when all are simultaneously due", due) - } + require.Len(t, due, 4) + require.Equal(t, "tight", due[0], "the tightest-cadence rule must be first when all are simultaneously due") } func TestScheduler_DueExcludesNotYetDue(t *testing.T) { @@ -222,9 +172,8 @@ func TestScheduler_DueExcludesNotYetDue(t *testing.T) { every: map[string]time.Duration{"soon": 10 * time.Second, "later": 10 * time.Second}, } due := s.Due(now) - if len(due) != 1 || due[0] != "soon" { - t.Fatalf("Due = %v, want only [soon]", due) - } + require.Len(t, due, 1) + require.Equal(t, "soon", due[0]) } func TestScheduler_MarkAdvancesNextDue(t *testing.T) { @@ -233,15 +182,9 @@ func TestScheduler_MarkAdvancesNextDue(t *testing.T) { next: map[string]time.Time{"r1": now}, every: map[string]time.Duration{"r1": 30 * time.Second}, } - if err := s.Mark("r1", now); err != nil { - t.Fatalf("Mark: unexpected error: %v", err) - } - if got := s.Due(now); len(got) != 0 { - t.Fatalf("Due right after Mark = %v, want none (next due is 30s out)", got) - } - if got := s.Due(now.Add(30 * time.Second)); len(got) != 1 { - t.Fatalf("Due at next-due time = %v, want [r1]", got) - } + require.NoError(t, s.Mark("r1", now), "Mark must succeed for a known rule") + require.Empty(t, s.Due(now), "next due is 30s out") + require.Len(t, s.Due(now.Add(30*time.Second)), 1) } func TestScheduler_MarkUnknownUIDFails(t *testing.T) { @@ -250,13 +193,11 @@ func TestScheduler_MarkUnknownUIDFails(t *testing.T) { next: map[string]time.Time{"r1": now}, every: map[string]time.Duration{"r1": 30 * time.Second}, } - if err := s.Mark("not-a-rule", now); err == nil { - t.Fatalf("Mark of an unknown uid: want error, got nil (a missing cadence must not read as zero and loop)") - } + err := s.Mark("not-a-rule", now) + require.Error(t, err, "a missing cadence must not read as zero and loop") // The failed Mark must not have inserted a bogus next-due entry. - if _, ok := s.next["not-a-rule"]; ok { - t.Errorf("Mark of an unknown uid inserted a next-due entry") - } + _, ok := s.next["not-a-rule"] + require.False(t, ok, "a failed Mark must not insert a next-due entry") } // TestScheduler_PerRuleCadenceOverTime simulates a run and counts how often @@ -279,24 +220,19 @@ func TestScheduler_PerRuleCadenceOverTime(t *testing.T) { now := start.Add(elapsed) for _, uid := range s.Due(now) { counts[uid]++ - if err := s.Mark(uid, now); err != nil { - t.Fatalf("Mark(%q): unexpected error: %v", uid, err) - } + require.NoErrorf(t, s.Mark(uid, now), "Mark(%q)", uid) } } // 900s of runtime: "tight" (10s cadence) polls ~90 times, "slack" (300s // cadence) ~3 times. Assert the ratio holds rather than an exact count, // since the staggered initial offset shifts each by up to one cadence. - if counts["tight"] < 85 || counts["tight"] > 91 { - t.Errorf("tight polled %d times over 900s, want ~90 (its own 10s cadence)", counts["tight"]) - } - if counts["slack"] < 2 || counts["slack"] > 4 { - t.Errorf("slack polled %d times over 900s, want ~3 (its own 300s cadence, not tight's)", counts["slack"]) - } - if counts["slack"] >= counts["tight"] { - t.Fatalf("slack polled as often as tight (%d vs %d) — schedules must be per rule, not a shared global cycle", counts["slack"], counts["tight"]) - } + require.GreaterOrEqual(t, counts["tight"], 85) + require.LessOrEqual(t, counts["tight"], 91) + require.GreaterOrEqual(t, counts["slack"], 2) + require.LessOrEqual(t, counts["slack"], 4) + require.Less(t, counts["slack"], counts["tight"], + "schedules must be per rule, not a shared global cycle") } func TestNewScheduler_StaggersWithinPollEvery(t *testing.T) { @@ -304,16 +240,14 @@ func TestNewScheduler_StaggersWithinPollEvery(t *testing.T) { rules := map[string]time.Duration{"r1": 100 * time.Second} s := NewScheduler(rules, now) offset := s.next["r1"].Sub(now) - if offset < 0 || offset >= 100*time.Second { - t.Fatalf("initial offset = %s, want within [0, 100s)", offset) - } + require.GreaterOrEqual(t, offset, time.Duration(0)) + require.Less(t, offset, 100*time.Second) } func TestScheduler_EarliestDueEmpty(t *testing.T) { s := &Scheduler{next: map[string]time.Time{}, every: map[string]time.Duration{}} - if _, ok := s.earliestDue(); ok { - t.Fatalf("earliestDue on an empty scheduler = ok=true, want false") - } + _, ok := s.earliestDue() + require.False(t, ok) } // A zero next-due time is real, not an empty scheduler. @@ -323,12 +257,8 @@ func TestScheduler_EarliestDueZeroTime(t *testing.T) { every: map[string]time.Duration{"r1": time.Second}, } earliest, ok := s.earliestDue() - if !ok { - t.Fatalf("earliestDue = ok=false, want true (the zero time is a real next-due, not an empty scheduler)") - } - if !earliest.IsZero() { - t.Errorf("earliestDue = %v, want the zero time", earliest) - } + require.True(t, ok, "the zero time is a real next-due, not an empty scheduler") + require.Truef(t, earliest.IsZero(), "earliestDue = %v, want the zero time", earliest) } func TestScheduler_EarliestDuePicksMinimum(t *testing.T) { @@ -341,12 +271,8 @@ func TestScheduler_EarliestDuePicksMinimum(t *testing.T) { every: map[string]time.Duration{"later": time.Minute, "soon": time.Minute}, } earliest, ok := s.earliestDue() - if !ok { - t.Fatalf("earliestDue = ok=false, want true") - } - if !earliest.Equal(now.Add(time.Minute)) { - t.Errorf("earliestDue = %v, want the earliest next-due time", earliest) - } + require.True(t, ok) + require.Truef(t, earliest.Equal(now.Add(time.Minute)), "earliestDue = %v, want the earliest next-due time", earliest) } // One rule at 10s beside twenty at 300s, all measured ~1.8s, must not error at @@ -359,9 +285,7 @@ func TestCheckBudget_MixedIntervalRegression(t *testing.T) { timings[uid] = ruleTimings{pollEvery: 150 * time.Second} measured[uid] = 1800 * time.Millisecond } - if err := CheckBudget(timings, measured, 1); err != nil { - t.Fatalf("CheckBudget = %v, want nil (utilization 0.6, burst bound 1.8s <= 5s)", err) - } + require.NoError(t, CheckBudget(timings, measured, 1)) } func TestCheckBudget_UtilizationExceeded(t *testing.T) { @@ -371,9 +295,7 @@ func TestCheckBudget_UtilizationExceeded(t *testing.T) { } measured := map[string]time.Duration{"a": 9 * time.Second, "b": 9 * time.Second} err := CheckBudget(timings, measured, 1) - if err == nil { - t.Fatal("CheckBudget = nil, want an error: utilization 1.8 > concurrency 1") - } + require.Error(t, err) assertBudgetMessage(t, err.Error()) } @@ -381,9 +303,7 @@ func TestCheckBudget_SingleRuleExceedsOwnCadence(t *testing.T) { timings := map[string]ruleTimings{"slow": {pollEvery: 5 * time.Second}} measured := map[string]time.Duration{"slow": 6 * time.Second} err := CheckBudget(timings, measured, 10) - if err == nil { - t.Fatal("CheckBudget = nil, want an error: measured 6s exceeds its own 5s poll-interval") - } + require.Error(t, err, "measured 6s exceeds its own 5s poll-interval") assertBudgetMessage(t, err.Error()) } @@ -397,12 +317,8 @@ func TestCheckBudget_BurstBoundViolation(t *testing.T) { } measured := map[string]time.Duration{"tight": 100 * time.Millisecond, "slow": 3 * time.Second} err := CheckBudget(timings, measured, 10) - if err == nil { - t.Fatal("CheckBudget = nil, want a burst-bound error: slow's 3s measured exceeds tight's 2s cadence") - } - if !strings.Contains(err.Error(), "burst bound") { - t.Errorf("error = %q, want it to name the burst bound", err.Error()) - } + require.Error(t, err, "slow's 3s measured exceeds tight's 2s cadence") + require.Contains(t, err.Error(), "burst bound") assertBudgetMessage(t, err.Error()) } @@ -412,17 +328,13 @@ func TestCheckBudget_BurstBoundOKWhenNotExceeded(t *testing.T) { "slow": {pollEvery: 100 * time.Second}, } measured := map[string]time.Duration{"tight": 100 * time.Millisecond, "slow": 1800 * time.Millisecond} - if err := CheckBudget(timings, measured, 10); err != nil { - t.Fatalf("CheckBudget = %v, want nil (1.8s <= 5s tightest cadence)", err) - } + require.NoError(t, CheckBudget(timings, measured, 10)) } func TestCheckBudget_MissingMeasurementIsAnError(t *testing.T) { timings := map[string]ruleTimings{"r1": {pollEvery: 30 * time.Second}} err := CheckBudget(timings, map[string]time.Duration{}, 10) - if err == nil { - t.Fatal("CheckBudget = nil, want an error: r1 was never measured (fail closed, not a silent zero)") - } + require.Error(t, err, "r1 was never measured (fail closed, not a silent zero)") } func TestCheckBudget_MissingMixedMeasurementIsAnError(t *testing.T) { @@ -431,15 +343,12 @@ func TestCheckBudget_MissingMixedMeasurementIsAnError(t *testing.T) { "slow": {pollEvery: 100 * time.Second}, } measured := map[string]time.Duration{"tight": 100 * time.Millisecond} - if err := CheckBudget(timings, measured, 10); err == nil { - t.Fatal("CheckBudget = nil, want an error: slow was never measured (fail closed, not a silent zero)") - } + require.Error(t, CheckBudget(timings, measured, 10), + "slow was never measured (fail closed, not a silent zero)") } func TestCheckBudget_EmptyScheduleIsFine(t *testing.T) { - if err := CheckBudget(nil, nil, 1); err != nil { - t.Fatalf("CheckBudget = %v, want nil for an empty schedule", err) - } + require.NoError(t, CheckBudget(nil, nil, 1)) } func TestCheckBudget_NonPositivePollIntervalIsAnError(t *testing.T) { @@ -447,12 +356,8 @@ func TestCheckBudget_NonPositivePollIntervalIsAnError(t *testing.T) { timings := map[string]ruleTimings{"r1": {pollEvery: pe}} measured := map[string]time.Duration{"r1": time.Second} err := CheckBudget(timings, measured, 1) - if err == nil { - t.Fatalf("CheckBudget(pollEvery=%s) = nil, want error (non-positive poll-interval would divide by zero)", pe) - } - if !strings.Contains(err.Error(), "non-positive") { - t.Errorf("error %q does not name the non-positive poll-interval", err.Error()) - } + require.Errorf(t, err, "pollEvery=%s would divide by zero", pe) + require.Contains(t, err.Error(), "non-positive", "the error must name the non-positive poll-interval") } } @@ -462,9 +367,7 @@ func TestCheckBudget_NonPositivePollIntervalIsAnError(t *testing.T) { func assertBudgetMessage(t *testing.T, msg string) { t.Helper() for _, want := range []string{"measured", "concurrency", "poll-interval", "fewer"} { - if !strings.Contains(msg, want) { - t.Errorf("message %q missing %q", msg, want) - } + require.Contains(t, msg, want) } } @@ -477,15 +380,9 @@ func TestStartupSummary_WarningWhenGraceTooLarge(t *testing.T) { to := from.Add(10 * time.Minute) global := globalTimings{transitionGrace: 5 * time.Minute, graceSource: "R (for=4m30s, interval=30s)", drainTimeout: time.Minute} summary, warning := StartupSummary(from, to, global) - if !strings.Contains(summary, "planned run time") { - t.Errorf("summary = %q, want it to name the planned run time", summary) - } - if warning == "" { - t.Fatal("warning = \"\", want one: transitionGrace (5m) > 1/4 of the 10m window") - } - if !strings.Contains(warning, "R (for=4m30s, interval=30s)") { - t.Errorf("warning = %q, want it to name the grace source", warning) - } + require.Contains(t, summary, "planned run time") + require.NotEmpty(t, warning, "transitionGrace (5m) > 1/4 of the 10m window") + require.Contains(t, warning, "R (for=4m30s, interval=30s)") } // The test above pins the warning formula with a hand-built globalTimings. @@ -495,22 +392,14 @@ func TestStartupSummary_WarningWhenGraceTooLarge(t *testing.T) { func TestStartupSummary_RealForOneWeekRuleTriggersWarning(t *testing.T) { defs := rulerDefs(t) _, global, notes := DeriveTimings(defs, 0) - if len(notes) != 0 { - t.Fatalf("notes = %v, want none: no --poll-interval override is given, so no override note should fire", notes) - } + require.Empty(t, notes) from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) to := from.Add(10 * time.Minute) // transitionGrace (>1w) dwarfs 1/4 of this window summary, warning := StartupSummary(from, to, global) - if !strings.Contains(summary, "planned run time") { - t.Errorf("summary = %q, want it to name the planned run time", summary) - } - if warning == "" { - t.Fatal("warning = \"\", want one: a real for:1w rule's transitionGrace vastly exceeds 1/4 of a 10m window") - } - if !strings.Contains(warning, "Example Failure Ratio Above 10 Percent Weekly") { - t.Errorf("warning = %q, want it to name rule0000010", warning) - } + require.Contains(t, summary, "planned run time") + require.NotEmpty(t, warning) + require.Contains(t, warning, "Example Failure Ratio Above 10 Percent Weekly") } func TestStartupSummary_NoWarningWhenGraceSmall(t *testing.T) { @@ -518,16 +407,12 @@ func TestStartupSummary_NoWarningWhenGraceSmall(t *testing.T) { to := from.Add(time.Hour) global := globalTimings{transitionGrace: time.Minute, graceSource: "R (for=30s, interval=30s)", drainTimeout: time.Minute} _, warning := StartupSummary(from, to, global) - if warning != "" { - t.Fatalf("warning = %q, want none: 1m grace is well under 1/4 of a 1h window", warning) - } + require.Empty(t, warning) } func TestStartupSummary_NoGraceSourceReadsNone(t *testing.T) { from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) to := from.Add(time.Hour) summary, _ := StartupSummary(from, to, globalTimings{}) - if !strings.Contains(summary, "none") { - t.Fatalf("summary = %q, want it to read \"none\" when no rule set the grace", summary) - } + require.Contains(t, summary, "none") } diff --git a/grafana-alertcheck/internal/gate/source.go b/grafana-alertcheck/internal/gate/source.go index bebb1839e..721151e21 100644 --- a/grafana-alertcheck/internal/gate/source.go +++ b/grafana-alertcheck/internal/gate/source.go @@ -44,9 +44,10 @@ type Observation struct { Latency time.Duration // t_send through the full body read — see requestResult.Latency } -// TransportError marks a failure worth retrying: a non-2xx response, a network -// failure, or a body that failed to parse. Not a deleted rule (an authoritative -// 2xx) and not a clock problem (a hard error — see doRequest). +// TransportError marks a failure worth retrying: a 5xx/429 response, a network +// failure, or a body that failed to parse. Not a 4xx (wrong auth, missing +// resource), not a deleted rule (an authoritative 2xx) and not a clock problem +// (a hard error — see doRequest). type TransportError struct { Err error Status int // 0 when the failure never got a status (network/transport failure) @@ -111,17 +112,7 @@ func parseGrafanaVersion(s string) (grafanaVersion, error) { var v grafanaVersion fields := [3]*int{&v.major, &v.minor, &v.patch} for i, field := range fields { - // Trim any trailing non-digit suffix (prerelease/build metadata, e.g. - // "0+security") rather than requiring an exact numeric match. - digits := parts[i] - j := 0 - for j < len(digits) && digits[j] >= '0' && digits[j] <= '9' { - j++ - } - if j == 0 { - return grafanaVersion{}, fmt.Errorf("unparseable version %q", s) - } - n, err := strconv.Atoi(digits[:j]) + n, err := strconv.Atoi(parts[i]) if err != nil { return grafanaVersion{}, fmt.Errorf("unparseable version %q: %w", s, err) } @@ -259,10 +250,11 @@ type requestResult struct { Latency time.Duration } -// doRequest performs one HTTP GET and classifies the outcome: network failure, -// non-2xx, or body-read failure is retryable (*TransportError); a missing or -// unparseable Date header or a skew beyond SkewHardLimit is a hard error — -// retrying can never fix either, so neither enters the backoff loop. +// doRequest performs one HTTP GET and classifies the outcome: a 5xx/429, a +// network failure, or a body-read failure is retryable (*TransportError); a +// 4xx (wrong auth, missing resource — retrying cannot fix it), a missing or +// unparseable Date header, or a skew beyond SkewHardLimit is a hard error, so +// none of those enters the backoff loop. // // The Date/skew check runs on every endpoint (even /api/health): a skew only // noticed once RuleState starts polling has already masked earlier reads, so it @@ -298,7 +290,11 @@ func (s *httpSource) doRequest(ctx context.Context, path string) (requestResult, latency := tBodyRead.Sub(tSend) if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return requestResult{}, &TransportError{Err: fmt.Errorf("unexpected status %d", resp.StatusCode), Status: resp.StatusCode} + err := fmt.Errorf("unexpected status %d", resp.StatusCode) + if resp.StatusCode >= 400 && resp.StatusCode < 500 && resp.StatusCode != http.StatusTooManyRequests { + return requestResult{}, err + } + return requestResult{}, &TransportError{Err: err, Status: resp.StatusCode} } dateHeader := resp.Header.Get("Date") diff --git a/grafana-alertcheck/internal/gate/source_test.go b/grafana-alertcheck/internal/gate/source_test.go index 39181cfca..e5f24f46f 100644 --- a/grafana-alertcheck/internal/gate/source_test.go +++ b/grafana-alertcheck/internal/gate/source_test.go @@ -7,11 +7,12 @@ import ( "net/http" "net/http/httptest" "net/url" - "strings" "sync" "sync/atomic" "testing" "time" + + "github.com/stretchr/testify/require" ) func healthBody(version string) string { @@ -79,20 +80,18 @@ func TestCheckGrafanaVersion(t *testing.T) { {"", true, nil}, } for _, c := range cases { - err := CheckGrafanaVersion(c.version) - if c.wantErr && err == nil { - t.Errorf("CheckGrafanaVersion(%q): want error, got nil", c.version) - continue - } - if !c.wantErr && err != nil { - t.Errorf("CheckGrafanaVersion(%q): unexpected error: %v", c.version, err) - continue - } - for _, want := range c.wantContains { - if !strings.Contains(err.Error(), want) { - t.Errorf("CheckGrafanaVersion(%q): error %q does not mention %q — it must name both what was found and what is supported", c.version, err.Error(), want) + t.Run(c.version, func(t *testing.T) { + err := CheckGrafanaVersion(c.version) + if c.wantErr { + require.Errorf(t, err, "CheckGrafanaVersion(%q)", c.version) + } else { + require.NoErrorf(t, err, "CheckGrafanaVersion(%q)", c.version) } - } + for _, want := range c.wantContains { + require.Contains(t, err.Error(), want, + "must name both what was found and what is supported") + } + }) } } @@ -102,20 +101,14 @@ func TestBackoffDelay(t *testing.T) { maxWithJitter := maxDelay + maxDelay/5 + time.Millisecond for n := 1; n <= 10; n++ { d := backoffDelay(base, maxDelay, n) - if d <= 0 { - t.Fatalf("backoffDelay(_, _, %d) = %v, want > 0", n, d) - } - if d > maxWithJitter { - t.Fatalf("backoffDelay(_, _, %d) = %v, want <= ~%v", n, d, maxWithJitter) - } + require.Positive(t, d) + require.LessOrEqual(t, d, maxWithJitter) } } func TestHTTPSource_Version_HappyPath(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/health" { - t.Errorf("path = %q, want /api/health", r.URL.Path) - } + require.Equal(t, "/api/health", r.URL.Path) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(healthBody("13.1.0"))) })) @@ -124,20 +117,14 @@ func TestHTTPSource_Version_HappyPath(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) v, err := src.Version(context.Background()) - if err != nil { - t.Fatalf("Version(): unexpected error: %v", err) - } - if v != "13.1.0" { - t.Fatalf("Version() = %q, want 13.1.0", v) - } + require.NoError(t, err) + require.Equal(t, "13.1.0", v) } func TestHTTPSource_Version_NeverLogsToken(t *testing.T) { const secret = "super-secret-token" srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if got := r.Header.Get("Authorization"); got != "Bearer "+secret { - t.Errorf("Authorization = %q, want Bearer %s", got, secret) - } + require.Equal(t, "Bearer "+secret, r.Header.Get("Authorization")) w.WriteHeader(http.StatusInternalServerError) })) defer srv.Close() @@ -145,12 +132,8 @@ func TestHTTPSource_Version_NeverLogsToken(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, secret, clock) _, err := src.Version(context.Background()) - if err == nil { - t.Fatalf("Version(): want error, got nil") - } - if strings.Contains(err.Error(), secret) { - t.Fatalf("error %q leaks the token", err.Error()) - } + require.Error(t, err) + require.NotContains(t, err.Error(), secret) } func TestHTTPSource_RuleState_EmptyIsNotAnError(t *testing.T) { @@ -163,24 +146,16 @@ func TestHTTPSource_RuleState_EmptyIsNotAnError(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) obs, err := src.RuleState(context.Background(), "Anything") - if err != nil { - t.Fatalf("RuleState(): unexpected error: %v", err) - } - if len(obs.Rules) != 0 { - t.Fatalf("Rules = %+v, want empty (an authoritative 2xx is not a transport error)", obs.Rules) - } - if obs.GrafanaNow.IsZero() { - t.Fatalf("GrafanaNow is zero, want the response's Date header value") - } + require.NoError(t, err) + require.Empty(t, obs.Rules, "an authoritative 2xx is not a transport error") + require.False(t, obs.GrafanaNow.IsZero(), "want the response's Date header value") } func TestHTTPSource_RuleState_EscapesRuleName(t *testing.T) { var gotQuery string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { gotQuery = r.URL.RawQuery - if r.URL.Path != "/api/prometheus/grafana/api/v1/rules" { - t.Errorf("path = %q, want /api/prometheus/grafana/api/v1/rules", r.URL.Path) - } + require.Equal(t, "/api/prometheus/grafana/api/v1/rules", r.URL.Path) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(emptyStateBody())) })) @@ -189,24 +164,16 @@ func TestHTTPSource_RuleState_EscapesRuleName(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) title := "[JD] No Job Proposals & More" - if _, err := src.RuleState(context.Background(), title); err != nil { - t.Fatalf("RuleState(): unexpected error: %v", err) - } - want := "rule_name=" + url.QueryEscape(title) - if gotQuery != want { - t.Fatalf("query = %q, want %q", gotQuery, want) - } + _, err := src.RuleState(context.Background(), title) + require.NoError(t, err) + require.Equal(t, "rule_name="+url.QueryEscape(title), gotQuery) } func TestHTTPSource_Definitions_HappyPath(t *testing.T) { body := readFixture(t, "ruler_rules.json") srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/api/ruler/grafana/api/v1/rules" { - t.Errorf("path = %q, want /api/ruler/grafana/api/v1/rules", r.URL.Path) - } - if r.URL.RawQuery != "" { - t.Errorf("query = %q, want none — Definitions reads the ruler API unfiltered", r.URL.RawQuery) - } + require.Equal(t, "/api/ruler/grafana/api/v1/rules", r.URL.Path) + require.Empty(t, r.URL.RawQuery, "Definitions reads the ruler API unfiltered") w.Header().Set("Content-Type", "application/json") _, _ = w.Write(body) })) @@ -215,12 +182,8 @@ func TestHTTPSource_Definitions_HappyPath(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) defs, err := src.Definitions(context.Background()) - if err != nil { - t.Fatalf("Definitions(): unexpected error: %v", err) - } - if len(defs) == 0 { - t.Fatalf("Definitions(): got 0 definitions from a fixture known to have some") - } + require.NoError(t, err) + require.NotEmpty(t, defs) } func TestHTTPSource_Skew(t *testing.T) { @@ -249,14 +212,13 @@ func TestHTTPSource_Skew(t *testing.T) { }) src := NewHTTPSource(srv.URL, "", clock) _, err := src.Version(context.Background()) - if c.wantErr && err == nil { - t.Fatalf("Version(): want error, got nil") - } - if !c.wantErr && err != nil { - t.Fatalf("Version(): unexpected error: %v", err) + if c.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) } - if c.wantErr && calls.Load() != 1 { - t.Fatalf("calls = %d, want 1 — a skew hard error must never be retried", calls.Load()) + if c.wantErr { + require.Equal(t, int32(1), calls.Load(), "a skew hard error must never be retried") } }) } @@ -273,12 +235,8 @@ func TestHTTPSource_MissingDateHeader(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) _, err := src.Version(context.Background()) - if err == nil { - t.Fatalf("Version(): want error, got nil: a missing Date header is a hard error") - } - if calls.Load() != 1 { - t.Fatalf("calls = %d, want 1 — a missing Date header must never be retried", calls.Load()) - } + require.Error(t, err, "a missing Date header is a hard error") + require.Equal(t, int32(1), calls.Load(), "a missing Date header must never be retried") } func TestHTTPSource_UnparseableDateHeader(t *testing.T) { @@ -293,12 +251,8 @@ func TestHTTPSource_UnparseableDateHeader(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) _, err := src.Version(context.Background()) - if err == nil { - t.Fatalf("Version(): want error, got nil: an unparseable Date header is a hard error") - } - if calls.Load() != 1 { - t.Fatalf("calls = %d, want 1 — an unparseable Date header must never be retried", calls.Load()) - } + require.Error(t, err, "an unparseable Date header is a hard error") + require.Equal(t, int32(1), calls.Load(), "an unparseable Date header must never be retried") } // TestHTTPSource_ObservationTiming pins the arithmetic behind Observation's @@ -333,18 +287,10 @@ func TestHTTPSource_ObservationTiming(t *testing.T) { }) src := NewHTTPSource(srv.URL, "", clock) obs, err := src.RuleState(context.Background(), "Anything") - if err != nil { - t.Fatalf("RuleState(): unexpected error: %v", err) - } - if obs.Skew != c.drift { - t.Errorf("Skew = %v, want %v", obs.Skew, c.drift) - } - if obs.SkewBound != time.Second { - t.Errorf("SkewBound = %v, want 1s (RTT/2 with a 2s round trip to headers)", obs.SkewBound) - } - if obs.Latency != 4*time.Second { - t.Errorf("Latency = %v, want 4s (send through full body read) — not just the 2s header round trip", obs.Latency) - } + require.NoError(t, err) + require.Equal(t, c.drift, obs.Skew) + require.Equal(t, time.Second, obs.SkewBound, "RTT/2 with a 2s round trip to headers") + require.Equal(t, 4*time.Second, obs.Latency, "send through full body read — not just the 2s header round trip") }) } } @@ -378,15 +324,10 @@ func TestHTTPSourceStalenessNeverFalsePositiveUnderSkew(t *testing.T) { src := NewHTTPSource(srv.URL, "", clock) obs, err := src.RuleState(context.Background(), def.Title) - if err != nil { - t.Fatalf("RuleState(): %v", err) - } - if !obs.GrafanaNow.Equal(serverDate) { - t.Fatalf("GrafanaNow = %s, want the Date header %s, never the runner's clock %s", obs.GrafanaNow, serverDate, runnerNow) - } - if len(obs.Rules) != 1 { - t.Fatalf("Rules = %+v, want exactly one", obs.Rules) - } + require.NoError(t, err) + require.True(t, obs.GrafanaNow.Equal(serverDate), + "want the Date header, never the runner's clock") + require.Len(t, obs.Rules, 1) rt := newRuleTimings(30*time.Second, 60) // evalStaleAfter = 120s from := serverDate.Add(-10 * time.Minute) @@ -396,10 +337,8 @@ func TestHTTPSourceStalenessNeverFalsePositiveUnderSkew(t *testing.T) { sentinel := to res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) - if res.Unobservable { - t.Fatalf("Coverage = %+v, want no violation: 100s behind Grafana's TRUE now is under the 120s limit — "+ - "only a runner-clock leak (skewed +30s here) would push this over", res) - } + require.False(t, res.Unobservable, + "100s behind Grafana's TRUE now is under the 120s limit — only a runner-clock leak (skewed +30s here) would push this over") } func TestHTTPSource_Retry_TransientRecovers(t *testing.T) { @@ -422,18 +361,12 @@ func TestHTTPSource_Retry_TransientRecovers(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) v, err := src.Version(context.Background()) - if err != nil { - t.Fatalf("Version(): unexpected error after a transient failure: %v", err) - } - if v != "13.1.0" { - t.Fatalf("Version() = %q, want 13.1.0", v) - } + require.NoError(t, err) + require.Equal(t, "13.1.0", v) mu.Lock() n := calls mu.Unlock() - if n != 3 { - t.Fatalf("calls = %d, want 3 (2 failures + 1 success)", n) - } + require.Equal(t, 3, n) } func TestHTTPSource_Retry_ExceedsLimit(t *testing.T) { @@ -447,12 +380,9 @@ func TestHTTPSource_Retry_ExceedsLimit(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) _, err := src.Version(context.Background()) - if err == nil { - t.Fatalf("Version(): want error, got nil") - } - if n := calls.Load(); n != 6 { - t.Fatalf("calls = %d, want 6 (maxSequentialFailures=5 tolerates 5, gives up on the 6th)", n) - } + require.Error(t, err) + require.Equal(t, int32(6), calls.Load(), + "maxSequentialFailures=5 tolerates 5, gives up on the 6th") assertRetryExhausted(t, err, 6) } @@ -479,18 +409,12 @@ func TestHTTPSource_RuleState_GarbageBodyRetries(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) obs, err := src.RuleState(context.Background(), "Anything") - if err != nil { - t.Fatalf("RuleState(): unexpected error after a transient garbage body: %v", err) - } - if len(obs.Rules) != 0 { - t.Fatalf("Rules = %+v, want empty", obs.Rules) - } + require.NoError(t, err) + require.Empty(t, obs.Rules) mu.Lock() n := calls mu.Unlock() - if n != 3 { - t.Fatalf("calls = %d, want 3 (2 unparseable bodies + 1 valid one)", n) - } + require.Equal(t, 3, n) } func TestHTTPSource_Definitions_GarbageBodyGivesUp(t *testing.T) { @@ -505,12 +429,9 @@ func TestHTTPSource_Definitions_GarbageBodyGivesUp(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) _, err := src.Definitions(context.Background()) - if err == nil { - t.Fatalf("Definitions(): want error, got nil") - } - if n := calls.Load(); n != 6 { - t.Fatalf("calls = %d, want 6 — a persistently unparseable 2xx body retries like any other transport failure", n) - } + require.Error(t, err) + require.Equal(t, int32(6), calls.Load(), + "a persistently unparseable 2xx body retries like any other transport failure") assertRetryExhausted(t, err, 6) } @@ -534,17 +455,11 @@ func TestHTTPSource_ResponseBodyTooLarge(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource(srv.URL, "", clock) _, err := src.Version(context.Background()) - if err == nil { - t.Fatalf("Version(): want error, got nil (an oversized body must fail loudly)") - } - if !strings.Contains(err.Error(), "exceeded") { - t.Fatalf("error %q does not name the size limit", err.Error()) - } + require.Error(t, err, "an oversized body must fail loudly") + require.Contains(t, err.Error(), "exceeded", "the error must name the size limit") // An oversized body is a stable condition, not a transient one: it must // fail hard on the first attempt, never burning retries re-reading it. - if n := calls.Load(); n != 1 { - t.Fatalf("calls = %d, want 1 — an oversized body must never be retried", n) - } + require.Equal(t, int32(1), calls.Load(), "an oversized body must never be retried") } func TestHTTPSource_NetworkFailureRetries(t *testing.T) { @@ -554,9 +469,7 @@ func TestHTTPSource_NetworkFailureRetries(t *testing.T) { clock := newFakeClock(time.Now()) src := NewHTTPSource("http://127.0.0.1:1", "", clock) _, err := src.Version(context.Background()) - if err == nil { - t.Fatalf("Version(): want error, got nil") - } + require.Error(t, err) assertRetryExhausted(t, err, 6) } @@ -568,39 +481,26 @@ func TestHTTPSource_NetworkFailureRetries(t *testing.T) { func assertRetryExhausted(t *testing.T, err error, wantFailures int) { t.Helper() var reErr *RetryExhaustedError - if !errors.As(err, &reErr) { - t.Fatalf("error %v (%T): want a *RetryExhaustedError", err, err) - } - if reErr.Failures != wantFailures { - t.Errorf("RetryExhaustedError.Failures = %d, want %d", reErr.Failures, wantFailures) - } - if !strings.Contains(err.Error(), fmt.Sprintf("gave up after %d", wantFailures)) { - t.Errorf("error %q does not name the failure count", err.Error()) - } - if _, ok := errors.AsType[*TransportError](err); ok { - t.Fatalf("error %v (%T) is classified as *TransportError — an exhausted retry must be a terminal, non-retryable error", err, err) - } + require.ErrorAs(t, err, &reErr) + require.Equal(t, wantFailures, reErr.Failures) + require.Contains(t, err.Error(), fmt.Sprintf("gave up after %d", wantFailures)) + _, ok := errors.AsType[*TransportError](err) + require.False(t, ok, "an exhausted retry must be a terminal, non-retryable error") } func TestFakeClock(t *testing.T) { start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) c := newFakeClock(start) - if !c.Now().Equal(start) { - t.Fatalf("Now() = %v, want %v", c.Now(), start) - } + require.True(t, c.Now().Equal(start)) c.Advance(5 * time.Minute) want := start.Add(5 * time.Minute) - if !c.Now().Equal(want) { - t.Fatalf("Now() after Advance = %v, want %v", c.Now(), want) - } + require.True(t, c.Now().Equal(want)) select { case fired := <-c.After(time.Hour): - if !fired.Equal(want.Add(time.Hour)) { - t.Fatalf("After fired with %v, want %v", fired, want.Add(time.Hour)) - } + require.True(t, fired.Equal(want.Add(time.Hour))) default: - t.Fatalf("After(1h) did not fire immediately") + require.Fail(t, "After(1h) did not fire immediately") } } @@ -610,37 +510,29 @@ func TestFakeSource(t *testing.T) { f.defs = []Definition{{UID: "u1", Title: "Rule One"}} ctx := context.Background() - if v, err := f.Version(ctx); err != nil || v != "13.1.0" { - t.Fatalf("Version() = (%q, %v), want (13.1.0, nil)", v, err) - } - if defs, err := f.Definitions(ctx); err != nil || len(defs) != 1 { - t.Fatalf("Definitions() = (%v, %v), want one definition", defs, err) - } + v, err := f.Version(ctx) + require.NoError(t, err) + require.Equal(t, "13.1.0", v) + defs, err := f.Definitions(ctx) + require.NoError(t, err) + require.Len(t, defs, 1) f.script("Rule One", Observation{Rules: []StateRule{{UID: "u1"}}}, nil) f.script("Rule One", Observation{}, fmt.Errorf("boom")) f.script("Rule One", Observation{Rules: nil}, nil) obs, err := f.RuleState(ctx, "Rule One") - if err != nil || len(obs.Rules) != 1 { - t.Fatalf("RuleState() call 1 = (%v, %v), want one rule, no error", obs, err) - } - if _, err := f.RuleState(ctx, "Rule One"); err == nil { - t.Fatalf("RuleState() call 2: want the scripted error, got nil") - } + require.NoError(t, err) + require.Len(t, obs.Rules, 1) + _, err = f.RuleState(ctx, "Rule One") + require.Error(t, err, "RuleState() call 2: want the scripted error, got nil") obs, err = f.RuleState(ctx, "Rule One") - if err != nil { - t.Fatalf("RuleState() call 3: unexpected error: %v", err) - } - if obs.Rules != nil { - t.Fatalf("RuleState() call 3: Rules = %v, want nil (last script entry, then repeats)", obs.Rules) - } + require.NoError(t, err) + require.Nil(t, obs.Rules, "last script entry, then repeats") obs, err = f.RuleState(ctx, "Rule One") - if err != nil || obs.Rules != nil { - t.Fatalf("RuleState() call 4: want the last scripted entry to repeat, got (%v, %v)", obs, err) - } + require.NoError(t, err) + require.Nil(t, obs.Rules) - if _, err := f.RuleState(ctx, "Unscripted Rule"); err == nil { - t.Fatalf("RuleState() for an unscripted title: want an error, got nil") - } + _, err = f.RuleState(ctx, "Unscripted Rule") + require.Error(t, err) } diff --git a/grafana-alertcheck/internal/gate/watch_daemon_test.go b/grafana-alertcheck/internal/gate/watch_daemon_test.go index c3a91cd12..73a6f3d7a 100644 --- a/grafana-alertcheck/internal/gate/watch_daemon_test.go +++ b/grafana-alertcheck/internal/gate/watch_daemon_test.go @@ -15,6 +15,8 @@ import ( "syscall" "testing" "time" + + "github.com/stretchr/testify/require" ) // TestMain doubles this test binary as the detached recorder. Watch spawns @@ -163,37 +165,25 @@ func grafanaTestServer(t *testing.T) *httptest.Server { func patchedStateBody(t *testing.T) []byte { t.Helper() var body map[string]any - if err := json.Unmarshal(readFixture(t, "state_one_instance.json"), &body); err != nil { - t.Fatalf("unmarshal state fixture: %v", err) - } + require.NoError(t, json.Unmarshal(readFixture(t, "state_one_instance.json"), &body)) data, ok := body["data"].(map[string]any) - if !ok { - t.Fatal("state fixture: no data object") - } + require.True(t, ok, "state fixture: no data object") groups, ok := data["groups"].([]any) - if !ok || len(groups) == 0 { - t.Fatal("state fixture: no groups") - } + require.True(t, ok, "state fixture: no groups") + require.NotEmpty(t, groups, "state fixture: no groups") group, ok := groups[0].(map[string]any) - if !ok { - t.Fatal("state fixture: group 0 is not an object") - } + require.True(t, ok, "state fixture: group 0 is not an object") rules, ok := group["rules"].([]any) - if !ok || len(rules) == 0 { - t.Fatal("state fixture: group 0 has no rules") - } + require.True(t, ok, "state fixture: group 0 has no rules") + require.NotEmpty(t, rules, "state fixture: group 0 has no rules") rule, ok := rules[0].(map[string]any) - if !ok { - t.Fatal("state fixture: rule 0 is not an object") - } + require.True(t, ok, "state fixture: rule 0 is not an object") rule["uid"] = watchActiveUID rule["name"] = watchActiveTitle rule["lastEvaluation"] = time.Now().UTC().Format(time.RFC3339Nano) b, err := json.Marshal(body) - if err != nil { - t.Fatalf("marshal patched state fixture: %v", err) - } + require.NoError(t, err) return b } @@ -209,7 +199,7 @@ func waitFor(t *testing.T, what string, timeout time.Duration, cond func() bool) } time.Sleep(20 * time.Millisecond) } - t.Fatalf("timed out after %s waiting for %s", timeout, what) + require.Fail(t, fmt.Sprintf("timed out after %s waiting for %s", timeout, what)) } // The one watch integration test: everything from the version gate to the @@ -237,9 +227,7 @@ func TestWatchSpawnsADetachedRecorder(t *testing.T) { Notes: ¬es, } - if err := Watch(context.Background(), cfg); err != nil { - t.Fatalf("Watch: %v\nnotes:\n%s", err, notes.String()) - } + require.NoError(t, Watch(context.Background(), cfg)) t.Cleanup(func() { if t.Failed() { t.Logf("notes:\n%s", notes.String()) @@ -248,20 +236,14 @@ func TestWatchSpawnsADetachedRecorder(t *testing.T) { }) pid, err := ReadPidFile(out + ".pid") - if err != nil { - t.Fatalf("ReadPidFile: %v", err) - } - if err := syscall.Kill(pid, 0); err != nil { - t.Fatalf("recorder pid %d is not running right after Watch returned: %v", pid, err) - } + require.NoError(t, err) + require.NoError(t, syscall.Kill(pid, 0), "recorder pid %d is not running right after Watch returned", pid) // Setsid, not a bare `&`: a session leader's process group id is its own // pid. Without this the child would still share the parent's process group // and die with the step that started it. - if pgid, err := syscall.Getpgid(pid); err != nil { - t.Errorf("Getpgid(%d): %v", pid, err) - } else if pgid != pid { - t.Errorf("recorder pgid = %d, want %d: it did not get its own session", pgid, pid) - } + pgid, err := syscall.Getpgid(pid) + require.NoError(t, err) + require.Equal(t, pid, pgid, "it did not get its own session") // The parent already wrote the first heartbeat before it returned; // these later ones prove the detached child is the one appending now. @@ -271,35 +253,24 @@ func TestWatchSpawnsADetachedRecorder(t *testing.T) { }) // Stop it exactly the way check does. - if err := syscall.Kill(pid, syscall.SIGTERM); err != nil { - t.Fatalf("SIGTERM %d: %v", pid, err) - } + require.NoError(t, syscall.Kill(pid, syscall.SIGTERM)) waitFor(t, "the stopped sentinel", 10*time.Second, func() bool { _, _, sentinel, err := ReadLog(out) return err == nil && sentinel != nil }) header, polls, sentinel, err := ReadLog(out) - if err != nil { - t.Fatalf("ReadLog: %v", err) - } - if header.URL != srv.URL || header.GrafanaVersion != "13.1.0" { - t.Errorf("header identity = %q/%q, want %q/13.1.0", header.URL, header.GrafanaVersion, srv.URL) - } - if len(header.Rules) != 1 || header.Rules[0].PollEverySeconds != 0.2 { - t.Errorf("header rules = %+v, want one rule recorded at 0.2s", header.Rules) - } + require.NoError(t, err) + require.Equal(t, srv.URL, header.URL) + require.Equal(t, "13.1.0", header.GrafanaVersion) + require.Len(t, header.Rules, 1) + require.Equal(t, float64(0.2), header.Rules[0].PollEverySeconds) for i, p := range polls { - if p.RuleUID != watchActiveUID || !p.Found { - t.Fatalf("poll %d = %+v, want a found observation of %s", i, p, watchActiveUID) - } - if p.GrafanaNow.IsZero() { - t.Fatalf("poll %d has no grafana_now; every poll needs the Date header of its own response", i) - } - } - if sentinel.Before(header.StartedAt) { - t.Errorf("sentinel at %s precedes the record start %s", sentinel, header.StartedAt) + require.Equalf(t, watchActiveUID, p.RuleUID, "poll %d", i) + require.Truef(t, p.Found, "poll %d", i) + require.Falsef(t, p.GrafanaNow.IsZero(), "poll %d has no grafana_now; every poll needs the Date header of its own response", i) } + require.False(t, sentinel.Before(header.StartedAt), "sentinel precedes the record start") waitFor(t, "the recorder to exit", 10*time.Second, func() bool { return syscall.Kill(pid, 0) != nil @@ -317,27 +288,17 @@ func TestDaemonChildRejectsAnAlreadyFinishedLog(t *testing.T) { clock := newFakeClock(testNow) w, err := NewWriter(path, clock) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - if err := w.WriteHeader(testHeader()); err != nil { - t.Fatalf("WriteHeader: %v", err) - } - if err := w.Stop(); err != nil { - t.Fatalf("Stop: %v", err) - } + require.NoError(t, err) + require.NoError(t, w.WriteHeader(testHeader())) + require.NoError(t, w.Stop()) err = RunDaemonChild(context.Background(), DaemonChildConfig{ URL: testHeader().URL, Out: path, Clock: clock, }) - if err == nil { - t.Fatal("RunDaemonChild: no error against a log that already carries a stopped sentinel") - } - if !strings.Contains(err.Error(), "sentinel") { - t.Errorf("error = %v, want it to name the stopped sentinel", err) - } + require.Error(t, err, "no error against a log that already carries a stopped sentinel") + require.Contains(t, err.Error(), "sentinel") } // TestWatchFailsWhenTheChildCannotStartRecording is the other half of the @@ -365,13 +326,9 @@ func TestWatchFailsWhenTheChildCannotStartRecording(t *testing.T) { Concurrency: 2, Notes: ¬es, }) - if err == nil { - t.Fatal("Watch: no error, but the child could never have started recording") - } - if !strings.Contains(err.Error(), "records url") { - t.Errorf("error does not quote the child's own reason:\n%v", err) - } - if _, statErr := os.Stat(out + ".pid"); !os.IsNotExist(statErr) { - t.Errorf("a pidfile survived a failed detach (%v); pids are reused, so the next step would signal a stranger", statErr) - } + require.Error(t, err, "the child could never have started recording") + require.Contains(t, err.Error(), "records url") + _, statErr := os.Stat(out + ".pid") + require.True(t, os.IsNotExist(statErr), + "a pidfile survived a failed detach; pids are reused, so the next step would signal a stranger") } diff --git a/grafana-alertcheck/internal/gate/watch_test.go b/grafana-alertcheck/internal/gate/watch_test.go index 33c455af7..41ee3b09b 100644 --- a/grafana-alertcheck/internal/gate/watch_test.go +++ b/grafana-alertcheck/internal/gate/watch_test.go @@ -4,14 +4,14 @@ import ( "context" "errors" "fmt" - "maps" "os" "path/filepath" - "slices" "strings" "sync" "testing" "time" + + "github.com/stretchr/testify/require" ) // The two fixture rules every prepareWatch test below uses: one live, one @@ -73,12 +73,8 @@ func testStateRule(uid, title string, interval time.Duration, grafanaNow time.Ti func newLoopWriter(t *testing.T, path string, clock Clock) *Writer { t.Helper() w, err := NewWriter(path, clock) - if err != nil { - t.Fatalf("NewWriter: %v", err) - } - if err := w.WriteHeader(testHeader()); err != nil { - t.Fatalf("WriteHeader: %v", err) - } + require.NoError(t, err) + require.NoError(t, w.WriteHeader(testHeader())) return w } @@ -123,29 +119,21 @@ func TestWatchLoopPollsEachRuleAtItsOwnCadence(t *testing.T) { Concurrency: 2, Clock: clock, }) - if err != nil { - t.Fatalf("watchLoop: %v", err) - } + require.NoError(t, err) _, polls, sentinel, readErr := ReadLog(path) - if readErr != nil { - t.Fatalf("ReadLog: %v", readErr) - } - if sentinel == nil { - t.Fatal("no stopped sentinel after a clean stop") - } - if sentinel.Before(testNow.Add(300 * time.Second)) { - t.Errorf("sentinel at %s, want >= the stop time %s", sentinel, testNow.Add(300*time.Second)) - } + require.NoError(t, readErr) + require.NotNil(t, sentinel, "no stopped sentinel after a clean stop") + require.False(t, sentinel.Before(testNow.Add(300*time.Second))) // 300s of window at 5s and 150s, minus the initial stagger offset of up to // one cadence: 59-60 and 1-2. The assertion is the ratio, not the exact // count — a single global cycle would give both rules the same number. - if got := countPolls(polls, tightUID); got < 59 || got > 61 { - t.Errorf("tight rule polled %d times, want ~60 (300s at 5s)", got) - } - if got := countPolls(polls, slackUID); got < 1 || got > 3 { - t.Errorf("slack rule polled %d times, want ~2 (300s at 150s)", got) - } + got := countPolls(polls, tightUID) + require.GreaterOrEqual(t, got, 59) + require.LessOrEqual(t, got, 61) + got = countPolls(polls, slackUID) + require.GreaterOrEqual(t, got, 1) + require.LessOrEqual(t, got, 3) } // Fail-closed from the recorder's side: a recorder that dies must look exactly @@ -174,20 +162,12 @@ func TestWatchLoopHardErrorLeavesNoSentinel(t *testing.T) { Concurrency: 1, Clock: clock, }) - if !errors.Is(err, boom) { - t.Fatalf("watchLoop error = %v, want %v", err, boom) - } + require.ErrorIs(t, err, boom) _, polls, sentinel, readErr := ReadLog(path) - if readErr != nil { - t.Fatalf("ReadLog: %v", readErr) - } - if sentinel != nil { - t.Errorf("sentinel at %s after a failed recording; check would read that as a finished window", sentinel) - } - if len(polls) != 1 { - t.Errorf("kept %d polls, want the 1 that succeeded before the failure", len(polls)) - } + require.NoError(t, readErr) + require.Nil(t, sentinel, "check would read that as a finished window") + require.Len(t, polls, 1, "want the 1 that succeeded before the failure") } // SIGTERM arriving while a poll is in flight is a clean stop, so the aborted @@ -211,7 +191,7 @@ func TestWatchLoopSignalDuringPollIsACleanStop(t *testing.T) { return observation(now, testStateRule("r1", title, time.Minute, now)), nil }) - if err := watchLoop(ctx, watchLoopConfig{ + require.NoError(t, watchLoop(ctx, watchLoopConfig{ Src: src, Writer: w, Reducer: NewReducer(), @@ -219,15 +199,11 @@ func TestWatchLoopSignalDuringPollIsACleanStop(t *testing.T) { Cadence: map[string]time.Duration{"r1": 30 * time.Second}, Concurrency: 1, Clock: clock, - }); err != nil { - t.Fatalf("watchLoop: %v", err) - } + })) - if _, _, sentinel, err := ReadLog(path); err != nil { - t.Fatalf("ReadLog: %v", err) - } else if sentinel == nil { - t.Error("no sentinel after a signalled stop; check would call a fully observed window unobservable") - } + _, _, sentinel, err := ReadLog(path) + require.NoError(t, err) + require.NotNil(t, sentinel, "no sentinel after a signalled stop; check would call a fully observed window unobservable") } // TestWatchLoopWithNothingToPollStillFinishesTheLog covers the every-rule-is- @@ -242,7 +218,7 @@ func TestWatchLoopWithNothingToPollStillFinishesTheLog(t *testing.T) { return Observation{}, fmt.Errorf("nothing should be polled, got %q", title) }) - if err := watchLoop(context.Background(), watchLoopConfig{ + require.NoError(t, watchLoop(context.Background(), watchLoopConfig{ Src: src, Writer: w, Reducer: NewReducer(), @@ -251,20 +227,12 @@ func TestWatchLoopWithNothingToPollStillFinishesTheLog(t *testing.T) { Until: testNow.Add(time.Minute), Concurrency: 1, Clock: clock, - }); err != nil { - t.Fatalf("watchLoop: %v", err) - } + })) _, polls, sentinel, err := ReadLog(path) - if err != nil { - t.Fatalf("ReadLog: %v", err) - } - if len(polls) != 0 { - t.Errorf("wrote %d polls with nothing to poll", len(polls)) - } - if sentinel == nil { - t.Error("no sentinel: check cannot tell this recording from one that died") - } + require.NoError(t, err) + require.Empty(t, polls) + require.NotNil(t, sentinel, "no sentinel: check cannot tell this recording from one that died") } // TestWatchLoopPollBatchKeepsTheHeartbeatsItGot: one rule's failure must not @@ -293,20 +261,13 @@ func TestWatchLoopPollBatchKeepsTheHeartbeatsItGot(t *testing.T) { Concurrency: 2, Clock: clock, } - if err := cfg.pollBatch(context.Background(), []string{"ok", "bad"}); !errors.Is(err, boom) { - t.Fatalf("pollBatch error = %v, want %v", err, boom) - } - if err := w.Close(); err != nil { - t.Fatalf("Close: %v", err) - } + require.ErrorIs(t, cfg.pollBatch(context.Background(), []string{"ok", "bad"}), boom) + require.NoError(t, w.Close()) _, polls, _, err := ReadLog(path) - if err != nil { - t.Fatalf("ReadLog: %v", err) - } - if len(polls) != 1 || polls[0].RuleUID != "ok" { - t.Errorf("polls = %+v, want the one heartbeat that was actually observed", polls) - } + require.NoError(t, err) + require.Len(t, polls, 1) + require.Equal(t, "ok", polls[0].RuleUID) } // The vanish-versus-clear distinction at the one seam the parent/child handoff @@ -326,27 +287,20 @@ func TestReducerSeedFromKeepsMarkersAcrossTheHandoff(t *testing.T) { r := NewReducer() r.seedFrom([]Poll{parentPoll}) p := r.Reduce("r1", childObs) - if !slices.Contains(p.Vanished, key) { - t.Errorf("vanished = %v, want it to contain %q", p.Vanished, key) - } - if len(p.Cleared) != 0 { - t.Errorf("cleared = %v, want none: a vanish is not a recovery", p.Cleared) - } + require.Contains(t, p.Vanished, key) + require.Empty(t, p.Cleared, "a vanish is not a recovery") }) t.Run("unseeded loses the transition", func(t *testing.T) { p := NewReducer().Reduce("r1", childObs) - if len(p.Vanished) != 0 { - t.Fatalf("vanished = %v; this subtest exists to show the seed is what produces the marker", p.Vanished) - } + require.Empty(t, p.Vanished, "this subtest exists to show the seed is what produces the marker") }) t.Run("a not-found poll does not clear the seed", func(t *testing.T) { r := NewReducer() r.seedFrom([]Poll{parentPoll, {RuleUID: "r1", Found: false}}) - if p := r.Reduce("r1", childObs); !slices.Contains(p.Vanished, key) { - t.Errorf("vanished = %v, want it to contain %q: an absent rule leaves the abnormal set untouched", p.Vanished, key) - } + p := r.Reduce("r1", childObs) + require.Contains(t, p.Vanished, key, "an absent rule leaves the abnormal set untouched") }) } @@ -392,50 +346,34 @@ func TestPrepareWatchDoesNotWaitForPausedRules(t *testing.T) { src := watchTestSource(t, liveObservation(testNow)) prep, err := prepareWatch(context.Background(), cfg, src) - if err != nil { - t.Fatalf("prepareWatch: %v", err) - } - if err := prep.writer.Close(); err != nil { - t.Fatalf("Close: %v", err) - } + require.NoError(t, err) + require.NoError(t, prep.writer.Close()) header, polls, sentinel, err := ReadLog(cfg.Out) - if err != nil { - t.Fatalf("ReadLog: %v", err) - } - if sentinel != nil { - t.Error("the parent wrote a sentinel; that would tell check the recording ended before the child started") - } + require.NoError(t, err) + require.Nil(t, sentinel, "the parent wrote a sentinel; that would tell check the recording ended before the child started") - if len(header.Rules) != 2 { - t.Fatalf("header names %d rules, want both the live and the paused one", len(header.Rules)) - } + require.Len(t, header.Rules, 2) for _, lr := range header.Rules { - if lr.PollEverySeconds <= 0 { - t.Errorf("header rule %s records poll_every_seconds=%v; check needs a positive cadence to derive maxGap from", lr.UID, lr.PollEverySeconds) - } - if lr.UID == watchPausedUID && !lr.IsPaused { - t.Errorf("header rule %s: is_paused = false, want the resolve-time snapshot to say true", lr.UID) + require.Positive(t, lr.PollEverySeconds, + "check needs a positive cadence to derive maxGap from") + if lr.UID == watchPausedUID { + require.True(t, lr.IsPaused, "want the resolve-time snapshot to say true") } } // One poll, for the live rule only — and it is already in the log before // prepareWatch returned, which is the whole point of the record step. - if len(polls) != 1 || polls[0].RuleUID != watchActiveUID { - t.Fatalf("polls = %+v, want exactly one first observation of %s", polls, watchActiveUID) - } - if !polls[0].Found || !polls[0].GrafanaNow.Equal(testNow) { - t.Errorf("first poll = %+v, want a found observation at %s", polls[0], testNow) - } + require.Len(t, polls, 1) + require.Equal(t, watchActiveUID, polls[0].RuleUID) + require.True(t, polls[0].Found) + require.True(t, polls[0].GrafanaNow.Equal(testNow)) // The poll record holds the state histogram, asserted through a real // prepareWatch()/Reducer call rather than log_test.go's hand-built // Writer/ReadLog round trip. - if want := map[string]int{"normal": 1}; !maps.Equal(polls[0].Histogram, want) { - t.Errorf("Histogram = %v, want %v: watch must record the state histogram on every poll it writes", polls[0].Histogram, want) - } - if !strings.Contains(notes.String(), watchPausedTitle) || !strings.Contains(notes.String(), "paused") { - t.Errorf("notes do not mention the paused rule:\n%s", notes.String()) - } + require.Equal(t, map[string]int{"normal": 1}, polls[0].Histogram) + require.Contains(t, notes.String(), watchPausedTitle) + require.Contains(t, notes.String(), "paused") } // One authority for the cadence, from the writing side: whatever @@ -448,20 +386,13 @@ func TestPrepareWatchHeaderRecordsTheOverriddenCadence(t *testing.T) { src := watchTestSource(t, liveObservation(testNow)) prep, err := prepareWatch(context.Background(), cfg, src) - if err != nil { - t.Fatalf("prepareWatch: %v", err) - } + require.NoError(t, err) defer prep.writer.Close() - if got := prep.header.Rules[0].PollEverySeconds; got != 120 { - t.Errorf("header poll_every_seconds = %v, want 120 (the override, used verbatim and never clamped)", got) - } - if got := prep.timings[watchActiveUID].maxGap; got != 240*time.Second { - t.Errorf("maxGap = %s, want 240s (2 x the recorded cadence)", got) - } - if !strings.Contains(notes.String(), "--poll-interval") { - t.Errorf("notes do not report that the override exceeds half the evaluation interval:\n%s", notes.String()) - } + require.Equal(t, float64(120), prep.header.Rules[0].PollEverySeconds, + "the override, used verbatim and never clamped") + require.Equal(t, 240*time.Second, prep.timings[watchActiveUID].maxGap) + require.Contains(t, notes.String(), "--poll-interval") } // The budget check runs on the latencies the parent just measured, before the @@ -475,9 +406,7 @@ func TestPrepareWatchFailsWhenTheScheduleDoesNotFit(t *testing.T) { src := watchTestSource(t, obs) _, err := prepareWatch(context.Background(), cfg, src) - if err == nil { - t.Fatal("prepareWatch: no error on a schedule that cannot hold its own cadence") - } + require.Error(t, err, "a schedule cannot hold its own cadence") assertBudgetMessage(t, err.Error()) } @@ -493,20 +422,14 @@ func TestPrepareWatchVerifiesNormalInstancesAreVisible(t *testing.T) { src := watchTestSource(t, observation(testNow, rule)) _, err := prepareWatch(context.Background(), cfg, src) - if err == nil { - t.Fatal("prepareWatch: no error when totals claim normal instances the response omitted") - } - if !strings.Contains(err.Error(), "no longer returns normal instances") { - t.Errorf("error does not say the endpoint stopped returning normal instances: %v", err) - } + require.Error(t, err, "totals claim normal instances the response omitted") + require.Contains(t, err.Error(), "no longer returns normal instances") // The failure happens before any poll is appended, so the log holds a // header and nothing else. - if _, polls, _, readErr := ReadLog(cfg.Out); readErr != nil { - t.Fatalf("ReadLog: %v", readErr) - } else if len(polls) != 0 { - t.Errorf("wrote %d polls from an observation it refused to trust", len(polls)) - } + _, polls, _, readErr := ReadLog(cfg.Out) + require.NoError(t, readErr) + require.Empty(t, polls) } func TestPrepareWatchRejectsAnUnsupportedGrafana(t *testing.T) { @@ -515,11 +438,10 @@ func TestPrepareWatchRejectsAnUnsupportedGrafana(t *testing.T) { src := watchTestSource(t, liveObservation(testNow)) src.version = "12.4.0" - if _, err := prepareWatch(context.Background(), cfg, src); err == nil { - t.Fatal("prepareWatch: no error on an unsupported grafana version") - } else if !strings.Contains(err.Error(), "12.4.0") || !strings.Contains(err.Error(), "13.0.0") { - t.Errorf("error names neither what was found nor what is supported: %v", err) - } + _, err := prepareWatch(context.Background(), cfg, src) + require.Error(t, err) + require.Contains(t, err.Error(), "12.4.0") + require.Contains(t, err.Error(), "13.0.0") } // A rule that resolved in the ruler API but is absent from the state endpoint @@ -531,23 +453,14 @@ func TestPrepareWatchNotesAnAbsentRule(t *testing.T) { src := watchTestSource(t, observation(testNow)) // an authoritative, empty 2xx prep, err := prepareWatch(context.Background(), cfg, src) - if err != nil { - t.Fatalf("prepareWatch: %v", err) - } - if err := prep.writer.Close(); err != nil { - t.Fatalf("Close: %v", err) - } + require.NoError(t, err) + require.NoError(t, prep.writer.Close()) _, polls, _, err := ReadLog(cfg.Out) - if err != nil { - t.Fatalf("ReadLog: %v", err) - } - if len(polls) != 1 || polls[0].Found { - t.Fatalf("polls = %+v, want one poll recorded as not found", polls) - } - if !strings.Contains(notes.String(), "absent from the state endpoint") { - t.Errorf("notes do not warn about the absent rule:\n%s", notes.String()) - } + require.NoError(t, err) + require.Len(t, polls, 1) + require.False(t, polls[0].Found, "want one poll recorded as not found") + require.Contains(t, notes.String(), "absent from the state endpoint") } func TestWatchConfigValidation(t *testing.T) { @@ -575,26 +488,16 @@ func TestWatchConfigValidation(t *testing.T) { cfg := base() tc.mutate(&cfg) err := cfg.withDefaults().validate() - if err == nil { - t.Fatalf("validate: no error, want one naming %q", tc.want) - } - if !strings.Contains(err.Error(), tc.want) { - t.Errorf("validate error = %v, want it to name %q", err, tc.want) - } + require.Errorf(t, err, "validate: no error, want one naming %q", tc.want) + require.Containsf(t, err.Error(), tc.want, "validate error") }) } t.Run("defaults derive the pidfile and daemon log from the log path", func(t *testing.T) { cfg := base().withDefaults() - if cfg.PidFile != cfg.Out+".pid" { - t.Errorf("PidFile = %q, want %q — check finds the recorder by this convention", cfg.PidFile, cfg.Out+".pid") - } - if cfg.DaemonLog == "" { - t.Error("DaemonLog is empty: a detached child would have nowhere to explain a failure") - } - if err := cfg.validate(); err != nil { - t.Errorf("validate: %v", err) - } + require.Equal(t, cfg.Out+".pid", cfg.PidFile) + require.NotEmpty(t, cfg.DaemonLog, "a detached child would have nowhere to explain a failure") + require.NoError(t, cfg.validate()) }) } @@ -609,15 +512,10 @@ func TestChildScheduleUsesTheRecordedCadence(t *testing.T) { }} titles, cadence, err := childSchedule(h) - if err != nil { - t.Fatalf("childSchedule: %v", err) - } - if _, ok := titles["paused"]; ok { - t.Error("the child scheduled a rule that was paused when the window opened") - } - if got := cadence["fast"]; got != 5*time.Second { - t.Errorf("pollEvery = %s, want 5s from the header, not %s from the interval", got, defaultPollEvery(300)) - } + require.NoError(t, err) + _, ok := titles["paused"] + require.False(t, ok, "the child scheduled a rule that was paused when the window opened") + require.Equal(t, 5*time.Second, cadence["fast"]) } func TestChildScheduleRejectsAnUnusableHeader(t *testing.T) { @@ -641,11 +539,9 @@ func TestChildScheduleRejectsAnUnusableHeader(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - if _, _, err := childSchedule(tc.h); err == nil { - t.Fatalf("childSchedule: no error, want one naming %q", tc.want) - } else if !strings.Contains(err.Error(), tc.want) { - t.Errorf("error = %v, want it to name %q", err, tc.want) - } + _, _, err := childSchedule(tc.h) + require.Errorf(t, err, "childSchedule: no error, want one naming %q", tc.want) + require.Contains(t, err.Error(), tc.want) }) } } @@ -670,85 +566,55 @@ func TestChildArgsCarryNoSecretsAndNoRuleSet(t *testing.T) { joined := strings.Join(args, " ") for _, want := range []string{DaemonChildFlag, "--out /tmp/log.jsonl", "--concurrency 3", "--until ", ReadyFDFlag + " 3"} { - if !strings.Contains(joined, want) { - t.Errorf("child args %q do not contain %q", joined, want) - } + require.Contains(t, joined, want) } for _, forbidden := range []string{"secret-token", "Example", "--folder", "--poll-interval", "--pidfile"} { - if strings.Contains(joined, forbidden) { - t.Errorf("child args %q contain %q, which must not reach argv", joined, forbidden) - } + require.NotContains(t, joined, forbidden) } } func TestPidFileRoundTrip(t *testing.T) { path := filepath.Join(t.TempDir(), "log.jsonl.pid") - if err := writePidFile(path, 4242); err != nil { - t.Fatalf("writePidFile: %v", err) - } + require.NoError(t, writePidFile(path, 4242)) pid, err := ReadPidFile(path) - if err != nil { - t.Fatalf("ReadPidFile: %v", err) - } - if pid != 4242 { - t.Errorf("pid = %d, want 4242", pid) - } + require.NoError(t, err) + require.Equal(t, 4242, pid) t.Run("garbage is an error, never a pid", func(t *testing.T) { bad := filepath.Join(t.TempDir(), "bad.pid") - if err := os.WriteFile(bad, []byte("not-a-pid\n"), 0o644); err != nil { - t.Fatalf("write: %v", err) - } - if _, err := ReadPidFile(bad); err == nil { - t.Error("ReadPidFile: no error on an unparseable pidfile") - } + require.NoError(t, os.WriteFile(bad, []byte("not-a-pid\n"), 0o644)) + _, err := ReadPidFile(bad) + require.Error(t, err, "no error on an unparseable pidfile") }) } func TestDaemonLogTail(t *testing.T) { t.Run("missing file is unreadable", func(t *testing.T) { out := daemonLogTail(filepath.Join(t.TempDir(), "nope.daemon.log"), 0) - if !strings.Contains(out, "unreadable") { - t.Errorf("daemonLogTail = %q, want it to name the file as unreadable", out) - } + require.Contains(t, out, "unreadable") }) t.Run("small file returns its content", func(t *testing.T) { path := filepath.Join(t.TempDir(), "small.daemon.log") - if err := os.WriteFile(path, []byte("line one\nline two\n"), 0o644); err != nil { - t.Fatalf("write: %v", err) - } - if out := daemonLogTail(path, 0); out != "line one\nline two" { - t.Errorf("daemonLogTail = %q, want the full trimmed content", out) - } + require.NoError(t, os.WriteFile(path, []byte("line one\nline two\n"), 0o644)) + require.Equal(t, "line one\nline two", daemonLogTail(path, 0)) }) t.Run("large file keeps only the tail", func(t *testing.T) { path := filepath.Join(t.TempDir(), "large.daemon.log") prefix := strings.Repeat("P", 1000) suffix := strings.Repeat("S", daemonLogTailBytes) - if err := os.WriteFile(path, []byte(prefix+suffix), 0o644); err != nil { - t.Fatalf("write: %v", err) - } - out := daemonLogTail(path, 0) - if out != suffix { - t.Errorf("daemonLogTail = %q, want exactly the trailing %d bytes (the %d leading bytes dropped)", out, daemonLogTailBytes, len(prefix)) - } + require.NoError(t, os.WriteFile(path, []byte(prefix+suffix), 0o644)) + require.Equal(t, suffix, daemonLogTail(path, 0)) }) t.Run("offset skips a previous run's content", func(t *testing.T) { path := filepath.Join(t.TempDir(), "shared.daemon.log") prior := strings.Repeat("p", 2000) - if err := os.WriteFile(path, []byte(prior), 0o644); err != nil { - t.Fatalf("write: %v", err) - } + require.NoError(t, os.WriteFile(path, []byte(prior), 0o644)) from := int64(len(prior)) thisRun := "this run's output\n" - if err := os.WriteFile(path, []byte(prior+thisRun), 0o644); err != nil { - t.Fatalf("write: %v", err) - } - if out := daemonLogTail(path, from); out != "this run's output" { - t.Errorf("daemonLogTail = %q, want only this run's bytes after offset %d", out, from) - } + require.NoError(t, os.WriteFile(path, []byte(prior+thisRun), 0o644)) + require.Equal(t, "this run's output", daemonLogTail(path, from)) }) }