diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a9c2e68ec..a80b0af79c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,6 +77,7 @@ * [BUGFIX] Querier: Fix panic due to request tracker truncating multi-byte UTF-8 character #7640 * [BUGFIX] Ingester: Fix panic (`HistogramProtoToHistogram called with a float histogram`) when ingesting a float native histogram with a zero count (e.g. a staleness marker or empty histogram). The decoder is now selected by histogram type via `IsFloatHistogram()` instead of by count value. #7645 * [BUGFIX] Querier: Fix parquet queryable fallback returning a nil error instead of the actual query error in `LabelValues` and `LabelNames`. #7638 +* [BUGFIX] Querier: Fix querier leak in the remote read handler by closing the per-query `storage.Querier` once each sub-query completes. Previously every remote read sub-query leaked its querier and the resources it held (inflight-query trackers, store-gateway clients, index readers) for the lifetime of the process. #7670 ## 1.21.0 2026-04-24 diff --git a/pkg/querier/remote_read.go b/pkg/querier/remote_read.go index 5067f1a8f5..cfa8efab15 100644 --- a/pkg/querier/remote_read.go +++ b/pkg/querier/remote_read.go @@ -46,6 +46,7 @@ func RemoteReadHandler(q storage.Queryable, logger log.Logger) http.Handler { errors <- err return } + defer querier.Close() params := &storage.SelectHints{ Start: int64(from), diff --git a/pkg/querier/remote_read_test.go b/pkg/querier/remote_read_test.go index b3e7c0c28b..1707674505 100644 --- a/pkg/querier/remote_read_test.go +++ b/pkg/querier/remote_read_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/http/httptest" + "sync/atomic" "testing" "github.com/go-kit/log" @@ -88,6 +89,33 @@ func TestRemoteReadHandler(t *testing.T) { require.Equal(t, expected, response) } +func TestRemoteReadHandler_Closes_Querier(t *testing.T) { + t.Parallel() + var closed atomic.Int64 + q := storage.QueryableFunc(func(mint, maxt int64) (storage.Querier, error) { + return &closeCountingQuerier{closed: &closed}, nil + }) + handler := RemoteReadHandler(q, log.NewNopLogger()) + + const numQueries = 3 + queries := make([]*client.QueryRequest, numQueries) + for i := range queries { + queries[i] = &client.QueryRequest{StartTimestampMs: 0, EndTimestampMs: 10} + } + requestBody, err := proto.Marshal(&client.ReadRequest{Queries: queries}) + require.NoError(t, err) + requestBody = snappy.Encode(nil, requestBody) + request, err := http.NewRequest("GET", "/query", bytes.NewReader(requestBody)) + require.NoError(t, err) + request.Header.Set("X-Prometheus-Remote-Read-Version", "0.1.0") + + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + + require.Equal(t, 200, recorder.Result().StatusCode) + require.Equal(t, int64(numQueries), closed.Load(), "each per-query querier must be closed") +} + type mockQuerier struct { matrix model.Matrix } @@ -110,3 +138,15 @@ func (m mockQuerier) LabelNames(ctx context.Context, _ *storage.LabelHints, matc func (mockQuerier) Close() error { return nil } + +// closeCountingQuerier records how many times Close is called so a test can +// assert the remote read handler releases every querier it creates. +type closeCountingQuerier struct { + mockQuerier + closed *atomic.Int64 +} + +func (q *closeCountingQuerier) Close() error { + q.closed.Add(1) + return nil +}