Skip to content

Commit dcfbb6f

Browse files
committed
feat(stovepipe): Apply stage configuration
1 parent 5f0d193 commit dcfbb6f

20 files changed

Lines changed: 339 additions & 66 deletions

File tree

‎doc/rfc/stovepipe/steps/buildanalysis.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,6 @@ Status-polling DLQ recovery does not publish a build-completion notice. The anal
1818

1919
## Analysis errors and dead letters
2020

21-
Analysis errors are returned to the normal consumer whether the Build is running or terminal; errors do not use the two-minute polling hold. The reference service uses backend error classification and the extended retry configuration: a maximum of 10 attempts, with retry delays of 5, 10, 20, 40, then 60 seconds, capped at 60 seconds. A successful hold resets failure accounting. Non-retryable errors and exhausted retries send the message to the analysis DLQ, so final analysis is attempted rather than guaranteed to succeed.
21+
Analysis errors are returned to the normal consumer whether the Build is running or terminal; errors do not use the configured polling hold. The reference service uses backend error classification and the extended retry configuration: a maximum of 10 attempts, with retry delays of 5, 10, 20, 40, then 60 seconds, capped at 60 seconds. A successful hold resets failure accounting. Non-retryable errors and exhausted retries send the message to the analysis DLQ, so final analysis is attempted rather than guaranteed to succeed.
2222

2323
The analysis DLQ uses the existing DLQ consumer, with its always-retryable processor and unlimited-retry subscription. The recovery controller records incomplete analysis in Request history without rerunning detection or changing the validation outcome. Only state reads and history persistence retry; unusable messages are logged and acknowledged. See [DLQ recovery](../../../../stovepipe/controller/buildanalysis/README.md#dlq-recovery) for the history identity and error-handling rules.

‎doc/rfc/stovepipe/steps/buildsignal.md‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,10 +115,10 @@ Two-tier cadence, matching SubmitQueue:
115115

116116
| Status | Delay | Rationale |
117117
|---|---|---|
118-
| `accepted` | `PollDelayAcceptedMs` (default 5000ms) | Queued by the runner, not started — poll infrequently. |
119-
| `running` | `PollDelayRunningMs` (default 2000ms) | Actively executing — poll more often. |
118+
| `accepted` | `BuildSignal.AcceptedPollDelayMs` (default 5000ms) | Queued by the runner, not started — poll infrequently. |
119+
| `running` | `BuildSignal.RunningPollDelayMs` (default 2000ms) | Actively executing — poll more often. |
120120

121-
Package-level `var`s (not `const`s) so tests can shorten them; the server always uses the defaults. Future: move these behind a `queueconfig`-style extension so operators can tune cadence per queue without a code change (the same TODO SubmitQueue's buildsignal carries).
121+
Each delivery resolves these settings through the queue configuration provider. The example server uses a single YAML file containing shared stage defaults and per-queue overrides; other hosts can supply resolved settings through the same interface. See the [configuration guide](../../../../service/stovepipe/README.md#queue-and-stage-configuration).
122122

123123
## Error classification
124124

‎service/stovepipe/server/BUILD.bazel‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,16 +101,19 @@ go_test(
101101
"//platform/extension/messagequeue/mock:go_default_library",
102102
"//stovepipe/controller/dlq:go_default_library",
103103
"//stovepipe/core/messagequeue:go_default_library",
104+
"//stovepipe/core/queueconfig:go_default_library",
104105
"//stovepipe/core/requestlog:go_default_library",
105106
"//stovepipe/entity:go_default_library",
106107
"//stovepipe/extension/buildissue/noop:go_default_library",
107108
"//stovepipe/extension/queueconfig:go_default_library",
109+
"//stovepipe/extension/queueconfig/mock:go_default_library",
108110
"//stovepipe/extension/queueconfig/yaml:go_default_library",
109111
"//stovepipe/extension/storage:go_default_library",
110112
"//stovepipe/extension/storage/mock:go_default_library",
111113
"@com_github_stretchr_testify//assert:go_default_library",
112114
"@com_github_stretchr_testify//require:go_default_library",
113115
"@com_github_uber_go_tally//:go_default_library",
116+
"@org_golang_google_protobuf//proto:go_default_library",
114117
"@org_uber_go_mock//gomock:go_default_library",
115118
"@org_uber_go_zap//zaptest:go_default_library",
116119
],

‎service/stovepipe/server/main.go‎

Lines changed: 32 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -455,19 +455,48 @@ func registerPrimaryControllers(
455455
}
456456
count++
457457

458-
buildSignalController := buildsignal.NewController(logger, scope, store, materializer, brf, registry, stovepipemq.TopicKeyBuildSignal, "stovepipe-buildsignal")
458+
buildSignalController := buildsignal.NewController(
459+
logger,
460+
scope,
461+
store,
462+
configs,
463+
materializer,
464+
brf,
465+
registry,
466+
stovepipemq.TopicKeyBuildSignal,
467+
"stovepipe-buildsignal",
468+
)
459469
if err := c.Register(buildSignalController); err != nil {
460470
return count, fmt.Errorf("failed to register buildsignal controller: %w", err)
461471
}
462472
count++
463473

464-
analysisController := buildanalysis.NewController(logger, scope, store, materializer, detectorFactory, registry, stovepipemq.TopicKeyBuildAnalysis, "stovepipe-buildanalysis")
474+
analysisController := buildanalysis.NewController(
475+
logger,
476+
scope,
477+
store,
478+
configs,
479+
materializer,
480+
detectorFactory,
481+
registry,
482+
stovepipemq.TopicKeyBuildAnalysis,
483+
"stovepipe-buildanalysis",
484+
)
465485
if err := c.Register(analysisController); err != nil {
466486
return count, fmt.Errorf("failed to register buildanalysis controller: %w", err)
467487
}
468488
count++
469489

470-
issueTrackController := issuetrack.NewController(logger, scope, store, materializer, registry, stovepipemq.TopicKeyIssueTrack, "stovepipe-issuetrack")
490+
issueTrackController := issuetrack.NewController(
491+
logger,
492+
scope,
493+
store,
494+
configs,
495+
materializer,
496+
registry,
497+
stovepipemq.TopicKeyIssueTrack,
498+
"stovepipe-issuetrack",
499+
)
471500
if err := c.Register(issueTrackController); err != nil {
472501
return count, fmt.Errorf("failed to register issuetrack controller: %w", err)
473502
}

‎service/stovepipe/server/main_test.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,15 +30,18 @@ import (
3030
queuemock "github.com/uber/submitqueue/platform/extension/messagequeue/mock"
3131
"github.com/uber/submitqueue/stovepipe/controller/dlq"
3232
stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue"
33+
stageconfig "github.com/uber/submitqueue/stovepipe/core/queueconfig"
3334
"github.com/uber/submitqueue/stovepipe/core/requestlog"
3435
"github.com/uber/submitqueue/stovepipe/entity"
3536
buildissuenoop "github.com/uber/submitqueue/stovepipe/extension/buildissue/noop"
3637
"github.com/uber/submitqueue/stovepipe/extension/queueconfig"
38+
queueconfigmock "github.com/uber/submitqueue/stovepipe/extension/queueconfig/mock"
3739
queueconfigyaml "github.com/uber/submitqueue/stovepipe/extension/queueconfig/yaml"
3840
"github.com/uber/submitqueue/stovepipe/extension/storage"
3941
storagemock "github.com/uber/submitqueue/stovepipe/extension/storage/mock"
4042
"go.uber.org/mock/gomock"
4143
"go.uber.org/zap/zaptest"
44+
"google.golang.org/protobuf/proto"
4245
)
4346

4447
// recordingConsumer captures what the host registers instead of subscribing.
@@ -363,3 +366,59 @@ func TestBuildAnalysisRegistration(t *testing.T) {
363366
assert.True(t, cfg.DLQ.Enabled)
364367
assert.Equal(t, "_dlq", cfg.DLQ.TopicSuffix)
365368
}
369+
370+
func TestStageControllersUseHostConfiguration(t *testing.T) {
371+
const queue = "configured-queue"
372+
373+
for _, tt := range []struct {
374+
name string
375+
topic consumer.TopicKey
376+
notification proto.Message
377+
partition string
378+
}{
379+
{
380+
"build signal",
381+
stovepipemq.TopicKeyBuildSignal,
382+
&stovepipemq.BuildSignal{Id: "build", QueueName: queue},
383+
"build",
384+
},
385+
{
386+
"build analysis",
387+
stovepipemq.TopicKeyBuildAnalysis,
388+
&stovepipemq.BuildAnalysis{Id: "build", QueueName: queue},
389+
"build",
390+
},
391+
{
392+
"issue tracking",
393+
stovepipemq.TopicKeyIssueTrack,
394+
&stovepipemq.IssueTrack{RequestId: "1", QueueName: queue},
395+
queue,
396+
},
397+
} {
398+
t.Run(tt.name, func(t *testing.T) {
399+
ctrl := gomock.NewController(t)
400+
configs := queueconfigmock.NewMockStore(ctrl)
401+
configs.EXPECT().Get(gomock.Any(), queue).Return(stageconfig.QueueConfig{}, assert.AnError)
402+
403+
_, controllers, _ := registeredControllersWithConfig(t, configs)
404+
405+
payload, err := stovepipemq.Marshal(tt.notification)
406+
require.NoError(t, err)
407+
408+
message := entityqueue.NewMessage("notification", payload, tt.partition, nil)
409+
message.Tenant = queue
410+
411+
delivery := consumermock.NewMockDelivery(ctrl)
412+
delivery.EXPECT().Message().Return(message)
413+
414+
for _, controller := range controllers {
415+
if controller.TopicKey() == tt.topic {
416+
require.ErrorIs(t, controller.Process(context.Background(), delivery), assert.AnError)
417+
return
418+
}
419+
}
420+
421+
require.Fail(t, "stage controller was not registered")
422+
})
423+
}
424+
}

‎stovepipe/controller/build/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,15 @@ go_test(
4141
"//stovepipe/controller/buildanalysis:go_default_library",
4242
"//stovepipe/controller/buildsignal:go_default_library",
4343
"//stovepipe/core/messagequeue:go_default_library",
44+
"//stovepipe/core/queueconfig:go_default_library",
4445
"//stovepipe/core/requestlog:go_default_library",
4546
"//stovepipe/core/requestlog/mock:go_default_library",
4647
"//stovepipe/entity:go_default_library",
4748
"//stovepipe/extension/buildissue:go_default_library",
4849
"//stovepipe/extension/buildissue/mock:go_default_library",
4950
"//stovepipe/extension/buildrunner:go_default_library",
5051
"//stovepipe/extension/buildrunner/mock:go_default_library",
52+
"//stovepipe/extension/queueconfig/mock:go_default_library",
5153
"//stovepipe/extension/storage:go_default_library",
5254
"//stovepipe/extension/storage/mock:go_default_library",
5355
"@com_github_stretchr_testify//assert:go_default_library",

‎stovepipe/controller/build/build_analysis_test.go‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@ import (
2626
"github.com/uber/submitqueue/stovepipe/controller/buildanalysis"
2727
"github.com/uber/submitqueue/stovepipe/controller/buildsignal"
2828
stovepipemq "github.com/uber/submitqueue/stovepipe/core/messagequeue"
29+
stageconfig "github.com/uber/submitqueue/stovepipe/core/queueconfig"
2930
"github.com/uber/submitqueue/stovepipe/core/requestlog"
3031
"github.com/uber/submitqueue/stovepipe/entity"
3132
"github.com/uber/submitqueue/stovepipe/extension/buildissue"
3233
buildissuemock "github.com/uber/submitqueue/stovepipe/extension/buildissue/mock"
3334
"github.com/uber/submitqueue/stovepipe/extension/buildrunner"
35+
queueconfigmock "github.com/uber/submitqueue/stovepipe/extension/queueconfig/mock"
3436
"github.com/uber/submitqueue/stovepipe/extension/storage"
3537
storagemock "github.com/uber/submitqueue/stovepipe/extension/storage/mock"
3638
"go.uber.org/mock/gomock"
@@ -142,11 +144,33 @@ func TestBuildAnalysisPipeline(t *testing.T) {
142144
detector.EXPECT().Detect(gomock.Any(), entity.BuildID{ID: testBuildID}).Return([]entity.BuildIssue{a, b}, nil),
143145
)
144146

145-
analysisController := buildanalysis.NewController(zap.NewNop().Sugar(), tally.NoopScope,
146-
staticStorageFactory{store: m.store}, requestlog.NewMaterializer(tally.NoopScope), detectorFactory, buildController.registry, stovepipemq.TopicKeyBuildAnalysis, "analysis")
147-
statusController := buildsignal.NewController(zap.NewNop().Sugar(), tally.NoopScope,
148-
staticStorageFactory{store: m.store}, m.materializer, m.runnerFactory, buildController.registry,
149-
stovepipemq.TopicKeyBuildSignal, "signal")
147+
configs := queueconfigmock.NewMockStore(ctrl)
148+
cfg := stageconfig.QueueConfig{Name: testQueue}
149+
cfg.Stages.BuildAnalysis.PollDelayMs = 137
150+
configs.EXPECT().Get(gomock.Any(), testQueue).Return(cfg, nil).AnyTimes()
151+
152+
analysisController := buildanalysis.NewController(
153+
zap.NewNop().Sugar(),
154+
tally.NoopScope,
155+
staticStorageFactory{store: m.store},
156+
configs,
157+
requestlog.NewMaterializer(tally.NoopScope),
158+
detectorFactory,
159+
buildController.registry,
160+
stovepipemq.TopicKeyBuildAnalysis,
161+
"analysis",
162+
)
163+
statusController := buildsignal.NewController(
164+
zap.NewNop().Sugar(),
165+
tally.NoopScope,
166+
staticStorageFactory{store: m.store},
167+
configs,
168+
m.materializer,
169+
m.runnerFactory,
170+
buildController.registry,
171+
stovepipemq.TopicKeyBuildSignal,
172+
"signal",
173+
)
150174

151175
delivered := func(msg entityqueue.Message) *consumermock.MockDelivery {
152176
d := consumermock.NewMockDelivery(ctrl)
@@ -159,13 +183,13 @@ func TestBuildAnalysisPipeline(t *testing.T) {
159183
require.Len(t, published["buildsignal"], 1)
160184

161185
firstPoll := delivered(published["buildanalysis"][0])
162-
firstPoll.EXPECT().Hold(buildanalysis.PollDelayMs)
186+
firstPoll.EXPECT().Hold(cfg.Stages.BuildAnalysis.PollDelayMs)
163187
require.NoError(t, analysisController.Process(ctx, firstPoll))
164188
assert.True(t, persisted[a])
165189
assert.False(t, persisted[b])
166190

167191
secondPoll := delivered(published["buildanalysis"][0])
168-
secondPoll.EXPECT().Hold(buildanalysis.PollDelayMs)
192+
secondPoll.EXPECT().Hold(cfg.Stages.BuildAnalysis.PollDelayMs)
169193
require.NoError(t, analysisController.Process(ctx, secondPoll))
170194
assert.True(t, persisted[b])
171195

‎stovepipe/controller/buildanalysis/BUILD.bazel‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ go_library(
1818
"//stovepipe/core/requestlog:go_default_library",
1919
"//stovepipe/entity:go_default_library",
2020
"//stovepipe/extension/buildissue:go_default_library",
21+
"//stovepipe/extension/queueconfig:go_default_library",
2122
"//stovepipe/extension/storage:go_default_library",
2223
"@com_github_uber_go_tally//:go_default_library",
2324
"@org_uber_go_zap//:go_default_library",
@@ -41,11 +42,13 @@ go_test(
4142
"//platform/extension/messagequeue/mock:go_default_library",
4243
"//platform/metrics:go_default_library",
4344
"//stovepipe/core/messagequeue:go_default_library",
45+
"//stovepipe/core/queueconfig:go_default_library",
4446
"//stovepipe/core/requestlog:go_default_library",
4547
"//stovepipe/core/requestlog/mock:go_default_library",
4648
"//stovepipe/entity:go_default_library",
4749
"//stovepipe/extension/buildissue:go_default_library",
4850
"//stovepipe/extension/buildissue/mock:go_default_library",
51+
"//stovepipe/extension/queueconfig/mock:go_default_library",
4952
"//stovepipe/extension/storage:go_default_library",
5053
"//stovepipe/extension/storage/mock:go_default_library",
5154
"@com_github_stretchr_testify//assert:go_default_library",

‎stovepipe/controller/buildanalysis/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,15 @@ All analysis deliveries for a build share its build-ID partition and consumer gr
1818

1919
## Polling and completion
2020

21-
Every delivery reloads the Build and its Request before detection. After a successful pass, a nonterminal Build holds the delivery for two minutes; a terminal Build completes after persistence and completion history once its Request outcome is known, or the Request has ended without one. A pass that begins with a nonterminal Build always holds, even if the Build finishes during detection, so another delivery analyzes it after observing terminal status. More than one delivery may analyze the same terminal Build; build-issue persistence remains idempotent.
21+
Every delivery reloads the Build and its Request before detection. After a successful pass, a nonterminal Build holds the delivery for the configured polling interval (two minutes by default); a terminal Build completes after persistence and completion history once its Request outcome is known, or the Request has ended without one. A pass that begins with a nonterminal Build always holds, even if the Build finishes during detection, so another delivery analyzes it after observing terminal status. More than one delivery may analyze the same terminal Build; build-issue persistence remains idempotent.
2222

2323
If the Request is terminal but the loaded Build is nonterminal, analysis reloads the Build before deciding to stop. The Build may have finished between the initial Build read and the Request read; reloading avoids skipping its final analysis. If it is still nonterminal, analysis retains abandonment and stops without detection or a hold. This can occur when the status-polling DLQ handler fails the Request without learning the actual build result, leaving the Build at its last observed status.
2424

2525
Status polling and updates remain the responsibility of buildsignal. Analysis never changes Request state or releases build slots.
2626

2727
## Errors
2828

29-
Detection, persistence, configuration, state-loading, and message-validation errors are returned to the consumer for its normal retry or DLQ policy, whether the Build is running or terminal. Analysis errors do not use the two-minute polling hold. The consumer classifies errors and applies its configured retry limits and backoff; a final analysis can still fail and dead-letter.
29+
Detection, persistence, configuration, state-loading, and message-validation errors are returned to the consumer for its normal retry or DLQ policy, whether the Build is running or terminal. Analysis errors do not use the configured polling hold. The consumer classifies errors and applies its configured retry limits and backoff; a final analysis can still fail and dead-letter.
3030

3131
## DLQ recovery
3232

‎stovepipe/controller/buildanalysis/buildanalysis.go‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,17 +32,15 @@ import (
3232
"github.com/uber/submitqueue/stovepipe/core/requestlog"
3333
"github.com/uber/submitqueue/stovepipe/entity"
3434
"github.com/uber/submitqueue/stovepipe/extension/buildissue"
35+
"github.com/uber/submitqueue/stovepipe/extension/queueconfig"
3536
"github.com/uber/submitqueue/stovepipe/extension/storage"
3637
"go.uber.org/zap"
3738
)
3839

39-
// PollDelayMs is the delay in milliseconds between successful polling passes.
40-
// Two minutes limits provider traffic during long-running builds.
41-
const PollDelayMs int64 = 120000
42-
4340
// Controller accumulates build issues while builds run and after completion.
4441
// Terminal analysis must persist all build issues and its completion event before acknowledging.
4542
type Controller struct {
43+
queueConfigs queueconfig.Store
4644
logger *zap.SugaredLogger
4745
metricsScope tally.Scope
4846
stores storage.Factory
@@ -57,13 +55,15 @@ func NewController(
5755
logger *zap.SugaredLogger,
5856
scope tally.Scope,
5957
stores storage.Factory,
58+
queueConfigs queueconfig.Store,
6059
materializer requestlog.Materializer,
6160
detectorFactory buildissue.DetectorFactory,
6261
registry consumer.TopicRegistry,
6362
topicKey consumer.TopicKey,
6463
consumerGroup string,
6564
) *Controller {
6665
return &Controller{
66+
queueConfigs: queueConfigs,
6767
logger: logger.Named("buildanalysis_controller"),
6868
metricsScope: scope.SubScope("buildanalysis_controller"),
6969
stores: stores,
@@ -98,6 +98,11 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) (r
9898
return fmt.Errorf("build analysis must be partitioned by build %q", analysis.GetId())
9999
}
100100

101+
cfg, err := c.queueConfigs.Get(ctx, analysis.GetQueueName())
102+
if err != nil {
103+
return fmt.Errorf("load queue configuration: %w", err)
104+
}
105+
101106
store, err := c.stores.For(storage.Config{QueueName: analysis.GetQueueName()})
102107
if err != nil {
103108
return fmt.Errorf("failed to resolve storage for queue %q: %w", analysis.GetQueueName(), err)
@@ -207,7 +212,7 @@ func (c *Controller) Process(ctx context.Context, delivery consumer.Delivery) (r
207212
}
208213

209214
if hold {
210-
delivery.Hold(PollDelayMs)
215+
delivery.Hold(cfg.Stages.BuildAnalysis.PollDelayMs)
211216
}
212217

213218
c.logger.Debugw("build analysis pass complete",

0 commit comments

Comments
 (0)