From 30e1270fe9e6fc7560c0aea3ebd9d928f217b3e4 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Thu, 16 Jul 2026 18:33:16 -0700 Subject: [PATCH 1/3] feat(ci): promote Trigger.dev tasks in lockstep with the ECS traffic cutover --- .github/scripts/wait-for-ecs-cutover.sh | 105 +++++++++ .github/workflows/ci.yml | 287 ++++++++++++++++++++++++ 2 files changed, 392 insertions(+) create mode 100755 .github/scripts/wait-for-ecs-cutover.sh diff --git a/.github/scripts/wait-for-ecs-cutover.sh b/.github/scripts/wait-for-ecs-cutover.sh new file mode 100755 index 00000000000..4ec90125804 --- /dev/null +++ b/.github/scripts/wait-for-ecs-cutover.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Waits for the ECS blue/green deploy triggered by a specific app image push to +# reach its traffic cutover (CodeDeploy AllowTraffic == Succeeded), then exits 0. +# +# ECR app images use a floating tag (latest/staging) with no git SHA, so the +# only durable key linking this CI push to its ECS deploy is the image DIGEST. +# Correlation: image digest -> CodePipeline execution (AppEcrImage revision) -> +# Deploy action externalExecutionId (== CodeDeploy deployment id) -> AllowTraffic. +# +# Usage: wait-for-ecs-cutover.sh +# Requires: awscli v2, configured credentials with codedeploy + codepipeline read. +set -euo pipefail + +PIPELINE="${1:?pipeline name required}" +DIGEST="${2:?image digest required}" + +POLL_INTERVAL="${POLL_INTERVAL:-15}" +# 70 min covers a prod deploy whose Deploy stage is queued behind a prior +# deploy's ~50-min termination bake before its own traffic shift begins. +OVERALL_TIMEOUT="${OVERALL_TIMEOUT:-4200}" + +deadline=$(( $(date +%s) + OVERALL_TIMEOUT )) +remaining() { echo $(( deadline - $(date +%s) )); } +log() { echo "[wait-for-ecs-cutover] $*"; } +fail_if_expired() { + if [ "$(remaining)" -le 0 ]; then + log "ERROR: timed out after ${OVERALL_TIMEOUT}s waiting for: $1" + exit 1 + fi +} + +log "Pipeline: $PIPELINE" +log "Target app image digest: $DIGEST" + +# Phase A: find the pipeline execution whose ECR source revision matches our +# digest. --max-items bounds the fetch (the CLI otherwise auto-paginates the whole +# execution history); our push is the newest execution, so it's on the first page. +# The revisionId match is done server-side via JMESPath; grep isolates the UUID +# from any trailing pagination-token line in text output. +EXECUTION_ID="" +while [ -z "$EXECUTION_ID" ]; do + fail_if_expired "pipeline execution matching digest" + EXECUTION_ID=$(aws codepipeline list-pipeline-executions \ + --pipeline-name "$PIPELINE" --max-items 30 \ + --query "pipelineExecutionSummaries[?sourceRevisions[?actionName=='ECR_Source' && revisionId=='$DIGEST']].pipelineExecutionId" \ + --output text 2>/dev/null | tr '\t ' '\n\n' | grep -Em1 '^[0-9a-f-]{36}$' || true) + if [ -z "$EXECUTION_ID" ]; then + log "No matching pipeline execution yet; retry in ${POLL_INTERVAL}s (remaining $(remaining)s)" + sleep "$POLL_INTERVAL" + fi +done +log "Matched pipeline execution: $EXECUTION_ID" + +# Phase B: resolve the CodeDeploy deployment id from the Deploy action. This may +# stay empty for a while if the Deploy stage is queued behind a prior deploy. +DEPLOYMENT_ID="" +while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; do + fail_if_expired "CodeDeploy deployment id (Deploy stage may be queued behind a prior deploy's bake)" + status=$(aws codepipeline get-pipeline-execution \ + --pipeline-name "$PIPELINE" --pipeline-execution-id "$EXECUTION_ID" \ + --query 'pipelineExecution.status' --output text 2>/dev/null || true) + case "$status" in + Failed|Stopped|Superseded) + log "ERROR: pipeline execution $EXECUTION_ID ended in status $status before deploy" + exit 1 + ;; + esac + DEPLOYMENT_ID=$(aws codepipeline list-action-executions \ + --pipeline-name "$PIPELINE" \ + --filter pipelineExecutionId="$EXECUTION_ID" \ + --query "actionExecutionDetails[?stageName=='Deploy'].output.executionResult.externalExecutionId | [0]" \ + --output text 2>/dev/null || true) + if [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; then + log "Deploy stage not started yet (pipeline status: $status); retry in ${POLL_INTERVAL}s (remaining $(remaining)s)" + sleep "$POLL_INTERVAL" + fi +done +log "CodeDeploy deployment: $DEPLOYMENT_ID" + +# Phase C: wait for the traffic cutover (AllowTraffic lifecycle event Succeeded). +while true; do + fail_if_expired "AllowTraffic (traffic cutover)" + dstatus=$(aws deploy get-deployment --deployment-id "$DEPLOYMENT_ID" \ + --query 'deploymentInfo.status' --output text 2>/dev/null || true) + case "$dstatus" in + Failed|Stopped) + log "ERROR: CodeDeploy deployment $DEPLOYMENT_ID ended in status $dstatus; not promoting" + exit 1 + ;; + esac + target_id=$(aws deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \ + --query 'targetIds[0]' --output text 2>/dev/null || true) + at_status="" + if [ -n "$target_id" ] && [ "$target_id" != "None" ]; then + at_status=$(aws deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$target_id" \ + --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \ + --output text 2>/dev/null || true) + if [ "$at_status" = "Succeeded" ]; then + log "Traffic cutover complete (AllowTraffic Succeeded) for $DEPLOYMENT_ID" + exit 0 + fi + fi + log "Deployment $DEPLOYMENT_ID status=$dstatus AllowTraffic=${at_status:-pending}; wait ${POLL_INTERVAL}s (remaining $(remaining)s)" + sleep "$POLL_INTERVAL" +done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca2901c7871..9571ac30c5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,6 +182,118 @@ jobs: fi bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim + # Staging: build & upload the Trigger.dev task version WITHOUT promoting it + # (--skip-promotion). New runs keep executing the old version until the promote + # job flips it at the ECS traffic cutover, so this deploy carries zero + # app<->task skew and runs in parallel with the image build. The captured + # version output is consumed by promote-trigger-staging. + deploy-trigger-staging: + name: Deploy Trigger.dev (Staging) + needs: [migrate] + if: github.event_name == 'push' && github.ref == 'refs/heads/staging' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 15 + outputs: + version: ${{ steps.deploy.outputs.version }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Deploy to Trigger.dev (skip promotion) + id: deploy + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + bunx trigger.dev@4.4.3 deploy --env staging --skip-promotion 2>&1 | tee deploy.log + # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation. + VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) + if [ -z "$VERSION" ]; then + echo "ERROR: could not parse deployed version from deploy output" >&2 + exit 1 + fi + echo "Captured deployed version: $VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + # Production: same skip-promotion deploy as staging, for the prod environment. + deploy-trigger-production: + name: Deploy Trigger.dev (Production) + needs: [migrate] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 15 + outputs: + version: ${{ steps.deploy.outputs.version }} + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Deploy to Trigger.dev (skip promotion) + id: deploy + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + bunx trigger.dev@4.4.3 deploy --env prod --skip-promotion 2>&1 | tee deploy.log + # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation. + VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) + if [ -z "$VERSION" ]; then + echo "ERROR: could not parse deployed version from deploy output" >&2 + exit 1 + fi + echo "Captured deployed version: $VERSION" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR. # Runs in parallel with tests — only immutable sha tags are pushed here, and # the CodePipeline EventBridge triggers filter on exactly the @@ -270,6 +382,7 @@ jobs: echo "tags=${TAGS}" >> $GITHUB_OUTPUT - name: Build and push images + id: build uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2 with: context: . @@ -280,6 +393,25 @@ jobs: provenance: false sbom: false + # Publish the app image digest so promote-trigger-* can correlate this push + # to its ECS CodePipeline execution. promote-images retags this same sha + # image to latest/staging (preserving the digest), so the pipeline's ECR + # source revision equals this digest — the only durable key (the deploy tag + # is floating). App leg only. + - name: Publish app image digest + if: matrix.ecr_repo_secret == 'ECR_APP' + run: | + mkdir -p digest + echo "${{ steps.build.outputs.digest }}" > digest/app-image-digest.txt + + - name: Upload app image digest + if: matrix.ecr_repo_secret == 'ECR_APP' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: app-image-digest + path: digest/app-image-digest.txt + retention-days: 1 + # Promote the sha-tagged ECR images to the deploy tags once tests and # migrations pass. Pushing the ECR latest/staging tag is what triggers # CodePipeline, so this seconds-long manifest retag is the deploy gate — @@ -297,6 +429,11 @@ jobs: permissions: contents: read id-token: write + outputs: + # Whether the deploy tag was actually moved (false on a stale-run guard + # skip). promote-trigger-* keys off this so tasks are never promoted when + # the app itself wasn't. + promoted: ${{ steps.guard.outputs.fresh }} steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 @@ -356,6 +493,156 @@ jobs: "${REGISTRY}/${repo}:${{ github.sha }}" done + # Staging: promote the skip-promoted Trigger.dev version at the exact moment the + # ECS app deploy shifts traffic (CodeDeploy AllowTraffic), so tasks and app cut + # over in lockstep. The promote-images retag above is what triggers the ECS + # pipeline; this job polls it (via the app image digest) and promotes at cutover. + # Skipped when promote-images skipped the tag move (stale run) — tasks then + # correctly stay on the old version. If the app deploy fails or never cuts over, + # promote never fires and this job fails visibly. + promote-trigger-staging: + name: Promote Trigger.dev (Staging) + needs: [promote-images, deploy-trigger-staging] + if: >- + github.event_name == 'push' && github.ref == 'refs/heads/staging' && + needs.promote-images.outputs.promoted == 'true' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 75 + permissions: + contents: read + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Download app image digest + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: app-image-digest + path: digest + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ secrets.STAGING_AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.STAGING_AWS_REGION }} + + - name: Wait for ECS traffic cutover + run: | + set -eo pipefail + DIGEST=$(cat digest/app-image-digest.txt) + bash .github/scripts/wait-for-ecs-cutover.sh sim-staging-us-east-1-app-deployment "$DIGEST" + + - name: Promote Trigger.dev version + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + VERSION: ${{ needs.deploy-trigger-staging.outputs.version }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + if [ -z "$VERSION" ]; then + echo "ERROR: no deployed version passed from deploy-trigger-staging" >&2 + exit 1 + fi + echo "Promoting Trigger.dev version $VERSION (staging) at ECS cutover" + bunx trigger.dev@4.4.3 promote "$VERSION" --env staging + + # Production: same lockstep promotion as staging, for the prod environment. + promote-trigger-production: + name: Promote Trigger.dev (Production) + needs: [promote-images, deploy-trigger-production] + if: >- + github.event_name == 'push' && github.ref == 'refs/heads/main' && + needs.promote-images.outputs.promoted == 'true' + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 75 + permissions: + contents: read + id-token: write + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.13 + + - name: Cache Bun dependencies + uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 + with: + path: | + ~/.bun/install/cache + node_modules + **/node_modules + key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} + restore-keys: | + ${{ runner.os }}-bun- + + - name: Install dependencies + run: bun install --frozen-lockfile + + - name: Download app image digest + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: app-image-digest + path: digest + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Wait for ECS traffic cutover + run: | + set -eo pipefail + DIGEST=$(cat digest/app-image-digest.txt) + bash .github/scripts/wait-for-ecs-cutover.sh sim-production-us-east-1-app-deployment "$DIGEST" + + - name: Promote Trigger.dev version + working-directory: ./apps/sim + env: + TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + VERSION: ${{ needs.deploy-trigger-production.outputs.version }} + run: | + set -eo pipefail + if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then + echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 + exit 1 + fi + if [ -z "$VERSION" ]; then + echo "ERROR: no deployed version passed from deploy-trigger-production" >&2 + exit 1 + fi + echo "Promoting Trigger.dev version $VERSION (production) at ECS cutover" + bunx trigger.dev@4.4.3 promote "$VERSION" --env prod + # Build ARM64 images for GHCR (main branch only, runs in parallel with # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64 # are applied by create-ghcr-manifests after the gate, so a failing run From b207a52e7c1c446e69a5ce78a15f2ee566a36dc7 Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Sun, 19 Jul 2026 12:17:49 -0700 Subject: [PATCH 2/3] =?UTF-8?q?fix(ci):=20harden=20Trigger.dev=20cutover?= =?UTF-8?q?=20gate=20=E2=80=94=20reject=20stale=20executions,=20verify=20a?= =?UTF-8?q?ll=20ECS=20targets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/scripts/wait-for-ecs-cutover.sh | 93 +++++++---- .github/workflows/ci.yml | 195 ++++++------------------ 2 files changed, 112 insertions(+), 176 deletions(-) diff --git a/.github/scripts/wait-for-ecs-cutover.sh b/.github/scripts/wait-for-ecs-cutover.sh index 4ec90125804..f7ea7970567 100755 --- a/.github/scripts/wait-for-ecs-cutover.sh +++ b/.github/scripts/wait-for-ecs-cutover.sh @@ -1,23 +1,32 @@ #!/usr/bin/env bash # Waits for the ECS blue/green deploy triggered by a specific app image push to -# reach its traffic cutover (CodeDeploy AllowTraffic == Succeeded), then exits 0. +# reach its traffic cutover (CodeDeploy AllowTraffic == Succeeded on every ECS +# target), then exits 0. # # ECR app images use a floating tag (latest/staging) with no git SHA, so the # only durable key linking this CI push to its ECS deploy is the image DIGEST. -# Correlation: image digest -> CodePipeline execution (AppEcrImage revision) -> +# Correlation: image digest -> CodePipeline execution (ECR_Source revision) -> # Deploy action externalExecutionId (== CodeDeploy deployment id) -> AllowTraffic. # -# Usage: wait-for-ecs-cutover.sh -# Requires: awscli v2, configured credentials with codedeploy + codepipeline read. +# The digest alone is ambiguous: a prior run with the same image could match an +# older, already-cutover execution and promote too early. SINCE_EPOCH (the time +# the deploy tag was retagged, i.e. when THIS push's pipeline was triggered) +# disambiguates — only an execution that started at/after the retag is ours. +# +# Usage: wait-for-ecs-cutover.sh +# Requires: awscli v2, python3, credentials with codedeploy + codepipeline read. set -euo pipefail PIPELINE="${1:?pipeline name required}" DIGEST="${2:?image digest required}" +SINCE_EPOCH="${3:?since-epoch (retag time) required}" POLL_INTERVAL="${POLL_INTERVAL:-15}" # 70 min covers a prod deploy whose Deploy stage is queued behind a prior # deploy's ~50-min termination bake before its own traffic shift begins. OVERALL_TIMEOUT="${OVERALL_TIMEOUT:-4200}" +# Tolerate minor clock skew between the runner (retag time) and CodePipeline. +SINCE_SKEW="${SINCE_SKEW:-120}" deadline=$(( $(date +%s) + OVERALL_TIMEOUT )) remaining() { echo $(( deadline - $(date +%s) )); } @@ -31,21 +40,44 @@ fail_if_expired() { log "Pipeline: $PIPELINE" log "Target app image digest: $DIGEST" +log "Requiring execution started at/after epoch $SINCE_EPOCH (minus ${SINCE_SKEW}s skew)" -# Phase A: find the pipeline execution whose ECR source revision matches our -# digest. --max-items bounds the fetch (the CLI otherwise auto-paginates the whole -# execution history); our push is the newest execution, so it's on the first page. -# The revisionId match is done server-side via JMESPath; grep isolates the UUID -# from any trailing pagination-token line in text output. +# Phase A: find the newest pipeline execution whose ECR source revision matches +# our digest AND that started at/after the retag. The since filter rejects a +# stale historical execution reusing the same digest. --max-items bounds the +# fetch (the CLI otherwise auto-paginates the whole history). EXECUTION_ID="" while [ -z "$EXECUTION_ID" ]; do - fail_if_expired "pipeline execution matching digest" - EXECUTION_ID=$(aws codepipeline list-pipeline-executions \ + fail_if_expired "pipeline execution matching digest since retag" + matches=$(aws codepipeline list-pipeline-executions \ --pipeline-name "$PIPELINE" --max-items 30 \ - --query "pipelineExecutionSummaries[?sourceRevisions[?actionName=='ECR_Source' && revisionId=='$DIGEST']].pipelineExecutionId" \ - --output text 2>/dev/null | tr '\t ' '\n\n' | grep -Em1 '^[0-9a-f-]{36}$' || true) + --query "pipelineExecutionSummaries[?sourceRevisions[?actionName=='ECR_Source' && revisionId=='$DIGEST']].[startTime, pipelineExecutionId]" \ + --output text 2>/dev/null || true) + EXECUTION_ID=$(printf '%s\n' "$matches" | SINCE="$SINCE_EPOCH" SKEW="$SINCE_SKEW" python3 -c ' +import sys, os, datetime +since = float(os.environ["SINCE"]) - float(os.environ["SKEW"]) +best_epoch = None +best_id = None +for line in sys.stdin: + parts = line.rstrip("\n").split("\t") + if len(parts) < 2: + continue + ts, eid = parts[0].strip(), parts[1].strip() + if not ts or not eid or "-" not in eid: + continue + try: + epoch = float(ts) + except ValueError: + try: + epoch = datetime.datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp() + except ValueError: + continue + if epoch >= since and (best_epoch is None or epoch > best_epoch): + best_epoch, best_id = epoch, eid +print(best_id or "") +' 2>/dev/null || true) if [ -z "$EXECUTION_ID" ]; then - log "No matching pipeline execution yet; retry in ${POLL_INTERVAL}s (remaining $(remaining)s)" + log "No matching post-retag pipeline execution yet; retry in ${POLL_INTERVAL}s (remaining $(remaining)s)" sleep "$POLL_INTERVAL" fi done @@ -77,9 +109,11 @@ while [ -z "$DEPLOYMENT_ID" ] || [ "$DEPLOYMENT_ID" = "None" ]; do done log "CodeDeploy deployment: $DEPLOYMENT_ID" -# Phase C: wait for the traffic cutover (AllowTraffic lifecycle event Succeeded). +# Phase C: wait for the traffic cutover. Require AllowTraffic == Succeeded on +# EVERY ECS target, so a multi-target deploy can't promote while one target is +# still mid-cutover or failed. while true; do - fail_if_expired "AllowTraffic (traffic cutover)" + fail_if_expired "AllowTraffic (traffic cutover) on all targets" dstatus=$(aws deploy get-deployment --deployment-id "$DEPLOYMENT_ID" \ --query 'deploymentInfo.status' --output text 2>/dev/null || true) case "$dstatus" in @@ -88,18 +122,25 @@ while true; do exit 1 ;; esac - target_id=$(aws deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \ - --query 'targetIds[0]' --output text 2>/dev/null || true) - at_status="" - if [ -n "$target_id" ] && [ "$target_id" != "None" ]; then - at_status=$(aws deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$target_id" \ - --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \ - --output text 2>/dev/null || true) - if [ "$at_status" = "Succeeded" ]; then - log "Traffic cutover complete (AllowTraffic Succeeded) for $DEPLOYMENT_ID" + target_ids=$(aws deploy list-deployment-targets --deployment-id "$DEPLOYMENT_ID" \ + --query 'targetIds' --output text 2>/dev/null || true) + if [ -n "$target_ids" ] && [ "$target_ids" != "None" ]; then + all_ok=1 + ntargets=0 + for tid in $target_ids; do + ntargets=$((ntargets + 1)) + at_status=$(aws deploy get-deployment-target --deployment-id "$DEPLOYMENT_ID" --target-id "$tid" \ + --query "deploymentTarget.ecsTarget.lifecycleEvents[?lifecycleEventName=='AllowTraffic'].status | [0]" \ + --output text 2>/dev/null || true) + if [ "$at_status" != "Succeeded" ]; then + all_ok=0 + fi + done + if [ "$ntargets" -gt 0 ] && [ "$all_ok" = "1" ]; then + log "Traffic cutover complete (AllowTraffic Succeeded on all $ntargets target(s)) for $DEPLOYMENT_ID" exit 0 fi fi - log "Deployment $DEPLOYMENT_ID status=$dstatus AllowTraffic=${at_status:-pending}; wait ${POLL_INTERVAL}s (remaining $(remaining)s)" + log "Deployment $DEPLOYMENT_ID status=$dstatus; not all targets past AllowTraffic; wait ${POLL_INTERVAL}s (remaining $(remaining)s)" sleep "$POLL_INTERVAL" done diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9571ac30c5e..0e004d2d2d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,69 +182,19 @@ jobs: fi bunx trigger.dev@4.4.3 deploy --env preview --branch dev-sim - # Staging: build & upload the Trigger.dev task version WITHOUT promoting it - # (--skip-promotion). New runs keep executing the old version until the promote - # job flips it at the ECS traffic cutover, so this deploy carries zero - # app<->task skew and runs in parallel with the image build. The captured - # version output is consumed by promote-trigger-staging. - deploy-trigger-staging: - name: Deploy Trigger.dev (Staging) + # Main/staging: build & upload the Trigger.dev task version WITHOUT promoting it + # (--skip-promotion). New runs keep executing the OLD promoted version until + # promote-trigger flips it at the ECS traffic cutover — so the app cutting over + # never changes which task version runs until promote-trigger (which depends on + # this job) promotes the version uploaded here. Runs in parallel with the build; + # intentionally NOT gating the app deploy on it, to avoid coupling every app / + # realtime / pii / migration deploy to trigger.dev availability. + deploy-trigger: + name: Deploy Trigger.dev needs: [migrate] - if: github.event_name == 'push' && github.ref == 'refs/heads/staging' - runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 15 - outputs: - version: ${{ steps.deploy.outputs.version }} - steps: - - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - - name: Setup Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - with: - bun-version: 1.3.13 - - - name: Cache Bun dependencies - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: | - ~/.bun/install/cache - node_modules - **/node_modules - key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} - restore-keys: | - ${{ runner.os }}-bun- - - - name: Install dependencies - run: bun install --frozen-lockfile - - - name: Deploy to Trigger.dev (skip promotion) - id: deploy - working-directory: ./apps/sim - env: - TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} - TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} - run: | - set -eo pipefail - if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then - echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 - exit 1 - fi - bunx trigger.dev@4.4.3 deploy --env staging --skip-promotion 2>&1 | tee deploy.log - # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation. - VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) - if [ -z "$VERSION" ]; then - echo "ERROR: could not parse deployed version from deploy output" >&2 - exit 1 - fi - echo "Captured deployed version: $VERSION" - echo "version=$VERSION" >> "$GITHUB_OUTPUT" - - # Production: same skip-promotion deploy as staging, for the prod environment. - deploy-trigger-production: - name: Deploy Trigger.dev (Production) - needs: [migrate] - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: >- + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 outputs: @@ -278,13 +228,14 @@ jobs: env: TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + TRIGGER_ENV: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }} run: | set -eo pipefail if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 exit 1 fi - bunx trigger.dev@4.4.3 deploy --env prod --skip-promotion 2>&1 | tee deploy.log + bunx trigger.dev@4.4.3 deploy --env "$TRIGGER_ENV" --skip-promotion 2>&1 | tee deploy.log # Extract the deployed version (e.g. 20260715.2) tied to THIS invocation. VERSION=$(sed -E 's/\x1b\[[0-9;]*m//g' deploy.log | grep -oE '20[0-9]{6}\.[0-9]+' | tail -n1 || true) if [ -z "$VERSION" ]; then @@ -431,9 +382,13 @@ jobs: id-token: write outputs: # Whether the deploy tag was actually moved (false on a stale-run guard - # skip). promote-trigger-* keys off this so tasks are never promoted when + # skip). promote-trigger keys off this so tasks are never promoted when # the app itself wasn't. promoted: ${{ steps.guard.outputs.fresh }} + # Epoch when the deploy tag was retagged (this push's ECS pipeline trigger). + # promote-trigger passes it to the poll script so a stale pipeline execution + # reusing the same image digest can't satisfy the cutover gate. + retag_epoch: ${{ steps.promote.outputs.retag_epoch }} steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 @@ -463,6 +418,7 @@ jobs: fi - name: Promote images to deploy tags + id: promote if: steps.guard.outputs.fresh == 'true' env: ECR_REPOS: >- @@ -471,6 +427,11 @@ jobs: ${{ secrets.ECR_REALTIME }} ${{ secrets.ECR_PII }} run: | + # Record the retag time BEFORE moving any tag — this is when the ECS + # pipeline for this push is triggered. promote-trigger uses it to + # reject an older pipeline execution reusing the same image digest. + echo "retag_epoch=$(date +%s)" >> "$GITHUB_OUTPUT" + REGISTRY="${{ steps.login-ecr.outputs.registry }}" if [ "${{ github.ref }}" = "refs/heads/main" ]; then @@ -493,18 +454,20 @@ jobs: "${REGISTRY}/${repo}:${{ github.sha }}" done - # Staging: promote the skip-promoted Trigger.dev version at the exact moment the - # ECS app deploy shifts traffic (CodeDeploy AllowTraffic), so tasks and app cut - # over in lockstep. The promote-images retag above is what triggers the ECS - # pipeline; this job polls it (via the app image digest) and promotes at cutover. + # Main/staging: promote the skip-promoted Trigger.dev version at the exact moment + # the ECS app deploy shifts traffic (CodeDeploy AllowTraffic on every target), so + # tasks and app cut over in lockstep. The promote-images retag is what triggers + # the ECS pipeline; this job correlates it via the app image digest + retag epoch + # (rejecting a stale execution reusing the digest) and promotes at cutover. # Skipped when promote-images skipped the tag move (stale run) — tasks then # correctly stay on the old version. If the app deploy fails or never cuts over, # promote never fires and this job fails visibly. - promote-trigger-staging: - name: Promote Trigger.dev (Staging) - needs: [promote-images, deploy-trigger-staging] + promote-trigger: + name: Promote Trigger.dev + needs: [promote-images, deploy-trigger] if: >- - github.event_name == 'push' && github.ref == 'refs/heads/staging' && + github.event_name == 'push' && + (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') && needs.promote-images.outputs.promoted == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 75 @@ -543,93 +506,25 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 with: - role-to-assume: ${{ secrets.STAGING_AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.STAGING_AWS_REGION }} + role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }} + aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }} - name: Wait for ECS traffic cutover - run: | - set -eo pipefail - DIGEST=$(cat digest/app-image-digest.txt) - bash .github/scripts/wait-for-ecs-cutover.sh sim-staging-us-east-1-app-deployment "$DIGEST" - - - name: Promote Trigger.dev version - working-directory: ./apps/sim env: - TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} - TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} - VERSION: ${{ needs.deploy-trigger-staging.outputs.version }} - run: | - set -eo pipefail - if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then - echo "ERROR: TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 - exit 1 - fi - if [ -z "$VERSION" ]; then - echo "ERROR: no deployed version passed from deploy-trigger-staging" >&2 - exit 1 - fi - echo "Promoting Trigger.dev version $VERSION (staging) at ECS cutover" - bunx trigger.dev@4.4.3 promote "$VERSION" --env staging - - # Production: same lockstep promotion as staging, for the prod environment. - promote-trigger-production: - name: Promote Trigger.dev (Production) - needs: [promote-images, deploy-trigger-production] - if: >- - github.event_name == 'push' && github.ref == 'refs/heads/main' && - needs.promote-images.outputs.promoted == 'true' - runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 75 - permissions: - contents: read - id-token: write - steps: - - name: Checkout code - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - - name: Setup Bun - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 - with: - bun-version: 1.3.13 - - - name: Cache Bun dependencies - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 - with: - path: | - ~/.bun/install/cache - node_modules - **/node_modules - key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }} - restore-keys: | - ${{ runner.os }}-bun- - - - name: Install dependencies - run: bun install --frozen-lockfile - - - name: Download app image digest - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - name: app-image-digest - path: digest - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Wait for ECS traffic cutover + PIPELINE: sim-${{ github.ref == 'refs/heads/main' && 'production' || 'staging' }}-us-east-1-app-deployment + RETAG_EPOCH: ${{ needs.promote-images.outputs.retag_epoch }} run: | set -eo pipefail DIGEST=$(cat digest/app-image-digest.txt) - bash .github/scripts/wait-for-ecs-cutover.sh sim-production-us-east-1-app-deployment "$DIGEST" + bash .github/scripts/wait-for-ecs-cutover.sh "$PIPELINE" "$DIGEST" "$RETAG_EPOCH" - name: Promote Trigger.dev version working-directory: ./apps/sim env: TRIGGER_ACCESS_TOKEN: ${{ secrets.TRIGGER_ACCESS_TOKEN }} TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} - VERSION: ${{ needs.deploy-trigger-production.outputs.version }} + TRIGGER_ENV: ${{ github.ref == 'refs/heads/main' && 'prod' || 'staging' }} + VERSION: ${{ needs.deploy-trigger.outputs.version }} run: | set -eo pipefail if [ -z "$TRIGGER_ACCESS_TOKEN" ] || [ -z "$TRIGGER_PROJECT_ID" ]; then @@ -637,11 +532,11 @@ jobs: exit 1 fi if [ -z "$VERSION" ]; then - echo "ERROR: no deployed version passed from deploy-trigger-production" >&2 + echo "ERROR: no deployed version passed from deploy-trigger" >&2 exit 1 fi - echo "Promoting Trigger.dev version $VERSION (production) at ECS cutover" - bunx trigger.dev@4.4.3 promote "$VERSION" --env prod + echo "Promoting Trigger.dev version $VERSION ($TRIGGER_ENV) at ECS cutover" + bunx trigger.dev@4.4.3 promote "$VERSION" --env "$TRIGGER_ENV" # Build ARM64 images for GHCR (main branch only, runs in parallel with # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64 From f7135b058aa5d798f0ca4b7ad9aa5822aad91b2c Mon Sep 17 00:00:00 2001 From: Theodore Li Date: Sun, 19 Jul 2026 13:55:34 -0700 Subject: [PATCH 3/3] fix(ci): widen promote-trigger job timeout above the poll budget --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e004d2d2d0..2160b50fec1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -470,7 +470,10 @@ jobs: (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') && needs.promote-images.outputs.promoted == 'true' runs-on: blacksmith-4vcpu-ubuntu-2404 - timeout-minutes: 75 + # Must exceed the poll script's OVERALL_TIMEOUT (70 min, covering a prod deploy + # queued behind a ~50-min bake) PLUS runner setup + the final promote step, so + # the Actions timeout never kills the job before the script's own deadline. + timeout-minutes: 90 permissions: contents: read id-token: write