Upgrade Elasticsearch to 9.5.3

.github/workflows/build.yaml

@@ -83,6 +83,8 @@ jobs:
     timeout-minutes: 30
     outputs:
       version: ${{ steps.version.outputs.version }}
+      elasticsearch_image_tag: ${{ steps.elasticsearch_image.outputs.tag }}
+      build_elasticsearch_image: ${{ steps.elasticsearch_image.outputs.build_locally }}
       should_publish: ${{ (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'dev-preview')) && secrets.DOCKER_USERNAME != '' && secrets.DOCKER_PASSWORD != '' }}
       is_prod_deploy: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name != 'pull_request' }}
       is_dev_deploy: ${{ (github.event_name == 'repository_dispatch' && github.event.action == 'preview') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'dev-preview')) }}
@@ -129,16 +131,77 @@ jobs:
           echo "version=$version" >> $GITHUB_OUTPUT
           echo "### $version" >> $GITHUB_STEP_SUMMARY
 
+      - name: Resolve Elasticsearch image
+        id: elasticsearch_image
+        env:
+          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          PR_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }}
+          PREVIEW_BASE_SHA: ${{ github.event.client_payload.base_sha }}
+          PUSH_BEFORE_SHA: ${{ github.event.before }}
+        run: |
+          version=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile)
+          tag=$version
+          image_changed=false
+          build_locally=false
+
+          if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] &&
+            ! git diff --quiet "$PR_BASE_SHA"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_changed=true
+          elif [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == "refs/heads/main" ]] &&
+            ! git diff --quiet "$PUSH_BEFORE_SHA"..HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_changed=true
+          elif [[ "$GITHUB_EVENT_NAME" == "repository_dispatch" ]] &&
+            ! git diff --quiet "${PREVIEW_BASE_SHA:-origin/main}"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_changed=true
+          elif [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]] &&
+            ! git diff --quiet origin/main...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then
+            image_changed=true
+          fi
+
+          if [[ "$image_changed" == "true" ]]; then
+            image_sha=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1)
+            tag="$version-sha256-$image_sha"
+            image="exceptionless/elasticsearch:$tag"
+
+            if [[ "$GITHUB_EVENT_NAME" == "pull_request" && "$PR_HEAD_REPOSITORY" != "$GITHUB_REPOSITORY" ]]; then
+              build_locally=true
+            else
+              for attempt in {1..150}; do
+                if docker manifest inspect "$image" > /dev/null 2>&1; then
+                  break
+                fi
+
+                if [[ "$attempt" -eq 150 ]]; then
+                  echo "::error::Timed out waiting for $image to be published."
+                  exit 1
+                fi
+
+                sleep 10
+              done
+            fi
+          fi
+
+          echo "tag=$tag" >> "$GITHUB_OUTPUT"
+          echo "build_locally=$build_locally" >> "$GITHUB_OUTPUT"
+          echo "### Elasticsearch image: exceptionless/elasticsearch:$tag" >> "$GITHUB_STEP_SUMMARY"
+
   test-api:
+    needs: version
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    env:
+      Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }}
 
     steps:
       - name: Checkout
         uses: actions/checkout@v7
         with:
           ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }}
 
+      - name: Build fork Elasticsearch candidate locally
+        if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }}
+        run: docker build --tag "exceptionless/elasticsearch:${{ needs.version.outputs.elasticsearch_image_tag }}" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x
+
       - name: Setup .NET Core
         uses: actions/setup-dotnet@v6
         with:
@@ -223,12 +286,21 @@ jobs:
         run: echo "npm run test:integration"
 
   test-e2e:
+    needs: version
     runs-on: ubuntu-latest
     timeout-minutes: 45
+    env:
+      Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }}
 
     steps:
       - name: Checkout
         uses: actions/checkout@v6
+        with:
+          ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }}
+
+      - name: Build fork Elasticsearch candidate locally
+        if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }}
+        run: docker build --tag "exceptionless/elasticsearch:${{ needs.version.outputs.elasticsearch_image_tag }}" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x
 
       - name: Setup .NET Core
         uses: actions/setup-dotnet@v6

.github/workflows/elasticsearch-docker-8.yml

@@ -40,7 +40,13 @@ jobs:
         with:
           platforms: linux/amd64,linux/arm64
       - name: Build custom Elasticsearch 8.x docker image
-        working-directory: build/docker/elasticsearch/8.x
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
         run: |
-          VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile)
-          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION --tag exceptionless/elasticsearch:latest
+          VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/8.x/Dockerfile)
+          IMAGE_SHA=$(git ls-files -- build/docker/elasticsearch/8.x .github/workflows/elasticsearch-docker-8.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1)
+          TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA")
+          if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then
+            TAGS+=(--tag "exceptionless/elasticsearch:$VERSION")
+          fi
+          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file build/docker/elasticsearch/8.x/Dockerfile build/docker/elasticsearch/8.x "${TAGS[@]}"

.github/workflows/elasticsearch-docker-9.yml

@@ -0,0 +1,52 @@
+name: Elasticsearch 9.x Docker Image CI
+
+on:
+  push:
+    paths:
+      - "build/docker/elasticsearch/9.x/**"
+      - ".github/workflows/elasticsearch-docker-9.yml"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: startsWith(github.ref, 'refs/tags/v') != true
+
+    steps:
+      - uses: actions/checkout@v7
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v5
+        with:
+          dotnet-version: 10.0.301
+      - name: Build Reason
+        env:
+          GITHUB_EVENT: ${{ toJson(github) }}
+        run: "echo ref: ${{github.ref}} event: ${{github.event_name}}"
+      - name: Build Version
+        run: |
+          dotnet tool install --global minver-cli --version 7.0.0
+          version=$(minver --tag-prefix v)
+          echo "MINVERVERSIONOVERRIDE=$version" >> $GITHUB_ENV
+          echo "VERSION=$version" >> $GITHUB_ENV
+          echo "### Version: $version" >> $GITHUB_STEP_SUMMARY
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v4
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+        with:
+          platforms: linux/amd64,linux/arm64
+      - name: Build custom Elasticsearch 9.x docker image
+        env:
+          DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        run: |
+          VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile)
+          IMAGE_SHA=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1)
+          TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA")
+          if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then
+            TAGS+=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest")
+          fi
+          docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x "${TAGS[@]}"

.github/workflows/preview-command.yml

@@ -77,6 +77,7 @@ jobs:
             const headRepository = pullRequest.head.repo.full_name;
             const headRef = pullRequest.head.ref;
             const headSha = pullRequest.head.sha;
+            const baseSha = pullRequest.base.sha;
             const headLabel = pullRequest.head.label;
             const headShortSha = headSha.slice(0, 12);
 
@@ -95,6 +96,7 @@ jobs:
             core.setOutput("head-ref", headRef);
             core.setOutput("head-label", headLabel);
             core.setOutput("head-sha", headSha);
+            core.setOutput("base-sha", baseSha);
             core.setOutput("head-short-sha", headShortSha);
 
             const previewLabel = "dev-preview";
@@ -142,6 +144,7 @@ jobs:
           HEAD_REF: ${{ steps.preview.outputs.head-ref }}
           HEAD_LABEL: ${{ steps.preview.outputs.head-label }}
           HEAD_SHA: ${{ steps.preview.outputs.head-sha }}
+          BASE_SHA: ${{ steps.preview.outputs.base-sha }}
         with:
           script: |
             await github.rest.repos.createDispatchEvent({
@@ -152,7 +155,8 @@ jobs:
                 pr_number: Number(process.env.PR_NUMBER),
                 head_ref: process.env.HEAD_REF,
                 head_label: process.env.HEAD_LABEL,
-                head_sha: process.env.HEAD_SHA
+                head_sha: process.env.HEAD_SHA,
+                base_sha: process.env.BASE_SHA
               }
             });
 

Dockerfile

@@ -100,7 +100,7 @@ ENTRYPOINT ["/app/app-docker-entrypoint.sh"]
 
 # completely self-contained
 
-FROM exceptionless/elasticsearch:8.19.15 AS exceptionless
+FROM exceptionless/elasticsearch:9.5.3 AS exceptionless
 
 WORKDIR /app
 COPY --from=job-publish /app/src/Exceptionless.Job/out ./
@@ -113,21 +113,23 @@ COPY ./build/supervisord.conf /etc/
 USER root
 
 # install dotnet and supervisor
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends \
-    supervisor \
+RUN microdnf install -y \
     wget \
     dos2unix \
     ca-certificates \
+    python3-pip \
     \
     # .NET dependencies
-    libc6 \
-    libgcc-s1 \
-    libicu74 \
-    libssl3 \
-    libstdc++6 \
+    glibc \
+    gzip \
+    libgcc \
+    libicu \
+    openssl-libs \
+    libstdc++ \
+    tar \
     tzdata \
-    && rm -rf /var/lib/apt/lists/* \
+    && pip3 install --no-cache-dir supervisor==4.3.0 \
+    && microdnf clean all \
     && dos2unix /app/docker-entrypoint.sh
 
 ENV discovery.type=single-node \

build/docker/elasticsearch/8.x/Dockerfile

@@ -1,5 +1,4 @@
 # https://www.docker.elastic.co/
-FROM docker.elastic.co/elasticsearch/elasticsearch:8.19.15
+FROM docker.elastic.co/elasticsearch/elasticsearch:8.19.21
 
 RUN elasticsearch-plugin install -b mapper-size
-

build/docker/elasticsearch/9.x/Dockerfile

@@ -0,0 +1,4 @@
+# https://www.docker.elastic.co/
+FROM docker.elastic.co/elasticsearch/elasticsearch:9.5.3
+
+RUN elasticsearch-plugin install -b mapper-size

docker/docker-compose.apm.yml

@@ -2,7 +2,7 @@ version: "2.2"
 
 services:
   setup:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.5.3
     volumes:
       - certs:/usr/share/elasticsearch/config/certs
     user: "0"
@@ -53,7 +53,7 @@ services:
     depends_on:
       setup:
         condition: service_healthy
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15
+    image: docker.elastic.co/elasticsearch/elasticsearch:9.5.3
     volumes:
       - certs:/usr/share/elasticsearch/config/certs
       - esdata:/usr/share/elasticsearch/data
@@ -98,7 +98,7 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.5.3
     volumes:
       - certs:/usr/share/kibana/config/certs
     ports:
@@ -124,7 +124,7 @@ services:
     depends_on:
       elasticsearch:
         condition: service_healthy
-    image: docker.elastic.co/apm/apm-server:8.19.15
+    image: docker.elastic.co/apm/apm-server:9.5.3
     volumes:
       - certs:/usr/share/apm-server/certs
     ports:

docker/docker-compose.dev.yml

@@ -50,7 +50,7 @@ services:
       - appdata:/app/storage
 
   elasticsearch:
-    image: exceptionless/elasticsearch:8.19.15
+    image: exceptionless/elasticsearch:9.5.3
     environment:
       discovery.type: single-node
       xpack.security.enabled: "false"
@@ -59,6 +59,8 @@ services:
       - 9200:9200
       - 9300:9300
     volumes:
+      # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9.
+      # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance
       - esdata7:/usr/share/elasticsearch/data
     healthcheck:
       test:
@@ -74,7 +76,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.5.3
     ports:
       - 5601:5601
 

docker/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   elasticsearch:
-    image: exceptionless/elasticsearch:8.19.15
+    image: exceptionless/elasticsearch:9.5.3
     environment:
       node.name: elasticsearch
       cluster.name: exceptionless
@@ -11,6 +11,8 @@ services:
     ports:
       - 9200:9200
     volumes:
+      # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9.
+      # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance
       - esdata:/usr/share/elasticsearch/data
     healthcheck:
       test:
@@ -26,7 +28,7 @@ services:
   kibana:
     depends_on:
       - elasticsearch
-    image: docker.elastic.co/kibana/kibana:8.19.15
+    image: docker.elastic.co/kibana/kibana:9.5.3
     environment:
       xpack.security.enabled: "false"
     ports: