diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 1d0725d975..69db68ae5a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -83,6 +83,8 @@ jobs: timeout-minutes: 30 outputs: version: ${{ steps.version.outputs.version }} + elasticsearch_image_tag: ${{ steps.elasticsearch_image.outputs.tag }} + build_elasticsearch_image: ${{ steps.elasticsearch_image.outputs.build_locally }} should_publish: ${{ (github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'dev-preview')) && secrets.DOCKER_USERNAME != '' && secrets.DOCKER_PASSWORD != '' }} is_prod_deploy: ${{ startsWith(github.ref, 'refs/tags/v') && github.event_name != 'pull_request' }} is_dev_deploy: ${{ (github.event_name == 'repository_dispatch' && github.event.action == 'preview') || (github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'dev-preview')) }} @@ -129,9 +131,66 @@ jobs: echo "version=$version" >> $GITHUB_OUTPUT echo "### $version" >> $GITHUB_STEP_SUMMARY + - name: Resolve Elasticsearch image + id: elasticsearch_image + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} + PREVIEW_BASE_SHA: ${{ github.event.client_payload.base_sha }} + PUSH_BEFORE_SHA: ${{ github.event.before }} + run: | + version=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile) + tag=$version + image_changed=false + build_locally=false + + if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]] && + ! git diff --quiet "$PR_BASE_SHA"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + elif [[ "$GITHUB_EVENT_NAME" == "push" && "$GITHUB_REF" == "refs/heads/main" ]] && + ! git diff --quiet "$PUSH_BEFORE_SHA"..HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + elif [[ "$GITHUB_EVENT_NAME" == "repository_dispatch" ]] && + ! git diff --quiet "${PREVIEW_BASE_SHA:-origin/main}"...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + elif [[ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]] && + ! git diff --quiet origin/main...HEAD -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml; then + image_changed=true + fi + + if [[ "$image_changed" == "true" ]]; then + image_sha=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1) + tag="$version-sha256-$image_sha" + image="exceptionless/elasticsearch:$tag" + + if [[ "$GITHUB_EVENT_NAME" == "pull_request" && "$PR_HEAD_REPOSITORY" != "$GITHUB_REPOSITORY" ]]; then + build_locally=true + else + for attempt in {1..150}; do + if docker manifest inspect "$image" > /dev/null 2>&1; then + break + fi + + if [[ "$attempt" -eq 150 ]]; then + echo "::error::Timed out waiting for $image to be published." + exit 1 + fi + + sleep 10 + done + fi + fi + + echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "build_locally=$build_locally" >> "$GITHUB_OUTPUT" + echo "### Elasticsearch image: exceptionless/elasticsearch:$tag" >> "$GITHUB_STEP_SUMMARY" + test-api: + needs: version runs-on: ubuntu-latest timeout-minutes: 30 + env: + Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }} steps: - name: Checkout @@ -139,6 +198,10 @@ jobs: with: ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }} + - name: Build fork Elasticsearch candidate locally + if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }} + run: docker build --tag "exceptionless/elasticsearch:${{ needs.version.outputs.elasticsearch_image_tag }}" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x + - name: Setup .NET Core uses: actions/setup-dotnet@v6 with: @@ -223,12 +286,21 @@ jobs: run: echo "npm run test:integration" test-e2e: + needs: version runs-on: ubuntu-latest timeout-minutes: 45 + env: + Elasticsearch__ImageTag: ${{ needs.version.outputs.elasticsearch_image_tag }} steps: - name: Checkout uses: actions/checkout@v6 + with: + ref: ${{ (github.event_name == 'repository_dispatch' && github.event.client_payload.head_sha) || github.event.pull_request.head.sha || github.sha }} + + - name: Build fork Elasticsearch candidate locally + if: ${{ needs.version.outputs.build_elasticsearch_image == 'true' }} + run: docker build --tag "exceptionless/elasticsearch:${{ needs.version.outputs.elasticsearch_image_tag }}" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x - name: Setup .NET Core uses: actions/setup-dotnet@v6 diff --git a/.github/workflows/elasticsearch-docker-8.yml b/.github/workflows/elasticsearch-docker-8.yml index ef74055e42..770c497fad 100644 --- a/.github/workflows/elasticsearch-docker-8.yml +++ b/.github/workflows/elasticsearch-docker-8.yml @@ -40,7 +40,13 @@ jobs: with: platforms: linux/amd64,linux/arm64 - name: Build custom Elasticsearch 8.x docker image - working-directory: build/docker/elasticsearch/8.x + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} run: | - VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' Dockerfile) - docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file ./Dockerfile . --tag exceptionless/elasticsearch:$VERSION --tag exceptionless/elasticsearch:latest + VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/8.x/Dockerfile) + IMAGE_SHA=$(git ls-files -- build/docker/elasticsearch/8.x .github/workflows/elasticsearch-docker-8.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1) + TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA") + if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then + TAGS+=(--tag "exceptionless/elasticsearch:$VERSION") + fi + docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file build/docker/elasticsearch/8.x/Dockerfile build/docker/elasticsearch/8.x "${TAGS[@]}" diff --git a/.github/workflows/elasticsearch-docker-9.yml b/.github/workflows/elasticsearch-docker-9.yml new file mode 100644 index 0000000000..a911616691 --- /dev/null +++ b/.github/workflows/elasticsearch-docker-9.yml @@ -0,0 +1,52 @@ +name: Elasticsearch 9.x Docker Image CI + +on: + push: + paths: + - "build/docker/elasticsearch/9.x/**" + - ".github/workflows/elasticsearch-docker-9.yml" + +jobs: + build: + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') != true + + steps: + - uses: actions/checkout@v7 + - name: Setup .NET Core + uses: actions/setup-dotnet@v5 + with: + dotnet-version: 10.0.301 + - name: Build Reason + env: + GITHUB_EVENT: ${{ toJson(github) }} + run: "echo ref: ${{github.ref}} event: ${{github.event_name}}" + - name: Build Version + run: | + dotnet tool install --global minver-cli --version 7.0.0 + version=$(minver --tag-prefix v) + echo "MINVERVERSIONOVERRIDE=$version" >> $GITHUB_ENV + echo "VERSION=$version" >> $GITHUB_ENV + echo "### Version: $version" >> $GITHUB_STEP_SUMMARY + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + - name: Login to DockerHub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + with: + platforms: linux/amd64,linux/arm64 + - name: Build custom Elasticsearch 9.x docker image + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + VERSION=$(sed -n 's/.*elasticsearch:\([^ ]*\).*/\1/p' build/docker/elasticsearch/9.x/Dockerfile) + IMAGE_SHA=$(git ls-files -- build/docker/elasticsearch/9.x .github/workflows/elasticsearch-docker-9.yml | sort | xargs sha256sum | sha256sum | cut -d " " -f 1) + TAGS=(--tag "exceptionless/elasticsearch:$VERSION-sha256-$IMAGE_SHA") + if [[ "$GITHUB_REF" == "refs/heads/$DEFAULT_BRANCH" ]]; then + TAGS+=(--tag "exceptionless/elasticsearch:$VERSION" --tag "exceptionless/elasticsearch:latest") + fi + docker buildx build --platform linux/amd64,linux/arm64 --output "type=image,push=true" --file build/docker/elasticsearch/9.x/Dockerfile build/docker/elasticsearch/9.x "${TAGS[@]}" diff --git a/.github/workflows/preview-command.yml b/.github/workflows/preview-command.yml index fe94875393..e4dd285c7f 100644 --- a/.github/workflows/preview-command.yml +++ b/.github/workflows/preview-command.yml @@ -77,6 +77,7 @@ jobs: const headRepository = pullRequest.head.repo.full_name; const headRef = pullRequest.head.ref; const headSha = pullRequest.head.sha; + const baseSha = pullRequest.base.sha; const headLabel = pullRequest.head.label; const headShortSha = headSha.slice(0, 12); @@ -95,6 +96,7 @@ jobs: core.setOutput("head-ref", headRef); core.setOutput("head-label", headLabel); core.setOutput("head-sha", headSha); + core.setOutput("base-sha", baseSha); core.setOutput("head-short-sha", headShortSha); const previewLabel = "dev-preview"; @@ -142,6 +144,7 @@ jobs: HEAD_REF: ${{ steps.preview.outputs.head-ref }} HEAD_LABEL: ${{ steps.preview.outputs.head-label }} HEAD_SHA: ${{ steps.preview.outputs.head-sha }} + BASE_SHA: ${{ steps.preview.outputs.base-sha }} with: script: | await github.rest.repos.createDispatchEvent({ @@ -152,7 +155,8 @@ jobs: pr_number: Number(process.env.PR_NUMBER), head_ref: process.env.HEAD_REF, head_label: process.env.HEAD_LABEL, - head_sha: process.env.HEAD_SHA + head_sha: process.env.HEAD_SHA, + base_sha: process.env.BASE_SHA } }); diff --git a/Dockerfile b/Dockerfile index 4904cdf6f9..b3c4bdc53c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,7 @@ ENTRYPOINT ["/app/app-docker-entrypoint.sh"] # completely self-contained -FROM exceptionless/elasticsearch:8.19.15 AS exceptionless +FROM exceptionless/elasticsearch:9.5.3 AS exceptionless WORKDIR /app COPY --from=job-publish /app/src/Exceptionless.Job/out ./ @@ -113,21 +113,23 @@ COPY ./build/supervisord.conf /etc/ USER root # install dotnet and supervisor -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - supervisor \ +RUN microdnf install -y \ wget \ dos2unix \ ca-certificates \ + python3-pip \ \ # .NET dependencies - libc6 \ - libgcc-s1 \ - libicu74 \ - libssl3 \ - libstdc++6 \ + glibc \ + gzip \ + libgcc \ + libicu \ + openssl-libs \ + libstdc++ \ + tar \ tzdata \ - && rm -rf /var/lib/apt/lists/* \ + && pip3 install --no-cache-dir supervisor==4.3.0 \ + && microdnf clean all \ && dos2unix /app/docker-entrypoint.sh ENV discovery.type=single-node \ diff --git a/build/docker/elasticsearch/8.x/Dockerfile b/build/docker/elasticsearch/8.x/Dockerfile index bbab4cc3bc..3be7362ff5 100644 --- a/build/docker/elasticsearch/8.x/Dockerfile +++ b/build/docker/elasticsearch/8.x/Dockerfile @@ -1,5 +1,4 @@ # https://www.docker.elastic.co/ -FROM docker.elastic.co/elasticsearch/elasticsearch:8.19.15 +FROM docker.elastic.co/elasticsearch/elasticsearch:8.19.21 RUN elasticsearch-plugin install -b mapper-size - diff --git a/build/docker/elasticsearch/9.x/Dockerfile b/build/docker/elasticsearch/9.x/Dockerfile new file mode 100644 index 0000000000..75eb1700b9 --- /dev/null +++ b/build/docker/elasticsearch/9.x/Dockerfile @@ -0,0 +1,4 @@ +# https://www.docker.elastic.co/ +FROM docker.elastic.co/elasticsearch/elasticsearch:9.5.3 + +RUN elasticsearch-plugin install -b mapper-size diff --git a/docker/docker-compose.apm.yml b/docker/docker-compose.apm.yml index bd55972eb9..c9cdfa81f7 100644 --- a/docker/docker-compose.apm.yml +++ b/docker/docker-compose.apm.yml @@ -2,7 +2,7 @@ version: "2.2" services: setup: - image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15 + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.3 volumes: - certs:/usr/share/elasticsearch/config/certs user: "0" @@ -53,7 +53,7 @@ services: depends_on: setup: condition: service_healthy - image: docker.elastic.co/elasticsearch/elasticsearch:8.19.15 + image: docker.elastic.co/elasticsearch/elasticsearch:9.5.3 volumes: - certs:/usr/share/elasticsearch/config/certs - esdata:/usr/share/elasticsearch/data @@ -98,7 +98,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.5.3 volumes: - certs:/usr/share/kibana/config/certs ports: @@ -124,7 +124,7 @@ services: depends_on: elasticsearch: condition: service_healthy - image: docker.elastic.co/apm/apm-server:8.19.15 + image: docker.elastic.co/apm/apm-server:9.5.3 volumes: - certs:/usr/share/apm-server/certs ports: diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index a18d81b4cf..5a1d298220 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -50,7 +50,7 @@ services: - appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:8.19.15 + image: exceptionless/elasticsearch:9.5.3 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -59,6 +59,8 @@ services: - 9200:9200 - 9300:9300 volumes: + # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9. + # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance - esdata7:/usr/share/elasticsearch/data healthcheck: test: @@ -74,7 +76,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.5.3 ports: - 5601:5601 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c81594d41f..101ea52335 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,6 +1,6 @@ services: elasticsearch: - image: exceptionless/elasticsearch:8.19.15 + image: exceptionless/elasticsearch:9.5.3 environment: node.name: elasticsearch cluster.name: exceptionless @@ -11,6 +11,8 @@ services: ports: - 9200:9200 volumes: + # Complete the Elasticsearch 8.19 upgrade preflight before reusing this volume with Elasticsearch 9. + # See https://exceptionless.com/docs/self-hosting/upgrading-self-hosted-instance - esdata:/usr/share/elasticsearch/data healthcheck: test: @@ -26,7 +28,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.5.3 environment: xpack.security.enabled: "false" ports: diff --git a/docs/docs/self-hosting/upgrading-self-hosted-instance.md b/docs/docs/self-hosting/upgrading-self-hosted-instance.md index dd0974989f..ff3d7cb665 100644 --- a/docs/docs/self-hosting/upgrading-self-hosted-instance.md +++ b/docs/docs/self-hosting/upgrading-self-hosted-instance.md @@ -8,6 +8,68 @@ title: "Upgrading" **If you are upgrading from v1 or [v2](https://github.com/exceptionless/Exceptionless/releases/tag/v2.0.0) you will need to upgrade to [v3.0](https://github.com/exceptionless/Exceptionless/releases/tag/v3.0.0) before upgrading to the latest release.** +## Upgrading from v8 to v9 + +Exceptionless v9 uses Elasticsearch 9. Do not point an Elasticsearch 9 node at an existing data volume until the cluster has been prepared with Elasticsearch 8.19. Elasticsearch 9 can fail to start when incompatible indices created before Elasticsearch 8 remain. + +Use this upgrade path for an existing self-hosted installation: + +1. Take a current Elasticsearch snapshot or other verified backup and test that it can be restored. Elasticsearch does not support downgrading a data directory after it has been upgraded. +2. Upgrade Elasticsearch and Kibana to the latest 8.19.x patch release first, using the existing data volume. Do not start Elasticsearch 9 yet. +3. Stop the Exceptionless app and job services, but leave Elasticsearch and Kibana 8.19 running. This prevents writes while legacy indices are reindexed. All-in-one installations must use the Elasticsearch-only procedure below; killing the app process is insufficient because its supervisor restarts it. +4. Open Kibana's **Upgrade Assistant** and resolve every critical issue. Reindex every active Exceptionless index created before Elasticsearch 8. Delete only indices you have confirmed are no longer needed; do not mark active Exceptionless indices as read-only. +5. If this data volume previously ran Elasticsearch 7, temporarily disable the GeoIP downloader while still on Elasticsearch 8.19. Elasticsearch deletes its downloaded `.geoip_databases` system index when this setting is disabled; Exceptionless data is not affected. + + ```bash + curl -fsS -X PUT "http://localhost:9200/_cluster/settings" \ + -H "Content-Type: application/json" \ + -d '{"persistent":{"ingest.geoip.downloader.enabled":false}}' + ``` + +6. Confirm that the deprecation API reports no critical issues: + + ```bash + curl -fsS "http://localhost:9200/_migration/deprecations?pretty" + ``` + +7. Stop Elasticsearch and Kibana 8.19 without deleting their data volume. Update the Elasticsearch and Kibana images to the v9 versions, start them, and verify cluster health before restarting the Exceptionless app and jobs. In Docker Compose, do not run `docker compose down -v` because `-v` deletes the data volume. +8. After Elasticsearch 9 is healthy, restore the default GeoIP downloader behavior: + + ```bash + curl -fsS -X PUT "http://localhost:9200/_cluster/settings" \ + -H "Content-Type: application/json" \ + -d '{"persistent":{"ingest.geoip.downloader.enabled":null}}' + ``` + +See Elastic's [prepare-to-upgrade guide](https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade) for the supported 8.x to 9.x upgrade requirements and Upgrade Assistant details. + +### All-in-one: keep Elasticsearch running without the app + +For `samples/docker-compose.all-in-one.yml`, run these commands from the existing deployment directory with its existing Compose project name and environment. Do not create a new project or change the volume mapping: that can silently select an empty data volume. Take and restore-test the snapshot first. Stop external Exceptionless jobs, ingestion consumers, and other writers too. + +1. Pin `exceptionless` to an approved **8.x all-in-one application image containing Elasticsearch 8.19.21**, and `kibana` to `docker.elastic.co/kibana/kibana:8.19.21`. Do not use `latest` or an Elasticsearch 9 image during preparation. Keep the original volume, security settings, and resource limits. +2. Stop both existing containers without removing their volumes: + + ```powershell + docker compose -f docker-compose.all-in-one.yml stop kibana exceptionless + ``` + +3. Start only Elasticsearch from the all-in-one image in a dedicated foreground terminal. Overriding the entrypoint bypasses the supervisor entirely, so neither the app nor its in-process jobs start. `--service-ports` and `--use-aliases` preserve the service's ports and Kibana's `exceptionless` hostname; `--no-deps` prevents other services from starting. + + ```powershell + docker compose -f docker-compose.all-in-one.yml run --rm --no-deps --service-ports --use-aliases --entrypoint /usr/local/bin/docker-entrypoint.sh exceptionless eswrapper + ``` + + Verify `GET /` reports 8.19.21 and the expected cluster UUID, and check cluster health, index counts, and representative records before doing maintenance. Confirm no application process is running. Never start the regular `exceptionless` service while this maintenance container holds its data volume. +4. In another terminal, start only Kibana and complete steps 4–6 above, keeping all application writers stopped: + + ```powershell + docker compose -f docker-compose.all-in-one.yml up -d --no-deps kibana + ``` + +5. Stop Kibana, then press Ctrl+C in the maintenance terminal and wait for Elasticsearch to exit cleanly. Pin the approved v9 all-in-one image and matching Kibana version. Repeat the Elasticsearch-only command to perform the 9 upgrade and verify health before any app startup; perform step 8 above against this node. +6. Stop that Elasticsearch-only container cleanly. Only after the migration checks pass, start the regular all-in-one service and matching Kibana with `docker compose -f docker-compose.all-in-one.yml up -d`. Resume external writers after application verification. Never use `down -v`, run two nodes against the same volume, or restart an 8.x image against a volume already opened by 9. + ## Upgrading from v7.1 to v8 We simplified the self hosting process by integrating the UI into the existing app images. As such `exceptionless/ui` docker images are deprecated and we recommend using `exceptionless/app`. diff --git a/docs/elasticsearch-9-production-migration.md b/docs/elasticsearch-9-production-migration.md new file mode 100644 index 0000000000..3bf7750e7c --- /dev/null +++ b/docs/elasticsearch-9-production-migration.md @@ -0,0 +1,142 @@ +# Elasticsearch 9 production migration plan + +Status: proposed; no production changes have been made. Inventory observations below are from September 6, 2026. This is an operator-run migration, not an application-startup migration. + +**Stack PR #2511 is an experiment only, not the production implementation or a planned production release.** Its queries, endpoint changes, and lookup schema are proof-of-concept evidence, not an approved architecture. The proper stack/event query refactor still needs to be designed and implemented in the Foundatio.Repositories PR, then integrated into Exceptionless through a separately reviewed application change. Neither the Elasticsearch 9 server upgrade nor index-format maintenance depends on shipping #2511. + +## Separate the three changes + +| Change | Release boundary | Required data work | +| --- | --- | --- | +| Run Elasticsearch 9 with existing application queries | Base PR [#2416](https://github.com/exceptionless/Exceptionless/pull/2416) alone | Resolve unsupported pre-8 indexes before starting 9; do not rewrite all 8-created indexes | +| Recreate indexes in the current server's index format | Explicit maintenance after the server upgrade stabilizes | Selected retained indexes, independently of application schema versions | +| Properly refactor stack/event queries and stack pagination | Future repositories-led implementation and separately reviewed Exceptionless integration; [#2511](https://github.com/exceptionless/Exceptionless/pull/2511) is experimental evidence only | Determine index/migration requirements from the approved repository design; validate mixed-generation event sources and production-scale queries | + +An Elasticsearch server version, `index.version.created`, the repository's schema version, and `index.mode` are different things. A force merge or server restart is not an index recreation. Do not bump the daily event schema version just to trigger a cluster-wide rewrite. + +The base PR retains the application query/index schema behavior and the Elasticsearch 8 client compatibility bridge; it must run independently of the experimental query services. There is no application-level legacy/JOIN switch. Deploy the base application release independently. Do not deploy the experiment; any future query release must use the approved repositories implementation and satisfy its own migration and validation gates. + +Elastic permits the supported previous-major index format on the next major. The documented LOOKUP JOIN constraint applies to the lookup-side index, not a blanket requirement to recreate every source event index. Confirm the exact expression joins used by the experiment against real 8-created event partitions on the target server before treating this as a production guarantee. + +### Local evidence on the rebased PRs + +Both solution builds passed with zero warnings/errors. The base PR independently passed all 829 API endpoint tests against the isolated custom Elasticsearch 9.5.0 image. + +A separate disposable cluster created `migration-events-v1` on 8.19.15 (`index.version.created=8537000`), then started 9.5.0 on that same fixture volume without recreating the event index. A newly created `migration-stacks-v2` had lookup mode and creation version `9107000`. The following expression join returned the expected `[1, "stack-1"]` row with `is_partial=false`: + +```esql +FROM migration-events-v1 +| LOOKUP JOIN migration-stacks-v2 ON stack_id == id AND is_deleted == false AND QSTR("status:open") +| WHERE id IS NOT NULL +| STATS total = COUNT(*) BY stack_id +| SORT total DESC, stack_id ASC +| LIMIT 26 +``` + +This proves mixed-generation support for that query shape, not production mapping coverage, throughput, or readiness. The restored production-data rehearsal remains required. + +## Observed production topology and missing sizing evidence + +- Production is green on Elasticsearch 8.19.15: four ready data/ingest/master nodes, each with 18 GiB container memory, a 9 GiB JVM heap, and a 600 GiB premium managed disk claim. +- Total provisioned storage is 2,400 GiB. This is **not** measured used storage or available migration headroom. Replica copies, shard placement, watermarks, growth, and recovery reserve all matter. +- The deployed ECK operator is 3.3.2. Production Kibana and the separate monitoring Elasticsearch cluster are on 8.19.15. +- The available Kubernetes identity can read resource metadata and logs, but cannot open an Elasticsearch port-forward or service proxy. Index creation versions, document counts, store sizes, shard distribution, and actual per-node free disk have **not** been measured. No duration or capacity promise is possible yet. + +Obtain an existing read-only Elasticsearch connection or have an operator export the following. Use monitoring/metadata privileges, not write or reindex privileges. Run requests against the intended cluster and record the cluster UUID; do not put credentials or complete source documents in reports. + +```http +GET / +GET /_cluster/health +GET /_cluster/settings?include_defaults=true&flat_settings=true +GET /_nodes/stats/fs,jvm,indices?filter_path=nodes.*.name,nodes.*.fs.total,nodes.*.jvm.mem,nodes.*.indices.indexing,nodes.*.indices.search,nodes.*.indices.merges +GET /_cat/allocation?format=json&bytes=b +GET /_cat/shards?format=json&bytes=b&h=index,shard,prirep,state,docs,store,node +GET /_cat/indices?format=json&bytes=b&expand_wildcards=all&h=index,health,status,pri,rep,docs.count,docs.deleted,pri.store.size,store.size +GET /_all/_settings?expand_wildcards=all&flat_settings=true&filter_path=*.settings.index.version.*,*.settings.index.mode,*.settings.index.number_of_*,*.settings.index.blocks.*,*.settings.index.lifecycle.*,*.settings.index.default_pipeline,*.settings.index.final_pipeline +GET /_all/_alias?expand_wildcards=all +GET /_migration/deprecations +GET /_snapshot/_all +GET /_slm/stats +``` + +System-index metadata may require an operator's separate access. Inventory those indexes through Upgrade Assistant rather than assuming application preflight covers them. Obtain latest successful snapshot details and a restore-test record; a configured repository alone does not prove a usable backup. + +Build a manifest with one row per concrete index: canonical aliases, application owner, creation version, mode, date partition, primary bytes, total bytes, exact live document count for migration candidates, shard/replica counts, retention deadline, last observed writes, pipelines, and chosen action. CAT document counts can include nested documents; use `_count` for copy verification. Capture mappings for selected pilot/copy candidates, including `_source` availability and dynamic fields. + +Measure normal and peak ingestion, search latency, disk growth, merge I/O, queue age, and recovery throughput over a representative workload window. Confirm configured maximum retention and actual cleanup behavior; do not assume an old daily partition is immutable or already expired. + +## Phase 1: rehearsal and pre-upgrade work on 8.19 + +1. Pin the exact approved target patch and container digest. The base PR now targets 9.5.3, and the separate 8.x rollout PR targets 8.19.21. These releases were published September 3 and September 2, 2026, respectively, and are less than two weeks old at this review: allow for elevated early-release risk in staging/soak acceptance. Review security fixes, known issues, plugins, client behavior, ECK support, and release-date upgrade compatibility before approving production. Test the same image that will be deployed, including the Exceptionless plugins. Earlier 9.5.0 test evidence above must not be treated as validation of 9.5.3. Do not silently substitute a production image during execution. +2. Patch Elasticsearch and Kibana to the latest approved 8.19 release first. Run Upgrade Assistant and resolve critical deprecations. Inventory all application, retained schema/error, hidden, monitoring, and system indexes. +3. Reindex any writable pre-8 application indexes **on Elasticsearch 8** before starting 9. Delete expired data only under the existing retention policy and explicit operator approval. Archive/read-only options are not replacements for writable Exceptionless indexes. Let Elastic's tooling own system-index migrations. +4. Restore a current snapshot to an isolated rehearsal cluster with matching topology/settings. Verify restore permissions, encryption keys, repository access, and recovery time. Restrict network access and apply production-data handling controls. +5. Run the base application on the rehearsed 8 cluster, upgrade that cluster to the target 9 patch, and rerun ingest, event/stack queries, stack status changes, jobs, saved views, aliases, retention, and deletion checks. Include existing records with old/missing fields and all retained creation versions. +6. As separate experimental research, create a lookup stack index in the rehearsal environment and test the experimental expression JOIN against **unchanged 8-created event indexes**. Compare status/deleted-stack filtering, tenant isolation, counts/charts, date boundaries, forward/backward cursors, and hydrated result identity. These findings inform the repositories PR; they neither approve the experiment for production nor block the independent base upgrade. Repeat correctness and performance validation against the eventual repository implementation. + +No production load tests or reindex experiments are authorized by this plan. + +## Phase 2: server upgrade with the base PR only + +1. Deploy and soak the base-compatible application independently of the server change while keeping production infrastructure pinned to 8.19. Coordinate GitOps/release manifests so an application deployment cannot unintentionally apply the major-version infrastructure change. +2. Approve rollback RPO/RTO and the treatment of writes accepted after the backup boundary. Take a fresh successful snapshot and verify the restore procedure. If replay of post-snapshot events is required, demonstrate durable queue retention/replay and idempotency first; do not assume the current pipeline can recreate every stack/status mutation. +3. Upgrade the monitoring cluster and supporting components in the supported order before the monitored production cluster where required. Keep Kibana matched to its Elasticsearch version. Follow the ECK rolling-upgrade procedure; do not hand-delete pods or change shard allocation independently of the operator without an approved runbook. +4. Upgrade one production node at a time. Because all four nodes have the same roles, confirm voting quorum and recovery capacity with one node unavailable. Wait for each node's shard recovery and health before proceeding. Halt on sustained unassigned shards, disk watermark pressure, write/search errors, queue growth, or latency outside the agreed SLO. +5. Validate the base application against the upgraded cluster with its existing queries. Do not deploy #2511 or run compatibility reindexing during this initial stabilization window. + +Rollback is **not** a downgrade of the upgraded disks or reverting the ECK version field. Recover on an older-version cluster from the pre-upgrade snapshot, with the approved replay/data-loss procedure. Preserve that recovery path until the upgrade has been accepted. + +## Phase 3: upgrade index formats without rewriting the world at once + +Prefer new daily event partitions created naturally on 9 and let eligible old partitions expire through verified retention. That upgrades the active data progressively with no bulk copy. Reindex only retained partitions that will outlive the agreed migration deadline or need a demonstrated format-specific feature. All remaining long-lived non-event indexes need an explicit current-format migration plan too. + +Blake's [Foundatio.Repositories PR #307](https://github.com/FoundatioFx/Foundatio.Repositories/pull/307), reviewed at `aec4fb78652c1dae8d08085e5895e28fdf10a2a5`, is a promising operator primitive, but is still open and is not integrated into the base PR's package. It separates compatibility upgrades from normal schema/configuration startup. It preserves canonical aliases while replacing physical indexes and exposes inspection/recovery for interrupted operations. + +Important constraints of that implementation: + +- It fences writes for the entire copy and requires writers, consumers, retention/index maintenance, and alias managers to be stopped. It is not a zero-downtime CDC or dual-write solution. +- It copies one exact index into `reindexed-v9-` using `_create_from` (an Elastic Technical Preview API), verifies the task, counts, mappings/settings, and alias topology, then atomically deletes the source and assigns its aliases to the target. +- Its copy task is unsliced and throttled. Do not size it assuming shard-parallel slicing. Partial/ambiguous operations require its evidence-based inspection/recovery, not blind retry or manual unblock. +- It rejects non-standard modes and several managed topologies. It preserves creation settings rather than changing them, so it does **not** implement the standard-to-lookup stack conversion. +- Canonical names become aliases. Prove exact-ID reads/patches, routing, mapping discovery, daily alias maintenance, old-schema discovery, retention deletion, and operational scripts still work. Restart/drain clients whose concurrency tokens refer to the retired physical index. + +Before adoption, finish review/release of #307, consume the approved package in a separate maintenance change, and test its failure/recovery cases with Exceptionless. Do not hide it in `ConfigureIndexesAsync` or add an unconditional global schema bump. + +The reviewed #307 scope is index-format maintenance. This plan does not claim that it already contains the proper stack query refactor. That query design and implementation remain work to resolve in the repositories PR, independently of the reindex primitive. + +For each candidate: + +1. Start with a small representative partition, then a larger/high-field-count partition. Measure sustained copy throughput **with** the intended production workload and throttle on the rehearsal cluster. +2. Reserve capacity for the complete target plus configured replica restoration, temporary segment/merge overhead, ingestion growth, and a node-recovery margin. Check fit per node and per shard against configured watermarks, not just aggregate free bytes. Provision additional capacity before copying if needed. +3. Stop all affected writers and index managers; drain in-flight work. Historical event dates can still receive late ingestion, deletions, and cleanup. A timestamp alone does not establish safety. If writers cannot be paused per partition with proven routing and queueing, the library's current safe procedure implies a broader maintenance outage. Decide that tradeoff explicitly before scheduling a massive copy. +4. Recompute the preflight; snapshot; copy only the approved concrete index with conservative throttling. Persist task identity, progress, baseline counts, aliases/settings, and recovery evidence outside the process. +5. Verify completion, exact counts, representative content/queries, replica recovery, alias identity, and application read/write/delete behavior. Release the write pause only after these gates pass. Then proceed to the next index; keep concurrency at one initially. +6. Abort/pause on the agreed disk/latency/queue limits. Use the library's inspection and verified cancellation path. A completed atomic cutover cannot be undone by canceling the task; snapshot restore is the recovery boundary. + +Sizing worksheet: `copy duration ≈ primary source bytes / measured effective source-byte throughput`, plus refresh, validation, replica recovery, and cutover. Alternatively use exact documents divided by measured documents/second for matching document distributions. Estimate the **write-pause duration**, queue accumulation (`arrival rate × pause duration`), and catch-up time separately. Do not estimate from raw disk bandwidth or promise a universal 2× free-space rule. + +## Phase 4: future repositories-led query refactor, not deployment of #2511 + +First settle the production query design in the Foundatio.Repositories PR: repository-level filtering/JOIN composition, grouped stack queries and counts, sorting/cursor semantics, result contracts, and index lifecycle support. Exceptionless should consume that capability rather than promote the experiment's application-side ES|QL service into the production architecture. Retain the intended endpoint ownership: stacks come from stacks endpoints, events from events endpoints, without application-side filtering joins. + +Review and release that repository implementation, then create a separately reviewed Exceptionless integration and migration plan. #2511 is only a source of feasibility evidence and regression scenarios. Passing its tests is not approval to merge or deploy it as the production refactor. + +The experiment defines stack schema version 2 with `index.mode=lookup` and one primary shard. These are provisional choices, not production migration instructions. The following requirements apply **only if the approved repositories design retains that lookup topology**; revise them if the design changes. Do not initiate this conversion through ordinary startup. + +1. Measure the current stack primary size, document cardinality, growth, update rate, largest tenant, and heap needed by representative joins. The single lookup primary is a hard capacity/throughput constraint; replicas can distribute reads but do not shard primary writes. Establish whether this design fits the forecast, not just today's sample. +2. Use a dedicated, reviewed schema-conversion maintenance operation that creates the approved repository implementation's mapping/settings and copies existing stack documents without changing IDs or relationships. It must handle partial copies, missing/default fields, retained aliases, and rollback. #307's currently reviewed format upgrade is not this operation. Avoid copying stacks twice merely to reach current format and then lookup mode. +3. Pause ingestion consumers and all stack-mutating APIs/jobs/maintenance, drain in-flight writes, snapshot, copy and verify, then perform an explicit atomic alias cutover. Rehearse full outage/recovery behavior; do not assume the existing generic schema reindexer's catch-up pass proves no missed deletes or concurrent status changes. +4. Before allowing traffic, verify the actual lookup mode, mapping, one-primary setting, complete stack IDs/counts, aliases, replica health, and existing-record semantics. Deploy only the separately approved repository-backed application implementation after migration success, never the experimental PR. Keep the base release available as an application rollback candidate, but rehearse its writes against the new mapping; application rollback does not revert the index conversion. +5. Benchmark representative tenant/time-range/skew combinations for event status filtering and stack grouping/count/charts/paging. Capture p50/p95/p99 latency, CPU, heap/breakers, I/O, concurrent ingestion impact, and cursor correctness under changes. Cursor pagination does not remove the cost of filtering/grouping the qualifying event population, and it is not a point-in-time snapshot. +6. Require correctness and SLO acceptance before the JOIN production rollout. If the canonical one-primary stack index does not fit, stop and redesign the lookup topology; do not deploy on the strength of tiny local benchmarks. + +## Decisions required before scheduling + +- Read-only index/disk/snapshot inventory and representative workload measurements. +- Approved source/target patches, image digests, ECK/component compatibility, and rehearsal evidence. +- Retention-based completion deadline versus retained event partitions that must be copied. +- Additional capacity and acceptable per-index/global write outage, including queue/replay limits. +- Adoption of #307's reindex capability where needed; completion of the proper stack/event query refactor in the repositories PR, followed by separately reviewed Exceptionless integration and any required schema migration. #2511 is not a production deliverable. +- Restore-tested RPO/RTO, cutover/abort thresholds, and named operator/approval owner for each stage. + +References: Elastic's [upgrade preparation](https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade), [rolling upgrade and rollback guidance](https://www.elastic.co/docs/deploy-manage/upgrade/deployment-or-cluster/elasticsearch), [LOOKUP JOIN constraints](https://www.elastic.co/docs/reference/query-languages/esql/esql-lookup-join), and [snapshot compatibility](https://www.elastic.co/docs/deploy-manage/tools/snapshot-and-restore). diff --git a/k8s/elastic-monitor.yaml b/k8s/elastic-monitor.yaml index 943cde867d..7da6e53e19 100644 --- a/k8s/elastic-monitor.yaml +++ b/k8s/elastic-monitor.yaml @@ -4,7 +4,7 @@ metadata: name: elastic-monitor namespace: elastic-system spec: - version: 8.19.15 + version: 9.5.3 podDisruptionBudget: {} nodeSets: - name: main @@ -228,7 +228,7 @@ metadata: name: kibana-monitor namespace: elastic-system spec: - version: 8.19.15 + version: 9.5.3 count: 1 http: tls: @@ -364,7 +364,7 @@ metadata: name: fleet-server namespace: elastic-system spec: - version: 8.19.15 + version: 9.5.3 kibanaRef: name: kibana-monitor elasticsearchRefs: @@ -388,7 +388,7 @@ metadata: name: elastic-agent namespace: elastic-system spec: - version: 8.19.15 + version: 9.5.3 kibanaRef: name: kibana-monitor fleetServerRef: diff --git a/k8s/ex-dev-elasticsearch.yaml b/k8s/ex-dev-elasticsearch.yaml index 46f14cc5eb..532328e54e 100644 --- a/k8s/ex-dev-elasticsearch.yaml +++ b/k8s/ex-dev-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 8.19.15 - image: exceptionless/elasticsearch:8.19.15 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.5.3 + image: exceptionless/elasticsearch:9.5.3 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch secureSettings: - secretName: ex-dev-snapshots http: @@ -68,7 +68,7 @@ metadata: name: ex-dev namespace: ex-dev spec: - version: 8.19.15 + version: 9.5.3 count: 1 elasticsearchRef: name: ex-dev diff --git a/k8s/ex-prod-elasticsearch.yaml b/k8s/ex-prod-elasticsearch.yaml index f159ae3046..155cddc58c 100644 --- a/k8s/ex-prod-elasticsearch.yaml +++ b/k8s/ex-prod-elasticsearch.yaml @@ -14,8 +14,8 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 8.19.15 - image: exceptionless/elasticsearch:8.19.15 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch + version: 9.5.3 + image: exceptionless/elasticsearch:9.5.3 # https://github.com/exceptionless/Exceptionless/tree/main/build/docker/elasticsearch monitoring: metrics: elasticsearchRefs: @@ -79,7 +79,7 @@ metadata: name: ex-prod namespace: ex-prod spec: - version: 8.19.15 + version: 9.5.3 count: 1 elasticsearchRef: name: ex-prod diff --git a/k8s/exceptionless/values.yaml b/k8s/exceptionless/values.yaml index db1c202926..0b55332862 100644 --- a/k8s/exceptionless/values.yaml +++ b/k8s/exceptionless/values.yaml @@ -36,7 +36,7 @@ elasticsearch: connectionString: image: repository: exceptionless/elasticsearch - tag: 8.19.15 + tag: 9.5.3 pullPolicy: IfNotPresent redis: diff --git a/samples/docker-compose.all-in-one.yml b/samples/docker-compose.all-in-one.yml index 5b1caf2d42..b4148c5f69 100644 --- a/samples/docker-compose.all-in-one.yml +++ b/samples/docker-compose.all-in-one.yml @@ -19,10 +19,12 @@ services: # Runs Kibana for working with Elasticsearch data directly. This is normally not needed and takes up resources when running. kibana: depends_on: - - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + - exceptionless + image: docker.elastic.co/kibana/kibana:9.5.3 ports: - 5601:5601 + environment: + ELASTICSEARCH_HOSTS: http://exceptionless:9200 volumes: ex_esdata: diff --git a/samples/docker-compose.yml b/samples/docker-compose.yml index d73e0518c9..b6101dd41d 100644 --- a/samples/docker-compose.yml +++ b/samples/docker-compose.yml @@ -44,7 +44,7 @@ services: - ex_appdata:/app/storage elasticsearch: - image: exceptionless/elasticsearch:8.19.15 + image: exceptionless/elasticsearch:9.5.3 environment: discovery.type: single-node xpack.security.enabled: "false" @@ -58,7 +58,7 @@ services: kibana: depends_on: - elasticsearch - image: docker.elastic.co/kibana/kibana:8.19.15 + image: docker.elastic.co/kibana/kibana:9.5.3 ports: - 5601:5601 diff --git a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs index ec5c5e4031..ed25ed161d 100644 --- a/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs +++ b/src/Exceptionless.AppHost/Extensions/ElasticsearchExtensions.cs @@ -13,7 +13,7 @@ public static class ElasticsearchBuilderExtensions private const int KibanaPort = 5601; /// - /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 8.19.15 tag of the Elasticsearch container image + /// Adds a Elasticsearch container to the application model. The default image is "docker.elastic.co/elasticsearch/elasticsearch". This version the package defaults to the 9.5.3 tag of the Elasticsearch container image /// /// The . /// The name of the resource. This name will be used as the connection string name when referenced in a dependency. @@ -60,7 +60,11 @@ public static IResourceBuilder AddElasticsearch(this IDis .PublishAsConnectionString(); } - public static IResourceBuilder WithKibana(this IResourceBuilder builder, Action>? configureContainer = null, string? containerName = null) + public static IResourceBuilder WithKibana( + this IResourceBuilder builder, + Action>? configureContainer = null, + string? containerName = null, + int? port = null) { ArgumentNullException.ThrowIfNull(builder); @@ -79,7 +83,7 @@ public static IResourceBuilder WithKibana(this IResourceB var resourceBuilder = builder.ApplicationBuilder.AddResource(resource) .WithImage(ElasticsearchContainerImageTags.KibanaImage, ElasticsearchContainerImageTags.Tag) .WithImageRegistry(ElasticsearchContainerImageTags.KibanaRegistry) - .WithHttpEndpoint(targetPort: KibanaPort, name: containerName) + .WithHttpEndpoint(targetPort: KibanaPort, port: port, name: containerName) .WithUrlForEndpoint(containerName, u => u.DisplayText = "Kibana") .WithEnvironment("xpack.security.enabled", "false") .WithEnvironment(ctx => @@ -121,7 +125,7 @@ internal static class ElasticsearchContainerImageTags public const string Image = "exceptionless/elasticsearch"; public const string KibanaRegistry = "docker.elastic.co"; public const string KibanaImage = "kibana/kibana"; - public const string Tag = "8.19.15"; + public const string Tag = "9.5.3"; } internal sealed class ElasticsearchConnectionHealthCheck(Func connectionStringFactory) : IHealthCheck @@ -134,9 +138,17 @@ public async Task CheckHealthAsync(HealthCheckContext context using var settings = new ElasticsearchClientSettings(new Uri(connectionString)); var client = new ElasticsearchClient(settings); - var response = await client.PingAsync(cancellationToken); - return response.IsValidResponse - ? HealthCheckResult.Healthy() - : new HealthCheckResult(context.Registration.FailureStatus, $"Elasticsearch ping failed: {response.DebugInformation}"); + var response = await client.Cluster.HealthAsync( + request => request.WaitForStatus(Elastic.Clients.Elasticsearch.HealthStatus.Yellow), + cancellationToken); + bool isReady = response.IsValidResponse + && !response.TimedOut + && response.Status is Elastic.Clients.Elasticsearch.HealthStatus.Yellow or Elastic.Clients.Elasticsearch.HealthStatus.Green; + if (isReady) + return HealthCheckResult.Healthy(); + + return new HealthCheckResult( + context.Registration.FailureStatus, + $"Elasticsearch cluster health check failed. Timed out: {response.TimedOut}; status: {response.Status}. {response.DebugInformation}"); } } diff --git a/src/Exceptionless.AppHost/Program.cs b/src/Exceptionless.AppHost/Program.cs index b12b48aefe..ef33ee8333 100644 --- a/src/Exceptionless.AppHost/Program.cs +++ b/src/Exceptionless.AppHost/Program.cs @@ -13,6 +13,13 @@ bool servicesOnly = HasArgument("--services-only"); bool ciE2E = HasArgument("--ci-e2e"); bool includeDevTools = !ciE2E; +int elasticsearchPort = GetPort("Elasticsearch:Port", 9200); +string elasticsearchImageTag = builder.Configuration["Elasticsearch:ImageTag"] ?? ElasticsearchContainerImageTags.Tag; +string kibanaImageTag = builder.Configuration["Elasticsearch:KibanaImageTag"] ?? ElasticsearchContainerImageTags.Tag; +string elasticsearchContainerName = builder.Configuration["Elasticsearch:ContainerName"] ?? "Exceptionless-Elasticsearch"; +string elasticsearchDataVolume = builder.Configuration["Elasticsearch:DataVolume"] ?? "exceptionless.data.v1"; +string kibanaContainerName = builder.Configuration["Elasticsearch:KibanaContainerName"] ?? "Exceptionless-Kibana"; +int kibanaPort = GetPort("Elasticsearch:KibanaPort", 5601); int oldAppHttpPort = worktreePorts?.OldAppHttp ?? 7120; int oldAppPort = worktreePorts?.OldAppHttps ?? 7121; int oldAppLiveReloadPort = worktreePorts?.OldAppLiveReload ?? 35729; @@ -24,8 +31,9 @@ string exceptionlessServerUrl = worktreePorts?.ApiHttpsUrl ?? $"https://api-ex.dev.localhost:{DefaultApiHttpsPort}"; const string SharedEmailConnectionString = "smtp://localhost:1026"; -var elastic = builder.AddElasticsearch("Elasticsearch", port: 9200) - .WithDataVolume("exceptionless.data.v1") +var elastic = builder.AddElasticsearch("Elasticsearch", port: elasticsearchPort) + .WithImageTag(elasticsearchImageTag) + .WithDataVolume(elasticsearchDataVolume) .WithEndpointProxySupport(false); var storage = builder.AddAzureStorage("Storage") @@ -70,15 +78,17 @@ var ownedElastic = elastic; elastic = ownedElastic .WithLifetime(ContainerLifetime.Persistent) - .WithContainerName("Exceptionless-Elasticsearch"); + .WithContainerName(elasticsearchContainerName); if (!servicesOnly && includeDevTools) { elastic = elastic.WithKibana(b => b + .WithImageTag(kibanaImageTag) .WithLifetime(ContainerLifetime.Persistent) .WithEndpointProxySupport(false) - .WithContainerName("Exceptionless-Kibana") - .WithParentRelationship(ownedElastic)); + .WithContainerName(kibanaContainerName) + .WithParentRelationship(ownedElastic), + port: kibanaPort); } var ownedCache = cache; @@ -242,3 +252,15 @@ await builder.Build().RunAsync(); bool HasArgument(string name) => args.Any(arg => StringComparer.OrdinalIgnoreCase.Equals(arg, name) || StringComparer.OrdinalIgnoreCase.Equals(arg, name.TrimStart('-'))); + +int GetPort(string key, int defaultValue) +{ + string? value = builder.Configuration[key]; + if (String.IsNullOrWhiteSpace(value)) + return defaultValue; + + if (!Int32.TryParse(value, out int port) || port is < 1 or > 65535) + throw new InvalidOperationException($"Configuration value '{key}' must be a valid TCP port."); + + return port; +} diff --git a/tests/Exceptionless.Tests/AppHostConfigurationTests.cs b/tests/Exceptionless.Tests/AppHostConfigurationTests.cs new file mode 100644 index 0000000000..04087739c9 --- /dev/null +++ b/tests/Exceptionless.Tests/AppHostConfigurationTests.cs @@ -0,0 +1,30 @@ +using Aspire.Hosting; +using Aspire.Hosting.ApplicationModel; +using Aspire.Hosting.Testing; +using Xunit; + +namespace Exceptionless.Tests; + +public class AppHostConfigurationTests +{ + [Fact] + public async Task CreateAsync_WithSeparateElasticsearchAndKibanaOverrides_UsesIndependentImageTags() + { + const string elasticsearchImageTag = "9.5.3-sha256-candidate"; + const string kibanaImageTag = "9.5.3"; + var appHost = await DistributedApplicationTestingBuilder.CreateAsync( + [ + $"--Elasticsearch:ImageTag={elasticsearchImageTag}", + $"--Elasticsearch:KibanaImageTag={kibanaImageTag}" + ], + TestContext.Current.CancellationToken); + + var elasticsearch = Assert.Single(appHost.Resources.OfType()); + var kibana = Assert.Single(appHost.Resources.OfType()); + var elasticsearchImage = Assert.Single(elasticsearch.Annotations.OfType()); + var kibanaImage = Assert.Single(kibana.Annotations.OfType()); + + Assert.Equal(elasticsearchImageTag, elasticsearchImage.Tag); + Assert.Equal(kibanaImageTag, kibanaImage.Tag); + } +} diff --git a/tests/Exceptionless.Tests/AppWebHostFactory.cs b/tests/Exceptionless.Tests/AppWebHostFactory.cs index c08f21687f..8f6fbdbd75 100644 --- a/tests/Exceptionless.Tests/AppWebHostFactory.cs +++ b/tests/Exceptionless.Tests/AppWebHostFactory.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using System.Net; +using System.Text.Json; using Aspire.Hosting; using Aspire.Hosting.Testing; using Exceptionless.Core; @@ -21,7 +22,7 @@ namespace Exceptionless.Tests; public class AppWebHostFactory : WebApplicationFactory, IAsyncLifetime { - private const string SharedElasticsearchUrl = "http://localhost:9200"; + private static readonly string SharedElasticsearchUrl = GetSharedElasticsearchUrl(); private static readonly TimeSpan SharedElasticsearchStartupTimeout = TimeSpan.FromMinutes(3); private static int s_counter = -1; private static readonly Lazy> s_sharedAppHost = new(StartSharedAppHostAsync, LazyThreadSafetyMode.ExecutionAndPublication); @@ -58,22 +59,40 @@ private static async Task StartSharedAppHostAsync() return app; } + private static string GetSharedElasticsearchUrl() + { + const int defaultPort = 9200; + string? configuredPort = Environment.GetEnvironmentVariable("Elasticsearch__Port"); + if (String.IsNullOrWhiteSpace(configuredPort)) + return $"http://localhost:{defaultPort}"; + + if (!Int32.TryParse(configuredPort, out int port) || port is < 1 or > 65535) + throw new InvalidOperationException("Environment variable 'Elasticsearch__Port' must be a valid TCP port."); + + return $"http://localhost:{port}"; + } + private static async Task WaitForElasticsearchAsync(Uri elasticsearchUri) { - using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(1) }; + using var client = new HttpClient { Timeout = TimeSpan.FromSeconds(2) }; var deadline = TimeProvider.System.GetUtcNow() + SharedElasticsearchStartupTimeout; + var healthUri = new Uri(elasticsearchUri, "/_cluster/health?wait_for_status=yellow&timeout=1s"); while (TimeProvider.System.GetUtcNow() < deadline) { try { - using var response = await client.GetAsync(elasticsearchUri); - if (response.StatusCode == HttpStatusCode.OK) + using var response = await client.GetAsync(healthUri); + using var document = await JsonDocument.ParseAsync(await response.Content.ReadAsStreamAsync()); + if (IsElasticsearchReady(response.StatusCode, document.RootElement)) return; } catch (HttpRequestException) { } + catch (JsonException) + { + } catch (TaskCanceledException) { } @@ -84,6 +103,20 @@ private static async Task WaitForElasticsearchAsync(Uri elasticsearchUri) throw new TimeoutException("Timed out waiting for the shared Elasticsearch container to be ready."); } + internal static bool IsElasticsearchReady(HttpStatusCode statusCode, JsonElement health) + { + if (statusCode != HttpStatusCode.OK) + return false; + + bool requestCompleted = health.TryGetProperty("timed_out", out var timedOut) + && timedOut.ValueKind == JsonValueKind.False; + bool clusterReady = health.TryGetProperty("status", out var status) + && status.ValueKind == JsonValueKind.String + && status.GetString() is "yellow" or "green"; + + return requestCompleted && clusterReady; + } + protected override void ConfigureWebHost(IWebHostBuilder builder) { builder.UseEnvironment(Environments.Development); diff --git a/tests/Exceptionless.Tests/AppWebHostFactoryTests.cs b/tests/Exceptionless.Tests/AppWebHostFactoryTests.cs index 7fef487faa..539785b681 100644 --- a/tests/Exceptionless.Tests/AppWebHostFactoryTests.cs +++ b/tests/Exceptionless.Tests/AppWebHostFactoryTests.cs @@ -1,4 +1,6 @@ +using System.Net; using System.Text; +using System.Text.Json; using Foundatio.Storage; using Microsoft.Extensions.DependencyInjection; using Xunit; @@ -7,6 +9,22 @@ namespace Exceptionless.Tests; public sealed class AppWebHostFactoryTests { + [Theory] + [InlineData(HttpStatusCode.OK, """{"timed_out":false,"status":"yellow"}""", true)] + [InlineData(HttpStatusCode.OK, """{"timed_out":false,"status":"green"}""", true)] + [InlineData(HttpStatusCode.OK, """{"timed_out":true,"status":"yellow"}""", false)] + [InlineData(HttpStatusCode.OK, """{"timed_out":false,"status":"red"}""", false)] + [InlineData(HttpStatusCode.OK, """{"timed_out":false,"status":1}""", false)] + [InlineData(HttpStatusCode.ServiceUnavailable, """{"timed_out":false,"status":"yellow"}""", false)] + public void IsElasticsearchReady_ClusterHealthResponse_ReturnsExpectedResult(HttpStatusCode statusCode, string json, bool expected) + { + using var document = JsonDocument.Parse(json); + + bool isReady = AppWebHostFactory.IsElasticsearchReady(statusCode, document.RootElement); + + Assert.Equal(expected, isReady); + } + [Fact] public async Task ConfigureWebHost_MultipleFactories_IsolatesFileStorageByAppScope() {