diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index a8089cef..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,438 +0,0 @@ -image: ${REGISTRY_DOCKER_URL}/stackstate/stackstate-agent-integrations-runner:${RUNNER_IMAGE_TAG} -stages: - - build - - test - - publish - -variables: - CONDA_ENV: "stackstate-agent-integrations-py-3" - PYTHON_VERSION: 3.13.14 - RUNNER_IMAGE_TAG: "20260625-py313" - -.rules: - - &pull_requests - if: $CI_EXTERNAL_PULL_REQUEST_IID - - &base_changes - if: $CI_EXTERNAL_PULL_REQUEST_IID - changes: - - stackstate_checks_base/* - - stackstate_checks_base/**/* - - stackstate_checks_dev/* - - stackstate_checks_dev/**/* - - stackstate_checks_tests_helper/* - - stackstate_checks_tests_helper/**/* - - .gitlab-ci.yml - - $CHECK/* - - $CHECK/**/* - - &master_branch - if: '$CI_COMMIT_BRANCH == "stackstate-7.78.2"' - - &release_branch - if: $CI_COMMIT_TAG - - &base_manual_changes - - if: $CI_EXTERNAL_PULL_REQUEST_IID - changes: - - stackstate_checks_base/* - - stackstate_checks_base/**/* - - stackstate_checks_dev/* - - stackstate_checks_dev/**/* - - stackstate_checks_tests_helper/* - - stackstate_checks_tests_helper/**/* - - .gitlab-ci.yml - - $CHECK/* - - $CHECK/**/* - when: manual - allow_failure: true - - &splunk_base_build_rule - if: $CI_EXTERNAL_PULL_REQUEST_IID - changes: - - splunk_base/* - - splunk_base/**/* - - &master_manual_branch - if: '$CI_COMMIT_BRANCH == "stackstate-7.78.2"' - when: manual - allow_failure: true - - &release_manual_branch - if: $CI_COMMIT_TAG - when: manual - allow_failure: true - -.base_integration_rules: - &base_integration_rules - rules: - - <<: *base_changes - - <<: *master_branch - - <<: *release_branch - -.base_splunk_integration_rules: - &base_splunk_integration_rules - rules: - - <<: *base_changes - - <<: *master_branch - - <<: *release_branch - - <<: *splunk_base_build_rule - -.base_integration_manual_rules: - &base_integration_manual_rules - rules: - - <<: *base_manual_changes - - <<: *master_manual_branch - - <<: *release_manual_branch - -.base_job_rules: - &base_job_rules - rules: - - <<: *pull_requests - - <<: *master_branch - - <<: *release_branch - -.manual_job_rules: - &manual_job_rules - rules: - - <<: *pull_requests - when: manual - allow_failure: true - - <<: *master_branch - when: manual - allow_failure: true - - <<: *release_branch - when: manual - allow_failure: true - -.linux_env: &linux_env - before_script: - - source .setup-scripts/setup_env.sh - -.windows_env: &windows_env - tags: - - windows_agent7_ps1 - before_script: - - Import-Module C:\tools\miniconda3\shell\condabin\conda-hook.ps1 - - ./.setup-scripts/conda_env.ps1 $CONDA_ENV $PYTHON_VERSION - - conda activate $CONDA_ENV - - python -m pip install .\stackstate_checks_dev[cli] - -print_env: - <<: *linux_env - <<: *manual_job_rules - stage: build - script: - - printenv - interruptible: true - -docker: - image: ${REGISTRY_DOCKER_URL}/library/docker:20-git - stage: build - rules: - - if: $CI_COMMIT_MESSAGE =~ /Build new docker image/i - when: on_success - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - changes: - - .setup-scripts/image/**/* - - .setup-scripts/setup_env.sh - when: manual - allow_failure: true - - if: $CI_EXTERNAL_PULL_REQUEST_IID - when: manual - allow_failure: true - - if: '$CI_COMMIT_BRANCH == "stackstate-7.78.2"' - when: manual - allow_failure: true - - if: $CI_COMMIT_TAG - when: manual - allow_failure: true - script: - - echo "${docker_password}" | docker login --username=${docker_user} --password-stdin docker.io - - echo "${quay_password}" | docker login --username=${quay_user} --password-stdin quay.io - - echo "${REGISTRY_PASSWORD}" | docker login --username=${REGISTRY_USER} --password-stdin ${REGISTRY_HOST} - - apk add make - - cd .setup-scripts/image - # Pull base from docker.io during build; mirror may not have 3.13.14-bookworm yet. - - make build push PYTHON_BASE_IMAGE=python:3.13.14-bookworm - - RUNNER_TAG=$(make -s print_tag) - - echo "Published ${REGISTRY_DOCKER_URL}/stackstate/stackstate-agent-integrations-runner:${RUNNER_TAG}" - - echo "Set RUNNER_IMAGE_TAG to ${RUNNER_TAG} (setup_env.sh/conda_env.ps1 already target python3.13/3.13.14), then re-run the pipeline." - - | - if [ "${CI_COMMIT_REF_NAME}" = "stackstate-7.78.2" ]; then - make tag_latest push_latest - fi - services: - - alias: docker - command: - - --experimental - - --tls=false - name: ${REGISTRY_DOCKER_URL}/library/docker:20-dind - variables: - DOCKER_DRIVER: overlay2 - DOCKER_HOST: tcp://docker:2375 - DOCKER_TLS_CERTDIR: "" - - -linux_deps: - <<: *linux_env - <<: *base_job_rules - stage: build - interruptible: true - script: - - source .setup-scripts/load_deps.sh - artifacts: - paths: - - venv/ - expire_in: 1 week - -.linux_test: &linux_test - <<: *linux_env - <<: *base_integration_rules - stage: test - needs: - - linux_deps - services: - - alias: docker - command: - - --experimental - - --tls=false - name: ${REGISTRY_DOCKER_URL}/library/docker:24-dind - script: - - .setup-scripts/setup_artifact_registry.sh - - .setup-scripts/setup_artifactory_docker.sh - - export COMPOSE_HTTP_TIMEOUT=300 # Increase timeout in case containers are slow to start up. - - checksdev test --cov ${CHECK} - - checksdev test ${CHECK} --bench || true - # [BS] we skip this until https://github.com/PyCQA/pylint/pull/2681/commits/253e5d05e3943d3bc8e4470804a138cb20049548 lands in pylint stable version - # - if [ -n "$PYTHON3" ]; then checksdev validate py3 ${CHECK}; fi - interruptible: true - -.linux_splunk_test: &linux_splunk_test - <<: *linux_test - <<: *base_splunk_integration_rules - script: - - .setup-scripts/setup_artifact_registry.sh - - .setup-scripts/setup_artifactory_docker.sh - - docker pull registry.tooling.stackstate.io/docker/splunk/splunk:latest # Pull splunk to aovid pulling during compose, which breaks in python - - export COMPOSE_HTTP_TIMEOUT=300 # Increase timeout in case splunk is slow to start up. - - checksdev test --cov ${CHECK} - - checksdev test ${CHECK} --bench || true - # [BS] we skip this until https://github.com/PyCQA/pylint/pull/2681/commits/253e5d05e3943d3bc8e4470804a138cb20049548 lands in pylint stable version - # - if [ -n "$PYTHON3" ]; then checksdev validate py3 ${CHECK}; fi - -test_stackstate_checks_base: - <<: *linux_test - variables: - CHECK: "stackstate_checks_base" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - script: - - checksdev validate config - - checksdev validate dep - - checksdev validate manifest --include-extras - - checksdev validate metadata - - checksdev validate service-checks - - checksdev test --cov ${CHECK} - - checksdev test ${CHECK} --bench || true - -test_stackstate_checks_dev: - <<: *linux_test - variables: - CHECK: "stackstate_checks_dev" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - tags: - - sts-k8s-xl-no-docker-runner - -test_dynatrace_base: - <<: *linux_test - variables: - CHECK: "dynatrace_base" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_dynatrace_topology: - <<: *linux_test - variables: - CHECK: "dynatrace_topology" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_dynatrace_health: - <<: *linux_test - variables: - CHECK: "dynatrace_health" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_kubelet: - <<: *linux_test - variables: - CHECK: "kubelet" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_openmetrics: - <<: *linux_test - variables: - CHECK: "openmetrics" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_postgres: - <<: *linux_test - variables: - CHECK: "postgres" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - tags: - - sts-k8s-xl-no-docker-runner - -test_splunk_topology: - <<: *linux_splunk_test - variables: - CHECK: "splunk_topology" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - tags: - - sts-k8s-xl-no-docker-runner - -test_splunk_health: - <<: *linux_splunk_test - variables: - CHECK: "splunk_health" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - tags: - - sts-k8s-xl-no-docker-runner - -test_splunk_metric: - <<: *linux_splunk_test - variables: - CHECK: "splunk_metric" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - tags: - - sts-k8s-xl-no-docker-runner - -test_agent_integration_sample: - <<: *linux_test - variables: - CHECK: "agent_integration_sample" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_agent_v2_integration_sample: - <<: *linux_test - variables: - CHECK: "agent_v2_integration_sample" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_agent_v2_integration_stateful_sample: - <<: *linux_test - variables: - CHECK: "agent_v2_integration_stateful_sample" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_agent_v2_integration_transactional_sample: - <<: *linux_test - variables: - CHECK: "agent_v2_integration_transactional_sample" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_vsphere: - <<: *linux_test - variables: - CHECK: "vsphere" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_zabbix: - <<: *linux_test - variables: - CHECK: "zabbix" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_splunk_base: - <<: *linux_splunk_test - variables: - CHECK: "splunk_base" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - DOCKER_TLS_CERTDIR: "" - tags: - - sts-k8s-xl-no-docker-runner - -test_servicenow: - <<: *linux_test - variables: - CHECK: "servicenow" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_static_topology: - <<: *linux_test - variables: - CHECK: "static_topology" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -test_static_health: - <<: *linux_test - variables: - CHECK: "static_health" - PYTHON3: "true" - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - -publish-checks-dev: - <<: *linux_env - stage: publish - interruptible: true - needs: - - job: linux_deps - artifacts: true - script: - - .setup-scripts/setup_artifact_registry.sh - - .setup-scripts/setup_artifact_publishing.sh - - export VERSION=`./.setup-scripts/version.sh` - - echo "__version__ = \"$VERSION\"" > stackstate_checks_dev/stackstate_checks/dev/__about__.py - - pip install twine - - cd stackstate_checks_dev && python setup.py sdist bdist_wheel && twine upload --repository gitlab dist/* - rules: - - if: '$CI_COMMIT_BRANCH =~ /^STAC-/' - when: manual - allow_failure: true - - <<: *pull_requests - when: manual - allow_failure: true - - <<: *master_branch - when: manual - allow_failure: true - - <<: *release_branch - when: manual - allow_failure: true diff --git a/.setup-scripts/image/Dockerfile b/.setup-scripts/image/Dockerfile deleted file mode 100644 index aa9de252..00000000 --- a/.setup-scripts/image/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# Match agent embedded CPython 3.13.14. Docker Hub publishes 3.13.14 on bookworm, not bullseye. -# CI `docker` job passes PYTHON_BASE_IMAGE=python:3.13.14-bookworm (docker.io is logged in there). -ARG PYTHON_BASE_IMAGE=registry.tooling.stackstate.io/docker/python:3.13.14-bookworm -FROM ${PYTHON_BASE_IMAGE} -ARG PIP_VERSION=24.3.1 -ARG SETUPTOOLS_VERSION=75.8.2 -RUN apt-get update && \ - apt-get install -y apt-transport-https ca-certificates curl gnupg2 lsb-release && \ - apt-get clean && \ - pip3 install "pip==${PIP_VERSION}" "setuptools==${SETUPTOOLS_VERSION}" virtualenv codecov wheel - -RUN install -m 0755 -d /etc/apt/keyrings && \ - curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \ - chmod a+r /etc/apt/keyrings/docker.asc && \ - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \ - apt-get update && \ - apt-get install -y docker-ce docker-ce-cli containerd.io && \ - apt-get clean - -RUN curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose -RUN chmod +x /usr/local/bin/docker-compose diff --git a/.setup-scripts/image/Makefile b/.setup-scripts/image/Makefile deleted file mode 100644 index 644bcc65..00000000 --- a/.setup-scripts/image/Makefile +++ /dev/null @@ -1,41 +0,0 @@ -RUNTIMETAG := $(shell date +%Y%m%d) -PYTHON_SUFFIX := py313 -IMAGE_REPO ?= stackstate/stackstate-agent-integrations-runner -# CI `docker` job overrides with python:3.13.14-bookworm (docker.io login); local default is the mirror. -PYTHON_BASE_IMAGE ?= registry.tooling.stackstate.io/docker/python:3.13.14-bookworm -include ../python_tool_versions.env -export - -#build: -# docker build -t stackstate/stackstate-agent-integrations-runner:$(RUNTIMETAG) . -#push: -# docker push stackstate/stackstate-agent-integrations-runner:$(RUNTIMETAG) -# -#tag_latest: -# docker tag stackstate/stackstate-agent-integrations-runner:$(RUNTIMETAG) stackstate/stackstate-agent-integrations-runner:latest -# -#push_latest: -# docker push stackstate/stackstate-agent-integrations-runner:latest - -# ^^^ Old runner -# ---------------------------------------------- -# vvv New runner - -build: - docker build --build-arg PYTHON_BASE_IMAGE=$(PYTHON_BASE_IMAGE) \ - --build-arg PIP_VERSION=$(PIP_VERSION) \ - --build-arg SETUPTOOLS_VERSION=$(SETUPTOOLS_VERSION) \ - -t $(IMAGE_REPO):$(RUNTIMETAG)-$(PYTHON_SUFFIX) . - -push: - docker push $(IMAGE_REPO):$(RUNTIMETAG)-$(PYTHON_SUFFIX) - -tag_latest: - docker tag $(IMAGE_REPO):$(RUNTIMETAG)-$(PYTHON_SUFFIX) $(IMAGE_REPO):latest-$(PYTHON_SUFFIX) - -push_latest: - docker push $(IMAGE_REPO):latest-$(PYTHON_SUFFIX) - -# Print the tag CI should set as RUNNER_IMAGE_TAG after a successful publish. -print_tag: - @echo $(RUNTIMETAG)-$(PYTHON_SUFFIX) diff --git a/.setup-scripts/python_tool_versions.env b/.setup-scripts/python_tool_versions.env index f4ccde95..62f06081 100644 --- a/.setup-scripts/python_tool_versions.env +++ b/.setup-scripts/python_tool_versions.env @@ -1,3 +1,3 @@ -# Python 3.13-compatible tool pins shared by CI setup, runner image, and tox. +# Python 3.13-compatible tool pins shared by CI setup and tox. PIP_VERSION=24.3.1 SETUPTOOLS_VERSION=75.8.2 diff --git a/.setup-scripts/setup_artifact_publishing.sh b/.setup-scripts/setup_artifact_publishing.sh deleted file mode 100755 index 34c54255..00000000 --- a/.setup-scripts/setup_artifact_publishing.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -missing= -for var in GITLAB_PACKAGE_REGISTRY_PYPI_URL GITLAB_PACKAGE_REGISTRY_USER GITLAB_PACKAGE_REGISTRY_PASSWORD; do - if [ -z "${!var}" ]; then - missing="$missing $var" - fi -done -if [ -n "$missing" ]; then - echo "ERROR: Required environment variables not set:$missing" >&2 - exit 1 -fi - -echo "→ Configuring .pypirc for publishing to GitLab Package Registry..." - -PYPI_REPOSITORY_URL="${GITLAB_PACKAGE_REGISTRY_PYPI_URL}" -if [[ "${PYPI_REPOSITORY_URL}" != http://* && "${PYPI_REPOSITORY_URL}" != https://* ]]; then - PYPI_REPOSITORY_URL="https://${PYPI_REPOSITORY_URL}" -fi -echo "GitLab PyPI URL: ${PYPI_REPOSITORY_URL}" - -# setup .pypirc -cat > ~/.pypirc <&2 - exit 1 -fi - -echo "→ Configuring pip to pull from GitLab Package Registry (simple URL)..." -echo "GitLab PyPI simple URL: $GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL" - -# setup pip.conf -mkdir -p ~/.pip -cat > ~/.pip/pip.conf < ~/.netrc < - $ export GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL=gitlab.com/api/v4/projects/71271774/packages/pypi/simple - $ export GITLAB_PACKAGE_REGISTRY_TOKEN= - $ export GITLAB_PACKAGE_REGISTRY_PYPI_URL=gitlab.com/api/v4/projects/71271774/packages/pypi - -In order to have a working setup the email you feed these environment variables should be the one you access our GitLab systems with, and the token should a personal access token with packages reading capabilities. - -Then proceed to source the setup scripts to correctly load the dependencies and setup the virtual environment: +Source the setup scripts to load the dependencies and set up the virtual environment: $ source .setup-scripts/load_deps.sh $ source .setup-scripts/setup_env.sh @@ -77,12 +68,11 @@ then you can run just once the check as you would do with a real agent: You can optionally pass a log level parameter, if not passed logging is disabled. -## CI image +## CI Python -The CI image is built from `.setup-scripts/image` and published to -`registry.tooling.stackstate.io` (referenced by `RUNNER_IMAGE_TAG` in -`.gitlab-ci.yml`). **Do not build or push it locally** except for ad-hoc -validation — use the GitLab CI `docker` job (build stage, manual). +CI runs the check suites directly in the SUSE BCI Python image, pinned by digest +in `BCI_PYTHON_IMAGE` at the top of `.github/workflows/checks-tests.yml`. There is +no bespoke runner image to build or publish. Its Python **must match the CPython that the StackState Agent embeds** — the agent's `omnibus/config/software/python3.rb` `default_version`, mirrored here in @@ -90,16 +80,10 @@ agent's `omnibus/config/software/python3.rb` `default_version`, mirrored here in not ad-hoc work: see the agent repo's `UPSTREAM_MERGE.md`, section "Integrations repo: CI runner image (embedded Python bump)". -After merging Dockerfile / Makefile changes: - -1. Run the manual **`docker`** job on the pipeline (it builds from - `python:3.13.14-bookworm` and pushes `YYYYMMDD-py313` to the registry). -2. In a follow-up commit, set `RUNNER_IMAGE_TAG` in `.gitlab-ci.yml` to the tag - the job prints, and ensure `setup_env.sh` uses `python3.13` and - `conda_env.ps1` defaults to `3.13.14` (must land together with the runner switch). -3. Re-run the pipeline so test jobs use the new runner. +To bump it, update `BCI_PYTHON_IMAGE` to the new digest, and ensure `.python-version`, +`setup_env.sh` (`python3.13`) and `conda_env.ps1` agree. They must land together. -Keeping runner Python in sync ensures CI tests run on the same interpreter the +Keeping CI Python in sync ensures CI tests run on the same interpreter the agent ships, so version-specific bugs (e.g. pydantic validation differences) are caught before release rather than in production. diff --git a/run_gitlab_local.sh b/run_gitlab_local.sh deleted file mode 100755 index 21c53dc0..00000000 --- a/run_gitlab_local.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -echo "Run a gitlab build step on the local machine" - -if [[ $(type gitlab-runner) -eq 1 ]]; then - echo "The cmd gilab-runner looks not available, do you want to install it ?" - sudo wget -O /usr/local/bin/gitlab-runner https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64 - sudo chmod +x /usr/local/bin/gitlab-runner -fi - -gitlab-runner exec docker \ - --cache-type s3 \ - --cache-s3-server-address s3.amazonaws.com \ - --cache-s3-bucket-name ci-runner-cache-eu1 \ - --cache-s3-bucket-location eu-west-1 \ - --cache-s3-access-key $AWS_ACCESS_KEY \ - --cache-s3-secret-key $AWS_SECRET_KEY \ - --docker-volumes /var/run/docker.sock:/var/run/docker.sock \ - --env GITLAB_PACKAGE_REGISTRY_PYPI_SIMPLE_URL=gitlab.com/api/v4/projects/71271774/packages/pypi/simple \ - --env GITLAB_PACKAGE_REGISTRY_USER=$GITLAB_PACKAGE_REGISTRY_USER \ - --env GITLAB_PACKAGE_REGISTRY_TOKEN=$GITLAB_PACKAGE_REGISTRY_TOKEN \ - "$@" -