diff --git a/.github/workflows/python-coverage.yml b/.github/workflows/python-coverage.yml new file mode 100644 index 0000000..68d4ae3 --- /dev/null +++ b/.github/workflows/python-coverage.yml @@ -0,0 +1,230 @@ +name: Python coverage + +# Reusable workflow: runs the unit tests with pytest, measures the coverage and +# fails below a threshold. The report goes to the job summary and the XML/HTML +# reports are kept as an artifact. +# +# Runs at the repository root by default; set WORKING_DIRECTORY for a +# sub-project (monorepo). The project dependencies are installed with uv; +# pytest, pytest-cov and coverage are added on the fly, so they need not be +# dependencies of the project. +# +# Threshold: +# - MIN_COVERAGE, when set, wins; +# - otherwise the project's own coverage configuration ([tool.coverage.*] in +# pyproject.toml, .coveragerc) decides, including its `fail_under`; +# - otherwise the default of 80 % applies. +# +# Tests that need a database, a cache...: COMPOSE_FILE starts them, ENV points +# the tests at them, SETUP_COMMAND prepares them (migrations). + +on: + workflow_call: + inputs: + WORKING_DIRECTORY: + description: Project directory (relative to the repository root) + type: string + required: false + default: . + PYTHON_VERSION: + description: Python version + type: string + required: false + default: "3.12" + SYNC_ARGS: + description: Arguments for `uv sync` (for example `--group dev`, `--extra foo`) + type: string + required: false + default: --all-groups + TEST_PATH: + description: Path of the tests, relative to WORKING_DIRECTORY. Empty lets pytest discover them. + type: string + required: false + default: "" + PYTEST_ARGS: + description: Extra arguments for pytest (for example `-x -m "not slow"`) + type: string + required: false + default: "" + COVERAGE_SOURCE: + description: >- + Comma-separated packages or directories to measure (for example app). + Empty measures the project code, not the installed dependencies. + type: string + required: false + default: "" + MIN_COVERAGE: + description: >- + Minimum total coverage in percent, 0 to disable. Empty: the project's + coverage configuration, or 80 when it has none. + type: string + required: false + default: "" + BRANCH_COVERAGE: + description: Measure branch coverage as well as line coverage + type: boolean + required: false + default: false + COMPOSE_FILE: + description: Docker Compose file whose services the tests need, started before them (relative to WORKING_DIRECTORY) + type: string + required: false + default: "" + COMPOSE_SERVICES: + description: Space-separated services to start. Empty starts all of them. + type: string + required: false + default: "" + ENV: + description: Environment variables for the tests, one KEY=value per line. Not for secrets. + type: string + required: false + default: "" + SETUP_COMMAND: + description: Command run once before the tests, in WORKING_DIRECTORY (for example `uv run alembic upgrade head`) + type: string + required: false + default: "" + UPLOAD_REPORTS: + description: Keep coverage.xml and the HTML report as a workflow artifact + type: boolean + required: false + default: true + RUNS_ON: + description: Runner labels as JSON array + type: string + required: false + default: '["ubuntu-24.04"]' + outputs: + coverage: + description: Total coverage in percent, for example 87 + value: ${{ jobs.coverage.outputs.coverage }} + +jobs: + coverage: + name: Tests and coverage + runs-on: ${{ fromJSON(inputs.RUNS_ON) }} + permissions: + contents: read + defaults: + run: + working-directory: ${{ inputs.WORKING_DIRECTORY }} + outputs: + coverage: ${{ steps.total.outputs.coverage }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + python-version: ${{ inputs.PYTHON_VERSION }} + enable-cache: true + cache-dependency-glob: ${{ inputs.WORKING_DIRECTORY }}/uv.lock + + - name: Install dependencies + env: + SYNC_ARGS: ${{ inputs.SYNC_ARGS }} + # Unquoted on purpose: SYNC_ARGS holds several arguments. + run: uv sync $SYNC_ARGS + + - name: Set the test environment + if: ${{ inputs.ENV != '' }} + env: + TEST_ENV: ${{ inputs.ENV }} + run: | + # Through the file, not the shell: a value cannot run as a command. + printf '%s\n' "$TEST_ENV" >> "$GITHUB_ENV" + + - name: Start the services + if: ${{ inputs.COMPOSE_FILE != '' }} + env: + COMPOSE_FILE_PATH: ${{ inputs.COMPOSE_FILE }} + COMPOSE_SERVICES: ${{ inputs.COMPOSE_SERVICES }} + # --wait blocks until the services are healthy (they need a healthcheck). + run: docker compose -f "$COMPOSE_FILE_PATH" up -d --wait $COMPOSE_SERVICES + + - name: Prepare + if: ${{ inputs.SETUP_COMMAND != '' }} + env: + SETUP_COMMAND: ${{ inputs.SETUP_COMMAND }} + run: bash -euo pipefail -c "$SETUP_COMMAND" + + - name: Run the tests + id: tests + env: + COVERAGE_SOURCE: ${{ inputs.COVERAGE_SOURCE }} + MIN_COVERAGE: ${{ inputs.MIN_COVERAGE }} + BRANCH_COVERAGE: ${{ inputs.BRANCH_COVERAGE }} + TEST_PATH: ${{ inputs.TEST_PATH }} + PYTEST_ARGS: ${{ inputs.PYTEST_ARGS }} + DEFAULT_MIN_COVERAGE: "80" + run: | + set -euo pipefail + args=() + + if [ -n "$COVERAGE_SOURCE" ]; then + IFS=',' read -ra sources <<<"$COVERAGE_SOURCE" + for source in "${sources[@]}"; do args+=("--cov=${source// /}"); done + else + args+=(--cov) + fi + [ "$BRANCH_COVERAGE" != "true" ] || args+=(--cov-branch) + + # A project that configures coverage keeps its own threshold. + if [ -f .coveragerc ] || { [ -f pyproject.toml ] && grep -q '^\[tool\.coverage' pyproject.toml; }; then + has_config=true + else + has_config=false + fi + if [ -n "$MIN_COVERAGE" ]; then + threshold="$MIN_COVERAGE" + elif [ "$has_config" = "false" ]; then + threshold="$DEFAULT_MIN_COVERAGE" + else + threshold="" + echo "Using the project's coverage configuration" + fi + [ -z "$threshold" ] || args+=("--cov-fail-under=$threshold") + + args+=(--cov-report=term-missing --cov-report=xml:coverage.xml --cov-report=html:htmlcov) + + # Word splitting is wanted for PYTEST_ARGS and TEST_PATH. + # shellcheck disable=SC2086 + uv run --no-sync --with pytest --with pytest-cov pytest "${args[@]}" $PYTEST_ARGS $TEST_PATH + + # Runs even when the tests or the threshold failed: that is when the report is read. + - name: Coverage report + id: total + if: ${{ !cancelled() }} + run: | + set -euo pipefail + [ -f .coverage ] || { echo "::warning::No coverage data was produced."; exit 0; } + # --fail-under=0: only pytest applies the threshold. Without it, a project + # `fail_under` makes this command fail and hides the report. + total=$(uv run --no-sync --with coverage coverage report --fail-under=0 --format=total) + echo "coverage=$total" >> "$GITHUB_OUTPUT" + { + echo "### Coverage: ${total} %" + echo + uv run --no-sync --with coverage coverage report --fail-under=0 --format=markdown --skip-covered + } >> "$GITHUB_STEP_SUMMARY" + + # Artifact names must be unique in a run, and several projects of one + # repository can each call this workflow. + - name: Name the artifact + id: artifact + if: ${{ !cancelled() && inputs.UPLOAD_REPORTS }} + env: + NAME: ${{ inputs.WORKING_DIRECTORY }}-py${{ inputs.PYTHON_VERSION }} + run: echo "name=coverage-$(printf '%s' "$NAME" | tr -c 'A-Za-z0-9._-' '-')" >> "$GITHUB_OUTPUT" + + - name: Upload the reports + if: ${{ !cancelled() && inputs.UPLOAD_REPORTS }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.artifact.outputs.name }} + path: | + ${{ inputs.WORKING_DIRECTORY }}/coverage.xml + ${{ inputs.WORKING_DIRECTORY }}/htmlcov + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/python-deadcode.yml b/.github/workflows/python-deadcode.yml index 29473df..2277f3b 100644 --- a/.github/workflows/python-deadcode.yml +++ b/.github/workflows/python-deadcode.yml @@ -93,7 +93,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: python-version: ${{ inputs.PYTHON_VERSION }} diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 000fee4..8b2ac15 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -83,7 +83,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: python-version: ${{ inputs.PYTHON_VERSION }} diff --git a/.github/workflows/python-typecheck.yml b/.github/workflows/python-typecheck.yml index 703b391..63029f8 100644 --- a/.github/workflows/python-typecheck.yml +++ b/.github/workflows/python-typecheck.yml @@ -66,7 +66,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0 + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 with: python-version: ${{ inputs.PYTHON_VERSION }} enable-cache: true diff --git a/.github/workflows/scan-gitleaks.yml b/.github/workflows/scan-gitleaks.yml index 76f5761..5c67610 100644 --- a/.github/workflows/scan-gitleaks.yml +++ b/.github/workflows/scan-gitleaks.yml @@ -160,7 +160,7 @@ jobs: - name: Upload findings to the Security tab id: upload if: ${{ !cancelled() && inputs.SECURITY_TAB && steps.scan.outcome != 'skipped' }} - uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 + uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 with: sarif_file: gitleaks-results.sarif category: ${{ inputs.CATEGORY }} diff --git a/.github/workflows/scan-trivy.yml b/.github/workflows/scan-trivy.yml index c19c0a4..e75c51b 100644 --- a/.github/workflows/scan-trivy.yml +++ b/.github/workflows/scan-trivy.yml @@ -191,7 +191,7 @@ jobs: - name: Upload findings to the Security tab id: upload if: ${{ !cancelled() && inputs.SECURITY_TAB && steps.scan.outcome != 'skipped' }} - uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 + uses: github/codeql-action/upload-sarif@2892aa5e19bbd11bc0cff5427e3b750a04d9e3c2 # v4.38.2 with: sarif_file: trivy-results.sarif category: ${{ inputs.CATEGORY || format('trivy-{0}', inputs.SCAN_TYPE) }} diff --git a/README.md b/README.md index c600873..7700c19 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ Reusable GitHub Actions workflows, independent from any organisation's shared ca | [path-filter.yml](.github/workflows/path-filter.yml) | Which folders changed, to skip the jobs that do not apply | | [status-gate.yml](.github/workflows/status-gate.yml) | One required check that aggregates the other jobs | | [lint-commits.yml](.github/workflows/lint-commits.yml) | commitlint: commits follow Conventional Commits | +| [python-coverage.yml](.github/workflows/python-coverage.yml) | pytest + coverage with a threshold, reports kept | | [python-deadcode.yml](.github/workflows/python-deadcode.yml) | vulture: unused code | Also provided: a shared [Renovate preset](renovate/default.json) and a Dependabot config @@ -47,7 +48,7 @@ Runs commitlint (pinned) on the commits of a pull request, or of a push. Add it scope requirement and header length are inputs; `CONFIG_FILE` swaps in your own commitlint config. To block merging on failure, mark the job as a required status check in the branch protection. -### python-lint / python-typecheck / python-deadcode +### python-lint / python-typecheck / python-deadcode / python-coverage Both run at the repository root by default; pass `WORKING_DIRECTORY` to target a sub-project. See [examples/python/caller.yml](examples/python/caller.yml) for all inputs. diff --git a/docs/README.md b/docs/README.md index 0e25275..d2fdc09 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,6 +8,7 @@ | [sync-prerelease-branch](sync-prerelease-branch.md) | Keeping `dev` on top of `main` after a release | | [python-lint](python-lint.md) | ruff, default rules and overrides | | [python-typecheck](python-typecheck.md) | ty, mypy or pyright | +| [python-coverage](python-coverage.md) | pytest and coverage, threshold, tests with a database | | [python-deadcode](python-deadcode.md) | vulture, confidence levels, false positives | | [scan-gitleaks](scan-gitleaks.md) | gitleaks, what is scanned, allowlists | | [scan-trivy](scan-trivy.md) | Trivy: fs, config and image scans | diff --git a/docs/python-coverage.md b/docs/python-coverage.md new file mode 100644 index 0000000..31755d9 --- /dev/null +++ b/docs/python-coverage.md @@ -0,0 +1,107 @@ +# python-coverage.yml + +Runs the unit tests with [pytest](https://docs.pytest.org/), measures the coverage with +[pytest-cov](https://pytest-cov.readthedocs.io/) and fails below a threshold. The report goes to the job summary, +and the XML and HTML reports are kept as an artifact. + +```yaml +coverage: + uses: Mitchou10/github-workflow/.github/workflows/python-coverage.yml@v0 + permissions: + contents: read + with: + WORKING_DIRECTORY: backend # optional, default: repository root + MIN_COVERAGE: "85" # optional, default: 80 +``` + +The dependencies are installed with `uv sync`. pytest, pytest-cov and coverage are added on the fly: they do not +have to be dependencies of the project. The project must be managed by uv. + +## Inputs + +| Input | Default | Description | +| --- | --- | --- | +| `WORKING_DIRECTORY` | `.` | Project directory, relative to the repository root | +| `PYTHON_VERSION` | `3.12` | Python version | +| `SYNC_ARGS` | `--all-groups` | Arguments of `uv sync`, for example `--group dev` | +| `TEST_PATH` | empty | Path of the tests. Empty lets pytest discover them | +| `PYTEST_ARGS` | empty | Extra pytest arguments, for example `-x -m "not slow"` | +| `COVERAGE_SOURCE` | empty | Comma-separated packages or directories to measure, for example `app` | +| `MIN_COVERAGE` | empty | Minimum total coverage in percent, `0` to disable | +| `BRANCH_COVERAGE` | `false` | Measure branches as well as lines | +| `COMPOSE_FILE` | empty | Docker Compose file whose services the tests need | +| `COMPOSE_SERVICES` | empty | Services to start (all of them if empty) | +| `ENV` | empty | Environment of the tests, one `KEY=value` per line. Not for secrets | +| `SETUP_COMMAND` | empty | Command run once before the tests, for example a migration | +| `UPLOAD_REPORTS` | `true` | Keep `coverage.xml` and `htmlcov` as an artifact | +| `RUNS_ON` | `["ubuntu-24.04"]` | Runner labels, as a JSON array | + +Output: `coverage`, the total in percent (for example `87`). + +## Which threshold applies + +In this order: + +1. **`MIN_COVERAGE`**, when set (`0` turns the threshold off). +2. The project's own coverage configuration, when it has one (`[tool.coverage.*]` in `pyproject.toml`, or + `.coveragerc`), including its `fail_under`. +3. Otherwise the default: **80 %**. + +Only pytest applies the threshold. The report step never fails on it, so the summary is always written, which is +when you want to read it. + +## What is measured + +- With `COVERAGE_SOURCE` empty, the project code is measured, not the installed dependencies. Tests count too, + which raises the number: prefer naming the source (`COVERAGE_SOURCE: app`) for a meaningful figure. +- `BRANCH_COVERAGE` also checks that both outcomes of each `if` were run. The number is lower than line coverage. + Turn it on for an honest figure, and expect the first value to drop. +- Files that should not count (migrations, generated code) are excluded in the project configuration: + + ```toml + [tool.coverage.run] + omit = ["*/migrations/*", "tests/*"] + + [tool.coverage.report] + fail_under = 85 + exclude_also = ["if TYPE_CHECKING:", "raise NotImplementedError"] + ``` + +## Tests that need services + +A reusable workflow cannot declare `services:` for its caller. Three inputs replace them: + +```yaml +with: + COMPOSE_FILE: docker-compose-test.yaml + COMPOSE_SERVICES: postgres redis + ENV: | + DATABASE_URL=postgresql+asyncpg://app:app@localhost:5432/app + REDIS_URL=redis://localhost:6379/0 + SETUP_COMMAND: uv run alembic upgrade head +``` + +1. The services are started with `docker compose up -d --wait`, which waits until they are healthy: give them a + `healthcheck` in the compose file. +2. `ENV` is exported for the following steps. +3. `SETUP_COMMAND` runs once in `WORKING_DIRECTORY`, with `bash -e`. + +Ports must be published (`ports: ["5432:5432"]`) so the tests reach the services on `localhost`. + +## Output + +- **Job summary**: total, and a table of the files below 100 % (fully covered files are hidden). +- **Log**: pytest's `term-missing` report, with the line numbers that are not covered. +- **Artifact** `coverage--py`, kept 7 days: `coverage.xml` (for Codecov, SonarQube...) and + `htmlcov/`. Open `htmlcov/index.html` to browse the code line by line. + +## Several projects in one repository + +Call the workflow once per project, with its `WORKING_DIRECTORY`. Each call has its own cache, summary and artifact +(the name contains the directory). See also [path-filter](path-filter.md) to run only the projects that changed. + +## Notes + +- `pytest` is run directly, like `uv run pytest`. If the tests import the project code, configure the path in + `pyproject.toml` (`[tool.pytest.ini_options]` with `pythonpath = ["."]`), or install the project as a package. +- pytest options of the project (`[tool.pytest.ini_options]`, `addopts`) are honoured. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 937a667..e6992e3 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -104,3 +104,19 @@ The calling job of `path-filter` lacks `pull-requests: read`. Grant `contents: r - The filter name in `contains(fromJSON(...), 'name')` does not match the name in `FILTERS` (case matters). - The pattern is not quoted, or does not match from the repository root (`backend/**`, not `/backend/**`). - The pull request is a draft and `SKIP_DRAFT` is on (the default). + +## `ModuleNotFoundError` when pytest collects the tests + +The workflow runs `pytest` like `uv run pytest`, which does not put the project directory on the import path. Add +`pythonpath = ["."]` to `[tool.pytest.ini_options]` in `pyproject.toml`, or make the project an installable +package. + +## `FAIL Required test coverage of 80% not reached` + +The default threshold is 80 %. Raise the coverage, lower the bar (`MIN_COVERAGE`), or set `fail_under` in the +project's coverage configuration. See [python-coverage](python-coverage.md#which-threshold-applies). + +## The tests cannot reach the database + +`COMPOSE_FILE` services must publish their ports and have a healthcheck (`up --wait` relies on it), and `ENV` must +point at `localhost`. A migration goes in `SETUP_COMMAND`. diff --git a/examples/python/caller.yml b/examples/python/caller.yml index 42f9b8c..f6fd9e4 100644 --- a/examples/python/caller.yml +++ b/examples/python/caller.yml @@ -42,3 +42,22 @@ jobs: # IGNORE_NAMES: "visit_*,Meta" # IGNORE_DECORATORS: "@app.route,@pytest.fixture" # WHITELIST: whitelist.py + + coverage: + uses: Mitchou10/github-workflow/.github/workflows/python-coverage.yml@v0 + permissions: + contents: read + # with: + # WORKING_DIRECTORY: backend # default: repository root + # COVERAGE_SOURCE: app # default: the project code + # MIN_COVERAGE: "85" # default: 80 (or the project's fail_under) + # BRANCH_COVERAGE: true + # PYTEST_ARGS: -m "not slow" + # + # Tests that need a database: start it, point the tests at it, prepare it. + # COMPOSE_FILE: docker-compose-test.yaml + # COMPOSE_SERVICES: postgres redis + # ENV: | + # DATABASE_URL=postgresql+asyncpg://app:app@localhost:5432/app + # REDIS_URL=redis://localhost:6379/0 + # SETUP_COMMAND: uv run alembic upgrade head