Skip to content
CI

Merge pull request #468 from ContextLab/docs/catalogue-validation-bas… #77

Sign in to view logs

Merge pull request #468 from ContextLab/docs/catalogue-validation-bas…

Merge pull request #468 from ContextLab/docs/catalogue-validation-bas… #77

Workflow file for this run

.github/workflows/ci.yml at 758b5d3
name: CI
# Default CI is hermetic: no secrets, no external services, no model providers.
# Anything needing credentials or a live API lives in live-tests.yml and is
# triggered explicitly. See docs/adr/0001-product-contract.md.
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
env:
# Runtime package installation must never happen inside CI.
ORCHESTRATOR_AUTO_INSTALL: "0"
jobs:
pr-base:
name: Pull request targets main
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- name: Base branch must be main
# #452, #453 and #455 were each merged into a feature branch that had
# itself already been merged and deleted. GitHub reported MERGED for
# all three and none of the work reached main; it was found later by
# checking file contents, not badges.
#
# The badge cannot distinguish "merged into main" from "merged into a
# dead branch". This can, before the merge rather than after it.
env:
BASE_REF: ${{ github.base_ref }}
run: |
if [ "$BASE_REF" != "main" ]; then
echo "::error::This pull request targets '$BASE_REF', not 'main'."
echo ""
echo "Merging into a base that is itself unmerged -- or already"
echo "merged and deleted -- reports MERGED while the work never"
echo "reaches main. That silently discarded three PRs."
echo ""
echo "Retarget to main, or keep this a draft until its parent lands."
exit 1
fi
echo "Base branch is 'main'."
lint:
name: Lint and package metadata
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install tooling
run: python -m pip install --upgrade pip ruff build twine
- name: No source file is excluded by .gitignore
# An unanchored `debug_*` rule silently excluded a real module
# (orchestrator/quality/debug_artifact_detector.py) for ~11 months.
# Local trees still had the file, so only fresh clones broke.
run: |
ignored=$(git ls-files --others --ignored --exclude-standard -- 'src/**/*.py')
if [ -n "$ignored" ]; then
echo "::error::These source files are present but excluded by .gitignore:"
echo "$ignored"
exit 1
fi
echo "no source files are hidden by .gitignore"
- uses: astral-sh/setup-uv@v5
- name: Lockfile is in sync with pyproject.toml
# A lock that drifts from the manifest is worse than no lock: it looks
# like a pinned resolution while no longer describing what installs.
run: uv lock --check
- name: Correctness lint (blocking)
# Only correctness-class rules gate the build. Style cleanup is tracked
# separately; making ~10k style findings blocking would bury real bugs.
run: ruff check src/orchestrator --select E9,F63,F7,F82,F821,F823,F601,F811
- name: Syntax check
run: python -m compileall -q src/orchestrator
- name: Build sdist and wheel
run: python -m build
- name: Validate package metadata
run: twine check dist/*
- uses: actions/upload-artifact@v4
with:
name: dist
path: dist/
test:
name: Tests (py${{ matrix.python-version }}, ${{ matrix.os }})
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
python-version: ["3.11", "3.12", "3.13"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- uses: astral-sh/setup-uv@v5
- name: Install from the lockfile
# Installs the LOCKED dependency set, not whatever pip resolves today,
# so this job tests the environment users actually get. `uv lock
# --check` proving the lock is fresh is not the same as running
# against it.
#
# --no-default-groups + --group test keeps this to the locked core
# plus test runners, so the hermetic layers are still proven to work
# with no optional extra installed.
run: uv sync --locked --no-default-groups --group test --python ${{ matrix.python-version }}
- name: Package imports without optional extras
run: |
uv run --no-sync python -c "import orchestrator; print(orchestrator.__version__)"
uv run --no-sync python -c "import orchestrator; [getattr(orchestrator, n) for n in ['Orchestrator','YAMLCompiler','Pipeline','Task']]; print('core exports OK')"
- name: Collect the full suite
# Collection must succeed with no network and no extras installed.
run: uv run --no-sync python -m pytest --collect-only -q
- name: Hermetic tests (unit, contract, e2e) - BLOCKING
# The gate is the layer that has been brought under the product
# contract. Passing here means the supported surface works.
run: >
uv run --no-sync python -m pytest
-m "(unit or contract or e2e) and not live and not integration and not docker"
--cov=src/orchestrator --cov-report=xml --cov-report=term-missing
-q
- name: Upload coverage
if: matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest'
uses: actions/upload-artifact@v4
with:
name: coverage
path: coverage.xml
catalogue:
name: Example catalogue
runs-on: ubuntu-latest
timeout-minutes: 25
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- uses: astral-sh/setup-uv@v5
- name: Install from the lockfile
run: uv sync --locked --no-default-groups --group test --python 3.12
- name: Report the catalogue, fail on a named regression
# The headline count is deliberately NOT the contract. It moves for
# reasons that are not regressions -- fixing an `enhanced/` twin moves
# it by two, retiring a legacy file moves it by one -- so gating on it
# would punish ordinary progress and reward deleting hard examples.
#
# scripts/catalogue_validation_baseline.txt names the files that pass
# validation -- which is not the same as running correctly. A
# file LEAVING that list fails this job. A file joining it does not;
# it prints a reminder to run --update.
#
# The grouped counts are informational: they show where the remaining
# work actually is, grouped by what the validator says rather than by
# a taxonomy that would drift from it.
run: uv run --no-sync python scripts/catalogue_report.py | tee catalogue.txt
- name: Publish the report to the job summary
if: always()
run: |
{
echo '## Example catalogue'
echo ''
echo '```'
cat catalogue.txt || echo '(the catalogue step produced no output)'
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
legacy-suite:
name: Legacy suite (non-blocking backlog)
runs-on: ubuntu-latest
# Hard cap: at least one legacy async test blocks forever (observed wedging
# a local run at ~36% despite pytest-timeout), so this job must not be able
# to occupy a runner indefinitely.
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- uses: astral-sh/setup-uv@v5
- name: Install from the lockfile
run: uv sync --locked --no-default-groups --group test --python 3.12
- name: Run unmarked legacy tests
# These predate the product contract and were written against several
# superseded architectures; many fail. This job exists so the size of
# the backlog is VISIBLE rather than hidden behind a green check, and
# so regressions in the parts that do pass are noticeable.
#
# It is intentionally non-blocking. It must NOT be made blocking by
# deleting or skipping failing tests -- fix them or delete them
# deliberately with the reason recorded.
#
# `continue-on-error` (not `|| true`) is what makes it non-blocking.
# Both together made the STEP report success as well, so the backlog
# rendered as a green check -- the exact opposite of keeping it
# visible. The step is now allowed to fail honestly.
id: legacy
continue-on-error: true
# --timeout-method=signal because the default thread method cannot
# interrupt a coroutine blocked in the event loop selector.
run: |
set -o pipefail
uv run --no-sync python -m pytest \
-m "not unit and not contract and not e2e and not live and not integration and not docker and not slow" \
-q --timeout=60 --timeout-method=signal --tb=no -rfE \
| tee legacy-results.txt
- name: Fail if the suite dirtied the repository
# NOT continue-on-error, deliberately: the backlog is allowed to be
# red, but it is not allowed to modify tracked files. Tests used to
# write generated pipelines -- embedding whichever ephemeral port their
# mock server bound -- straight into tests/scenarios/test_pipelines/,
# so every run left a diff and whichever port was committed last became
# everyone else's fixture (#431). This is the guard that keeps it fixed.
if: always()
run: |
# --untracked-files=all, not `git diff`: a diff only sees
# modifications to tracked files, so a test could still scatter new
# files through the checkout and the guard would pass.
# legacy-results.txt is written by the step above, on purpose. It is
# the only expected artifact; anything else is a test misbehaving.
dirt=$(git status --porcelain --untracked-files=all \
| grep -v ' legacy-results\.txt$' || true)
if [ -n "$dirt" ]; then
echo "::error::The test suite left the repository dirty:"
echo "$dirt"
echo
echo "Tests must write generated files to a temp directory"
echo "(pytest's tmp_path), never into the repository."
exit 1
fi
- name: Publish backlog counts
# Runs even when the step above fails, which is the normal case.
#
# `continue-on-error` forces the step's *conclusion* to "success" no
# matter what pytest returned, so neither the job badge nor the step
# list can show this backlog. The summary and the warning annotation
# below are therefore the only honest surface for it.
if: always()
run: |
summary=""
if [ -f legacy-results.txt ]; then
# pytest's own tally is the last non-empty line of the run.
summary=$(grep -vE '^\s*$' legacy-results.txt | tail -n 1)
fi
{
echo "## Legacy suite backlog (non-blocking)"
echo
if [ -n "$summary" ]; then
echo '```'
echo "$summary"
echo '```'
echo
echo "Tracked in [#354](https://github.com/ContextLab/orchestrator/issues/354)."
echo "This job is **expected** to be failing. It is marked"
echo "\`continue-on-error\`, so the green check above reflects that"
echo "policy, NOT a passing suite. Do not 'fix' it by deleting or"
echo "skipping tests."
else
echo "The run produced no output -- it timed out or the runner died."
fi
} >> "$GITHUB_STEP_SUMMARY"
# An annotation surfaces on the run page even though the job is green.
if [ -n "$summary" ]; then
echo "::warning title=Legacy backlog (non-blocking)::${summary}"
else
echo "::warning title=Legacy suite produced no result::timed out or runner died"
fi
- name: Upload legacy results
if: always()
uses: actions/upload-artifact@v4
with:
name: legacy-suite-results
path: legacy-results.txt
if-no-files-found: warn
wheel-smoke:
name: Installed-wheel smoke test
needs: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- uses: actions/download-artifact@v4
with:
name: dist
path: dist/
- name: Install the built wheel into a clean environment
run: |
python -m venv /tmp/smoke
/tmp/smoke/bin/python -m pip install --upgrade pip
/tmp/smoke/bin/python -m pip install dist/*.whl
- name: Wheel ships its configuration resources
# The wheel previously contained no models.yaml/orchestrator.yaml/
# registry, so orchestrator-install-configs silently installed nothing.
# The golden pipeline does not need models, so it could not catch this.
run: |
/tmp/smoke/bin/python - <<'PY'
from importlib.resources import files
expected = {
"orchestrator.config": ["models.yaml", "orchestrator.yaml"],
"orchestrator.registry.models": ["default_registry.yaml"],
"orchestrator.registry.skills": ["default_registry.yaml"],
}
for package, names in expected.items():
present = {p.name for p in files(package).iterdir()}
missing = [n for n in names if n not in present]
assert not missing, f"{package} is missing {missing} in the wheel"
print("all configuration resources present in the installed wheel")
PY
- name: Config installer works from the installed wheel
run: |
export HOME=/tmp/smoke-home
mkdir -p "$HOME"
/tmp/smoke/bin/orchestrator-install-configs
test -d "$HOME/.orchestrator" || { echo "installer created nothing"; exit 1; }
find "$HOME/.orchestrator" -name '*.yaml' | grep -q . \
|| { echo "installer wrote no yaml configuration"; exit 1; }
echo "config installer populated ~/.orchestrator from the wheel"
- name: Run a golden pipeline through the installed console script
# Executed outside the repo, so the source tree cannot satisfy imports:
# this tests the artifact users actually install.
run: |
mkdir -p /tmp/smoke-run
cp tests/golden/basic.yaml /tmp/smoke-run/
cd /tmp/smoke-run
/tmp/smoke/bin/orchestrator validate basic.yaml
/tmp/smoke/bin/orchestrator run basic.yaml -i greeting=ci
test "$(cat golden_out/greeting.txt)" = "ci world"
echo "installed wheel executed a pipeline correctly"