diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml new file mode 100644 index 0000000..641e341 --- /dev/null +++ b/.github/workflows/conformance.yml @@ -0,0 +1,62 @@ +name: Conformance + +on: + pull_request: + branches: [ mainline, release, 'patch_*' ] + workflow_call: + inputs: + branch: + required: false + type: string + tag: + required: false + type: string + +jobs: + conformance: + name: Conformance (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - name: Checkout + uses: actions/checkout@v7 + with: + ref: ${{ inputs.tag }} + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + # The suite's job fixtures run `command: python`, which does not exist on a + # bare Ubuntu runner -- only `python3` does. A virtualenv provides both names + # plus the `openjd` entry point the runner invokes, so putting its bin + # directory on PATH covers all three. + - name: Create virtualenv + shell: bash + run: | + # Outside the checkout, so it cannot end up in a source distribution. + python -m venv "$RUNNER_TEMP/conformance-venv" + if [ -d "$RUNNER_TEMP/conformance-venv/Scripts" ]; then + echo "$RUNNER_TEMP/conformance-venv/Scripts" >> "$GITHUB_PATH" + else + echo "$RUNNER_TEMP/conformance-venv/bin" >> "$GITHUB_PATH" + fi + - name: Install openjd-cli + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install . + - name: Checkout openjd-specifications + uses: actions/checkout@v7 + with: + repository: OpenJobDescription/openjd-specifications + path: openjd-specifications + - name: Install uv + uses: astral-sh/setup-uv@v9.0.0 + - name: Run conformance tests + shell: bash + working-directory: openjd-specifications/conformance-tests + run: | + openjd --version + uv run run_openjd_cli_tests.py '2023-09/*' diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 54d7d5d..42f5f6c 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -115,6 +115,34 @@ log_cli_level = 10 ``` 3. Add logging statements to your tests as desired and run the test(s) that you are debugging. +### Running the conformance suite + +The [openjd-specifications](https://github.com/OpenJobDescription/openjd-specifications) conformance +suite checks this CLI's behaviour against the specification. It drives the installed `openjd` +command, so it exercises the entry point rather than importing the package. CI runs it on Linux, +macOS and Windows via `.github/workflows/conformance.yml`. + +To run it locally you need a checkout of `openjd-specifications`, [uv](https://docs.astral.sh/uv/), +and `openjd` on `PATH`. `hatch shell` provides the latter: + +```bash +hatch shell +cd /path/to/openjd-specifications/conformance-tests +uv run run_openjd_cli_tests.py '2023-09/*' # the whole suite +uv run run_openjd_cli_tests.py '2023-09/base/jobs' # one directory +uv run run_openjd_cli_tests.py '2023-09/base/jobs/1.1--basic-job-creation.test.yaml' +``` + +The job fixtures run `command: python`, so `python` — not just `python3` — has to resolve on +`PATH`. A virtualenv or `hatch shell` satisfies this; a bare system Python on Ubuntu does not. + +Two failure shapes are worth telling apart. A template test failing means `openjd check` accepted +something it should have rejected, or vice versa. A job test failing means `openjd run` produced the +wrong output, or exited non-zero when the fixture expected a clean run — many of the single-task job +fixtures assert their own output from inside the task and signal a mismatch through the task's exit +status, so a non-zero exit with `OPENJD_CONFORMANCE_ASSERT_FAILED` in the log is an output mismatch, +not a crash. + ## Things to Know ### The Package's Public Interface diff --git a/test/openjd/cli/templates/self_asserting_task.yaml b/test/openjd/cli/templates/self_asserting_task.yaml new file mode 100644 index 0000000..cd7aaab --- /dev/null +++ b/test/openjd/cli/templates/self_asserting_task.yaml @@ -0,0 +1,40 @@ +specificationVersion: "jobtemplate-2023-09" +name: Self Asserting Task +description: > + An onRun action that spawns its own child process, reproduces that child's + output, and turns a comparison of it into the task's exit status. This is the + shape the openjd-specifications conformance suite uses for its single-task job + fixtures, so losing grandchild output or dropping the action's exit status + silently weakens that whole suite. + +parameterDefinitions: + - name: Printed + type: STRING + default: "EXPECTED_VALUE" + +steps: + - name: Assert + script: + actions: + onRun: + command: python + args: ["{{Task.File.Assert}}"] + embeddedFiles: + - name: Assert + type: TEXT + data: | + import subprocess + import sys + + COMMAND = [sys.executable, "-c", "print(r'OUTPUT:{{Param.Printed}}')"] + EXPECTED = ["OUTPUT:EXPECTED_VALUE"] + + completed = subprocess.run(COMMAND, capture_output=True, text=True) + sys.stdout.write(completed.stdout) + sys.stderr.write(completed.stderr) + output = completed.stdout + completed.stderr + + missing = [line for line in EXPECTED if line not in output] + for _ in missing: + sys.stderr.write("ASSERT_FAILED: expected output line not found\n") + sys.exit(1 if missing else 0) diff --git a/test/openjd/cli/test_self_asserting_task.py b/test/openjd/cli/test_self_asserting_task.py new file mode 100644 index 0000000..0441c03 --- /dev/null +++ b/test/openjd/cli/test_self_asserting_task.py @@ -0,0 +1,42 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +"""The openjd-specifications conformance shape where a task asserts its own output. + +Its single-task job fixtures spawn the case's command as a child, reproduce its +output, and exit non-zero when the output does not match. That verdict only reaches +the runner if we capture a grandchild's output and propagate the action's exit status, +so both directions are pinned here rather than left to the external suite to catch. +""" + +from pathlib import Path + +from . import format_capsys_outerr, run_openjd_cli_main + +TEMPLATE_DIR = Path(__file__).parent / "templates" + + +def test_grandchild_output_captured_and_assertion_passes(capsys): + outerr = run_openjd_cli_main( + capsys, + args=["run", str(TEMPLATE_DIR / "self_asserting_task.yaml")], + expected_exit_code=0, + ) + # Printed by the grandchild and echoed by the action. Missing means output from a + # process we did not spawn ourselves was dropped. + assert "OUTPUT:EXPECTED_VALUE" in outerr.out, format_capsys_outerr(outerr) + assert "ASSERT_FAILED" not in outerr.out, format_capsys_outerr(outerr) + + +def test_assertion_failure_fails_the_run(capsys): + outerr = run_openjd_cli_main( + capsys, + args=[ + "run", + str(TEMPLATE_DIR / "self_asserting_task.yaml"), + "-p", + "Printed=WRONG_VALUE", + ], + expected_exit_code=1, + ) + output = outerr.out + outerr.err + assert "ASSERT_FAILED" in output, format_capsys_outerr(outerr) + assert "OUTPUT:WRONG_VALUE" in output, format_capsys_outerr(outerr)