diff --git a/.agents/skills/edg-python-style/SKILL.md b/.agents/skills/edg-python-style/SKILL.md index e8892db6be..bc8cc7e937 100644 --- a/.agents/skills/edg-python-style/SKILL.md +++ b/.agents/skills/edg-python-style/SKILL.md @@ -26,7 +26,8 @@ Prefer matching the file you are editing over inventing a new dialect. neighbors. - Package code lives under `dev_tools/pylibs//` (importable as `edgtest`, `edgutil`, `edgacknowledg`, …). Put `dev_tools/bin` on `PATH` - and `dev_tools/pylibs` on `PYTHONPATH` (direnv / [HACKING.md](../../../HACKING.md)). + and `dev_tools/pylibs` on `PYTHONPATH` (direnv / + [HACKING.md](../../../HACKING.md)). ## Indentation and wrapping @@ -36,41 +37,143 @@ Prefer matching the file you are editing over inventing a new dialect. - Wrap long signatures, calls, and `typing` imports with parentheses; align continuations with neighboring EDG Python, not Black/ruff defaults. -### Function parameter lists +### Function calls and wrapped message text + +Call-site wrapping differs from the declaration wrapping below. When a +call's arguments need multiple lines: + +- Break immediately after the opening `(`; put the arguments on following + lines, indented **two spaces** from the statement's indentation. +- Once a call is split over multiple lines, put **each positional and + keyword argument on its own line**, even when several would fit together. + A single argument may span multiple lines (for example, a long message), + but do not put another argument on any of those lines. +- Put the closing `)` on its own line, aligned with the statement's start + (including `raise`, `return`, or `await`). Keep suffixes such as `from + None` or a `with` statement's `:` after that closing parenthesis. +- Do not leave the first argument beside the opening parenthesis and hang + subsequent arguments under it. Declaration-style right alignment does + not apply to calls. + +After moving arguments onto new lines, **reflow their text for the new +available width**. Count indentation, quotes, and any `f` prefix against +79 columns. Merge adjacent message literals when the whole argument fits +on one line; otherwise split at word boundaries using the available width. +Do not preserve old string breaks just because they belonged to the old +hanging layout. Preserve exact spaces, interpolation, and rendered text; +implicit concatenation must not introduce a newline into the message. + +For example, replace this hanging call: -If a `def` / `async def` signature does not fit on one line, wrap the -parameter list in the same spirit as EDG C++ (`code_style.rst` — Wrapping -Function Declarations): +```python +raise ConfigError(f"{host_key} must be a host name, without a scheme, " + f"port, or path") +``` -1. Prefer keeping the `def` name and opening `(` on the first line when the - first parameter(s) still fit; continue parameters on following lines. -2. When that is not enough (long name, many parameters, long annotations), - break immediately after `(` and put parameters on the next lines. -3. **Right-indent** wrapped parameter lines so they end at **column 79** - (right-aligned to the margin), packing as many parameters per line as - fit. Do not use a fixed left hanging indent (PEP 8 / Black’s 4-space - continuation). +with this layout, whose message now fits on one argument line: + +```python +raise ConfigError( + f"{host_key} must be a host name, without a scheme, port, or path" +) +``` + +When the message still needs wrapping, reflow the words inside the call: + +```python +raise ConfigError( + f"{port_key} is required for a host other than localhost " + f"({TLS_PORT} for TLS)" +) +``` + +For a call with multiple arguments, keep each argument separate: + +```python +return replace( + self, + service_bind = service_bind, + builder_service_url = builder_service_url, + publisher_service_url = publisher_service_url +) +``` + +Review argument separation, parentheses, and text wrapping in added or +modified calls. An unchanged syntax tree verifies behavior, not readable +layout. + +### Multiline collections + +Optimize for readability and small future diffs, not fewer lines. In a +multiline list, tuple, set, or dictionary literal, put **one element or +key-value pair per line**, even when several would fit. Keep trailing +commas so adding or removing an element changes only its own line. This +also applies to collections passed to calls such as `frozenset`. +Naturally short single-line collections can remain on one line. + +```python +_INT_KEYS = frozenset(( + 'max_artifact_bytes', + 'max_extracted_bytes', + 'max_artifact_entries', +)) +``` + +### Function parameter lists + +For `def` / `async def` signatures, adapt the **Wrapping Function +Declarations** rules in +[the EDG C++ style guide](../../../doc/source/code_style.rst): + +1. First try wrapping with continuation parameters **left-aligned just to + the right of the opening `(`**, in the first parameter's column. Keep + initial parameters on the declaration line when they fit. +2. Use that layout only if every resulting line fits within **79 columns**. + If the declaration overflows and a parameter cannot fit at that column, + break immediately after the opening `(` and move all parameters below it. +3. In this fallback layout, use a **common indentation column chosen by + right-aligning the longest continuation line to column 79**. Shorter + lines start in that same column; do not right-justify each line + independently. Include closing punctuation and any return annotation + when measuring the longest line. +4. Treat the keyword-only separator `*,` as belonging to the preceding + parameter when wrapping. Keep it on that parameter's line whenever it + fits within 79 columns, **including the first declaration line**: + write `def with_wiring(self, *,` rather than putting `*,` below `self`. + The same applies after `cls` or another positional parameter, even when + all following keyword-only parameters wrap. Do not give `*,` its own + line when the preceding line has room for it. + +When reviewing a wrapped signature, check lines beginning with `*,` and +move the separator onto the preceding line if it fits. + +Normal left alignment (including `ForgeStore.open`): ```python def get_output_filename(env: Dict[str, str], case_number: int, command_number: int) -> str: +async def open(cls, config: DatabaseConfig, *, + password: Optional[str] = None) -> 'ForgeStore': + +def with_wiring(self, *, + service_bind: Optional[str] = None) -> 'ForgeServiceConfig': +``` + +Fallback when left alignment would overflow: + +```python async def invoke_recorded_process( - base_command: List[str], command_args: List[str], - working_dir: Path, env: Dict[str, str], - build_constants: BuildConstants, - output_file_handle, *, - filter_command: Optional[str] = None, debug: bool, - driver_debug: bool = False, - injected_command_args: List[str] = None) -> int: - -def parse_test_status_diff( - status_line: str) -> Tuple[str, ParsedTestStatusDiff]: + base_command: List[str], + command_args: List[str], + working_dir: Path, + injected_command_args: Optional[List[str]] = None) -> int: ``` Return annotations (`-> …`) stay on the last parameter line when they fit -within 79 columns; otherwise wrap similarly to neighbors. +within 79 columns; otherwise wrap similarly to neighbors. These declaration +rules do not change the call-site rules above. ## Spacing around `=` @@ -127,6 +230,14 @@ Typical order: Avoid unused imports; do not introduce heavy formatters that rewrite the whole tree. +## Count checks + +When a condition tests whether a collection's element count is zero or +nonzero, prefer explicit `len(items) == 0` or `len(items) != 0` over +collection truthiness (`if not items:` or `if items:`). For example, write +`if len(removed) != 0:` when deciding whether there are removed settings to +report. This rule concerns counts, not boolean flags or optional values. + ## Structure - Modules are flat and explicit; classes are fine for stateful helpers. @@ -139,9 +250,10 @@ whole tree. ## What not to do - Do not run Black/ruff/autopep8 in a way that converts the tree to 4-space - PEP 8, strips spaces around `=`, or replaces right-aligned-to-79 - parameter wraps with fixed hanging indents. -- Do not copy front-end **C** style (`edg-cpfe-style`) into Python. + PEP 8, strips spaces around `=`, or replaces the two declaration-wrapping + layouts above with fixed four-space continuations. +- Do not copy unrelated front-end **C** syntax/layout conventions into + Python; use the declaration-wrapping adaptation above. - Do not invent a second string-quoting scheme; follow `f"..."` / `'...'`. ## Related skills diff --git a/.github/workflows/acknowledg-forge-deploy.yml b/.github/workflows/acknowledg-forge-deploy.yml index bc6700e35a..0ea21b28e1 100644 --- a/.github/workflows/acknowledg-forge-deploy.yml +++ b/.github/workflows/acknowledg-forge-deploy.yml @@ -13,17 +13,15 @@ # installed copy of # dev_tools/services/acknowledg-forge/deploy-remote.py, the only # thing the deploy user may run as root) to point the service at -# the pushed digest, set its AcknowlEDG API and UI URLs, and -# restart it (only if any of those changed) +# the pushed digest and restart it when that changes # # The image is public and holds no credentials or configuration: the publish # job has no access to the deployment's secrets. Everything else about the # service (its unit, tokens, API key) is configured on the host. # # Host, user, and SSH credentials come from secrets in the -# `acknowledg-forge` environment; the public AcknowlEDG URLs come from -# repository variables shared with acknowledg-deploy.yml (see -# dev_tools/services/acknowledg-forge/README.md). +# `acknowledg-forge` environment. Application settings, including public +# AcknowlEDG endpoints, belong in the host administrator's TOML. name: acknowledg-forge Deploy @@ -130,11 +128,8 @@ jobs: env: SERVICE: ${{ secrets.ACKNOWLEDG_FORGE_SERVICE || 'acknowledg-forge-github' }} IMAGE: ${{ needs.publish.outputs.image }} - API_URL: ${{ vars.ACKNOWLEDG_API_HOST }} - UI_URL: ${{ vars.ACKNOWLEDG_UI_HOST }} run: | # The arguments are deliberately expanded (and quoted) here. # shellcheck disable=SC2029 ssh acknowledg-forge "sudo -n /usr/local/libexec/acknowledg-forge-deploy \ - $(printf '%q ' "${SERVICE}" "${IMAGE}" "${API_URL}" \ - "${UI_URL}")" + $(printf '%q ' "${SERVICE}" "${IMAGE}")" diff --git a/.github/workflows/build-and-test-ci.yml b/.github/workflows/build-and-test-ci.yml index 71b5400ba2..0d32a01cf4 100644 --- a/.github/workflows/build-and-test-ci.yml +++ b/.github/workflows/build-and-test-ci.yml @@ -308,7 +308,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7 + image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" diff --git a/.github/workflows/ci-toolchain.yml b/.github/workflows/ci-toolchain.yml index 87cfdb7cd8..9d92a72a52 100644 --- a/.github/workflows/ci-toolchain.yml +++ b/.github/workflows/ci-toolchain.yml @@ -66,7 +66,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: &ci-runner-image ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7 + image: &ci-runner-image ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7 defaults: &container-defaults run: shell: edg-container-exec "bash -eo pipefail {0}" @@ -267,6 +267,17 @@ jobs: edg-prefix-env-loader .tmp/extensive-ci.edgivk + # Reviews (acknowledg-forge, edgy-review) read only the .rt.tar.gz + # files changes.elist names; edgy -A archives every test's. Pruning + # must never keep the results from being uploaded, hence + # continue-on-error. + - name: Prune .rt.tar.gz results changes.elist doesn't name + if: ${{ failure() }} + continue-on-error: true + env: + RUN_TIMESTAMP: ${{ inputs.run_timestamp }} + run: edg-prune-test-run "tests/runs/${RUN_TIMESTAMP}" + - name: Upload partial elist, elog, and .rt.tar.gz results if: ${{ failure() }} uses: actions/upload-artifact@v7 diff --git a/.github/workflows/promote-test-cache.yml b/.github/workflows/promote-test-cache.yml index 85f0784ee4..29bb26eea8 100644 --- a/.github/workflows/promote-test-cache.yml +++ b/.github/workflows/promote-test-cache.yml @@ -72,7 +72,7 @@ jobs: # The test legs' image, so that actions/cache picks the same compression # (part of an entry's version) and the test legs can restore the entry. container: - image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7 + image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7 strategy: fail-fast: false matrix: diff --git a/.github/workflows/strict-build-check.yml b/.github/workflows/strict-build-check.yml index ebdb1f94ff..c74d38f5f1 100644 --- a/.github/workflows/strict-build-check.yml +++ b/.github/workflows/strict-build-check.yml @@ -158,7 +158,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7 + image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" @@ -185,7 +185,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7 + image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" @@ -212,7 +212,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7 + image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" @@ -239,7 +239,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 120 container: - image: ghcr.io/edgcpp/ci-runner@sha256:565d7b6581912008ca91c5dbc940280c6e28a02c8cb6487d88ee5f0d7e8c51d7 + image: ghcr.io/edgcpp/ci-runner@sha256:daae67adf04085b9ae54a816b35053b79086070434d2da4aafd6662f6ecb0bf7 defaults: run: shell: edg-container-exec "bash -eo pipefail {0}" diff --git a/dev_tools/bin/edg-prune-test-run b/dev_tools/bin/edg-prune-test-run new file mode 100755 index 0000000000..451f49fc5a --- /dev/null +++ b/dev_tools/bin/edg-prune-test-run @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM +# Exceptions. +# See https://edgcpp.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +'''Prune a test run down to what a review of it reads, e.g., before CI +uploads the run: + + edg-prune-test-run tests/runs/ + +Reviews of a run read only the .rt.tar files its changes.elist files name. +The replays directory is removed whole. In each configuration, a suite +with no test named there is removed whole (its directory, and its +suite- logs), as is every suite of a configuration without a +changes.elist; of the other suites, the .rt.tar files not named there are +removed. +''' + +import argparse +import sys + +import edgtest +import edgutil + +from pathlib import Path + +def _parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description = ( + "Remove the .rt.tar files of a test run that no configuration's " + 'changes.elist names.' + ) + ) + parser.add_argument( + 'run_dir', + type = Path, + help = 'the run directory (e.g., tests/runs/)' + ) + return parser.parse_args() + +def main() -> int: + args = _parse_args() + if not args.run_dir.is_dir(): + edgutil.eprint(f"error: {args.run_dir} is not a directory") + return 1 + + # Counts: of .rt.tar files kept and removed, and of suites removed. + kept, removed, suites = edgtest.prune_run_archives(args.run_dir) + print( + f"Kept {kept} .rt.tar files named by changes.elist files; removed " + f"{removed} others, and {suites} suites with no changes." + ) + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/dev_tools/bin/edg-rematerialize-test-run b/dev_tools/bin/edg-rematerialize-test-run index 4f2c99b7f1..d4c3948613 100755 --- a/dev_tools/bin/edg-rematerialize-test-run +++ b/dev_tools/bin/edg-rematerialize-test-run @@ -73,8 +73,14 @@ def build_diff_files(diff_file: Path, curr_diff_file_descr = None diff_file_to_handle = {} - for line in diff_file_handle: - line_match = GIT_DIFF_REGEX.match(line) + # Lines are read in bounded pieces so that an enormous one is never held + # whole; only a piece that starts a line can be a header, and every piece + # is written through unchanged. + for line, at_line_start in edgutil.iter_line_pieces( + diff_file_handle, + edgutil.MAX_LINE_CHARS + ): + line_match = GIT_DIFF_REGEX.match(line) if at_line_start else None if line_match is not None: if curr_diff_file_descr is not None: # Close the file handle for the previous diff file (if any) to @@ -180,8 +186,15 @@ def collect_affected_tests(test_home: Path, suite_name = file_path.stem # An expectations update. with open(diff_file.diff_file) as file_handle: - for line in file_handle: - if line.startswith('---') or line.startswith('+++'): + # edgutil.MAX_LINE_CHARS is far more than any genuine expectation + # entry needs, so a longer line is refused (as when the file is + # read), without being held whole. + try: + for line in edgutil.iter_limited_lines( + file_handle, + edgutil.MAX_LINE_CHARS + ): + if line.startswith('---') or line.startswith('+++'): # Do not take in lines like: # # --- a/tests/expectations/edg_x86_64_cp/modules.log @@ -189,13 +202,19 @@ def collect_affected_tests(test_home: Path, # continue - if line.startswith('-') or line.startswith('+'): - # Do a naive parse to get the test file name. - test_file_rel_path = PurePosixPath(line[1:].split(':')[0]) - result.tests.add( - tests_dir.joinpath(suite_name, test_file_rel_path) - ) - result.configs.add(config_name) + if line.startswith('-') or line.startswith('+'): + # Do a naive parse to get the test file name. + test_file_rel_path = PurePosixPath(line[1:].split(':')[0]) + result.tests.add( + tests_dir.joinpath(suite_name, test_file_rel_path) + ) + result.configs.add(config_name) + except edgutil.LineTooLongError: + eprint( + f"{diff_file.file_name}: a changed line is longer than " + f"{edgutil.MAX_LINE_CHARS} characters" + ) + sys.exit(1) return result @@ -548,10 +567,20 @@ async def simulate_test(test_home: Path, config_run_dir: Path, return test_status_lines def read_suite_expectation_lines(suite_file: Path) -> List[str]: + '''The lines of ``suite_file`` (none if there is no such file); exits if + one is longer than ``edgutil.MAX_LINE_CHARS``, rather than holding it. + ''' if not suite_file.exists(): return [] - with open(suite_file, 'r') as file_handle: - return file_handle.readlines() + try: + with open(suite_file, 'r') as file_handle: + return list(edgutil.iter_limited_lines( + file_handle, + edgutil.MAX_LINE_CHARS + )) + except edgutil.LineTooLongError as ex: + eprint(f"{suite_file}: {ex}") + sys.exit(1) def load_test_suite_expectations(all_expectations: Dict[str, List[str]], suite_name: str, expectations_dir: Path, diff --git a/dev_tools/docker/acknowledg-forge/Dockerfile b/dev_tools/docker/acknowledg-forge/Dockerfile index 89232aac5b..fe1e01d125 100644 --- a/dev_tools/docker/acknowledg-forge/Dockerfile +++ b/dev_tools/docker/acknowledg-forge/Dockerfile @@ -46,8 +46,5 @@ RUN useradd --create-home --uid 1000 acknowledg && \ mkdir -p /var/lib/acknowledg-forge && \ chown acknowledg:acknowledg /var/lib/acknowledg-forge -ENV ACKNOWLEDG_FORGE_TOOLS_DIR=/edg/tools -ENV ACKNOWLEDG_FORGE_HOME=/var/lib/acknowledg-forge - USER acknowledg WORKDIR /var/lib/acknowledg-forge diff --git a/dev_tools/pylibs/edgacknowledg/__init__.py b/dev_tools/pylibs/edgacknowledg/__init__.py index 9b21f50672..6ee7b8bdbe 100644 --- a/dev_tools/pylibs/edgacknowledg/__init__.py +++ b/dev_tools/pylibs/edgacknowledg/__init__.py @@ -121,12 +121,17 @@ def request_tags_with_prefix(prefix: str) -> FrozenSet[str]: ALLOWED_BENCH_EDIT_TAGS ) -PROTOCOL_REVISION = 20 +PROTOCOL_REVISION = 21 # 10MiB MAX_MESSAGE_SIZE = 10485760 # 8MiB MAX_DISPLAY_CONTENT_SIZE = 8388608 +# 16Ki characters: a longer line is cut for display (see lines_truncated). +MAX_DISPLAY_LINE_SIZE = 16384 +# 1Ki characters: a longer label (e.g. a status line's variant, at most a few +# hundred in practice) is cut for display, ending in an ellipsis. +MAX_DISPLAY_LABEL_SIZE = 1024 # Most lines shown on each side of the requested line by a ranged # ``test.src.show-src`` request. diff --git a/dev_tools/pylibs/edgacknowledg/budget.py b/dev_tools/pylibs/edgacknowledg/budget.py new file mode 100644 index 0000000000..11a70f1750 --- /dev/null +++ b/dev_tools/pylibs/edgacknowledg/budget.py @@ -0,0 +1,27 @@ +# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM +# Exceptions. +# See https://edgcpp.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +'''A size budget shared by readers that count bytes or characters.''' + + +class BudgetExceeded(Exception): + '''Raised when an operation would spend more than its budget allows.''' + + +class Budget: + def __init__(self, limit: int) -> None: + if limit < 0: + raise ValueError('budget limit must be nonnegative') + self.limit = limit + self.remaining = limit + self.exceeded = False + + def spend(self, count: int) -> None: + if count < 0: + raise ValueError('budget count must be nonnegative') + if count > self.remaining: + self.exceeded = True + raise BudgetExceeded() + self.remaining -= count diff --git a/dev_tools/pylibs/edgacknowledg/client/shared.py b/dev_tools/pylibs/edgacknowledg/client/shared.py index 9d47963490..722fa20f7a 100644 --- a/dev_tools/pylibs/edgacknowledg/client/shared.py +++ b/dev_tools/pylibs/edgacknowledg/client/shared.py @@ -7,6 +7,7 @@ import asyncio import errno +import io import os import signal import stat @@ -39,12 +40,15 @@ from edgacknowledg import ( MAX_DISPLAY_CONTENT_SIZE, MAX_SOURCE_RANGE_CONTEXT, + MAX_DISPLAY_LABEL_SIZE, + MAX_DISPLAY_LINE_SIZE, ProtocolMessage, RequestTags, RichName, decode_response, message, ) +from edgacknowledg.budget import Budget, BudgetExceeded if Protocol is not None: @@ -85,11 +89,12 @@ async def handle_request(self, channel: Channel) -> None: ReviewClientHandler = Any # type: ignore[misc,assignment] +import edgutil + from edgutil import get_running_event_loop, is_path_relative_to CoroFactory = Callable[[], Awaitable[Any]] - class BackgroundWorkerPool: '''Async worker pool used to regulate parallel background work.''' @@ -156,51 +161,162 @@ def source_range_window(line: int, context: int) -> Tuple[int, int]: context = max(0, min(context, MAX_SOURCE_RANGE_CONTEXT)) return max(1, line - context), max(1, line) + context -def slice_source_range(lines: Iterable[str], line: int, - context: int) -> Dict[str, Union[str, int]]: - '''Reply fields for the window around ``line`` of ``lines``. - ``lines`` may be a lazy iterable (such as an open file). It is consumed - in full to count the lines, but only the window is kept in memory. +def slice_source_range(handle: IO[str], line: int, + context: int) -> Dict[str, Any]: + '''Reply fields for the window around ``line`` of the text ``handle``, + with ``lines_truncated`` as ``display_content`` sets it (numbered within + the window), and ``truncated`` if the read budget is exhausted. + + ``handle`` is read no further than the window's last line, and only the + window is kept in memory, of each line at most ``MAX_DISPLAY_LINE_SIZE`` + characters. ''' first, last = source_range_window(line, context) shown: List[str] = [] - total = 0 - for total, text in enumerate(lines, start = 1): - if first <= total <= last: - shown.append(text.rstrip('\n')) - return { + lines_truncated: List[int] = [] + budget = Budget(MAX_DISPLAY_CONTENT_SIZE) + for line_number, (text, _, cut) in enumerate( + _display_lines( + edgutil.iter_line_pieces(handle, MAX_DISPLAY_LINE_SIZE + 1), + budget + ), + start = 1 + ): + if line_number >= first: + shown.append(text) + if cut: + lines_truncated.append(len(shown)) + if line_number == last: + break + result: Dict[str, Any] = { 'content': '\n'.join(shown), - 'first_line': min(first, max(total, 1)), - 'total_lines': total + 'first_line': first } + if lines_truncated: + result['lines_truncated'] = lines_truncated + if budget.exceeded: + result['truncated'] = True + return result + +def trunc_display_label(text: str) -> str: + '''``text`` (one line, e.g. a status) cut for UI display to + ``MAX_DISPLAY_LABEL_SIZE`` characters, a cut ending in an ellipsis. + ''' + if len(text) > MAX_DISPLAY_LABEL_SIZE: + text = text[:MAX_DISPLAY_LABEL_SIZE - 1] + '\N{HORIZONTAL ELLIPSIS}' + return text -def trunc_display_content(content: str) -> Dict[str, Union[str, bool]]: - '''Truncate ``content`` for UI display; may set ``truncated``.''' - max_content_size = MAX_DISPLAY_CONTENT_SIZE - if len(content) > max_content_size: - # Attempt to truncate at line boundaries. - # - # To save time, find the newline closest to the boundary before walking - # the file size back. - last_newline_permitted = content.find('\n', max_content_size) - if last_newline_permitted != -1: - content = content[:last_newline_permitted] - # Now walk the file size back. - while len(content) > max_content_size: - last_new_line_idx = content.rfind('\n') - if last_new_line_idx != -1: - content = content[:(last_new_line_idx - 1)] - else: - # If there are no more newlines, resort to truncating within - # the line. - content = content[:(max_content_size - 4)] - content += '...' - return { - 'truncated': True, - 'content': content - } - return { 'content': content } + +def _display_lines(pieces: Iterable[Tuple[str, bool]], + budget: Budget) -> Iterator[Tuple[str, str, bool]]: + '''Join ``(piece, at_line_start)`` pieces (as ``edgutil.iter_line_pieces`` + yields them) into lines, yielding ``(text, line_end, cut)`` for each: at + most ``MAX_DISPLAY_LINE_SIZE`` characters of its text (``cut`` if it had + more, which are dropped as they come), and its end ('\\n', or '' for a + last line without one). Stop once ``budget`` is exhausted. + + A piece that starts a line ends the one before it, so a text that doesn't + end in a line end (e.g. a command's output) never runs into the next. + ''' + # The line being joined (as much of it as is kept), and whether some was + # dropped; ``None`` between lines. + line: Optional[str] = None + cut = False + for piece, at_line_start in pieces: + try: + budget.spend(len(piece)) + except BudgetExceeded: + if not budget.remaining: + break + piece = piece[:budget.remaining] + budget.spend(len(piece)) + if line is not None and at_line_start: + yield line, '\n', cut + line = None + if line is None: + line = '' + cut = False + text = piece[:-1] if piece.endswith('\n') else piece + room = MAX_DISPLAY_LINE_SIZE - len(line) + if len(text) > room: + cut = True + line += text[:room] + if piece.endswith('\n'): + yield line, '\n', cut + line = None + if budget.exceeded: + break + if line is not None: + yield line, '', cut + + +def display_content(pieces: Iterable[Tuple[str, bool]]) -> Dict[str, Any]: + '''The ``content`` of a display payload, from ``(piece, at_line_start)`` + pieces (as ``edgutil.iter_line_pieces`` yields them), and as needed + ``truncated`` and ``lines_truncated``: + + - Each line is cut to ``MAX_DISPLAY_LINE_SIZE`` characters, its 1-based + number listed in ``lines_truncated``. + - The whole is cut to ``MAX_DISPLAY_CONTENT_SIZE``, after the last line + that fits (setting ``truncated``); no more pieces are taken then, so + what they come from is read no further. (Closing ``pieces``, should it + be a generator with files open, is left to the caller.) + Reading also stops at ``MAX_DISPLAY_CONTENT_SIZE`` characters and sets + ``truncated``, including characters discarded by the line limit. + + Neither key is set when nothing was cut, and nothing marks a cut in the + content itself: the UI does. + ''' + kept: List[str] = [] + size = 0 + lines_truncated: List[int] = [] + truncated = False + budget = Budget(MAX_DISPLAY_CONTENT_SIZE) + for line_number, (text, line_end, cut) in enumerate( + _display_lines(pieces, budget), + start = 1 + ): + num_chars_for_line = len(text) + len(line_end) + if size + num_chars_for_line > MAX_DISPLAY_CONTENT_SIZE: + truncated = True + if not kept: + # The first line alone doesn't fit (MAX_DISPLAY_LINE_SIZE is + # larger than MAX_DISPLAY_CONTENT_SIZE): keep what does of it, as + # a cut line. + kept.append(text[:MAX_DISPLAY_CONTENT_SIZE]) + lines_truncated.append(line_number) + break + kept.append(text + line_end) + size += num_chars_for_line + if cut: + lines_truncated.append(line_number) + + truncated = truncated or budget.exceeded + content = ''.join(kept) + if truncated and content.endswith('\n'): + # Without the end of the last line kept. + content = content[:-1] + result: Dict[str, Any] = { 'content': content } + if truncated: + result['truncated'] = True + if lines_truncated: + result['lines_truncated'] = lines_truncated + return result + + +def display_text(handle: IO[str]) -> Dict[str, Any]: + '''``display_content`` of the text ``handle`` reads: at most the display + budget plus one read piece, even if a line is longer. + ''' + return display_content( + edgutil.iter_line_pieces(handle, MAX_DISPLAY_LINE_SIZE + 1) + ) + + +def trunc_display_content(content: str) -> Dict[str, Any]: + '''``display_content`` of ``content``, already in memory.''' + return display_text(io.StringIO(content)) def reviewer_of(request: Any) -> Optional[str]: diff --git a/dev_tools/pylibs/edgacknowledg/client/test/__init__.py b/dev_tools/pylibs/edgacknowledg/client/test/__init__.py index e3a53770ca..e70279fb66 100644 --- a/dev_tools/pylibs/edgacknowledg/client/test/__init__.py +++ b/dev_tools/pylibs/edgacknowledg/client/test/__init__.py @@ -6,6 +6,10 @@ '''AcknowlEDG test-review provider package.''' import asyncio +import contextlib +import filecmp +import functools +import io import os import re import shutil @@ -27,7 +31,7 @@ ) from edgacknowledg import ( - MAX_DISPLAY_CONTENT_SIZE, + MAX_DISPLAY_LINE_SIZE, ProtocolMessage, RequestTags, WebSocketsMessageType, @@ -40,9 +44,12 @@ ConfinedPath, IllegalFileAccessError, SandboxContext, + display_content, + display_text, reviewer_of, slice_source_range, trunc_display_content, + trunc_display_label, ) from edgutil import eprint @@ -83,7 +90,13 @@ def _create_status_dict(elog_file_paths: List[ConfinedPath], *, for elog_file_path in elog_file_paths: config_name = elog_file_path.parent.name with elog_file_path.open() as file_handle: - for line in file_handle: + # edgutil.MAX_LINE_CHARS is far more than any genuine status line + # needs, so a longer line is refused (failing the request, as a line + # that isn't a status line does), without being held whole. + for line in edgutil.iter_limited_lines( + file_handle, + edgutil.MAX_LINE_CHARS + ): key, parsed_diff = ( edgtest.runtest.parse_test_status_diff(line) ) @@ -92,7 +105,7 @@ def _create_status_dict(elog_file_paths: List[ConfinedPath], *, pending[key] = {} lines_by_test_config[key] = {} if parsed_diff.remark: - remarks[key] = parsed_diff.remark + remarks[key] = trunc_display_label(parsed_diff.remark) if config_name not in pending[key]: rt_file_path = _rt_path_for(elog_file_path.parent, key) @@ -106,9 +119,9 @@ def _create_status_dict(elog_file_paths: List[ConfinedPath], *, lines_by_test_config[key][config_name] = [] lines_by_test_config[key][config_name].append({ - "qualifier": parsed_diff.qualifier, - "variant": parsed_diff.variant, - "status": parsed_diff.status.full_status + "qualifier": trunc_display_label(parsed_diff.qualifier), + "variant": trunc_display_label(parsed_diff.variant), + "status": trunc_display_label(parsed_diff.status.full_status) }) keyed_statuses: _StatusDict = {} @@ -311,13 +324,17 @@ def test_id_to_path(test_id: str) -> ConfinedPath: )) return - async def form_curr_diff_text(config_name: str, - case_rt_path: ConfinedPath) -> str: - '''Live-diff observed vs recordings per command. - - Per-command body content is diffed (recording command line stripped), - then equal bodies are collapsed with # command comments — same - presentation as static. + async def write_curr_diff(config_name: str, + case_rt_path: ConfinedPath, + out_path: Path) -> None: + '''Write the live diff of observed output vs recordings to + ``out_path``. + + Per-command body content is diffed (recording command line + stripped), then equal bodies are collapsed with # command + comments, the same presentation as static. The per-command diffs + are files of its own, gone once ``out_path`` is written; none is + held in memory. ''' suite_name = test_id.split('/')[0] suite_path = test_home / 'tests' / suite_name @@ -336,8 +353,7 @@ async def form_curr_diff_text(config_name: str, with tempfile.TemporaryDirectory() as tmp_dir_name: tmp_dir = Path(tmp_dir_name) - entries: List[Tuple[Dict[str, Any], List[str]]] = [] - + entries: List[Tuple[Dict[str, Any], Path]] = [] with _run_test_output(case_rt_path.path, max_rt_entries, max_rt_member_bytes) as rt_output: for case_number, case in enumerate( @@ -381,13 +397,9 @@ async def form_curr_diff_text(config_name: str, if diff_hash is None: continue - with open(change_diff_path, 'r', - errors = 'replace') as file_handle: - diff_lines = file_handle.readlines() - command_entry = dict(command) command_entry['diff_hash'] = diff_hash - entries.append((command_entry, diff_lines)) + entries.append((command_entry, change_diff_path)) if case.get('matches') is None: continue @@ -421,59 +433,90 @@ async def form_curr_diff_text(config_name: str, if matches_diff_hash is None: continue - with open(matches_change_diff, 'r', - errors = 'replace') as file_handle: - matches_diff_lines = file_handle.readlines() - matches_entry = { 'command': edgtest.runtest.case_matches_annotation( case_number ), 'diff_hash': matches_diff_hash } - entries.append((matches_entry, matches_diff_lines)) - - return ''.join( - edgtest.runtest.format_collapsed_case_diffs(entries) - ) + entries.append((matches_entry, matches_change_diff)) + + with open(out_path, 'w', encoding = 'utf-8') as out_handle: + with contextlib.closing( + edgtest.runtest.iter_collapsed_case_diff_pieces( + [ + (entry, + functools.partial( + open, + path, + 'r', + encoding = 'utf-8', + errors = 'replace' + )) + for entry, path in entries + ], + edgutil.MAX_LINE_CHARS + ) + ) as pieces: + for piece, _ in pieces: + out_handle.write(piece) + # Reads stop at the display budget, even for a long line or member + # of the run artifact. + piece_limit = MAX_DISPLAY_LINE_SIZE + 1 if request_tag == RequestTags.TEST_SHOW_DIFF: with _run_test_output(rt_file_path.path, max_rt_entries, max_rt_member_bytes) as rt_output: - static_text = ''.join(rt_output.combined_static_diff()) + with contextlib.closing( + rt_output.iter_static_diff_pieces(piece_limit) + ) as pieces: + shown = display_content(pieces) client_response = { 'id': configured_test_id, - **trunc_display_content(static_text) + **shown } elif request_tag == RequestTags.TEST_SHOW_RAW_OUTPUT: with _run_test_output(rt_file_path.path, max_rt_entries, max_rt_member_bytes) as rt_output: - raw_output = ''.join(rt_output.combined_raw_output()) + with contextlib.closing( + rt_output.iter_raw_output_pieces(piece_limit) + ) as pieces: + shown = display_content(pieces) client_response = { 'id': configured_test_id, - **trunc_display_content(raw_output) + **shown } elif request_tag == RequestTags.TEST_SHOW_CURR_DIFF: rep_config = changes_file_path.parent.name group_configs = request_value.get('configs') or [rep_config] - rep_curr_text = await form_curr_diff_text(rep_config, rt_file_path) divergent_configs: List[str] = [] - for config_name in group_configs: - if config_name == rep_config: - continue - sibling_changes = changes_files[config_name] - sibling_rt = _rt_path_for(sibling_changes.parent, test_id) - if not sibling_rt.exists(): - divergent_configs.append(config_name) - continue - sibling_text = await form_curr_diff_text(config_name, sibling_rt) - if sibling_text != rep_curr_text: - divergent_configs.append(config_name) + with tempfile.TemporaryDirectory() as tmp_dir_name: + rep_path = Path(tmp_dir_name, 'curr.diff') + await write_curr_diff(rep_config, rt_file_path, rep_path) + for n, config_name in enumerate(group_configs): + if config_name == rep_config: + continue + sibling_changes = changes_files[config_name] + sibling_rt = _rt_path_for(sibling_changes.parent, test_id) + if not sibling_rt.exists(): + divergent_configs.append(config_name) + continue + # Compared a chunk at a time, then gone. + # Use a distinct path: filecmp caches results by path, size, + # and mtime even with shallow=False; reuse can hide divergence. + sibling_path = Path(tmp_dir_name, f"sibling-{n}.diff") + await write_curr_diff(config_name, sibling_rt, sibling_path) + if not filecmp.cmp(rep_path, sibling_path, shallow = False): + divergent_configs.append(config_name) + sibling_path.unlink() + + with open(rep_path, encoding = 'utf-8') as curr_diff_handle: + shown = display_text(curr_diff_handle) client_response = { 'id': configured_test_id, - **trunc_display_content(rep_curr_text) + **shown } if len(divergent_configs) != 0: @@ -579,8 +622,9 @@ async def form_curr_diff_text(config_name: str, ifc_dump_output = result.output.decode(errors = 'replace') if is_ranged: assert range_line is not None and range_context is not None + # Universal newlines, as a file opened as text has. reply_fields = slice_source_range( - ifc_dump_output.splitlines(), + io.StringIO(ifc_dump_output, newline = None), range_line, range_context ) @@ -595,11 +639,10 @@ async def form_curr_diff_text(config_name: str, range_context ) else: - # Only as much as can be shown is read (one character more marks - # the content as truncated). + # Only as much as can be shown is read. with path_to_show.open(errors = 'replace') as file_handle: - shown = file_handle.read(MAX_DISPLAY_CONTENT_SIZE + 1) - reply_fields = trunc_display_content(shown) + shown = display_text(file_handle) + reply_fields = shown await channel.send(message( RequestTags.TEST_SHOW_SOURCE, diff --git a/dev_tools/pylibs/edgacknowledg/forge/config.py b/dev_tools/pylibs/edgacknowledg/forge/config.py index 0ace242519..7a3dd77bd1 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/config.py +++ b/dev_tools/pylibs/edgacknowledg/forge/config.py @@ -3,22 +3,28 @@ # See https://edgcpp.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -'''acknowledg-forge configuration, read from the environment.''' +'''Typed TOML settings, command-line wiring, and service-only credentials.''' +import math import os +import re +import tomllib -from dataclasses import dataclass +from dataclasses import dataclass, field, fields, replace from pathlib import Path -from typing import Mapping, Optional - -from edgacknowledg.client import ( - default_api_host, - default_api_port, - default_ui_host, - default_ui_port, -) +from typing import Any, Mapping, Optional, Sequence +from urllib.parse import urlsplit ENV_PREFIX = 'ACKNOWLEDG_FORGE_' +SETTINGS_VARIABLE = ENV_PREFIX + 'SETTINGS_FILE' + +_LEGACY_ENVIRONMENT_VARIABLES = frozenset(( + 'EDG_ACKNOWLEDG_API_HOST', + 'EDG_ACKNOWLEDG_API_PORT', + 'EDG_ACKNOWLEDG_UI_HOST', + 'EDG_ACKNOWLEDG_UI_PORT', + 'EDG_ACKNOWLEDG_API_KEY', +)) # The defaults of the limits on what a pull request and its CI artifacts # may make the service do (``workspace``), which the settings may raise for @@ -32,286 +38,456 @@ # suite's name, is short. DEFAULT_MAX_PATTERNS = 20_000 # The largest diff (of tests/, in bytes) a pull request may have: it is -# held in memory, and copied and split into lines more than once. -DEFAULT_MAX_DIFF_BYTES = 64 * 1024 * 1024 +# kept with the review's rematerialized run (on disk, never in memory +# whole), and the builder applies all of it. +DEFAULT_MAX_DIFF_BYTES = 512 * 1024 * 1024 # The most entries an artifact zip may have, and the largest its central # directory may be: zipfile reads all of that directory into memory, before # we can count. CI's test results reach about 170,000 entries, of about 106 # bytes each there. DEFAULT_MAX_ARTIFACT_ENTRIES = 500_000 DEFAULT_MAX_ARTIFACT_DIRECTORY_BYTES = 64 * 1024 * 1024 +DEFAULT_MAX_RT_ENTRIES = 10_000 +DEFAULT_MAX_RT_MEMBER_BYTES = 64 * 1024 * 1024 class ConfigError(Exception): - '''Raised for a missing or malformed setting.''' + '''Raised for a missing or malformed setting; never includes its value.''' + + +def check_environment(environ: Mapping[str, str]) -> None: + '''Reject removed application variables without disclosing their values.''' + removed = sorted( + name for name in environ + if ( + name.startswith(ENV_PREFIX) and name != SETTINGS_VARIABLE + ) or name in _LEGACY_ENVIRONMENT_VARIABLES + ) + if len(removed) != 0: + raise ConfigError( + 'removed environment settings: ' + ', '.join(removed) + + '; move application settings to lowercase TOML keys and secrets to ' + 'CREDENTIALS_DIRECTORY' + ) -def read_secret(credential_name: str, - env_name: str, - *, - environ: Mapping[str, str] = os.environ) -> Optional[str]: - '''Return a secret, preferring a systemd credential over the environment. +def settings_path(environ: Optional[Mapping[str, str]] = None) -> Path: + environ = os.environ if environ is None else environ + check_environment(environ) + filename = environ.get(SETTINGS_VARIABLE, '').strip() + if not filename: + raise ConfigError(f"{SETTINGS_VARIABLE} must name a TOML file") + return Path(filename).absolute() - Credentials (``LoadCredential=`` / ``LoadCredentialEncrypted=``) keep the - secret out of the process environment; ``env_name`` is the fallback for - local development. - ''' - credentials_dir = environ.get('CREDENTIALS_DIRECTORY') - if credentials_dir is not None: - path = Path(credentials_dir, credential_name) - if path.is_file(): - return path.read_text(encoding = 'utf-8').rstrip('\r\n') - return environ.get(env_name) - - -def service_url(name: str, - *, - environ: Mapping[str, str] = os.environ) -> Optional[str]: - '''The URL of the ``name`` service (another container's server) that - ``ACKNOWLEDG_FORGE__SERVICE_URL`` sets, if any. - ''' - url = environ.get(f"{ENV_PREFIX}{name}_SERVICE_URL", '').strip() - if url == '': + +def read_secret(name: str, directory: Optional[Path]) -> Optional[str]: + '''Read a credential, including a development launcher's memfd link.''' + if directory is None: return None - if not url.startswith('ws://'): - raise ConfigError(f"{ENV_PREFIX}{name}_SERVICE_URL must be a ws:// URL") - return url + try: + value = (directory / name).read_text(encoding = 'utf-8').rstrip('\r\n') + except FileNotFoundError: + return None + except (OSError, UnicodeError): + raise ConfigError(f"cannot read credential {name}") from None + return value or None -def _environment_secrets(environ: Mapping[str, str], - names: dict[str, str]) -> tuple[str, ...]: - '''The variables (of ``names``, credential name -> variable) a secret - is actually read from, for lack of a credential. - ''' - credentials_dir = environ.get('CREDENTIALS_DIRECTORY') - return tuple( - env_name for credential_name, env_name in names.items() - if environ.get(env_name) is not None and ( - credentials_dir is None or - not Path(credentials_dir, credential_name).is_file() +@dataclass(frozen = True) +class ForgeCredentials: + '''Service-only secrets, kept separate from shared application settings.''' + forge_token: Optional[str] = field(default = None, repr = False) + webhook_secret: Optional[str] = field(default = None, repr = False) + acknowledg_api_key: Optional[str] = field(default = None, repr = False) + database_password: Optional[str] = field(default = None, repr = False) + + @classmethod + def from_directory(cls, directory: Optional[Path]) -> 'ForgeCredentials': + return cls( + forge_token = read_secret('forge-token', directory), + webhook_secret = read_secret('webhook-secret', directory), + acknowledg_api_key = read_secret('acknowledg-api-key', directory), + database_password = read_secret('database-password', directory) ) - ) + + +def check_service_bind(value: str) -> None: + '''Validate the service's host:port listener address.''' + host, _, port = value.rpartition(':') + if (not host or not port.isascii() or not port.isdecimal() or + len(port) > 5 or not 0 <= int(port) <= 65535): + raise ConfigError('service_bind must be host:port with a valid port') + + +def check_service_url(value: str, key: str) -> str: + '''A container's server: ``ws://`` only, within the deployment.''' + url = urlsplit(value) + if url.scheme != 'ws' or not url.netloc: + raise ConfigError(f"{key} must be a ws:// URL") + if url.path not in ('', '/'): + raise ConfigError(f"{key} must have an empty path or '/'") + return value + + +# A DNS name or IPv4 address, nothing that could change a URL's shape. +_HOST_NAME = re.compile(r'[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?') +_LOOPBACK_HOSTS = frozenset(('localhost', '127.0.0.1', '::1')) +# The AcknowlEDG client selects TLS (wss://, https://) by this port alone. +TLS_PORT = 443 + + +@dataclass(frozen = True) +class SqliteDatabaseConfig: + '''Non-secret SQLite connection settings.''' + path: Path + + +@dataclass(frozen = True) +class PostgresqlUnixDatabaseConfig: + '''Non-secret PostgreSQL connection settings for a Unix socket.''' + socket_directory: Path + name: str + username: str + port: int = 5432 + + +@dataclass(frozen = True) +class PostgresqlTcpDatabaseConfig: + '''Non-secret PostgreSQL connection settings for TCP.''' + host: str + name: str + username: str + port: int = 5432 + require_tls: bool = True + + +DatabaseConfig = ( + SqliteDatabaseConfig | + PostgresqlUnixDatabaseConfig | + PostgresqlTcpDatabaseConfig +) @dataclass(frozen = True) class ForgeServiceConfig: - '''Settings shared by every forge service.''' - # Root for clones, worktrees, and (by default) the database. + '''Non-secret application settings shared by the service and workers.''' home: Path - database_url: str - repositories: tuple[str, ...] - # Only pull requests targeting these branches are served (e.g. not ones - # into feature branches). A pull request retargeted away from them has its - # review removed, as if it were closed. - base_branches: frozenset[str] - # Forge API base URL override (e.g. GitHub Enterprise, self-hosted GitLab). - api_url: Optional[str] - # Seconds between polls; 0 polls once at startup only. - poll_interval: float - # 'host:port' to serve the web app (serving) on: health checks, and webhooks - # if enabled; None serves none. - service_bind: Optional[str] - # Whether the web app receives webhooks. - webhooks: bool - comments: bool - # CI workflow (GitHub workflow file or id; GitLab: unused) whose runs are - # tracked; None disables CI tracking. - ci_workflow: Optional[str] - # Seconds to wait for CI before a pull request counts as settled anyway. - settle_timeout: float - test_artifact: str - bench_artifact: str - max_artifact_bytes: int - max_extracted_bytes: int - max_artifact_entries: int - max_artifact_directory_bytes: int - max_diff_bytes: int - max_sparse_patterns: int - max_concurrency: int - rematerialize_timeout: float - unsandboxed: bool - # The builder's runner server and the publisher server, in their own - # containers (see compose.yaml); required unless unsandboxed. - builder_service_url: Optional[str] - publisher_service_url: Optional[str] - # A dev_tools-style directory (bin/ and pylibs/) with the EDG tooling the - # service runs (edg-rematerialize-test-run, ifc-dump): a frozen, reviewed - # copy (the container image embeds one), never the pull request's. tools_dir: Path - forge_token: Optional[str] - webhook_secret: Optional[str] - acknowledg_api_host: str - acknowledg_api_port: int - acknowledg_api_key: Optional[str] - acknowledg_ui_host: str - acknowledg_ui_port: int - # Secrets read from environment variables rather than credentials; any - # process can read its own initial environment (/proc/self/environ). - environment_secrets: tuple[str, ...] = () - - @classmethod - def from_environment(cls, - environ: Mapping[str, str] = os.environ - ) -> 'ForgeServiceConfig': - def get(name: str) -> Optional[str]: - value = environ.get(ENV_PREFIX + name) - if value is None or value.strip() == '': - return None - return value.strip() - - def get_number(name: str, default: float) -> float: - value = get(name) - if value is None: - return default - try: - number = float(value) - except ValueError: - raise ConfigError(f"{ENV_PREFIX}{name} must be a number") from None - if number < 0: - raise ConfigError(f"{ENV_PREFIX}{name} must not be negative") - return number - - def get_flag(name: str) -> bool: - value = get(name) - if value is None: - return False - if value.lower() in ('1', 'true', 'yes', 'on'): - return True - if value.lower() in ('0', 'false', 'no', 'off'): - return False - raise ConfigError(f"{ENV_PREFIX}{name} must be a boolean (1/0)") - - def get_list(name: str) -> tuple[str, ...]: - value = get(name) - if value is None: - return () - return tuple(item.strip() for item in value.split(',') if item.strip()) - - home_value = get('HOME') - if home_value is None: - raise ConfigError(f"{ENV_PREFIX}HOME must be set") - home = Path(home_value).absolute() - - repositories = get_list('REPOSITORIES') - if len(repositories) == 0: - raise ConfigError(f"{ENV_PREFIX}REPOSITORIES must list a repository") - - tools_value = get('TOOLS_DIR') - if tools_value is None: - raise ConfigError(f"{ENV_PREFIX}TOOLS_DIR must be set") - tools_dir = Path(tools_value).absolute() - if not (tools_dir / 'bin' / 'edg-rematerialize-test-run').is_file(): - raise ConfigError( - f"{ENV_PREFIX}TOOLS_DIR ({tools_dir}) has no " - f"bin/edg-rematerialize-test-run" - ) - - max_concurrency = int(get_number('MAX_CONCURRENCY', 2)) - if max_concurrency < 1: - raise ConfigError(f"{ENV_PREFIX}MAX_CONCURRENCY must be at least 1") + repositories: tuple[str, ...] + database: DatabaseConfig + base_branches: frozenset[str] = frozenset(('main',)) + api_url: Optional[str] = None + poll_interval: float = 300 + webhooks: bool = False + comments: bool = False + ci_workflow: Optional[str] = None + settle_timeout: float = 2 * 60 * 60 + test_artifact: str = 'test-results-all' + bench_artifact: str = 'bench-results' + max_artifact_bytes: int = 256 * 1024 * 1024 + max_extracted_bytes: int = 2 * 1024 * 1024 * 1024 + max_artifact_entries: int = DEFAULT_MAX_ARTIFACT_ENTRIES + max_artifact_directory_bytes: int = DEFAULT_MAX_ARTIFACT_DIRECTORY_BYTES + max_rt_entries: int = DEFAULT_MAX_RT_ENTRIES + max_rt_member_bytes: int = DEFAULT_MAX_RT_MEMBER_BYTES + max_diff_bytes: int = DEFAULT_MAX_DIFF_BYTES + max_sparse_patterns: int = DEFAULT_MAX_PATTERNS + max_concurrency: int = 2 + rematerialize_timeout: float = 15 * 60 + unsandboxed: bool = False + publisher_processes: int = 4 + publisher_request_timeout: float = 150 + acknowledg_api_host: str = 'localhost' + acknowledg_api_port: int = 2727 + acknowledg_ui_host: str = 'localhost' + acknowledg_ui_port: int = 5173 + # The deployment's wiring, which keeps each server to its one network. + # compose.yaml sets it on the service's command line (``with_wiring``); + # the TOML can't, so a settings change can't widen a listener. + service_bind: Optional[str] = None + builder_service_url: Optional[str] = None + publisher_service_url: Optional[str] = None - service_bind = get('SERVICE_BIND') - if service_bind is not None: - host, _, port = service_bind.rpartition(':') - if host == '' or not port.isdigit(): - raise ConfigError(f"{ENV_PREFIX}SERVICE_BIND must be host:port") - webhooks = get_flag('WEBHOOKS') - if get('WEBHOOK_BIND') is not None: - raise ConfigError( - f"{ENV_PREFIX}WEBHOOK_BIND is gone: set {ENV_PREFIX}SERVICE_BIND " - f"(compose.yaml does) and {ENV_PREFIX}WEBHOOKS=1 instead" - ) - if webhooks and service_bind is None: - raise ConfigError( - f"{ENV_PREFIX}WEBHOOKS is set, but {ENV_PREFIX}SERVICE_BIND is not" - ) + @staticmethod + def from_file(path: Optional[Path] = None) -> 'ForgeServiceConfig': + '''Read typed shared TOML settings without accessing credentials. - unsandboxed = get_flag('UNSANDBOXED') - builder_service_url = service_url('BUILDER', environ = environ) - publisher_service_url = service_url('PUBLISHER', environ = environ) - if not unsandboxed and (builder_service_url is None or - publisher_service_url is None): + Each service validates the settings required for its own role. + ''' + path = settings_path() if path is None else path.absolute() + try: + with path.open('rb') as stream: + data = tomllib.load(stream) + except (OSError, ValueError): + # TOML parser exceptions can quote the input, which may contain secrets. raise ConfigError( - f"{ENV_PREFIX}BUILDER_SERVICE_URL and " - f"{ENV_PREFIX}PUBLISHER_SERVICE_URL must be set, as compose.yaml " - f"does; for local development only, set {ENV_PREFIX}UNSANDBOXED=1 " - f"instead" - ) + f"cannot read settings {path}: invalid or unreadable TOML" + ) from None + try: + return _decode_settings(data, path) + except ConfigError as ex: + raise ConfigError(f"{path}: {ex}") from None - return cls( - home = home, - # A credential, as a remote database's URL holds a password. - database_url = ( - read_secret( - 'database-url', - f"{ENV_PREFIX}DATABASE_URL", - environ = environ - ) or f"sqlite+aiosqlite:///{home / 'state.db'}" - ), - repositories = repositories, - base_branches = frozenset(get_list('BASE_BRANCHES') or ('main',)), - api_url = get('API_URL'), - poll_interval = get_number('POLL_INTERVAL', 300), + def with_wiring(self, *, + service_bind: Optional[str] = None, + builder_service_url: Optional[str] = None, + publisher_service_url: Optional[str] = None + ) -> 'ForgeServiceConfig': + '''These settings with the service's command-line wiring.''' + if service_bind is not None: + check_service_bind(service_bind) + for key, value in (('builder_service_url', builder_service_url), + ('publisher_service_url', publisher_service_url)): + if value is not None: + check_service_url(value, key) + return replace( + self, service_bind = service_bind, - webhooks = webhooks, - comments = get_flag('COMMENTS'), - ci_workflow = get('CI_WORKFLOW'), - settle_timeout = get_number('SETTLE_TIMEOUT', 2 * 60 * 60), - test_artifact = get('TEST_ARTIFACT') or 'test-results-all', - bench_artifact = get('BENCH_ARTIFACT') or 'bench-results', - max_artifact_bytes = int( - get_number('MAX_ARTIFACT_BYTES', 256 * 1024 * 1024) - ), - max_extracted_bytes = int( - get_number('MAX_EXTRACTED_BYTES', 2 * 1024 * 1024 * 1024) - ), - max_artifact_entries = int( - get_number('MAX_ARTIFACT_ENTRIES', DEFAULT_MAX_ARTIFACT_ENTRIES) - ), - max_artifact_directory_bytes = int(get_number( - 'MAX_ARTIFACT_DIRECTORY_BYTES', - DEFAULT_MAX_ARTIFACT_DIRECTORY_BYTES - )), - max_diff_bytes = int( - get_number('MAX_DIFF_BYTES', DEFAULT_MAX_DIFF_BYTES) - ), - max_sparse_patterns = int( - get_number('MAX_SPARSE_PATTERNS', DEFAULT_MAX_PATTERNS) - ), - max_concurrency = max_concurrency, - rematerialize_timeout = get_number('REMATERIALIZE_TIMEOUT', 15 * 60), - unsandboxed = unsandboxed, builder_service_url = builder_service_url, - publisher_service_url = publisher_service_url, - tools_dir = tools_dir, - forge_token = read_secret( - 'forge-token', - f"{ENV_PREFIX}TOKEN", - environ = environ - ), - webhook_secret = read_secret( - 'webhook-secret', - f"{ENV_PREFIX}WEBHOOK_SECRET", - environ = environ - ), - acknowledg_api_host = default_api_host(), - acknowledg_api_port = default_api_port(), - acknowledg_api_key = read_secret( - 'acknowledg-api-key', - 'EDG_ACKNOWLEDG_API_KEY', - environ = environ - ), - acknowledg_ui_host = default_ui_host(), - acknowledg_ui_port = default_ui_port(), - # (Not the database URL, which is only a secret if it holds a - # password, and then the documentation says to use the credential.) - environment_secrets = _environment_secrets(environ, { - 'forge-token': f"{ENV_PREFIX}TOKEN", - 'webhook-secret': f"{ENV_PREFIX}WEBHOOK_SECRET", - 'acknowledg-api-key': 'EDG_ACKNOWLEDG_API_KEY', - }), + publisher_service_url = publisher_service_url ) @property def ci_tracked(self) -> bool: return self.ci_workflow is not None + + +def check_credential_transport(config: ForgeServiceConfig, + credentials: 'ForgeCredentials') -> None: + '''Refuse to send the AcknowlEDG API key off this machine without TLS.''' + if (credentials.acknowledg_api_key is not None and + config.acknowledg_api_host not in _LOOPBACK_HOSTS and + config.acknowledg_api_port != TLS_PORT): + raise ConfigError( + f"the acknowledg-api-key credential is only sent over TLS: set " + f"acknowledg_api_port = {TLS_PORT} for a host other than localhost" + ) + + +_PATH_KEYS = frozenset(('home', 'tools_dir')) +_LIST_KEYS = frozenset(('repositories', 'base_branches')) +_FLAG_KEYS = frozenset(('webhooks', 'comments', 'unsandboxed')) +_NUMBER_KEYS = frozenset(( + 'poll_interval', + 'settle_timeout', + 'rematerialize_timeout', + 'publisher_request_timeout', +)) +_INT_KEYS = frozenset(( + 'max_artifact_bytes', + 'max_extracted_bytes', + 'max_artifact_entries', + 'max_artifact_directory_bytes', + 'max_rt_entries', + 'max_rt_member_bytes', + 'max_diff_bytes', + 'max_sparse_patterns', + 'max_concurrency', + 'publisher_processes', + 'acknowledg_api_port', + 'acknowledg_ui_port', +)) +_WIRING_KEYS = frozenset(( + 'service_bind', + 'builder_service_url', + 'publisher_service_url', +)) +# Former wiring settings, now command-line arguments of each server. +_REMOVED_WIRING_KEYS = frozenset(( + 'builder_bind', + 'publisher_bind', + 'tools_bind', + 'tools_service_url', +)) + + +def _check_api_url(value: str) -> None: + '''The forge token is sent here: HTTPS, or HTTP to this machine only. + + Errors never quote the URL, which could hold userinfo. + ''' + try: + url = urlsplit(value) + hostname = url.hostname + has_userinfo = url.username is not None or url.password is not None + # Only for its ValueError on a malformed port. + _ = url.port + except ValueError: + raise ConfigError('api_url is not a valid URL') from None + if url.scheme not in ('https', 'http') or not hostname: + raise ConfigError('api_url must be an https:// URL') + if url.scheme == 'http' and hostname not in _LOOPBACK_HOSTS: + raise ConfigError( + 'api_url must use https:// for a host other than localhost' + ) + if has_userinfo or url.query or url.fragment: + raise ConfigError( + 'api_url must not hold credentials, a query, or a fragment; put the ' + 'token in the forge-token credential' + ) + + +def _decode_settings(data: dict[str, Any], path: Path) -> ForgeServiceConfig: + allowed = {f.name for f in fields(ForgeServiceConfig)} - _WIRING_KEYS + values: dict[str, Any] = {} + for key, value in data.items(): + if key in _WIRING_KEYS or key in _REMOVED_WIRING_KEYS: + raise ConfigError( + f"{key} is deployment wiring, set on the command line (compose.yaml), " + 'not in settings' + ) + if key not in allowed: + raise ConfigError(f"unknown or secret setting {key}") + if key == 'database': + # Decode after home, which supplies SQLite's default path. + continue + if key in _LIST_KEYS: + if (not isinstance(value, list) or + not all(isinstance(v, str) and v.strip() for v in value)): + raise ConfigError(f"{key} must be an array of nonempty strings") + values[key] = ( + frozenset(value) if key == 'base_branches' + else tuple(value) + ) + elif key in _FLAG_KEYS: + if type(value) is not bool: + raise ConfigError(f"{key} must be a boolean") + values[key] = value + elif key in _INT_KEYS or key in _NUMBER_KEYS: + types = (int,) if key in _INT_KEYS else (int, float) + if type(value) not in types or value < 0 or ( + isinstance(value, float) and not math.isfinite(value)): + raise ConfigError( + f"{key} must be a nonnegative finite " + + ('integer' if key in _INT_KEYS else 'number') + ) + values[key] = value + else: + if not isinstance(value, str) or not value.strip(): + raise ConfigError(f"{key} must be a nonempty string") + if key in _PATH_KEYS: + directory = Path(value) + values[key] = (path.parent / directory).absolute() + else: + values[key] = value + for key in ('home', 'tools_dir', 'repositories'): + if key not in values or not values[key]: + raise ConfigError(f"{key} is required") + tool = values['tools_dir'] / 'bin' / 'edg-rematerialize-test-run' + if not tool.is_file(): + raise ConfigError('tools_dir has no bin/edg-rematerialize-test-run') + # The local AcknowlEDG's ports are the defaults; any other host needs its + # port stated, since the port alone selects TLS (never a silent 80). + for kind, local_port in (('api', 2727), ('ui', 5173)): + host_key = f"acknowledg_{kind}_host" + port_key = f"acknowledg_{kind}_port" + host = values.get(host_key, 'localhost') + if _HOST_NAME.fullmatch(host) is None: + raise ConfigError( + f"{host_key} must be a host name, without a scheme, port, or path" + ) + if port_key not in values: + if host != 'localhost': + raise ConfigError( + f"{port_key} is required for a host other than localhost " + f"({TLS_PORT} for TLS)" + ) + values[port_key] = local_port + if 'api_url' in values: + _check_api_url(values['api_url']) + values['database'] = _decode_database( + data.get('database', {}), path.parent, values['home'] / 'state.db' + ) + config = ForgeServiceConfig(**values) + for key in ('max_concurrency', 'publisher_processes'): + if getattr(config, key) < 1: + raise ConfigError(f"{key} must be at least 1") + if config.publisher_request_timeout <= 0: + raise ConfigError('publisher_request_timeout must be positive') + for key in ('acknowledg_api_port', 'acknowledg_ui_port'): + if not 1 <= getattr(config, key) <= 65535: + raise ConfigError(f"{key} must be between 1 and 65535") + return config + + +def _validate_database_fields(data: dict[str, Any], + keys: Sequence[str]) -> None: + for key, value in data.items(): + if key not in keys: + raise ConfigError(f"unknown or unsupported database setting {key}") + if key == 'port': + if type(value) is not int or not 1 <= value <= 65535: + raise ConfigError('database.port must be an integer from 1 to 65535') + elif key == 'require_tls': + if type(value) is not bool: + raise ConfigError('database.require_tls must be a boolean') + elif not isinstance(value, str) or not value.strip(): + raise ConfigError(f"database.{key} must be a nonempty string") + + +def _decode_database(data: Any, directory: Path, + default_path: Path) -> DatabaseConfig: + if not isinstance(data, dict): + raise ConfigError('database must be a table') + backend = data.get('backend', 'sqlite') + if backend not in ('sqlite', 'postgresql-unix', 'postgresql-tcp'): + raise ConfigError( + 'database.backend must be sqlite, postgresql-unix, or postgresql-tcp' + ) + if backend == 'sqlite': + _validate_database_fields(data, ('backend', 'path')) + filename = data.get('path') + if filename is None: + path = default_path + elif filename == ':memory:': + path = Path(':memory:') + else: + path = (directory / filename).absolute() + return SqliteDatabaseConfig(path = path) + elif backend == 'postgresql-unix': + _validate_database_fields( + data, + ( + 'backend', + 'socket_directory', + 'port', + 'name', + 'username', + ) + ) + for key in ('socket_directory', 'name', 'username'): + if key not in data: + raise ConfigError(f"database.{key} is required for postgresql-unix") + socket_directory = Path(data['socket_directory']) + if not socket_directory.is_absolute(): + raise ConfigError('database.socket_directory must be an absolute path') + return PostgresqlUnixDatabaseConfig( + socket_directory = socket_directory, + port = data.get('port', 5432), + name = data['name'], + username = data['username'] + ) + else: + _validate_database_fields( + data, + ( + 'backend', + 'host', + 'port', + 'name', + 'username', + 'require_tls', + ) + ) + for key in ('host', 'name', 'username'): + if key not in data: + raise ConfigError(f"database.{key} is required for postgresql-tcp") + if '/' in data['host']: + raise ConfigError('database.host must be a TCP host name or IP address') + return PostgresqlTcpDatabaseConfig( + host = data['host'], + port = data.get('port', 5432), + name = data['name'], + username = data['username'], + require_tls = data.get('require_tls', True) + ) diff --git a/dev_tools/pylibs/edgacknowledg/forge/db/__init__.py b/dev_tools/pylibs/edgacknowledg/forge/db/__init__.py index a64970de9e..396083dbdd 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/db/__init__.py +++ b/dev_tools/pylibs/edgacknowledg/forge/db/__init__.py @@ -7,14 +7,18 @@ from edgacknowledg.forge.db.store import ( AnnouncementRecord, + DatabaseConfigError, ForgeStore, PullRequestRecord, SectionRecord, + create_database_engine, ) __all__ = [ 'AnnouncementRecord', + 'DatabaseConfigError', 'ForgeStore', 'PullRequestRecord', 'SectionRecord', + 'create_database_engine', ] diff --git a/dev_tools/pylibs/edgacknowledg/forge/db/store.py b/dev_tools/pylibs/edgacknowledg/forge/db/store.py index 55093b8d77..4787bd4b92 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/db/store.py +++ b/dev_tools/pylibs/edgacknowledg/forge/db/store.py @@ -10,6 +10,7 @@ doesn't depend on SQLAlchemy. ''' +import ssl import uuid from dataclasses import dataclass, field @@ -18,8 +19,8 @@ from typing import Any, Optional, Sequence from sqlalchemy import delete, event, func, select -from sqlalchemy.engine import Connection, make_url -from sqlalchemy.exc import IntegrityError +from sqlalchemy.engine import Connection, URL +from sqlalchemy.exc import ArgumentError, IntegrityError, InvalidRequestError from sqlalchemy.ext.asyncio import ( AsyncEngine, AsyncSession, @@ -36,6 +37,12 @@ PullRequestInfo, PullRequestState, ) +from edgacknowledg.forge.config import ( + DatabaseConfig, + PostgresqlTcpDatabaseConfig, + PostgresqlUnixDatabaseConfig, + SqliteDatabaseConfig, +) from edgacknowledg.forge.db.models import ( Announcement, CIRun, @@ -46,6 +53,10 @@ _MIGRATIONS_DIR = Path(__file__).parent / 'migrations' +class DatabaseConfigError(Exception): + '''An invalid connection configuration, reported without its values.''' + + @dataclass(frozen = True) class PullRequestRecord: id: int @@ -184,6 +195,75 @@ def _run_migrations(connection: Connection) -> None: command.upgrade(config, 'head') +def _configure_sqlite(dbapi_connection: Any, _record: Any) -> None: + cursor = dbapi_connection.cursor() + cursor.execute('PRAGMA foreign_keys = ON') + cursor.execute('PRAGMA journal_mode = WAL') + cursor.execute('PRAGMA busy_timeout = 10000') + cursor.close() + + +def _sqlite_engine(config: SqliteDatabaseConfig, + password: Optional[str]) -> AsyncEngine: + if password is not None: + raise DatabaseConfigError('sqlite accepts no database-password credential') + url = URL.create('sqlite+aiosqlite', database = str(config.path)) + options: dict[str, Any] = {} + if url.database == ':memory:': + # An in-memory database exists per connection; share one. + options['poolclass'] = StaticPool + engine = create_async_engine(url, **options) + event.listen(engine.sync_engine, 'connect', _configure_sqlite) + return engine + + +def _postgresql_engine( + config: PostgresqlUnixDatabaseConfig | PostgresqlTcpDatabaseConfig, *, + host: str, + password: Optional[str], + tls: ssl.SSLContext | bool) -> AsyncEngine: + url = URL.create( + 'postgresql+asyncpg', + host = host, + port = config.port, + database = config.name, + username = config.username, + password = password + ) + return create_async_engine(url, connect_args = {'ssl': tls}) + + +def create_database_engine(config: DatabaseConfig, *, + password: Optional[str] = None) -> AsyncEngine: + '''Build an engine using typed settings and separate credentials.''' + try: + match config: + case SqliteDatabaseConfig(): + engine = _sqlite_engine(config, password) + case PostgresqlUnixDatabaseConfig(): + engine = _postgresql_engine( + config, + host = str(config.socket_directory), + password = password, + tls = False + ) + case PostgresqlTcpDatabaseConfig(): + engine = _postgresql_engine( + config, + host = config.host, + password = password, + tls = ssl.create_default_context() if config.require_tls else False + ) + case _: + raise DatabaseConfigError('unsupported database backend') + except (ArgumentError, InvalidRequestError, ValueError, ImportError): + # SQLAlchemy errors may quote connection details or credentials. + raise DatabaseConfigError( + 'database connection settings are invalid or unsupported' + ) from None + return engine + + class ForgeStore: '''Async access to the acknowledg-forge database.''' @@ -192,25 +272,10 @@ def __init__(self, engine: AsyncEngine) -> None: self._sessions = async_sessionmaker(engine, expire_on_commit = False) @classmethod - async def open(cls, url: str) -> 'ForgeStore': - '''Connect to ``url`` and bring its schema up to date.''' - parsed_url = make_url(url) - is_sqlite = parsed_url.get_backend_name() == 'sqlite' - engine_kwargs: dict[str, Any] = {} - if is_sqlite and parsed_url.database in (None, '', ':memory:'): - # An in-memory database exists per connection; share one. - engine_kwargs['poolclass'] = StaticPool - engine = create_async_engine(url, **engine_kwargs) - - if is_sqlite: - def configure_sqlite(dbapi_connection: Any, _record: Any) -> None: - cursor = dbapi_connection.cursor() - cursor.execute('PRAGMA foreign_keys = ON') - cursor.execute('PRAGMA journal_mode = WAL') - cursor.execute('PRAGMA busy_timeout = 10000') - cursor.close() - - event.listen(engine.sync_engine, 'connect', configure_sqlite) + async def open(cls, config: DatabaseConfig, *, + password: Optional[str] = None) -> 'ForgeStore': + '''Connect using typed settings and a separately supplied password.''' + engine = create_database_engine(config, password = password) async with engine.begin() as connection: await connection.run_sync(_run_migrations) diff --git a/dev_tools/pylibs/edgacknowledg/forge/dev.py b/dev_tools/pylibs/edgacknowledg/forge/dev.py new file mode 100644 index 0000000000..64b65fd76c --- /dev/null +++ b/dev_tools/pylibs/edgacknowledg/forge/dev.py @@ -0,0 +1,291 @@ +# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM +# Exceptions. +# See https://edgcpp.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +'''Linux development launcher: credential bytes stay in anonymous memory. + +Only symlinks to sealed parent-held memfds are placed in the temporary +credentials directory. The service can reopen them through /proc even +when uv or subprocess closes inherited descriptors. + +Note that this does not isolate processes running as the same user or prevent +the kernel swapping memory. +''' + +import argparse +import contextlib +import fcntl +import getpass +import json +import os +import signal +import subprocess +import sys +import tempfile +import time + +from pathlib import Path +from typing import Iterator, Mapping, Optional + +from edgacknowledg.forge.config import ( + ConfigError, SETTINGS_VARIABLE, check_environment, +) + + +# Linux UAPI (linux/fcntl.h). Some standalone Python builds omit these +# names from fcntl even though memfd_create and sealing are supported. +_F_ADD_SEALS = getattr(fcntl, 'F_ADD_SEALS', 1024 + 9) +_F_SEALS = (getattr(fcntl, 'F_SEAL_SEAL', 0x0001) | + getattr(fcntl, 'F_SEAL_SHRINK', 0x0002) | + getattr(fcntl, 'F_SEAL_GROW', 0x0004) | + getattr(fcntl, 'F_SEAL_WRITE', 0x0008)) +_CREDENTIAL_NAMES = ( + 'forge-token', + 'webhook-secret', + 'acknowledg-api-key', + 'database-password', +) +_CREDENTIAL_PROBE_TIMEOUT = 5 +_SHUTDOWN_GRACE_PERIOD = 10 + + +@contextlib.contextmanager +def memory_credentials(directory: Path, + values: Mapping[str, bytes]) -> Iterator[Path]: + '''Keep credentials readable until the caller's child has stopped.''' + descriptors = [] + created = False + try: + directory.mkdir(mode = 0o700) + created = True + for name, value in values.items(): + if name not in _CREDENTIAL_NAMES: + raise ValueError('unknown credential name') + fd = os.memfd_create( + 'forge-credential', + os.MFD_CLOEXEC | os.MFD_ALLOW_SEALING + ) + descriptors.append(fd) + os.fchmod(fd, 0o600) + with os.fdopen(os.dup(fd), 'wb') as stream: + stream.write(value) + fcntl.fcntl(fd, _F_ADD_SEALS, _F_SEALS) + (directory / name).symlink_to(f"/proc/{os.getpid()}/fd/{fd}") + yield directory + finally: + for fd in descriptors: + os.close(fd) + # This directory is created by this context, never supplied by a user. + if created: + for path in directory.iterdir(): + path.unlink() + directory.rmdir() + + +def supervise(command: list[str], environ: Mapping[str, str]) -> int: + '''Forward shutdown signals and reap the child before closing secrets.''' + child: Optional[subprocess.Popen[bytes]] = None + received: Optional[int] = None + shutdown_deadline: Optional[float] = None + + def forward(signum: int, frame: object) -> None: + nonlocal received, shutdown_deadline + received = signum + if shutdown_deadline is None: + shutdown_deadline = time.monotonic() + _SHUTDOWN_GRACE_PERIOD + if child is not None: + try: + os.killpg(child.pid, signum) + except ProcessLookupError: + pass + + previous = {sig: signal.signal(sig, forward) + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)} + try: + child = subprocess.Popen( + command, env = dict(environ), + start_new_session = True, close_fds = True + ) + if received is not None: + forward(received, None) + while True: + try: + returncode = child.wait(timeout = 0.1) + break + except subprocess.TimeoutExpired: + if (shutdown_deadline is not None and + time.monotonic() >= shutdown_deadline): + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + returncode = child.wait() + break + return returncode if returncode >= 0 else 128 - returncode + finally: + if child is not None and child.poll() is None: + os.killpg(child.pid, signal.SIGKILL) + child.wait() + for sig, handler in previous.items(): + signal.signal(sig, handler) + + +def development_settings(home: Path, tools_dir: Path) -> str: + '''A non-secret TOML for the read-only public-repository demo.''' + return ( + f"home = {json.dumps(str(home))}\n" + f"tools_dir = {json.dumps(str(tools_dir))}\n" + 'repositories = ["edgcpp/compiler"]\n' + 'ci_workflow = "build-and-test-ci.yml"\n' + 'poll_interval = 60\n' + 'unsandboxed = true\n' + 'comments = false\n' + 'acknowledg_api_host = "localhost"\n' + 'acknowledg_api_port = 2727\n' + 'acknowledg_ui_host = "localhost"\n' + 'acknowledg_ui_port = 5173\n' + ) + + +def _token(use_gh: bool) -> bytes: + if use_gh: + try: + result = subprocess.run( + ['gh', 'auth', 'token'], check = True, + stdout = subprocess.PIPE, + stderr = subprocess.DEVNULL + ) + except (OSError, subprocess.CalledProcessError): + raise ConfigError( + 'cannot read the GitHub CLI token; run gh auth login' + ) from None + token = result.stdout.strip() + else: + # Refuse getpass's fallback to an echoed stdin read without a terminal. + try: + with open('/dev/tty', 'rb'): + pass + except OSError: + raise ConfigError( + 'a terminal is required for the token prompt; use --use-gh-token ' + 'or CREDENTIALS_DIRECTORY' + ) from None + token = getpass.getpass('Read-only GitHub token: ').encode('utf-8') + if not token: + raise ConfigError('the forge-token credential must not be empty') + return token + + +class _Termination(Exception): + def __init__(self, signum: int) -> None: + self.signum = signum + + +@contextlib.contextmanager +def _startup_signals() -> Iterator[None]: + '''Unwind temporary resources if interrupted before child supervision.''' + def stop(signum: int, frame: object) -> None: + raise _Termination(signum) + + previous = {sig: signal.signal(sig, stop) + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP)} + try: + yield + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + + +def _check_credentials(directory: Path) -> None: + '''Check child access to credentials, including parent-held memfds.''' + if not directory.is_dir(): + raise ConfigError(f"credentials path is not a directory: {directory}") + try: + probe = subprocess.run( + [ + sys.executable, + '-c', + 'from pathlib import Path; import sys; ' + 'root = Path(sys.argv[1]); ' + 'values = {name: p.read_bytes() for name in sys.argv[2:] ' + 'if (p := root / name).exists()}; ' + 'sys.exit(0 if values.get("forge-token", b"").rstrip(b"\\r\\n") ' + 'else 3)', + str(directory), + *_CREDENTIAL_NAMES, + ], + env = {}, + stdout = subprocess.DEVNULL, + stderr = subprocess.DEVNULL, + timeout = _CREDENTIAL_PROBE_TIMEOUT + ) + except subprocess.TimeoutExpired: + raise ConfigError('the credential access check timed out') from None + if probe.returncode == 3: + raise ConfigError( + 'the forge-token credential is required and must not be empty' + ) + if probe.returncode != 0: + raise ConfigError('the child cannot read the credentials directory') + + +def main() -> int: + parser = argparse.ArgumentParser(description = __doc__) + parser.add_argument( + '--use-gh-token', action = 'store_true', + help = 'use gh auth token instead of prompting' + ) + args = parser.parse_args() + try: + check_environment(os.environ) + if not hasattr(os, 'memfd_create') or not Path('/proc/self/fd').is_dir(): + raise ConfigError( + 'development credentials require Linux memfd and /proc' + ) + if args.use_gh_token and os.environ.get('CREDENTIALS_DIRECTORY'): + raise ConfigError('choose --use-gh-token or CREDENTIALS_DIRECTORY') + # No secret bytes are written anywhere under this directory. + with (_startup_signals(), + tempfile.TemporaryDirectory( + prefix = 'acknowledg-forge-dev-' + ) as tmp): + root = Path(tmp) + # gh may use these as its source; Forge must not inherit them. + environ = {k: v for k, v in os.environ.items() if k not in ( + 'GH_TOKEN', 'GITHUB_TOKEN', 'GH_ENTERPRISE_TOKEN', + 'GITHUB_ENTERPRISE_TOKEN', + )} + if not environ.get(SETTINGS_VARIABLE): + settings = root / 'settings.toml' + tools_dir = Path(__file__).resolve().parents[3] + settings.write_text( + development_settings(root / 'state', tools_dir), + encoding = 'utf-8' + ) + environ[SETTINGS_VARIABLE] = str(settings) + with contextlib.ExitStack() as stack: + if not environ.get('CREDENTIALS_DIRECTORY'): + credentials = stack.enter_context( + memory_credentials( + root / 'credentials', {'forge-token': _token(args.use_gh_token)} + ) + ) + environ['CREDENTIALS_DIRECTORY'] = str(credentials) + _check_credentials(Path(environ['CREDENTIALS_DIRECTORY'])) + return supervise( + [sys.executable, '-m', 'acknowledg_forge_github'], + environ + ) + except (ConfigError, OSError) as ex: + # OS errors here identify paths/operations, never credential contents. + print(f"development launcher: {ex}", file = sys.stderr) + return 2 + except _Termination as ex: + return 128 + ex.signum + except KeyboardInterrupt: + return 130 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/dev_tools/pylibs/edgacknowledg/forge/http.py b/dev_tools/pylibs/edgacknowledg/forge/http.py index 1398076465..eab84ea3ed 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/http.py +++ b/dev_tools/pylibs/edgacknowledg/forge/http.py @@ -17,7 +17,7 @@ from http import HTTPStatus from pathlib import Path -from typing import Any, Mapping, Optional +from typing import Any, AsyncIterator, Mapping, Optional import httpx @@ -26,6 +26,10 @@ _log = logging.getLogger(__name__) PAGE_SIZE = 100 +# API response limit, checked after each decoded chunk; downloads have +# their own limit. HTTPX can allocate a larger chunk during decompression. +# See the accepted HTTP decompression limitation in SANDBOXING.md. +MAX_RESPONSE_BYTES = 32 * 1024 * 1024 MAX_RATE_LIMIT_WAIT = 15 * 60.0 MAX_ATTEMPTS = 4 @@ -88,17 +92,19 @@ async def request(self, params: Optional[Mapping[str, Any]] = None, json_body: Optional[Any] = None) -> httpx.Response: '''Send a request, retrying transient failures and waiting out rate - limits; raise ``ForgeError`` on failure. + limits; raise ``ForgeError`` on failure, or for a response over + ``MAX_RESPONSE_BYTES`` (checked after each decoded chunk). ''' for attempt in range(MAX_ATTEMPTS): last_attempt = attempt + 1 == MAX_ATTEMPTS try: - response = await self._client.request( + request = self._client.build_request( method, path, params = params, json = json_body ) + response = await _send_capped(self._client, request, method, path) except httpx.HTTPError as ex: if last_attempt: raise ForgeError(f"{method} {path} failed: {ex}") from ex @@ -129,13 +135,12 @@ async def get_json(self, async def paginate(self, path: str, - params: Optional[Mapping[str, Any]] = None, - *, - key: Optional[str] = None) -> list[Any]: - '''Collect every page of a list endpoint (``key`` selects the list in - wrapped responses such as ``{"workflow_runs": [...]}``). + params: Optional[Mapping[str, Any]] = None, *, + key: Optional[str] = None) -> AsyncIterator[Any]: + '''Yield list items as pages are requested, without accumulating pages. + ``key`` selects the list in wrapped responses such as + ``{"workflow_runs": [...]}``. Stopping iteration avoids later requests. ''' - items: list[Any] = [] next_path: Optional[str] = path next_params: Optional[Mapping[str, Any]] = { **(params or {}), @@ -144,11 +149,11 @@ async def paginate(self, while next_path is not None: response = await self.request('GET', next_path, params = next_params) payload = response.json() - items += payload[key] if key is not None else payload + for item in payload[key] if key is not None else payload: + yield item # The next link is absolute and carries its own query. next_path = response.links.get('next', {}).get('url') next_params = None - return items async def download(self, path: str, @@ -179,6 +184,45 @@ async def download(self, raise ForgeError(f"download of {path} failed: {ex}") from ex +async def _send_capped(client: httpx.AsyncClient, + request: httpx.Request, + method: str, + path: str) -> httpx.Response: + '''Send ``request``; accumulate decoded chunks, raising ``ForgeError`` + when the accumulated size exceeds ``MAX_RESPONSE_BYTES``. + HTTPX decompresses each chunk before this check, so this is not a hard + bound on peak memory usage. + ''' + response = await client.send(request, stream = True) + try: + body = bytearray() + async for chunk in response.aiter_bytes(): + body += chunk + if len(body) > MAX_RESPONSE_BYTES: + raise ForgeError( + f"{method} {path}: the response is over {MAX_RESPONSE_BYTES} bytes" + ) + finally: + await response.aclose() + # The same response with its body in place; it is decoded already, so + # without the headers that describe how it was sent. + headers = [ + (name, value) for name, value in response.headers.raw + if name.lower() not in ( + b'content-encoding', + b'content-length', + b'transfer-encoding', + ) + ] + return httpx.Response( + response.status_code, + headers = headers, + content = bytes(body), + request = request, + extensions = response.extensions + ) + + async def _write_capped(stream: httpx.Response, destination: Path, max_bytes: int, diff --git a/dev_tools/pylibs/edgacknowledg/forge/local.py b/dev_tools/pylibs/edgacknowledg/forge/local.py index 7b1e78320c..9c1d6bf96e 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/local.py +++ b/dev_tools/pylibs/edgacknowledg/forge/local.py @@ -3,7 +3,7 @@ # See https://edgcpp.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -'''Without containers (``ACKNOWLEDG_FORGE_UNSANDBOXED=1``, for local +'''Without containers (``unsandboxed = true``, for local development only): the servers the builder, tools, and publisher containers run, served from the service's own process instead. @@ -27,6 +27,10 @@ from quart import Blueprint from edgacknowledg.forge import serving +from edgacknowledg.forge.publisher.pool import ( + DEFAULT_PROCESSES, + DEFAULT_REQUEST_TIMEOUT_S, +) from edgacknowledg.forge.publisher.protocol import MESSAGE_LIMIT from edgacknowledg.forge.publisher.server import create_publisher from edgacknowledg.forge.runner.client import RunnerClient @@ -70,11 +74,17 @@ async def local_runner(role: str, @contextlib.asynccontextmanager -async def local_deployment(*, - home: Path, - tools_dir: Path) -> AsyncIterator[LocalDeployment]: +async def local_deployment( + *, + home: Path, + tools_dir: Path, + processes: int = DEFAULT_PROCESSES, + request_timeout: float = DEFAULT_REQUEST_TIMEOUT_S + ) -> AsyncIterator[LocalDeployment]: '''Serve the builder and tools runner servers and the publisher server - for the deployment in ``home``, until the context exits. + (with ``processes`` pool processes, each given ``request_timeout`` + seconds a request) for the deployment in ``home``, until the context + exits. ''' async with contextlib.AsyncExitStack() as stack: builder = await stack.enter_async_context( @@ -84,10 +94,16 @@ async def local_deployment(*, local_runner('tools', home = home, tools_dir = tools_dir) ) # Its pool runs while it serves. - listening = await stack.enter_async_context(_serving( - create_publisher(tools_service_url = tools.url), - MESSAGE_LIMIT - )) + listening = await stack.enter_async_context( + _serving( + create_publisher( + tools_service_url = tools.url, + processes = processes, + request_timeout = request_timeout + ), + MESSAGE_LIMIT + ) + ) yield LocalDeployment( builder = builder, tools = tools, diff --git a/dev_tools/pylibs/edgacknowledg/forge/processor.py b/dev_tools/pylibs/edgacknowledg/forge/processor.py index 53491af47a..590e2e3e1a 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/processor.py +++ b/dev_tools/pylibs/edgacknowledg/forge/processor.py @@ -20,7 +20,7 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path, PurePosixPath -from typing import Any, Awaitable, Callable, Iterable, Optional +from typing import Any, Awaitable, Callable, Iterable, Optional, TypedDict import edgtest import edgutil @@ -60,6 +60,13 @@ _log = logging.getLogger(__name__) + +class _TemporaryPathOptions(TypedDict): + prefix: str + dir: Path + suffix: str + + CHANGES_NAME = 'Test - Changes' FAILURES_NAME = 'Test - Failures' @@ -71,11 +78,13 @@ CI_METADATA_FILE = 'ci-meta.json' # The largest that file may be, and the most (in bytes) the changes.elist -# files of a failures artifact may hold, and their test ids take as JSON: -# they are kept in its section's details, which go to the publisher in one -# message. +# files of a failures artifact may hold, and their test ids take as JSON +# in the database's section bookkeeping. MAX_CI_METADATA_BYTES = 64 * 1024 MAX_TEST_IDS_BYTES = 4 * 1024 * 1024 +# The longest test id taken from CI (a path within a suite); a longer line +# names no test, and is skipped rather than carried along. +MAX_TEST_ID_CHARS = 4096 # Retries for a failing pull request: 1 min, 2 min, ... up to 1 h; give up # after this many attempts until the pull request changes again. @@ -103,6 +112,11 @@ def _file_sha256(path: Path) -> str: return digest.hexdigest() +def _diff_patterns(path: Path, max_patterns: int) -> list[str]: + with open(path, encoding = 'utf-8') as handle: + return changes_patterns(handle, max_patterns = max_patterns) + + def _reusable(section: Optional[SectionRecord], source_key: str) -> Optional[SectionRecord]: '''Return ``section`` if it was built from ``source_key`` and its files @@ -117,7 +131,8 @@ def _reusable(section: Optional[SectionRecord], def _changes_test_ids(changes_elists: Iterable[Path]) -> list[str]: '''The test ids ``changes_elists`` name, within ``MAX_TEST_IDS_BYTES`` - (together, both as read and as JSON); ``WorkspaceError`` beyond. + (together, both as read and as JSON); ``WorkspaceError`` beyond. A line + longer than ``MAX_TEST_ID_CHARS`` is skipped. ''' test_ids = [] remaining = MAX_TEST_IDS_BYTES @@ -130,7 +145,14 @@ def _changes_test_ids(changes_elists: Iterable[Path]) -> list[str]: f"CI's changes.elist files are over {MAX_TEST_IDS_BYTES} bytes" ) text = content.decode('utf-8', errors = 'replace') - test_ids += [line.strip() for line in text.splitlines() if line.strip()] + for line in text.splitlines(): + if len(line) > MAX_TEST_ID_CHARS: + _log.warning('skipped a changes.elist line of %d characters', + len(line)) + continue + test_id = line.strip() + if test_id: + test_ids.append(test_id) # JSON escapes control and non-ASCII characters (e.g. the U+FFFD of bytes # that aren't UTF-8), up to six bytes each. if len(json.dumps(test_ids)) > MAX_TEST_IDS_BYTES: @@ -397,97 +419,120 @@ async def _changes_section(self, if reused is not None: return reused - diff = await self._workspaces.diff( - info.repository, - merge_base, - head_sha, - ['tests/'] - ) - if diff.strip() == '': - return None - - worktree = await self._workspaces.checkout( - info, - record.review_id, - 'base', - merge_base, - changes_patterns( - diff, - max_patterns = self._config.max_sparse_patterns - ) - ) - runs_dir = ensure_real_directory(worktree, PurePosixPath('tests', 'runs')) - run_dir = runs_dir / f"remat-{_sha256(source_key.encode())[:16]}" - remove_tree(run_dir) - # The builder sees the worktree read-only, so it writes elsewhere (on # the same file system, for the rename into place). + run_name = f"remat-{_sha256(source_key.encode())[:16]}" staging_root = ensure_real_directory( self._workspaces.build_staging_dir.parent, PurePosixPath(self._workspaces.build_staging_dir.name) ) - staging = Path(tempfile.mkdtemp( - prefix = f"{record.review_id}-{run_dir.name}-", - dir = staging_root - )) + tmp_dir_opts: _TemporaryPathOptions = { + 'prefix': f"{record.review_id}-{run_name}-", + 'dir': staging_root, + 'suffix': '', + } + staging = Path(tempfile.mkdtemp(**tmp_dir_opts)) try: - diff_file = staging / 'pull-request.diff' - diff_file.write_text(diff, encoding = 'utf-8') - timestamp = self._clock().strftime(edgutil.RUN_TIMESTAMP_FORMAT) - result = await self._sandbox.run( - [ - 'python3', - str(self._config.tools_dir / 'bin' / 'edg-rematerialize-test-run'), - '--runs-dir', str(staging), - '--run-timestamp', timestamp, - str(diff_file), - ], - read_only = [worktree], - writable = [staging], - env = {'EDG_TEST_HOME': str(worktree / 'tests')}, - cwd = worktree, - timeout = self._config.rematerialize_timeout - ) - if not result.succeeded: - reason = 'timed out' if result.timed_out else ( - f"exited with {result.returncode}" + tmp_diff_opts: _TemporaryPathOptions = { + **tmp_dir_opts, + 'suffix': '.diff', + } + with tempfile.NamedTemporaryFile(**tmp_diff_opts) as diff_handle: + # Only ever in this file, never held in memory whole: it may be large. + diff_file = Path(diff_handle.name) + await self._workspaces.diff( + info.repository, + merge_base, + head_sha, + ['tests/'], + diff_handle ) - raise ProcessingError( - f"edg-rematerialize-test-run {reason}: " - f"{result.output.strip()[-2000:]}" + diff_handle.flush() + if diff_file.stat().st_size == 0: + return None + patterns = await asyncio.to_thread( + _diff_patterns, + diff_file, + self._config.max_sparse_patterns ) - # The builder saw only pull request content and the frozen tooling, - # but what it wrote is read by the service and the publisher server: - # it may hold only directories and regular files (no links to service - # files). - try: - await asyncio.to_thread( - check_plain_tree, - staging, - allow_hard_links = False + + worktree = await self._workspaces.checkout( + info, + record.review_id, + 'base', + merge_base, + patterns + ) + # Prepare where the run goes once built: tests/runs/ in the + # worktree (through real directories only, not links the pull request + # committed), cleared of any earlier attempt so that the staging + # directory can be renamed into place. + runs_dir = ensure_real_directory( + worktree, + PurePosixPath('tests', 'runs') ) - except WorkspaceError as ex: - raise ProcessingError( - f"edg-rematerialize-test-run wrote something unsafe: {ex}" - ) from ex - if len(edgtest.find_changes_files(staging / timestamp)) == 0: - _log.warning( - '%s#%d: edg-rematerialize-test-run produced no changes.elist; ' - 'not serving %s', info.repository, info.number, CHANGES_NAME + run_dir = runs_dir / run_name + remove_tree(run_dir) + + timestamp = self._clock().strftime(edgutil.RUN_TIMESTAMP_FORMAT) + result = await self._sandbox.run( + [ + 'python3', + str(self._config.tools_dir / 'bin' / 'edg-rematerialize-test-run'), + '--runs-dir', + str(staging), + '--run-timestamp', + timestamp, + str(diff_file), + ], + read_only = [worktree], + writable = [staging], + env = {'EDG_TEST_HOME': str(worktree / 'tests')}, + cwd = worktree, + timeout = self._config.rematerialize_timeout ) - remove_tree(staging) - return None + if not result.succeeded: + reason = 'timed out' if result.timed_out else ( + f"exited with {result.returncode}" + ) + raise ProcessingError( + f"edg-rematerialize-test-run {reason}: " + f"{result.output.strip()[-2000:]}" + ) + # The builder saw only pull request content and the frozen tooling, + # but what it wrote is read by the service and the publisher server: + # it may hold only directories and regular files (no links to service + # files). + try: + await asyncio.to_thread( + check_plain_tree, + staging, + allow_hard_links = False + ) + except WorkspaceError as ex: + raise ProcessingError( + f"edg-rematerialize-test-run wrote something unsafe: {ex}" + ) from ex + if len(edgtest.find_changes_files(staging / timestamp)) == 0: + _log.warning( + '%s#%d: edg-rematerialize-test-run produced no changes.elist; ' + 'not serving %s', + info.repository, + info.number, + CHANGES_NAME + ) + return None + fingerprint = await asyncio.to_thread(_file_sha256, diff_file) staging.rename(run_dir) - except BaseException: + finally: remove_tree(staging) - raise return SectionRecord( kind = KIND_TEST_CHANGES, name = CHANGES_NAME, slug = section_slug(CHANGES_NAME), source_key = source_key, - content_fingerprint = _sha256(diff.encode()), + content_fingerprint = fingerprint, root = run_dir / timestamp, home = worktree / 'tests', details = {'run_dir': str(run_dir)}, diff --git a/dev_tools/pylibs/edgacknowledg/forge/publisher/acknowledg.py b/dev_tools/pylibs/edgacknowledg/forge/publisher/acknowledg.py index b5754209db..0492e6adce 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/publisher/acknowledg.py +++ b/dev_tools/pylibs/edgacknowledg/forge/publisher/acknowledg.py @@ -154,7 +154,8 @@ async def sync(self, if (wanted is not None and still_registered and wanted.name == current.record.name and wanted.source_key == current.record.source_key and - wanted.root == current.record.root): + wanted.root == current.record.root and + wanted.details == current.record.details): continue if still_registered: await client_review.remove_section(current.handler) diff --git a/dev_tools/pylibs/edgacknowledg/forge/publisher/pool.py b/dev_tools/pylibs/edgacknowledg/forge/publisher/pool.py index c08a4be2bd..4187192ebf 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/publisher/pool.py +++ b/dev_tools/pylibs/edgacknowledg/forge/publisher/pool.py @@ -29,9 +29,12 @@ answered (its process stays busy with it until then), and the answer dropped. -Messages on a channel are JSON objects, one per line: a process first says -``{"ready": true}``; then the server sends ``{"id", "section": ..., -"request": ...}``, and the process answers ``{"id", "replies": [...]}``, +The tools runner URL is passed on each process's command line using +``--tools-service-url``, including ephemeral local runner addresses. +Messages on the channel are JSON objects, one per line: a process first +says ``{"ready": true}``, then the server sends +``{"id", "section": ..., "request": ...}``, and the process answers +``{"id", "replies": [...]}``, plus ``"error": "illegal-access" | "failed"`` (and ``"message"``, ``"reviewer_id"``) if the handler raised; an answer to another id means the two lost track, and the process is replaced. A process that exits @@ -60,7 +63,7 @@ IllegalFileAccessError, reviewer_of, ) -from edgacknowledg.forge.config import ENV_PREFIX +from edgacknowledg.forge.config import SETTINGS_VARIABLE, settings_path from edgacknowledg.forge.db import SectionRecord from edgacknowledg.forge.publisher import SUBSCRIPTION_TYPES from edgacknowledg.forge.publisher.protocol import ( @@ -111,17 +114,15 @@ def python_paths() -> list[Path]: return result -def _process_env(tools_service_url: str) -> dict[str, str]: - '''A pool process's whole environment: nothing of the server's but - ``PATH``, and the tools runner to use. - ''' +def _process_env() -> dict[str, str]: + '''The worker gets a settings path, never credentials or overrides.''' return { 'PATH': os.environ.get('PATH', '/usr/local/bin:/usr/bin:/bin'), 'HOME': '/tmp', 'PYTHONUTF8': '1', 'PYTHONDONTWRITEBYTECODE': '1', 'PYTHONPATH': os.pathsep.join(str(path) for path in python_paths()), - f"{ENV_PREFIX}TOOLS_SERVICE_URL": tools_service_url, + SETTINGS_VARIABLE: str(settings_path()), } @@ -350,9 +351,14 @@ async def _launch(self, slot: _Slot) -> None: parent_end, process_end = socket.socketpair() try: process = await asyncio.create_subprocess_exec( - sys.executable, '-m', POOL_PROCESS_MODULE, str(process_end.fileno()), + sys.executable, + '-m', + POOL_PROCESS_MODULE, + str(process_end.fileno()), + '--tools-service-url', + self._tools_service_url, stdin = asyncio.subprocess.DEVNULL, - env = _process_env(self._tools_service_url), + env = _process_env(), pass_fds = (process_end.fileno(),), start_new_session = True ) @@ -368,8 +374,10 @@ async def _launch(self, slot: _Slot) -> None: sock = parent_end, limit = MESSAGE_LIMIT ) - ready = await asyncio.wait_for(slot.messages.readline(), - _READY_TIMEOUT_S) + ready = await asyncio.wait_for( + slot.messages.readline(), + _READY_TIMEOUT_S + ) except asyncio.TimeoutError: ready = b'' except BaseException: diff --git a/dev_tools/pylibs/edgacknowledg/forge/publisher/pool_process.py b/dev_tools/pylibs/edgacknowledg/forge/publisher/pool_process.py index 95c48d1318..893fa87ad4 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/publisher/pool_process.py +++ b/dev_tools/pylibs/edgacknowledg/forge/publisher/pool_process.py @@ -11,14 +11,13 @@ ``edgacknowledg.client``), has it answer the request, and drops it. The handlers read pull request content without following links, only under the section's tree, and run their commands (ifc-dump, cg_annotate) in the -tools runner that ``ACKNOWLEDG_FORGE_TOOLS_SERVICE_URL`` names. +tools runner named by its ``--tools-service-url`` argument. ''' import argparse import asyncio import json import logging -import os import socket from pathlib import Path @@ -32,7 +31,11 @@ ReviewClientHandler, ) from edgacknowledg.client.test import TestSourceHandler -from edgacknowledg.forge.config import ENV_PREFIX +from edgacknowledg.forge.config import ( + ConfigError, + ForgeServiceConfig, + check_service_url, +) from edgacknowledg.forge.db import SectionRecord from edgacknowledg.forge.publisher import ( KIND_BENCH, @@ -51,13 +54,12 @@ # Parallel cg_annotate runs in one process (it answers one request at a # time, so this only matters to a request running several). _BENCH_WORKERS = 2 -_MAX_RT_ENTRIES = 10_000 -_MAX_RT_MEMBER_BYTES = 64 * 1024 * 1024 def build_handler(section: SectionRecord, sandbox: Sandbox, - bench_workers: BackgroundWorkerPool) -> ReviewClientHandler: + bench_workers: BackgroundWorkerPool, + config: ForgeServiceConfig) -> ReviewClientHandler: '''The stock handler serving ``section``. Files are read without following links, and commands run on reviewers' behalf (ifc-dump, cg_annotate) only see the review trees, read-only. @@ -70,8 +72,8 @@ def build_handler(section: SectionRecord, context, context.subpath(section.home), context.subpath(section.root), - max_rt_entries = _MAX_RT_ENTRIES, - max_rt_member_bytes = _MAX_RT_MEMBER_BYTES + max_rt_entries = config.max_rt_entries, + max_rt_member_bytes = config.max_rt_member_bytes ) if section.kind == KIND_BENCH: session = BenchReviewSession( @@ -105,12 +107,13 @@ async def send(self, msg: ProtocolMessage) -> None: async def answer(request: dict[str, Any], sandbox: Sandbox, - bench_workers: BackgroundWorkerPool) -> dict[str, Any]: + bench_workers: BackgroundWorkerPool, + config: ForgeServiceConfig) -> dict[str, Any]: '''Answer one request from the server.''' channel = _OneRequestChannel(request['request']) try: section = decode_section(request['section']) - handler = build_handler(section, sandbox, bench_workers) + handler = build_handler(section, sandbox, bench_workers, config) await handler.handle_request(channel) # type: ignore[arg-type] except IllegalFileAccessError as ex: return { @@ -129,7 +132,7 @@ async def answer(request: dict[str, Any], return {'replies': channel.replies} -async def _run(channel_fd: int) -> None: +async def _run(channel_fd: int, tools_service_url: str) -> None: sock = socket.socket(fileno = channel_fd) # Commands run for reviewers must not reach the server. sock.set_inheritable(False) @@ -137,11 +140,12 @@ async def _run(channel_fd: int) -> None: sock = sock, limit = MESSAGE_LIMIT ) + # One snapshot for this worker's lifetime. + config = ForgeServiceConfig.from_file() bench_workers = BackgroundWorkerPool(_BENCH_WORKERS) # One connection to the tools runner, for every command this process # runs. - async with RunnerClient( - os.environ.get(f"{ENV_PREFIX}TOOLS_SERVICE_URL", '')) as tools: + async with RunnerClient(tools_service_url) as tools: sandbox = Sandbox(tools = tools) try: channel.write(b'{"ready": true}\n') @@ -151,7 +155,7 @@ async def _run(channel_fd: int) -> None: if line == b'': return request = json.loads(line) - reply = await answer(request, sandbox, bench_workers) + reply = await answer(request, sandbox, bench_workers, config) # Which request it answers. reply['id'] = request.get('id') channel.write((json.dumps(reply) + '\n').encode()) @@ -169,13 +173,24 @@ def main() -> None: type = int, help = 'the inherited socket to the server' ) + parser.add_argument( + '--tools-service-url', + required = True, + help = 'the tools runner server (ws://)' + ) args = parser.parse_args() logging.basicConfig( level = logging.INFO, format = '%(levelname)s %(name)s (pool process): %(message)s' ) try: - asyncio.run(_run(args.channel_fd)) + tools_service_url = check_service_url( + args.tools_service_url, + '--tools-service-url' + ) + asyncio.run(_run(args.channel_fd, tools_service_url)) + except ConfigError as ex: + parser.error(str(ex)) except KeyboardInterrupt: pass diff --git a/dev_tools/pylibs/edgacknowledg/forge/publisher/protocol.py b/dev_tools/pylibs/edgacknowledg/forge/publisher/protocol.py index d795e952c9..38bf66c54f 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/publisher/protocol.py +++ b/dev_tools/pylibs/edgacknowledg/forge/publisher/protocol.py @@ -23,6 +23,11 @@ The server answers each ({'ok': true} or {'error': 'failed', 'message'}) once it has queued what to publish, whether or not it is connected to AcknowlEDG: it publishes what is queued once it is. + +Section messages contain common fields and a ``variant`` object, with +``kind`` as its discriminant. Test sections have an empty variant; benchmark +variants contain ``baseline_dir`` and ``tag``. Database-only bookkeeping +is never sent. The same encoding is used for requests to publisher workers. ''' import uuid @@ -32,7 +37,7 @@ from edgacknowledg import is_valid_rich_name from edgacknowledg.forge.db import SectionRecord -from edgacknowledg.forge.publisher import ReviewIdentity +from edgacknowledg.forge.publisher import KIND_BENCH, ReviewIdentity # Longest message (a sync of every section of a review is small; this is a # sanity limit). @@ -59,6 +64,12 @@ def decode_review(value: dict[str, Any]) -> ReviewIdentity: def encode_section(section: SectionRecord) -> dict[str, Any]: + variant = {} + if section.kind == KIND_BENCH: + variant = { + 'baseline_dir': str(section.details['baseline_dir']), + 'tag': str(section.details['tag']), + } return { 'kind': section.kind, 'name': section.name, @@ -67,18 +78,28 @@ def encode_section(section: SectionRecord) -> dict[str, Any]: 'content_fingerprint': section.content_fingerprint, 'root': str(section.root), 'home': str(section.home), - 'details': section.details, + 'variant': variant, } def decode_section(value: dict[str, Any]) -> SectionRecord: + kind = str(value['kind']) + variant = dict(value['variant']) + # SectionRecord also serves the database; only benchmark metadata belongs + # in a publisher's copy of its details. + details = {} + if kind == KIND_BENCH: + details = { + 'baseline_dir': str(variant['baseline_dir']), + 'tag': str(variant['tag']), + } return SectionRecord( - kind = str(value['kind']), + kind = kind, name = str(value['name']), slug = str(value['slug']), source_key = str(value['source_key']), content_fingerprint = str(value['content_fingerprint']), root = Path(value['root']), home = Path(value['home']), - details = dict(value['details']), + details = details, ) diff --git a/dev_tools/pylibs/edgacknowledg/forge/publisher/server.py b/dev_tools/pylibs/edgacknowledg/forge/publisher/server.py index 97b89f40fb..37b6c65c89 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/publisher/server.py +++ b/dev_tools/pylibs/edgacknowledg/forge/publisher/server.py @@ -28,7 +28,6 @@ import asyncio import json import logging -import os import sys import uuid @@ -40,7 +39,11 @@ from edgacknowledg import PROTOCOL_REVISION, IncompatibleProtocolVersion from edgacknowledg.forge import serving -from edgacknowledg.forge.config import ENV_PREFIX, ConfigError +from edgacknowledg.forge.config import ( + ConfigError, + ForgeServiceConfig, + check_service_url, +) from edgacknowledg.forge.publisher.acknowledg import ( Delist, Prune, @@ -213,6 +216,9 @@ def main() -> None: parser = argparse.ArgumentParser( description = 'Publish for the service, over its connections.' ) + # The deployment's wiring, which compose.yaml sets: never the shared + # settings, which could otherwise widen the listener to the tools + # runner's network. parser.add_argument( '--host', default = '127.0.0.1', @@ -221,34 +227,27 @@ def main() -> None: ) parser.add_argument('--port', type = int, required = True) parser.add_argument( - '--processes', - type = int, - default = DEFAULT_PROCESSES, - help = 'pool processes answering reviewers (default: %(default)s)' - ) - parser.add_argument( - '--request-timeout', - type = float, - default = DEFAULT_REQUEST_TIMEOUT_S, - help = 'seconds a pool process may take to answer a request, before ' - 'it is replaced (default: %(default)s)' + '--tools-service-url', metavar = 'URL', required = True, + help = 'the tools runner server (ws://)' ) args = parser.parse_args() logging.basicConfig( level = logging.INFO, format = '%(levelname)s %(name)s: %(message)s' ) - tools_service_url = os.environ.get( - f"{ENV_PREFIX}TOOLS_SERVICE_URL", '' - ).strip() - if not tools_service_url.startswith('ws://'): - print(f"configuration error: {ENV_PREFIX}TOOLS_SERVICE_URL must be a " - f"ws:// URL", file = sys.stderr) + try: + config = ForgeServiceConfig.from_file() + tools_service_url = check_service_url( + args.tools_service_url, + '--tools-service-url' + ) + except ConfigError as ex: + print(f"configuration error: {ex}", file = sys.stderr) sys.exit(2) publisher = create_publisher( tools_service_url = tools_service_url, - processes = args.processes, - request_timeout = args.request_timeout + processes = config.publisher_processes, + request_timeout = config.publisher_request_timeout ) try: asyncio.run(serving.serve( diff --git a/dev_tools/pylibs/edgacknowledg/forge/runner/roles.py b/dev_tools/pylibs/edgacknowledg/forge/runner/roles.py index dd336a4a95..5114e23828 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/runner/roles.py +++ b/dev_tools/pylibs/edgacknowledg/forge/runner/roles.py @@ -80,7 +80,7 @@ def describe(self) -> dict[str, Any]: BUILD_STAGING_DIR = 'build-staging' # Seconds a command may run at most, by role (the service's own limits, e.g. -# ACKNOWLEDG_FORGE_REMATERIALIZE_TIMEOUT, apply below these). +# rematerialize_timeout, apply below these). BUILD_MAX_TIMEOUT_S = 60 * 60.0 TOOLS_MAX_TIMEOUT_S = 120.0 diff --git a/dev_tools/pylibs/edgacknowledg/forge/runner/server.py b/dev_tools/pylibs/edgacknowledg/forge/runner/server.py index ff2b69f115..42837c29cb 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/runner/server.py +++ b/dev_tools/pylibs/edgacknowledg/forge/runner/server.py @@ -47,6 +47,7 @@ import shlex import shutil import signal +import sys from dataclasses import dataclass from pathlib import Path @@ -58,6 +59,7 @@ from quart.wrappers import Websocket from edgacknowledg.forge import serving +from edgacknowledg.forge.config import ConfigError, ForgeServiceConfig from edgacknowledg.forge.runner.roles import ROLES, RunnerPolicy _log = logging.getLogger('edgacknowledg.forge.runner.server') @@ -362,14 +364,8 @@ def main(role: str) -> None: description = f"Serve the {role} runner: run its allowed commands for " f"its client." ) - parser.add_argument( - '--home', - type = Path, - required = True, - help = "the deployment's state directory, as mounted (only the " - "role's part of it need be)" - ) - parser.add_argument('--tools-dir', type = Path, required = True) + # The deployment's wiring, which compose.yaml sets: never the shared + # settings, which could otherwise widen a listener to other networks. parser.add_argument( '--host', default = '127.0.0.1', @@ -388,10 +384,12 @@ def main(role: str) -> None: level = logging.INFO, format = '%(levelname)s %(name)s: %(message)s' ) - policy = ROLES[role]( - home = args.home.absolute(), - tools_dir = args.tools_dir.absolute() - ) + try: + config = ForgeServiceConfig.from_file() + except ConfigError as ex: + print(f"configuration error: {ex}", file = sys.stderr) + sys.exit(2) + policy = ROLES[role](home = config.home, tools_dir = config.tools_dir) try: asyncio.run(serving.serve( create_runner(policy, container = args.container), diff --git a/dev_tools/pylibs/edgacknowledg/forge/service.py b/dev_tools/pylibs/edgacknowledg/forge/service.py index 35516802eb..36e83ebbeb 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/service.py +++ b/dev_tools/pylibs/edgacknowledg/forge/service.py @@ -18,19 +18,25 @@ enabled, the webhook receiver (``webhooks``) and its event worker. ''' +import argparse import asyncio import contextlib import logging +import os import signal import sys from dataclasses import dataclass +from pathlib import Path from typing import Callable, Optional from edgacknowledg.forge import Forge, ForgeCommenter, ForgeWebhooks, serving from edgacknowledg.forge.announcer import Announcer -from edgacknowledg.forge.config import ConfigError, ForgeServiceConfig -from edgacknowledg.forge.db import ForgeStore +from edgacknowledg.forge.config import ( + ConfigError, ForgeCredentials, ForgeServiceConfig, + check_credential_transport, +) +from edgacknowledg.forge.db import DatabaseConfigError, ForgeStore from edgacknowledg.forge.ingest import Ingestor from edgacknowledg.forge.local import local_deployment from edgacknowledg.forge.poller import run_poller @@ -56,16 +62,27 @@ class ForgeComponents: async def run_service(config: ForgeServiceConfig, + credentials: ForgeCredentials, components: ForgeComponents) -> None: + if not config.unsandboxed and ( + config.builder_service_url is None or + config.publisher_service_url is None): + raise ConfigError( + '--builder-service-url and --publisher-service-url are required unless ' + 'unsandboxed is true' + ) + if config.webhooks and config.service_bind is None: + raise ConfigError('webhooks requires --service-bind') + check_credential_transport(config, credentials) if config.webhooks and components.webhooks is None: raise ConfigError( - 'ACKNOWLEDG_FORGE_WEBHOOKS is set, but webhooks are not configured ' - '(is the webhook secret set?)' + 'webhooks is enabled, but webhooks are not configured (is the webhook ' + 'secret set?)' ) if config.comments and components.commenter is None: raise ConfigError( - 'ACKNOWLEDG_FORGE_COMMENTS is set, but commenting is not configured ' - '(is the forge token set?)' + 'comments is enabled, but commenting is not configured (is the forge ' + 'token set?)' ) config.home.mkdir(parents = True, exist_ok = True) @@ -74,7 +91,11 @@ async def run_service(config: ForgeServiceConfig, _log.warning('running without containers: nothing confines what reads ' 'pull requests; for local development only') local = await stack.enter_async_context( - local_deployment(home = config.home, tools_dir = config.tools_dir) + local_deployment( + home = config.home, tools_dir = config.tools_dir, + processes = config.publisher_processes, + request_timeout = config.publisher_request_timeout + ) ) builder = local.builder publisher_service_url = local.publisher_service_url @@ -85,14 +106,14 @@ async def run_service(config: ForgeServiceConfig, RunnerClient(config.builder_service_url) ) publisher_service_url = config.publisher_service_url - if len(config.environment_secrets) != 0: - _log.warning('secrets come from the environment (%s); use ' - 'credentials in production', - ', '.join(config.environment_secrets)) - await _run_components(config, components, builder, publisher_service_url) + await _run_components( + config, credentials, components, builder, + publisher_service_url + ) async def _run_components(config: ForgeServiceConfig, + credentials: ForgeCredentials, components: ForgeComponents, builder: RunnerClient, publisher_service_url: str) -> None: @@ -100,7 +121,9 @@ async def _run_components(config: ForgeServiceConfig, sandbox = Sandbox(builder = builder) await sandbox.check() - store = await ForgeStore.open(config.database_url) + store = await ForgeStore.open( + config.database, password = credentials.database_password + ) try: wake = asyncio.Event() ingestor = Ingestor(store, forge, config, wake = wake) @@ -114,7 +137,7 @@ async def _run_components(config: ForgeServiceConfig, url = publisher_service_url, api_host = config.acknowledg_api_host, api_port = config.acknowledg_api_port, - api_key = config.acknowledg_api_key, + api_key = credentials.acknowledg_api_key, ui_host = config.acknowledg_ui_host, ui_port = config.acknowledg_ui_port, reviews_dir = workspaces.reviews_dir, @@ -174,18 +197,50 @@ async def _run_components(config: ForgeServiceConfig, await store.aclose() -def main(build: Callable[[ForgeServiceConfig], ForgeComponents]) -> None: +def main(build: Callable[[ForgeServiceConfig, ForgeCredentials], + ForgeComponents]) -> None: '''Entry point for a forge service: read the configuration, build the - forge components with ``build``, and run until SIGINT/SIGTERM. + forge components with ``build(config, credentials)``, and run until + SIGINT/SIGTERM. + + The command line carries the deployment's wiring (compose.yaml sets it), + which the shared settings can't. ''' + parser = argparse.ArgumentParser( + description = 'Serve pull requests to AcknowlEDG; settings come from ' + 'the TOML file ACKNOWLEDG_FORGE_SETTINGS_FILE names.' + ) + parser.add_argument( + '--service-bind', + metavar = 'HOST:PORT', + help = 'serve health checks (and webhooks, if enabled) here; in a ' + 'container, on its frontend network only' + ) + parser.add_argument( + '--builder-service-url', metavar = 'URL', + help = "the builder's runner server (ws://)" + ) + parser.add_argument( + '--publisher-service-url', metavar = 'URL', + help = 'the publisher server (ws://)' + ) + args = parser.parse_args() logging.basicConfig( level = logging.INFO, format = '%(levelname)s %(name)s: %(message)s' ) logging.getLogger('alembic').setLevel(logging.WARNING) try: - config = ForgeServiceConfig.from_environment() - components = build(config) + directory = os.environ.get('CREDENTIALS_DIRECTORY') + config = ForgeServiceConfig.from_file().with_wiring( + service_bind = args.service_bind, + builder_service_url = args.builder_service_url, + publisher_service_url = args.publisher_service_url + ) + credentials = ForgeCredentials.from_directory( + Path(directory) if directory else None + ) + components = build(config, credentials) except ConfigError as ex: print(f"configuration error: {ex}", file = sys.stderr) sys.exit(2) @@ -197,12 +252,12 @@ async def run() -> None: for signal_number in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(signal_number, task.cancel) try: - await run_service(config, components) + await run_service(config, credentials, components) except asyncio.CancelledError: _log.info('shutting down') try: asyncio.run(run()) - except (ConfigError, SandboxUnavailableError) as ex: + except (ConfigError, DatabaseConfigError, SandboxUnavailableError) as ex: print(f"error: {ex}", file = sys.stderr) sys.exit(2) diff --git a/dev_tools/pylibs/edgacknowledg/forge/workspace.py b/dev_tools/pylibs/edgacknowledg/forge/workspace.py index 1571aca917..0620cd9734 100644 --- a/dev_tools/pylibs/edgacknowledg/forge/workspace.py +++ b/dev_tools/pylibs/edgacknowledg/forge/workspace.py @@ -26,6 +26,7 @@ import asyncio import base64 +import codecs import logging import os import re @@ -38,18 +39,18 @@ from dataclasses import dataclass from pathlib import Path, PurePosixPath -from typing import Iterable, Optional, Sequence +from typing import IO, Callable, Iterable, Optional, Sequence import edgtest import edgutil +from edgacknowledg.budget import Budget, BudgetExceeded from edgacknowledg.forge import Forge, PullRequestInfo from edgacknowledg.forge.config import ( DEFAULT_MAX_ARTIFACT_DIRECTORY_BYTES, DEFAULT_MAX_ARTIFACT_ENTRIES, DEFAULT_MAX_DIFF_BYTES, DEFAULT_MAX_PATTERNS, - ENV_PREFIX, ) from edgacknowledg.forge.runner.roles import BUILD_STAGING_DIR, REVIEWS_DIR from edgtest.runtest import get_bare_name @@ -71,6 +72,14 @@ # files) before it is killed; well within docker stop's default 10. _TERMINATE_ALLOWANCE = 5.0 +# The most output a git command run for its result (not one whose output +# goes to a sink) may have; far more than a rev-parse or for-each-ref of a +# pull request's refs has. +_MAX_GIT_OUTPUT_BYTES = 16 * 1024 * 1024 +# How much of a git command's stderr is kept (the rest is read, and +# dropped): a fetch relays what the remote says, which needn't be short. +_MAX_GIT_STDERR_BYTES = 64 * 1024 + class WorkspaceError(Exception): '''Raised when a workspace operation fails or input is unsafe.''' @@ -80,6 +89,10 @@ class UnsafeArchiveError(WorkspaceError): '''Raised for an artifact that is too large or has unsafe entries.''' +class _OutputTooLarge(Exception): + '''Raised when a process's output is over its limit (it is stopped).''' + + @dataclass(frozen = True) class GitResult: returncode: int @@ -129,26 +142,40 @@ def _patterns_for_changed_test_file(path: PurePosixPath) -> list[str]: return [f"/{_escape_pattern(str(path.parent))}/"] -def changed_paths(diff: str) -> list[str]: - '''Return the paths a ``git diff --no-renames`` output touches.''' - paths = [] - for line in diff.splitlines(): - match = _DIFF_HEADER.match(line) - if match is not None: - paths.append(match.group(1)) - return paths +def _add_patterns(patterns: dict[str, None], + more: list[str], + max_patterns: int) -> None: + '''Add ``more`` to ``patterns`` (an ordered set), within + ``max_patterns``. + ''' + patterns.update(dict.fromkeys(more)) + if len(patterns) > max_patterns: + raise WorkspaceError( + f"more than {max_patterns} sparse-checkout patterns (too many tests; " + f"max_sparse_patterns)" + ) -def changes_patterns(diff: str, +def changes_patterns(diff: IO[str], *, max_patterns: int = DEFAULT_MAX_PATTERNS) -> list[str]: - '''Sparse-checkout patterns for rematerializing ``diff`` (of tests/) on - the base tree: the test home and every affected test; at most - ``max_patterns``. + '''Sparse-checkout patterns for rematerializing the diff (of tests/) read + from ``diff`` on the base tree: the test home and every affected test; at + most ``max_patterns``. ''' patterns = dict.fromkeys(_TEST_HOME_PATTERNS) current_suite: Optional[str] = None - for line in diff.splitlines(): + limit = edgutil.MAX_LINE_CHARS + for piece, at_line_start in edgutil.iter_line_pieces(diff, limit + 1): + # Only whole lines within edgutil.MAX_LINE_CHARS matter: no header or + # expectation line is longer, and a longer one (e.g. of a recording) is + # passed over in pieces, never held whole. + if not at_line_start: + continue + text_length = len(piece) - (1 if piece.endswith('\n') else 0) + if text_length > limit: + continue + line = piece.rstrip('\n') match = _DIFF_HEADER.match(line) if match is not None: path = PurePosixPath(match.group(1)) @@ -198,20 +225,6 @@ def failures_patterns(test_ids: Iterable[str], return list(patterns) -def _add_patterns(patterns: dict[str, None], - more: list[str], - max_patterns: int) -> None: - '''Add ``more`` to ``patterns`` (an ordered set), within - ``max_patterns``. - ''' - patterns.update(dict.fromkeys(more)) - if len(patterns) > max_patterns: - raise WorkspaceError( - f"more than {max_patterns} sparse-checkout patterns (too many tests; " - f"{ENV_PREFIX}MAX_SPARSE_PATTERNS)" - ) - - def ensure_real_directory(root: Path, relative: PurePosixPath) -> Path: '''Create ``root/relative`` one component at a time, refusing to go through anything that isn't a real directory (e.g. a symlink a pull @@ -302,22 +315,25 @@ def remove_tree(path: Path) -> None: class _Budget: '''What extracting an artifact may still spend: bytes, and entries.''' def __init__(self, max_bytes: int, max_entries: int) -> None: - self.bytes = max_bytes - self.max_entries = max_entries - self.entries = max_entries + self.byte_budget = Budget(max_bytes) + self.entry_budget = Budget(max_entries) def spend(self, count: int) -> None: - self.bytes -= count - if self.bytes < 0: - raise UnsafeArchiveError('artifact exceeds the extracted size limit') + try: + self.byte_budget.spend(count) + except BudgetExceeded as ex: + raise UnsafeArchiveError( + 'artifact exceeds the extracted size limit' + ) from ex def count_entry(self) -> None: - self.entries -= 1 - if self.entries < 0: + try: + self.entry_budget.spend(1) + except BudgetExceeded as ex: raise UnsafeArchiveError( - f"artifact has more than {self.max_entries} entries " - f"({ENV_PREFIX}MAX_ARTIFACT_ENTRIES)" - ) + f"artifact has more than {self.entry_budget.limit} entries " + f"(max_artifact_entries)" + ) from ex def _central_directory_size(zip_path: Path) -> Optional[int]: @@ -362,7 +378,7 @@ def extract_artifact( if directory_size is not None and directory_size > max_directory_bytes: raise UnsafeArchiveError( f"artifact's central directory is over {max_directory_bytes} bytes " - f"({ENV_PREFIX}MAX_ARTIFACT_DIRECTORY_BYTES)" + f"(max_artifact_directory_bytes)" ) budget = _Budget(max_bytes, max_entries) try: @@ -455,9 +471,15 @@ async def git(self, cwd: Optional[Path] = None, stdin: Optional[str] = None, check: bool = True, - max_output: Optional[int] = None) -> GitResult: - '''Run git; with ``max_output``, its output may be at most that many - bytes (it is stopped, and ``WorkspaceError`` raised, beyond). + max_output: Optional[int] = None, + stdout_sink: Optional[Callable[[bytes], None]] = None + ) -> GitResult: + '''Run git; with ``stdout_sink``, its output goes there as it comes, + rather than into the result, and may be at most ``max_output`` bytes + (it is stopped, and ``_OutputTooLarge`` raised, beyond: the caller + knows which limit that is). Without, its output goes into the result, + and may be at most ``_MAX_GIT_OUTPUT_BYTES`` (``WorkspaceError`` + beyond). Only the start of its stderr is kept. ''' command = ['git'] if git_dir is not None: @@ -474,18 +496,31 @@ async def git(self, cwd = cwd, env = self._git_env() ) - if max_output is None: - stdout, stderr = await process.communicate( - stdin.encode() if stdin is not None else None - ) - else: - assert stdin is None - bounded = await _communicate_bounded(process, max_output) - if bounded is None: - raise WorkspaceError( - f"git {args[0]}'s output is over {max_output} bytes" + stdin_bytes = stdin.encode() if stdin is not None else None + if stdout_sink is None: + assert max_output is None + collected = bytearray() + try: + stderr = await _communicate_bounded( + process, + _MAX_GIT_OUTPUT_BYTES, + collected.extend, + stdin_bytes ) - stdout, stderr = bounded + except _OutputTooLarge: + raise WorkspaceError( + f"git {' '.join(args)} wrote over {_MAX_GIT_OUTPUT_BYTES} bytes" + ) from None + stdout = bytes(collected) + else: + assert max_output is not None + stderr = await _communicate_bounded( + process, + max_output, + stdout_sink, + stdin_bytes + ) + stdout = b'' assert process.returncode is not None result = GitResult( process.returncode, @@ -586,15 +621,55 @@ async def diff(self, repository: str, base: str, head: str, - paths: Sequence[str]) -> str: + paths: Sequence[str], + handle: IO[bytes]) -> None: + '''Write the diff of ``paths`` from ``base`` to ``head`` to ``handle`` + as it comes (it may be large): UTF-8, with what isn't replaced (U+FFFD). + ''' + decoder = codecs.getincrementaldecoder('utf-8')(errors = 'replace') async with self.lock(repository): - result = await self.git( - 'diff', '--no-renames', '--no-color', '--no-ext-diff', '--no-textconv', - base, head, '--', *paths, - git_dir = self.clone_dir(repository), - max_output = self._max_diff_bytes - ) - return result.stdout + # Decoded and encoded again so that what isn't UTF-8 is replaced: + # the builder's edg-rematerialize-test-run (and changes_patterns) + # read it as UTF-8 text, and the Changes section's fingerprint + # (the file's) stays what it was when the diff was decoded whole. + # Incrementally, so a character split between chunks survives. + + written = 0 + + def write(chunk: bytes) -> None: + nonlocal written + decoded = decoder.decode(chunk).encode() + written += len(decoded) + if written > self._max_diff_bytes: + raise _OutputTooLarge() + handle.write(decoded) + + try: + await self.git( + 'diff', + '--no-renames', + '--no-color', + '--no-ext-diff', + '--no-textconv', + base, + head, + '--', + *paths, + git_dir = self.clone_dir(repository), + max_output = self._max_diff_bytes, + stdout_sink = write + ) + # Finish decoding: bytes the decoder still holds (a character the + # output ended in the middle of) are written too, replaced. + decoded = decoder.decode(b'', final = True).encode() + if written + len(decoded) > self._max_diff_bytes: + raise _OutputTooLarge() + handle.write(decoded) + except _OutputTooLarge: + raise WorkspaceError( + f"the diff is over {self._max_diff_bytes} bytes " + f"(max_diff_bytes)" + ) from None async def checkout(self, info: PullRequestInfo, @@ -678,32 +753,68 @@ async def collect_garbage(self, repository: str, open_count: int) -> None: await self.git('gc', '--auto', '--quiet', git_dir = clone_dir) +async def _read_start(stream: asyncio.StreamReader, keep: int) -> bytes: + '''Read ``stream`` to its end; return its first ``keep`` bytes (the rest + is dropped as it comes). + ''' + kept = bytearray() + while chunk := await stream.read(64 * 1024): + kept += chunk[:max(0, keep - len(kept))] + return bytes(kept) + + +async def _write_stdin(process: asyncio.subprocess.Process, + data: bytes) -> None: + '''Write ``data`` to ``process``'s stdin, then close it.''' + assert process.stdin is not None + try: + process.stdin.write(data) + await process.stdin.drain() + except (BrokenPipeError, ConnectionResetError): + # It exited without reading it all; its exit status tells. + pass + finally: + process.stdin.close() + + async def _communicate_bounded(process: asyncio.subprocess.Process, - limit: int) -> Optional[tuple[bytes, bytes]]: - '''``process``'s output (stdout, stderr), once it exits; ``None``, having - stopped it, should its stdout be over ``limit`` bytes. Should this be - canceled (or fail), it is stopped too, rather than left writing to a - pipe no one reads. + limit: int, + sink: Callable[[bytes], None], + stdin: Optional[bytes] = None) -> bytes: + '''Pass ``process``'s stdout to ``sink`` as it comes, in chunks (none held + here), having written ``stdin`` (if any) to it; return the start of its + stderr (``_MAX_GIT_STDERR_BYTES``) once it exits. Should its stdout be + over ``limit`` bytes, it is stopped, and ``_OutputTooLarge`` raised. + Should this be canceled (or fail), it is stopped too, rather than left + writing to a pipe no one reads. ''' assert process.stdout is not None and process.stderr is not None - errors = asyncio.create_task(process.stderr.read()) - chunks: list[bytes] = [] + reading_stderr = asyncio.create_task( + _read_start(process.stderr, _MAX_GIT_STDERR_BYTES) + ) + writing_stdin = ( + asyncio.create_task(_write_stdin(process, stdin)) + if stdin is not None else None + ) size = 0 try: while chunk := await process.stdout.read(64 * 1024): size += len(chunk) if size > limit: - await edgutil.async_terminate_within(process, _TERMINATE_ALLOWANCE) - return None - chunks.append(chunk) - output = b''.join(chunks), await errors + raise _OutputTooLarge() + sink(chunk) + if writing_stdin is not None: + await writing_stdin + stderr = await reading_stderr await process.wait() - return output + return stderr except BaseException: await edgutil.async_terminate_within(process, _TERMINATE_ALLOWANCE) raise finally: - errors.cancel() + reading_stderr.cancel() + if writing_stdin is not None: + writing_stdin.cancel() def basic_auth_header(user: str, password: str) -> str: diff --git a/dev_tools/pylibs/edgtest/__init__.py b/dev_tools/pylibs/edgtest/__init__.py index 0b8ef00085..3eba4d3f02 100644 --- a/dev_tools/pylibs/edgtest/__init__.py +++ b/dev_tools/pylibs/edgtest/__init__.py @@ -306,6 +306,89 @@ def find_changes_files(run_dir: Path) -> Dict[str, Path]: changes_files[dir_entry] = changes_file return changes_files +def _remove_run_tree(path: Path) -> int: + '''Remove the directory path (of a test run) whole; return how many + .rt.tar files it held. + ''' + count = 0 + for _, _, file_names in os.walk(path): + count += sum( + 1 for file_name in file_names + if file_name.endswith(runtest.RUN_TEST_OUTPUT_SUFFIX) + ) + shutil.rmtree(path) + return count + +def prune_run_archives(run_dir: Path) -> Tuple[int, int, int]: + '''Prune the run in run_dir down to what a review of it reads (the .rt.tar + files changes.elist names; see _scan_changes_file), e.g., to keep CI's + uploads small. The replays directory is removed whole. In each + configuration, a suite with no test changes.elist names (every suite, + without a changes.elist) is removed whole: its directory, and its + suite- logs. Of the other suites, the .rt.tar files changes.elist + doesn't name are removed. Return how many .rt.tar files were kept, how + many removed, and how many suites were removed. + ''' + kept = 0 + removed = 0 + suites = 0 + for dir_entry in os.listdir(run_dir): + config_dir = run_dir / dir_entry + if not config_dir.is_dir() or config_dir.is_symlink(): + continue + if dir_entry == 'replays': + removed += _remove_run_tree(config_dir) + continue + if dir_entry in ('recording-updates', 'patched-tests'): + continue + + # The .rt.tar files changes.elist names, by suite. + named: Dict[str, Set[Path]] = {} + changes_file = config_dir / 'changes.elist' + if changes_file.is_file(): + with open(changes_file, 'r') as file_handle: + test_ids = [ + test_id for test_id in (line.strip() for line in file_handle) + if test_id != '' + ] + for test_id in test_ids: + test_path = PurePosixPath(test_id) + if (test_path.is_absolute() or '..' in test_path.parts or + len(test_path.parts) < 2): + raise ValueError(f"invalid test id in {changes_file}: {test_id!r}") + named.setdefault(test_path.parts[0], set()).add( + config_dir / test_path.with_suffix(runtest.RUN_TEST_OUTPUT_SUFFIX) + ) + + # Each suite edgy ran (and logged) in the configuration. + for log_path in config_dir.glob('suite-*.elog'): + suite = log_path.stem[len('suite-'):] + suite_dir = config_dir / suite + walkable = suite_dir.is_dir() and not suite_dir.is_symlink() + if suite not in named: + if walkable: + removed += _remove_run_tree(suite_dir) + log_path.unlink() + try: + log_path.with_suffix('.log').unlink() + except FileNotFoundError: + pass + suites += 1 + continue + if not walkable: + continue + for root, _, file_names in os.walk(suite_dir): + for file_name in file_names: + if not file_name.endswith(runtest.RUN_TEST_OUTPUT_SUFFIX): + continue + file_path = Path(root, file_name) + if file_path in named[suite]: + kept += 1 + else: + file_path.unlink() + removed += 1 + return kept, removed, suites + def describe_rt_path(rt_file_path: Path, run_dir: Path) -> Tuple[str, str]: '''Return (relative_rt_path, config_name) for an .rt.tar under the run. diff --git a/dev_tools/pylibs/edgtest/exec/__init__.py b/dev_tools/pylibs/edgtest/exec/__init__.py index c5fcd8fbf4..5d1d3d3b4d 100644 --- a/dev_tools/pylibs/edgtest/exec/__init__.py +++ b/dev_tools/pylibs/edgtest/exec/__init__.py @@ -137,7 +137,8 @@ def write_recording_file(dest: Path, command: str, body_file: Path) -> None: '''Write a recording as command line + observed body content.''' dest.parent.mkdir(parents = True, exist_ok = True) with open(dest, 'wb') as out_handle, open(body_file, 'rb') as body_handle: - out_handle.write(f"{command}\n".encode('utf-8')) + command_line = runtest.format_cmd_for_display(command) + out_handle.write(f"{command_line}\n".encode('utf-8')) shutil.copyfileobj(body_handle, out_handle) async def check_output_mismatch( @@ -257,6 +258,7 @@ def make_status_message_with_explicit_status(test_path: Path, case_tag: str, status: str) -> str: test_type_str = runtest.test_type_to_string(test_type) + case_tag = runtest.format_cmd_for_display(case_tag) return f"{test_path}:{test_type_str}:{case_tag}:{status}:{test_remark}" def make_requirements_not_met_status(test_path: Path, diff --git a/dev_tools/pylibs/edgtest/runtest/__init__.py b/dev_tools/pylibs/edgtest/runtest/__init__.py index 1a22a0c1dc..600003b04c 100644 --- a/dev_tools/pylibs/edgtest/runtest/__init__.py +++ b/dev_tools/pylibs/edgtest/runtest/__init__.py @@ -12,6 +12,7 @@ import functools import hashlib import io +import itertools import json import os import re @@ -27,8 +28,8 @@ from collections import OrderedDict from pathlib import Path, PurePath from typing import ( - Any, Callable, Dict, IO, Iterator, List, Mapping, NamedTuple, Optional, - Pattern, Set, Tuple, cast + Any, Callable, Dict, Generator, IO, Iterator, List, Mapping, NamedTuple, + Optional, Pattern, Set, Tuple, cast ) def get_default_config_name(env: Optional[Dict[str, str]] = None) -> str: @@ -1119,9 +1120,13 @@ async def write_content_diff( env = env ) +def format_cmd_for_display(text: str) -> str: + '''Escape carriage returns and newlines for single-line text output.''' + return text.replace('\r', '\\r').replace('\n', '\\n') + def case_command_from_metadata(command: Dict[str, Any]) -> str: - '''Return the command string used for # annotations.''' - return command.get('command') or '' + '''Return the command string used for single-line # annotations.''' + return format_cmd_for_display(command.get('command') or '') def split_unified_diff_headers( lines: List[str]) -> Tuple[List[str], List[str]]: @@ -1204,6 +1209,59 @@ def emit_one(commands: List[Dict[str, Any]], result.extend(emit_one(group['commands'], group['diff_lines'])) return result +# A text that is opened when (and if) it is read. +TextOpener = Callable[[], IO[str]] + +def _hash_text_stream(open_text: TextOpener) -> str: + '''``hash_diff_text`` of the text ``open_text`` opens, read in chunks.''' + digest = hashlib.sha256() + with open_text() as handle: + while True: + chunk = handle.read(64 * 1024) + if not chunk: + break + digest.update(chunk.encode('utf-8', errors = 'replace')) + return digest.hexdigest() + +def iter_collapsed_case_diff_pieces( + command_diff_entries: List[Tuple[Dict[str, Any], TextOpener]], + limit: int) -> Generator[Tuple[str, bool], None, None]: + '''The collapsed ``format_collapsed_case_diffs`` of + ``command_diff_entries`` (with their stored labels), whose diffs are + opened rather than given as lines, as ``edgutil.iter_line_pieces`` pieces + of at most ``limit`` characters (each ``# command`` comment is one piece, + however long). A diff is read only as far as the pieces are taken, so + neither a long line nor a long diff is held whole. + ''' + # First-seen order of unique diff bodies (keyed by stored hash). + groups: Dict[str, Tuple[List[Dict[str, Any]], TextOpener]] = ( + OrderedDict() + ) + for command, open_text in command_diff_entries: + group_key = command.get('diff_hash') or _hash_text_stream(open_text) + if group_key not in groups: + groups[group_key] = ([], open_text) + groups[group_key][0].append(command) + + for commands, open_text in groups.values(): + with open_text() as handle: + pieces = edgutil.iter_line_pieces(handle, limit) + # The ---/+++ header, if it has one, comes before the comments; its + # lines are short (labels), so each must be one piece: a diff whose + # first lines are longer has no header (and isn't held to look). + head = list(itertools.islice(pieces, 2)) + if (len(head) == 2 and + all(at_line_start and piece.endswith('\n') + for piece, at_line_start in head) and + head[0][0].startswith('---') and + head[1][0].startswith('+++')): + yield from head + head = [] + for command in commands: + yield f"# {case_command_from_metadata(command)}\n", True + yield from head + yield from pieces + class RtTarContents: '''Manifest of filesystem files to pack into a .rt.tar[.gz]. @@ -1377,6 +1435,16 @@ def _require_tar(self) -> tarfile.TarFile: def _read_lines(self, member_name: str) -> List[str]: return _read_tar_lines_member(self._require_tar(), member_name) + def open_text_member(self, member_name: str) -> IO[str]: + '''Open a member as text (UTF-8, errors replaced), read as it is + decompressed: nothing is held but what is read. + ''' + return io.TextIOWrapper( + _extract_tar_member(self._require_tar(), member_name), + encoding = 'utf-8', + errors = 'replace' + ) + @property def metadata(self) -> Dict[str, Any]: if self._metadata is None: @@ -1458,9 +1526,9 @@ def iter_command_diffs(self) -> Iterator[List[str]]: for _entry, diff_lines in self.iter_command_diff_entries(): yield diff_lines - def iter_command_diff_entries( - self) -> Iterator[Tuple[Dict[str, Any], List[str]]]: - '''Yield (annotation metadata, change.diff lines) for diffs. + def iter_command_diff_members( + self) -> Iterator[Tuple[Dict[str, Any], str]]: + '''Yield (annotation metadata, change.diff member name) for diffs. Command diffs use the stored command metadata. Matches diffs use a synthetic command string ``Matches for case N``. @@ -1471,21 +1539,29 @@ def iter_command_diff_entries( case.get('commands', []), start = 1 ): - diff_lines = self.command_diff(case_number, command_number) - if diff_lines is not None: - yield command, diff_lines + member = f"cases/{case_number}/{command_number}/change.diff" + if member in self.member_names: + yield command, member matches = case.get('matches') if matches is None: continue - matches_diff_lines = self.case_matches_diff(case_number) - if matches_diff_lines is None: + member = f"cases/{case_number}/matches/change.diff" + if member not in self.member_names: continue matches_entry = { 'command': case_matches_annotation(case_number), 'diff_hash': matches.get('diff_hash') } - yield matches_entry, matches_diff_lines + yield matches_entry, member + + def iter_command_diff_entries( + self) -> Iterator[Tuple[Dict[str, Any], List[str]]]: + '''Yield (annotation metadata, change.diff lines) for diffs (see + ``iter_command_diff_members``). + ''' + for entry, member in self.iter_command_diff_members(): + yield entry, self._read_lines(member) def combined_static_diff(self, *, @@ -1503,6 +1579,48 @@ def combined_static_diff(self, collapse = collapse ) + def iter_static_diff_pieces( + self, + limit: int) -> Generator[Tuple[str, bool], None, None]: + '''``combined_static_diff()`` (collapsed, with the stored labels) as + ``edgutil.iter_line_pieces`` pieces of at most ``limit`` characters, + each member read only as far as the pieces are taken. + ''' + return iter_collapsed_case_diff_pieces( + [ + (entry, functools.partial(self.open_text_member, member)) + for entry, member in self.iter_command_diff_members() + ], + limit + ) + + def iter_raw_output_pieces( + self, + limit: int) -> Generator[Tuple[str, bool], None, None]: + '''``combined_raw_output()`` as ``edgutil.iter_line_pieces`` pieces of + at most ``limit`` characters (each ``# `` header line is one piece, + however long), each member read only as far as the pieces are taken. + ''' + for case_number, case in enumerate(self.metadata.get('cases', []), + start = 1): + for command_number, command in enumerate( + case.get('commands', []), + start = 1 + ): + command_str = case_command_from_metadata(command) + if command_str: + yield f"# {command_str}\n", True + with self.open_text_member( + f"cases/{case_number}/{command_number}/output.txt" + ) as handle: + yield from edgutil.iter_line_pieces(handle, limit) + + matches_member = f"cases/{case_number}/matches/output.txt" + if matches_member in self.member_names: + yield f"# {case_matches_annotation(case_number)}\n", True + with self.open_text_member(matches_member) as handle: + yield from edgutil.iter_line_pieces(handle, limit) + def combined_raw_output(self) -> List[str]: '''Concatenate command lines from metadata with each command's body. diff --git a/dev_tools/pylibs/edgutil/__init__.py b/dev_tools/pylibs/edgutil/__init__.py index a3cb0458a9..49ce5d8e63 100644 --- a/dev_tools/pylibs/edgutil/__init__.py +++ b/dev_tools/pylibs/edgutil/__init__.py @@ -27,7 +27,9 @@ Callable, Coroutine, Dict, + IO, Iterable, + Iterator, List, Optional, Set, @@ -193,6 +195,53 @@ def __exit__(self, def eprint(msg: str) -> None: print(msg, file = sys.stderr) +def iter_line_pieces(handle: IO[str], + limit: int) -> Iterator[Tuple[str, bool]]: + '''Yield ``(piece, at_line_start)`` for the text ``handle``, a line at a + time but never holding more than ``limit`` characters of one: a longer + line comes as several pieces, only the first of which has + ``at_line_start`` set. Concatenating the pieces gives back the text + exactly. + ''' + at_line_start = True + while True: + piece = handle.readline(limit) + if not piece: + break + yield piece, at_line_start + at_line_start = piece.endswith('\n') + +class LineTooLongError(ValueError): + '''Raised by ``iter_limited_lines`` for a line over its limit.''' + + def __init__(self, line_number: int, limit: int) -> None: + super().__init__( + f"line {line_number} is longer than {limit} characters" + ) + self.line_number = line_number + self.limit = limit + +def iter_limited_lines(handle: IO[str], limit: int) -> Iterator[str]: + '''Yield each line of the text ``handle``, with its line end ('\\n') if + it has one; raise ``LineTooLongError`` for a line longer than ``limit`` + characters (its end not counted), having read only ``limit + 1`` of + them. + ''' + # Every piece up to an overlong line's first is a whole line. + for line_number, (line, _) in enumerate( + iter_line_pieces(handle, limit + 1), + start = 1 + ): + text_length = len(line) - (1 if line.endswith('\n') else 0) + if text_length > limit: + raise LineTooLongError(line_number, limit) + yield line + +# The longest line a reader of untrusted line-oriented text (a pull +# request's diff, a CI artifact's files) need hold: far more than any path, +# test name, or status line, and small enough not to matter in memory. +MAX_LINE_CHARS = 64 * 1024 + def indent_text_block(text_block: str, n_times: int = 1) -> str: indent = ' ' * n_times return re.sub(r'^(.)', f"{indent}\\1", text_block, flags = re.MULTILINE) diff --git a/dev_tools/services/acknowledg-forge/README.md b/dev_tools/services/acknowledg-forge/README.md index 1e596424cc..660aed622a 100644 --- a/dev_tools/services/acknowledg-forge/README.md +++ b/dev_tools/services/acknowledg-forge/README.md @@ -14,7 +14,7 @@ these sections, when they apply: Optionally, a service account comments on the pull request whenever what's served changes in a meaningful way: once the pull request has *settled* (CI for -its head finished, or `ACKNOWLEDG_FORGE_SETTLE_TIMEOUT` passed), with every +its head finished, or `settle_timeout` passed), with every section that changed in a single comment. Restarts, closing, merging, and reopening with the same results post nothing. @@ -29,13 +29,13 @@ reopening with the same results post nothing. ``` - **Polling** always runs once at startup (catching up on anything missed - while the service was down) and then every `ACKNOWLEDG_FORGE_POLL_INTERVAL` + while the service was down) and then every `poll_interval` seconds, unless that is `0`. - **Webhooks** are optional. A delivery only identifies what may have changed; the service re-queries the forge, so polling and webhooks have the same effect. -- **The database** (SQLite by default, PostgreSQL by URL) records each pull - request, its latest CI run, and what's served for it. The processor +- **The database** (SQLite by default, PostgreSQL by configuration) records + each pull request, its latest CI run, and what's served for it. The processor reconciles each changed pull request: it builds missing or stale sections, publishes exactly the sections that apply, and removes everything when the pull request closes. @@ -91,45 +91,136 @@ unprivileged user namespaces, and no AppArmor changes. ## Configuration -Settings come from the environment. Secrets come from systemd credentials, or -from the environment variable shown as a fallback. - -| Variable | Default | Meaning | -| --- | --- | --- | -| `ACKNOWLEDG_FORGE_HOME` | (required; image: `/var/lib/acknowledg-forge`) | Storage root for clones, worktrees, and the default database | -| `ACKNOWLEDG_FORGE_TOOLS_DIR` | (required; image: `/edg/tools`) | A `dev_tools`-style directory (`bin/`, `pylibs/`) with the EDG tooling to run: `edg-rematerialize-test-run`, `ifc-dump` | -| `ACKNOWLEDG_FORGE_REPOSITORIES` | (required) | Comma-separated repositories: `owner/name` on GitHub, project paths on GitLab | -| `ACKNOWLEDG_FORGE_DATABASE_URL` | `sqlite+aiosqlite:///$HOME/state.db` | SQLAlchemy async URL, e.g. `postgresql+asyncpg://…` (the image includes the `postgresql` extra). Use the `database-url` credential instead when the URL holds a password | -| `ACKNOWLEDG_FORGE_BASE_BRANCHES` | `main` | Comma-separated target branches whose pull requests are served | -| `ACKNOWLEDG_FORGE_API_URL` | github.com / gitlab.com | Forge API URL, e.g. `https://ghe.example/api/v3` or `https://gitlab.example/api/v4` | -| `ACKNOWLEDG_FORGE_POLL_INTERVAL` | `300` | Seconds between polls; `0` polls only at startup | -| `ACKNOWLEDG_FORGE_SERVICE_BIND` | (unset; `compose.yaml`: `service-frontend:2728`) | `host:port` to serve the service's web app on, once the service has started: health checks (`GET /healthz`, for the container's healthcheck), and with webhooks, `POST /webhook`. Unset serves nothing | -| `ACKNOWLEDG_FORGE_WEBHOOKS` | `0` | `1` receives webhooks on the service bind (needs the webhook secret) | -| `ACKNOWLEDG_FORGE_CI_WORKFLOW` | (unset) | CI to track: the GitHub workflow file (e.g. `build-and-test-ci.yml`), or the GitLab pipeline name (`*` for any). Unset disables CI tracking (and the Failures and Bench sections) | -| `ACKNOWLEDG_FORGE_TEST_ARTIFACT` | `test-results-all` | Test results artifact (GitLab: job) name | -| `ACKNOWLEDG_FORGE_BENCH_ARTIFACT` | `bench-results` | Benchmark results artifact (GitLab: job) name | -| `ACKNOWLEDG_FORGE_COMMENTS` | `0` | `1` enables comments (needs the forge token) | -| `ACKNOWLEDG_FORGE_SETTLE_TIMEOUT` | `7200` | Seconds after a push to stop waiting for CI before commenting | -| `ACKNOWLEDG_FORGE_MAX_ARTIFACT_BYTES` | 256 MiB | Largest artifact download | -| `ACKNOWLEDG_FORGE_MAX_EXTRACTED_BYTES` | 2 GiB | Largest extracted artifact (including stored, compressed inner tarballs, but not their unextracted contents) | -| `ACKNOWLEDG_FORGE_MAX_ARTIFACT_ENTRIES` | `500000` | Most ZIP entries in an artifact. Inner tarballs are checked when read | -| `ACKNOWLEDG_FORGE_MAX_ARTIFACT_DIRECTORY_BYTES` | 64 MiB | Largest artifact zip central directory, which is read into memory whole (about 106 bytes per entry in CI's test results) | -| `ACKNOWLEDG_FORGE_MAX_DIFF_BYTES` | 64 MiB | Largest diff of a pull request's `tests/`, which is held in memory and copied more than once: raising it may need a higher `mem_limit` for the service ([`compose.yaml`](compose.yaml)) | -| `ACKNOWLEDG_FORGE_MAX_SPARSE_PATTERNS` | `20000` | Most sparse-checkout patterns in a review's worktree (up to two per test). Checking out takes time proportional to patterns times paths | -| `ACKNOWLEDG_FORGE_MAX_CONCURRENCY` | `2` | Pull requests processed at once | -| `ACKNOWLEDG_FORGE_REMATERIALIZE_TIMEOUT` | `900` | Seconds `edg-rematerialize-test-run` may run | -| `ACKNOWLEDG_FORGE_UNSANDBOXED` | `0` | `1` runs without containers: the service serves the runner servers and the publisher server itself, on `127.0.0.1`, where any local process can reach them. **Local development on a single-user machine only.** | -| `ACKNOWLEDG_FORGE_{BUILDER,PUBLISHER}_SERVICE_URL` | (set by `compose.yaml`) | The builder's runner server and the publisher server; required unless unsandboxed | -| `ACKNOWLEDG_FORGE_TOOLS_SERVICE_URL` | (set by `compose.yaml`) | The tools runner server, for the publisher server | -| `EDG_ACKNOWLEDG_API_HOST`, `EDG_ACKNOWLEDG_API_PORT` | `localhost`, `2727` | AcknowlEDG API | -| `EDG_ACKNOWLEDG_UI_HOST`, `EDG_ACKNOWLEDG_UI_PORT` | `localhost`, `5173` | AcknowlEDG UI, used for links in comments | - -| Credential | Fallback variable | Meaning | +Application settings come only from the TOML file named by +`ACKNOWLEDG_FORGE_SETTINGS_FILE`. Python's built-in `tomllib` reads native +strings, booleans, numbers, and arrays, without interpolation. Keys are flat, +lowercase names. `ForgeServiceConfig.from_file()` is the shared loader; +each service checks the settings required for its role. Unknown keys and +incorrect types are errors. Relative paths are resolved against the TOML +file's directory. Database connection components are typed fields in a +`[database]` table; the database module constructs a SQLAlchemy URL object +when connecting. Config itself has no SQLAlchemy dependency. + +Compose mounts `SETTINGS_FILE` at `/etc/acknowledg-forge.toml`, read-only in +all four containers. Publisher workers read `max_rt_entries` and +`max_rt_member_bytes` locally. Section messages use `kind` to discriminate +a `variant` payload: empty for tests, or the baseline directory and tag for +benchmarks. Processor bookkeeping stays in the database. Restart the entire +deployment after changing settings. Secrets are loaded into a separate +`ForgeCredentials` object from named files in +`CREDENTIALS_DIRECTORY`, mounted only in the service's container. The TOML +file is readable in every container, including the builder and tools +containers that handle pull request content, so it accepts no token or API +key settings. The `[database]` table accepts only non-secret components; +the password comes from the `database-password` credential, which only the +service can read. + +The deployment's wiring is not a setting: which address each server listens +on, and where the service and publisher find the other servers. That is +what keeps each server to its one network (see +[SANDBOXING.md](SANDBOXING.md)), so `compose.yaml` sets it on each +container's command line, and the TOML rejects it, so a settings change +can't widen a listener: + +| Server | Arguments | +| --- | --- | +| Service (`acknowledg-forge-github`, `acknowledg-forge-gitlab`) | `--service-bind=HOST:PORT` (the web app: health checks, and `POST /webhook` with webhooks; unset serves nothing), `--builder-service-url`, `--publisher-service-url` (`ws://`; required unless unsandboxed) | +| Builder and tools runners (`edgacknowledg.forge.builder`, `.tools`) | `--host` (default `127.0.0.1`), `--port`, `--container` | +| Publisher (`edgacknowledg.forge.publisher`) | `--host` (default `127.0.0.1`), `--port`, `--tools-service-url` (`ws://`) | + +Only `ACKNOWLEDG_FORGE_SETTINGS_FILE` and `CREDENTIALS_DIRECTORY` remain +application bootstrap environment variables. The host's Compose variables +(`FORGE`, `IMAGE`, `FORGE_UID`, `FORGE_GID`, `STATE_DIR`, `SETTINGS_FILE`, +`CREDENTIALS_DIR`, `WEBHOOK_PORT`) still select images, mounts, identities, +and published ports. Standard runtime variables (`PATH`, `HOME`, `PYTHON*`) +retain their usual purposes. + +Migration is explicit: convert old `KEY=value` files and uppercase TOML +keys to lowercase TOML with native types; remove all former Forge setting +and secret environment variables, including `EDG_ACKNOWLEDG_*` endpoint/key +variables. Forge rejects them with migration guidance. Move secrets into +credentials, and AcknowlEDG endpoints into the administrator-owned TOML. +Remove every secret from the old `.settings` file **before** +converting it or pointing `SETTINGS_FILE` at it: Compose mounts that file +into all four containers before Forge reads (and rejects) it, whereas the +old file was only ever the service container's. +Update the installed Compose file, deploy helper, workflow, configuration, +and image together. The deploy helper now takes only service and image; +existing endpoint arguments are rejected. No compatibility fallback or live +reload is provided. + +| TOML key | Default | Meaning | | --- | --- | --- | -| `forge-token` | `ACKNOWLEDG_FORGE_TOKEN` | Forge API token (also used for git) | -| `webhook-secret` | `ACKNOWLEDG_FORGE_WEBHOOK_SECRET` | Webhook secret; required for webhooks | -| `database-url` | `ACKNOWLEDG_FORGE_DATABASE_URL` | Database URL, for one holding a password (optional) | -| `acknowledg-api-key` | `EDG_ACKNOWLEDG_API_KEY` | AcknowlEDG API key, if the server requires one | +| `home` | (required) | Storage root for clones, worktrees, and the default database | +| `tools_dir` | (required) | A `dev_tools`-style directory (`bin/`, `pylibs/`) with the EDG tooling to run: `edg-rematerialize-test-run`, `ifc-dump` | +| `repositories` | (required) | Array of repositories: `owner/name` on GitHub, project paths on GitLab | +| `database` | SQLite at `/state.db` | Typed connection table; see below | +| `base_branches` | `["main"]` | Array of target branches whose pull requests are served | +| `api_url` | github.com / gitlab.com | Forge API URL, e.g. `https://ghe.example/api/v3` or `https://gitlab.example/api/v4`. The forge token is sent here, so it must be `https://` (plain `http://` only to `localhost`), with no credentials, query, or fragment | +| `poll_interval` | `300` | Seconds between polls; `0` polls only at startup | +| `webhooks` | `false` | `true` receives webhooks on `--service-bind` (needs the webhook secret) | +| `ci_workflow` | (omitted) | CI to track: the GitHub workflow file (e.g. `build-and-test-ci.yml`), or the GitLab pipeline name (`*` for any). Omit this key to disable CI tracking (and the Failures and Bench sections) | +| `test_artifact` | `test-results-all` | Test results artifact (GitLab: job) name | +| `bench_artifact` | `bench-results` | Benchmark results artifact (GitLab: job) name | +| `comments` | `false` | `true` enables comments (needs the forge token) | +| `settle_timeout` | `7200` | Seconds after a push to stop waiting for CI before commenting | +| `max_artifact_bytes` | 256 MiB | Largest artifact download | +| `max_extracted_bytes` | 2 GiB | Largest extracted artifact (including stored, compressed inner tarballs, but not their unextracted contents) | +| `max_artifact_entries` | `500000` | Most ZIP entries in an artifact. Inner tarballs are checked when read | +| `max_artifact_directory_bytes` | 64 MiB | Largest artifact zip central directory, which is read into memory whole (about 106 bytes per entry in CI's test results) | +| `max_rt_entries` | `10000` | Most entries in a run tarball opened for a reviewer | +| `max_rt_member_bytes` | 64 MiB | Largest member of a run tarball opened for a reviewer | +| `max_diff_bytes` | 512 MiB | Largest diff of a pull request's `tests/`. It is streamed to disk, not held in memory, and kept with the review's rematerialized run; the builder applies all of it within `rematerialize_timeout` | +| `max_sparse_patterns` | `20000` | Most sparse-checkout patterns in a review's worktree (up to two per test). Checking out takes time proportional to patterns times paths | +| `max_concurrency` | `2` | Pull requests processed at once | +| `rematerialize_timeout` | `900` | Seconds `edg-rematerialize-test-run` may run | +| `unsandboxed` | `false` | `true` runs without containers: the service serves the runner servers and the publisher server itself, on `127.0.0.1`, where any local process can reach them. **Local development on a single-user machine only.** | +| `acknowledg_api_host`, `acknowledg_api_port` | `localhost`, `2727` | AcknowlEDG API: a host name alone (no scheme, port, or path). Port 443 selects TLS (`wss://`), any other plain `ws://`, so a host other than `localhost` needs its port stated, and the `acknowledg-api-key` credential is only sent to one on 443 | +| `acknowledg_ui_host`, `acknowledg_ui_port` | `localhost`, `5173` | AcknowlEDG UI, used for links in comments; as for the API, 443 selects `https://`, and a host other than `localhost` needs its port stated | +| `publisher_processes` | `4` | Publisher worker count | +| `publisher_request_timeout` | `150` | Seconds per worker request | + +| Credential | Meaning | +| --- | --- | +| `forge-token` | Forge API token (also used for git) | +| `webhook-secret` | Webhook secret; required for webhooks | +| `database-password` | PostgreSQL password; SQLite accepts no password | +| `acknowledg-api-key` | AcknowlEDG API key, if required | + +### Database connection table + +Omit `[database]` to use SQLite at `/state.db`. To select another +SQLite file, append this table after the root settings: + +```toml +[database] +backend = "sqlite" +path = "state.db" +``` + +An explicit relative path is relative to the TOML file's directory; +`:memory:` selects an in-memory database. SQLite accepts only `backend` +and `path`. PostgreSQL has two backends; both require `name` and +`username`, and default `port` to 5432: + +- `backend = "postgresql-unix"` requires `socket_directory`, an absolute + path such as `/run/postgresql`. Mount that directory into the service + container (see step 6 below). Unix sockets do not use network ports, but + PostgreSQL includes the configured `port` in the socket filename: + `/run/postgresql/.s.PGSQL.5432` with the defaults. This selects among + PostgreSQL instances sharing a socket directory. Peer authentication + needs no `database-password` credential. This backend accepts neither + `host` nor `require_tls`. +- `backend = "postgresql-tcp"` requires `host`, a host name or IP address. + `require_tls` defaults to `true`, requiring TLS with certificate and + hostname verification using the system trust store. Set it to `false` + to disable TLS explicitly. This backend accepts no `socket_directory`. + +The drivers are fixed to `aiosqlite` and `asyncpg` respectively. + +Passwords belong only in the `database-password` credential. Arbitrary +connection URLs, query mappings, and driver argument mappings are not +configuration options; supported options must be explicit schema fields. ### GitHub @@ -141,9 +232,9 @@ from the environment variable shown as a fallback. individual events* and clear *Pushes*, which the service ignores; a push to a pull request arrives as a *Pull requests* event). Point it at `https:///webhook`, typically behind a reverse proxy to - `ACKNOWLEDG_FORGE_SERVICE_BIND` (published on the host's `127.0.0.1`; + `--service-bind` (published on the host's `127.0.0.1`; proxy only `/webhook`). -- **CI:** set `ACKNOWLEDG_FORGE_CI_WORKFLOW=build-and-test-ci.yml`. Its +- **CI:** set `ci_workflow = "build-and-test-ci.yml"`. Its `publish-test-results` job uploads `test-results-all`, including the `ci-meta.json` that records the tested (merge) commit. Benchmark results need a CI job uploading `bench-results` (`c/` and `cpp/` runs plus @@ -155,8 +246,8 @@ from the environment variable shown as a fallback. (`read_api` is enough without comments). - **Webhook:** use the secret token, with *Merge request events* and *Pipeline events*. -- **CI:** artifacts belong to jobs, so `ACKNOWLEDG_FORGE_TEST_ARTIFACT` and - `ACKNOWLEDG_FORGE_BENCH_ARTIFACT` name the jobs whose artifacts archives hold +- **CI:** artifacts belong to jobs, so `test_artifact` and + `bench_artifact` name the jobs whose artifacts archives hold the results, laid out like the GitHub artifacts (configurations or `c/`/`cpp/` at the top, plus `ci-meta.json`). @@ -237,12 +328,11 @@ URLs is configured here, by the host's administrator. As root: [Service] LoadCredentialEncrypted=forge-token:/etc/credstore.encrypted/acknowledg-forge.forge-token - # Webhooks (see step 4); remove to only poll. + # Webhooks (see step 4); remove this line when webhooks = false. LoadCredentialEncrypted=webhook-secret:/etc/credstore.encrypted/acknowledg-forge.webhook-secret #LoadCredentialEncrypted=acknowledg-api-key:/etc/credstore.encrypted/acknowledg-forge.acknowledg-api-key - # Optional: a remote PostgreSQL, whose URL (with its password) is a credential - # (see step 6). - #LoadCredentialEncrypted=database-url:/etc/credstore.encrypted/acknowledg-forge.database-url + # Optional: the password for a remote PostgreSQL database (see step 6). + #LoadCredentialEncrypted=database-password:/etc/credstore.encrypted/acknowledg-forge.database-password # The service container's credentials, in memory; removed on stop. RuntimeDirectory=acknowledg-forge-github # For STATE_DIR (step 5). @@ -282,9 +372,9 @@ URLs is configured here, by the host's administrator. As root: The webhook secret (the same value as in the forge's webhook settings; for example, generate one with `openssl rand -hex 32`). Webhooks are optional: - to only poll, skip this, and remove the unit's `webhook-secret` line and - the settings' `ACKNOWLEDG_FORGE_WEBHOOKS` (step 5), or the service - refuses to start: + to only poll, skip this, remove the unit's `webhook-secret` line, and set + `webhooks = false` in the TOML settings (step 5). Enabling webhooks + without the secret prevents the service from starting: ``` systemd-ask-password -n 'Webhook secret:' \ @@ -340,22 +430,32 @@ URLs is configured here, by the host's administrator. As root: FORGE_GID= # The state directory (step 2). STATE_DIR=/var/lib/acknowledg-forge/github - SETTINGS_FILE=/etc/acknowledg-forge/acknowledg-forge-github.settings + SETTINGS_FILE=/etc/acknowledg-forge/acknowledg-forge-github.toml CREDENTIALS_DIR=/run/acknowledg-forge-github/credentials # The webhook port on the host's 127.0.0.1, for the reverse proxy. WEBHOOK_PORT=2728 ``` - And the service's settings (see [Configuration](#configuration)) as - `/etc/acknowledg-forge/acknowledg-forge-github.settings`, replacing the + And the shared TOML settings (see [Configuration](#configuration)) as + `/etc/acknowledg-forge/acknowledg-forge-github.toml`, replacing the repository: - ``` - ACKNOWLEDG_FORGE_REPOSITORIES=/ - ACKNOWLEDG_FORGE_CI_WORKFLOW=build-and-test-ci.yml - ACKNOWLEDG_FORGE_COMMENTS=1 - # Webhooks (with the webhook secret, step 4); remove to only poll. - ACKNOWLEDG_FORGE_WEBHOOKS=1 + ```toml + home = "/var/lib/acknowledg-forge" + tools_dir = "/edg/tools" + repositories = ["/"] + # Your public AcknowlEDG endpoints; port 443 selects TLS. + acknowledg_api_host = "acknowledg-api.example" + acknowledg_api_port = 443 + acknowledg_ui_host = "acknowledg.example" + acknowledg_ui_port = 443 + ci_workflow = "build-and-test-ci.yml" + comments = true + # Webhooks (with the webhook secret, step 4); set to false to only poll. + webhooks = true + # Optional run tarball limits (the defaults). + max_rt_entries = 10000 + max_rt_member_bytes = 67108864 ``` 6. Optionally, use PostgreSQL instead of the default SQLite database (which @@ -384,36 +484,46 @@ URLs is configured here, by the host's administrator. As root: service: volumes: - /run/postgresql:/run/postgresql - environment: - ACKNOWLEDG_FORGE_DATABASE_URL: postgresql+asyncpg://acknowledg-forge@/acknowledg_forge?host=/run/postgresql ``` - - If using a a remote PostgreSQL database: + Add this table after the root settings in the shared TOML: + + ```toml + [database] + backend = "postgresql-unix" + socket_directory = "/run/postgresql" + port = 5432 + name = "acknowledg_forge" + username = "acknowledg-forge" + ``` + + No database password credential is needed for peer authentication. + + - If using a remote PostgreSQL database: Create a secure credential using systemd for the database password: ``` - systemd-ask-password -n 'Database URL:' \ - | systemd-creds encrypt --name=database-url - \ - /etc/credstore.encrypted/acknowledg-forge.database-url + systemd-ask-password -n 'Database password:' \ + | systemd-creds encrypt --name=database-password - \ + /etc/credstore.encrypted/acknowledg-forge.database-password ``` - Then uncomment the unit's `database-url` line in the systemd unit, - configure `/etc/acknowledg-forge/acknowledg-forge-github.override.yaml` - from the template below, and add `--file - /etc/acknowledg-forge/acknowledg-forge-github.override.yaml` after both of - the unit's `--file /etc/acknowledg-forge/compose.yaml` lines (in - `ExecStart=` and `ExecStop=`): - - ``` - services: - service: - volumes: - - /run/postgresql:/run/postgresql - environment: - ACKNOWLEDG_FORGE_DATABASE_URL: postgresql+asyncpg://:@:/ + Uncomment the unit's `database-password` credential line, and add the + non-secret connection table after the root settings: + + ```toml + [database] + backend = "postgresql-tcp" + host = "db.internal" + port = 5432 + name = "acknowledg_forge" + username = "acknowledg_forge" + require_tls = true ``` + No PostgreSQL socket mount is needed for a remote database. + 7. Start it: ``` @@ -447,17 +557,14 @@ It runs as root on the deploy user's behalf; read it before installing it. The helper ([`deploy-remote.py`](deploy-remote.py), which needs only the host's `python3`) manages a single root-owned file, `/etc/acknowledg-forge/acknowledg-forge-github.deploy.env`, which sets the -deployment's `IMAGE` and the `EDG_ACKNOWLEDG_{API,UI}_{HOST,PORT}` the -AcknowlEDG URLs translate to. It pulls the image and drives the service's -unit. +deployment's `IMAGE`. Application settings, including AcknowlEDG endpoints, +are owned by the administrator in TOML. It pulls the image and drives the +service's unit. - **Strict arguments:** it accepts only images from - `ghcr.io/edgcpp/acknowledg-forge` pinned by digest, `ws(s)://` and - `http(s)://` URLs with no path, and a service named `acknowledg-forge-*`. - The service picks TLS by port alone, so the helper also rejects URLs it - couldn't honor exactly: TLS on a port other than 443, or plain - `ws://`/`http://` on 443. A leaked deploy key can therefore only - deploy another published image. + `ghcr.io/edgcpp/acknowledg-forge` pinned by digest and a service named + `acknowledg-forge-*`. A leaked deploy key can only deploy another + published image; it cannot change application settings. - **What it does:** it pulls the image and restarts the service only if the file changed (otherwise it just makes sure the service is running), then checks that the service stays up, and removes the repository's other @@ -479,13 +586,9 @@ environment: | `ACKNOWLEDG_FORGE_SSH_KNOWN_HOSTS` | `ssh-keyscan` output for the host | | `ACKNOWLEDG_FORGE_SERVICE` | The unit's name (optional; defaults to `acknowledg-forge-github`) | -And these **repository variables** (public, so not secrets; the API URL is -shared with the AcknowlEDG deploy, which builds the web client with it): - -| Variable | Purpose | -| --- | --- | -| `ACKNOWLEDG_API_HOST` | Public AcknowlEDG API URL, e.g. `wss://acknowledg-api.example` | -| `ACKNOWLEDG_UI_HOST` | Public AcknowlEDG UI URL, e.g. `https://acknowledg.example`, for links in comments | +The Forge deploy no longer reads the `ACKNOWLEDG_API_HOST` or +`ACKNOWLEDG_UI_HOST` repository variables. Those may still be used by the +separate AcknowlEDG application deploy. The host needs Docker Engine 25 or later and Compose 2.20.2 or later (for several `--env-file`s, and the healthchecks' `start_interval`), and systemd @@ -510,7 +613,7 @@ Running the service directly on a host has no sandbox at all: it serves its runner servers and the publisher server itself, the same way, but commands that read pull request content run beside its credentials. That is only for local development (see below), with -`ACKNOWLEDG_FORGE_UNSANDBOXED=1`. +`unsandboxed = true`. ## Development @@ -520,6 +623,34 @@ tests/run-tests.sh # pytest: both services and the shared code typecheck.sh # mypy, as strict as typecheck-python.sh ``` +Database tests default to in-memory SQLite. To exercise another database, +set `ACKNOWLEDG_FORGE_TEST_SETTINGS` to a Forge TOML settings file; the tests +use its `[database]` table through the application settings loader. For +example (adjust `tools_dir` to your checkout): + +```toml +home = "/tmp/forge-test" +tools_dir = "/path/to/compiler/dev_tools" +repositories = ["edgcpp/compiler"] + +[database] +backend = "postgresql-tcp" +host = "localhost" +name = "forge_test" +username = "forge_test" +require_tls = true # Set to false for a local plaintext test server. +``` + +Use a dedicated test database: its schema is dropped before and after each +test using the store fixture. If it needs a password, put it in a +`database-password` credential file and set +`ACKNOWLEDG_FORGE_TEST_CREDENTIALS_DIRECTORY` to that directory. Both schema +reset and store connections use the same database settings and credentials, +including TLS verification. The `postgresql-unix` backend works with the +same settings as the application and requires no password for peer +authentication. These test variables replace +`ACKNOWLEDG_FORGE_TEST_DATABASE_URL`. + To try the service locally against `edgcpp/compiler`: 1. Create a [fine-grained personal access @@ -530,34 +661,33 @@ To try the service locally against `edgcpp/compiler`: permissions. 2. Start the AcknowlEDG server and client ([../acknowledg](../acknowledg/README.md)). -3. Set `ACKNOWLEDG_FORGE_TOKEN` to the token and run `run-dev-oss-repo.sh`: - - ``` - read -rs ACKNOWLEDG_FORGE_TOKEN && export ACKNOWLEDG_FORGE_TOKEN - ./run-dev-oss-repo.sh - ``` - - (In fish, `read -sx ACKNOWLEDG_FORGE_TOKEN`.) Reading it this way keeps - the token out of your shell history. - - If `ACKNOWLEDG_FORGE_TOKEN` isn't set, the script falls back to the GitHub - CLI's token (`gh auth token`), if you're logged in with `gh auth login`. - That's convenient, but the CLI's token usually has far broader scopes - (e.g. `repo`, `workflow`) than the script needs, so prefer the - fine-grained token. - -It serves the pull requests to the local AcknowlEDG, unsandboxed and without -commenting. Its state is kept in a temporary directory (printed when it -starts) and removed when it exits. - -For anything else, run `github/run-service.sh` with these set: - -- `ACKNOWLEDG_FORGE_HOME`; -- `ACKNOWLEDG_FORGE_REPOSITORIES`; -- `ACKNOWLEDG_FORGE_TOOLS_DIR` (e.g. this checkout's `dev_tools`); -- a token. - -Outside the containers, also set `ACKNOWLEDG_FORGE_UNSANDBOXED=1`. +3. Run `./run-dev-oss-repo.sh` and enter the fine-grained token at the + silent prompt. Alternatively, explicitly choose `./run-dev-oss-repo.sh + --use-gh-token` to use `gh auth token`. The CLI token can have broader + permissions than this demo needs. + +The Linux Python launcher holds credentials in sealed anonymous `memfd` +files. Its private temporary credentials directory contains only symlinks +to `/proc//fd/`, never credential bytes. The launcher stays +alive while Forge runs and forwards termination signals. After the first +signal it allows ten seconds for shutdown, then kills the child process +group and reaps the child if necessary before removing its temporary state. +Tokens are absent from child environments, arguments, logs, and generated +TOML. There is no disk-backed fallback if `memfd` or `/proc` is unavailable. +Memory can swap, as environment memory can; this does not isolate other +processes running as the same user. + +The default generated TOML serves `edgcpp/compiler`, unsandboxed, without +commenting. Its state lives in a temporary directory removed on exit. To +change settings or preserve state, set `ACKNOWLEDG_FORGE_SETTINGS_FILE` to +your own TOML with a persistent `home`. To supply existing credentials, set +`CREDENTIALS_DIRECTORY` to an existing directory containing a nonempty +`forge-token`; the launcher leaves that directory untouched. + +For other development configurations, run `github/run-service.sh` or +`gitlab/run-service.sh` with `ACKNOWLEDG_FORGE_SETTINGS_FILE` and +`CREDENTIALS_DIRECTORY`. Set `unsandboxed = true` in TOML when running +outside containers. Use this mode only on a single-user machine. Database schema changes go through Alembic. Add a revision under `edgacknowledg/forge/db/migrations/versions/` (see `migrations/env.py`). @@ -568,4 +698,4 @@ Implement `Forge`, and optionally `ForgeWebhooks` and `ForgeCommenter` (`edgacknowledg/forge/__init__.py`). `edgacknowledg.forge.http` handles the REST plumbing: rate limits, pagination, and credential-free downloads. Then add a service directory like `github/`, whose `__main__` passes a -`build(config)` to `edgacknowledg.forge.service.main`. +`build(config, credentials)` to `edgacknowledg.forge.service.main`. diff --git a/dev_tools/services/acknowledg-forge/SANDBOXING.md b/dev_tools/services/acknowledg-forge/SANDBOXING.md index 00ee19c935..4e201db32a 100644 --- a/dev_tools/services/acknowledg-forge/SANDBOXING.md +++ b/dev_tools/services/acknowledg-forge/SANDBOXING.md @@ -28,7 +28,7 @@ No single mechanism is relied on. In order, from outermost: files and networks its role needs. Everything that reads pull request content runs outside the service's container, away from its secrets. 2. **What enters the service.** The EDG tooling comes from a frozen, - reviewed copy (`ACKNOWLEDG_FORGE_TOOLS_DIR`; the image embeds one), never + reviewed copy (`tools_dir`; the image embeds one), never from a pull request. A pull request's diff is only ever input to `patch`. Checkouts and artifacts are checked before anything reads them (see [Symlinks and links](#symlinks-and-links)). @@ -153,7 +153,10 @@ All four containers run from the same image and get: Each mounts only its part of the deployment's state directory, at the same path in every container (`/var/lib/acknowledg-forge`), so paths in messages -need no translation. +need no translation. All four containers also mount the TOML settings +file read-only at `/etc/acknowledg-forge.toml`. Publisher pool +processes read run tarball limits from that file, rather than section +metadata. Credentials remain mounted only in the service container. | Container | Runs | Sees | Writes | Networks | Holds | | --- | --- | --- | --- | --- | --- | @@ -168,8 +171,10 @@ need no translation. and tools runner can reach nothing but their one client. A listener on a container with several networks binds only on its pair's (the publisher server on `publish`; the service's web app (health checks and webhooks), - with `ACKNOWLEDG_FORGE_SERVICE_BIND=service-frontend:2728`, on - `frontend`). + with `--service-bind=service-frontend:2728`, on + `frontend`). These listen addresses, and the URLs each client uses, are + command-line arguments in `compose.yaml`; the shared TOML settings + reject them, so a settings change can't widen a listener. That is what keeps each server to its one client: the connections carry no credentials of their own, so `tests/test_containers.py` checks every pairing, and a change to the networks in `compose.yaml` that breaks one @@ -196,7 +201,7 @@ carry, so commands run concurrently over it. For each, the runner: - runs it with a cleared environment (`PATH`, `HOME=/tmp`, `LC_ALL`, `PYTHONPATH` for the frozen tools), in a session of its own, and kills its process group when it ends, times out (the builder: the service's - `ACKNOWLEDG_FORGE_REMATERIALIZE_TIMEOUT`; tools: 2 min), its client + `rematerialize_timeout`; tools: 2 min), its client cancels it, or the connection goes (which kills every command on it); - once no command is running, kills every other process left in the container and empties `/tmp`. @@ -374,8 +379,10 @@ which its event loop never waits on for more than a request's deadline: - Each section is registered with a `UnixSocketHandler`, which stands in for the section's handler: it passes each request, with the section it is for, to the pool, and relays the replies. -- The pool keeps a fixed number of pool processes (`--processes`, 4), each - with one end of a socket pair as its channel. Requests queue, in order; +- The pool keeps a fixed number of pool processes (`publisher_processes`, 4), each + with one end of a socket pair as its channel. Its tools runner URL is + passed as `--tools-service-url`; the first channel message is the + worker's `ready` response. Requests queue, in order; each process has a dispatcher that gives it the next one only once it has answered the last, so a process answers one request at a time, building the section's handler for it and dropping it afterwards, and a @@ -383,15 +390,16 @@ which its event loop never waits on for more than a request's deadline: its answer carries back; a request abandoned before its turn (as when its section is removed) is skipped, and one abandoned while answered still keeps its process busy until then, its answer dropped. -- A request has a deadline (`--request-timeout`, 150 s, longer than the - tools runner's 2 min), counted from when its process is given it. A +- A request has a deadline (`publisher_request_timeout`, 150 s, longer than + the tools runner's 2 min), counted from when its process is given it. A process that misses it is killed, with whatever it started, and replaced; so is one that exits. Either way that request fails, and the client library answers the reviewer with its failure reply; a request for a file outside the section still gets the reviewer dropped. - A process that exits before it is ready has failed to start: the next try waits, doubling from 1 s up to 60 s; meanwhile the other processes - take the requests, and while none can, requests fail at once. The server doesn't start unless every process does. + take the requests, and while none can, requests fail at once. The server + doesn't start unless every process does. The client library answers one request at a time per section, so one section's requests occupy at most one process at a time. @@ -420,7 +428,7 @@ At startup, the service waits until it can reach the builder that run a command (`ifc-dump`, `cg_annotate`) need it, so while it can't be reached, those wait for it, and everything else is still served. -Without containers (`ACKNOWLEDG_FORGE_UNSANDBOXED=1`, for local +Without containers (`unsandboxed = true`, for local development only), all of this is the same: the service runs the publisher server and both runner servers itself (see [Development](#development)) and reaches them the same way. @@ -525,28 +533,64 @@ them. Several layers keep the service from reading through one: | Resource | Limit | | --- | --- | -| Artifact download | `ACKNOWLEDG_FORGE_MAX_ARTIFACT_BYTES` (256 MiB) | -| Extracted artifact | `ACKNOWLEDG_FORGE_MAX_EXTRACTED_BYTES` (2 GiB), counting the stored size of inner tarballs, not their unextracted contents | -| Artifact entries | `ACKNOWLEDG_FORGE_MAX_ARTIFACT_ENTRIES` (500,000) for ZIP entries, 10,000 per inner tarball when opened; and its zip's central directory, `ACKNOWLEDG_FORGE_MAX_ARTIFACT_DIRECTORY_BYTES` (64 MiB), checked before zipfile reads it | -| Inner tarball member | 64 MiB per member when a reviewer opens a tarball | -| A pull request's diff of `tests/` | `ACKNOWLEDG_FORGE_MAX_DIFF_BYTES` (64 MiB) | -| A review worktree's sparse-checkout patterns | `ACKNOWLEDG_FORGE_MAX_SPARSE_PATTERNS` (20,000) | +| Artifact download | `max_artifact_bytes` (256 MiB) | +| Extracted artifact | `max_extracted_bytes` (2 GiB), counting the stored size of inner tarballs, not their unextracted contents | +| Artifact entries | `max_artifact_entries` (500,000) for ZIP entries, `max_rt_entries` (10,000) per inner tarball when opened; and its zip's central directory, `max_artifact_directory_bytes` (64 MiB), checked before zipfile reads it | +| Inner tarball member | `max_rt_member_bytes` (64 MiB) per member when a reviewer opens a tarball | +| A pull request's diff of `tests/` | `max_diff_bytes` (512 MiB) | +| A review worktree's sparse-checkout patterns | `max_sparse_patterns` (20,000) | | A file read for a reviewer | 64 MiB; a larger one fails the request | -| A source shown to a reviewer | only as much as can be displayed is read | +| A source shown to a reviewer | reads stop at the response content limit (8 MiB for ASCII text, counted in characters) plus one read piece, even for an unterminated long line; the response is marked truncated | | Each container's `/tmp` | 512 MiB tmpfs; copies made for diffing are streamed there | -| Build commands | `ACKNOWLEDG_FORGE_REMATERIALIZE_TIMEOUT` (15 min) | +| Build commands | `rematerialize_timeout` (15 min) | | Reviewer commands | 2 min | | A reviewer's request | 150 s in a pool process, which is then replaced | | Each container | `mem_limit` (the service, builder, and publisher 2 GiB, tools 1 GiB) and 512 processes ([`compose.yaml`](compose.yaml)) | +### Accepted HTTP decompression limitation + +REST API responses are limited to 32 MiB, checked after each decoded +chunk. Artifact downloads also check decoded chunks against their size +limit. HTTPX decompresses a chunk before either check runs, so these +checks bound accepted content, not peak decompression memory. A chunk can +temporarily exceed the limit before the response is rejected. + +We trust the configured GitHub or GitLab server, authenticated by TLS, to +serve ordinary API responses. The service authenticates its requests with +the forge token; this does not make user-authored comment bodies trusted. +The REST calls fetch metadata and paginated comments, not file contents or +diffs. Comment lookup processes one page at a time and stops at the bot's +marked comment. Large individual pages remain possible, particularly on +GitLab. + +We accept the assumption that exhausting memory before the size check +would require substantial abuse: enough highly compressible content to +expand an individual chunk beyond the service's available memory before +the check can reject the response. This is an operational assumption, not +a guarantee supplied by authentication or pagination. Bounded +decompression can be added if this becomes a problem; the deployed +container memory limit remains the outer bound. + ## Secrets -Secrets belong in systemd credentials, which the host's unit decrypts and -[`prepare-runtime.sh`](prepare-runtime.sh) hands to the service container -only, in memory (`/run`), never on disk unencrypted. A secret passed in an -environment variable is readable from the process's own -`/proc/self/environ`, so the service warns at startup when sandboxing is on -and a secret came from the environment. +Secrets belong in systemd credentials, decrypted in memory (`/run`) and +handed only to the service by [`prepare-runtime.sh`](prepare-runtime.sh). +Environment secret fallbacks have been removed. Shared TOML is read-only +in all four containers, so it accepts no token, API key, or database +password settings. Its typed `[database]` table contains only non-secret +connection components; the separate `database-password` credential is +readable only by the service. Publisher workers receive the settings path, +without the service's credentials directory. + +The AcknowlEDG API key is only sent over TLS (port 443) to a host other +than `localhost`, and the forge token only to an `https://` `api_url` +(`http://` to `localhost` only); the service refuses to start otherwise. + +For Linux development, the launcher holds credentials in sealed anonymous +memfds and exposes only symlinks in a private temporary directory. It keeps +the descriptors open until the service exits; credential bytes never go +into ordinary temporary files or child-process environments. Memory may +swap, and unsandboxed development does not isolate same-user processes. Comments the service posts contain only fixed text, commit hashes, and links it generates. @@ -563,7 +607,7 @@ later, with AppArmor) need no changes, and no AppArmor profile. ## Development -`ACKNOWLEDG_FORGE_UNSANDBOXED=1` runs without containers, for local +`unsandboxed = true` runs without containers, for local development only. The control flow stays the same: the service serves the builder and tools runner servers and the publisher server itself (`edgacknowledg.forge.local`), on `127.0.0.1`, and uses them as it uses the diff --git a/dev_tools/services/acknowledg-forge/compose.yaml b/dev_tools/services/acknowledg-forge/compose.yaml index 806e5af7a9..b7340dac32 100644 --- a/dev_tools/services/acknowledg-forge/compose.yaml +++ b/dev_tools/services/acknowledg-forge/compose.yaml @@ -30,10 +30,9 @@ # IMAGE the image, pinned by digest (set by deploy-remote.py) # FORGE_UID/GID the host account owning STATE_DIR; every container runs # as it -# STATE_DIR the deployment's state (ACKNOWLEDG_FORGE_HOME) -# SETTINGS_FILE the service's settings (ACKNOWLEDG_FORGE_*) +# STATE_DIR the deployment's state (home) +# SETTINGS_FILE the TOML settings, shared read-only by all containers # CREDENTIALS_DIR the credentials systemd decrypted (the service's only) -# EDG_ACKNOWLEDG_{API,UI}_{HOST,PORT} AcknowlEDG (set by deploy-remote.py) name: acknowledg-forge @@ -56,6 +55,14 @@ x-environment: &environment HOME: /tmp PYTHONDONTWRITEBYTECODE: "1" PYTHONUTF8: "1" + ACKNOWLEDG_FORGE_SETTINGS_FILE: /etc/acknowledg-forge.toml + +x-settings: &settings + type: bind + source: ${SETTINGS_FILE:?} + target: /etc/acknowledg-forge.toml + read_only: true + bind: {create_host_path: false} x-python: &python /edg/acknowledg-forge/${FORGE:-github}/bin/python @@ -79,23 +86,19 @@ x-healthcheck: &healthcheck services: service: <<: *container - command: ["acknowledg-forge-${FORGE:-github}"] - env_file: - - ${SETTINGS_FILE:?} + command: + - acknowledg-forge-${FORGE:-github} + # The web app: health checks, and webhooks with webhooks = true. + # Only on frontend (see networks). The wiring is set here, never in + # the shared settings, so the settings can't widen a listener. + - --service-bind=service-frontend:2728 + - --builder-service-url=ws://builder:2730 + - --publisher-service-url=ws://publisher:2731 environment: <<: *environment - ACKNOWLEDG_FORGE_HOME: /var/lib/acknowledg-forge CREDENTIALS_DIRECTORY: /run/credentials - ACKNOWLEDG_FORGE_BUILDER_SERVICE_URL: ws://builder:2730 - ACKNOWLEDG_FORGE_PUBLISHER_SERVICE_URL: ws://publisher:2731 - # The web app: health checks, and webhooks with - # ACKNOWLEDG_FORGE_WEBHOOKS=1. Only on frontend (see networks). - ACKNOWLEDG_FORGE_SERVICE_BIND: service-frontend:2728 - EDG_ACKNOWLEDG_API_HOST: ${EDG_ACKNOWLEDG_API_HOST:-localhost} - EDG_ACKNOWLEDG_API_PORT: ${EDG_ACKNOWLEDG_API_PORT:-2727} - EDG_ACKNOWLEDG_UI_HOST: ${EDG_ACKNOWLEDG_UI_HOST:-localhost} - EDG_ACKNOWLEDG_UI_PORT: ${EDG_ACKNOWLEDG_UI_PORT:-5173} volumes: + - *settings # One bind for all of the state, so builds move into reviews by # rename. (Missing directories are an error, not created as root.) - type: bind @@ -115,7 +118,7 @@ services: # is published. - 127.0.0.1:${WEBHOOK_PORT:-2728}:2728 networks: - # The web app binds only here (ACKNOWLEDG_FORGE_SERVICE_BIND). + # The web app binds only here (--service-bind). frontend: aliases: [service-frontend] build: {} @@ -142,8 +145,6 @@ services: # A runner server with the builder role (runner.roles: what it # allows, as without containers). - edgacknowledg.forge.builder - - --home=/var/lib/acknowledg-forge - - --tools-dir=/edg/tools # Its only network is the one it shares with its client. - --host=0.0.0.0 - --port=2730 @@ -151,6 +152,7 @@ services: environment: <<: *environment volumes: + - *settings - type: bind source: ${STATE_DIR:?}/reviews target: /var/lib/acknowledg-forge/reviews @@ -176,14 +178,11 @@ services: # runner's). - --host=publisher-publish - --port=2731 - # Pool processes answering reviewers, and how long each may take on a - # request before it is replaced (the defaults). - - --processes=4 - - --request-timeout=150 + - --tools-service-url=ws://tools:2732 environment: <<: *environment - ACKNOWLEDG_FORGE_TOOLS_SERVICE_URL: ws://tools:2732 volumes: + - *settings - type: bind source: ${STATE_DIR:?}/reviews target: /var/lib/acknowledg-forge/reviews @@ -215,8 +214,6 @@ services: - -m # A runner server with the tools role. - edgacknowledg.forge.tools - - --home=/var/lib/acknowledg-forge - - --tools-dir=/edg/tools # Its only network is the one it shares with its client. - --host=0.0.0.0 - --port=2732 @@ -224,6 +221,7 @@ services: environment: <<: *environment volumes: + - *settings - type: bind source: ${STATE_DIR:?}/reviews target: /var/lib/acknowledg-forge/reviews diff --git a/dev_tools/services/acknowledg-forge/deploy-remote.py b/dev_tools/services/acknowledg-forge/deploy-remote.py index 1361b393e3..39f45b901d 100755 --- a/dev_tools/services/acknowledg-forge/deploy-remote.py +++ b/dev_tools/services/acknowledg-forge/deploy-remote.py @@ -12,20 +12,16 @@ with sudo, and nothing else as root: sudo /usr/local/libexec/acknowledg-forge-deploy \\ - - -The URLs are the public AcknowlEDG endpoints, e.g. wss://api.example and -https://ui.example (the same values the web client is built with). + The service is a systemd unit, /etc/systemd/system/.service, that runs the Docker Compose deployment (compose.yaml) with the settings in CONFIG_DIR, all of which the administrator owns (see README.md), along with its credentials. This script only manages one file there, .deploy.env, which sets the image (pinned by digest, from -IMAGE_REPOSITORY only) and the EDG_ACKNOWLEDG_{API,UI}_{HOST,PORT} the URLs -translate to, and restarts the service only when that file changes. Since -it runs as root on a caller's behalf, every argument is strictly -validated. +IMAGE_REPOSITORY only), and restarts the service only when that file +changes. Since it runs as root on a caller's behalf, every argument is +strictly validated. It uses only the standard library and runs isolated (python3 -I), so nothing from the environment or the caller's directory can change it. @@ -40,14 +36,12 @@ import time from pathlib import Path -from typing import NamedTuple -from urllib.parse import urlsplit, urlunsplit # The only repository images may come from. IMAGE_REPOSITORY = 'ghcr.io/edgcpp/acknowledg-forge' # The administrator's deployment configuration: compose.yaml, and each -# service's .env and .settings. +# deployment's .env and .toml. CONFIG_DIR = Path('/etc/acknowledg-forge') # Where the administrator installs each service's unit. UNITS_DIR = Path('/etc/systemd/system') @@ -57,66 +51,12 @@ _SERVICE_NAME = re.compile(r'acknowledg-forge-[a-z0-9-]+') _IMAGE = re.compile(re.escape(IMAGE_REPOSITORY) + r'@sha256:[0-9a-f]{64}') -_HOST_NAME = re.compile(r'[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?') -_PORT = re.compile(r'[0-9]{1,5}') - -TLS_PORT = 443 class DeployError(Exception): '''A deployment that can't go ahead; the message says why.''' -class Endpoint(NamedTuple): - host: str - port: int - - -def parse_url(url: str, *, plain: str, tls: str, label: str) -> Endpoint: - '''The host and port of ``url``, which must use the ``plain`` or ``tls`` - scheme (e.g. ws and wss), a host name, an optional port, and no path. - - The service picks TLS by port alone (443), so only URLs it can honor - exactly are accepted: TLS on 443 and plain on any other port, never a - silent downgrade. - ''' - shape_error = DeployError( - f"the AcknowlEDG {label} must be {plain}:// or {tls}:// followed by a " - f"host name and optional port, with no path" - ) - # urlsplit quietly drops some characters (tabs, newlines) and empty - # parts ('?', '#'); anything it wouldn't reproduce exactly is refused. - parts = urlsplit(url) - if (url != urlunsplit(parts) or - parts.scheme not in (plain, tls) or - parts.path not in ('', '/') or - parts.query != '' or - parts.fragment != ''): - raise shape_error - host, colon, port_text = parts.netloc.partition(':') - if _HOST_NAME.fullmatch(host) is None: - raise shape_error - if colon != '' and _PORT.fullmatch(port_text) is None: - raise shape_error - - is_tls = parts.scheme == tls - if colon == '': - port = TLS_PORT if is_tls else 80 - else: - port = int(port_text) - if not 1 <= port <= 65535: - raise DeployError(f"the AcknowlEDG {label} has an invalid port") - if is_tls and port != TLS_PORT: - raise DeployError( - f"the AcknowlEDG {label} can only use {tls}:// on port {TLS_PORT}" - ) - if not is_tls and port == TLS_PORT: - raise DeployError( - f"the AcknowlEDG {label} must use {tls}:// on port {TLS_PORT}" - ) - return Endpoint(host, port) - - def check_service(service: str) -> None: if _SERVICE_NAME.fullmatch(service) is None: raise DeployError('the service must be named acknowledg-forge-') @@ -129,18 +69,12 @@ def check_image(image: str) -> None: ) -def deploy_env_content(image: str, api: Endpoint, ui: Endpoint) -> str: - '''The Compose variables pointing the deployment at ``image`` and the - AcknowlEDG endpoints. - ''' +def deploy_env_content(image: str) -> str: + '''Compose's image selection; application settings belong in TOML.''' return ( '# Managed by the acknowledg-forge-deploy workflow; changes are ' 'overwritten.\n' f"IMAGE={image}\n" - f"EDG_ACKNOWLEDG_API_HOST={api.host}\n" - f"EDG_ACKNOWLEDG_API_PORT={api.port}\n" - f"EDG_ACKNOWLEDG_UI_HOST={ui.host}\n" - f"EDG_ACKNOWLEDG_UI_PORT={ui.port}\n" ) @@ -200,14 +134,11 @@ def remove_old_images(current: str) -> None: ) -def deploy(service: str, image: str, api_url: str, ui_url: str) -> None: +def deploy(service: str, image: str) -> None: if os.geteuid() != 0: raise DeployError('must run as root (via sudo)') check_service(service) check_image(image) - api = parse_url(api_url, plain = 'ws', tls = 'wss', label = 'API URL') - ui = parse_url(ui_url, plain = 'http', tls = 'https', label = 'UI URL') - unit = UNITS_DIR / f"{service}.service" for required in (unit, CONFIG_DIR / 'compose.yaml'): if not required.is_file(): @@ -222,7 +153,7 @@ def deploy(service: str, image: str, api_url: str, ui_url: str) -> None: ) deploy_env = CONFIG_DIR / f"{service}.deploy.env" - changed = write_if_changed(deploy_env, deploy_env_content(image, api, ui)) + changed = write_if_changed(deploy_env, deploy_env_content(image)) service_unit = f"{service}.service" is_active = ['systemctl', 'is-active', '--quiet', service_unit] @@ -272,24 +203,13 @@ def parse_args() -> argparse.Namespace: 'image', help = f"the image, {IMAGE_REPOSITORY}@sha256:" ) - parser.add_argument( - 'api_url', - metavar = 'acknowledg-api-url', - help = 'the public AcknowlEDG API, ws://[:] or wss://' - ) - parser.add_argument( - 'ui_url', - metavar = 'acknowledg-ui-url', - help = 'the public AcknowlEDG UI, http://[:] or ' - 'https://' - ) return parser.parse_args() def main() -> int: args = parse_args() try: - deploy(args.service, args.image, args.api_url, args.ui_url) + deploy(args.service, args.image) except DeployError as ex: print(f"error: {ex}", file = sys.stderr) return 1 diff --git a/dev_tools/services/acknowledg-forge/github/acknowledg_forge_github/__init__.py b/dev_tools/services/acknowledg-forge/github/acknowledg_forge_github/__init__.py index 55b3dd76e7..c6ed735b35 100644 --- a/dev_tools/services/acknowledg-forge/github/acknowledg_forge_github/__init__.py +++ b/dev_tools/services/acknowledg-forge/github/acknowledg_forge_github/__init__.py @@ -126,11 +126,13 @@ def _pull_request(self, repository: str, payload: Any) -> PullRequestInfo: async def list_open_pull_requests(self, repository: str) -> list[PullRequestInfo]: - payloads = await self._client.paginate( + payloads = self._client.paginate( f"/repos/{repository}/pulls", {'state': 'open'} ) - return [self._pull_request(repository, payload) for payload in payloads] + return [ + self._pull_request(repository, payload) async for payload in payloads + ] async def get_pull_request(self, repository: str, @@ -165,7 +167,7 @@ async def latest_ci_run(self, str(run.get('conclusion')), CIConclusion.OTHER ) - artifacts = tuple( + artifacts = tuple([ ArtifactInfo( name = str(artifact['name']), id = str(artifact['id']), @@ -173,11 +175,11 @@ async def latest_ci_run(self, digest = artifact.get('digest'), expired = bool(artifact.get('expired', False)), ) - for artifact in await self._client.paginate( + async for artifact in self._client.paginate( f"/repos/{repository}/actions/runs/{run['id']}/artifacts", key = 'artifacts' ) - ) + ]) else: status = ( CIStatus.RUNNING if run['status'] == 'in_progress' @@ -281,8 +283,9 @@ async def find_comment(self, number: int, marker: str) -> Optional[str]: login = await self._own_login() - for comment in await self._client.paginate( - f"/repos/{repository}/issues/{number}/comments"): + async for comment in self._client.paginate( + f"/repos/{repository}/issues/{number}/comments" + ): author = (comment.get('user') or {}).get('login') if login is not None and author != login: continue diff --git a/dev_tools/services/acknowledg-forge/github/acknowledg_forge_github/__main__.py b/dev_tools/services/acknowledg-forge/github/acknowledg_forge_github/__main__.py index bede2d9b03..44076c1735 100644 --- a/dev_tools/services/acknowledg-forge/github/acknowledg_forge_github/__main__.py +++ b/dev_tools/services/acknowledg-forge/github/acknowledg_forge_github/__main__.py @@ -6,7 +6,7 @@ '''Run acknowledg-forge against GitHub (configuration: see README.md).''' from edgacknowledg.forge import service -from edgacknowledg.forge.config import ForgeServiceConfig +from edgacknowledg.forge.config import ForgeCredentials, ForgeServiceConfig from acknowledg_forge_github import ( DEFAULT_API_URL, @@ -17,18 +17,19 @@ ) -def build(config: ForgeServiceConfig) -> service.ForgeComponents: +def build(config: ForgeServiceConfig, + credentials: ForgeCredentials) -> service.ForgeComponents: client = GitHubClient( api_url = config.api_url or DEFAULT_API_URL, - token = config.forge_token + token = credentials.forge_token ) return service.ForgeComponents( forge = GitHubForge(client, workflow = config.ci_workflow), webhooks = ( - GitHubWebhooks(config.webhook_secret) if config.webhook_secret + GitHubWebhooks(credentials.webhook_secret) if credentials.webhook_secret else None ), - commenter = GitHubCommenter(client) if config.forge_token else None, + commenter = GitHubCommenter(client) if credentials.forge_token else None, ) diff --git a/dev_tools/services/acknowledg-forge/github/run-service.sh b/dev_tools/services/acknowledg-forge/github/run-service.sh index dfa39fd853..970558945c 100755 --- a/dev_tools/services/acknowledg-forge/github/run-service.sh +++ b/dev_tools/services/acknowledg-forge/github/run-service.sh @@ -5,8 +5,7 @@ # See https://edgcpp.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # -# Run acknowledg-forge-github for local development. Configuration comes -# from the environment (see ../README.md); at least ACKNOWLEDG_FORGE_HOME and -# ACKNOWLEDG_FORGE_REPOSITORIES must be set. +# Run acknowledg-forge-github with ACKNOWLEDG_FORGE_SETTINGS_FILE naming +# its TOML configuration and CREDENTIALS_DIRECTORY naming its credentials. exec uv run acknowledg-forge-github "$@" diff --git a/dev_tools/services/acknowledg-forge/gitlab/acknowledg_forge_gitlab/__init__.py b/dev_tools/services/acknowledg-forge/gitlab/acknowledg_forge_gitlab/__init__.py index ddeffa4e50..7754acd12c 100644 --- a/dev_tools/services/acknowledg-forge/gitlab/acknowledg_forge_gitlab/__init__.py +++ b/dev_tools/services/acknowledg-forge/gitlab/acknowledg_forge_gitlab/__init__.py @@ -13,9 +13,9 @@ - Repositories are project paths ('group/project'). - CI artifacts belong to jobs, so an artifact name (e.g. - ``ACKNOWLEDG_FORGE_TEST_ARTIFACT``) names the *job* whose artifacts + ``test_artifact``) names the *job* whose artifacts archive holds the results. -- ``ACKNOWLEDG_FORGE_CI_WORKFLOW`` names the pipeline to track (its +- ``ci_workflow`` names the pipeline to track (its ``workflow:name``), or '*' for the newest pipeline of any name. ''' @@ -134,11 +134,13 @@ def _merge_request(self, repository: str, payload: Any) -> PullRequestInfo: async def list_open_pull_requests(self, repository: str) -> list[PullRequestInfo]: - payloads = await self._client.paginate( + payloads = self._client.paginate( f"/projects/{_project(repository)}/merge_requests", {'state': 'opened'} ) - return [self._merge_request(repository, payload) for payload in payloads] + return [ + self._merge_request(repository, payload) async for payload in payloads + ] async def get_pull_request(self, repository: str, @@ -203,8 +205,9 @@ async def _job_artifacts(self, ) -> tuple[ArtifactInfo, ...]: now = self._clock() artifacts = [] - for job in await self._client.paginate( - f"/projects/{project}/pipelines/{pipeline_id}/jobs"): + async for job in self._client.paginate( + f"/projects/{project}/pipelines/{pipeline_id}/jobs" + ): name = str(job['name']) if name not in self._artifact_jobs: continue @@ -309,9 +312,10 @@ async def find_comment(self, number: int, marker: str) -> Optional[str]: username = await self._own_username() - for note in await self._client.paginate( - f"/projects/{_project(repository)}/merge_requests/{number}/notes", - {'sort': 'asc'}): + async for note in self._client.paginate( + f"/projects/{_project(repository)}/merge_requests/{number}/notes", + {'sort': 'asc'} + ): author = (note.get('author') or {}).get('username') if username is not None and author != username: continue diff --git a/dev_tools/services/acknowledg-forge/gitlab/acknowledg_forge_gitlab/__main__.py b/dev_tools/services/acknowledg-forge/gitlab/acknowledg_forge_gitlab/__main__.py index 1def62cb95..30f4ceefba 100644 --- a/dev_tools/services/acknowledg-forge/gitlab/acknowledg_forge_gitlab/__main__.py +++ b/dev_tools/services/acknowledg-forge/gitlab/acknowledg_forge_gitlab/__main__.py @@ -6,7 +6,7 @@ '''Run acknowledg-forge against GitLab (configuration: see README.md).''' from edgacknowledg.forge import service -from edgacknowledg.forge.config import ForgeServiceConfig +from edgacknowledg.forge.config import ForgeCredentials, ForgeServiceConfig from acknowledg_forge_gitlab import ( DEFAULT_API_URL, @@ -17,10 +17,11 @@ ) -def build(config: ForgeServiceConfig) -> service.ForgeComponents: +def build(config: ForgeServiceConfig, + credentials: ForgeCredentials) -> service.ForgeComponents: client = GitLabClient( api_url = config.api_url or DEFAULT_API_URL, - token = config.forge_token + token = credentials.forge_token ) return service.ForgeComponents( forge = GitLabForge( @@ -29,10 +30,10 @@ def build(config: ForgeServiceConfig) -> service.ForgeComponents: artifact_jobs = (config.test_artifact, config.bench_artifact) ), webhooks = ( - GitLabWebhooks(config.webhook_secret) if config.webhook_secret + GitLabWebhooks(credentials.webhook_secret) if credentials.webhook_secret else None ), - commenter = GitLabCommenter(client) if config.forge_token else None, + commenter = GitLabCommenter(client) if credentials.forge_token else None, ) diff --git a/dev_tools/services/acknowledg-forge/gitlab/run-service.sh b/dev_tools/services/acknowledg-forge/gitlab/run-service.sh index 44ae14aeca..916b4939c9 100755 --- a/dev_tools/services/acknowledg-forge/gitlab/run-service.sh +++ b/dev_tools/services/acknowledg-forge/gitlab/run-service.sh @@ -5,8 +5,7 @@ # See https://edgcpp.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # -# Run acknowledg-forge-gitlab for local development. Configuration comes -# from the environment (see ../README.md); at least ACKNOWLEDG_FORGE_HOME and -# ACKNOWLEDG_FORGE_REPOSITORIES must be set. +# Run acknowledg-forge-gitlab with ACKNOWLEDG_FORGE_SETTINGS_FILE naming +# its TOML configuration and CREDENTIALS_DIRECTORY naming its credentials. exec uv run acknowledg-forge-gitlab "$@" diff --git a/dev_tools/services/acknowledg-forge/run-dev-oss-repo.sh b/dev_tools/services/acknowledg-forge/run-dev-oss-repo.sh index e6325ad103..25fe151e1b 100755 --- a/dev_tools/services/acknowledg-forge/run-dev-oss-repo.sh +++ b/dev_tools/services/acknowledg-forge/run-dev-oss-repo.sh @@ -5,57 +5,10 @@ # See https://edgcpp.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # -# Run a development acknowledg-forge-github against edgcpp/compiler, serving -# reviews to the local AcknowlEDG server (../acknowledg/server/ -# run-server.sh) and client. It runs unsandboxed and never comments on pull -# requests, so ACKNOWLEDG_FORGE_TOKEN only needs to be a fine-grained personal -# access token with read-only access to public repositories (see README.md). -# Without it, the GitHub CLI's token (gh auth token) is used. -# -# State (clones, reviews, the database) is kept in a new temporary directory, -# removed when the service exits, unless ACKNOWLEDG_FORGE_HOME names one to -# keep. +# Run the public-repository demo with TOML and anonymous-memory credentials. +# The Python launcher owns cleanup and signal forwarding. No token is put +# in the shell environment, arguments, or a disk-backed temporary file. set -euo pipefail - cd "$(dirname "$0")/github" -dev_tools="$(realpath ../../..)" - -if [[ -z "${ACKNOWLEDG_FORGE_TOKEN:-}" ]]; then - if ! command -v gh > /dev/null; then - echo 'error: set ACKNOWLEDG_FORGE_TOKEN to a GitHub personal access' \ - 'token (see README.md)' >&2 - exit 1 - fi - if ! ACKNOWLEDG_FORGE_TOKEN="$(gh auth token 2>/dev/null)"; then - echo 'error: set ACKNOWLEDG_FORGE_TOKEN to a GitHub personal access' \ - 'token, or log in with gh auth login (see README.md)' >&2 - exit 1 - fi - echo 'Using the GitHub CLI token (gh auth token).' >&2 -fi -export ACKNOWLEDG_FORGE_TOKEN - -if [[ -z "${ACKNOWLEDG_FORGE_HOME:-}" ]]; then - ACKNOWLEDG_FORGE_HOME="$(mktemp -d -t acknowledg-forge-dev.XXXXXXXXXX)" - trap 'rm -rf "${ACKNOWLEDG_FORGE_HOME}"' EXIT -fi -export ACKNOWLEDG_FORGE_HOME -echo "acknowledg-forge home: ${ACKNOWLEDG_FORGE_HOME}" >&2 -export ACKNOWLEDG_FORGE_TOOLS_DIR="${dev_tools}" -export ACKNOWLEDG_FORGE_REPOSITORIES='edgcpp/compiler' -export ACKNOWLEDG_FORGE_CI_WORKFLOW='build-and-test-ci.yml' -export ACKNOWLEDG_FORGE_POLL_INTERVAL="${ACKNOWLEDG_FORGE_POLL_INTERVAL:-60}" -export ACKNOWLEDG_FORGE_UNSANDBOXED=1 -export ACKNOWLEDG_FORGE_COMMENTS=0 -unset ACKNOWLEDG_FORGE_API_URL ACKNOWLEDG_FORGE_SERVICE_BIND -unset ACKNOWLEDG_FORGE_WEBHOOKS -unset ACKNOWLEDG_FORGE_DATABASE_URL - -export EDG_ACKNOWLEDG_API_HOST='localhost' -export EDG_ACKNOWLEDG_API_PORT=2727 -export EDG_ACKNOWLEDG_UI_HOST='localhost' -export EDG_ACKNOWLEDG_UI_PORT=5173 - -# Not exec, so the temporary home is removed afterwards. -uv run acknowledg-forge-github "$@" +exec uv run python -m edgacknowledg.forge.dev "$@" diff --git a/dev_tools/services/acknowledg-forge/tests/conftest.py b/dev_tools/services/acknowledg-forge/tests/conftest.py index d4b486a4a7..949a4f9fb3 100644 --- a/dev_tools/services/acknowledg-forge/tests/conftest.py +++ b/dev_tools/services/acknowledg-forge/tests/conftest.py @@ -16,7 +16,7 @@ import zipfile from pathlib import Path -from typing import AsyncIterator, Iterator, Optional +from typing import Any, AsyncIterator, Iterator, Optional import pytest import pytest_asyncio @@ -32,8 +32,13 @@ PullRequestState, ) from edgacknowledg.forge.announcer import Announcer -from edgacknowledg.forge.config import ForgeServiceConfig -from edgacknowledg.forge.db import ForgeStore +from edgacknowledg.forge.config import ( + DatabaseConfig, + ForgeCredentials, + ForgeServiceConfig, + SqliteDatabaseConfig, +) +from edgacknowledg.forge.db import ForgeStore, create_database_engine from edgacknowledg.forge.ingest import Ingestor from edgacknowledg.forge.local import local_runner from edgacknowledg.forge.poller import sync_all @@ -232,7 +237,7 @@ def make_tools_dir(path: Path) -> Path: # The builder runner the local_builder fixture serves for the test, which # make_config points configurations at. -_BUILDER_SERVICE_URL = 'ACKNOWLEDG_FORGE_BUILDER_SERVICE_URL' +_builder_service_url: Optional[str] = None # The test's other clients of that runner (each Harness's), which # local_builder closes before it stops the runner: the runner's shutdown # waits for their connections. @@ -248,33 +253,65 @@ async def local_builder(tmp_path: Path, own without containers; the containers are tested on their own (test_containers.py). ''' + global _builder_service_url + # Test controls are read at collection, never application settings. + for name in tuple(os.environ): + if name.startswith('ACKNOWLEDG_FORGE_TEST_'): + monkeypatch.delenv(name) + settings = tmp_path / 'settings.toml' + write_settings(settings, { + 'home': str(tmp_path / 'home'), + 'tools_dir': str(make_tools_dir(tmp_path / 'tools')), + 'repositories': [REPOSITORY], + 'unsandboxed': True, + }) + monkeypatch.setenv('ACKNOWLEDG_FORGE_SETTINGS_FILE', str(settings)) async with local_runner( 'builder', home = tmp_path / 'home', tools_dir = make_tools_dir(tmp_path / 'tools')) as client: - monkeypatch.setenv(_BUILDER_SERVICE_URL, client.url) + _builder_service_url = client.url try: yield client finally: + _builder_service_url = None while len(_builder_clients) != 0: await _builder_clients.pop().close() -def make_config(home: Path, **overrides: str) -> ForgeServiceConfig: - environ = { - 'ACKNOWLEDG_FORGE_HOME': str(home), - 'ACKNOWLEDG_FORGE_TOOLS_DIR': str(make_tools_dir(home.parent / 'tools')), - 'ACKNOWLEDG_FORGE_REPOSITORIES': REPOSITORY, - 'ACKNOWLEDG_FORGE_CI_WORKFLOW': 'ci.yml', - 'ACKNOWLEDG_FORGE_COMMENTS': '1', - 'ACKNOWLEDG_FORGE_UNSANDBOXED': '1', +def write_settings(path: Path, values: dict[str, Any]) -> None: + def encode(value: Any) -> str: + if isinstance(value, dict): + return '{' + ', '.join( + f"{json.dumps(key)} = {encode(item)}" for key, item in value.items() + if item is not None + ) + '}' + return json.dumps(value) + + path.write_text(''.join( + f"{key} = {encode(value)}\n" + for key, value in values.items() if value is not None + ), encoding = 'utf-8') + + +def make_config(home: Path, **overrides: Any) -> ForgeServiceConfig: + values: dict[str, Any] = { + 'home': str(home), + 'tools_dir': str(make_tools_dir(home.parent / 'tools')), + 'repositories': [REPOSITORY], + 'ci_workflow': 'ci.yml', + 'comments': True, + 'unsandboxed': True, } - if _BUILDER_SERVICE_URL in os.environ: - environ[_BUILDER_SERVICE_URL] = os.environ[_BUILDER_SERVICE_URL] - environ.update( - {f"ACKNOWLEDG_FORGE_{key}": value for key, value in overrides.items()} - ) - return ForgeServiceConfig.from_environment(environ) + # The wiring comes from the command line, as compose.yaml sets it. + wiring = {'builder_service_url': _builder_service_url} + for key in ('service_bind', 'builder_service_url', 'publisher_service_url'): + if key in overrides: + wiring[key] = overrides.pop(key) + values.update(overrides) + path = home.parent / 'service-settings.toml' + write_settings(path, values) + return ForgeServiceConfig.from_file(path).with_wiring(**wiring) class Harness: @@ -381,34 +418,57 @@ def forge(origin: Origin) -> FakeForge: return FakeForge(origin.path) -# Set to run the database tests against another database (e.g. -# postgresql+asyncpg://...); its schema is reset after each test. -TEST_DATABASE_URL = os.environ.get('ACKNOWLEDG_FORGE_TEST_DATABASE_URL') +# Capture test controls before local_builder removes them from worker envs. +_TEST_SETTINGS = os.environ.get('ACKNOWLEDG_FORGE_TEST_SETTINGS') +_TEST_CREDENTIALS_DIRECTORY = os.environ.get( + 'ACKNOWLEDG_FORGE_TEST_CREDENTIALS_DIRECTORY' +) -async def _reset_database(url: str) -> None: +def _database_settings( + filename: Optional[str], + directory: Optional[str]) -> tuple[DatabaseConfig, ForgeCredentials]: + if filename is None: + return SqliteDatabaseConfig(path = Path(':memory:')), ForgeCredentials() + config = ForgeServiceConfig.from_file(Path(filename)) + credentials = ForgeCredentials.from_directory( + Path(directory) if directory is not None else None + ) + return config.database, credentials + + +async def _reset_database(config: DatabaseConfig, *, + password: Optional[str] = None) -> None: from sqlalchemy import text - from sqlalchemy.ext.asyncio import create_async_engine from edgacknowledg.forge.db.models import Base - engine = create_async_engine(url) - async with engine.begin() as connection: - await connection.run_sync(Base.metadata.drop_all) - await connection.execute(text('DROP TABLE IF EXISTS alembic_version')) - await engine.dispose() + engine = create_database_engine(config, password = password) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.drop_all) + await connection.execute(text('DROP TABLE IF EXISTS alembic_version')) + finally: + await engine.dispose() @pytest_asyncio.fixture async def store() -> AsyncIterator[ForgeStore]: - url = TEST_DATABASE_URL or 'sqlite+aiosqlite://' - if TEST_DATABASE_URL: - await _reset_database(url) - store = await ForgeStore.open(url) - yield store - await store.aclose() - if TEST_DATABASE_URL: - await _reset_database(url) + database, credentials = _database_settings( + _TEST_SETTINGS, + _TEST_CREDENTIALS_DIRECTORY + ) + password = credentials.database_password + external = _TEST_SETTINGS is not None + if external: + await _reset_database(database, password = password) + store = await ForgeStore.open(database, password = password) + try: + yield store + finally: + await store.aclose() + if external: + await _reset_database(database, password = password) @pytest.fixture diff --git a/dev_tools/services/acknowledg-forge/tests/test_config.py b/dev_tools/services/acknowledg-forge/tests/test_config.py new file mode 100644 index 0000000000..5a0427e6ce --- /dev/null +++ b/dev_tools/services/acknowledg-forge/tests/test_config.py @@ -0,0 +1,526 @@ +# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM +# Exceptions. +# See https://edgcpp.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +'''Typed TOML settings, migration errors, and credentials-only secrets.''' + + +from pathlib import Path +from typing import Any + +import pytest + +from edgacknowledg.forge.config import ( + ConfigError, + DatabaseConfig, + ForgeCredentials, + ForgeServiceConfig, + PostgresqlTcpDatabaseConfig, + PostgresqlUnixDatabaseConfig, + SqliteDatabaseConfig, + check_credential_transport, + check_environment, + settings_path, +) +from edgacknowledg.forge.publisher.pool import _process_env + +from conftest import make_tools_dir, write_settings + + +def settings_file(root: Path, **overrides: Any) -> Path: + path = root / 'config.toml' + values = { + 'home': str(root / 'home'), + 'tools_dir': str(make_tools_dir(root / 'tools')), + 'repositories': ['edg/compiler'], + 'unsandboxed': True, + **overrides, + } + write_settings(path, values) + return path + + +def test_native_settings_and_defaults(tmp_path: Path) -> None: + path = settings_file(tmp_path) + defaults = ForgeServiceConfig.from_file(path) + assert (defaults.max_rt_entries, defaults.max_rt_member_bytes) == ( + 10_000, 64 * 1024 * 1024 + ) + assert defaults.base_branches == frozenset(('main',)) + assert defaults.database == SqliteDatabaseConfig( + path = tmp_path / 'home' / 'state.db' + ) + assert defaults.acknowledg_api_port == 2727 + path = settings_file( + tmp_path, max_rt_entries = 12, + max_rt_member_bytes = 1024, comments = True, + poll_interval = 1.5, + base_branches = ['main', 'release'], + test_artifact = 'literal ${NAME}', + acknowledg_api_host = 'api.example', + acknowledg_api_port = 443, + api_url = 'https://ghe.example/api/v3' + ) + config = ForgeServiceConfig.from_file(path) + assert (config.max_rt_entries, config.max_rt_member_bytes) == (12, 1024) + assert config.comments and config.poll_interval == 1.5 + assert config.base_branches == frozenset(('main', 'release')) + assert config.test_artifact == 'literal ${NAME}' + assert config.acknowledg_api_port == 443 + # A local forge (e.g. a test double) may use plain HTTP. + path = settings_file(tmp_path, api_url = 'http://127.0.0.1:8080/api/v3') + assert ForgeServiceConfig.from_file(path).api_url.startswith('http://') + + +def test_postgresql_settings(tmp_path: Path) -> None: + config = ForgeServiceConfig.from_file( + settings_file( + tmp_path, + database = { + 'backend': 'postgresql-tcp', + 'host': 'db.internal', + 'name': 'acknowledg_forge', + 'username': 'acknowledg_forge', + } + ) + ) + assert config.database == PostgresqlTcpDatabaseConfig( + host = 'db.internal', + port = 5432, + name = 'acknowledg_forge', + username = 'acknowledg_forge' + ) + assert not hasattr(config.database, 'password') + + +def test_passwordless_postgresql_socket(tmp_path: Path) -> None: + config = ForgeServiceConfig.from_file( + settings_file( + tmp_path, + database = { + 'backend': 'postgresql-unix', + 'socket_directory': '/run/postgresql', + 'name': 'acknowledg_forge', + 'username': 'acknowledg-forge', + } + ) + ) + credentials = ForgeCredentials.from_directory(tmp_path) + assert credentials.database_password is None + assert isinstance(config.database, PostgresqlUnixDatabaseConfig) + assert config.database.socket_directory == Path('/run/postgresql') + assert config.database.port == 5432 + assert not hasattr(config.database, 'host') + assert not hasattr(config.database, 'require_tls') + + +@pytest.mark.parametrize('require_tls', ['true', 1, 0]) +def test_postgresql_tls_requires_boolean(tmp_path: Path, + require_tls: Any) -> None: + with pytest.raises(ConfigError, match = 'database.require_tls'): + ForgeServiceConfig.from_file( + settings_file( + tmp_path, + database = { + 'backend': 'postgresql-tcp', + 'host': 'db.internal', + 'name': 'forge', + 'username': 'forge', + 'require_tls': require_tls, + } + ) + ) + + +@pytest.mark.parametrize( + 'fields', + [ + {}, + {'socket_directory': 'relative/path'}, + {'socket_directory': ''}, + {'socket_directory': 1}, + {'socket_directory': '/run/postgresql', 'host': 'localhost'}, + {'socket_directory': '/run/postgresql', 'require_tls': True}, + {'socket_directory': '/run/postgresql', 'require_tls': False}, + {'socket_directory': '/run/postgresql', 'port': 0}, + {'socket_directory': '/run/postgresql', 'name': ''}, + {'socket_directory': '/run/postgresql', 'username': ''}, + ] +) +def test_postgresql_unix_schema(tmp_path: Path, + fields: dict[str, Any]) -> None: + with pytest.raises(ConfigError, match = 'database'): + ForgeServiceConfig.from_file( + settings_file( + tmp_path, + database = { + 'backend': 'postgresql-unix', + 'name': 'forge', + 'username': 'forge', + **fields, + } + ) + ) + + +@pytest.mark.parametrize( + 'fields', + [ + {'host': '/run/postgresql'}, + {'socket_directory': '/run/postgresql'}, + ] +) +def test_postgresql_tcp_rejects_socket_settings(tmp_path: Path, + fields: dict[str, Any]) -> None: + with pytest.raises(ConfigError, match = 'database'): + ForgeServiceConfig.from_file( + settings_file( + tmp_path, + database = { + 'backend': 'postgresql-tcp', + 'host': 'db.internal', + 'name': 'forge', + 'username': 'forge', + **fields, + } + ) + ) + + +@pytest.mark.parametrize('key, value', [ + ('max_rt_entries', -1), ('max_rt_entries', 1.5), + ('max_rt_entries', True), ('max_rt_entries', '12'), + ('max_rt_member_bytes', 'no'), ('comments', 'true'), + ('repositories', 'edg/compiler'), ('repositories', []), + ('base_branches', [1]), ('poll_interval', float('inf')), + ('max_concurrency', 0), ('publisher_processes', 0), + ('publisher_request_timeout', 0), ('acknowledg_api_port', 65536), + ('webhook_bind', 'localhost:1'), + ('forge_token', 'synthetic-secret'), ('webhook_secret', 'synthetic-secret'), + ('acknowledg_api_key', 'synthetic-secret'), + # Wiring is compose.yaml's, so settings can't widen a listener. + ('service_bind', '0.0.0.0:2728'), ('builder_service_url', 'ws://b:1'), + ('publisher_service_url', 'ws://p:1'), ('publisher_bind', '0.0.0.0:2731'), + ('builder_bind', '0.0.0.0:2730'), ('tools_bind', '0.0.0.0:2732'), + ('tools_service_url', 'ws://tools:2732'), + # A host other than localhost needs its port (which selects TLS) stated. + ('acknowledg_api_host', 'api.example'), + ('acknowledg_ui_host', 'ui.example'), + ('acknowledg_api_host', 'evil.example/x?'), + ('acknowledg_api_host', 'wss://api.example'), + ('acknowledg_ui_host', 'ui.example:443'), + ('api_url', 'http://ghe.example/api/v3'), + ('api_url', 'ftp://ghe.example/api/v3'), + ('api_url', 'https://user:synthetic-secret@ghe.example/api/v3'), + ('api_url', 'https://ghe.example/api/v3?token=synthetic-secret'), + ('api_url', 'https://ghe.example:synthetic-secret/api/v3'), + ('api_url', 'synthetic-secret'), +]) +def test_invalid_settings(tmp_path: Path, key: str, value: Any) -> None: + path = settings_file(tmp_path) + # JSON's Infinity is not TOML, so write native TOML for this case. + if isinstance(value, float) and value == float('inf'): + with path.open('a') as stream: + stream.write(f"{key} = inf\n") + else: + path = settings_file(tmp_path, **{key: value}) + with pytest.raises(ConfigError) as failure: + ForgeServiceConfig.from_file(path) + assert str(path) in str(failure.value) + assert 'synthetic-secret' not in str(failure.value) + + +def test_missing_and_malformed_settings(tmp_path: Path) -> None: + path = tmp_path / 'missing.toml' + with pytest.raises(ConfigError, match = 'cannot read settings'): + ForgeServiceConfig.from_file(path) + path.write_text('bad = "synthetic-secret') + with pytest.raises(ConfigError) as failure: + ForgeServiceConfig.from_file(path) + assert 'synthetic-secret' not in str(failure.value) + path.write_text('max_rt_entries = 12\n') + with pytest.raises(ConfigError, match = 'home is required'): + ForgeServiceConfig.from_file(path) + + +@pytest.mark.parametrize('name', [ + 'ACKNOWLEDG_FORGE_MAX_RT_ENTRIES', 'ACKNOWLEDG_FORGE_HOME', + 'ACKNOWLEDG_FORGE_TOOLS_SERVICE_URL', 'ACKNOWLEDG_FORGE_TOKEN', + 'ACKNOWLEDG_FORGE_WEBHOOK_SECRET', 'ACKNOWLEDG_FORGE_DATABASE_URL', + 'EDG_ACKNOWLEDG_API_KEY', 'EDG_ACKNOWLEDG_API_HOST', + 'EDG_ACKNOWLEDG_API_PORT', 'EDG_ACKNOWLEDG_UI_HOST', + 'EDG_ACKNOWLEDG_UI_PORT', +]) +def test_removed_environment(name: str) -> None: + with pytest.raises(ConfigError) as failure: + check_environment({name: 'synthetic-secret'}) + assert name in str(failure.value) + assert 'synthetic-secret' not in str(failure.value) + assert 'TOML' in str(failure.value) + + +def test_bootstrap_and_worker_environment(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch + ) -> None: + with pytest.raises(ConfigError, match = 'must name a TOML file'): + settings_path({}) + path = settings_file(tmp_path) + monkeypatch.setenv('ACKNOWLEDG_FORGE_SETTINGS_FILE', str(path)) + monkeypatch.setenv('CREDENTIALS_DIRECTORY', '/private/credentials') + worker = _process_env() + assert worker['ACKNOWLEDG_FORGE_SETTINGS_FILE'] == str(path) + assert 'CREDENTIALS_DIRECTORY' not in worker + assert not any(name.startswith('EDG_ACKNOWLEDG_') for name in worker) + assert ForgeServiceConfig.from_file().max_rt_entries == 10_000 + + +def test_credentials_are_service_only(tmp_path: Path) -> None: + path = settings_file( + tmp_path, database = { + 'backend': 'postgresql-tcp', 'host': 'db.internal', + 'name': 'acknowledg_forge', 'username': 'acknowledg_forge', + } + ) + credentials = tmp_path / 'credentials' + credentials.mkdir(mode = 0o700) + for name in ('forge-token', 'webhook-secret', 'acknowledg-api-key'): + (credentials / name).write_text('synthetic-secret\n') + (credentials / 'database-password').write_text('synthetic-secret\n') + secrets = ForgeCredentials.from_directory(credentials) + assert secrets.forge_token == secrets.webhook_secret == 'synthetic-secret' + assert secrets.acknowledg_api_key == 'synthetic-secret' + assert secrets.database_password == 'synthetic-secret' + plain = ForgeServiceConfig.from_file(path) + assert not hasattr(plain, 'forge_token') + assert not hasattr(plain, 'webhook_secret') + assert not hasattr(plain, 'acknowledg_api_key') + assert isinstance(plain.database, PostgresqlTcpDatabaseConfig) + assert plain.database.host == 'db.internal' + assert not hasattr(plain.database, 'password') + assert 'synthetic-secret' not in repr(secrets) + assert ForgeCredentials.from_directory(None) == ForgeCredentials() + (credentials / 'forge-token').write_bytes(b'\xff') + with pytest.raises( + ConfigError, + match = 'cannot read credential forge-token' + ): + ForgeCredentials.from_directory(credentials) + + +@pytest.mark.parametrize('wiring', [ + {'service_bind': 'bad'}, {'service_bind': 'host:²'}, + {'builder_service_url': 'http://builder:2730'}, + {'publisher_service_url': 'wss://publisher:2731'}, + {'builder_service_url': 'ws:builder'}, + {'builder_service_url': 'ws:///missing-host'}, + {'builder_service_url': 'ws://builder:2730/missing'}, + {'publisher_service_url': 'ws://publisher:2731/missing'}, +]) +def test_invalid_wiring(tmp_path: Path, wiring: dict[str, str]) -> None: + config = ForgeServiceConfig.from_file(settings_file(tmp_path)) + with pytest.raises(ConfigError, match = next(iter(wiring))): + config.with_wiring(**wiring) + + +@pytest.mark.parametrize('path', ['', '/']) +def test_root_service_urls(tmp_path: Path, path: str) -> None: + config = ForgeServiceConfig.from_file(settings_file(tmp_path)) + builder = f"ws://builder:2730{path}" + publisher = f"ws://publisher:2731{path}" + wired = config.with_wiring( + builder_service_url = builder, + publisher_service_url = publisher + ) + assert wired.builder_service_url == builder + assert wired.publisher_service_url == publisher + + +@pytest.mark.parametrize('host, port, key, allowed', [ + ('localhost', 2727, 'synthetic-secret', True), + ('api.example', 443, 'synthetic-secret', True), + ('api.example', 80, None, True), + ('api.example', 80, 'synthetic-secret', False), + ('api.example', 2727, 'synthetic-secret', False), +]) +def test_api_key_is_only_sent_over_tls(tmp_path: Path, host: str, port: int, + key: str | None, + allowed: bool) -> None: + config = ForgeServiceConfig.from_file( + settings_file( + tmp_path, acknowledg_api_host = host, acknowledg_api_port = port + ) + ) + credentials = ForgeCredentials(acknowledg_api_key = key) + if allowed: + check_credential_transport(config, credentials) + else: + with pytest.raises(ConfigError, match = 'only sent over TLS') as failure: + check_credential_transport(config, credentials) + assert 'synthetic-secret' not in str(failure.value) + + +async def test_service_refuses_the_api_key_without_tls(tmp_path: Path) -> None: + from edgacknowledg.forge import service + from fakes import FakeForge + + config = ForgeServiceConfig.from_file( + settings_file( + tmp_path, acknowledg_api_host = 'api.example', acknowledg_api_port = 80 + ) + ) + with pytest.raises(ConfigError, match = 'only sent over TLS'): + await service.run_service( + config, ForgeCredentials(acknowledg_api_key = 'synthetic-secret'), + service.ForgeComponents(FakeForge(tmp_path)) + ) + + +@pytest.mark.parametrize('wiring', [ + {}, {'builder_service_url': 'ws://builder:2730'}, + {'publisher_service_url': 'ws://publisher:2731'}, +]) +async def test_sandboxed_service_needs_endpoints(tmp_path: Path, + wiring: dict[str, str] + ) -> None: + from edgacknowledg.forge import service + from fakes import FakeForge + + path = settings_file(tmp_path, unsandboxed = False) + # Shared loading succeeds for roles that do not need these endpoints. + config = ForgeServiceConfig.from_file(path).with_wiring(**wiring) + with pytest.raises(ConfigError, match = '--builder-service-url'): + await service.run_service( + config, ForgeCredentials(), service.ForgeComponents(FakeForge(tmp_path)) + ) + + +@pytest.mark.parametrize('forge', ['github', 'gitlab']) +async def test_forge_builders_receive_credentials_separately(tmp_path: Path, + forge: str + ) -> None: + import importlib + + module = importlib.import_module(f"acknowledg_forge_{forge}.__main__") + config = ForgeServiceConfig.from_file( + settings_file( + tmp_path, comments = True, webhooks = True + ) + ).with_wiring(service_bind = '127.0.0.1:0') + credentials = ForgeCredentials( + forge_token = 'synthetic-token', + webhook_secret = 'synthetic-webhook' + ) + components = module.build(config, credentials) + try: + assert components.webhooks is not None + assert components.commenter is not None + assert not hasattr(config, 'forge_token') + finally: + await components.forge.aclose() + anonymous = module.build(config, ForgeCredentials()) + try: + assert anonymous.webhooks is anonymous.commenter is None + finally: + await anonymous.forge.aclose() + + +@pytest.mark.parametrize('password', [None, 'synthetic-secret']) +async def test_database_password_is_passed_separately( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + password: str | None) -> None: + from edgacknowledg.forge import service + from fakes import FakeForge + + config = ForgeServiceConfig.from_file(settings_file(tmp_path)) + credentials = ForgeCredentials(database_password = password) + opened = [] + + class DatabaseOpened(Exception): + pass + + async def check(sandbox: object) -> None: + pass + + async def open_database(database: DatabaseConfig, + *, password: str | None = None) -> None: + opened.append((database, password)) + # Stop before constructing the rest of the service. + raise DatabaseOpened + + monkeypatch.setattr(service.Sandbox, 'check', check) + monkeypatch.setattr(service.ForgeStore, 'open', open_database) + with pytest.raises(DatabaseOpened): + await service._run_components( + config, credentials, service.ForgeComponents(FakeForge(tmp_path)), + None, 'unused' + ) + assert opened == [(config.database, password)] + assert isinstance(config.database, SqliteDatabaseConfig) + assert config.database.path == tmp_path / 'home' / 'state.db' + + +@pytest.mark.parametrize( + 'database', + [ + 'not-a-table', + {'backend': 'mysql'}, + {'backend': True}, + {'path': ''}, + {'path': 1}, + {'password': 'synthetic-secret'}, + {'query': {'password': 'synthetic-secret'}}, + {'connect_args': {'password': 'synthetic-secret'}}, + {'backend': 'sqlite', 'host': 'db.internal'}, + {'backend': 'postgresql-tcp', 'path': 'state.db'}, + {'backend': 'postgresql-tcp'}, + {'backend': 'postgresql-tcp', 'host': 'db.internal'}, + {'backend': 'postgresql-tcp', 'host': 'db.internal', 'name': 'forge'}, + { + 'backend': 'postgresql-tcp', + 'host': '', + 'name': 'forge', + 'username': 'forge', + }, + { + 'backend': 'postgresql-tcp', + 'host': 'db', + 'name': 'forge', + 'username': 'forge', + 'port': True, + }, + { + 'backend': 'postgresql-tcp', + 'host': 'db', + 'name': 'forge', + 'username': 'forge', + 'port': '5432', + }, + { + 'backend': 'postgresql-tcp', + 'host': 'db', + 'name': 'forge', + 'username': 'forge', + 'port': 65536, + }, + ] +) +def test_database_schema_rejects_unsupported_fields(tmp_path: Path, + database: Any) -> None: + with pytest.raises(ConfigError) as failure: + ForgeServiceConfig.from_file(settings_file(tmp_path, database = database)) + assert 'database' in str(failure.value) + assert 'synthetic-secret' not in str(failure.value) + + +def test_sqlite_paths(tmp_path: Path) -> None: + path = settings_file(tmp_path) + with path.open('a') as stream: + stream.write('[database]\nbackend = "sqlite"\npath = "local.db"\n') + assert ForgeServiceConfig.from_file(path).database == SqliteDatabaseConfig( + path = tmp_path / 'local.db' + ) + path = settings_file(tmp_path, database = {'path': ':memory:'}) + assert ForgeServiceConfig.from_file(path).database == SqliteDatabaseConfig( + path = Path(':memory:') + ) diff --git a/dev_tools/services/acknowledg-forge/tests/test_containers.py b/dev_tools/services/acknowledg-forge/tests/test_containers.py index 7e158558a9..c87d64361f 100644 --- a/dev_tools/services/acknowledg-forge/tests/test_containers.py +++ b/dev_tools/services/acknowledg-forge/tests/test_containers.py @@ -30,6 +30,8 @@ from edgacknowledg.forge.sandbox import TMPFS_SIZE +from conftest import write_settings + IMAGE = os.environ.get('ACKNOWLEDG_FORGE_TEST_IMAGE') COMPOSE_FILE = Path(__file__).resolve().parent.parent / 'compose.yaml' PYTHON = '/edg/acknowledg-forge/github/bin/python' @@ -106,8 +108,11 @@ def deployment(tmp_path_factory: pytest.TempPathFactory credentials = root / 'credentials' credentials.mkdir() (credentials / 'forge-token').write_text('forge-token-value\n') - settings = root / 'settings.env' - settings.write_text('ACKNOWLEDG_FORGE_REPOSITORIES=edg/compiler\n') + settings = root / 'settings.toml' + write_settings(settings, { + 'home': STATE, 'tools_dir': '/edg/tools', + 'repositories': ['edg/compiler'], 'max_rt_entries': 12, + }) deployment.env_file.write_text( f"IMAGE={IMAGE}\n" f"FORGE=github\n" @@ -129,6 +134,22 @@ def deployment(tmp_path_factory: pytest.TempPathFactory print(logs[-20000:]) +@pytest.mark.parametrize('container', CONTAINERS) +def test_settings_are_shared_read_only(deployment: Deployment, + container: str) -> None: + result = deployment.python( + container, ( + 'import os; from pathlib import Path; ' + 'from edgacknowledg.forge.config import ForgeServiceConfig; ' + 'assert ForgeServiceConfig.from_file().max_rt_entries == 12; ' + 'Path(os.environ["ACKNOWLEDG_FORGE_SETTINGS_FILE"])' + '.write_text("changed")' + ) + ) + assert result.returncode != 0, result + assert 'Read-only file system' in result.stderr, result + + # What each container is @@ -211,8 +232,8 @@ def test_networks_pair_containers(deployment: Deployment) -> None: def test_webhooks_bound_to_the_frontend_reach_only_the_host( deployment: Deployment) -> None: - # As the service does with ACKNOWLEDG_FORGE_SERVICE_BIND= - # service-frontend:2728 (a stand-in server; the service isn't running). + # As the service does with --service-bind=service-frontend:2728 (a + # stand-in server; the service isn't running). deployment.compose( 'exec', '--detach', 'service', PYTHON, '-m', 'http.server', '--bind', 'service-frontend', '--directory', '/tmp', '2728' diff --git a/dev_tools/services/acknowledg-forge/tests/test_database.py b/dev_tools/services/acknowledg-forge/tests/test_database.py new file mode 100644 index 0000000000..68a4734e16 --- /dev/null +++ b/dev_tools/services/acknowledg-forge/tests/test_database.py @@ -0,0 +1,230 @@ +# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM +# Exceptions. +# See https://edgcpp.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +'''Connection components remain separate through SQLAlchemy engine creation.''' + +import ssl + +from pathlib import Path +from typing import Any, Optional + +import pytest + +from sqlalchemy.engine import URL +from sqlalchemy.exc import ArgumentError + +from edgacknowledg.forge.config import ( + DatabaseConfig, + PostgresqlTcpDatabaseConfig, + PostgresqlUnixDatabaseConfig, + SqliteDatabaseConfig, +) +from edgacknowledg.forge.db import DatabaseConfigError, ForgeStore +from edgacknowledg.forge.db import store as store_module + +from conftest import ( + _database_settings, + _reset_database, + make_tools_dir, + write_settings, +) + + +async def test_postgresql_components_are_not_parsed_as_a_url( + monkeypatch: pytest.MonkeyPatch) -> None: + config = PostgresqlUnixDatabaseConfig( + socket_directory = Path('/run/postgresql'), + port = 5433, + name = 'forge?name#fragment', + username = 'user:@/%' + ) + password = 'synthetic-secret:@/?#%' + captured = [] + + class EngineCreated(Exception): + pass + + def engine(url: URL, **options: Any) -> None: + captured.append(url) + raise EngineCreated + + monkeypatch.setattr(store_module, 'create_async_engine', engine) + with pytest.raises(EngineCreated): + await ForgeStore.open(config, password = password) + url = captured[0] + assert isinstance(url, URL) + assert url.drivername == 'postgresql+asyncpg' + assert url.username == config.username + assert url.password == password + assert url.host == str(config.socket_directory) + assert url.port == config.port + assert url.database == config.name + assert not url.query + # Exercise the actual asyncpg dialect's argument construction, too. + _, options = url.get_dialect()().create_connect_args(url) + assert options['password'] == password + assert options['host'] == '/run/postgresql' + assert options['database'] == config.name + + +async def test_sqlite_path_is_literal(tmp_path: Path) -> None: + path = tmp_path / 'state?mode=ro#fragment.db' + store = await ForgeStore.open(SqliteDatabaseConfig(path = path)) + try: + assert path.is_file() + assert not (tmp_path / 'state').exists() + finally: + await store.aclose() + + +@pytest.mark.parametrize('password', [None, 'synthetic-secret']) +@pytest.mark.parametrize('reset', [False, True]) +@pytest.mark.parametrize('backend', ['postgresql-tcp', 'postgresql-unix']) +@pytest.mark.parametrize('require_tls', [True, False]) +async def test_postgresql_reset_and_store_connection_options( + monkeypatch: pytest.MonkeyPatch, + password: Optional[str], + reset: bool, + backend: str, + require_tls: bool) -> None: + config: DatabaseConfig + if backend == 'postgresql-unix': + config = PostgresqlUnixDatabaseConfig( + socket_directory = Path('/run/postgresql'), + port = 5432, + name = 'forge', + username = 'forge' + ) + else: + config = PostgresqlTcpDatabaseConfig( + host = 'db.internal', + port = 5432, + name = 'forge', + username = 'forge', + require_tls = require_tls + ) + captured = [] + + class EngineCreated(Exception): + pass + + def engine(url: URL, **options: Any) -> None: + _, arguments = url.get_dialect()().create_connect_args(url) + arguments.update(options['connect_args']) + captured.append(arguments) + raise EngineCreated + + monkeypatch.setattr(store_module, 'create_async_engine', engine) + with pytest.raises(EngineCreated): + if reset: + await _reset_database(config, password = password) + else: + await ForgeStore.open(config, password = password) + arguments = captured[0] + assert arguments['host'] == ( + '/run/postgresql' if backend == 'postgresql-unix' else 'db.internal' + ) + assert arguments.get('password') == password + tls = arguments['ssl'] + if require_tls and backend == 'postgresql-tcp': + assert isinstance(tls, ssl.SSLContext) + assert tls.verify_mode == ssl.CERT_REQUIRED + assert tls.check_hostname + else: + assert tls is False + + +async def test_engine_configuration_errors_do_not_expose_passwords( + monkeypatch: pytest.MonkeyPatch) -> None: + config = PostgresqlTcpDatabaseConfig( + host = 'db.internal', + port = 5432, + name = 'forge', + username = 'forge' + ) + + def engine(url: URL, **options: Any) -> None: + raise ArgumentError('synthetic-secret') + + monkeypatch.setattr(store_module, 'create_async_engine', engine) + with pytest.raises(DatabaseConfigError) as failure: + await ForgeStore.open(config, password = 'synthetic-secret') + assert str(failure.value) == ( + 'database connection settings are invalid or unsupported' + ) + + +async def test_sqlite_does_not_ignore_a_database_password( + tmp_path: Path) -> None: + with pytest.raises(DatabaseConfigError, match = 'accepts no'): + await ForgeStore.open( + SqliteDatabaseConfig(path = tmp_path / 'state.db'), + password = 'synthetic-secret' + ) + assert not (tmp_path / 'state.db').exists() + + +@pytest.mark.parametrize('require_tls', [True, False]) +def test_database_fixture_loads_toml_and_separate_credentials( + tmp_path: Path, + require_tls: bool) -> None: + settings = tmp_path / 'database.toml' + write_settings( + settings, + { + 'home': str(tmp_path / 'home'), + 'tools_dir': str(make_tools_dir(tmp_path / 'tools')), + 'repositories': ['edg/compiler'], + 'database': { + 'backend': 'postgresql-tcp', + 'host': 'db.internal', + 'port': 5433, + 'name': 'forge', + 'username': 'forge', + 'require_tls': require_tls, + }, + } + ) + directory = tmp_path / 'credentials' + directory.mkdir() + (directory / 'database-password').write_text('synthetic-secret\n') + database, credentials = _database_settings(str(settings), str(directory)) + assert database == PostgresqlTcpDatabaseConfig( + host = 'db.internal', + port = 5433, + name = 'forge', + username = 'forge', + require_tls = require_tls + ) + assert credentials.database_password == 'synthetic-secret' + + +def test_database_fixture_defaults_to_in_memory_sqlite() -> None: + database, credentials = _database_settings(None, None) + assert database == SqliteDatabaseConfig(path = Path(':memory:')) + assert credentials.database_password is None + + +async def test_reset_database_uses_configured_sqlite_path( + tmp_path: Path) -> None: + from sqlalchemy import inspect + + from edgacknowledg.forge.db import create_database_engine + + config = SqliteDatabaseConfig(path = tmp_path / 'state.db') + store = await ForgeStore.open(config) + await store.aclose() + await _reset_database(config) + engine = create_database_engine(config) + try: + async with engine.connect() as connection: + tables = await connection.run_sync( + lambda c: inspect(c).get_table_names() + ) + assert tables == [] + finally: + await engine.dispose() + store = await ForgeStore.open(config) + await store.aclose() diff --git a/dev_tools/services/acknowledg-forge/tests/test_deploy.py b/dev_tools/services/acknowledg-forge/tests/test_deploy.py index d3b3470be4..ce5f941e88 100644 --- a/dev_tools/services/acknowledg-forge/tests/test_deploy.py +++ b/dev_tools/services/acknowledg-forge/tests/test_deploy.py @@ -34,58 +34,6 @@ def _load_deploy() -> ModuleType: IMAGE = f"ghcr.io/edgcpp/acknowledg-forge@{DIGEST}" -@pytest.mark.parametrize('url, expected', [ - ('wss://api.example', ('api.example', 443)), - ('wss://api.example/', ('api.example', 443)), - ('wss://api.example:443', ('api.example', 443)), - ('ws://localhost', ('localhost', 80)), - ('ws://10.0.0.1:8080', ('10.0.0.1', 8080)), - ('ws://api.example:0080', ('api.example', 80)), - ('ws://a', ('a', 80)), -]) -def test_urls_accepted(url: str, expected: tuple[str, int]) -> None: - assert deploy.parse_url( - url, plain = 'ws', tls = 'wss', label = 'API URL' - ) == expected - - -@pytest.mark.parametrize('url', [ - 'https://api.example', - 'WSS://api.example', - 'wss://', - 'wss://api.example/path', - 'wss://api.example?', - 'wss://api.example?x=1', - 'wss://api.example#', - 'wss://user@api.example', - 'wss://api.example:', - 'wss://api.example:+80', - 'wss://api.example:123456', - 'wss://-api.example', - 'wss://api.example.', - 'wss://[::1]', - 'wss://api.exa mple', - 'wss://api.exa\nmple', - 'wss://api.example\n', - 'wss://api.example\tEnvironment=X', - ' wss://api.example', -]) -def test_urls_refused(url: str) -> None: - with pytest.raises(deploy.DeployError, match = 'host name and optional'): - deploy.parse_url(url, plain = 'ws', tls = 'wss', label = 'API URL') - - -@pytest.mark.parametrize('url, message', [ - ('ws://api.example:0', 'invalid port'), - ('ws://api.example:65536', 'invalid port'), - ('wss://api.example:8443', 'only use wss:// on port 443'), - ('ws://api.example:443', 'must use wss:// on port 443'), -]) -def test_ports_refused(url: str, message: str) -> None: - with pytest.raises(deploy.DeployError, match = message): - deploy.parse_url(url, plain = 'ws', tls = 'wss', label = 'API URL') - - def test_service_and_image_names() -> None: deploy.check_service('acknowledg-forge-github') for service in ('acknowledg-forge-', 'acknowledg-forge-GitHub', @@ -108,18 +56,8 @@ def test_service_and_image_names() -> None: def test_deploy_env(tmp_path: Path) -> None: - content = deploy.deploy_env_content( - IMAGE, - deploy.Endpoint('api.example', 443), - deploy.Endpoint('ui.example', 8080) - ) - assert content.splitlines()[1:] == [ - f"IMAGE={IMAGE}", - 'EDG_ACKNOWLEDG_API_HOST=api.example', - 'EDG_ACKNOWLEDG_API_PORT=443', - 'EDG_ACKNOWLEDG_UI_HOST=ui.example', - 'EDG_ACKNOWLEDG_UI_PORT=8080', - ] + content = deploy.deploy_env_content(IMAGE) + assert content.splitlines()[1:] == [f"IMAGE={IMAGE}"] deploy_env = ( tmp_path / 'acknowledg-forge' / 'acknowledg-forge-github.deploy.env' @@ -143,8 +81,7 @@ def test_main_checks_usage_and_root(capsys: pytest.CaptureFixture[str], assert 'usage:' in capsys.readouterr().err if os.geteuid() != 0: monkeypatch.setattr(sys, 'argv', [ - 'deploy', 'acknowledg-forge-github', IMAGE, 'wss://api.example', - 'https://ui.example' + 'deploy', 'acknowledg-forge-github', IMAGE ]) assert deploy.main() == 1 assert 'must run as root' in capsys.readouterr().err diff --git a/dev_tools/services/acknowledg-forge/tests/test_dev.py b/dev_tools/services/acknowledg-forge/tests/test_dev.py new file mode 100644 index 0000000000..704ef083c5 --- /dev/null +++ b/dev_tools/services/acknowledg-forge/tests/test_dev.py @@ -0,0 +1,285 @@ +# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM +# Exceptions. +# See https://edgcpp.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +'''The development launcher's anonymous-memory credentials and lifetime.''' + +import os +import select +import signal +import stat +import subprocess +import sys +import textwrap + +from pathlib import Path +from typing import Mapping + +import pytest + +from edgacknowledg.forge import dev +from edgacknowledg.forge.config import ( + ConfigError, + ForgeServiceConfig, + read_secret, +) + +from conftest import make_tools_dir + +pytestmark = pytest.mark.skipif( + not hasattr(os, 'memfd_create') or not Path('/proc/self/fd').is_dir(), + reason = 'needs Linux memfd and /proc' +) + + +def test_memory_credentials(tmp_path: Path) -> None: + directory = tmp_path / 'credentials' + values = {'forge-token': b'synthetic-secret'} + with dev.memory_credentials(directory, values) as credentials: + assert stat.S_IMODE(directory.stat().st_mode) == 0o700 + link = credentials / 'forge-token' + assert link.is_symlink() + assert stat.S_IMODE(link.stat().st_mode) == 0o600 + assert read_secret('forge-token', credentials) == 'synthetic-secret' + assert read_secret('forge-token', credentials) == 'synthetic-secret' + assert read_secret('webhook-secret', credentials) is None + with pytest.raises(PermissionError): + link.write_bytes(b'changed') + # The child inherits no descriptor: it reopens the parent-held memfd. + child = subprocess.run( + [ + sys.executable, '-c', + 'from pathlib import Path; import os; ' + 'p = Path(os.environ["CREDENTIALS_DIRECTORY"]) / "forge-token"; ' + 'assert len(p.read_bytes()) == 16; ' + 'assert len(p.read_bytes()) == 16; ' + 'assert not any("synthetic-secret" in v for v in os.environ.values())', + ], + env = {'CREDENTIALS_DIRECTORY': str(credentials)}, close_fds = True, + capture_output = True + ) + assert child.returncode == 0, child.stderr + # Only the link is on disk; its target identifies a deleted memfd. + assert os.readlink(link).startswith(f"/proc/{os.getpid()}/fd/") + target = os.readlink(os.readlink(link)) + assert 'memfd:forge-credential' in target + assert list(directory.iterdir()) == [link] + assert not directory.exists() + with pytest.raises(FileNotFoundError): + link.read_bytes() + + +def test_failed_launch_cleans_credentials(tmp_path: Path) -> None: + directory = tmp_path / 'credentials' + with pytest.raises(FileNotFoundError): + with dev.memory_credentials( + directory, {'forge-token': b'synthetic-secret'} + ): + dev.supervise(['/nonexistent/forge-command'], {}) + assert not directory.exists() + + +def test_caller_directory_is_not_deleted(tmp_path: Path) -> None: + directory = tmp_path / 'credentials' + directory.mkdir() + marker = directory / 'existing' + marker.write_text('keep') + with pytest.raises(FileExistsError): + with dev.memory_credentials(directory, {}): + pass + assert marker.read_text() == 'keep' + + +@pytest.mark.parametrize('sig', [signal.SIGTERM, signal.SIGINT, signal.SIGHUP]) +@pytest.mark.parametrize('ignores_shutdown', [False, True]) +def test_shutdown_reaps_child_and_removes_credentials( + tmp_path: Path, + sig: int, + ignores_shutdown: bool) -> None: + # The child announces readiness only after reading the anonymous secret. + original = set(tmp_path.iterdir()) + program = textwrap.dedent(''' + import os, sys, tempfile + from pathlib import Path + from edgacknowledg.forge import dev + from edgacknowledg.forge.dev import memory_credentials, supervise + dev._SHUTDOWN_GRACE_PERIOD = 0.2 + with tempfile.TemporaryDirectory(dir = sys.argv[1]) as root: + with memory_credentials(Path(root) / 'credentials', + {'forge-token': b'synthetic-secret'} + ) as directory: + code = ( + 'import os, signal, time; from pathlib import Path; ' + + ( + '[signal.signal(s, signal.SIG_IGN) for s in ' + '(signal.SIGTERM, signal.SIGINT, signal.SIGHUP)]; ' + if sys.argv[2] == 'True' else '' + ) + + 'assert len((Path(os.environ["CREDENTIALS_DIRECTORY"]) / ' + '"forge-token").read_bytes()) == 16; ' + 'print(os.getpid(), flush=True); time.sleep(60)' + ) + status = supervise([sys.executable, '-c', code], + {'CREDENTIALS_DIRECTORY': str(directory)}) + sys.exit(status) + ''') + process = subprocess.Popen( + [sys.executable, '-c', program, str(tmp_path), str(ignores_shutdown)], + stdout = subprocess.PIPE, + stderr = subprocess.PIPE, + start_new_session = True + ) + try: + assert process.stdout is not None + readable, _, _ = select.select([process.stdout], [], [], 10) + assert readable, 'launcher did not start its child' + line = process.stdout.readline() + assert line.strip().isdigit(), (line, process.communicate(timeout = 5)) + child_pid = int(line) + process.send_signal(sig) + stdout, stderr = process.communicate(timeout = 10) + expected = signal.SIGKILL if ignores_shutdown else sig + assert process.returncode == 128 + expected, (stdout, stderr) + assert set(tmp_path.iterdir()) == original + assert not Path(f"/proc/{child_pid}").exists() + assert b'synthetic-secret' not in stdout + stderr + finally: + if process.poll() is None: + process.kill() + process.communicate(timeout = 10) + + +def test_generated_toml_contains_no_credentials(tmp_path: Path) -> None: + tools = make_tools_dir(tmp_path / 'tools') + path = tmp_path / 'development.toml' + path.write_text(dev.development_settings(tmp_path / 'state', tools)) + config = ForgeServiceConfig.from_file(path) + assert config.unsandboxed and not config.comments + assert config.repositories == ('edgcpp/compiler',) + assert not hasattr(config, 'forge_token') + + +def test_launcher_owns_secret_lifetime(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str] + ) -> None: + monkeypatch.delenv('ACKNOWLEDG_FORGE_SETTINGS_FILE') + monkeypatch.delenv('CREDENTIALS_DIRECTORY', raising = False) + monkeypatch.setattr(sys, 'argv', ['forge-dev', '--use-gh-token']) + monkeypatch.setenv('GH_TOKEN', 'synthetic-secret') + monkeypatch.setenv('GITHUB_TOKEN', 'synthetic-secret') + choices = [] + roots = [] + + def token(use_gh: bool) -> bytes: + choices.append(use_gh) + return b'synthetic-secret' + + def child(command: list[str], environ: Mapping[str, str]) -> int: + assert command == [sys.executable, '-m', 'acknowledg_forge_github'] + credentials = Path(environ['CREDENTIALS_DIRECTORY']) + roots.append(credentials.parent) + assert read_secret('forge-token', credentials) == 'synthetic-secret' + config = ForgeServiceConfig.from_file( + Path(environ['ACKNOWLEDG_FORGE_SETTINGS_FILE']) + ) + assert config.unsandboxed and not config.comments + assert not any('synthetic-secret' in v for v in environ.values()) + assert not any('synthetic-secret' in p.read_text() + for p in credentials.parent.rglob('*') + if p.is_file() and not p.is_symlink()) + return 7 + + monkeypatch.setattr(dev, '_token', token) + monkeypatch.setattr(dev, 'supervise', child) + assert dev.main() == 7 + assert choices == [True] + assert roots and not roots[0].exists() + captured = capsys.readouterr() + assert 'synthetic-secret' not in captured.out + captured.err + + +def test_interrupted_startup_cleans_credentials(tmp_path: Path) -> None: + directory = tmp_path / 'credentials' + with pytest.raises(dev._Termination): + with dev._startup_signals(): + with dev.memory_credentials( + directory, {'forge-token': b'synthetic-secret'} + ): + os.kill(os.getpid(), signal.SIGTERM) + assert not directory.exists() + + +def test_launcher_preserves_caller_credentials(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch + ) -> None: + directory = tmp_path / 'existing' + directory.mkdir(mode = 0o700) + marker = directory / 'forge-token' + marker.write_text('synthetic-secret') + # Unrelated entries must not be opened or prevent startup. + (directory / 'unrelated-directory').mkdir() + os.mkfifo(directory / 'unrelated-fifo') + monkeypatch.setenv('CREDENTIALS_DIRECTORY', str(directory)) + monkeypatch.setattr(sys, 'argv', ['forge-dev']) + monkeypatch.setattr(dev, 'supervise', lambda command, environ: 0) + assert dev.main() == 0 + assert marker.read_text() == 'synthetic-secret' + + +def test_credential_probe_rejects_unreadable_credential( + tmp_path: Path) -> None: + (tmp_path / 'forge-token').mkdir() + with pytest.raises(ConfigError, match = 'child cannot read'): + dev._check_credentials(tmp_path) + + +def test_credential_probe_times_out(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch) -> None: + os.mkfifo(tmp_path / 'forge-token') + monkeypatch.setattr(dev, '_CREDENTIAL_PROBE_TIMEOUT', 0.1) + with pytest.raises(ConfigError, match = 'access check timed out'): + dev._check_credentials(tmp_path) + + +@pytest.mark.parametrize('kind', [ + 'missing', + 'file', + 'no-token', + 'empty-token', +]) +def test_launcher_rejects_invalid_credentials( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + kind: str) -> None: + directory = tmp_path / 'credentials' + if kind == 'file': + directory.write_text('not a directory', encoding = 'utf-8') + elif kind != 'missing': + directory.mkdir() + if kind == 'empty-token': + (directory / 'forge-token').write_bytes(b'\r\n') + monkeypatch.setenv('CREDENTIALS_DIRECTORY', str(directory)) + monkeypatch.setattr(sys, 'argv', ['forge-dev']) + + def unexpected_child(command: list[str], environ: Mapping[str, str]) -> int: + pytest.fail('Forge must not start with invalid credentials') + + monkeypatch.setattr(dev, 'supervise', unexpected_child) + assert dev.main() == 2 + error = capsys.readouterr().err + if kind in ('missing', 'file'): + assert f"credentials path is not a directory: {directory}" in error + else: + assert 'forge-token credential is required' in error + + +def test_no_memory_file_support_has_no_disk_fallback( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: + monkeypatch.delattr(os, 'memfd_create') + monkeypatch.setattr(sys, 'argv', ['forge-dev']) + assert dev.main() == 2 + assert 'require Linux memfd' in capsys.readouterr().err diff --git a/dev_tools/services/acknowledg-forge/tests/test_display.py b/dev_tools/services/acknowledg-forge/tests/test_display.py new file mode 100644 index 0000000000..38baeea035 --- /dev/null +++ b/dev_tools/services/acknowledg-forge/tests/test_display.py @@ -0,0 +1,210 @@ +# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM +# Exceptions. +# See https://edgcpp.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +'''Content cut for display to reviewers: whole, and line by line.''' + +import contextlib +import io +import time + +from typing import Iterator, Optional + +import pytest + +from edgacknowledg import MAX_DISPLAY_LABEL_SIZE +from edgacknowledg.budget import Budget, BudgetExceeded +from edgacknowledg.client import shared +from edgacknowledg.client.shared import ( + display_content, + display_text, + slice_source_range, + trunc_display_content, + trunc_display_label, +) + + +@pytest.fixture +def small_limits(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(shared, 'MAX_DISPLAY_CONTENT_SIZE', 40) + monkeypatch.setattr(shared, 'MAX_DISPLAY_LINE_SIZE', 8) + + +def test_budget_counts_exactly_and_reports_overflow() -> None: + budget = Budget(5) + budget.spend(2) + budget.spend(3) + assert budget.limit == 5 + assert budget.remaining == 0 + assert not budget.exceeded + with pytest.raises(BudgetExceeded): + budget.spend(1) + assert budget.exceeded + assert budget.remaining == 0 + + budget = Budget(2) + with pytest.raises(BudgetExceeded): + budget.spend(3) + assert budget.remaining == 2 + assert budget.exceeded + + +def test_short_content_is_untouched(small_limits: None) -> None: + for content in ('', 'a', 'a\nb\n', 'a\n\n\nb', '12345678\n'): + assert trunc_display_content(content) == {'content': content} + + +def test_long_lines_are_cut_and_numbered(small_limits: None) -> None: + result = trunc_display_content('short\n123456789\nok\n' + 'x' * 20) + assert result == { + 'content': 'short\n12345678\nok\n' + 'x' * 8, + 'lines_truncated': [2, 4], + } + + +def test_long_content_stops_at_read_limit(small_limits: None) -> None: + # Six complete lines and four characters of the next fit in 40. + lines = [f"line{n}" for n in range(8)] + result = trunc_display_content('\n'.join(lines)) + assert result == { + 'content': '\n'.join(lines[:6]) + '\nline', + 'truncated': True, + } + + +def test_cut_lines_count_as_cut(small_limits: None) -> None: + # Discarded line characters also consume the read budget. + content = '\n'.join(['y' * 20] * 10) + result = trunc_display_content(content) + assert result['truncated'] + assert result['content'] == '\n'.join(['y' * 8] * 2) + assert result['lines_truncated'] == [1, 2] + assert len(result['content']) <= 40 + + +def test_one_line_larger_than_everything( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(shared, 'MAX_DISPLAY_CONTENT_SIZE', 10) + monkeypatch.setattr(shared, 'MAX_DISPLAY_LINE_SIZE', 100) + assert trunc_display_content('z' * 50 + '\nmore') == { + 'content': 'z' * 10, + 'truncated': True, + } + + +def test_no_characters_are_lost_between_lines(small_limits: None) -> None: + # Every character of each kept line survives the line joins. + lines = ['abcdefg'] * 10 + content = trunc_display_content('\n'.join(lines))['content'] + assert all(line == 'abcdefg' for line in content.split('\n')) + + +def test_large_content_is_linear() -> None: + # Many short lines past the content limit, and a huge one, go quickly. + content = 'a\n' * 8_000_000 + 'b' * 20_000_000 + start = time.monotonic() + result = trunc_display_content(content) + assert time.monotonic() - start < 20 + assert result['truncated'] + assert 'lines_truncated' not in result + + +def test_labels_end_in_an_ellipsis() -> None: + assert trunc_display_label('PASS') == 'PASS' + fits = 'x' * MAX_DISPLAY_LABEL_SIZE + assert trunc_display_label(fits) == fits + label = trunc_display_label(fits + 'x') + assert len(label) == MAX_DISPLAY_LABEL_SIZE + assert label.endswith('\N{HORIZONTAL ELLIPSIS}') + + +def test_display_reads_only_what_fits(small_limits: None) -> None: + taken: list[str] = [] + + def pieces() -> Iterator[tuple[str, bool]]: + for n in range(1000): + taken.append(f"{n}") + yield f"line{n}\n", True + + with contextlib.closing(pieces()) as generator: + shown = display_content(generator) + assert shown['truncated'] + # Up to the first line that doesn't fit, and no more. + assert len(taken) == 7 + assert shown['content'] == '\n'.join(f"line{n}" for n in range(6)) + '\nline' + + +def test_display_cuts_lines_across_pieces(small_limits: None) -> None: + # A line in several pieces; then a text without a last line end, which + # the next line doesn't run into. + pieces = [ + ('abcde', True), ('fghij', False), ('k\n', False), + ('no end', True), + ('# next\n', True), + ] + assert display_content(pieces) == { + 'content': 'abcdefgh\nno end\n# next\n', + 'lines_truncated': [1], + } + + +def test_source_ranges_cut_lines(small_limits: None) -> None: + text = 'a\n' + 'b' * 20 + '\nc\n' + 'd' * 9 + '\ne\n' + # Numbered within the window. + assert slice_source_range(io.StringIO(text), 3, 1) == { + 'content': 'b' * 8 + '\nc\n' + 'd' * 8, + 'first_line': 2, + 'lines_truncated': [1, 3], + } + # Cut lines outside the window aren't listed. + assert slice_source_range(io.StringIO(text), 5, 0) == { + 'content': 'e', + 'first_line': 5, + } + # A window past the end of the file is empty, where it was asked for. + assert slice_source_range(io.StringIO(text), 9, 1) == { + 'content': '', + 'first_line': 8, + } + + +def test_source_ranges_read_no_further(small_limits: None) -> None: + reads: list[int] = [] + + class Handle(io.StringIO): + def readline(self, size: Optional[int] = -1) -> str: + assert size is not None + reads.append(size) + return super().readline(size) + + # Of each line, no more than a piece at a time; and nothing past the + # window. + handle = Handle('x' * 20 + '\ny\n' + 'z' * 100 + '\n') + shown = slice_source_range(handle, 1, 1) + assert shown == { + 'content': 'x' * 8 + '\ny', + 'first_line': 1, + 'lines_truncated': [1], + } + assert set(reads) == {shared.MAX_DISPLAY_LINE_SIZE + 1} + assert handle.read() == 'z' * 100 + '\n' + + +def test_unterminated_long_lines_stop_at_read_budget( + small_limits: None, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(shared, 'MAX_DISPLAY_CONTENT_SIZE', 40) + text = 'x' * 1000 + handle = io.StringIO(text) + shown = display_text(handle) + assert shown == { + 'content': 'x' * 8, + 'truncated': True, + 'lines_truncated': [1], + } + assert handle.tell() <= 40 + 9 + + handle = io.StringIO(text) + shown = slice_source_range(handle, 2, 0) + assert shown == {'content': '', 'first_line': 2, 'truncated': True} + assert handle.tell() <= 40 + 9 diff --git a/dev_tools/services/acknowledg-forge/tests/test_github.py b/dev_tools/services/acknowledg-forge/tests/test_github.py index 3c333825a7..5bfdb7ceba 100644 --- a/dev_tools/services/acknowledg-forge/tests/test_github.py +++ b/dev_tools/services/acknowledg-forge/tests/test_github.py @@ -11,7 +11,7 @@ from http import HTTPStatus from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Optional import httpx import pytest @@ -235,3 +235,79 @@ def handler(request: httpx.Request) -> httpx.Response: assert posted == [{'body': 'hello'}] assert await commenter.find_comment(REPO, 1, 'MARK') == '2' assert await commenter.find_comment(REPO, 1, 'OTHER') is None + + +async def test_responses_are_read_only_so_far( + monkeypatch: pytest.MonkeyPatch) -> None: + from edgacknowledg.forge import ForgeError + from edgacknowledg.forge import http as forge_http + import gzip + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == '/small': + # Compressed, and paginated: decoded once, links kept. + return httpx.Response( + 200, + headers = [ + (b'content-encoding', b'gzip'), + (b'link', f'<{API}/small?page=2>; rel="next"'.encode('ascii')), + (b'x-note', b'caf\xe9'), + (b'x-note', b'second value'), + ], + content = gzip.compress(json.dumps({'ok': True}).encode()) + ) + return httpx.Response(200, json = {'pad': 'x' * 1000}) + + client = forge_http.ForgeHttpClient( + base_url = API, + headers = {}, + transport = httpx.MockTransport(handler) + ) + try: + monkeypatch.setattr(forge_http, 'MAX_RESPONSE_BYTES', 100) + response = await client.request('GET', '/small') + assert response.json() == {'ok': True} + assert response.links['next']['url'] == f"{API}/small?page=2" + assert [value for name, value in response.headers.raw + if name == b'x-note'] == [b'caf\xe9', b'second value'] + with pytest.raises(ForgeError, match = 'over 100 bytes'): + await client.get_json('/large') + finally: + await client.aclose() + + +@pytest.mark.parametrize('match_page', [1, 2, None]) +async def test_comment_lookup_stops_fetching_pages( + match_page: Optional[int]) -> None: + pages: list[int] = [] + path = f"/repos/{REPO}/issues/1/comments" + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith('/user'): + return httpx.Response(200, json = {'login': 'bot'}) + page = int(request.url.params.get('page', '1')) + pages.append(page) + headers = {} + if page < 3: + headers['link'] = f'<{API}{path}?page={page + 1}>; rel="next"' + return httpx.Response( + 200, + headers = headers, + json = [{ + 'id': page, + 'user': {'login': 'bot'}, + 'body': 'MARK' if page == match_page else 'other', + }] + ) + + client = GitHubClient( + api_url = API, + token = 't', + transport = httpx.MockTransport(handler) + ) + try: + result = await GitHubCommenter(client).find_comment(REPO, 1, 'MARK') + assert result == (str(match_page) if match_page is not None else None) + assert pages == list(range(1, (match_page or 3) + 1)) + finally: + await client.aclose() diff --git a/dev_tools/services/acknowledg-forge/tests/test_gitlab.py b/dev_tools/services/acknowledg-forge/tests/test_gitlab.py index fd7393aa60..ba38c5e8c5 100644 --- a/dev_tools/services/acknowledg-forge/tests/test_gitlab.py +++ b/dev_tools/services/acknowledg-forge/tests/test_gitlab.py @@ -10,9 +10,10 @@ from datetime import datetime, timezone from http import HTTPStatus from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Optional import httpx +import pytest from edgacknowledg.forge import ( ArtifactInfo, @@ -226,3 +227,42 @@ def handler(request: httpx.Request) -> httpx.Response: commenter = GitLabCommenter(client) assert await commenter.post_comment(PROJECT, 4, 'hi') == '7' assert await commenter.find_comment(PROJECT, 4, 'MARK') == '6' + + +@pytest.mark.parametrize('match_page', [1, 2, None]) +async def test_comment_lookup_stops_fetching_pages( + match_page: Optional[int]) -> None: + pages: list[int] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith('/user'): + return httpx.Response(200, json = {'username': 'bot'}) + page = int(request.url.params.get('page', '1')) + pages.append(page) + headers = {} + if page < 3: + headers['link'] = ( + f'<{API}/projects/{ENCODED}/merge_requests/1/notes' + f'?page={page + 1}>; rel="next"' + ) + return httpx.Response( + 200, + headers = headers, + json = [{ + 'id': page, + 'author': {'username': 'bot'}, + 'body': 'MARK' if page == match_page else 'other', + }] + ) + + client = GitLabClient( + api_url = API, + token = 't', + transport = httpx.MockTransport(handler) + ) + try: + result = await GitLabCommenter(client).find_comment(PROJECT, 1, 'MARK') + assert result == (str(match_page) if match_page is not None else None) + assert pages == list(range(1, (match_page or 3) + 1)) + finally: + await client.aclose() diff --git a/dev_tools/services/acknowledg-forge/tests/test_pool.py b/dev_tools/services/acknowledg-forge/tests/test_pool.py index 2962ce57a9..99f7261b71 100644 --- a/dev_tools/services/acknowledg-forge/tests/test_pool.py +++ b/dev_tools/services/acknowledg-forge/tests/test_pool.py @@ -11,6 +11,8 @@ import errno import os import signal +import socket +import sys import time import uuid @@ -83,6 +85,48 @@ async def env(tmp_path: Path) -> AsyncIterator[_Env]: yield _Env(section, tools) +async def test_worker_is_ready_without_bootstrap(env: _Env) -> None: + parent_end, worker_end = socket.socketpair() + process = None + channel = None + try: + process = await asyncio.create_subprocess_exec( + sys.executable, + '-m', + pool_module.POOL_PROCESS_MODULE, + str(worker_end.fileno()), + '--tools-service-url', + env.tools.url, + stdin = asyncio.subprocess.DEVNULL, + env = pool_module._process_env(), + pass_fds = (worker_end.fileno(),) + ) + worker_end.close() + messages, channel = await asyncio.open_unix_connection( + sock = parent_end + ) + # No parent message is sent before the worker reports readiness. + ready = await asyncio.wait_for( + messages.readline(), + 10 + ) + assert ready == b'{"ready": true}\n' + finally: + worker_end.close() + if channel is not None: + channel.close() + await channel.wait_closed() + else: + parent_end.close() + if process is not None: + try: + await asyncio.wait_for(process.wait(), 10) + except asyncio.TimeoutError: + process.kill() + await process.wait() + raise + + @pytest_asyncio.fixture async def make_pool(env: _Env) -> AsyncIterator[Any]: pools: list[ProcessPool] = [] diff --git a/dev_tools/services/acknowledg-forge/tests/test_publisher.py b/dev_tools/services/acknowledg-forge/tests/test_publisher.py index 5978f23b5f..9c161cc7c4 100644 --- a/dev_tools/services/acknowledg-forge/tests/test_publisher.py +++ b/dev_tools/services/acknowledg-forge/tests/test_publisher.py @@ -39,6 +39,10 @@ ) from edgacknowledg.client.shared import SandboxContext from edgacknowledg.forge import serving +from edgacknowledg.forge.config import ( + ForgeServiceConfig, + SqliteDatabaseConfig, +) from edgacknowledg.forge.db import ( ForgeStore, PullRequestRecord, @@ -47,23 +51,28 @@ from edgacknowledg.forge.ingest import utc_now from edgacknowledg.forge.local import LocalDeployment, local_deployment from edgacknowledg.forge.publisher import ( + KIND_BENCH, KIND_TEST_CHANGES, + KIND_TEST_FAILURES, TITLE_ELLIPSIS, ReviewIdentity, review_identity, ) from edgacknowledg.forge.publisher.client import PublisherClient +from edgacknowledg.forge.publisher.pool import _process_env from edgacknowledg.forge.publisher.pool_process import build_handler from edgacknowledg.forge.publisher.protocol import ( MESSAGE_LIMIT, decode_review, + decode_section, encode_review, + encode_section, ) from edgacknowledg.forge.publisher.server import create_publisher from edgacknowledg.forge.sandbox_client import ServiceUnavailableError from edgacknowledg.forge.serving import HEALTH_PATH, Listening -from conftest import make_tools_dir, pull_request_info +from conftest import make_tools_dir, pull_request_info, write_settings from fakes import REPOSITORY, FakeForge from test_acknowledg_integration import ( Server, @@ -177,32 +186,115 @@ def _empty_sync() -> dict[str, Any]: } -async def test_sync_does_not_send_rt_limits(tmp_path: Path, - store: ForgeStore, - monkeypatch: pytest.MonkeyPatch - ) -> None: +@pytest.mark.parametrize( + 'kind', + [KIND_TEST_CHANGES, KIND_TEST_FAILURES] +) +async def test_sync_sends_empty_test_variant(tmp_path: Path, + store: ForgeStore, + monkeypatch: pytest.MonkeyPatch, + kind: str) -> None: reviews, section = _layout(tmp_path) - publisher = _client('ws://127.0.0.1:1', reviews, store, - api_port = 1) + details = { + 'run_dir': str(section.root), + 'tests': ['suite/a.sft.cpp'], + 'tested_sha': 'a' * 40, + 'max_rt_entries': 12, + 'max_rt_member_bytes': 1024, + } + section = dataclasses.replace( + section, + kind = kind, + details = details + ) + publisher = _client( + 'ws://127.0.0.1:1', + reviews, + store, + api_port = 1 + ) requests: list[dict[str, Any]] = [] async def capture(request: dict[str, Any]) -> None: requests.append(request) monkeypatch.setattr(publisher, '_request', capture) - await publisher._sync(ReviewIdentity(uuid.uuid4(), NAME, ''), [section]) - details = requests[0]['sections'][0]['details'] - assert details == section.details + await publisher._sync( + ReviewIdentity(uuid.uuid4(), NAME, ''), + [section] + ) + encoded = requests[0]['sections'][0] + assert 'details' not in encoded + assert encoded['variant'] == {} + assert all(key not in encoded for key in details) + assert decode_section(encoded).details == {} + assert section.details == details -def test_pool_uses_fixed_rt_limits(tmp_path: Path) -> None: +def test_bench_section_protocol(tmp_path: Path) -> None: + _, section = _layout(tmp_path) + baseline = tmp_path / 'benchmarks' / 'baselines' / 'project' / 'release-c' + tag = 'release-c' + section = dataclasses.replace( + section, + kind = KIND_BENCH, + details = { + 'run_dir': str(section.root), + 'baseline_dir': str(baseline), + 'tag': tag, + } + ) + encoded = encode_section(section) + assert 'details' not in encoded + assert 'run_dir' not in encoded + assert 'baseline_dir' not in encoded + assert 'tag' not in encoded + assert encoded['variant'] == { + 'baseline_dir': str(baseline), + 'tag': tag, + } + decoded = decode_section(encoded) + assert decoded.details == { + 'baseline_dir': str(baseline), + 'tag': tag, + } + assert decoded.root == section.root + assert decoded.home == section.home + + +def test_pool_uses_configured_rt_limits(tmp_path: Path, + monkeypatch: pytest.MonkeyPatch + ) -> None: + settings = tmp_path / 'settings.toml' + write_settings(settings, { + 'home': str(tmp_path / 'home'), + 'tools_dir': str(make_tools_dir(tmp_path / 'tools')), + 'repositories': [REPOSITORY], + 'max_rt_entries': 12, + 'max_rt_member_bytes': 1024, + }) + monkeypatch.setenv('ACKNOWLEDG_FORGE_SETTINGS_FILE', str(settings)) + # The pool's sanitized environment must retain the file's location. + worker_env = _process_env() + assert worker_env['ACKNOWLEDG_FORGE_SETTINGS_FILE'] == str(settings) _, section = _layout(tmp_path) + section = dataclasses.replace( + section, details = { + 'max_rt_entries': 999_999, + 'max_rt_member_bytes': 999_999, + } + ) sandbox = SimpleNamespace( context_for = lambda home: SandboxContext(home) ) - handler = build_handler(section, sandbox, None) - assert handler._max_rt_entries == 10_000 - assert handler._max_rt_member_bytes == 64 * 1024 * 1024 + handler = build_handler( + section, + sandbox, + None, + ForgeServiceConfig.from_file() + ) + assert handler._max_rt_entries == 12 + assert handler._max_rt_member_bytes == 1024 async def _wait_for(condition: Callable[[], bool]) -> None: @@ -304,9 +396,16 @@ async def test_connecting_leaves_out_what_is_too_big( store: ForgeStore) -> None: local, reviews, section = deployment # First: the server would close the connection over it, every time. - too_big = await _store(store, 1, [ - dataclasses.replace(section, details = {'tests': ['x' * MESSAGE_LIMIT]}) - ]) + too_big = await _store( + store, + 1, + [ + dataclasses.replace( + section, + name = 'x' * MESSAGE_LIMIT + ) + ] + ) served = await _store(store, 2, [section]) for record in await store.open_pull_requests(FakeForge.name): await store.mark_processed(record.id, record.generation) @@ -407,7 +506,9 @@ async def test_a_new_service_drops_what_it_does_not_store( # The publisher still serves it, until a service whose database doesn't # store it connects. await _wait_for_sources(server.port, review_id, ['test-changes']) - other_store = await ForgeStore.open('sqlite+aiosqlite://') + other_store = await ForgeStore.open( + SqliteDatabaseConfig(path = Path(':memory:')) + ) later = _client(local.publisher_service_url, reviews, other_store, api_port = server.port) later.start() diff --git a/dev_tools/services/acknowledg-forge/tests/test_rematerialize.py b/dev_tools/services/acknowledg-forge/tests/test_rematerialize.py index 7123d85ca2..fc456bb35f 100644 --- a/dev_tools/services/acknowledg-forge/tests/test_rematerialize.py +++ b/dev_tools/services/acknowledg-forge/tests/test_rematerialize.py @@ -12,8 +12,20 @@ import sys from pathlib import Path, PurePosixPath +from typing import Any + +import pytest import edgtest +import edgutil + +# (Not 'from ... import TestSourceHandler', which pytest would collect.) +import edgacknowledg.client.test as test_provider + +from edgacknowledg import RequestTags, message +from edgacknowledg.client.shared import SandboxContext + +from test_symlinks import FakeChannel DEV_TOOLS = Path(__file__).resolve().parents[3] REMATERIALIZE = DEV_TOOLS / 'bin' / 'edg-rematerialize-test-run' @@ -155,7 +167,16 @@ def static_diff(test: str) -> str: edgtest.runtest.RUN_TEST_OUTPUT_SUFFIX ) with edgtest.runtest.RunTestOutput(rt_path) as rt_output: - return ''.join(rt_output.combined_static_diff()) + text = ''.join(rt_output.combined_static_diff()) + # As pieces (for display), the same, header lines fitting in one. + for limit in (len('--- Expected Output\n'), 1000): + assert ''.join( + piece for piece, _ in rt_output.iter_static_diff_pieces(limit) + ) == text + assert ''.join( + piece for piece, _ in rt_output.iter_raw_output_pieces(limit) + ) == ''.join(rt_output.combined_raw_output()) + return text c_diff = static_diff('suite/c.sft.cpp') assert '# cpfe -DTEST_NUMBER=1 Test_name.c\n' in c_diff @@ -179,3 +200,174 @@ def static_diff(test: str) -> str: for config, changes_file in changes_files.items() for test in changes_file.read_text().split() ) + + +def _rematerialize_change(tmp_path: Path, + change: dict[str, str]) -> tuple[ + subprocess.CompletedProcess[str], Path]: + '''Run the tool on the diff ``change`` makes to a small test home; return + its result and the run's directory. + ''' + repo = tmp_path / 'repo' + repo.mkdir() + git(repo, 'init', '--quiet', '--initial-branch=main') + write_files(repo, { + 'tests/.edgy/config.json': '{}\n', + 'tests/expectations/edg_x86_64/suite.log': '', + 'tests/expectations/edg_x86_64_cp/suite.log': '', + 'tests/tests/suite/a.sft.cpp': '//type:cp\nint a;\n', + 'tests/tests/suite/.a.rto/default.1.1.txt': ( + 'cpfe -DTEST_NUMBER=1 Test_name.c\nold output\n' + ), + }) + git(repo, 'add', '-A') + git(repo, 'commit', '--quiet', '-m', 'base') + write_files(repo, change) + git(repo, 'add', '-A') + git(repo, 'commit', '--quiet', '-m', 'change') + diff_file = tmp_path / 'pr.diff' + diff_file.write_text(git( + repo, 'diff', '--no-renames', '--no-color', 'HEAD~1', 'HEAD', '--', 'tests' + )) + git(repo, 'checkout', '--quiet', 'HEAD~1') + + runs_dir = tmp_path / 'runs' + timestamp = '2026.01.02-03.04.05' + env = dict(os.environ) + env['EDG_TEST_HOME'] = str(repo / 'tests') + env['PYTHONPATH'] = str(DEV_TOOLS / 'pylibs') + env.pop('RUN_TEST_DEFAULT_CONFIG', None) + result = subprocess.run( + [sys.executable, str(REMATERIALIZE), + '--runs-dir', str(runs_dir), + '--run-timestamp', timestamp, + str(diff_file)], + cwd = repo, + env = env, + capture_output = True, + text = True + ) + return result, runs_dir / timestamp + + +def test_rematerialize_passes_huge_lines_through(tmp_path: Path) -> None: + # Far longer than a line is ever held whole; written (and patched) in + # full. + huge = 'h' * (5 * edgutil.MAX_LINE_CHARS) + result, run_dir = _rematerialize_change(tmp_path, { + 'tests/tests/suite/.a.rto/default.1.1.txt': ( + f"cpfe -DTEST_NUMBER=1 Test_name.c\n{huge}\n" + ), + }) + assert result.returncode == 0, result.stdout + result.stderr + rt_path = (run_dir / 'edg_x86_64' / 'suite/a.sft.cpp').with_suffix( + edgtest.runtest.RUN_TEST_OUTPUT_SUFFIX + ) + with edgtest.runtest.RunTestOutput(rt_path) as rt_output: + static_diff = ''.join(rt_output.combined_static_diff()) + assert f"+{huge}\n" in static_diff + + +def test_rematerialize_refuses_huge_expectation_lines(tmp_path: Path) -> None: + result, _ = _rematerialize_change(tmp_path, { + 'tests/expectations/edg_x86_64/suite.log': ( + 'a.sft.cpp:cp:-DTEST_NUMBER=1:FAIL:' + + 'r' * edgutil.MAX_LINE_CHARS + '\n' + ), + }) + assert result.returncode == 1 + assert ( + 'tests/expectations/edg_x86_64/suite.log: a changed line is longer than' + ) in result.stderr + + +async def test_curr_diff_is_shown_and_compared( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + # Emulate a non-UTF-8 locale for text reads without an explicit encoding. + def locale_open(file: Any, mode: str = 'r', **kwargs: Any) -> Any: + if 'b' not in mode: + kwargs.setdefault('encoding', 'ascii') + return open(file, mode, **kwargs) + + monkeypatch.setattr(test_provider, 'open', locale_open, raising = False) + result, run_dir = _rematerialize_change(tmp_path, { + 'tests/tests/suite/.a.rto/default.1.1.txt': ( + 'cpfe -DTEST_NUMBER=1 Test_name.c\nnew output\n' + ), + # Brings the other configuration into the run (which inherits the + # recording's change). + 'tests/expectations/edg_x86_64_cp/suite.log': ( + 'a.sft.cpp:cp:-DTEST_NUMBER=1:FAIL:\n' + ), + }) + assert result.returncode == 0, result.stdout + result.stderr + context = SandboxContext(tmp_path) + handler = test_provider.TestSourceHandler( + 'Test', + 'test', + context, + context.subpath(tmp_path / 'repo' / 'tests'), + context.subpath(run_dir) + ) + + async def show_curr_diff( + configs: tuple[str, ...] = ('edg_x86_64', 'edg_x86_64_cp') + ) -> dict[str, Any]: + channel = FakeChannel(message( + RequestTags.TEST_SHOW_CURR_DIFF, + value = { + 'id': '(edg_x86_64) suite/a.sft.cpp', + 'configs': list(configs), + 'reviewer_id': 'r', + } + )) + await handler.handle_request(channel) # type: ignore[arg-type] + [reply] = channel.sent + response: dict[str, Any] = reply['v']['client_response'] + return response + + # Observed against the run's (the pull request's) recordings, which it + # was made from: no difference. + response = await show_curr_diff() + assert response['content'] == '' + assert 'divergent_configs' not in response + + # Recordings changed since: the difference is shown, the same for both + # configurations (the other uses the default recording). + recordings = run_dir / 'patched-tests' / 'suite' / '.a.rto' + (recordings / 'default.1.1.txt').write_text( + 'cpfe -DTEST_NUMBER=1 Test_name.c\nnewer café output\n', + encoding = 'utf-8' + ) + response = await show_curr_diff() + assert response['content'] == ( + '--- Expected Output\n' + '+++ Observed Output\n' + '# cpfe -DTEST_NUMBER=1 Test_name.c\n' + '@@ -1 +1 @@\n' + '-newer café output\n' + '+new output\n' + ) + assert 'divergent_configs' not in response + + # A recording of its own makes the other configuration's differ. + (recordings / 'edg_x86_64_cp.1.1.txt').write_text( + 'cpfe -DTEST_NUMBER=1 Test_name.c\nnew output\n' + ) + response = await show_curr_diff() + assert '-newer café output\n+new output' in response['content'] + assert response['divergent_configs'] == ['edg_x86_64_cp'] + + compared: list[Path] = [] + original_cmp = test_provider.filecmp.cmp + + def cmp(first: Path, second: Path, *, shallow: bool) -> bool: + compared.append(second) + return original_cmp(first, second, shallow = shallow) + + monkeypatch.setattr(test_provider.filecmp, 'cmp', cmp) + await show_curr_diff( + ('edg_x86_64', 'edg_x86_64_cp', 'edg_x86_64_cp') + ) + assert len(compared) == 2 + assert compared[0] != compared[1] diff --git a/dev_tools/services/acknowledg-forge/tests/test_safety.py b/dev_tools/services/acknowledg-forge/tests/test_safety.py index 14c275e5c4..3442567e17 100644 --- a/dev_tools/services/acknowledg-forge/tests/test_safety.py +++ b/dev_tools/services/acknowledg-forge/tests/test_safety.py @@ -18,14 +18,19 @@ import pytest import edgtest +import edgutil from edgacknowledg.client.test import _run_test_output +from edgacknowledg.forge import workspace + from edgacknowledg.forge import processor from edgacknowledg.forge.processor import _changes_test_ids, _read_ci_metadata from edgacknowledg.forge.sandbox import Sandbox, SandboxUnavailableError from edgacknowledg.forge.workspace import ( UnsafeArchiveError, WorkspaceError, + Workspaces, + _OutputTooLarge, _central_directory_size, _communicate_bounded, changes_patterns, @@ -35,6 +40,9 @@ patterns_for_test, ) +from conftest import Origin, pull_request_info +from fakes import REPOSITORY, FakeForge + def _zip(path: Path, entries: dict[str, bytes]) -> Path: with zipfile.ZipFile(path, 'w') as archive: for name, data in entries.items(): @@ -74,7 +82,8 @@ def test_extract_caps_total_size(tmp_path: Path) -> None: archive = _zip(tmp_path / 'a.zip', {'big.txt': b'0' * 100_000}) destination = tmp_path / 'out' destination.mkdir() - with pytest.raises(UnsafeArchiveError): + with pytest.raises(UnsafeArchiveError, + match = 'artifact exceeds the extracted size limit'): extract_artifact(archive, destination, max_bytes = 10_000) @@ -90,7 +99,9 @@ def test_extract_counts_inner_tarballs_compressed_size(tmp_path: Path) -> None: destination.mkdir() compressed_size = len(payload.getvalue()) extract_artifact(archive, destination, max_bytes = compressed_size) - assert (destination / 'cfg' / 't.rt.tar.gz').read_bytes() == payload.getvalue() + assert ( + (destination / 'cfg' / 't.rt.tar.gz').read_bytes() == payload.getvalue() + ) smaller = tmp_path / 'smaller' smaller.mkdir() @@ -267,7 +278,7 @@ def test_changes_patterns_cover_recordings_and_expectations() -> None: '-dir/b.sft.cpp: PASS', '+dir/b.sft.cpp: FAIL', ]) - patterns = changes_patterns(diff) + patterns = changes_patterns(io.StringIO(diff)) assert '/tests/tests/s/.a.rto/' in patterns assert '/tests/tests/s/a.*' in patterns assert '/tests/tests/other/dir/b.sft.cpp' in patterns @@ -284,6 +295,63 @@ def test_test_patterns_are_linear_in_the_path() -> None: ] +def test_line_pieces_are_bounded_and_lose_nothing() -> None: + text = ( + 'diff --git a/tests/tests/s/a.sft.cpp b/tests/tests/s/a.sft.cpp\n' + '+' + 'x' * 1000 + '\n\n+last' + ) + pieces = list(edgutil.iter_line_pieces(io.StringIO(text), 16)) + assert ''.join(piece for piece, _ in pieces) == text + assert all(len(piece) <= 16 for piece, _ in pieces) + # Only a line's first piece starts it. + assert [piece for piece, at_line_start in pieces if at_line_start] == [ + 'diff --git a/tes', '+xxxxxxxxxxxxxxx', '\n', '+last' + ] + + +def test_limited_lines_refuse_overlong_lines() -> None: + # Up to the limit (its end not counted), lines come whole. + text = '0123456789\nshort\n' + 'y' * 11 + '\nnever read\n' + lines = edgutil.iter_limited_lines(io.StringIO(text), 10) + assert next(lines) == '0123456789\n' + assert next(lines) == 'short\n' + with pytest.raises(edgutil.LineTooLongError, + match = 'line 3 is longer than 10 characters'): + next(lines) + # A last line without an end is checked too. + def limited(text: str) -> list[str]: + return list(edgutil.iter_limited_lines(io.StringIO(text), 10)) + + assert limited('a\n' + 'b' * 10) == ['a\n', 'b' * 10] + with pytest.raises(edgutil.LineTooLongError): + limited('a\n' + 'b' * 11) + + +def test_changes_patterns_pass_over_huge_lines_in_pieces() -> None: + # A line far over the limit, in an expectations file, names no test (and + # is never held whole); the lines after it still count. + diff = ( + 'diff --git a/tests/expectations/x/s.log b/tests/expectations/x/s.log\n' + '+++ b/tests/expectations/x/s.log\n' + '+big.sft.cpp:' + 'q' * (4 * edgutil.MAX_LINE_CHARS) + '\n' + '+ok.sft.cpp:cp:-DTEST_NUMBER=1:PASS:\n' + ) + handle = io.StringIO(diff) + real_readline = handle.readline + sizes: list[int] = [] + + def readline(size: int = -1) -> str: + piece = real_readline(size) + sizes.append(len(piece)) + return piece + + handle.readline = readline # type: ignore[method-assign] + patterns = changes_patterns(handle) + assert '/tests/tests/s/ok.sft.cpp' in patterns + assert not any('big' in pattern for pattern in patterns) + assert max(sizes) <= edgutil.MAX_LINE_CHARS + 1 + + def test_changes_patterns_skip_suites_no_suite_may_be_named() -> None: # A suite is in the patterns of every test its expectation file names. def expectations(suite: str) -> str: @@ -295,10 +363,10 @@ def expectations(suite: str) -> str: '+a.sft.cpp: PASS', ]) - patterns = changes_patterns(expectations('cwg')) + patterns = changes_patterns(io.StringIO(expectations('cwg'))) assert '/tests/tests/cwg/a.sft.cpp' in patterns for suite in ['s' * 65, 'a.b', 'a b']: - assert changes_patterns(expectations(suite)) == [ + assert changes_patterns(io.StringIO(expectations(suite))) == [ '/tests/.edgy/', '/tests/expectations/', ] @@ -317,34 +385,136 @@ def test_patterns_are_capped() -> None: *(f"+{n}.sft.cpp: FAIL" for n in range(cap)), ]) with pytest.raises(WorkspaceError, match = 'sparse-checkout patterns'): - changes_patterns(diff, max_patterns = cap) + changes_patterns(io.StringIO(diff), max_patterns = cap) # Within a higher cap. - changes_patterns(diff, max_patterns = 3 * cap) + changes_patterns(io.StringIO(diff), max_patterns = 3 * cap) async def test_git_output_can_be_bounded() -> None: - async def run(*argv: str, limit: int) -> object: - process = await asyncio.create_subprocess_exec( - *argv, - stdout = asyncio.subprocess.PIPE, - stderr = asyncio.subprocess.PIPE - ) - result = await _communicate_bounded(process, limit) - assert process.returncode is not None - return result - - assert await run('sh', '-c', 'printf out; printf err >&2', - limit = 3) == (b'out', b'err') + # Stdout goes to the sink as it comes; stderr is returned. + sunk: list[bytes] = [] + process = await asyncio.create_subprocess_exec( + 'sh', '-c', 'printf out; printf err >&2', + stdout = asyncio.subprocess.PIPE, + stderr = asyncio.subprocess.PIPE + ) + assert await _communicate_bounded(process, 3, sunk.append) == b'err' + assert process.returncode is not None + assert b''.join(sunk) == b'out' # One that would never stop is stopped (letting it clean up). process = await asyncio.create_subprocess_exec( 'yes', stdout = asyncio.subprocess.PIPE, stderr = asyncio.subprocess.PIPE ) - assert await _communicate_bounded(process, 1024 * 1024) is None + with pytest.raises(_OutputTooLarge): + await _communicate_bounded(process, 1024 * 1024, sunk.append) assert process.returncode == -signal.SIGTERM +async def test_git_stdin_is_written_and_stderr_bounded() -> None: + sunk: list[bytes] = [] + # It reads its input (larger than a pipe holds) while writing more + # stderr than is kept. + process = await asyncio.create_subprocess_exec( + 'sh', '-c', 'cat; head -c 1000000 /dev/zero | tr "\\0" e >&2', + stdin = asyncio.subprocess.PIPE, + stdout = asyncio.subprocess.PIPE, + stderr = asyncio.subprocess.PIPE + ) + data = b'pattern\n' * 100_000 + stderr = await _communicate_bounded( + process, + len(data), + sunk.append, + data + ) + assert process.returncode == 0 + assert b''.join(sunk) == data + assert stderr == b'e' * workspace._MAX_GIT_STDERR_BYTES + + +async def test_git_results_are_bounded(tmp_path: Path, + forge: FakeForge, + monkeypatch: pytest.MonkeyPatch + ) -> None: + workspaces = Workspaces(tmp_path / 'home', forge) + assert (await workspaces.git('version')).stdout.startswith('git version') + monkeypatch.setattr(workspace, '_MAX_GIT_OUTPUT_BYTES', 4) + with pytest.raises(WorkspaceError, match = 'wrote over 4 bytes'): + await workspaces.git('version') + + +async def test_diffs_are_written_as_they_come(tmp_path: Path, + origin: Origin, + forge: FakeForge) -> None: + # Longer than a chunk git's output comes in, with characters of several + # bytes across them. + lines = [f"{n}.sft.cpp: {'é€😀' * 10}" for n in range(2000)] + head = origin.push_pull_request(1, { + 'tests/expectations/x/s.log': ''.join(f"{line}\n" for line in lines), + }) + workspaces = Workspaces(tmp_path / 'home', forge) + info = pull_request_info(1, head, origin.main_sha) + head, _ = await workspaces.fetch_pull_request(info) + diff_file = tmp_path / 'pull-request.diff' + with diff_file.open('xb') as handle: + await workspaces.diff( + REPOSITORY, + origin.main_sha, + head, + ['tests/'], + handle + ) + diff = diff_file.read_text(encoding = 'utf-8') + assert len(diff_file.read_bytes()) > 64 * 1024 + assert '\ufffd' not in diff + added = [ + line[1:] for line in diff.splitlines() + if line.startswith('+') and not line.startswith('+++') + ] + assert added == lines + # Its patterns, read from the file. + with open(diff_file, encoding = 'utf-8') as handle: + assert '/tests/tests/s/1999.sft.cpp' in changes_patterns(handle) + + # Too long: refused. + workspaces = Workspaces(tmp_path / 'home', forge, max_diff_bytes = 1024) + with pytest.raises( + WorkspaceError, + match = r'over 1024 bytes \(max_diff_bytes\)' + ): + with (tmp_path / 'too-long.diff').open('xb') as handle: + await workspaces.diff( + REPOSITORY, + origin.main_sha, + head, + ['tests/'], + handle + ) + + +@pytest.mark.parametrize('raw', [b'\xff\xff', b'\xe2']) +async def test_diff_limit_includes_replacement_bytes( + tmp_path: Path, + forge: FakeForge, + monkeypatch: pytest.MonkeyPatch, + raw: bytes) -> None: + workspaces = Workspaces(tmp_path / 'home', forge, max_diff_bytes = 2) + + async def git(*args: str, **kwargs: object) -> None: + sink = kwargs['stdout_sink'] + assert callable(sink) + sink(raw) + + monkeypatch.setattr(workspaces, 'git', git) + destination = tmp_path / 'expanded.diff' + with pytest.raises(WorkspaceError, match = 'over 2 bytes'): + with destination.open('xb') as handle: + await workspaces.diff(REPOSITORY, 'base', 'head', ['tests/'], handle) + assert destination.stat().st_size <= 2 + + async def test_cancelled_git_output_reads_stop_git() -> None: # As when the service stops while git still has output to write: left # writing to a pipe no one reads, it would never exit. @@ -353,7 +523,9 @@ async def test_cancelled_git_output_reads_stop_git() -> None: stdout = asyncio.subprocess.PIPE, stderr = asyncio.subprocess.PIPE ) - reading = asyncio.create_task(_communicate_bounded(process, 1 << 40)) + reading = asyncio.create_task( + _communicate_bounded(process, 1 << 40, lambda chunk: None) + ) await asyncio.sleep(0.2) reading.cancel() await asyncio.wait_for(asyncio.gather(reading, return_exceptions = True), 10) @@ -389,6 +561,16 @@ def test_ci_reads_are_capped(tmp_path: Path, _read_ci_metadata(metadata) +def test_ci_test_ids_skip_overlong_lines(tmp_path: Path) -> None: + elist = tmp_path / 'a.elist' + elist.write_text( + 's/a.sft.cpp\n' + + 's/' + 'x' * processor.MAX_TEST_ID_CHARS + '\n' + + 's/b.sft.cpp\n' + ) + assert _changes_test_ids([elist]) == ['s/a.sft.cpp', 's/b.sft.cpp'] + + def test_failures_patterns_ignore_escaping_ids() -> None: patterns = failures_patterns(['s/a.sft.cpp', '../../etc/passwd']) assert '/tests/tests/s/a.sft.cpp' in patterns diff --git a/dev_tools/services/acknowledg-forge/tests/test_state.py b/dev_tools/services/acknowledg-forge/tests/test_state.py index 2b45050ae7..b478aa3e61 100644 --- a/dev_tools/services/acknowledg-forge/tests/test_state.py +++ b/dev_tools/services/acknowledg-forge/tests/test_state.py @@ -138,9 +138,14 @@ async def test_publisher_outage_leaves_serving_to_connecting( record = await store.find_pull_request('fake', REPOSITORY, 1) assert record is not None assert not record.is_dirty and record.error_count == 0 - assert [s.name for s in await store.sections(record.id)] == [ + sections = await store.sections(record.id) + assert [s.name for s in sections] == [ 'Test - Changes' ] + # The completed run contains only its results, with no temporary diff. + [section] = sections + assert list(section.root.parent.iterdir()) == [section.root] + assert list(harness.workspaces.build_staging_dir.iterdir()) == [] assert harness.publisher.served_names(record.review_id) == [] harness.publisher.available = True @@ -254,7 +259,7 @@ async def test_settle_deadline_announces_without_ci(tmp_path: Path, forge: FakeForge, store: ForgeStore, clock: Clock) -> None: - config = make_config(tmp_path / 'home', SETTLE_TIMEOUT = '600') + config = make_config(tmp_path / 'home', settle_timeout = 600) harness = Harness(store, forge, config, clock) await harness.start() head = origin.push_pull_request(1, PR_FILES) @@ -273,7 +278,7 @@ async def test_comment_not_duplicated_after_crash(tmp_path: Path, forge: FakeForge, store: ForgeStore, clock: Clock) -> None: - config = make_config(tmp_path / 'home', CI_WORKFLOW = '') + config = make_config(tmp_path / 'home', ci_workflow = None) harness = Harness(store, forge, config, clock) await harness.start() head = origin.push_pull_request(1, PR_FILES) @@ -300,7 +305,7 @@ async def test_fresh_database_adopts_existing_pull_requests( forge: FakeForge, store: ForgeStore, clock: Clock) -> None: - config = make_config(tmp_path / 'home', CI_WORKFLOW = '') + config = make_config(tmp_path / 'home', ci_workflow = None) head1 = origin.push_pull_request(1, PR_FILES) forge.set_pull_request(pull_request_info(1, head1, origin.main_sha, clock = clock)) @@ -404,7 +409,7 @@ async def test_webhook_endpoint_verifies_and_queues() -> None: async def test_app_without_webhooks_answers_only_health_checks() -> None: - # As the service serves it without ACKNOWLEDG_FORGE_WEBHOOKS. + # As the service serves it with webhooks disabled. client = create_app().test_client() response = await client.get(HEALTH_PATH) assert response.status_code == HTTPStatus.OK @@ -413,24 +418,36 @@ async def test_app_without_webhooks_answers_only_health_checks() -> None: assert response.status_code == HTTPStatus.NOT_FOUND -def test_webhooks_need_the_service_bind(tmp_path: Path) -> None: - config = make_config(tmp_path / 'home', SERVICE_BIND = '127.0.0.1:0', - WEBHOOKS = '1') +async def test_webhooks_need_the_service_bind(tmp_path: Path) -> None: + from edgacknowledg.forge.config import ForgeCredentials + from edgacknowledg.forge.service import ForgeComponents, run_service + + config = make_config( + tmp_path / 'home', service_bind = '127.0.0.1:0', + webhooks = True + ) assert config.service_bind == '127.0.0.1:0' and config.webhooks - config = make_config(tmp_path / 'home', SERVICE_BIND = '127.0.0.1:0') + config = make_config(tmp_path / 'home', service_bind = '127.0.0.1:0') assert not config.webhooks - with pytest.raises(ConfigError, match = 'SERVICE_BIND is not'): - make_config(tmp_path / 'home', WEBHOOKS = '1') + config = make_config(tmp_path / 'home', webhooks = True) + with pytest.raises(ConfigError, match = 'webhooks requires --service-bind'): + await run_service( + config, ForgeCredentials(), + ForgeComponents(FakeForge(tmp_path)) + ) # The old setting isn't silently ignored. - with pytest.raises(ConfigError, match = 'WEBHOOK_BIND is gone'): - make_config(tmp_path / 'home', WEBHOOK_BIND = '127.0.0.1:0') + with pytest.raises( + ConfigError, + match = 'unknown or secret setting webhook_bind' + ): + make_config(tmp_path / 'home', webhook_bind = '127.0.0.1:0') @pytest.mark.parametrize('setting, problem', [ - ('MAX_DIFF_BYTES', "git diff's output is over 1 bytes"), - ('MAX_SPARSE_PATTERNS', 'MAX_SPARSE_PATTERNS'), - ('MAX_ARTIFACT_ENTRIES', 'MAX_ARTIFACT_ENTRIES'), - ('MAX_ARTIFACT_DIRECTORY_BYTES', 'MAX_ARTIFACT_DIRECTORY_BYTES'), + ('max_diff_bytes', 'max_diff_bytes'), + ('max_sparse_patterns', 'max_sparse_patterns'), + ('max_artifact_entries', 'max_artifact_entries'), + ('max_artifact_directory_bytes', 'max_artifact_directory_bytes'), ]) async def test_limits_are_settings(tmp_path: Path, origin: Origin, @@ -440,7 +457,7 @@ async def test_limits_are_settings(tmp_path: Path, setting: str, problem: str) -> None: # A pull request (with CI results) within the defaults... - config = make_config(tmp_path / 'home', **{setting: '1'}) + config = make_config(tmp_path / 'home', **{setting: 1}) harness = Harness(store, forge, config, clock) await harness.start() head = origin.push_pull_request(1, PR_FILES) diff --git a/dev_tools/services/acknowledg-forge/tests/test_symlinks.py b/dev_tools/services/acknowledg-forge/tests/test_symlinks.py index c10e0472f5..41be601c46 100644 --- a/dev_tools/services/acknowledg-forge/tests/test_symlinks.py +++ b/dev_tools/services/acknowledg-forge/tests/test_symlinks.py @@ -557,7 +557,6 @@ async def test_show_source_reads_only_what_is_shown( monkeypatch: pytest.MonkeyPatch) -> None: home, root = _test_tree(tmp_path, secret) (home / 'tests' / 'suite' / 'big.sft.cpp').write_text('int x;\n' * 1000) - monkeypatch.setattr(test_provider, 'MAX_DISPLAY_CONTENT_SIZE', 64) monkeypatch.setattr(shared, 'MAX_DISPLAY_CONTENT_SIZE', 64) reads: list[int] = [] real_open = shared.ConfinedPath.open @@ -566,12 +565,17 @@ def open_recording_reads(self: shared.ConfinedPath, *args: Any, **kwargs: Any) -> Any: handle = real_open(self, *args, **kwargs) - real_read = handle.read + real_readline = handle.readline + + def readline(size: int = -1) -> Any: + line = real_readline(size) + reads.append(len(line)) + return line def read(size: int = -1) -> Any: - reads.append(size) - return real_read(size) + raise AssertionError('read whole') + handle.readline = readline handle.read = read return handle @@ -594,7 +598,8 @@ def read(size: int = -1) -> Any: response = reply['v']['client_response'] assert response['truncated'] and len(response['content']) <= 64 assert response['content'].startswith('int x;\n') - assert reads == [65] + # A line at a time, and no more than one past what fits. + assert reads == [7] * 10 async def test_show_source_range(tmp_path: Path, secret: Path) -> None: @@ -625,7 +630,7 @@ async def ask(test_id: str, line: int, ctx: int) -> dict[str, Any]: # The window is returned, and the request echoed. response = await ask('suite/ten.sft.cpp', 5, 2) assert response['content'] == 'line 3\nline 4\nline 5\nline 6\nline 7' - assert response['first_line'] == 3 and response['total_lines'] == 10 + assert response['first_line'] == 3 and 'total_lines' not in response assert response['line'] == 5 and response['context'] == 2 assert 'truncated' not in response diff --git a/dev_tools/services/acknowledg/PROTOCOL.md b/dev_tools/services/acknowledg/PROTOCOL.md index b40dd98ddd..41c39556e1 100644 --- a/dev_tools/services/acknowledg/PROTOCOL.md +++ b/dev_tools/services/acknowledg/PROTOCOL.md @@ -3,7 +3,7 @@ This document describes the WebSocket protocol used by the AcknowlEDG server, the Vue client, and the `edgy-review` / `edg-bench-review` / `edg-acknowledg-cli` providers. -The current protocol revision is **20**. +The current protocol revision is **21**. All frames are JSON text. Implementations live in `edgacknowledg` (`dev_tools/pylibs/edgacknowledg`) and must stay in sync with the JavaScript @@ -74,15 +74,15 @@ work can flow immediately. Providers may optionally send identity updates at any time (on dedicated sockets or nested in `batch.event`): ```json -{ "t": "meta.section.name", "r": 20, "v": "" } +{ "t": "meta.section.name", "r": 21, "v": "" } ``` ```json -{ "t": "meta.review.name", "r": 20, "v": [ ] } +{ "t": "meta.review.name", "r": 21, "v": [ ] } ``` ```json -{ "t": "meta.review.timestamp", "r": 20, "v": "" } +{ "t": "meta.review.timestamp", "r": 21, "v": "" } ``` - `meta.section.name` sets the display name for the section tab (shown in @@ -329,7 +329,7 @@ content that isn't the provider user's own, one reached through a symlink), the provider doesn't answer the request. It reports the reviewer instead: ```json -{ "t": "illegal-access", "r": 20, "v": { "reviewer_id": "" } } +{ "t": "illegal-access", "r": 21, "v": { "reviewer_id": "" } } ``` The server then closes that reviewer's socket with code **3001**, without @@ -345,7 +345,7 @@ access), it doesn't answer the request either. It tells the reviewer it failed, through the server, and keeps serving the section: ```json -{ "t": "invalid", "r": 20, "v": { "reviewer_id": "" } } +{ "t": "invalid", "r": 21, "v": { "reviewer_id": "" } } ``` The server sends `invalid` to that reviewer alone, with `v` `null` and `s` @@ -358,10 +358,26 @@ section is served. ### Source `client_response` shapes (reviewer `v`) Display payloads that carry file text use `content` (string or `null` if -missing) and may set `truncated: true` when the body exceeds 8 MiB -(`MAX_DISPLAY_CONTENT_SIZE`). Protocol messages themselves are capped at +missing) and may set `truncated: true` when the content or the text consumed +while reading exceeds `MAX_DISPLAY_CONTENT_SIZE` (8 MiB for ASCII text; +the implementation counts characters). The read budget counts text skipped +before a requested range and text discarded by the line limit. Protocol +messages themselves are capped at 10 MiB (`MAX_MESSAGE_SIZE`). +Each line of `content` is at most 16 Ki characters +(`MAX_DISPLAY_LINE_SIZE`). A provider cuts a longer line to that length and +lists its 1-based line number in `lines_truncated` (sorted ascending, and +omitted when no line was cut). Nothing in `content` itself marks a cut. The +reviewer UI appends `…` (U+2026) to each listed line and shows a "Some Lines +Truncated" notice. Line cuts happen before the 8 MiB cap, so `truncated` and +`lines_truncated` may both be set; `lines_truncated` lists only lines +present in `content`. + +Single-line labels (the `qualifier`, `variant`, and `status` of +`test.src.list-statuses` lines, and its `remark`) are cut to 1 Ki characters +(`MAX_DISPLAY_LABEL_SIZE`), ending in `…`. + **`test.src.list-statuses`** ```json @@ -392,7 +408,12 @@ missing) and may set `truncated: true` when the body exceeds 8 MiB **`test.src.show-diff`**, **`test.src.show-output`** ```json -{ "id": "", "content": "", "truncated": false } +{ + "id": "", + "content": "", + "truncated": false, + "lines_truncated": [1] +} ``` **`test.src.show-diff-curr`** — same as above, plus optional @@ -421,7 +442,8 @@ any Unicode name is allowed). The primary file itself is not in `files`; it is "id": "", "assoc_file": "", "content": "", - "truncated": false + "truncated": false, + "lines_truncated": [1] } ``` @@ -430,8 +452,10 @@ neither (otherwise the server answers `invalid`). Then `content` holds only lines `line - context` through `line + context` (clamped to the file, joined with `\n`, no trailing newline), `line` and `context` are echoed back so the reviewer can match the reply, and the reply adds `first_line` (the 1-based -number of the first returned line) and `total_lines`. `truncated` is not -sent. A missing file still yields `content: null`. +number of the first requested line, `content` being empty should the file +end before it). `truncated: true` is sent if the read budget stops reading +before the window ends. `lines_truncated` numbers lines within `content` +(1 for the line at `first_line`). A missing file still yields `content: null`. **`bench.src.list-statuses`** @@ -451,7 +475,12 @@ sent. A missing file still yields `content: null`. **`bench.src.show-annotate`** ```json -{ "id": "", "content": "", "truncated": false } +{ + "id": "", + "content": "", + "truncated": false, + "lines_truncated": [1] +} ``` ### Editor `v` as seen by the reviewer diff --git a/dev_tools/services/acknowledg/client/src/BenchReviewView.vue b/dev_tools/services/acknowledg/client/src/BenchReviewView.vue index b414c930b4..cccff6704f 100644 --- a/dev_tools/services/acknowledg/client/src/BenchReviewView.vue +++ b/dev_tools/services/acknowledg/client/src/BenchReviewView.vue @@ -22,6 +22,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception } from './bench-metrics.js' import { formatBenchAnchor, parseBenchAnchor } from './review-anchor.js' import { protocolMessage } from './sockets.js' + import { displayLines } from './display-content.js' import AppButton from './AppButton.vue' import AppSelect from './AppSelect.vue' import BenchStatus from './BenchStatus.vue' @@ -47,8 +48,14 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception ] const selectedBenchmarkId = ref(null) const displayTruncated = ref(false) + // 1-based numbers of the displayed lines the provider cut. + const displayedLinesTruncated = ref([]) const displayedOutput = ref(null) + const displayedOutputLines = computed(() => { + return displayLines(displayedOutput.value, displayedLinesTruncated.value) + }) + provide('activeBenchmarkId', selectedBenchmarkId) provide('benchSortMode', sortMode) @@ -215,12 +222,14 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception } else if (message.t === 'bench.src.show-annotate') { if (message.v.id === selectedBenchmarkId.value) { displayTruncated.value = false + displayedLinesTruncated.value = [] if (message.v.content === null) { displayedOutput.value = 'Annotation unavailable!' } else { if (message.v.truncated) { displayTruncated.value = true } + displayedLinesTruncated.value = message.v.lines_truncated ?? [] displayedOutput.value = message.v.content } } @@ -409,12 +418,17 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception class="truncation-warning"> Content Too Long: Display Truncated +
+ Some Lines Truncated +
- 
{{ displayedOutput }}
+ 
{{ displayedOutputLines.join('\n') }}
still loading - // { missing: true } -> the file cannot be shown - // { content, firstLine } -> the lines to show + // Still loading: + // null + // The file cannot be shown: + // { missing: true } + // The lines to show: + // { content, firstLine, linesTruncated, truncated } import AnchoredPopover from './AnchoredPopover.vue' import LoadingSpinner from './LoadingSpinner.vue' import NumberedSource from './NumberedSource.vue' @@ -39,13 +42,23 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception {{ location.file }}:{{ location.line }}
-
+
+ +
{{ source.reason ?? 'File not found!' }}
- + @@ -60,4 +73,15 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception padding: 8px 0; text-align: center; } + .loading { + font-size: 28px; + } + .content-notice { + color: white; + background-color: orange; + border-radius: 10px; + text-align: center; + padding: 5px; + margin-bottom: 8px; + } diff --git a/dev_tools/services/acknowledg/client/src/TestReviewView.vue b/dev_tools/services/acknowledg/client/src/TestReviewView.vue index a7a8fd6a08..31dd02c982 100644 --- a/dev_tools/services/acknowledg/client/src/TestReviewView.vue +++ b/dev_tools/services/acknowledg/client/src/TestReviewView.vue @@ -25,6 +25,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception parseDiffDiagnostics, resolveDiagnosticFile } from './diagnostics.js' + import { displayLines } from './display-content.js' import DiffLine from './DiffLine.vue' import LoadingSpinner from './LoadingSpinner.vue' @@ -55,11 +56,17 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception const displayedTestPrimaryName = ref(DEFAULT_PRIMARY_SOURCE_NAME) const displayedTestAssocFileName = ref(null) const displayTruncated = ref(false) + // 1-based numbers of the displayed lines the provider cut. + const displayedLinesTruncated = ref([]) const currDiffDivergenceNotice = ref(null) const displayedOutput = ref("") const outputViewModeTarget = ref("test.src.show-diff") const isMultiConfig = ref(false) + const displayedOutputLines = computed(() => { + return displayLines(displayedOutput.value, displayedLinesTruncated.value) + }) + const displayedTestConfig = computed(() => { return displayedGroupConfigs.value[0] ?? "" }) @@ -150,7 +157,15 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception } const source = reply.content === null ? { missing: true } : - { content: reply.content, firstLine: reply.first_line } + { + content: displayLines( + reply.content, + reply.lines_truncated ?? [] + ).join('\n'), + firstLine: reply.first_line, + linesTruncated: reply.lines_truncated ?? [], + truncated: reply.truncated ?? false + } sourceSnippetCache.set(snippetKey(reply.assoc_file, reply.line), source) const request = sourcePopover.value.request if (sourcePopover.value.open && request !== null && @@ -444,6 +459,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception message.t === "test.src.show-output") { if (message.v.id === configuredIdForDisplayedTest()) { displayTruncated.value = false + displayedLinesTruncated.value = [] currDiffDivergenceNotice.value = null if (message.v.content === null) { displayedOutput.value = "Missing .rt.tar file!" @@ -451,6 +467,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception if (message.v.truncated) { displayTruncated.value = true } + displayedLinesTruncated.value = message.v.lines_truncated ?? [] displayedOutput.value = message.v.content } if (message.t === "test.src.show-diff-curr" && @@ -473,6 +490,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception } else if (message.t === "test.src.show-src") { if (message.v.id === displayedTestName.value) { displayTruncated.value = false + displayedLinesTruncated.value = [] currDiffDivergenceNotice.value = null if (message.v.content === null) { displayedOutput.value = "File not found!" @@ -480,6 +498,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception if (message.v.truncated) { displayTruncated.value = true } + displayedLinesTruncated.value = message.v.lines_truncated ?? [] displayedOutput.value = message.v.content } } @@ -655,6 +674,11 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception class="content-notice"> Content Too Long: Display Truncated +
+ Some Lines Truncated +
{{ currDiffDivergenceNotice }} @@ -673,15 +697,15 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception padding-left: 0; --content-inset: 5px;"> + :text="displayedOutputLines.join('\n')" /> diff --git a/dev_tools/services/acknowledg/client/src/display-content.js b/dev_tools/services/acknowledg/client/src/display-content.js new file mode 100644 index 0000000000..242223e2a0 --- /dev/null +++ b/dev_tools/services/acknowledg/client/src/display-content.js @@ -0,0 +1,27 @@ +/* +Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM +Exceptions. +See https://edgcpp.org/LICENSE.txt for license information. +SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +*/ + +// Display content as providers send it: `content`, maybe with +// `lines_truncated`, the 1-based numbers of the lines they cut (see +// MAX_DISPLAY_LINE_SIZE in PROTOCOL.md). Nothing in the content marks a cut; +// the lines are marked here. + +// U+2026 HORIZONTAL ELLIPSIS (JavaScript has no named escapes). +const TRUNCATION_MARK = '\u2026' + +// The lines of `content`, those numbered in `linesTruncated` ending in an +// ellipsis; none for null content. +export const displayLines = (content, linesTruncated) => { + if (content === null || content === undefined) { + return [] + } + const lines = content.split('\n') + for (const lineNumber of linesTruncated) { + lines[lineNumber - 1] += TRUNCATION_MARK + } + return lines +} diff --git a/dev_tools/services/acknowledg/client/src/sockets.js b/dev_tools/services/acknowledg/client/src/sockets.js index bea1823d0e..cf927dd8ef 100644 --- a/dev_tools/services/acknowledg/client/src/sockets.js +++ b/dev_tools/services/acknowledg/client/src/sockets.js @@ -5,7 +5,7 @@ See https://edgcpp.org/LICENSE.txt for license information. SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -const PROTOCOL_REVISION = 20 +const PROTOCOL_REVISION = 21 const getSocketHost = () => { if (process.env.NODE_ENV === "development") { diff --git a/dev_tools/spelling_allowlist.json b/dev_tools/spelling_allowlist.json index 3899d26036..da702f93c1 100644 --- a/dev_tools/spelling_allowlist.json +++ b/dev_tools/spelling_allowlist.json @@ -474,6 +474,8 @@ "fullmatch", "gzip", "hostname", + "http", + "https", "inbounds", "ingestor", "init", @@ -484,6 +486,7 @@ "loopback", "metadatas", "metavar", + "mkdir", "monospace", "normalizer", "parseable", @@ -491,16 +494,24 @@ "poller", "postgresql", "prebuild", + "proc", "reflectable", "rematerialize", "repo", + "returncode", "sandboxing", "serializer", "splitable", "splittable", "staticmethod", + "stderr", + "stdout", "str", + "subprocess", "symlinks", + "toml", + "tty", + "ui", "uncomment", "unsandboxed", "untrusted", @@ -513,6 +524,7 @@ "paths": { "bin": { "words": [ + "argparse", "commandline", "deduplication", "subparsers" diff --git a/dev_tools/tests/test_edgtest_prune.py b/dev_tools/tests/test_edgtest_prune.py new file mode 100644 index 0000000000..6e1a54d49a --- /dev/null +++ b/dev_tools/tests/test_edgtest_prune.py @@ -0,0 +1,134 @@ +# Part of the EDG Compiler Project, under the Apache License v2.0 with LLVM +# Exceptions. +# See https://edgcpp.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +'''Tests for pruning a test run's .rt.tar files (edgtest.prune_run_archives) +and edg-prune-test-run. + +Run with: python3 -m unittest discover -s dev_tools/tests +(with dev_tools/pylibs on PYTHONPATH). +''' + +import os +import subprocess +import sys +import tempfile +import unittest + +from pathlib import Path +from typing import List + +import edgtest + +from edgtest import runtest + +DEV_TOOLS_DIR = Path(__file__).resolve().parents[1] +PRUNE_TOOL_PATH = DEV_TOOLS_DIR / 'bin' / 'edg-prune-test-run' + +SUFFIX = runtest.RUN_TEST_OUTPUT_SUFFIX + +def _touch(path: Path) -> Path: + path.parent.mkdir(parents = True, exist_ok = True) + path.write_bytes(b'') + return path + +def _files(root: Path) -> List[str]: + return sorted( + Path(directory, name).relative_to(root).as_posix() + for directory, _, names in os.walk(root) + for name in names + ) + +class PruneRunArchivesTests(unittest.TestCase): + + def setUp(self) -> None: + temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(temp_dir.cleanup) + self.run_dir = Path(temp_dir.name) / 'run' + # One configuration whose changes.elist names two tests of one suite + # (and one without results), with every test's results... + config = self.run_dir / 'edg_x86_64' + _touch(config / 'changes.elist').write_text( + 'suite/a.sft.cpp\nsuite/dir/b.sft.c\nsuite/none.sft.cpp\n\n' + ) + for name in ('changes.elog', 'executed.elist', 'suite-suite.elog', + 'suite-suite.log', 'suite-other.elog', 'suite-other.log'): + _touch(config / name) + for name in ('suite/a.sft', 'suite/dir/b.sft', 'suite/c.sft', + 'other/a.sft'): + _touch(config / f"{name}{SUFFIX}") + # ...one without a changes.elist... + config = self.run_dir / 'edg_x86_32' + for name in ('executed.elist', 'suite-suite.elog', 'suite-suite.log', + f"suite/a.sft{SUFFIX}"): + _touch(config / name) + # ...and the run's replays (removed whole). + _touch(self.run_dir / 'replays' / f"suite/c.sft{SUFFIX}") + + def test_keeps_only_what_changes_elist_names(self) -> None: + changes_file = self.run_dir / 'edg_x86_64' / 'changes.elist' + read = list(edgtest._scan_changes_file(changes_file)) + + self.assertEqual(edgtest.prune_run_archives(self.run_dir), (2, 4, 2)) + # Suites with no changes go whole, and unnamed results with them. + self.assertEqual(_files(self.run_dir), [ + 'edg_x86_32/executed.elist', + 'edg_x86_64/changes.elist', + 'edg_x86_64/changes.elog', + 'edg_x86_64/executed.elist', + 'edg_x86_64/suite-suite.elog', + 'edg_x86_64/suite-suite.log', + 'edg_x86_64/suite/a.sft' + SUFFIX, + 'edg_x86_64/suite/dir/b.sft' + SUFFIX, + ]) + # A review reads what it read before. + self.assertEqual(list(edgtest._scan_changes_file(changes_file)), read) + + # Pruning again changes nothing. + self.assertEqual(edgtest.prune_run_archives(self.run_dir), (2, 0, 0)) + + def test_keeps_nothing_without_changes_elist(self) -> None: + (self.run_dir / 'edg_x86_64' / 'changes.elist').unlink() + self.assertEqual(edgtest.prune_run_archives(self.run_dir), (0, 6, 3)) + self.assertEqual(_files(self.run_dir), [ + 'edg_x86_32/executed.elist', + 'edg_x86_64/changes.elog', + 'edg_x86_64/executed.elist', + ]) + + def test_rejects_invalid_test_ids(self) -> None: + changes_file = self.run_dir / 'edg_x86_64' / 'changes.elist' + for test_id in ('a.sft.cpp', '/suite/a.sft.cpp', + 'suite/../a.sft.cpp', '../suite/a.sft.cpp'): + with self.subTest(test_id = test_id): + changes_file.write_text(f"{test_id}\n") + with self.assertRaisesRegex(ValueError, 'invalid test id'): + edgtest.prune_run_archives(self.run_dir) + self.assertTrue( + (self.run_dir / 'edg_x86_64' / f"suite/a.sft{SUFFIX}").exists() + ) + + def test_command(self) -> None: + env = dict(os.environ) + env['PYTHONPATH'] = str(DEV_TOOLS_DIR / 'pylibs') + result = subprocess.run( + [sys.executable, str(PRUNE_TOOL_PATH), str(self.run_dir)], + capture_output = True, + text = True, + env = env + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn('Kept 2', result.stdout) + self.assertIn('removed 4 others, and 2 suites', result.stdout) + + missing = subprocess.run( + [sys.executable, str(PRUNE_TOOL_PATH), str(self.run_dir / 'missing')], + capture_output = True, + text = True, + env = env + ) + self.assertEqual(missing.returncode, 1) + +if __name__ == '__main__': + unittest.main()