diff --git a/.mise.toml b/.mise.toml index 33d7aedf..3cbfdb70 100644 --- a/.mise.toml +++ b/.mise.toml @@ -33,6 +33,7 @@ run = [ "bash ./tests/renovate-custom-manager-coverage.sh", "bash ./tests/renovate-apm-manager.sh", "bash ./tests/renovate-annotation-managers.sh", + "bash ./tests/renovate-shell-manager.sh", "bash ./tests/renovate-dprint-plugins.sh", "bash ./tests/renovate-batching-contract.sh", "bash ./tests/renovate-engines-range.sh", diff --git a/.renovate/customManagers.json5 b/.renovate/customManagers.json5 index 4074c5ba..4ffcf566 100644 --- a/.renovate/customManagers.json5 +++ b/.renovate/customManagers.json5 @@ -2,13 +2,32 @@ $schema: "https://docs.renovatebot.com/renovate-schema.json", customManagers: [ { + // Annotated versions in shell scripts (and chezmoi .tmpl copies), e.g. + // # renovate: datasource=github-releases depName=bats-core/bats-assert + // BATS_ASSERT_VERSION="v2.1.0" + // # renovate: datasource=docker depName=docker.io/library/postgres + // POSTGRES_IMAGE="docker.io/library/postgres:18.3-alpine@sha256:<64 hex>" + // Optional packageName=, versioning=, extractVersion= and registryUrl= + // follow depName= in that order, like Renovate's own *Versions presets. + // An image value keeps its `[:port]//:` prefix; only + // the tag and digest are tracked, so images need a path + // (`docker.io/library/redis:7.4`, not `redis:7.4`). Without a path a value + // keeps a leading numeric Debian epoch (`5:29.6.2-1~…`). Three exclusive + // patterns: double-quoted, single-quoted and unquoted values; each must end + // at the quote or a command separator, so a value that is not a plain + // literal (`"1.2.3 ${X}"`, `1.2.3$S`, a backtick substitution) is not + // matched. Replaces the truenas-apps-only SOPS manager for scripts/dccd.sh, + // which this covers. customType: "regex", - description: "Process SOPS version in dccd.sh", - managerFilePatterns: ["/scripts/dccd\\.sh$/"], + description: "Process annotated versions in shell scripts", + managerFilePatterns: ["/\\.(?:sh|bash|zsh|bats)(?:\\.tmpl)?$/"], matchStrings: [ - "# renovate: datasource=github-releases depName=(?\\S+)\\nSOPS_VERSION=\"(?[^\"]+)\"", + "#[ \\t]*renovate:[ \\t]*datasource=(?\\S+)[ \\t]+depName=(?\\S+)(?:[ \\t]+packageName=(?\\S+))?(?:[ \\t]+versioning=(?\\S+))?(?:[ \\t]+extractVersion=(?\\S+))?(?:[ \\t]+registryUrl=(?\\S+))?[ \\t]*\\r?\\n[ \\t]*(?:(?:export|local|readonly|declare(?:[ \\t]+-[a-zA-Z]+)*)[ \\t]+)?[A-Za-z_][A-Za-z0-9_]*=\"(?:[^\\s\"'@$;`]*/[^\\s\"'@$;`/]*:)?(?(?:[0-9]+:)?[A-Za-z0-9][^\\s\"'@$;`:/]*)(?:@(?sha256:[a-f0-9]{64}))?\"(?:[ \\t;&|]|\\r?\\n|$)", + "#[ \\t]*renovate:[ \\t]*datasource=(?\\S+)[ \\t]+depName=(?\\S+)(?:[ \\t]+packageName=(?\\S+))?(?:[ \\t]+versioning=(?\\S+))?(?:[ \\t]+extractVersion=(?\\S+))?(?:[ \\t]+registryUrl=(?\\S+))?[ \\t]*\\r?\\n[ \\t]*(?:(?:export|local|readonly|declare(?:[ \\t]+-[a-zA-Z]+)*)[ \\t]+)?[A-Za-z_][A-Za-z0-9_]*='(?:[^\\s\"'@$;]*/[^\\s\"'@$;/]*:)?(?(?:[0-9]+:)?[A-Za-z0-9][^\\s\"'@$;:/]*)(?:@(?sha256:[a-f0-9]{64}))?'(?:[ \\t;&|]|\\r?\\n|$)", + "#[ \\t]*renovate:[ \\t]*datasource=(?\\S+)[ \\t]+depName=(?\\S+)(?:[ \\t]+packageName=(?\\S+))?(?:[ \\t]+versioning=(?\\S+))?(?:[ \\t]+extractVersion=(?\\S+))?(?:[ \\t]+registryUrl=(?\\S+))?[ \\t]*\\r?\\n[ \\t]*(?:(?:export|local|readonly|declare(?:[ \\t]+-[a-zA-Z]+)*)[ \\t]+)?[A-Za-z_][A-Za-z0-9_]*=(?:[^\\s\"'@$;`&|<>()#]*/[^\\s\"'@$;`&|<>()#/]*:)?(?(?:[0-9]+:)?[A-Za-z0-9][^\\s\"'@$;`&|<>()#:/]*)(?:@(?sha256:[a-f0-9]{64}))?(?:[ \\t;&|]|\\r?\\n|$)", ], - datasourceTemplate: "github-releases", + datasourceTemplate: "{{{datasource}}}", + versioningTemplate: "{{#if versioning}}{{{versioning}}}{{/if}}", }, { customType: "regex", @@ -18,7 +37,7 @@ "/workflow-templates/.+\\.ya?ml$/", ], matchStrings: [ - "# renovate: datasource=(?\\S+)(?: depName=(?\\S+))?\\n\\s+[\\w-]*version: \\[?\"?(?[^\\s\"\\[\\],]+)\"?\\]?[ \\t]*(?:#[^\\n]*)?(?:\\r?\\n|$)", + "# renovate: datasource=(?\\S+)(?: depName=(?\\S+))?\\r?\\n\\s+[\\w-]*version: \\[?\"?(?[^\\s\"\\[\\],]+)\"?\\]?[ \\t]*(?:#[^\\n]*)?(?:\\r?\\n|$)", ], datasourceTemplate: "{{{datasource}}}", // `depName=` may be omitted for the runtime datasources, e.g. @@ -44,7 +63,7 @@ "/workflow-templates/.+\\.ya?ml$/", ], matchStrings: [ - "# renovate: datasource=(?\\S+) depName=(?\\S+)\\n\\s+[\\w-]+:\\s*\"?[^\\s\"@]+:(?[^\\s\"@]+)@(?sha256:[a-f0-9]+)\"?", + "# renovate: datasource=(?\\S+) depName=(?\\S+)\\r?\\n\\s+[\\w-]+:\\s*\"?[^\\s\"@]+:(?[^\\s\"@]+)@(?sha256:[a-f0-9]+)\"?", ], datasourceTemplate: "{{{datasource}}}", }, @@ -53,7 +72,7 @@ description: "Process min_version in .mise.toml", managerFilePatterns: ["/\\.mise\\.toml$/"], matchStrings: [ - "# renovate: datasource=github-releases depName=(?\\S+)\\nmin_version = \"(?[^\"]+)\"", + "# renovate: datasource=github-releases depName=(?\\S+)\\r?\\nmin_version = \"(?[^\"]+)\"", ], datasourceTemplate: "github-releases", }, @@ -64,7 +83,7 @@ "/ansible/inventory/.+\\.ya?ml$/", ], matchStrings: [ - "# renovate: datasource=(?\\S+) depName=(?\\S+)(?: registryUrl=(?\\S+))?[ \\t]*\\n\\s*[\\w-]+:\\s*\"?(?[^\\s\"]+)\"?", + "# renovate: datasource=(?\\S+) depName=(?\\S+)(?: registryUrl=(?\\S+))?[ \\t]*\\r?\\n\\s*[\\w-]+:\\s*\"?(?[^\\s\"]+)\"?", ], // Any datasource, with an optional registryUrl, and any key on the next // line: the annotation marks the value. For example, Debian packages diff --git a/docs/architecture.md b/docs/architecture.md index 1266f6f5..76824690 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1233,6 +1233,18 @@ can still assign Code Owners if an automerge PR cannot merge cleanly. - **Ansible inventory** (`ansible/inventory/`): any datasource, an optional `registryUrl=` and any key, e.g. Debian packages pinned per suite with `# renovate: datasource=deb depName=docker-ce registryUrl=https://…`. +- **Shell scripts** (`.sh`, `.bash`, `.zsh`, `.bats`, and their chezmoi + `.tmpl` copies): the next line is a `NAME=value` assignment, optionally after + `export`, `local`, `readonly` or `declare`. The annotation can add + `packageName=`, `versioning=`, `extractVersion=` and `registryUrl=`, in that + order. For an image value (`"docker.io/library/postgres:18.3-alpine@sha256:…"`) + only the tag and digest are tracked; the image needs a registry or namespace + path, so write `docker.io/library/redis:7.4` rather than `redis:7.4`. Other + values keep a leading Debian epoch (`5:29.6.2-1~…`). Values that aren't plain + literals (`"${OTHER}"`, backtick substitutions) are skipped. Scripts without one of those extensions are not + scanned. + +All annotation managers accept CRLF line endings as well as LF. ### dprint plugins diff --git a/tests/renovate-shell-manager.sh b/tests/renovate-shell-manager.sh new file mode 100644 index 00000000..b98edd93 --- /dev/null +++ b/tests/renovate-shell-manager.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# +# Checks the Renovate custom manager for `# renovate:` annotations in shell +# scripts (.renovate/customManagers.json5, "Process annotated versions in shell +# scripts"): +# +# - plain versions, images with a digest, optional annotation fields and +# assignment keywords are extracted; non-literal values are not; +# - CRLF files match like LF files, for every annotation manager; +# - no annotated line is extracted by two managers, which would make Renovate +# update the same value twice (the old dccd.sh SOPS manager is replaced). + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +custom_managers="${repo_root}/.renovate/customManagers.json5" + +if ! command -v pyjson5 >/dev/null 2>&1; then + echo "error: 'pyjson5' not found on PATH. Run via 'mise run test'." >&2 + exit 1 +fi + +MANAGERS="$(pyjson5 --as-json "${custom_managers}" | tr -d '\r')" python3 - <<'PY' +import json +import os +import re +import sys + +managers = json.loads(os.environ["MANAGERS"])["customManagers"] +errors = [] + + +def compile_js(pattern): + # Renovate uses JavaScript named groups; Python spells them (?P...). + return re.compile(pattern.replace("(?<", "(?P<")) + + +shell = [m for m in managers if m["description"] == "Process annotated versions in shell scripts"] +if len(shell) != 1: + print("error: expected one shell annotation manager", file=sys.stderr) + sys.exit(1) +shell = shell[0] +if any(m["description"].startswith("Process SOPS version") for m in managers): + errors.append("the dccd.sh SOPS manager duplicates the shell manager; remove it") +for m in managers: + for p in m.get("matchStrings", []): + # Renovate runs custom-manager regexes on RE2, which has no lookaround. + if re.search(r"\(\? 1: + errors.append(f"shell manager ({label}): {text!r} matched by {len(matches)} patterns") + got = None if not matches else tuple(matches[0].groupdict().get(n) for n in names) + if got != want: + errors.append(f"shell manager ({label}): {text!r}: want {want}, got {got}") + + +def search(text): + return next((m for m in (p.search(text) for p in patterns) if m), None) + + +m = search(f"# renovate: datasource=docker depName=app packageName=ghcr.io/o/app versioning=loose registryUrl=https://ghcr.io\nAPP=1.2\n") +if not m or (m["packageName"], m["versioning"], m["registryUrl"]) != ("ghcr.io/o/app", "loose", "https://ghcr.io"): + errors.append(f"shell manager: optional annotation fields not captured: {m and m.groupdict()}") + +file_patterns = [compile_js(p.strip("/")) for p in shell["managerFilePatterns"]] +for path, want in { + "tests/setup_libs.sh": True, + "scripts/dccd.sh": True, + "home/dot_local/bin/executable_x.sh.tmpl": True, + "tests/foo.bats": True, + "a.bash": True, + "README.md": False, + "Makefile": False, +}.items(): + if any(p.search(path) for p in file_patterns) != want: + errors.append(f"shell manager managerFilePatterns: {path} should {'' if want else 'not '}match") + +# Every annotation manager matches CRLF files, and each annotated line is +# extracted by exactly one manager. +samples = { + ".github/workflows/ci.yml": ' # renovate: datasource=github-releases depName=jdx/mise\n version: "2026.10.19"\n', + ".github/workflows/x.yml": f' # renovate: datasource=docker depName=mcr.microsoft.com/devcontainers/base\n source: "mcr.microsoft.com/devcontainers/base:debian-12@{digest}"\n', + ".mise.toml": '# renovate: datasource=github-releases depName=jdx/mise\nmin_version = "2026.4.3"\n', + "ansible/inventory/group_vars/all.yml": ' # renovate: datasource=github-releases depName=getsops/sops\n sops_version: "v3.11.0"\n', + "scripts/dccd.sh": f'{gr}getsops/sops\nSOPS_VERSION="v3.13.3"\n', +} +for path, text in samples.items(): + for variant, label in ((text, "LF"), (text.replace("\n", "\r\n"), "CRLF")): + hits = [] + for m in managers: + if not any(compile_js(p.strip("/")).search(path) for p in m["managerFilePatterns"]): + continue + if any(compile_js(s).search(variant) for s in m.get("matchStrings", [])): + hits.append(m["description"]) + if len(hits) != 1: + errors.append(f"{path} ({label}): expected exactly one manager to extract it, got {hits}") + +if errors: + print("error: Renovate shell annotation manager:\n " + "\n ".join(errors), file=sys.stderr) + sys.exit(1) +print("Renovate shell annotation manager: OK") +PY