Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/core-protocol.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
test "$(docker exec "$POSTGRES_CONTAINER" psql -X -U postgres -d postgres -v ON_ERROR_STOP=1 \
-Atqc 'SELECT pg_reload_conf()')" = t
# pg_reload_conf sends SIGHUP; read the actual setting until applied.
for attempt in {1..50}; do
for _ in {1..50}; do
docker exec "$POSTGRES_CONTAINER" psql -X -U postgres -d postgres -v ON_ERROR_STOP=1 \
-Atqc "SELECT setting, pending_restart FROM pg_settings WHERE name = 'autovacuum'" \
> "$RUNNER_TEMP/core-postgres-offline-settings.txt"
Expand Down
38 changes: 32 additions & 6 deletions .github/workflows/linux-packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ jobs:
git rev-parse HEAD | tee out/evidence/revision.txt
nfpm --version | tee out/evidence/nfpm-version.txt
node --test scripts/workflow-topology.test.mjs
python3 -B packaging/linux/test_extore_receipts.py
python3 -B packaging/linux/test_native.py -v 2>&1 | tee out/evidence/native-tests.log
go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 -color .github/workflows/linux-packages.yml
- name: Build the three product binaries for this architecture
Expand All @@ -99,6 +100,7 @@ jobs:
docker buildx build --platform "linux/$ARCH" -f deployment/docker/extensions.Dockerfile \
--target binaries-musl --output type=local,dest=out/extensions-musl .
- name: Build verified source archives and native packages
id: packages
run: |
revision=$(git rev-parse HEAD)
SOURCE_DATE_EPOCH=$(git show -s --format=%ct HEAD)
Expand All @@ -109,6 +111,7 @@ jobs:
done
python3 scripts/build-distribution.py build --component extensions \
--version 0.1.0-preview --revision "$revision" --platform "linux-musl-$ARCH" --input out/extensions-musl
package_paths=()
for component in core extensions cli; do
package="lmm-$component"
if [[ "$component" == cli ]]; then package=lmm; fi
Expand All @@ -118,12 +121,19 @@ jobs:
archive="out/distributions/${package}-0.1.0-preview-${platform}.tar.gz"
digest=$(sha256sum "$archive" | cut -d ' ' -f 1)
for release in 1 2; do
python3 packaging/linux/build.py --archive "$archive" --sha256 "$digest" \
--revision "$revision" --component "$component" --format "$format" --release "$release"
built=$(python3 packaging/linux/build.py --archive "$archive" --sha256 "$digest" \
--revision "$revision" --component "$component" --format "$format" --release "$release")
package_paths+=("$built" "$built.sha256" "$built.buildinfo.json")
done
done
done
(cd out/linux-packages && sha256sum --check ./*.sha256)
for path in "${package_paths[@]}"; do test -s "$path"; done
{
echo 'paths<<END_PACKAGES'
printf '%s\n' "${package_paths[@]}"
echo 'END_PACKAGES'
} >> "$GITHUB_OUTPUT"
- name: Install, upgrade, run and remove in disposable distribution containers
run: |
set -o pipefail
Expand Down Expand Up @@ -159,23 +169,39 @@ jobs:
-f packages/extore/tests/native_plugin_alpine.Dockerfile --target fixture \
--build-arg "EXTORE_RUST_TARGET=$RUST_TARGET" -t lmm-extore-alpine-fixture packages/extore
docker image inspect lmm-extore-alpine-fixture --format '{{.Id}}' > out/evidence/extore-fixture-image.txt
umask 077
staging=$(mktemp -d "$RUNNER_TEMP/extore-receipts.XXXXXXXX")
report=extore-native-alpine.json
touch "$staging/$report" "$staging/$report.package.json"
timeout 300 docker run --rm --network none --add-host localhost.localdomain:127.0.0.1 \
--cap-add NET_ADMIN --cpus 1 --memory 768m --pids-limit 256 \
--security-opt no-new-privileges:true -e LMM_PACKAGING_ISOLATED=1 \
-v "$PWD/out/linux-packages:/packages:ro" -v "$PWD/out/native-probe:/probe:ro" \
-v "$PWD/out/evidence:/evidence" \
-v "$staging/$report:/receipts/$report" \
-v "$staging/$report.package.json:/receipts/$report.package.json" \
-v "$PWD/packaging/linux/extore-plugin-smoke.py:/package-probe.py:ro" \
--entrypoint python3 lmm-extore-alpine-fixture /package-probe.py \
--arch "$ARCH" --revision "$(git rev-parse HEAD)" \
--parent-netns "$(readlink /proc/self/ns/net)" \
--report /evidence/extore-native-alpine.json 2>&1 | tee out/evidence/extore-native-alpine.log
--report "/receipts/$report" > "$staging/fixture.log" 2>&1
for receipt in "$staging/$report" "$staging/$report.package.json"; do
test -s "$receipt"
test -O "$receipt"
done
mv -- "$staging/$report" "$staging/$report.package.json" out/evidence/
- name: Preserve preview packages and scoped evidence, never publish
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
with:
name: native-linux-${{ matrix.arch }}-${{ github.sha }}
path: |
out/linux-packages/
out/evidence/
${{ steps.packages.outputs.paths }}
out/evidence/revision.txt
out/evidence/nfpm-version.txt
out/evidence/install-matrix.txt
out/evidence/image-digests.txt
out/evidence/extore-fixture-image.txt
out/evidence/extore-native-alpine.json
out/evidence/extore-native-alpine.json.package.json
if-no-files-found: error
retention-days: 7
55 changes: 46 additions & 9 deletions packaging/linux/extore-plugin-smoke.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,34 @@
import urllib.request


def probe_receipt(report, host_sha256, plugin_sha256):
"""Export only reviewed acceptance facts from the private probe report."""
def confirmed(actual, expected):
for key, value in expected.items():
if type(actual.get(key)) is not type(value) or actual[key] != value:
raise ValueError(f"unconfirmed native acceptance field: {key}")
return {key: actual[key] for key in expected}

receipt = confirmed(report, {
"status": "passed", "consumer_exit_code": 0,
"host_library_sha256": host_sha256, "plugin_library_sha256": plugin_sha256,
"actual_rust_health": True, "isolated_network_namespace": True,
"tls_certificate_verification": True, "authorized_catalog": True,
"cross_shop_http_detail_denied": True, "revoked_token_denied": True,
"catalog_calls_did_not_issue_cards": True, "actual_sqlite_grant_usage_recorded": True,
})
receipt["consumer"] = confirmed(report["consumer"], {
"status": "passed", "host_abi": 1, "interface": "extore.commerce",
"real_extore_https_sqlite": True, "native_digest_path_binding": True,
"native_repeated_open_call_release_close": True, "store_sync_persisted": True,
"http_catalog_fallback_calls": 0, "secret_values_logged": False,
"production_accounts_created": False,
"store_repository": "transactional memory test repository",
"core_authority": "existing Store test double; independent CoreSession acceptance not run",
})
return receipt


def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--arch", choices=("amd64", "arm64"), required=True)
Expand Down Expand Up @@ -104,7 +132,12 @@ def main():
time.sleep(0.1)
if not ready:
raise RuntimeError("installed extensions did not start without native configuration")
if os.readlink(f"/proc/{process.pid}/exe") != "/usr/bin/lmm-extensions":
# Procfs checks credentials; container root cannot inspect another UID.
identity = subprocess.run(
["readlink", "-n", f"/proc/{process.pid}/exe"],
check=True, stdout=subprocess.PIPE, timeout=5, user=account.pw_uid,
group=account.pw_gid, extra_groups=[])
if identity.stdout != b"/usr/bin/lmm-extensions":
raise RuntimeError("default process is not the installed program")
finally:
process.terminate()
Expand All @@ -127,15 +160,19 @@ def main():
"service_uid": account.pw_uid, "service_gid": account.pw_gid,
"default_process_stopped": True,
}
with tempfile.TemporaryDirectory(prefix="extore-private-probe-") as directory:
private_report = Path(directory) / "probe.json"
command = [sys.executable, "/work/tests/native_plugin_probe.py",
"--inside-netns", "--parent-netns", args.parent_netns,
"--binary", "/opt/extore/extore-server",
"--host-library", host, "--plugin-library", plugin,
"--report", str(private_report), "--consumer", str(args.consumer),
"-test.run", "^TestNativePluginCatalogStore$", "-test.v"]
subprocess.run(command, check=True)
accepted = probe_receipt(json.loads(private_report.read_text()),
info["files"][host]["sha256"], info["files"][plugin]["sha256"])
args.report.write_text(json.dumps(accepted, indent=2) + "\n")
Path(str(args.report) + ".package.json").write_text(json.dumps(receipt, indent=2) + "\n")
command = [sys.executable, "/work/tests/native_plugin_probe.py",
"--inside-netns", "--parent-netns", args.parent_netns,
"--binary", "/opt/extore/extore-server",
"--host-library", "/usr/lib/lmm/extensions/extore/libextore_plugin_host.so",
"--plugin-library", "/usr/lib/lmm/extensions/extore/libextore_commerce_plugin.so",
"--report", str(args.report), "--consumer", str(args.consumer),
"-test.run", "^TestNativePluginCatalogStore$", "-test.v"]
subprocess.run(command, check=True)


if __name__ == "__main__":
Expand Down
47 changes: 47 additions & 0 deletions packaging/linux/test_extore_receipts.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
"""Check the upload boundary with synthetic private probe output."""
import runpy
import unittest
from pathlib import Path


probe_receipt = runpy.run_path(str(Path(__file__).with_name("extore-plugin-smoke.py")))["probe_receipt"]


class ProbeReceiptTests(unittest.TestCase):
def setUp(self):
self.host, self.plugin = "a" * 64, "b" * 64
self.consumer = {
"status": "passed", "host_abi": 1, "interface": "extore.commerce",
"real_extore_https_sqlite": True, "native_digest_path_binding": True,
"native_repeated_open_call_release_close": True, "store_sync_persisted": True,
"http_catalog_fallback_calls": 0, "secret_values_logged": False,
"production_accounts_created": False,
"store_repository": "transactional memory test repository",
"core_authority": "existing Store test double; independent CoreSession acceptance not run",
}
self.report = {
"status": "passed", "consumer_exit_code": 0, "consumer": self.consumer,
"host_library_sha256": self.host, "plugin_library_sha256": self.plugin,
"actual_rust_health": True, "isolated_network_namespace": True,
"tls_certificate_verification": True, "authorized_catalog": True,
"cross_shop_http_detail_denied": True, "revoked_token_denied": True,
"catalog_calls_did_not_issue_cards": True, "actual_sqlite_grant_usage_recorded": True,
}

def test_only_reviewed_fields_cross_the_boundary(self):
private = dict(self.report, stdout="private fixture output",
consumer=dict(self.consumer, credentials={"test": "private"}))
self.assertEqual(probe_receipt(private, self.host, self.plugin), self.report)

def test_incomplete_failed_or_mistyped_facts_cannot_be_exported(self):
missing = self.report.copy()
del missing["consumer_exit_code"]
for raw in [missing, dict(self.report, status="failed"), dict(self.report, consumer_exit_code=False),
dict(self.report, host_library_sha256=self.plugin),
dict(self.report, consumer=dict(self.consumer, core_authority="unreviewed text"))]:
with self.subTest(raw=raw), self.assertRaises(ValueError):
probe_receipt(raw, self.host, self.plugin)


if __name__ == "__main__":
unittest.main()
2 changes: 1 addition & 1 deletion scripts/workflow-topology.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ test('workflows initialize only their required pinned submodules', () => {
lmm: ['apps/cli'],
'microservice-distribution': ['apps/cli', 'apps/cli', 'packages/extore'],
'linux-packages': ['apps/cli', 'packages/extore'],
'core-protocol': ['apps/cli'],
'core-protocol': ['apps/cli', 'packages/extore'],
deploy: ['packages/extore'],
};
for (const [name, paths] of Object.entries(dependencies)) {
Expand Down