diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 58d8b9e0..7c8d2969 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -112,6 +112,11 @@ jobs: java: true cluster: true helm: false + - name: "Tier 19: Dependency CVE remediation" + tiers: --tier 19 + java: true + cluster: true + helm: false steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -136,6 +141,12 @@ jobs: PYTHONDONTWRITEBYTECODE: "1" run: python3 -m unittest discover -s integration-tests/e2e -p 'nodeprofile_fixtures_test.py' -v + - name: Test CVE sweep logic + if: contains(matrix.tiers, '--tier 19') + env: + PYTHONDONTWRITEBYTECODE: "1" + run: python3 -m unittest discover -s integration-tests/e2e -p 'cve_sweep_test.py' -v + - name: Set up Helm if: matrix.helm uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 diff --git a/docs/managed-dependency-bundles.md b/docs/managed-dependency-bundles.md index b188ec38..e6b315d0 100644 --- a/docs/managed-dependency-bundles.md +++ b/docs/managed-dependency-bundles.md @@ -377,6 +377,35 @@ layout and cache behavior. --- +## Responding to a dependency CVE + +When a bundled library (for example `log4j-core`) receives a critical CVE: + +1. **Identify affected workloads from what is running.** Brewlet does not track + bundles on `JavaApplication` or on pods. Instead, map each running pod's + digest-pinned image to its managed-dependency evidence with + `brewlet inspect`. That evidence names the bundle and SBOM digests. Read the + components and purls from the bundle's CycloneDX SBOM referrer. Treat any + image you cannot trace to an SBOM as unverified, not clean. +2. **Publish a patched bundle** from the updated BOM under a new version or tag. + Bundles are immutable, and publishing one does not change running + applications. +3. **Recompose and roll out.** Run `brewlet push` again with the *same* thin + application JAR, `--dependency-bundle` pointing at the patched bundle, and + its lock. No application rebuild is required. The application layer and + `applicationJarDigest` stay the same, and only the managed classpath layer + changes. Then roll each workload to the new image digest with a + zero-downtime rolling update (for example, `maxUnavailable: 0`). Brewlet does + not swap classpath layers inside a running JVM, so a new pod on the new digest + is the unit of remediation. +4. **Verify** by repeating step 1 against the running pods. + +E2E tier 19 (`integration-tests/e2e/tier19-cve-remediation.sh`) runs this +procedure end to end. It uses `integration-tests/e2e/cve_sweep.py` as the +reference SBOM sweep. + +--- + ## Failure behavior Managed publication fails closed. It does not silently rebuild dependency diff --git a/integration-tests/AGENTS.md b/integration-tests/AGENTS.md index d3da3017..9dcf3388 100644 --- a/integration-tests/AGENTS.md +++ b/integration-tests/AGENTS.md @@ -40,15 +40,15 @@ The harness does not switch branches or modify component sources. It uses | Tool | Tiers | |---|---| | Go | all | -| Python 3 | 2, 4, 12, 13, 16, 17 | -| JDK 21+ | 2, 3, 7, 8, 9, 12, 14-18 | -| Docker | 3, 6, 7, 8-12, 14-18 | -| kubectl and a reachable cluster | 4-18 | +| Python 3 | 2, 4, 12, 13, 16, 17, 19 | +| JDK 21+ | 2, 3, 7, 8, 9, 12, 14-19 | +| Docker | 3, 6, 7, 8-12, 14-19 | +| kubectl and a reachable cluster | 4-19 | | Helm | 4 (optional), 10, 15, 17 | | OpenSSL | 4, 5, 6, 10, 11 | Host-only tiers 1-3 need no cluster. Tiers 4-7 and 13 exercise API-server -behavior. Tiers 6, 8-12, and 14-18 require local containerd nodes that +behavior. Tiers 6, 8-12, and 14-19 require local containerd nodes that Docker can enter, such as kind. Managed clusters skip those node-side paths. Tier 13 also proves the control-plane guard: a NodeProfile that does not set `nodePool.includeControlPlane` never counts or schedules onto a control-plane @@ -87,6 +87,18 @@ checks that the running pod keeps its original JDK and container, and that a retired-root sweep keeps the root that pod still uses. Finally, it asserts that `kubectl rollout restart` moves the same application image onto the patched JDK, after which an idle sweep reclaims the retired root. +Tier 19 rehearses a critical CVE in a library shipped by a managed dependency +bundle. It publishes a bundle with a stub `log4j-core` 2.14.1, composes three +thin-JAR apps (two on that bundle, one control on an unrelated bundle), and +deploys them to a prod and a staging namespace. `e2e/cve_sweep.py` then traces +every running Brewlet pod from its digest-pinned image through `brewlet inspect` +evidence to the bundle's CycloneDX SBOM. It flags exactly the affected prod +workloads and fails closed on images it cannot trace. The tier publishes a +patched bundle (2.17.1) and recomposes the *unchanged* thin JARs onto it, +asserting that only the classpath layer changed. It rolls the Deployments by +digest (`maxUnavailable: 0`) while a client polls the Service, and asserts that +no request was dropped and that the post-remediation prod sweep is clean. There +is no in-place JVM swap: a "hot" redeploy is a recompose plus a rolling update. **Architecture coverage.** Tiers pick up the node's architecture automatically, but hosted runners are amd64, so `.github/workflows/e2e.yml` runs the host-only @@ -111,6 +123,13 @@ PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover \ -s integration-tests/e2e -p 'nodeprofile_fixtures_test.py' -v ``` +Run tier 19's SBOM sweep logic offline (fake CLI, no cluster) with: + +```bash +PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover \ + -s integration-tests/e2e -p 'cve_sweep_test.py' -v +``` + Run `./e2e/run.sh --reset` before repeating Kubernetes tiers. It removes only Brewlet-owned labels, annotations, CRDs, runtime classes, webhook configuration, RBAC, and fixed test namespaces. It does not delete application workloads outside @@ -124,7 +143,7 @@ Common environment-specific skips: - Tier 5 skips when a cluster cannot reach a host-bound webhook; tier 6 covers the same assertions in-cluster. -- Tiers 8, 9, 12, and 14-18 skip if no schedulable local containerd node +- Tiers 8, 9, 12, and 14-19 skip if no schedulable local containerd node can be provisioned. - Tier 12 skips when the node's `ctr` supports neither `images unpack` nor import-time unpack; tier 16 applies the same rule. diff --git a/integration-tests/e2e/cve_sweep.py b/integration-tests/e2e/cve_sweep.py new file mode 100644 index 00000000..459babde --- /dev/null +++ b/integration-tests/e2e/cve_sweep.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""SBOM-driven CVE sweep over running Brewlet workloads (tier 18). + +``sweep`` answers "which Brewlet applications running in the cluster ship a +vulnerable third-party library?" without trusting image tags or application +metadata: + + running Pod (runtimeClassName=brewlet) + -> digest-pinned image + -> ``brewlet inspect`` managed-dependency evidence (bundle + SBOM digests) + -> the bundle's single CycloneDX SBOM referrer, digest-verified + -> components matching the advisory's Maven coordinates and version range + +``compare`` reports which layers two runnable images share, proving a +remediation recomposition reused the application layer and replaced only the +managed classpath layer. + +Both read a local OCI layout (the tier's registry stand-in). The logic is the +same against a registry: resolve the digest, read the evidence annotation, and +fetch the bundle's CycloneDX referrer. +""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys + +REF_NAME = "org.opencontainers.image.ref.name" +REFERRER_SUBJECT = "brewlet.sh/referrer-subject" +CYCLONEDX = "application/vnd.cyclonedx+json" +EVIDENCE_HEADER = "== managed dependency evidence (unsigned) ==" +TERMINAL_PHASES = {"Succeeded", "Failed"} + + +def version_key(version): + """Order Maven-style versions: 2.0-beta9 < 2.0 < 2.14.1 < 2.17.1.""" + release, _, qualifier = version.partition("-") + numbers = [] + for part in release.split("."): + match = re.match(r"\d+", part) + numbers.append(int(match.group()) if match else 0) + numbers += [0] * (4 - len(numbers)) + if not qualifier: + return tuple(numbers), (1,) + match = re.match(r"([A-Za-z]*)(\d*)", qualifier) + label, number = match.group(1).lower(), int(match.group(2) or 0) + return tuple(numbers), (0, label, number) + + +def is_affected(version, introduced, fixed): + key = version_key(version) + return version_key(introduced) <= key < version_key(fixed) + + +def image_digest(image): + if "@sha256:" in image: + return "sha256:" + image.rsplit("@sha256:", 1)[1] + if image.startswith("sha256:"): + return image + return "" + + +def workload_of(pod): + meta = pod.get("metadata", {}) + owners = meta.get("ownerReferences") or [] + if not owners: + return "Pod", meta.get("name", "") + owner = owners[0] + template_hash = meta.get("labels", {}).get("pod-template-hash", "") + if owner.get("kind") == "ReplicaSet" and template_hash and \ + owner.get("name", "").endswith("-" + template_hash): + return "Deployment", owner["name"][: -len(template_hash) - 1] + return owner.get("kind", ""), owner.get("name", "") + + +class Layout: + def __init__(self, root): + self.root = root + with open(os.path.join(root, "index.json"), encoding="utf-8") as stream: + self.index = json.load(stream).get("manifests", []) + + def blob(self, digest): + algorithm, _, value = digest.partition(":") + with open(os.path.join(self.root, "blobs", algorithm, value), "rb") as stream: + raw = stream.read() + if "sha256:" + hashlib.sha256(raw).hexdigest() != digest: + raise ValueError(f"blob {digest} failed digest verification") + return raw + + def ref_for_digest(self, digest): + for desc in self.index: + ref = desc.get("annotations", {}).get(REF_NAME) + if desc.get("digest") == digest and ref: + return ref + return "" + + def descriptor_for_ref(self, ref): + for desc in self.index: + if desc.get("annotations", {}).get(REF_NAME) == ref: + return desc + raise KeyError(f"ref {ref!r} not found in {self.root}") + + def platform_manifest(self, ref, arch): + desc = self.descriptor_for_ref(ref) + document = json.loads(self.blob(desc["digest"])) + if "manifests" in document: + for entry in document["manifests"]: + platform = entry.get("platform", {}) + if platform.get("os") == "linux" and platform.get("architecture") == arch: + return json.loads(self.blob(entry["digest"])) + raise KeyError(f"{ref} has no linux/{arch} manifest") + return document + + def bundle_sbom(self, bundle_digest, sbom_digest): + referrers = [ + desc for desc in self.index + if desc.get("artifactType") == CYCLONEDX + and desc.get("annotations", {}).get(REFERRER_SUBJECT) == bundle_digest + ] + if len(referrers) != 1: + raise ValueError( + f"bundle {bundle_digest} has {len(referrers)} CycloneDX referrers, want exactly 1") + manifest = json.loads(self.blob(referrers[0]["digest"])) + if (manifest.get("subject") or {}).get("digest") != bundle_digest: + raise ValueError(f"SBOM referrer subject does not bind bundle {bundle_digest}") + layers = manifest.get("layers") or [] + if len(layers) != 1 or layers[0].get("digest") != sbom_digest: + raise ValueError(f"SBOM referrer does not carry evidence SBOM {sbom_digest}") + return json.loads(self.blob(sbom_digest)) + + +def inspect_evidence(brewlet, store, ref): + out = subprocess.run( + [brewlet, "inspect", ref, "--store", store], + check=True, capture_output=True, text=True).stdout + if EVIDENCE_HEADER not in out: + return None + return json.JSONDecoder().raw_decode(out.split(EVIDENCE_HEADER, 1)[1].lstrip())[0] + + +def scan_image(layout, brewlet, store, digest, group, artifact, introduced, fixed): + ref = layout.ref_for_digest(digest) + if not ref: + raise LookupError(f"image {digest} is not in the artifact store") + evidence = inspect_evidence(brewlet, store, ref) + if evidence is None: + raise LookupError(f"image {ref} carries no managed dependency evidence") + sbom = layout.bundle_sbom(evidence["dependencyBundleDigest"], evidence["sbomDigest"]) + findings, components = [], [] + for component in sbom.get("components", []): + if component.get("group") != group or component.get("name") != artifact: + continue + version = component.get("version", "") + components.append(version) + if is_affected(version, introduced, fixed): + findings.append({"purl": component.get("purl", ""), "version": version}) + return { + "ref": ref, + "digest": digest, + "applicationJarDigest": evidence["applicationJarDigest"], + "dependencyBundleDigest": evidence["dependencyBundleDigest"], + "sbomDigest": evidence["sbomDigest"], + "sourceBom": evidence.get("sourceBom", ""), + "componentVersions": components, + "findings": findings, + } + + +def sweep(args): + group, _, artifact = args.component.partition(":") + with open(args.pods, encoding="utf-8") as stream: + pods = json.load(stream).get("items", []) + layout = Layout(args.store) + cache, workloads, unscannable, brewlet_pods = {}, {}, [], 0 + for pod in pods: + meta, spec = pod.get("metadata", {}), pod.get("spec", {}) + if spec.get("runtimeClassName") != "brewlet": + continue + if pod.get("status", {}).get("phase") in TERMINAL_PHASES: + continue + namespace = meta.get("namespace", "") + if args.namespace and namespace not in args.namespace: + continue + brewlet_pods += 1 + kind, name = workload_of(pod) + entry = workloads.setdefault((namespace, kind, name), { + "namespace": namespace, "kind": kind, "name": name, + "pods": [], "images": {}, "vulnerable": False, + }) + entry["pods"].append(meta.get("name", "")) + statuses = {s.get("name"): s for s in pod.get("status", {}).get("containerStatuses") or []} + for container in spec.get("containers", []): + image = container.get("image", "") + digest = image_digest(image) or image_digest( + statuses.get(container.get("name"), {}).get("imageID", "")) + if not digest: + unscannable.append({"namespace": namespace, "pod": meta.get("name", ""), + "image": image, "reason": "image is not digest-pinned"}) + continue + if digest not in cache: + try: + cache[digest] = scan_image(layout, args.brewlet, args.store, digest, + group, artifact, args.introduced, args.fixed) + except (LookupError, ValueError, OSError, subprocess.CalledProcessError) as error: + cache[digest] = {"error": str(error)} + result = cache[digest] + if "error" in result: + unscannable.append({"namespace": namespace, "pod": meta.get("name", ""), + "image": image, "reason": result["error"]}) + continue + entry["images"][digest] = result + entry["vulnerable"] = entry["vulnerable"] or bool(result["findings"]) + report = { + "advisory": {"component": args.component, "introduced": args.introduced, + "fixed": args.fixed}, + "brewletPods": brewlet_pods, + "workloads": [], + "unscannable": unscannable, + } + for key in sorted(workloads): + entry = workloads[key] + entry["pods"].sort() + entry["images"] = [entry["images"][d] for d in sorted(entry["images"])] + report["workloads"].append(entry) + report["vulnerable"] = sorted( + f"{w['namespace']}/{w['name']}" for w in report["workloads"] if w["vulnerable"]) + json.dump(report, sys.stdout, indent=2) + print() + if args.fail_on_findings and (report["vulnerable"] or unscannable): + return 3 + return 0 + + +def compare(args): + layout = Layout(args.store) + layers = [] + for ref in (args.ref_a, args.ref_b): + manifest = layout.platform_manifest(ref, args.arch) + layers.append({ + layer["digest"]: layer.get("annotations", {}).get("brewlet.sh/layer", "") + for layer in manifest.get("layers", []) + }) + a, b = layers + json.dump({ + "shared": sorted(f"{a[d]}={d}" for d in a if d in b), + "onlyA": sorted(f"{a[d]}={d}" for d in a if d not in b), + "onlyB": sorted(f"{b[d]}={d}" for d in b if d not in a), + }, sys.stdout, indent=2) + print() + return 0 + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + commands = parser.add_subparsers(dest="command", required=True) + s = commands.add_parser("sweep", help="find running Brewlet workloads affected by an advisory") + s.add_argument("--store", required=True, help="OCI layout holding the deployed images") + s.add_argument("--brewlet", required=True, help="brewlet CLI binary") + s.add_argument("--pods", required=True, help="output of `kubectl get pods -A -o json`") + s.add_argument("--component", required=True, help="Maven groupId:artifactId") + s.add_argument("--introduced", required=True, help="first affected version (inclusive)") + s.add_argument("--fixed", required=True, help="first fixed version (exclusive bound)") + s.add_argument("--namespace", action="append", default=[], + help="limit the sweep to this namespace (repeatable)") + s.add_argument("--fail-on-findings", action="store_true", + help="exit 3 when a workload is affected or cannot be scanned") + s.set_defaults(func=sweep) + c = commands.add_parser("compare", help="diff the layers of two runnable images") + c.add_argument("--store", required=True) + c.add_argument("--arch", required=True) + c.add_argument("ref_a") + c.add_argument("ref_b") + c.set_defaults(func=compare) + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/integration-tests/e2e/cve_sweep_test.py b/integration-tests/e2e/cve_sweep_test.py new file mode 100644 index 00000000..5bece082 --- /dev/null +++ b/integration-tests/e2e/cve_sweep_test.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Offline checks for cve_sweep.py; no cluster or brewlet build required.""" + +import contextlib +import hashlib +import io +import json +import os +import stat +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import cve_sweep # noqa: E402 + +GROUP, ARTIFACT = "org.apache.logging.log4j", "log4j-core" + + +class Store: + def __init__(self, root): + self.root = root + self.index = [] + os.makedirs(os.path.join(root, "blobs", "sha256")) + + def blob(self, document): + raw = document if isinstance(document, bytes) else json.dumps(document).encode() + digest = "sha256:" + hashlib.sha256(raw).hexdigest() + with open(os.path.join(self.root, "blobs", "sha256", digest[7:]), "wb") as stream: + stream.write(raw) + return digest + + def bundle(self, name, version): + bundle = self.blob({"bundle": name, "version": version}) + sbom = self.blob({"bomFormat": "CycloneDX", "components": [{ + "group": GROUP, "name": ARTIFACT, "version": version, + "purl": f"pkg:maven/{GROUP}/{ARTIFACT}@{version}"}]}) + referrer = self.blob({"subject": {"digest": bundle}, "layers": [{"digest": sbom}]}) + self.index.append({"digest": bundle, "annotations": {cve_sweep.REF_NAME: name}}) + self.index.append({"digest": referrer, "artifactType": cve_sweep.CYCLONEDX, + "annotations": {cve_sweep.REFERRER_SUBJECT: bundle}}) + return bundle, sbom + + def image(self, ref, bundle, sbom): + digest = self.blob({"image": ref}) + self.index.append({"digest": digest, "annotations": {cve_sweep.REF_NAME: ref}}) + return digest, {"schemaVersion": 1, "thinJar": True, + "applicationJarDigest": "sha256:app", + "dependencyBundleDigest": bundle, "sbomDigest": sbom, + "sourceBom": "com.example:bom:1"} + + def save(self): + with open(os.path.join(self.root, "index.json"), "w", encoding="utf-8") as stream: + json.dump({"manifests": self.index}, stream) + + +def pod(name, namespace, image, owner=None, runtime="brewlet", phase="Running"): + metadata = {"name": name, "namespace": namespace} + if owner: + metadata["labels"] = {"pod-template-hash": "abc12"} + metadata["ownerReferences"] = [{"kind": "ReplicaSet", "name": f"{owner}-abc12"}] + return {"metadata": metadata, + "spec": {"runtimeClassName": runtime, + "containers": [{"name": "app", "image": image}]}, + "status": {"phase": phase}} + + +class VersionTest(unittest.TestCase): + def test_log4shell_range(self): + affected = lambda v: cve_sweep.is_affected(v, "2.0-beta9", "2.17.1") + for version in ("2.0-beta9", "2.0", "2.14.1", "2.15.0", "2.17.0"): + self.assertTrue(affected(version), version) + for version in ("1.2.17", "2.0-beta8", "2.17.1", "2.24.3"): + self.assertFalse(affected(version), version) + + +class SweepTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + root = self.tmp.name + self.store = Store(os.path.join(root, "oci")) + bad_bundle, bad_sbom = self.store.bundle("platform/logging:1", "2.14.1") + good_bundle, good_sbom = self.store.bundle("platform/logging:2", "2.17.1") + self.vulnerable, bad_evidence = self.store.image("apps/orders:1", bad_bundle, bad_sbom) + self.patched, good_evidence = self.store.image("apps/orders:2", good_bundle, good_sbom) + self.plain = self.store.image("apps/plain:1", "", "")[0] + self.store.save() + evidence = {"apps/orders:1": bad_evidence, "apps/orders:2": good_evidence} + self.brewlet = os.path.join(root, "brewlet") + with open(self.brewlet, "w", encoding="utf-8") as stream: + stream.write("#!/usr/bin/env python3\nimport json, sys\n" + f"evidence = {json.dumps(evidence)!r}\n" + "print('== kind ==\\nrunnable OCI image\\n')\n" + "ref = sys.argv[2]\n" + "if ref in json.loads(evidence):\n" + f" print({cve_sweep.EVIDENCE_HEADER!r})\n" + " print(json.dumps(json.loads(evidence)[ref], indent=2))\n") + os.chmod(self.brewlet, os.stat(self.brewlet).st_mode | stat.S_IEXEC) + self.pods = os.path.join(root, "pods.json") + + def tearDown(self): + self.tmp.cleanup() + + def run_sweep(self, pods, *extra): + with open(self.pods, "w", encoding="utf-8") as stream: + json.dump({"items": pods}, stream) + out = io.StringIO() + with contextlib.redirect_stdout(out): + code = cve_sweep.main([ + "sweep", "--store", self.store.root, "--brewlet", self.brewlet, + "--pods", self.pods, "--component", f"{GROUP}:{ARTIFACT}", + "--introduced", "2.0-beta9", "--fixed", "2.17.1", *extra]) + return code, json.loads(out.getvalue()) + + def test_identifies_only_vulnerable_brewlet_workloads_in_scope(self): + code, report = self.run_sweep([ + pod("orders-abc12-x", "prod", f"reg/apps/orders@{self.vulnerable}", owner="orders"), + pod("orders-abc12-y", "prod", f"reg/apps/orders@{self.vulnerable}", owner="orders"), + pod("billing-abc12-x", "prod", f"reg/apps/billing@{self.patched}", owner="billing"), + pod("orders-abc12-z", "staging", f"reg/apps/orders@{self.vulnerable}", owner="orders"), + pod("web", "prod", f"reg/apps/orders@{self.vulnerable}", runtime="runc"), + pod("done", "prod", f"reg/apps/orders@{self.vulnerable}", phase="Succeeded"), + ], "--namespace", "prod", "--fail-on-findings") + self.assertEqual(code, 3) + self.assertEqual(report["vulnerable"], ["prod/orders"]) + self.assertEqual(report["brewletPods"], 3) + orders = next(w for w in report["workloads"] if w["name"] == "orders") + self.assertEqual(orders["kind"], "Deployment") + self.assertEqual(orders["pods"], ["orders-abc12-x", "orders-abc12-y"]) + self.assertEqual(orders["images"][0]["findings"][0]["version"], "2.14.1") + self.assertEqual(report["unscannable"], []) + + def test_clean_fleet_passes(self): + code, report = self.run_sweep( + [pod("orders-abc12-x", "prod", f"reg/apps/orders@{self.patched}", owner="orders")], + "--fail-on-findings") + self.assertEqual(code, 0) + self.assertEqual(report["vulnerable"], []) + self.assertEqual(report["workloads"][0]["images"][0]["componentVersions"], ["2.17.1"]) + + def test_unverifiable_images_fail_closed(self): + code, report = self.run_sweep([ + pod("tagged", "prod", "reg/apps/orders:1"), + pod("unknown", "prod", "reg/apps/x@sha256:" + "0" * 64), + pod("plain", "prod", f"reg/apps/plain@{self.plain}"), + ], "--fail-on-findings") + self.assertEqual(code, 3) + reasons = sorted(entry["reason"] for entry in report["unscannable"]) + self.assertIn("image is not digest-pinned", reasons) + self.assertTrue(any("not in the artifact store" in r for r in reasons)) + self.assertTrue(any("no managed dependency evidence" in r for r in reasons)) + + def test_conflicting_sbom_referrers_fail_closed(self): + bundle = self.store.index[0]["digest"] + extra = self.store.blob({"subject": {"digest": bundle}, "layers": [{"digest": "x"}]}) + self.store.index.append({"digest": extra, "artifactType": cve_sweep.CYCLONEDX, + "annotations": {cve_sweep.REFERRER_SUBJECT: bundle}}) + self.store.save() + _, report = self.run_sweep( + [pod("orders", "prod", f"reg/apps/orders@{self.vulnerable}")]) + self.assertIn("2 CycloneDX referrers", report["unscannable"][0]["reason"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/integration-tests/e2e/reset.sh b/integration-tests/e2e/reset.sh index feebe70b..bcd7f782 100755 --- a/integration-tests/e2e/reset.sh +++ b/integration-tests/e2e/reset.sh @@ -31,7 +31,7 @@ fi # Fixed namespaces the tiers use for their fixtures. Throwaway per-run # namespaces (tiers 5/6/10/11) are random and self-cleaned by their RETURN # traps; we only need to sweep the deterministic ones here. -E2E_FIXED_NS=(brewlet brewlet-e2e brewlet-e2e-app brewlet-custom-jdk brewlet-metrics-e2e brewlet-stagegc-e2e brewlet-jdk-patch) +E2E_FIXED_NS=(brewlet brewlet-e2e brewlet-e2e-app brewlet-custom-jdk brewlet-metrics-e2e brewlet-stagegc-e2e brewlet-jdk-patch brewlet-cve-prod brewlet-cve-staging) # scrub_node_labels: remove every brewlet.sh/* label and annotation from every # node. These are advertised by the provisioner (or simulated by tiers) and are diff --git a/integration-tests/e2e/run.sh b/integration-tests/e2e/run.sh index 78aec1c0..d03e3153 100755 --- a/integration-tests/e2e/run.sh +++ b/integration-tests/e2e/run.sh @@ -32,7 +32,7 @@ mkdir -p "$WORK" for arg in "$@"; do case "$arg" in --list) - printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster) 18 jdk-patch(in-cluster)\n' + printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster) 18 jdk-patch(in-cluster) 19 cve-remediation(in-cluster)\n' exit 0 ;; esac @@ -99,6 +99,7 @@ source "$E2E_DIR/tier15-metrics-incluster.sh" source "$E2E_DIR/tier16-failure-modes.sh" source "$E2E_DIR/tier17-stage-gc.sh" source "$E2E_DIR/tier18-jdk-patch.sh" +source "$E2E_DIR/tier19-cve-remediation.sh" declare -a TIERS=() DO_RESET=0 @@ -108,7 +109,7 @@ while [[ $# -gt 0 ]]; do case "$1" in --tier) TIERS+=("$2"); shift 2 ;; --reset) DO_RESET=1; shift ;; - --list) printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster) 18 jdk-patch(in-cluster)\n'; exit 0 ;; + --list) printf '1 unit 2 cli 3 runc 4 k8s 5 webhook(host) 6 webhook(in-cluster) 7 petclinic 8 appcds(in-cluster) 9 serving(in-cluster) 10 helm(in-cluster) 11 webhook-resilience 12 runnable-image(in-cluster) 13 nodeprofile 14 custom-jdk(in-cluster) 15 metrics(in-cluster) 16 failure-modes(in-cluster) 17 stage-gc(in-cluster) 18 jdk-patch(in-cluster) 19 cve-remediation(in-cluster)\n'; exit 0 ;; -h|--help) usage; exit 0 ;; *) warn "unknown arg: $1"; usage; exit 2 ;; esac @@ -121,7 +122,7 @@ if [[ "$DO_RESET" -eq 1 ]]; then [[ ${#TIERS[@]} -eq 0 ]] && exit 0 fi -[[ ${#TIERS[@]} -eq 0 ]] && TIERS=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18) +[[ ${#TIERS[@]} -eq 0 ]] && TIERS=(1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19) section "Brewlet E2E — environment" info "harness : $REPO_DIR" @@ -181,6 +182,7 @@ for t in "${TIERS[@]}"; do 16) tier16_failure_modes ;; 17) tier17_stage_gc ;; 18) tier18_jdk_patch ;; + 19) tier19_cve_remediation ;; *) warn "no such tier: $t" ;; esac done diff --git a/integration-tests/e2e/tier19-cve-remediation.sh b/integration-tests/e2e/tier19-cve-remediation.sh new file mode 100644 index 00000000..8c64dd89 --- /dev/null +++ b/integration-tests/e2e/tier19-cve-remediation.sh @@ -0,0 +1,385 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Tier 19 — CVE remediation of a third-party library shipped in a managed +# dependency bundle ("Log4Shell day"), end to end on a live node. +# +# SCENARIO. Platform publishes an approved bundle containing log4j-core 2.14.1. +# Several applications compose thin JARs onto it and run in "prod". A 10.0 CVE +# lands against log4j-core [2.0-beta9, 2.17.1). The platform team must: +# +# 1. IDENTIFY every running Brewlet workload in prod that ships the affected +# component — from what is actually running, not from tags or app repos: +# Pod -> digest-pinned image -> `brewlet inspect` managed-dependency +# evidence -> the bundle's single CycloneDX SBOM referrer -> components. +# (cve_sweep.py sweep). A workload whose image cannot be traced to an SBOM +# fails the sweep closed rather than being reported clean. +# 2. RELEASE a patched bundle (log4j-core 2.17.1) under a new version/tag. +# Bundles are immutable, and updating a bundle does NOT roll out +# applications built from the old one (docs/managed-dependency-bundles.md). +# 3. REDEPLOY without downtime. There is no in-place update of a running JVM +# (SPECIFICATION §11), so "hot redeploy" is: recompose the SAME thin +# application JAR onto the patched bundle (no application rebuild; the app +# layer is byte-identical and deduplicated, only the classpath layer +# changes), then roll the Deployment to the new digest with +# maxUnavailable=0 while a client polls the Service continuously. +# +# WHAT THIS TIER ASSERTS +# (a) the prod-scoped sweep flags exactly prod/orders and prod/billing; the +# control workload on an unrelated bundle and the staging copy of orders +# are excluded from the prod verdict; a fleet-wide sweep sees staging too. +# (b) the recomposed images keep the applicationJarDigest and the app layer, +# and replace only the managed classpath layer. +# (c) the rolling update drops zero requests, the running JVMs report the +# patched library, and no pod on the vulnerable digest remains. +# (d) the post-remediation prod sweep is clean while staging is still flagged. +# +# Prereqs: same as tier 12 (kubectl + reachable kind-style cluster, docker, go, +# python3, JDK 17+, network access to pull eclipse-temurin:21). SKIPs otherwise. + +T19_PROD="brewlet-cve-prod" +T19_STAGING="brewlet-cve-staging" +T19_PORT=8080 +T19_COMPONENT="org.apache.logging.log4j:log4j-core" +T19_INTRODUCED="2.0-beta9" +T19_FIXED="2.17.1" +T19_RC_CREATED="" +T19_NODE="" +declare -a T19_PROVISIONED_NODES=() + +_t19_cleanup() { + info "tier19: cleaning up" + kubectl delete ns "$T19_PROD" "$T19_STAGING" --ignore-not-found --wait=true --timeout=120s >/dev/null 2>&1 || true + [[ -n "$T19_RC_CREATED" ]] && kubectl delete runtimeclass brewlet --ignore-not-found >/dev/null 2>&1 || true + for n in ${T19_PROVISIONED_NODES[@]+"${T19_PROVISIONED_NODES[@]}"}; do + label_node "$n" brewlet.sh/runtime- "brewlet.sh/jdk.$T9_JDK-" "brewlet.sh/jdk-feature.${T9_JDK##*-}-" brewlet.sh/launcher.java- >/dev/null 2>&1 || true + annotate_node "$n" brewlet.sh/jdks- brewlet.sh/launchers- >/dev/null 2>&1 || true + done +} + +_t19_sha256() { + if have sha256sum; then sha256sum "$1" | awk '{print $1}'; else shasum -a 256 "$1" | awk '{print $1}'; fi +} + +# _t19_bundle JAR REF VERSION GROUP ARTIFACT LIBVERSION: publish an unsigned +# managed dependency bundle holding one JAR and write its canonical lock to +# $WORK/t19-bundles/.lock.json. +_t19_bundle() { + local jar="$1" ref="$2" version="$3" group="$4" artifact="$5" libversion="$6" + local dir="$WORK/t19-bundles/$version" name + name="$(basename "$jar")" + mkdir -p "$dir/lib" + cp "$jar" "$dir/lib/$name" + COPYFILE_DISABLE=1 tar -cf "$dir/deps.tar" -C "$dir/lib" "$name" + printf '{"schemaVersion":1,"artifacts":[{"groupId":"%s","artifactId":"%s","version":"%s","type":"jar","scope":"runtime","fileName":"%s","sha256":"%s"}]}\n' \ + "$group" "$artifact" "$libversion" "$name" "$(_t19_sha256 "$dir/lib/$name")" \ + >"$WORK/t19-bundles/$version.lock.json" + "$T19_BREWLET" dependency-bundle "$dir/deps.tar" "$ref" --store "$T19_STORE" \ + --name "${ref%%:*}" --version "$version" \ + --source-bom "com.example.platform:approved-bom:$version" \ + --lock "$WORK/t19-bundles/$version.lock.json" >>"$WORK/t19-build.log" 2>&1 +} + +# _t19_compose JAR REF BUNDLE_REF BUNDLE_VERSION: compose a thin JAR onto a bundle. +_t19_compose() { + "$T19_BREWLET" push "$1" "$2" --store "$T19_STORE" \ + --dependency-bundle "$3" --dependency-lock "$WORK/t19-bundles/$4.lock.json" \ + --main-class com.example.cveapp.CveApp >>"$WORK/t19-build.log" 2>&1 +} + +# _t19_pin REF: print the node's digest-pinned CRI reference for a store ref. +_t19_pin() { + local digest + digest="$(oci_layout_digest "$T19_STORE" "$1")" || return 1 + [[ -n "$digest" ]] || return 1 + pin_image_for_cri "$T19_NODE" "$1" "$digest" "$WORK/t19-import.log" +} + +# _t19_apply NS NAME IMAGE REPLICAS: a zero-downtime Deployment + Service. +_t19_apply() { + local ns="$1" name="$2" image="$3" replicas="$4" + kubectl apply -n "$ns" -f - >>"$WORK/t19-deploy.log" 2>&1 < 0 )); do + if body="$(kubectl exec -n "$T19_PROD" t19-client -- \ + wget -q -O- -T 5 "http://$name.$ns.svc.cluster.local:$T19_PORT/version" 2>/dev/null)" \ + && [[ -n "$body" ]]; then + printf '%s' "$body"; return 0 + fi + sleep 1 + done + return 1 +} + +# _t19_sweep OUT [--namespace NS]...: snapshot running pods and run the SBOM +# sweep. Prints the sweep's exit code; the JSON report is written to OUT. +_t19_sweep() { + local out="$1"; shift + local pods="$out.pods.json" code=0 + kubectl get pods -A -o json >"$pods" 2>>"$WORK/t19-deploy.log" || { echo 99; return 0; } + python3 "$E2E_DIR/cve_sweep.py" sweep --store "$T19_STORE" --brewlet "$T19_BREWLET" \ + --pods "$pods" --component "$T19_COMPONENT" --introduced "$T19_INTRODUCED" \ + --fixed "$T19_FIXED" --fail-on-findings "$@" >"$out" 2>>"$WORK/t19-deploy.log" || code=$? + echo "$code" +} + +# _t19_json FILE EXPR: evaluate a Python expression over the report bound to `r`. +_t19_json() { + python3 -c 'import json, sys; r = json.load(open(sys.argv[1])); print(eval(sys.argv[2]))' "$1" "$2" +} + +# _t19_wait_digest_gone NS DIGEST: wait until no pod in NS runs DIGEST, including +# terminating pods that may still be serving. +_t19_wait_digest_gone() { + local ns="$1" digest="$2" tries=90 + while (( tries-- > 0 )); do + kubectl get pods -n "$ns" -o jsonpath='{range .items[*]}{.spec.containers[0].image}{"\n"}{end}' \ + 2>/dev/null | grep -qF "$digest" || return 0 + sleep 1 + done + return 1 +} + +tier19_cve_remediation() { + section "Tier 19 — SBOM-driven CVE sweep and zero-downtime bundle remediation" + local name="tier19: CVE remediation" + if ! have kubectl || ! k8s_reachable; then skip "$name" "no reachable cluster"; return 0; fi + if ! have docker || ! docker info >/dev/null 2>&1; then skip "$name" "docker daemon not available"; return 0; fi + if ! have go; then skip "$name" "go not installed"; return 0; fi + if ! have python3; then skip "$name" "python3 not installed"; return 0; fi + if ! have java; then skip "$name" "no JDK to build fixtures (set JAVA_HOME, JDK 17+)"; return 0; fi + T19_NODE="$(pick_provisionable_node)" + if [[ -z "$T19_NODE" ]]; then skip "$name" "no node is a local containerd docker container (need kind/CI)"; return 0; fi + if ! node_schedulable "$T19_NODE"; then skip "$name" "only provisionable node ($T19_NODE) is unschedulable"; return 0; fi + local arch; arch="$(_t9_node_arch "$T19_NODE")" + if [[ -z "$arch" ]]; then skip "$name" "unknown node arch"; return 0; fi + info "tier19: node=$T19_NODE arch=$arch" + + trap _t19_cleanup RETURN + + # --- build shim, CLI and fixture -------------------------------------------- + local shimbin="$WORK/t19-shim-$arch" + T19_BREWLET="$WORK/t19-brewlet" + T19_STORE="$WORK/t19-oci" + rm -rf "$T19_STORE" "$WORK/t19-bundles" + if ! ( cd "$BREWLET_CORE_DIR" && GOOS=linux GOARCH="$arch" go build -o "$shimbin" ./shim/cmd/containerd-shim-brewlet-v2 \ + && go build -o "$T19_BREWLET" ./cmd/brewlet ) >"$WORK/t19-build.log" 2>&1; then + fail "tier19: build shim and brewlet CLI" "see $WORK/t19-build.log"; return 0 + fi + local jh; jh="$(resolve_java_home)" + local fx="$FIXTURES_DIR/log4j-cve-app/target" + if ! env JAVA_HOME="$jh" PATH="$jh/bin:$PATH" "$FIXTURES_DIR/log4j-cve-app/build.sh" \ + >>"$WORK/t19-build.log" 2>&1; then + fail "tier19: build CVE fixture" "see $WORK/t19-build.log"; return 0 + fi + # billing is a different application on the same bundle: distinct JAR bytes. + local billing_jar="$WORK/t19-billing.jar" + cp "$fx/app.jar" "$billing_jar" + printf 'billing\n' >"$WORK/t19-billing-marker.txt" + "$jh/bin/jar" --update --file "$billing_jar" -C "$WORK" t19-billing-marker.txt >>"$WORK/t19-build.log" 2>&1 + + # --- day 0: vulnerable bundle, control bundle, three prod apps --------------- + if ! _t19_bundle "$fx/log4j-core-2.14.1.jar" platform/java-deps/logging:2026.09 2026.09 \ + org.apache.logging.log4j log4j-core 2.14.1 \ + || ! _t19_bundle "$fx/safe-lib-1.0.jar" platform/java-deps/safe:1.0 1.0 \ + com.example.platform safe-lib 1.0 \ + || ! _t19_compose "$fx/app.jar" apps/orders:1.0.0 platform/java-deps/logging:2026.09 2026.09 \ + || ! _t19_compose "$billing_jar" apps/billing:3.2.0 platform/java-deps/logging:2026.09 2026.09 \ + || ! _t19_compose "$fx/app.jar" apps/inventory:2.0.0 platform/java-deps/safe:1.0 1.0; then + fail "tier19: publish day-0 bundles and applications" "see $WORK/t19-build.log"; return 0 + fi + pass "tier19: published log4j-core 2.14.1 bundle and composed orders, billing, inventory" + + # --- provision node and deploy ------------------------------------------------ + if ! import_oci_layout "$T19_NODE" "$T19_STORE" "$WORK/t19-import.log"; then + fail "tier19: import images into node content store" "see $WORK/t19-import.log"; return 0 + fi + local orders_v1 billing_v1 inventory_img + if ! orders_v1="$(_t19_pin apps/orders:1.0.0)" || ! billing_v1="$(_t19_pin apps/billing:3.2.0)" \ + || ! inventory_img="$(_t19_pin apps/inventory:2.0.0)"; then + fail "tier19: pin day-0 images for CRI" "see $WORK/t19-import.log"; return 0 + fi + docker cp "$shimbin" "$T19_NODE":"$T9_SHIM_DST" >>"$WORK/t19-prov.log" 2>&1 + docker exec "$T19_NODE" chmod +x "$T9_SHIM_DST" >>"$WORK/t19-prov.log" 2>&1 + docker exec "$T19_NODE" mkdir -p "$T9_CACHE" >>"$WORK/t19-prov.log" 2>&1 + if ! _t9_stage_jdk "$T19_NODE" "$arch"; then + skip "$name" "could not stage temurin JDK userland (see $WORK/t9-jdk.log)"; return 0 + fi + if ! _t9_patch_containerd "$T19_NODE" || ! _t9_advertise "$T19_NODE" >>"$WORK/t19-prov.log" 2>&1; then + fail "tier19: provision brewlet node" "see $WORK/t9-containerd.log and $WORK/t19-prov.log"; return 0 + fi + T19_PROVISIONED_NODES+=("$T19_NODE") + if ! kubectl get runtimeclass brewlet >/dev/null 2>&1; then + printf 'apiVersion: node.k8s.io/v1\nkind: RuntimeClass\nmetadata: { name: brewlet }\nhandler: brewlet\n' \ + | kubectl create -f - >/dev/null 2>&1 && T19_RC_CREATED=1 + fi + kubectl create namespace "$T19_PROD" >/dev/null 2>&1 || true + kubectl create namespace "$T19_STAGING" >/dev/null 2>&1 || true + kubectl run t19-client -n "$T19_PROD" --image=busybox:1.36 --restart=Never \ + --command -- sleep 3600 >>"$WORK/t19-deploy.log" 2>&1 || true + + _t19_apply "$T19_PROD" orders "$orders_v1" 2 + _t19_apply "$T19_PROD" billing "$billing_v1" 1 + _t19_apply "$T19_PROD" inventory "$inventory_img" 1 + _t19_apply "$T19_STAGING" orders "$orders_v1" 1 + local d + for d in "$T19_PROD/orders" "$T19_PROD/billing" "$T19_PROD/inventory" "$T19_STAGING/orders"; do + if ! kubectl rollout status -n "${d%/*}" deploy/"${d#*/}" --timeout=180s >>"$WORK/t19-deploy.log" 2>&1; then + fail "tier19: day-0 rollout of $d" "$(save_pod_diag t19-day0 "${d%/*}")"; return 0 + fi + done + if ! kubectl wait -n "$T19_PROD" --for=condition=Ready pod/t19-client --timeout=60s >>"$WORK/t19-deploy.log" 2>&1; then + fail "tier19: in-cluster client Ready" "see $WORK/t19-deploy.log"; return 0 + fi + assert_contains "tier19: prod/orders JVM loaded log4j-core 2.14.1 from its bundle" \ + "$(_t19_version "$T19_PROD" orders)" "log4j-core=2.14.1" + assert_contains "tier19: prod/inventory JVM has no log4j-core on its classpath" \ + "$(_t19_version "$T19_PROD" inventory)" "log4j-core=absent" + + # --- (a) identify affected prod workloads from their SBOMs -------------------- + local before="$WORK/t19-sweep-before.json" code + code="$(_t19_sweep "$before" --namespace "$T19_PROD")" + assert_eq "tier19: prod sweep fails with findings (exit 3)" "$code" "3" + assert_eq "tier19: prod sweep identifies exactly the affected workloads" \ + "$(_t19_json "$before" "','.join(r['vulnerable'])")" "$T19_PROD/billing,$T19_PROD/orders" + assert_eq "tier19: every running Brewlet pod in prod was traced to an SBOM" \ + "$(_t19_json "$before" "len(r['unscannable'])")" "0" + assert_eq "tier19: control workload on an unrelated bundle is scanned and clean" \ + "$(_t19_json "$before" "[w['vulnerable'] for w in r['workloads'] if w['name'] == 'inventory']")" "[False]" + assert_eq "tier19: finding cites the SBOM purl" \ + "$(_t19_json "$before" "[w for w in r['workloads'] if w['name'] == 'orders'][0]['images'][0]['findings'][0]['purl']")" \ + "pkg:maven/org.apache.logging.log4j/log4j-core@2.14.1" + local fleet="$WORK/t19-sweep-fleet.json" + _t19_sweep "$fleet" >/dev/null + assert_contains "tier19: fleet-wide sweep also sees the staging copy" \ + "$(_t19_json "$fleet" "r['vulnerable']")" "$T19_STAGING/orders" + + # --- release the patched bundle and recompose (no app rebuild) --------------- + if ! _t19_bundle "$fx/log4j-core-2.17.1.jar" platform/java-deps/logging:2026.09.1 2026.09.1 \ + org.apache.logging.log4j log4j-core 2.17.1 \ + || ! _t19_compose "$fx/app.jar" apps/orders:1.0.0-deps.2026.09.1 platform/java-deps/logging:2026.09.1 2026.09.1 \ + || ! _t19_compose "$billing_jar" apps/billing:3.2.0-deps.2026.09.1 platform/java-deps/logging:2026.09.1 2026.09.1; then + fail "tier19: publish patched bundle and recompose applications" "see $WORK/t19-build.log"; return 0 + fi + pass "tier19: released patched bundle (log4j-core 2.17.1) and recomposed the unchanged thin JARs" + local diff_json="$WORK/t19-layers.json" + python3 "$E2E_DIR/cve_sweep.py" compare --store "$T19_STORE" --arch "$arch" \ + apps/orders:1.0.0 apps/orders:1.0.0-deps.2026.09.1 >"$diff_json" 2>>"$WORK/t19-build.log" + assert_eq "tier19: recomposed image reuses the application layer" \ + "$(_t19_json "$diff_json" "[s.split('=')[0] for s in r['shared']]")" "['app']" + assert_eq "tier19: recomposed image replaces only the managed classpath layer" \ + "$(_t19_json "$diff_json" "[s.split('=')[0] for s in r['onlyA'] + r['onlyB']]")" "['classpath', 'classpath']" + + if ! import_oci_layout "$T19_NODE" "$T19_STORE" "$WORK/t19-import.log"; then + fail "tier19: import patched images" "see $WORK/t19-import.log"; return 0 + fi + local orders_v2 billing_v2 + if ! orders_v2="$(_t19_pin apps/orders:1.0.0-deps.2026.09.1)" \ + || ! billing_v2="$(_t19_pin apps/billing:3.2.0-deps.2026.09.1)"; then + fail "tier19: pin patched images for CRI" "see $WORK/t19-import.log"; return 0 + fi + + # --- (c) zero-downtime rolling redeploy under continuous traffic ------------- + local url="http://orders.$T19_PROD.svc.cluster.local:$T19_PORT/version" + kubectl exec -n "$T19_PROD" t19-client -- sh -c \ + "rm -f /tmp/poll /tmp/stop; nohup sh -c 'while [ ! -f /tmp/stop ]; do wget -q -O- -T 2 $url || echo FAIL; sleep 0.2; done' >/tmp/poll 2>&1 >"$WORK/t19-deploy.log" 2>&1 + sleep 3 + kubectl set image -n "$T19_PROD" deploy/orders app="$orders_v2" >>"$WORK/t19-deploy.log" 2>&1 + kubectl set image -n "$T19_PROD" deploy/billing app="$billing_v2" >>"$WORK/t19-deploy.log" 2>&1 + for d in orders billing; do + if ! kubectl rollout status -n "$T19_PROD" deploy/"$d" --timeout=240s >>"$WORK/t19-deploy.log" 2>&1; then + fail "tier19: rolling redeploy of prod/$d" "$(save_pod_diag t19-rollout "$T19_PROD" "app=$d")"; return 0 + fi + done + if ! _t19_wait_digest_gone "$T19_PROD" "${orders_v1##*@}" || ! _t19_wait_digest_gone "$T19_PROD" "${billing_v1##*@}"; then + fail "tier19: old vulnerable pods drained" "$(save_pod_diag t19-drain "$T19_PROD")"; return 0 + fi + sleep 3 + kubectl exec -n "$T19_PROD" t19-client -- touch /tmp/stop >/dev/null 2>&1 + sleep 1 + local poll; poll="$(kubectl exec -n "$T19_PROD" t19-client -- cat /tmp/poll 2>/dev/null)" + printf '%s\n' "$poll" >"$WORK/t19-poll.log" + local total failed first last + total="$(grep -c 'log4j-core=' <<<"$poll" || true)" + failed="$(grep -c 'FAIL' <<<"$poll" || true)" + first="$(grep -m1 'log4j-core=' <<<"$poll")" + last="$(grep 'log4j-core=' <<<"$poll" | tail -1)" + if (( total >= 20 )); then + pass "tier19: client polled prod/orders throughout the rollout ($total responses)" + else + fail "tier19: client polled prod/orders throughout the rollout" "only $total responses; see $WORK/t19-poll.log" + fi + assert_eq "tier19: rolling redeploy dropped zero requests" "$failed" "0" + assert_contains "tier19: traffic was served by the vulnerable build before the rollout" "$first" "log4j-core=2.14.1" + assert_contains "tier19: traffic is served by the patched build after the rollout" "$last" "log4j-core=2.17.1" + assert_contains "tier19: prod/billing JVM now loads log4j-core 2.17.1" \ + "$(_t19_version "$T19_PROD" billing)" "log4j-core=2.17.1" + + # --- (d) verify remediation from the SBOMs of what is now running ------------- + local after="$WORK/t19-sweep-after.json" + code="$(_t19_sweep "$after" --namespace "$T19_PROD")" + assert_eq "tier19: post-remediation prod sweep is clean (exit 0)" "$code" "0" + assert_eq "tier19: prod runs only the fixed log4j-core version" \ + "$(_t19_json "$after" "sorted({v for w in r['workloads'] for i in w['images'] for v in i['componentVersions']})")" \ + "['2.17.1']" + assert_eq "tier19: remediation kept the application JAR identity" \ + "$(_t19_json "$after" "[i['applicationJarDigest'] for w in r['workloads'] if w['name'] == 'orders' for i in w['images']]")" \ + "$(_t19_json "$before" "[i['applicationJarDigest'] for w in r['workloads'] if w['name'] == 'orders' for i in w['images']]")" + _t19_sweep "$fleet" >/dev/null + assert_eq "tier19: fleet-wide sweep still tracks unremediated staging" \ + "$(_t19_json "$fleet" "','.join(r['vulnerable'])")" "$T19_STAGING/orders" +} diff --git a/integration-tests/fixtures/log4j-cve-app/build.sh b/integration-tests/fixtures/log4j-cve-app/build.sh new file mode 100755 index 00000000..ef73927e --- /dev/null +++ b/integration-tests/fixtures/log4j-cve-app/build.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +# Builds the CVE-remediation fixture with only the JDK: +# target/log4j-core-2.14.1.jar vulnerable stand-in library +# target/log4j-core-2.17.1.jar patched stand-in library (same API) +# target/safe-lib-1.0.jar unrelated library for a control bundle +# target/app.jar thin application JAR (no nested dependencies) +set -euo pipefail + +root="$(cd "$(dirname "$0")" && pwd)" +target="$root/target" +rm -rf "$target" +mkdir -p "$target" + +build_library() { + local version="$1" jar="$2" work="$target/lib-$1" + mkdir -p "$work/src/com/example/stublog" "$work/classes" + sed "s/@VERSION@/$version/" "$root/src/com/example/stublog/LibraryVersion.java.in" \ + >"$work/src/com/example/stublog/LibraryVersion.java" + javac --release 17 -d "$work/classes" "$work/src/com/example/stublog/LibraryVersion.java" + jar --create --file "$target/$jar" -C "$work/classes" . +} + +build_library 2.14.1 log4j-core-2.14.1.jar +build_library 2.17.1 log4j-core-2.17.1.jar + +mkdir -p "$target/safe/com/example/safelib" +printf 'safe-lib 1.0\n' >"$target/safe/com/example/safelib/VERSION" +jar --create --file "$target/safe-lib-1.0.jar" -C "$target/safe" . + +mkdir -p "$target/app-classes" +javac --release 17 -d "$target/app-classes" "$root/src/com/example/cveapp/CveApp.java" +jar --create --file "$target/app.jar" --main-class com.example.cveapp.CveApp \ + -C "$target/app-classes" . diff --git a/integration-tests/fixtures/log4j-cve-app/src/com/example/cveapp/CveApp.java b/integration-tests/fixtures/log4j-cve-app/src/com/example/cveapp/CveApp.java new file mode 100644 index 00000000..e48b08cf --- /dev/null +++ b/integration-tests/fixtures/log4j-cve-app/src/com/example/cveapp/CveApp.java @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.example.cveapp; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; + +/** + * Thin application JAR for the CVE-remediation tier. It carries no + * dependencies; the logging library comes from the managed dependency bundle + * under /app/lib, so the SAME application JAR can be recomposed onto a patched + * bundle and report which library version the running JVM actually loaded. + */ +public final class CveApp { + private CveApp() { + } + + public static void main(String[] args) throws IOException { + int port = Integer.getInteger("server.port", 8080); + String app = System.getenv().getOrDefault("APP_NAME", "app"); + HttpServer server = HttpServer.create(new InetSocketAddress(port), 0); + server.createContext("/healthz", exchange -> respond(exchange, "ok\n")); + server.createContext("/version", exchange -> + respond(exchange, app + " log4j-core=" + loggingVersion() + "\n")); + server.start(); + + // Keep serving briefly after SIGTERM so kube-proxy can drop this + // endpoint before the listener closes; a rollout must not drop requests. + long drainMillis = Long.getLong("brewlet.test.drainMillis", 5000L); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + Thread.sleep(drainMillis); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + server.stop(1); + })); + System.out.println(app + " started with log4j-core=" + loggingVersion()); + } + + private static String loggingVersion() { + try { + Class library = Class.forName("com.example.stublog.LibraryVersion"); + return (String) library.getMethod("version").invoke(null); + } catch (ReflectiveOperationException e) { + return "absent"; + } + } + + private static void respond(HttpExchange exchange, String body) throws IOException { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "text/plain; charset=utf-8"); + exchange.sendResponseHeaders(200, bytes.length); + try (OutputStream out = exchange.getResponseBody()) { + out.write(bytes); + } + } +} diff --git a/integration-tests/fixtures/log4j-cve-app/src/com/example/stublog/LibraryVersion.java.in b/integration-tests/fixtures/log4j-cve-app/src/com/example/stublog/LibraryVersion.java.in new file mode 100644 index 00000000..ee3df979 --- /dev/null +++ b/integration-tests/fixtures/log4j-cve-app/src/com/example/stublog/LibraryVersion.java.in @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package com.example.stublog; + +/** + * Test-only stand-in for a third-party logging library. build.sh compiles one + * copy per version and packages it under Maven coordinates that mimic + * org.apache.logging.log4j:log4j-core, so SBOM-driven CVE sweeps have a + * realistic component to match without downloading the real library. + */ +public final class LibraryVersion { + private LibraryVersion() { + } + + public static String version() { + return "@VERSION@"; + } +}