diff --git a/src/publish.py b/src/publish.py index 8df7552..44f3992 100644 --- a/src/publish.py +++ b/src/publish.py @@ -4,6 +4,7 @@ from copy import deepcopy from dataclasses import dataclass import hashlib +import json from pathlib import Path import re from typing import Any @@ -26,6 +27,10 @@ "btc_cycle_indicators.json", ) +LIVE_POOL_LEGACY_EXACT_BYTES_CONTRACT_VERSION = ( + "qsl.crypto_live_pool_legacy_exact_bytes.v1" +) + @dataclass(frozen=True) class PublishSettings: @@ -129,6 +134,10 @@ def _sha256_file(path: Path) -> str: return digest.hexdigest() +def _reject_non_standard_json_constant(value: str) -> None: + raise ValueError(f"Non-standard JSON constant is not allowed: {value}") + + def _validate_runtime_evidence_identity( *, identity: Any, @@ -304,6 +313,56 @@ def build_firestore_payload( ) -> dict[str, Any]: symbol_map = dict(artifacts.live_pool_legacy["symbols"]) symbols = list(symbol_map.keys()) + exact_bytes = artifacts.live_pool_legacy_path.read_bytes() + try: + exact_text = exact_bytes.decode("utf-8") + except UnicodeDecodeError as exc: + raise ValueError("live_pool_legacy.json exact bytes must be valid UTF-8.") from exc + try: + exact_payload = json.loads( + exact_text, + parse_constant=_reject_non_standard_json_constant, + ) + except ValueError as exc: + raise ValueError("live_pool_legacy.json exact bytes must contain valid JSON.") from exc + if not isinstance(exact_payload, dict): + raise ValueError("live_pool_legacy.json exact bytes must contain a JSON object.") + + exact_digest = hashlib.sha256(exact_bytes).hexdigest() + manifest_entry = artifacts.artifact_manifest.get("artifacts", {}).get( + "live_pool_legacy", {} + ) + identity_entry = artifacts.runtime_evidence_identity.get("artifacts", {}).get( + "live_pool_legacy", {} + ) + if ( + manifest_entry.get("sha256") != exact_digest + or identity_entry.get("sha256") != exact_digest + ): + raise ValueError( + "live_pool_legacy.json exact bytes digest mismatch with artifact manifest or runtime identity." + ) + + if ( + exact_payload.get("as_of_date") != artifacts.as_of_date + or exact_payload.get("version") != artifacts.version + or exact_payload.get("mode") != settings.mode + or exact_payload.get("symbols") != artifacts.live_pool_legacy.get("symbols") + or exact_payload.get("symbol_map") + != artifacts.live_pool_legacy.get("symbol_map") + or exact_payload.get("pool_size") + != artifacts.live_pool_legacy.get("pool_size") + or exact_payload.get("pool_size") != artifacts.live_pool.get("pool_size") + or exact_payload.get("source_project") + != artifacts.live_pool_legacy.get("source_project") + or exact_payload.get("source_project") != settings.source_project + or list(exact_payload.get("symbols", {})) != symbols + or exact_payload.get("symbol_map") != symbol_map + ): + raise ValueError( + "live_pool_legacy.json exact bytes do not match Firestore convenience fields." + ) + generated_at = pd.Timestamp.now(tz="UTC").isoformat() return { "as_of_date": artifacts.as_of_date, @@ -325,6 +384,11 @@ def build_firestore_payload( "generated_at": generated_at, "source_project": settings.source_project, "runtime_evidence_identity": deepcopy(artifacts.runtime_evidence_identity), + "live_pool_legacy_exact_bytes": { + "contract_version": LIVE_POOL_LEGACY_EXACT_BYTES_CONTRACT_VERSION, + "encoding": "utf-8", + "utf8_text": exact_text, + }, } @@ -370,6 +434,8 @@ def upload_release_artifacts( settings: PublishSettings, artifacts: ReleaseArtifacts, storage_layout: dict[str, Any], + *, + live_pool_legacy_exact_bytes: bytes, ) -> None: if settings.dry_run: return @@ -398,9 +464,14 @@ def upload_release_artifacts( files[opt_file] = opt_path for filename, local_path in files.items(): object_info = storage_layout["objects"][filename] - store.write_bytes(object_info["release_uri"], local_path.read_bytes()) + upload_bytes = ( + live_pool_legacy_exact_bytes + if filename == "live_pool_legacy.json" + else local_path.read_bytes() + ) + store.write_bytes(object_info["release_uri"], upload_bytes) if settings.upload_current_pointer: - store.write_bytes(object_info["current_uri"], local_path.read_bytes()) + store.write_bytes(object_info["current_uri"], upload_bytes) def publish_firestore_summary(settings: PublishSettings, firestore_payload: dict[str, Any]) -> None: @@ -450,6 +521,9 @@ def run_release_publish( artifacts = load_release_artifacts(config["paths"].output_dir, settings.mode) storage_layout = build_storage_layout(settings, artifacts) firestore_payload = build_firestore_payload(settings, artifacts, storage_layout) + live_pool_legacy_exact_bytes = firestore_payload["live_pool_legacy_exact_bytes"][ + "utf8_text" + ].encode("utf-8") manifest = build_release_manifest(settings, artifacts, storage_layout, firestore_payload) manifest_path = write_release_manifest(artifacts.output_dir, manifest) validation = assert_release_outputs( @@ -469,8 +543,18 @@ def run_release_publish( or firestore_payload.get("runtime_evidence_identity") != artifacts.runtime_evidence_identity ): raise ValueError("Runtime evidence identity changed before publish.") + if ( + live_pool_legacy_exact_bytes + != artifacts.live_pool_legacy_path.read_bytes() + ): + raise ValueError("Validated live_pool_legacy.json exact bytes changed before publish.") - upload_release_artifacts(settings, artifacts, storage_layout) + upload_release_artifacts( + settings, + artifacts, + storage_layout, + live_pool_legacy_exact_bytes=live_pool_legacy_exact_bytes, + ) publish_firestore_summary(settings, firestore_payload) return { diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 644dbf4..62dc222 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -5,16 +5,19 @@ import tempfile import unittest from pathlib import Path +from unittest.mock import patch import pandas as pd from src.publish import ( PublishSettings, + ReleaseArtifacts, build_firestore_payload, build_release_manifest, build_storage_layout, ensure_publish_preflight, load_release_artifacts, + upload_release_artifacts, ) from src.release_contract import validate_release_outputs @@ -31,6 +34,21 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def build_publish_settings() -> PublishSettings: + return PublishSettings( + enabled=False, + dry_run=True, + mode="core_major", + project_id=None, + cloud_bucket=None, + cloud_root_prefix="crypto-live-pool-pipelines", + firestore_collection="strategy", + firestore_document="CRYPTO_LIVE_POOL_ROTATION_LIVE_POOL", + source_project="crypto-live-pool-pipelines", + upload_current_pointer=False, + ) + + class ReleaseContractValidationTests(unittest.TestCase): def build_outputs( self, @@ -177,6 +195,21 @@ def build_outputs( }, ) + def build_runtime_identity_outputs(self, root: Path) -> Path: + self.build_outputs( + root, + include_manifest=True, + include_runtime_evidence_identity=True, + ) + return root / "data" / "output" + + def load_publish_context( + self, output_dir: Path + ) -> tuple[ReleaseArtifacts, PublishSettings, dict[str, object]]: + artifacts = load_release_artifacts(output_dir, "core_major") + settings = build_publish_settings() + return artifacts, settings, build_storage_layout(settings, artifacts) + def test_validate_release_outputs_accepts_consistent_contract(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: root = Path(tmp_dir) @@ -236,6 +269,211 @@ def test_identity_is_canonical_across_artifact_release_and_firestore(self) -> No self.assertEqual(release_manifest["runtime_evidence_identity"], identity) self.assertEqual(firestore_payload["runtime_evidence_identity"], identity) + def test_firestore_payload_preserves_exact_legacy_artifact_bytes(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + output_dir = self.build_runtime_identity_outputs(root) + legacy_path = output_dir / "live_pool_legacy.json" + legacy_payload = json.loads(legacy_path.read_text(encoding="utf-8")) + exact_bytes = ("\n" + json.dumps(legacy_payload, separators=(", ", ": ")) + "\n").encode( + "utf-8" + ) + legacy_path.write_bytes(exact_bytes) + + manifest_path = output_dir / "artifact_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + exact_digest = hashlib.sha256(exact_bytes).hexdigest() + manifest["artifacts"]["live_pool_legacy"]["sha256"] = exact_digest + manifest["runtime_evidence_identity"]["artifacts"]["live_pool_legacy"][ + "sha256" + ] = exact_digest + write_json(manifest_path, manifest) + + artifacts, settings, storage_layout = self.load_publish_context(output_dir) + firestore_payload = build_firestore_payload( + settings, + artifacts, + storage_layout, + ) + + handoff = firestore_payload["live_pool_legacy_exact_bytes"] + self.assertEqual( + handoff["contract_version"], + "qsl.crypto_live_pool_legacy_exact_bytes.v1", + ) + self.assertEqual(handoff["encoding"], "utf-8") + self.assertEqual(handoff["utf8_text"].encode("utf-8"), exact_bytes) + + def test_upload_reuses_validated_legacy_snapshot_after_path_mutation(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + output_dir = self.build_runtime_identity_outputs(root) + artifacts = load_release_artifacts(output_dir, "core_major") + settings = PublishSettings( + enabled=True, + dry_run=False, + mode="core_major", + project_id="test-project", + cloud_bucket="test-bucket", + cloud_root_prefix="crypto-live-pool-pipelines", + firestore_collection="strategy", + firestore_document="CRYPTO_LIVE_POOL_ROTATION_LIVE_POOL", + source_project="crypto-live-pool-pipelines", + upload_current_pointer=True, + ) + storage_layout = build_storage_layout(settings, artifacts) + firestore_payload = build_firestore_payload( + settings, + artifacts, + storage_layout, + ) + validated_bytes = firestore_payload["live_pool_legacy_exact_bytes"][ + "utf8_text" + ].encode("utf-8") + artifacts.live_pool_legacy_path.write_bytes(b'{"mutated": true}\n') + + uploaded: dict[str, bytes] = {} + + class FakeStore: + def write_bytes(self, uri: str, payload: bytes) -> None: + uploaded[uri] = payload + + with patch( + "quant_platform_kit.cloud.get_object_store", + return_value=FakeStore(), + ): + upload_release_artifacts( + settings, + artifacts, + storage_layout, + live_pool_legacy_exact_bytes=validated_bytes, + ) + + legacy_objects = storage_layout["objects"]["live_pool_legacy.json"] + release_bytes = uploaded[legacy_objects["release_uri"]] + current_bytes = uploaded[legacy_objects["current_uri"]] + manifest_digest = artifacts.artifact_manifest["artifacts"][ + "live_pool_legacy" + ]["sha256"] + identity_digest = artifacts.runtime_evidence_identity["artifacts"][ + "live_pool_legacy" + ]["sha256"] + self.assertEqual(release_bytes, validated_bytes) + self.assertEqual(current_bytes, validated_bytes) + self.assertEqual(hashlib.sha256(validated_bytes).hexdigest(), manifest_digest) + self.assertEqual(hashlib.sha256(release_bytes).hexdigest(), manifest_digest) + self.assertEqual(identity_digest, manifest_digest) + + def test_firestore_payload_rejects_mutated_legacy_bytes_with_unchanged_identity(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + output_dir = self.build_runtime_identity_outputs(root) + artifacts, settings, storage_layout = self.load_publish_context(output_dir) + with artifacts.live_pool_legacy_path.open("ab") as handle: + handle.write(b"\n") + + with self.assertRaisesRegex(ValueError, "digest mismatch"): + build_firestore_payload( + settings, + artifacts, + storage_layout, + ) + + def test_firestore_payload_rejects_top_level_convenience_field_mismatch(self) -> None: + for field in ("symbols", "symbol_map"): + with self.subTest(field=field), tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + output_dir = self.build_runtime_identity_outputs(root) + artifacts, settings, storage_layout = self.load_publish_context(output_dir) + if field == "symbols": + artifacts.live_pool_legacy[field].pop("TRXUSDT") + else: + artifacts.live_pool_legacy[field]["TRXUSDT"] = { + "base_asset": "MISMATCH" + } + + with self.assertRaisesRegex(ValueError, "convenience fields"): + build_firestore_payload( + settings, + artifacts, + storage_layout, + ) + + def test_firestore_payload_rejects_pool_size_or_source_project_mismatch(self) -> None: + cases = { + "pool_size_mismatch": ("pool_size", 4), + "pool_size_missing": ("pool_size", None), + "source_project_mismatch": ("source_project", "wrong-source"), + "source_project_missing": ("source_project", None), + } + for label, (field, value) in cases.items(): + with self.subTest(label=label), tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + output_dir = self.build_runtime_identity_outputs(root) + legacy_path = output_dir / "live_pool_legacy.json" + legacy_payload = json.loads(legacy_path.read_text(encoding="utf-8")) + if value is None: + legacy_payload.pop(field) + else: + legacy_payload[field] = value + write_json(legacy_path, legacy_payload) + + manifest_path = output_dir / "artifact_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + digest = sha256_file(legacy_path) + manifest["artifacts"]["live_pool_legacy"]["sha256"] = digest + manifest["runtime_evidence_identity"]["artifacts"][ + "live_pool_legacy" + ]["sha256"] = digest + write_json(manifest_path, manifest) + + artifacts, settings, storage_layout = self.load_publish_context( + output_dir + ) + with self.assertRaisesRegex(ValueError, "convenience fields"): + build_firestore_payload( + settings, + artifacts, + storage_layout, + ) + + def test_firestore_payload_rejects_invalid_legacy_artifact_bytes(self) -> None: + cases = { + "invalid_utf8": (b"\xff", "valid UTF-8"), + "invalid_json": (b"{", "valid JSON"), + "nan": (b'{"value": NaN}', "valid JSON"), + "infinity": (b'{"value": Infinity}', "valid JSON"), + "negative_infinity": (b'{"value": -Infinity}', "valid JSON"), + "non_object": (b"[]", "JSON object"), + } + for label, (invalid_bytes, expected_error) in cases.items(): + with self.subTest(label=label), tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + output_dir = self.build_runtime_identity_outputs(root) + artifacts, settings, storage_layout = self.load_publish_context(output_dir) + artifacts.live_pool_legacy_path.write_bytes(invalid_bytes) + + with self.assertRaisesRegex(ValueError, expected_error): + build_firestore_payload( + settings, + artifacts, + storage_layout, + ) + + def test_firestore_payload_rejects_missing_legacy_artifact(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + output_dir = self.build_runtime_identity_outputs(root) + artifacts, settings, storage_layout = self.load_publish_context(output_dir) + artifacts.live_pool_legacy_path.unlink() + + with self.assertRaises(FileNotFoundError): + build_firestore_payload( + settings, + artifacts, + storage_layout, + ) + def test_artifact_byte_mutation_breaks_runtime_identity_before_publish(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: root = Path(tmp_dir)