diff --git a/proto/worker_scheduler.proto b/proto/worker_scheduler.proto index 3a66ee803..1a95647cc 100644 --- a/proto/worker_scheduler.proto +++ b/proto/worker_scheduler.proto @@ -332,9 +332,6 @@ message DesiredResidency { uint64 generation = 1; repeated string disk_refs = 2; repeated DesiredInstance hot = 3; - // th#1941: keyed by COMPOSED MANIFEST DIGEST, not by canonical ref. Two - // resolutions of one repo that differ in their component sources are two - // entries with two digests and cannot interfere. map snapshots = 4; // th#1087/th#1085: hub-authoritative desired state is the PAIR // (release_id, config_generation). config_generation is the release's @@ -819,10 +816,9 @@ message RunJob { MediaBytes media_bytes = 9; // client's `Prefer: bytes=inline|url` media-delivery preference ResolvedCompute compute = 10; repeated ModelBinding models = 11; // slot -> ref bindings resolved by the orchestrator - // th#1941: presigned snapshots keyed by COMPOSED MANIFEST DIGEST, not by - // canonical ref. Ordinary warm-local bindings may be omitted; - // required_compile retains every exact model digest plus the selected graph - // so the worker can revalidate immutable execution identity. + // Presigned snapshots keyed by canonical ref. Ordinary warm-local bindings + // may be omitted; required_compile retains every exact model digest plus the + // selected graph so the worker can revalidate immutable execution identity. map snapshots = 12; // Unique exact current-state fence selected by Tensorhub. Multiple // applicable targets fail closed. Mandatory for W8A8; when present on any @@ -903,11 +899,10 @@ message RequiredCompileExecution { // th#2082: this is the client's `Prefer: bytes=` header and nothing else — how // MEDIA outputs are delivered, decided before the job runs. It is NOT the -// function's output cardinality (`EndpointSpec.output_mode`, pgw#1320) and NOT -// the SSE terminal payload's `delivery` key (th#2079); all three wore one -// generic name until those issues split them. Field 9 and the enum numbers are -// unchanged by the rename, so the wire is untouched and a hub or worker on an -// older build still parses it. +// function's output cardinality and NOT the SSE terminal payload's `delivery` +// key (th#2079); all three wore one generic name until those two issues split +// them. Field 9 and the enum numbers are unchanged by the rename, so the wire +// is untouched and a worker on an older wheel still parses it. enum MediaBytes { MEDIA_BYTES_UNSPECIFIED = 0; MEDIA_BYTES_URL = 1; // upload media, return refs (default behavior) @@ -940,9 +935,14 @@ message ModelBinding { // (precedence: explicit payload value > this > endpoint fallback preset). // Additive: old workers ignore this field. string inference_defaults = 4; - // th#1941: `components` was the SUBTRACTION — "load the base composition, - // then substitute", the only wire syntax for "load X but not part of X". - // The hub now composes hub-side and sends a resolved manifest. + // th#2106 / pgw#1246: `components` was the SUBTRACTION — "load the base + // composition, then substitute" — and it never had a reader. gen-worker + // deleted the whole load-then-substitute apparatus and reserved this number + // in its own copy of this file; a hub still stamping it would be writing a + // promise the far side cannot read. The substitution is now applied hub-side + // by COMPOSING the manifest the slot ships (runtimestore.ComposeComponents), + // so what crosses the wire is one resolved tree whose component paths already + // hold the substitute's bytes. reserved 5; reserved "components"; // th#1139 (pgw#654) objective split, retiring th#1017's never-landed regime @@ -963,28 +963,21 @@ message ModelBinding { // beside inference_defaults — (model, defaults) per bound checkpoint — and // the worker's read-side typed decode (`ctx.defaults`, pgw#1377) is the // authority on the pair; the hub never interprets either. Additive. - // - // pgw#1415: NUMBER 9 IS THE HUB'S, and it is the SHIPPING one. th#1941 took - // 9 here for `manifest_digest` while tensorhub still had 8 free, th#2140 then - // took 9 there for `model`, and the two protos meant different things by the - // same tag — the hub stamped a model name that this worker read as a fetch - // pointer. The tie breaks toward the field that has a SENDER: tensorhub - // stamps `model` on every dispatch (scheduler_dispatch.go, `mb.Model`), - // and NOTHING has ever sent `manifest_digest` — th#1941's hub leg is still a - // PARKED DRAFT (tensorhub PR #1219). So this side moves, and the hub's - // number stands. string model = 9; - // The composed manifest digest this slot fetches (th#1941). Key into - // DesiredResidency.snapshots / RunJob.snapshots. `ref` above survives as - // the display/audit label; THIS is the fetch pointer. - // - // pgw#1415: moved 9 -> 10 (see above). tensorhub owes the SENDER at THIS - // number — th#1941, whose parked draft PR #1219 predates th#2140 and would - // RE-ADD the collision at 9 if it ever landed as authored. Until it lands a - // v2 dispatch carries no fetch pointer at all - // and `HubBindingResolver` refuses naming it, which is the correct state: - // the alternative was reading the hub's `model` string as a digest. - string manifest_digest = 10; + // th#2208 hardcut: pgw once declared `manifest_digest = 10`, but the hub + // never had that field or a sender for it. The worker now resolves the tree + // the plain-config pull materialized. Reserve both markers before adding the + // Bind Contract carrier so a retired fetch pointer cannot silently become a + // different fact inside this major. + reserved 10; + reserved "manifest_digest"; + // th#2290 / pgw#1653: the immutable Bind Contract selected for THIS exact + // resolved checkpoint config tree. Digest is its repo-CAS address; URL is a + // bounded presigned GET for those bytes. Both are required together. The + // document itself carries (release, derive image digest, config digest), so + // a census refusal is attributable to the bind rather than to this pod. + string bind_contract_digest = 11; + string bind_contract_url = 12; } // One LoRA adapter overlay: a resolved tensorhub ref + its adapter weight. @@ -1013,36 +1006,8 @@ message LoraOverlay { // never calls tensorhub for ref resolution; the orchestrator is the only // resolver and ships snapshots on DesiredResidency / RunJob. message Snapshot { - // th#1941 — the COMPOSED manifest digest: sha256 over the sorted - // (path, digest, size) triples of `files`. For an uncomposed variant it - // equals that variant's own storage digest. Fetch identity, dispatch key - // AND the worker's snapshot directory name — one key, one meaning. - string digest = 1; - // THE WHOLE TRUTH. Every file the pod puts on disk for this entry, already - // composed hub-side: a component sourced from another release appears here - // remapped under its subfolder, and the displaced files are simply ABSENT. - // Nothing here can express subtraction because there is nothing to subtract - // from — the list is the answer, not an input to one. + string digest = 1; // snapshot digest the download is pinned to repeated SnapshotFile files = 2; - // Audit/display ONLY. A worker that reads this to decide what to fetch has - // reintroduced the bug this deletes; `files` is complete by construction. - ResolvedProvenance provenance = 3; -} - -// th#1941 — where a composed manifest's files came from. Audit/display only. -message ResolvedProvenance { - string repo = 1; // owner/repo - string release = 2; // author-chosen release identifier (th#1936) - string variant_id = 3; // the variant the contract-spec resolved to (th#1938) - // component -> where that component's files came from, for components NOT - // supplied by variant_id. - map component_sources = 4; -} - -message ComponentOrigin { - string repo = 1; - string release = 2; - string variant_id = 3; } message SnapshotFile { diff --git a/src/gen_worker/bind_contract.py b/src/gen_worker/bind_contract.py new file mode 100644 index 000000000..cc9526800 --- /dev/null +++ b/src/gen_worker/bind_contract.py @@ -0,0 +1,235 @@ +"""The immutable Bind Contract: fetch, address verification, and refusal report. + +The hub chooses the contract for (release, derive image digest, config digest) +and sends its CAS address plus a bounded URL. This module verifies the address +before decoding. Tensor semantics stay in ``serving.streaming.census``; the +wire layer only carries the decoded expected census and its bind identity. +""" + +from __future__ import annotations + +import hashlib +import json +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any, Callable, Mapping, Optional + +from . import worker_credential +from .serving.streaming.census import Census, CensusMismatch + +BIND_CONTRACT_KIND = "tensorhub.bind-contract@1" +BIND_CONTRACT_VERSION = 1 +MAX_BIND_CONTRACT_BYTES = 4 << 20 +REPORT_PATH = "/v1/worker/release-bind-refusals" + + +class BindContractError(RuntimeError): + """The selected bind document is absent, corrupt, or unreadable.""" + + +@dataclass(frozen=True, slots=True) +class BindIdentity: + release_id: str + derive_image_digest: str + config_digest: str + + +@dataclass(frozen=True, slots=True) +class BindContract: + digest: str + identity: BindIdentity + release_contract_digest: str + census: Census + graphs: tuple[Mapping[str, Any], ...] + + +def _sha256(ref: str) -> str: + algorithm, separator, digest = str(ref or "").strip().partition(":") + if separator != ":" or algorithm.lower() != "sha256": + raise BindContractError( + f"bind contract address {ref!r} is not sha256:<64 lowercase hex>" + ) + if len(digest) != 64 or any(ch not in "0123456789abcdef" for ch in digest): + raise BindContractError( + f"bind contract address {ref!r} is not sha256:<64 lowercase hex>" + ) + return digest + + +def decode(raw: bytes, *, digest: str) -> BindContract: + expected = _sha256(digest) + actual = hashlib.sha256(raw).hexdigest() + if actual != expected: + raise BindContractError( + f"bind contract {digest} fetched bytes hashing to sha256:{actual}" + ) + try: + document = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise BindContractError( + f"bind contract {digest} is not JSON: {exc}" + ) from exc + if not isinstance(document, dict): + raise BindContractError(f"bind contract {digest} is not a JSON object") + if document.get("v") != BIND_CONTRACT_VERSION: + raise BindContractError( + f"bind contract {digest} states v={document.get('v')!r}; " + f"this worker reads v={BIND_CONTRACT_VERSION}" + ) + if document.get("kind") != BIND_CONTRACT_KIND: + raise BindContractError( + f"bind contract {digest} states kind={document.get('kind')!r}; " + f"this worker reads {BIND_CONTRACT_KIND!r}" + ) + identity = document.get("identity") + if not isinstance(identity, dict): + raise BindContractError(f"bind contract {digest} has no identity object") + release_id = str(identity.get("release_id") or "").strip() + image = str(identity.get("derive_image_digest") or "").strip() + config = str(identity.get("config_digest") or "").strip() + if not release_id or not image or not config: + raise BindContractError( + f"bind contract {digest} identity lacks release_id, " + "derive_image_digest, or config_digest" + ) + release_contract_digest = str( + document.get("release_contract_digest") or "" + ).strip() + _sha256(release_contract_digest) + census_row = document.get("construction_census") + if not isinstance(census_row, dict): + raise BindContractError( + f"bind contract {digest} has no construction_census object" + ) + try: + census = Census.from_document(census_row) + except Exception as exc: + raise BindContractError( + f"bind contract {digest} carries an unreadable construction " + f"census: {type(exc).__name__}: {exc}" + ) from exc + graph_rows = document.get("graphs") + if not isinstance(graph_rows, list): + raise BindContractError(f"bind contract {digest} has no graphs array") + graphs: list[Mapping[str, Any]] = [] + seen_graphs: set[tuple[str, str]] = set() + for index, row in enumerate(graph_rows): + if not isinstance(row, dict): + raise BindContractError( + f"bind contract {digest} graphs[{index}] is not an object" + ) + lane = str(row.get("lane") or "").strip() + graph = str(row.get("graph_hash") or "").strip() + if not lane or not graph: + raise BindContractError( + f"bind contract {digest} graphs[{index}] lacks lane or graph_hash" + ) + key = (lane, graph) + if key in seen_graphs: + raise BindContractError( + f"bind contract {digest} repeats graph {graph!r} in lane {lane!r}" + ) + seen_graphs.add(key) + graphs.append(dict(row)) + return BindContract( + digest=digest, + identity=BindIdentity(release_id, image, config), + release_contract_digest=release_contract_digest, + census=census, + graphs=tuple(graphs), + ) + + +def fetch( + digest: str, + url: str, + *, + token: Optional[str] = None, + opener: Callable[..., Any] = urllib.request.urlopen, +) -> BindContract: + _sha256(digest) + if not str(url or "").strip(): + raise BindContractError(f"bind contract {digest} has no fetch URL") + bearer = str(token if token is not None else worker_credential.current() or "").strip() + if not bearer: + raise BindContractError( + f"bind contract {digest} fetch has no worker credential" + ) + request = urllib.request.Request(str(url), method="GET") + request.add_header("Accept-Encoding", "identity") + request.add_header("Authorization", f"Bearer {bearer}") + try: + with opener(request, timeout=60.0) as response: + raw = response.read(MAX_BIND_CONTRACT_BYTES + 1) + except (OSError, urllib.error.URLError) as exc: + raise BindContractError(f"bind contract {digest} fetch failed: {exc}") from exc + if len(raw) > MAX_BIND_CONTRACT_BYTES: + raise BindContractError( + f"bind contract {digest} exceeds {MAX_BIND_CONTRACT_BYTES} bytes" + ) + return decode(bytes(raw), digest=digest) + + +def refusal_payload(contract: BindContract, mismatch: CensusMismatch) -> bytes: + """Stable report body; no caller parses the mismatch's prose.""" + + body: Mapping[str, Any] = { + "release_id": contract.identity.release_id, + "derive_image_digest": contract.identity.derive_image_digest, + "config_digest": contract.identity.config_digest, + "bind_contract_digest": contract.digest, + "code": "bind_contract_census_mismatch", + "invariant": mismatch.invariant, + "component": mismatch.component, + "tensor": mismatch.tensor, + "detail": str(mismatch), + } + return json.dumps(body, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def report_refusal( + hub_base_url: str, + contract: BindContract, + mismatch: CensusMismatch, + *, + token: Optional[str] = None, + opener: Callable[..., Any] = urllib.request.urlopen, +) -> None: + base = str(hub_base_url or "").strip().rstrip("/") + if not base: + raise BindContractError("cannot report bind refusal: tensorhub URL is absent") + bearer = str(token if token is not None else worker_credential.current() or "").strip() + if not bearer: + raise BindContractError("cannot report bind refusal: worker credential is absent") + request = urllib.request.Request( + base + REPORT_PATH, + data=refusal_payload(contract, mismatch), + headers={ + "Authorization": f"Bearer {bearer}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with opener(request, timeout=15.0) as response: + status = int(getattr(response, "status", 200)) + if status < 200 or status >= 300: + raise BindContractError( + f"bind refusal report answered HTTP {status}" + ) + except (OSError, urllib.error.URLError) as exc: + raise BindContractError(f"bind refusal report failed: {exc}") from exc + + +__all__ = [ + "BIND_CONTRACT_KIND", + "BIND_CONTRACT_VERSION", + "BindContract", + "BindContractError", + "BindIdentity", + "decode", + "fetch", + "refusal_payload", + "report_refusal", +] diff --git a/src/gen_worker/models/checkpoint_juggle.py b/src/gen_worker/models/checkpoint_juggle.py index ca5632143..a12a7e9e5 100644 --- a/src/gen_worker/models/checkpoint_juggle.py +++ b/src/gen_worker/models/checkpoint_juggle.py @@ -89,7 +89,7 @@ from dataclasses import dataclass from enum import Enum from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Sequence, Set, Tuple +from typing import Any, Callable, Dict, List, Optional, Set, Tuple from .._vendor.tensorfs.layout2 import ExpectedHeader, LayoutTensor from .arena_residency import ( @@ -100,11 +100,16 @@ dlpack_dtype, ) from .safetensors_header import header_len_ok +from ..serving.streaming.fill_client import ( + AddressSource, + CudaFillClient, + Destination, + FileSource, + HostFillClient, +) logger = logging.getLogger(__name__) -_DL_UINT8 = (1, 8) - DEFAULT_HOST_FLOOR_BYTES = 4 << 30 _SAFETENSORS_DTYPES: Dict[str, Tuple[int, int]] = { @@ -420,7 +425,6 @@ def __init__( *, torch_mod: Any, varena_mod: Any = None, - engine: Any = None, pin: bool = True, ) -> None: self.checkpoint_id = checkpoint_id @@ -456,27 +460,43 @@ def __init__( if buffer is None: buffer = torch_mod.empty(n, dtype=torch_mod.uint8) self.buffer = buffer - self._build(engine) + self._build() - def _build(self, engine: Any) -> None: + def _build(self) -> None: torch = self._torch - straight: List[Tuple[str, int, int, int, int]] = [] casts: List[Tuple[SlotSpec, SlotSource]] = [] base_ptr = int(self.buffer.data_ptr()) - for region in self.layout.regions: - for slot in region.slots: - src = self.manifest[_slot_key(slot)] - if (src.dtype_code, src.dtype_bits) == (slot.dtype_code, slot.dtype_bits): - straight.append( - (str(src.path), src.offset, src.length, base_ptr + slot.offset, 0) - ) - else: - casts.append((slot, src)) - if straight: - if engine is not None: - engine.submit(straight, 0).wait() + sources: List[FileSource] = [] + cursor = 0 + slots = sorted( + (slot for region in self.layout.regions for slot in region.slots), + key=lambda slot: slot.offset, + ) + for slot in slots: + if slot.offset > cursor: + sources.append(FileSource(None, 0, slot.offset - cursor)) + src = self.manifest[_slot_key(slot)] + if (src.dtype_code, src.dtype_bits) == (slot.dtype_code, slot.dtype_bits): + sources.append(FileSource(str(src.path), src.offset, src.length)) else: - self._pread(straight, base_ptr) + sources.append(FileSource(None, 0, slot.nbytes)) + casts.append((slot, src)) + cursor = slot.offset + slot.nbytes + if cursor < self.layout.virtual_bytes: + sources.append(FileSource(None, 0, self.layout.virtual_bytes - cursor)) + if sources: + HostFillClient().fill_files( + sources, + Destination( + name=self.checkpoint_id, + pointer=base_ptr, + capacity=self.layout.virtual_bytes, + source_offset=0, + shape=(self.layout.virtual_bytes,), + element_bytes=1, + layout="torch.contiguous@1", + ), + ) for slot, src in casts: file_dtype = _torch_dtype(torch, src.dtype_code, src.dtype_bits) lane_dtype = _torch_dtype(torch, slot.dtype_code, slot.dtype_bits) @@ -495,21 +515,6 @@ def _build(self, engine: Any) -> None: for region in self.layout.regions: self.region_digests[region.name] = self.region_digest(region) - def _pread( - self, requests: Sequence[Tuple[str, int, int, int, int]], base_ptr: int - ) -> None: - mv = memoryview(self.buffer.numpy()) - for path, offset, length, host_ptr, _dev in requests: - start = host_ptr - base_ptr - with open(path, "rb") as fh: - fh.seek(offset) - got = fh.readinto(mv[start : start + length]) - if got != length: - raise JuggleRefusal( - f"{self.checkpoint_id}: short read from {path} " - f"({got} of {length} bytes)" - ) - def region_bytes(self, region: RegionSpec) -> Any: return self.buffer[region.offset : region.offset + region.span] @@ -559,14 +564,12 @@ def __init__( *, torch_mod: Any, varena_mod: Any = None, - engine_factory: Optional[Callable[[], Any]] = None, host_floor_bytes: int = DEFAULT_HOST_FLOOR_BYTES, mem_available: Callable[[], int] = _mem_available_bytes, ) -> None: self.layout = layout self._torch = torch_mod self._varena = varena_mod - self._engine_factory = engine_factory self.host_floor_bytes = int(host_floor_bytes) self._mem_available = mem_available self.manifests: Dict[str, Dict[str, SlotSource]] = {} @@ -630,10 +633,9 @@ def ensure_warm(self, checkpoint_id: str) -> Optional[CheckpointImage]: self.host_floor_bytes / (1 << 30), self.pressure_epoch, ) return None - engine = self._engine_factory() if self._engine_factory else None image = CheckpointImage( checkpoint_id, self.layout, manifest, - torch_mod=self._torch, varena_mod=self._varena, engine=engine, + torch_mod=self._torch, varena_mod=self._varena, ) self.images[checkpoint_id] = image self._lru.append(checkpoint_id) @@ -775,7 +777,6 @@ def __init__( self.layout, torch_mod=residency._torch, varena_mod=residency._varena, - engine_factory=residency._engine_for, ) if not residency.adopted: raise ValueError( @@ -791,6 +792,9 @@ def __init__( self.rearms = 0 self.reports: List[SwitchReport] = [] self._residue_bytes = self._count_residue() + self._fill_client = CudaFillClient( + 64 << 20, int(residency.device.index or 0) + ) def admit(self, checkpoint_id: str, manifest: Dict[str, SlotSource]) -> None: self.catalog.admit(checkpoint_id, manifest) @@ -838,18 +842,13 @@ def switch_to(self, checkpoint_id: str) -> SwitchReport: from_id = self.serving_id device = self.residency.device - stream = torch.cuda.Stream(device=device) - stream.wait_stream(torch.cuda.current_stream(device)) with torch.no_grad(): self.residency.ring.drain() for region in self.layout.regions: if self.residency.is_resident(region.name): self.ledger.begin(region.name, checkpoint_id) try: - with torch.cuda.stream(stream): - bytes_moved += self._refill_backed( - region, image, manifest, stream - ) + bytes_moved += self._refill_backed(region, image, manifest) except Exception: self.ledger.poison(region.name) logger.error( @@ -876,7 +875,6 @@ def switch_to(self, checkpoint_id: str) -> SwitchReport: key: (src.path, src.offset, src.length) for key, src in manifest.items() } - torch.cuda.current_stream(device).wait_stream(stream) torch.cuda.synchronize(device) backing_verified = self._verify_backing(checkpoint_id) @@ -923,17 +921,31 @@ def _refill_backed( region: RegionSpec, image: Optional[CheckpointImage], manifest: Dict[str, SlotSource], - stream: Any, ) -> int: - if image is not None: - dst = self._device_bytes(region) - src = image.region_bytes(region) - dst.copy_(src, non_blocking=image.pinned) - return region.span base = int(self.residency.reservation.base_ptr) - requests = [] - moved = 0 - for slot in region.slots: + destination = Destination( + name=region.name, + pointer=base + region.offset, + capacity=region.span, + source_offset=0, + shape=(region.span,), + element_bytes=1, + layout="torch.contiguous@1", + ) + if image is not None: + stats = self._fill_client.fill_address( + AddressSource( + pointer=int(image.buffer.data_ptr()) + region.offset, + capacity=region.span, + ), + destination, + ) + return int(stats.destination_bytes) + sources: List[FileSource] = [] + cursor = region.offset + for slot in sorted(region.slots, key=lambda item: item.offset): + if slot.offset > cursor: + sources.append(FileSource(None, 0, slot.offset - cursor)) src = manifest[_slot_key(slot)] if (src.dtype_code, src.dtype_bits) != (slot.dtype_code, slot.dtype_bits): raise JuggleRefusal( @@ -941,23 +953,13 @@ def _refill_backed( f"flight; warm this checkpoint first (the image is where " f"casts happen, once)" ) - requests.append( - (str(src.path), src.offset, src.length, 0, base + slot.offset) - ) - moved += src.length - handle = self.residency._engine_for().submit( - requests, int(stream.cuda_stream) if stream is not None else 0 - ) - handle.wait() - return moved - - def _device_bytes(self, region: RegionSpec) -> Any: - torch = self.residency._torch - return torch.from_dlpack( - self.residency.reservation.tensor( - region.offset, [region.span], *_DL_UINT8 - ) - ) + sources.append(FileSource(str(src.path), src.offset, src.length)) + cursor = slot.offset + slot.nbytes + end = region.offset + region.span + if cursor < end: + sources.append(FileSource(None, 0, end - cursor)) + stats = self._fill_client.fill_files(sources, destination) + return int(stats.destination_bytes) def _count_residue(self) -> int: from .stream_residency import own_tensors, tensor_bytes diff --git a/src/gen_worker/pb/worker_scheduler_pb2.py b/src/gen_worker/pb/worker_scheduler_pb2.py index 0aa679eae..7d6337d21 100644 --- a/src/gen_worker/pb/worker_scheduler_pb2.py +++ b/src/gen_worker/pb/worker_scheduler_pb2.py @@ -24,7 +24,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16worker_scheduler.proto\x12\x11\x63ozy.scheduler.v1\"\xf7\x05\n\rWorkerMessage\x12)\n\x05hello\x18\x01 \x01(\x0b\x32\x18.cozy.scheduler.v1.HelloH\x00\x12\x34\n\x0bstate_delta\x18\x02 \x01(\x0b\x32\x1d.cozy.scheduler.v1.StateDeltaH\x00\x12\x36\n\x0cjob_accepted\x18\x03 \x01(\x0b\x32\x1e.cozy.scheduler.v1.JobAcceptedH\x00\x12\x32\n\njob_result\x18\x04 \x01(\x0b\x32\x1c.cozy.scheduler.v1.JobResultH\x00\x12\x36\n\x0cjob_progress\x18\x05 \x01(\x0b\x32\x1e.cozy.scheduler.v1.JobProgressH\x00\x12\x34\n\x0bmodel_event\x18\x06 \x01(\x0b\x32\x1d.cozy.scheduler.v1.ModelEventH\x00\x12:\n\x0e\x66n_unavailable\x18\x07 \x01(\x0b\x32 .cozy.scheduler.v1.FnUnavailableH\x00\x12\x34\n\x0b\x66n_degraded\x18\x08 \x01(\x0b\x32\x1d.cozy.scheduler.v1.FnDegradedH\x00\x12<\n\x0f\x61\x63tivity_update\x18\t \x01(\x0b\x32!.cozy.scheduler.v1.ActivityUpdateH\x00\x12\x44\n\x13hardware_unsuitable\x18\n \x01(\x0b\x32%.cozy.scheduler.v1.HardwareUnsuitableH\x00\x12\x36\n\x0cgoal_receipt\x18\x0b \x01(\x0b\x32\x1e.cozy.scheduler.v1.GoalReceiptH\x00\x12\x42\n\x12lifecycle_snapshot\x18\x0c \x01(\x0b\x32$.cozy.scheduler.v1.LifecycleSnapshotH\x00\x12\x32\n\nboot_phase\x18\r \x01(\x0b\x32\x1c.cozy.scheduler.v1.BootPhaseH\x00\x42\x05\n\x03msg\"\x8f\x03\n\x10SchedulerMessage\x12\x30\n\thello_ack\x18\x01 \x01(\x0b\x32\x1b.cozy.scheduler.v1.HelloAckH\x00\x12,\n\x07run_job\x18\x02 \x01(\x0b\x32\x19.cozy.scheduler.v1.RunJobH\x00\x12\x32\n\ncancel_job\x18\x03 \x01(\x0b\x32\x1c.cozy.scheduler.v1.CancelJobH\x00\x12.\n\x08model_op\x18\x04 \x01(\x0b\x32\x1a.cozy.scheduler.v1.ModelOpH\x00\x12)\n\x05\x64rain\x18\x05 \x01(\x0b\x32\x18.cozy.scheduler.v1.DrainH\x00\x12\x38\n\rtoken_refresh\x18\x06 \x01(\x0b\x32\x1f.cozy.scheduler.v1.TokenRefreshH\x00\x12\x38\n\rserve_posture\x18\x08 \x01(\x0b\x32\x1f.cozy.scheduler.v1.ServePostureH\x00\x42\x05\n\x03msgJ\x04\x08\x07\x10\x08R\x0brun_attempt\"\xb3\x03\n\x05Hello\x12<\n\x10protocol_version\x18\x01 \x01(\x0e\x32\".cozy.scheduler.v1.ProtocolVersion\x12\x11\n\tworker_id\x18\x02 \x01(\t\x12\x12\n\nrelease_id\x18\x03 \x01(\t\x12\x35\n\tresources\x18\x04 \x01(\x0b\x32\".cozy.scheduler.v1.WorkerResources\x12,\n\x05state\x18\x05 \x01(\x0b\x32\x1d.cozy.scheduler.v1.StateDelta\x12\x31\n\x06models\x18\x06 \x03(\x0b\x32!.cozy.scheduler.v1.ModelResidency\x12\x31\n\tin_flight\x18\x07 \x03(\x0b\x32\x1e.cozy.scheduler.v1.InFlightJob\x12\x1d\n\x15heartbeat_interval_ms\x18\x08 \x01(\x04\x12\x19\n\x11worker_session_id\x18\t \x01(\t\x12@\n\x12lifecycle_snapshot\x18\n \x01(\x0b\x32$.cozy.scheduler.v1.LifecycleSnapshot\"\x9b\x03\n\x0fWorkerResources\x12\x11\n\tgpu_count\x18\x01 \x01(\x05\x12\x18\n\x10vram_total_bytes\x18\x02 \x01(\x03\x12\x10\n\x08gpu_name\x18\x03 \x01(\t\x12\x0e\n\x06gpu_sm\x18\x04 \x01(\t\x12\x16\n\x0einstalled_libs\x18\x05 \x03(\t\x12\x14\n\x0cimage_digest\x18\x06 \x01(\t\x12\x12\n\ngit_commit\x18\x07 \x01(\t\x12\x13\n\x0binstance_id\x18\x08 \x01(\t\x12\x32\n\x0bhost_canary\x18\t \x01(\x0b\x32\x1d.cozy.scheduler.v1.HostCanary\x12\x15\n\rtorch_version\x18\n \x01(\t\x12\x1a\n\x12gen_worker_version\x18\x0b \x01(\t\x12\x16\n\x0e\x64river_version\x18\r \x01(\t\x12\x14\n\x0c\x63uda_version\x18\x0e \x01(\t\x12\x1f\n\x17\x63\x61pability_reason_class\x18\x0f \x01(\t\x12\x19\n\x11\x63\x61pability_detail\x18\x10 \x01(\tJ\x04\x08\x0c\x10\rR\x0bworker_mode\"\xa2\x02\n\x12HardwareUnsuitable\x12\x11\n\tworker_id\x18\x01 \x01(\t\x12\x12\n\nrelease_id\x18\x02 \x01(\t\x12\x14\n\x0creason_class\x18\x03 \x01(\t\x12\x0e\n\x06\x64\x65tail\x18\x04 \x01(\t\x12\x16\n\x0e\x64river_version\x18\x05 \x01(\t\x12\x10\n\x08gpu_name\x18\x06 \x01(\t\x12\x15\n\rtorch_version\x18\x07 \x01(\t\x12\x1a\n\x12torch_cuda_version\x18\x08 \x01(\t\x12\x1a\n\x12gen_worker_version\x18\t \x01(\t\x12\x14\n\x0cimage_digest\x18\n \x01(\t\x12\x13\n\x0binstance_id\x18\x0b \x01(\t\x12\x1b\n\x13reported_at_unix_ms\x18\x0c \x01(\x03\"\x88\x02\n\nHostCanary\x12\x13\n\x0bmemcpy_gbps\x18\x01 \x01(\x01\x12\x10\n\x08\x64\x32h_gbps\x18\x02 \x01(\x01\x12\x17\n\x0fpinned_alloc_ok\x18\x03 \x01(\x08\x12\x17\n\x0f\x63pu_single_mbps\x18\x04 \x01(\x01\x12\x16\n\x0e\x63pu_multi_mbps\x18\x05 \x01(\x01\x12\r\n\x05vcpus\x18\x06 \x01(\x05\x12\x14\n\x0cram_total_gb\x18\x07 \x01(\x01\x12\x13\n\x0b\x64uration_ms\x18\x08 \x01(\x03\x12\x14\n\x0cinterconnect\x18\t \x01(\t\x12\x11\n\tpeer_gbps\x18\n \x01(\x01\x12\x13\n\x0bpeer_access\x18\x0b \x01(\x08\x12\x11\n\ttopo_link\x18\x0c \x01(\t\"\x98\x01\n\x0eModelResidency\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12.\n\x04tier\x18\x02 \x01(\x0e\x32 .cozy.scheduler.v1.ResidencyTier\x12\x12\n\nvram_bytes\x18\x03 \x01(\x03\x12\x17\n\x0fsnapshot_digest\x18\x04 \x01(\t\x12\x1c\n\x14residency_generation\x18\x05 \x01(\x04\"2\n\x0bInFlightJob\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\"\x9f\x02\n\x08HelloAck\x12<\n\x10protocol_version\x18\x01 \x01(\x0e\x32\".cozy.scheduler.v1.ProtocolVersion\x12\x15\n\rfile_base_url\x18\x02 \x01(\t\x12\x37\n\x0bresolutions\x18\x03 \x03(\x0b\x32\".cozy.scheduler.v1.ModelResolution\x12>\n\x11\x64\x65sired_residency\x18\x04 \x01(\x0b\x32#.cozy.scheduler.v1.DesiredResidency\x12\x45\n\x15\x64\x65sired_state_command\x18\x05 \x01(\x0b\x32&.cozy.scheduler.v1.DesiredStateCommand\"\xaf\x02\n\x10\x44\x65siredResidency\x12\x12\n\ngeneration\x18\x01 \x01(\x04\x12\x11\n\tdisk_refs\x18\x02 \x03(\t\x12/\n\x03hot\x18\x03 \x03(\x0b\x32\".cozy.scheduler.v1.DesiredInstance\x12\x45\n\tsnapshots\x18\x04 \x03(\x0b\x32\x32.cozy.scheduler.v1.DesiredResidency.SnapshotsEntry\x12\x12\n\nrelease_id\x18\x05 \x01(\t\x12\x19\n\x11\x63onfig_generation\x18\x06 \x01(\x04\x1aM\n\x0eSnapshotsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.cozy.scheduler.v1.Snapshot:\x02\x38\x01\"\x99\x03\n\x13\x44\x65siredStateCommand\x12\x19\n\x11worker_session_id\x18\x01 \x01(\t\x12\x13\n\x0b\x63ommand_seq\x18\x02 \x01(\x04\x12\x0f\n\x07goal_id\x18\x03 \x01(\t\x12\x12\n\nrelease_id\x18\x04 \x01(\t\x12\x19\n\x11\x63onfig_generation\x18\x05 \x01(\x04\x12\x15\n\rconfig_digest\x18\x06 \x01(\x0c\x12\x19\n\x11issued_at_unix_ms\x18\x07 \x01(\x03\x12\x19\n\x11\x61\x63\x63\x65pt_by_unix_ms\x18\x08 \x01(\x03\x12\x1f\n\x17\x66irst_action_by_unix_ms\x18\t \x01(\x03\x12\x31\n\x07intents\x18\n \x03(\x0b\x32 .cozy.scheduler.v1.DesiredIntent\x12\x11\n\tmandatory\x18\x0b \x01(\x08\x12\x42\n\x16\x63hanged_config_classes\x18\x0c \x01(\x0b\x32\".cozy.scheduler.v1.ConfigClassMask\x12\x1a\n\x12parameter_snapshot\x18\r \x01(\x0c\"\x95\x03\n\rDesiredIntent\x12\x11\n\tintent_id\x18\x01 \x01(\t\x12\x32\n\x04kind\x18\x02 \x01(\x0e\x32$.cozy.scheduler.v1.DesiredIntentKind\x12\x34\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32%.cozy.scheduler.v1.DesiredIntentCause\x12\x15\n\rfunction_name\x18\x04 \x01(\t\x12\x0b\n\x03ref\x18\x05 \x01(\t\x12\x17\n\x0fsnapshot_digest\x18\x06 \x01(\x0c\x12\x36\n\x0c\x64\x65sired_tier\x18\x07 \x01(\x0e\x32 .cozy.scheduler.v1.ResidencyTier\x12\x16\n\x0e\x62inding_digest\x18\x08 \x01(\x0c\x12\x18\n\x10parent_intent_id\x18\t \x01(\t\x12;\n\x10waiting_requests\x18\n \x03(\x0b\x32!.cozy.scheduler.v1.RequestAttempt\x12\x10\n\x08priority\x18\x0b \x01(\x05\x12\x11\n\tmandatory\x18\x0c \x01(\x08\"5\n\x0eRequestAttempt\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\"\xd0\x02\n\x0bGoalReceipt\x12\x19\n\x11worker_session_id\x18\x01 \x01(\t\x12\x13\n\x0b\x63ommand_seq\x18\x02 \x01(\x04\x12\x0f\n\x07goal_id\x18\x03 \x01(\t\x12\x12\n\nrelease_id\x18\x04 \x01(\t\x12\x34\n\x06status\x18\x05 \x01(\x0e\x32$.cozy.scheduler.v1.GoalReceiptStatus\x12\x39\n\nerror_code\x18\x06 \x01(\x0e\x32%.cozy.scheduler.v1.LifecycleErrorCode\x12\x36\n\nrejections\x18\x07 \x03(\x0b\x32\".cozy.scheduler.v1.IntentRejection\x12\x0e\n\x06\x64\x65tail\x18\x08 \x01(\t\x12\x1b\n\x13received_at_unix_ms\x18\t \x01(\x03\x12\x16\n\x0e\x63ommand_digest\x18\n \x01(\x0c\"o\n\x0fIntentRejection\x12\x11\n\tintent_id\x18\x01 \x01(\t\x12\x39\n\nerror_code\x18\x02 \x01(\x0e\x32%.cozy.scheduler.v1.LifecycleErrorCode\x12\x0e\n\x06\x64\x65tail\x18\x03 \x01(\t\"R\n\x11LifecycleProgress\x12\x0c\n\x04\x64one\x18\x01 \x01(\x01\x12\r\n\x05total\x18\x02 \x01(\x01\x12\x0c\n\x04unit\x18\x03 \x01(\t\x12\x12\n\nrate_per_s\x18\x04 \x01(\x01\"\x98\x05\n\x0bIntentState\x12\x19\n\x11worker_session_id\x18\x01 \x01(\t\x12\x11\n\tstate_seq\x18\x02 \x01(\x04\x12\x0f\n\x07goal_id\x18\x03 \x01(\t\x12\x11\n\tintent_id\x18\x04 \x01(\t\x12\x12\n\nrelease_id\x18\x05 \x01(\t\x12\x19\n\x11\x63onfig_generation\x18\x06 \x01(\x04\x12\x38\n\x06status\x18\x07 \x01(\x0e\x32(.cozy.scheduler.v1.LifecycleIntentStatus\x12\x36\n\x05stage\x18\x08 \x01(\x0e\x32\'.cozy.scheduler.v1.LifecycleIntentStage\x12\x36\n\x06reason\x18\t \x01(\x0e\x32&.cozy.scheduler.v1.LifecycleWaitReason\x12\x15\n\rsince_unix_ms\x18\n \x01(\x03\x12\x1a\n\x12updated_at_unix_ms\x18\x0b \x01(\x03\x12\x1d\n\x15next_retry_at_unix_ms\x18\x0c \x01(\x03\x12\x1b\n\x13\x64\x65\x61\x64line_at_unix_ms\x18\r \x01(\x03\x12\x19\n\x11\x62locker_intent_id\x18\x0e \x01(\t\x12:\n\x0f\x62locker_request\x18\x0f \x01(\x0b\x32!.cozy.scheduler.v1.RequestAttempt\x12\x36\n\x08progress\x18\x10 \x01(\x0b\x32$.cozy.scheduler.v1.LifecycleProgress\x12\x39\n\nerror_code\x18\x11 \x01(\x0e\x32%.cozy.scheduler.v1.LifecycleErrorCode\x12\x0e\n\x06\x64\x65tail\x18\x12 \x01(\t\x12\x15\n\ractual_digest\x18\x13 \x01(\x0c\"\x83\x01\n\rModelIdentity\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12\x17\n\x0fsnapshot_digest\x18\x02 \x01(\x0c\x12.\n\x04tier\x18\x03 \x01(\x0e\x32 .cozy.scheduler.v1.ResidencyTier\x12\x1c\n\x14residency_generation\x18\x04 \x01(\x04\"\xaa\x02\n\x12\x46unctionCapability\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12\x12\n\nrelease_id\x18\x02 \x01(\t\x12\x19\n\x11\x63onfig_generation\x18\x03 \x01(\x04\x12\x16\n\x0e\x62inding_digest\x18\x04 \x01(\x0c\x12\x0c\n\x04lane\x18\x05 \x01(\t\x12\x30\n\x06models\x18\x06 \x03(\x0b\x32 .cozy.scheduler.v1.ModelIdentity\x12%\n\x1d\x63ompile_target_incarnation_id\x18\x07 \x01(\t\x12\x39\n\x05state\x18\x08 \x01(\x0e\x32*.cozy.scheduler.v1.FunctionCapabilityState\x12\x14\n\x0cserving_tier\x18\t \x01(\t\"E\n\x0f\x43onfigClassMask\x12\x12\n\nparameters\x18\x01 \x01(\x08\x12\x10\n\x08\x62indings\x18\x02 \x01(\x08\x12\x0c\n\x04\x62oot\x18\x03 \x01(\x08\"\xf3\x02\n\x11\x43onfigApplication\x12\x12\n\nrelease_id\x18\x01 \x01(\t\x12\x19\n\x11target_generation\x18\x02 \x01(\x04\x12\x1b\n\x13received_generation\x18\x03 \x01(\x04\x12%\n\x1dparameter_snapshot_generation\x18\x04 \x01(\x04\x12 \n\x18\x62inding_ready_generation\x18\x05 \x01(\x04\x12\x17\n\x0f\x62oot_generation\x18\x06 \x01(\x04\x12\x38\n\x05state\x18\x07 \x01(\x0e\x32).cozy.scheduler.v1.ConfigApplicationState\x12;\n\x0fpending_classes\x18\x08 \x01(\x0b\x32\".cozy.scheduler.v1.ConfigClassMask\x12\x39\n\nerror_code\x18\t \x01(\x0e\x32%.cozy.scheduler.v1.LifecycleErrorCode\"\x89\x02\n\x0f\x44rainProjection\x12\x0f\n\x07goal_id\x18\x01 \x01(\t\x12\x11\n\tintent_id\x18\x02 \x01(\t\x12\x37\n\x06status\x18\x03 \x01(\x0e\x32\'.cozy.scheduler.v1.DrainLifecycleStatus\x12\x15\n\rsince_unix_ms\x18\x04 \x01(\x03\x12\x1a\n\x12updated_at_unix_ms\x18\x05 \x01(\x03\x12\x1b\n\x13\x64\x65\x61\x64line_at_unix_ms\x18\x06 \x01(\x03\x12\x39\n\nerror_code\x18\x07 \x01(\x0e\x32%.cozy.scheduler.v1.LifecycleErrorCode\x12\x0e\n\x06\x64\x65tail\x18\x08 \x01(\t\"\x8f\x03\n\x11LifecycleSnapshot\x12\x19\n\x11worker_session_id\x18\x01 \x01(\t\x12\x11\n\tstate_seq\x18\x02 \x01(\x04\x12/\n\x07intents\x18\x03 \x03(\x0b\x32\x1e.cozy.scheduler.v1.IntentState\x12;\n\x0c\x63\x61pabilities\x18\x04 \x03(\x0b\x32%.cozy.scheduler.v1.FunctionCapability\x12@\n\x12\x63onfig_application\x18\x05 \x01(\x0b\x32$.cozy.scheduler.v1.ConfigApplication\x12\x35\n\rgoal_receipts\x18\x06 \x03(\x0b\x32\x1e.cozy.scheduler.v1.GoalReceipt\x12\x31\n\x05\x64rain\x18\x07 \x01(\x0b\x32\".cozy.scheduler.v1.DrainProjection\x12\x14\n\x0c\x66ull_replace\x18\x08 \x01(\x08\x12\x1c\n\x14generated_at_unix_ms\x18\t \x01(\x03\"Y\n\x0f\x44\x65siredInstance\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12/\n\x06models\x18\x02 \x03(\x0b\x32\x1f.cozy.scheduler.v1.ModelBinding\"e\n\x0fModelResolution\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12\x14\n\x0cresolved_ref\x18\x02 \x01(\t\x12\x0c\n\x04\x63\x61st\x18\x03 \x01(\t\x12\x0c\n\x04lane\x18\x04 \x01(\t\x12\x13\n\x0blane_pinned\x18\x05 \x01(\x08\"\xab\x03\n\nStateDelta\x12-\n\x05phase\x18\x01 \x01(\x0e\x32\x1e.cozy.scheduler.v1.WorkerPhase\x12\x1b\n\x13\x61vailable_functions\x18\x02 \x03(\t\x12\x19\n\x11loading_functions\x18\x03 \x03(\t\x12\x17\n\x0f\x66ree_vram_bytes\x18\x04 \x01(\x03\x12\x17\n\x0f\x66inalizing_jobs\x18\x05 \x01(\x05\x12%\n\x1dobserved_residency_generation\x18\x06 \x01(\x04\x12\x39\n\x0f\x63ompile_targets\x18\x07 \x03(\x0b\x32 .cozy.scheduler.v1.CompileTarget\x12\x46\n\x16\x63ompiled_graph_lookups\x18\x08 \x03(\x0b\x32&.cozy.scheduler.v1.CompiledGraphLookup\x12\x36\n\ndisk_usage\x18\t \x01(\x0b\x32\".cozy.scheduler.v1.DiskUsageReport\x12\"\n\x1aobserved_config_generation\x18\n \x01(\x04\"\xac\x01\n\x10StorageTierUsage\x12,\n\x04tier\x18\x01 \x01(\x0e\x32\x1e.cozy.scheduler.v1.StorageTier\x12\x12\n\nmount_path\x18\x02 \x01(\t\x12\x13\n\x0btotal_bytes\x18\x03 \x01(\x03\x12\x12\n\nfree_bytes\x18\x04 \x01(\x03\x12\x12\n\nused_bytes\x18\x05 \x01(\x03\x12\x19\n\x11reclaimable_bytes\x18\x06 \x01(\x03\"b\n\x0f\x44iskUsageReport\x12\x32\n\x05tiers\x18\x01 \x03(\x0b\x32#.cozy.scheduler.v1.StorageTierUsage\x12\x1b\n\x13\x63\x61pacity_generation\x18\x02 \x01(\x04\"A\n\x13\x43ompiledGraphLookup\x12\x0e\n\x06\x66\x61mily\x18\x01 \x01(\t\x12\x1a\n\x12\x63ompiled_graph_key\x18\x02 \x01(\t\"\xe1\x02\n\rCompileTarget\x12\x16\n\x0eincarnation_id\x18\x01 \x01(\t\x12\x0e\n\x06\x66\x61mily\x18\x02 \x01(\t\x12\x1c\n\x14pipeline_weight_lane\x18\x03 \x01(\t\x12\x13\n\x0blora_bucket\x18\x04 \x01(\x05\x12\x17\n\x0f\x63ontract_digest\x18\x05 \x01(\t\x12\x1a\n\x12\x61\x63tive_compile_ref\x18\x06 \x01(\t\x12&\n\x1e\x61\x63tive_compile_snapshot_digest\x18\x07 \x01(\t\x12\x16\n\x0e\x66unction_names\x18\x08 \x03(\t\x12?\n\x0emodel_bindings\x18\t \x03(\x0b\x32\'.cozy.scheduler.v1.CompileTargetBinding\x12$\n\x1crequested_compiled_graph_key\x18\n \x01(\tJ\x04\x08\x0b\x10\x0cR\x13requested_cell_axes\"J\n\x14\x43ompileTargetBinding\x12\x0c\n\x04slot\x18\x01 \x01(\t\x12\x0b\n\x03ref\x18\x02 \x01(\t\x12\x17\n\x0fsnapshot_digest\x18\x03 \x01(\t\"\x9a\x06\n\x06RunJob\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\x12\x15\n\rfunction_name\x18\x03 \x01(\t\x12\x15\n\rinput_payload\x18\x04 \x01(\x0c\x12\x12\n\ntimeout_ms\x18\x05 \x01(\x03\x12\x0b\n\x03org\x18\x06 \x01(\t\x12\x12\n\ninvoker_id\x18\x07 \x01(\t\x12\x18\n\x10\x63\x61pability_token\x18\x08 \x01(\t\x12\x32\n\x0bmedia_bytes\x18\t \x01(\x0e\x32\x1d.cozy.scheduler.v1.MediaBytes\x12\x33\n\x07\x63ompute\x18\n \x01(\x0b\x32\".cozy.scheduler.v1.ResolvedCompute\x12/\n\x06models\x18\x0b \x03(\x0b\x32\x1f.cozy.scheduler.v1.ModelBinding\x12;\n\tsnapshots\x18\x0c \x03(\x0b\x32(.cozy.scheduler.v1.RunJob.SnapshotsEntry\x12\x45\n\x10required_compile\x18\r \x01(\x0b\x32+.cozy.scheduler.v1.RequiredCompileExecution\x12\x0c\n\x04lane\x18\x0e \x01(\t\x12\x33\n\x0cinput_assets\x18\x0f \x03(\x0b\x32\x1d.cozy.scheduler.v1.InputAsset\x12\x19\n\x11\x63onfig_generation\x18\x10 \x01(\x04\x12\x15\n\rconfig_params\x18\x11 \x01(\x0c\x12\x39\n\x0bintent_kind\x18\x12 \x01(\x0e\x32$.cozy.scheduler.v1.DesiredIntentKind\x12\x11\n\tintent_id\x18\x13 \x01(\t\x12\x0f\n\x07goal_id\x18\x14 \x01(\t\x12\x16\n\x0ephase_budget_s\x18\x15 \x01(\x03\x12\x15\n\rfile_base_url\x18\x16 \x01(\t\x1aM\n\x0eSnapshotsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.cozy.scheduler.v1.Snapshot:\x02\x38\x01\"w\n\nInputAsset\x12\x10\n\x08\x61sset_id\x18\x01 \x01(\t\x12\x12\n\nsource_ref\x18\x02 \x01(\t\x12\x0e\n\x06\x62lake3\x18\x03 \x01(\t\x12\x12\n\nsize_bytes\x18\x04 \x01(\x03\x12\x0c\n\x04kind\x18\x05 \x01(\t\x12\x11\n\tmime_type\x18\x06 \x01(\t\"\x96\x01\n\x18RequiredCompileExecution\x12\x1d\n\x15target_incarnation_id\x18\x01 \x01(\t\x12\x1a\n\x12\x63ompiled_graph_ref\x18\x02 \x01(\t\x12&\n\x1e\x63ompiled_graph_snapshot_digest\x18\x03 \x01(\t\x12\x17\n\x0f\x63ontract_digest\x18\x04 \x01(\t\"9\n\x0fResolvedCompute\x12\x13\n\x0b\x61\x63\x63\x65lerator\x18\x01 \x01(\t\x12\x11\n\tgpu_index\x18\x02 \x01(\x05\"\xee\x01\n\x0cModelBinding\x12\x0c\n\x04slot\x18\x01 \x01(\t\x12\x0b\n\x03ref\x18\x02 \x01(\t\x12-\n\x05loras\x18\x03 \x03(\x0b\x32\x1e.cozy.scheduler.v1.LoraOverlay\x12\x1a\n\x12inference_defaults\x18\x04 \x01(\t\x12\x11\n\tobjective\x18\x06 \x01(\t\x12\x11\n\tdistilled\x18\x07 \x01(\x08\x12\x18\n\x10\x64istilled_status\x18\x08 \x01(\t\x12\r\n\x05model\x18\t \x01(\t\x12\x17\n\x0fmanifest_digest\x18\n \x01(\tJ\x04\x08\x05\x10\x06R\ncomponents\"U\n\x0bLoraOverlay\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12\x0e\n\x06weight\x18\x02 \x01(\x01\x12\x1a\n\x12inference_defaults\x18\x03 \x01(\t\x12\r\n\x05model\x18\x04 \x01(\t\"\x85\x01\n\x08Snapshot\x12\x0e\n\x06\x64igest\x18\x01 \x01(\t\x12.\n\x05\x66iles\x18\x02 \x03(\x0b\x32\x1f.cozy.scheduler.v1.SnapshotFile\x12\x39\n\nprovenance\x18\x03 \x01(\x0b\x32%.cozy.scheduler.v1.ResolvedProvenance\"\xfc\x01\n\x12ResolvedProvenance\x12\x0c\n\x04repo\x18\x01 \x01(\t\x12\x0f\n\x07release\x18\x02 \x01(\t\x12\x12\n\nvariant_id\x18\x03 \x01(\t\x12V\n\x11\x63omponent_sources\x18\x04 \x03(\x0b\x32;.cozy.scheduler.v1.ResolvedProvenance.ComponentSourcesEntry\x1a[\n\x15\x43omponentSourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x31\n\x05value\x18\x02 \x01(\x0b\x32\".cozy.scheduler.v1.ComponentOrigin:\x02\x38\x01\"D\n\x0f\x43omponentOrigin\x12\x0c\n\x04repo\x18\x01 \x01(\t\x12\x0f\n\x07release\x18\x02 \x01(\t\x12\x12\n\nvariant_id\x18\x03 \x01(\t\"\xa4\x01\n\x0cSnapshotFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x12\n\nsize_bytes\x18\x02 \x01(\x03\x12\x0e\n\x06\x62lake3\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\x0e\n\x06\x64igest\x18\x05 \x01(\t\x12\x18\n\x10\x63hunk_size_bytes\x18\x06 \x01(\x03\x12+\n\x06\x63hunks\x18\x07 \x03(\x0b\x32\x1b.cozy.scheduler.v1.ChunkRef\"4\n\x08\x43hunkRef\x12\x0e\n\x06sha256\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12\x0b\n\x03len\x18\x03 \x01(\x03\"2\n\x0bJobAccepted\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\"\x9b\x02\n\tJobResult\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\x12,\n\x06status\x18\x03 \x01(\x0e\x32\x1c.cozy.scheduler.v1.JobStatus\x12\x10\n\x06inline\x18\x04 \x01(\x0cH\x00\x12\x12\n\x08\x62lob_ref\x18\x05 \x01(\tH\x00\x12\x14\n\x0csafe_message\x18\x06 \x01(\t\x12.\n\x07metrics\x18\x07 \x01(\x0b\x32\x1d.cozy.scheduler.v1.JobMetrics\x12\x32\n\x0b\x61\x64justments\x18\x08 \x03(\x0b\x32\x1d.cozy.scheduler.v1.Adjustment\x12\x11\n\ttraceback\x18\t \x01(\tB\x08\n\x06output\"O\n\nAdjustment\x12\r\n\x05\x66ield\x18\x01 \x01(\t\x12\x11\n\trequested\x18\x02 \x01(\t\x12\x0f\n\x07\x61pplied\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\"\x8d\x06\n\nJobMetrics\x12\x12\n\nruntime_ms\x18\x01 \x01(\x03\x12\x10\n\x08queue_ms\x18\x02 \x01(\x03\x12\x18\n\x10rss_at_end_bytes\x18\x03 \x01(\x03\x12\x17\n\x0fpeak_vram_bytes\x18\x04 \x01(\x03\x12\x1c\n\x14\x63oncurrency_at_start\x18\x05 \x01(\x05\x12\x1f\n\x17output_media_duration_s\x18\x06 \x01(\x01\x12\x14\n\x0cinput_tokens\x18\x07 \x01(\x03\x12\x1b\n\x13input_cached_tokens\x18\x08 \x01(\x03\x12\x15\n\routput_tokens\x18\t \x01(\x03\x12\x14\n\x0coutput_count\x18\n \x01(\x03\x12\x14\n\x0cslot_held_ms\x18\x0b \x01(\x03\x12\x18\n\x10\x66inalize_wall_ms\x18\x0c \x01(\x03\x12\x0c\n\x04lane\x18\r \x01(\t\x12\x46\n\rruntime_terms\x18\x0e \x03(\x0b\x32/.cozy.scheduler.v1.JobMetrics.RuntimeTermsEntry\x12<\n\x08stage_ms\x18\x0f \x03(\x0b\x32*.cozy.scheduler.v1.JobMetrics.StageMsEntry\x12\x14\n\x0cserving_mode\x18\x10 \x01(\t\x12!\n\x19served_compiled_graph_ref\x18\x11 \x01(\t\x12\x1d\n\x15served_eager_fallback\x18\x12 \x01(\x08\x12\x17\n\x0f\x66\x61llback_reason\x18\x13 \x01(\t\x12\n\n\x02sm\x18\x14 \x01(\t\x12\r\n\x05steps\x18\x15 \x01(\x03\x12\r\n\x05width\x18\x16 \x01(\x03\x12\x0e\n\x06height\x18\x17 \x01(\x03\x12\x33\n\x07posture\x18\x18 \x01(\x0b\x32\".cozy.scheduler.v1.MeasuredPosture\x1a\x33\n\x11RuntimeTermsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x01:\x02\x38\x01\x1a.\n\x0cStageMsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x03:\x02\x38\x01\"\xdb\x02\n\x0fMeasuredPosture\x12\x16\n\x0e\x65xecution_lane\x18\x01 \x01(\t\x12\x19\n\x11\x61ttention_backend\x18\x02 \x01(\t\x12 \n\x18\x61ttention_backend_wanted\x18\x03 \x01(\t\x12\x15\n\rcompile_state\x18\x04 \x01(\t\x12\x1c\n\x14\x63ompile_state_wanted\x18\x05 \x01(\t\x12\x16\n\x0eresidency_mode\x18\x06 \x01(\t\x12\x34\n\x07\x61pplied\x18\x07 \x03(\x0b\x32#.cozy.scheduler.v1.AppliedTechnique\x12\x37\n\ncomponents\x18\x08 \x03(\x0b\x32#.cozy.scheduler.v1.ComponentPosture\x12\x37\n\tshortfall\x18\t \x01(\x0b\x32$.cozy.scheduler.v1.ResourceShortfall\"i\n\x10\x41ppliedTechnique\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tcomponent\x18\x02 \x01(\t\x12\x0e\n\x06wanted\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x14\n\x0c\x65st_slowdown\x18\x05 \x01(\x01\"s\n\x10\x43omponentPosture\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x15\n\rapplied_quant\x18\x02 \x01(\t\x12\x13\n\x0b\x62ound_quant\x18\x03 \x01(\t\x12\x11\n\tplacement\x18\x04 \x01(\t\x12\r\n\x05\x62ytes\x18\x05 \x01(\x03\"g\n\x11ResourceShortfall\x12\x10\n\x08resource\x18\x01 \x01(\t\x12\x11\n\tcomponent\x18\x02 \x01(\t\x12\x14\n\x0cneeded_bytes\x18\x03 \x01(\x03\x12\x17\n\x0f\x61vailable_bytes\x18\x04 \x01(\x03\"c\n\x0bJobProgress\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\x12\x0b\n\x03seq\x18\x03 \x01(\x03\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x05 \x01(\t\"0\n\tCancelJob\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\"\xa6\x01\n\x07ModelOp\x12*\n\x02op\x18\x01 \x01(\x0e\x32\x1e.cozy.scheduler.v1.ModelOpKind\x12\x0b\n\x03ref\x18\x02 \x01(\t\x12-\n\x08snapshot\x18\x03 \x01(\x0b\x32\x1b.cozy.scheduler.v1.Snapshot\x12\x14\n\x0coperation_id\x18\x04 \x01(\t\x12\x1d\n\x15target_incarnation_id\x18\x05 \x01(\t\"\x9e\x04\n\nModelEvent\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12,\n\x05state\x18\x02 \x01(\x0e\x32\x1d.cozy.scheduler.v1.ModelState\x12\x12\n\nvram_bytes\x18\x03 \x01(\x03\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x12\n\nbytes_done\x18\x05 \x01(\x03\x12\x13\n\x0b\x62ytes_total\x18\x06 \x01(\x03\x12\x13\n\x0b\x64uration_ms\x18\x07 \x01(\x03\x12\x12\n\ncache_hits\x18\x08 \x01(\x03\x12\x14\n\x0c\x63\x61\x63he_misses\x18\t \x01(\x03\x12\x10\n\x08warmup_s\x18\n \x01(\x01\x12\x1f\n\x17host_ram_required_bytes\x18\x0b \x01(\x03\x12\'\n\x1fhost_ram_available_before_bytes\x18\x0c \x01(\x03\x12&\n\x1ehost_ram_available_after_bytes\x18\r \x01(\x03\x12\x1d\n\x15host_ram_evicted_refs\x18\x0e \x03(\t\x12$\n\x1chost_ram_capacity_generation\x18\x0f \x01(\x04\x12\x17\n\x0fsnapshot_digest\x18\x10 \x01(\t\x12\x1c\n\x14residency_generation\x18\x11 \x01(\x04\x12\x14\n\x0coperation_id\x18\x12 \x01(\t\x12\x1d\n\x15target_incarnation_id\x18\x13 \x01(\t\x12\x15\n\rnetwork_bytes\x18\x14 \x01(\x03\"\xbe\x03\n\x0e\x41\x63tivityUpdate\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12\r\n\x05phase\x18\x02 \x01(\t\x12\x0c\n\x04step\x18\x03 \x01(\x03\x12\x13\n\x0btotal_steps\x18\x04 \x01(\x03\x12\x0b\n\x03seq\x18\x05 \x01(\x04\x12/\n\x05state\x18\x06 \x01(\x0e\x32 .cozy.scheduler.v1.ActivityState\x12\r\n\x05\x65rror\x18\x07 \x01(\t\x12\x0e\n\x06\x64\x65tail\x18\x08 \x01(\t\x12\x1a\n\x12updated_at_unix_ms\x18\t \x01(\x03\x12\x0f\n\x07\x63ounter\x18\n \x01(\t\x12\x14\n\x0c\x63ounter_unit\x18\x0b \x01(\t\x12\x14\n\x0c\x63ounter_done\x18\x0c \x01(\x01\x12\x15\n\rcounter_total\x18\r \x01(\x01\x12\x12\n\nrate_per_s\x18\x0e \x01(\x01\x12\x14\n\x0cself_stalled\x18\x0f \x01(\x08\x12\x16\n\x0estalled_for_ms\x18\x10 \x01(\x03\x12\x13\n\x0b\x64uration_ms\x18\x11 \x01(\x03\x12\x0e\n\x06\x66\x61mily\x18\x12 \x01(\t\x12\x1a\n\x12\x63ompiled_graph_key\x18\x13 \x01(\t\x12\x1c\n\x14graph_specialization\x18\x14 \x01(\t\"\xe2\x02\n\tBootPhase\x12\x0f\n\x07\x62oot_id\x18\x01 \x01(\t\x12\x0f\n\x07ordinal\x18\x02 \x01(\x03\x12\x16\n\x0eparent_ordinal\x18\x03 \x01(\x03\x12\r\n\x05phase\x18\x04 \x01(\t\x12\x10\n\x08terminal\x18\x05 \x01(\x08\x12\x1a\n\x12started_at_unix_ms\x18\x06 \x01(\x03\x12\x13\n\x0b\x64uration_ms\x18\x07 \x01(\x03\x12\x19\n\x11process_uptime_ms\x18\x08 \x01(\x03\x12\r\n\x05\x62ytes\x18\t \x01(\x03\x12\x0e\n\x06source\x18\n \x01(\t\x12\x0b\n\x03ref\x18\x0b \x01(\t\x12\x15\n\rartifact_kind\x18\x0c \x01(\t\x12\x14\n\x0c\x61rtifact_key\x18\r \x01(\t\x12\x10\n\x08\x66unction\x18\x0e \x01(\t\x12\x0f\n\x07outcome\x18\x0f \x01(\t\x12\x0e\n\x06reason\x18\x10 \x01(\t\x12\x0e\n\x06\x64\x65tail\x18\x11 \x01(\t\x12\x12\n\ncumulative\x18\x12 \x01(\x08\"\xad\x01\n\rFnUnavailable\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x0e\n\x06\x64\x65tail\x18\x03 \x01(\t\x12\x38\n\x04\x61xes\x18\x04 \x03(\x0b\x32*.cozy.scheduler.v1.FnUnavailable.AxesEntry\x1a+\n\tAxesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"p\n\nFnDegraded\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12\x0e\n\x06wanted\x18\x02 \x01(\t\x12\x0b\n\x03ran\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x1e\n\x16\x65st_latency_multiplier\x18\x05 \x01(\x01\"\x1c\n\x05\x44rain\x12\x13\n\x0b\x64\x65\x61\x64line_ms\x18\x01 \x01(\x03\"6\n\x0cTokenRefresh\x12\r\n\x05token\x18\x01 \x01(\t\x12\x17\n\x0f\x65xpires_at_unix\x18\x02 \x01(\x03\"A\n\x0cServePosture\x12\x12\n\neager_only\x18\x01 \x01(\x08\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\r\n\x05\x61\x63tor\x18\x03 \x01(\t*Q\n\x0fProtocolVersion\x12 \n\x1cPROTOCOL_VERSION_UNSPECIFIED\x10\x00\x12\x1c\n\x18PROTOCOL_VERSION_CURRENT\x10\x01*y\n\rResidencyTier\x12\x1e\n\x1aRESIDENCY_TIER_UNSPECIFIED\x10\x00\x12\x17\n\x13RESIDENCY_TIER_DISK\x10\x01\x12\x16\n\x12RESIDENCY_TIER_RAM\x10\x02\x12\x17\n\x13RESIDENCY_TIER_VRAM\x10\x03*\x92\x02\n\x11\x44\x65siredIntentKind\x12#\n\x1f\x44\x45SIRED_INTENT_KIND_UNSPECIFIED\x10\x00\x12#\n\x1f\x44\x45SIRED_INTENT_KIND_MATERIALIZE\x10\x01\x12&\n\"DESIRED_INTENT_KIND_FUNCTION_READY\x10\x02\x12$\n DESIRED_INTENT_KIND_CONFIG_APPLY\x10\x03\x12%\n!DESIRED_INTENT_KIND_COMPILE_ADOPT\x10\x04\x12\x1d\n\x19\x44\x45SIRED_INTENT_KIND_DRAIN\x10\x05\x12\x1f\n\x1b\x44\x45SIRED_INTENT_KIND_RUN_JOB\x10\x06*\x95\x02\n\x12\x44\x65siredIntentCause\x12$\n DESIRED_INTENT_CAUSE_UNSPECIFIED\x10\x00\x12\"\n\x1e\x44\x45SIRED_INTENT_CAUSE_COLD_BOOT\x10\x01\x12 \n\x1c\x44\x45SIRED_INTENT_CAUSE_REQUEST\x10\x02\x12$\n DESIRED_INTENT_CAUSE_PREPOSITION\x10\x03\x12&\n\"DESIRED_INTENT_CAUSE_CONFIG_CHANGE\x10\x04\x12$\n DESIRED_INTENT_CAUSE_REPLACEMENT\x10\x05\x12\x1f\n\x1b\x44\x45SIRED_INTENT_CAUSE_RETIRE\x10\x06*|\n\x11GoalReceiptStatus\x12#\n\x1fGOAL_RECEIPT_STATUS_UNSPECIFIED\x10\x00\x12 \n\x1cGOAL_RECEIPT_STATUS_ACCEPTED\x10\x01\x12 \n\x1cGOAL_RECEIPT_STATUS_REJECTED\x10\x02*\xc9\x02\n\x15LifecycleIntentStatus\x12\'\n#LIFECYCLE_INTENT_STATUS_UNSPECIFIED\x10\x00\x12$\n LIFECYCLE_INTENT_STATUS_ACCEPTED\x10\x01\x12#\n\x1fLIFECYCLE_INTENT_STATUS_WAITING\x10\x02\x12#\n\x1fLIFECYCLE_INTENT_STATUS_RUNNING\x10\x03\x12%\n!LIFECYCLE_INTENT_STATUS_SUCCEEDED\x10\x04\x12\"\n\x1eLIFECYCLE_INTENT_STATUS_FAILED\x10\x05\x12$\n LIFECYCLE_INTENT_STATUS_CANCELED\x10\x06\x12&\n\"LIFECYCLE_INTENT_STATUS_SUPERSEDED\x10\x07*\xc9\x08\n\x14LifecycleIntentStage\x12&\n\"LIFECYCLE_INTENT_STAGE_UNSPECIFIED\x10\x00\x12%\n!LIFECYCLE_INTENT_STAGE_VALIDATING\x10\x01\x12+\n\'LIFECYCLE_INTENT_STAGE_WAIT_TENANT_IDLE\x10\x02\x12(\n$LIFECYCLE_INTENT_STAGE_WAIT_REF_LOCK\x10\x03\x12)\n%LIFECYCLE_INTENT_STAGE_WAIT_LOAD_LOCK\x10\x04\x12(\n$LIFECYCLE_INTENT_STAGE_WAIT_GPU_SLOT\x10\x05\x12(\n$LIFECYCLE_INTENT_STAGE_WAIT_SNAPSHOT\x10\x06\x12-\n)LIFECYCLE_INTENT_STAGE_WAIT_DISK_HEADROOM\x10\x07\x12(\n$LIFECYCLE_INTENT_STAGE_WAIT_HOST_RAM\x10\x08\x12-\n)LIFECYCLE_INTENT_STAGE_WAIT_NETWORK_RETRY\x10\t\x12+\n\'LIFECYCLE_INTENT_STAGE_WAIT_REPLACEMENT\x10\n\x12#\n\x1fLIFECYCLE_INTENT_STAGE_FETCHING\x10\x0b\x12$\n LIFECYCLE_INTENT_STAGE_VERIFYING\x10\x0c\x12\"\n\x1eLIFECYCLE_INTENT_STAGE_ON_DISK\x10\r\x12\'\n#LIFECYCLE_INTENT_STAGE_LOADING_HOST\x10\x0e\x12)\n%LIFECYCLE_INTENT_STAGE_LOADING_DEVICE\x10\x0f\x12\"\n\x1eLIFECYCLE_INTENT_STAGE_WARMING\x10\x10\x12$\n LIFECYCLE_INTENT_STAGE_COMPILING\x10\x11\x12#\n\x1fLIFECYCLE_INTENT_STAGE_ADOPTING\x10\x12\x12/\n+LIFECYCLE_INTENT_STAGE_CONFIG_MATERIALIZING\x10\x13\x12\x33\n/LIFECYCLE_INTENT_STAGE_CONFIG_BINDINGS_APPLYING\x10\x14\x12,\n(LIFECYCLE_INTENT_STAGE_CONFIG_BOOT_STALE\x10\x15\x12 \n\x1cLIFECYCLE_INTENT_STAGE_READY\x10\x16\x12#\n\x1fLIFECYCLE_INTENT_STAGE_DRAINING\x10\x17\x12%\n!LIFECYCLE_INTENT_STAGE_FINALIZING\x10\x18\x12#\n\x1fLIFECYCLE_INTENT_STAGE_FLUSHING\x10\x19*\xc0\x03\n\x13LifecycleWaitReason\x12%\n!LIFECYCLE_WAIT_REASON_UNSPECIFIED\x10\x00\x12%\n!LIFECYCLE_WAIT_REASON_TENANT_WORK\x10\x01\x12\"\n\x1eLIFECYCLE_WAIT_REASON_REF_LOCK\x10\x02\x12#\n\x1fLIFECYCLE_WAIT_REASON_LOAD_LOCK\x10\x03\x12\"\n\x1eLIFECYCLE_WAIT_REASON_GPU_SLOT\x10\x04\x12\"\n\x1eLIFECYCLE_WAIT_REASON_SNAPSHOT\x10\x05\x12\'\n#LIFECYCLE_WAIT_REASON_DISK_HEADROOM\x10\x06\x12\"\n\x1eLIFECYCLE_WAIT_REASON_HOST_RAM\x10\x07\x12\'\n#LIFECYCLE_WAIT_REASON_NETWORK_RETRY\x10\x08\x12%\n!LIFECYCLE_WAIT_REASON_REPLACEMENT\x10\t\x12-\n)LIFECYCLE_WAIT_REASON_SINGLE_FLIGHT_OWNER\x10\n*\xa6\x05\n\x12LifecycleErrorCode\x12$\n LIFECYCLE_ERROR_CODE_UNSPECIFIED\x10\x00\x12\x30\n,LIFECYCLE_ERROR_CODE_MISSING_MANDATORY_FIELD\x10\x01\x12\x30\n,LIFECYCLE_ERROR_CODE_WORKER_SESSION_MISMATCH\x10\x02\x12)\n%LIFECYCLE_ERROR_CODE_RELEASE_MISMATCH\x10\x03\x12/\n+LIFECYCLE_ERROR_CODE_COMMAND_SEQ_REGRESSION\x10\x04\x12-\n)LIFECYCLE_ERROR_CODE_COMMAND_SEQ_CONFLICT\x10\x05\x12*\n&LIFECYCLE_ERROR_CODE_CONFIG_REGRESSION\x10\x06\x12)\n%LIFECYCLE_ERROR_CODE_UNKNOWN_FUNCTION\x10\x07\x12\x32\n.LIFECYCLE_ERROR_CODE_SNAPSHOT_IDENTITY_MISSING\x10\x08\x12+\n\'LIFECYCLE_ERROR_CODE_UNSUPPORTED_INTENT\x10\t\x12\x32\n.LIFECYCLE_ERROR_CODE_UNKNOWN_MANDATORY_COMMAND\x10\n\x12\x31\n-LIFECYCLE_ERROR_CODE_PROTOCOL_UNREPORTED_WAIT\x10\x0b\x12\x35\n1LIFECYCLE_ERROR_CODE_CONFIG_SNAPSHOT_WRITE_FAILED\x10\x0c\x12%\n!LIFECYCLE_ERROR_CODE_DRAIN_FAILED\x10\r*\xe1\x01\n\x17\x46unctionCapabilityState\x12)\n%FUNCTION_CAPABILITY_STATE_UNSPECIFIED\x10\x00\x12#\n\x1f\x46UNCTION_CAPABILITY_STATE_READY\x10\x01\x12&\n\"FUNCTION_CAPABILITY_STATE_APPLYING\x10\x02\x12(\n$FUNCTION_CAPABILITY_STATE_BOOT_STALE\x10\x03\x12$\n FUNCTION_CAPABILITY_STATE_FAILED\x10\x04*\xdf\x01\n\x16\x43onfigApplicationState\x12(\n$CONFIG_APPLICATION_STATE_UNSPECIFIED\x10\x00\x12%\n!CONFIG_APPLICATION_STATE_APPLYING\x10\x01\x12&\n\"CONFIG_APPLICATION_STATE_CONVERGED\x10\x02\x12\'\n#CONFIG_APPLICATION_STATE_BOOT_STALE\x10\x03\x12#\n\x1f\x43ONFIG_APPLICATION_STATE_FAILED\x10\x04*\x9b\x02\n\x14\x44rainLifecycleStatus\x12&\n\"DRAIN_LIFECYCLE_STATUS_UNSPECIFIED\x10\x00\x12#\n\x1f\x44RAIN_LIFECYCLE_STATUS_ACCEPTED\x10\x01\x12#\n\x1f\x44RAIN_LIFECYCLE_STATUS_DRAINING\x10\x02\x12%\n!DRAIN_LIFECYCLE_STATUS_FINALIZING\x10\x03\x12#\n\x1f\x44RAIN_LIFECYCLE_STATUS_FLUSHING\x10\x04\x12\"\n\x1e\x44RAIN_LIFECYCLE_STATUS_DRAINED\x10\x05\x12!\n\x1d\x44RAIN_LIFECYCLE_STATUS_FAILED\x10\x06*v\n\x0bStorageTier\x12\x1c\n\x18STORAGE_TIER_UNSPECIFIED\x10\x00\x12\x1a\n\x16STORAGE_TIER_CONTAINER\x10\x01\x12\x17\n\x13STORAGE_TIER_VOLUME\x10\x02\x12\x14\n\x10STORAGE_TIER_NFS\x10\x03*\xd8\x01\n\x0bWorkerPhase\x12\x1c\n\x18WORKER_PHASE_UNSPECIFIED\x10\x00\x12\x18\n\x14WORKER_PHASE_BOOTING\x10\x01\x12#\n\x1fWORKER_PHASE_DOWNLOADING_MODELS\x10\x02\x12\"\n\x1eWORKER_PHASE_LOADING_PIPELINES\x10\x03\x12\x18\n\x14WORKER_PHASE_WARMING\x10\x04\x12\x16\n\x12WORKER_PHASE_READY\x10\x05\x12\x16\n\x12WORKER_PHASE_ERROR\x10\x06*V\n\nMediaBytes\x12\x1b\n\x17MEDIA_BYTES_UNSPECIFIED\x10\x00\x12\x13\n\x0fMEDIA_BYTES_URL\x10\x01\x12\x16\n\x12MEDIA_BYTES_INLINE\x10\x02*\x9b\x01\n\tJobStatus\x12\x1a\n\x16JOB_STATUS_UNSPECIFIED\x10\x00\x12\x11\n\rJOB_STATUS_OK\x10\x01\x12\x16\n\x12JOB_STATUS_INVALID\x10\x02\x12\x18\n\x14JOB_STATUS_RETRYABLE\x10\x03\x12\x14\n\x10JOB_STATUS_FATAL\x10\x04\x12\x17\n\x13JOB_STATUS_CANCELED\x10\x05*S\n\x0bModelOpKind\x12\x1d\n\x19MODEL_OP_KIND_UNSPECIFIED\x10\x00\x12%\n!MODEL_OP_KIND_ADOPT_COMPILE_CACHE\x10\x01*\x82\x02\n\nModelState\x12\x1b\n\x17MODEL_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17MODEL_STATE_DOWNLOADING\x10\x01\x12\x17\n\x13MODEL_STATE_ON_DISK\x10\x02\x12\x16\n\x12MODEL_STATE_IN_RAM\x10\x03\x12\x17\n\x13MODEL_STATE_IN_VRAM\x10\x04\x12\x17\n\x13MODEL_STATE_EVICTED\x10\x05\x12\x16\n\x12MODEL_STATE_FAILED\x10\x06\x12\x17\n\x13MODEL_STATE_ADOPTED\x10\x07\x12&\n\"MODEL_STATE_HOST_CAPACITY_PROGRESS\x10\x08*\x84\x01\n\rActivityState\x12\x1e\n\x1a\x41\x43TIVITY_STATE_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x41\x43TIVITY_STATE_RUNNING\x10\x01\x12\x1c\n\x18\x41\x43TIVITY_STATE_COMPLETED\x10\x02\x12\x19\n\x15\x41\x43TIVITY_STATE_FAILED\x10\x03\x32g\n\x0fWorkerScheduler\x12T\n\x07\x43onnect\x12 .cozy.scheduler.v1.WorkerMessage\x1a#.cozy.scheduler.v1.SchedulerMessage(\x01\x30\x01\x42YZWgithub.com/cozy-creator/tensorhub/internal/orchestrator/grpc/pb/schedulerv1;schedulerv1b\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x16worker_scheduler.proto\x12\x11\x63ozy.scheduler.v1\"\xf7\x05\n\rWorkerMessage\x12)\n\x05hello\x18\x01 \x01(\x0b\x32\x18.cozy.scheduler.v1.HelloH\x00\x12\x34\n\x0bstate_delta\x18\x02 \x01(\x0b\x32\x1d.cozy.scheduler.v1.StateDeltaH\x00\x12\x36\n\x0cjob_accepted\x18\x03 \x01(\x0b\x32\x1e.cozy.scheduler.v1.JobAcceptedH\x00\x12\x32\n\njob_result\x18\x04 \x01(\x0b\x32\x1c.cozy.scheduler.v1.JobResultH\x00\x12\x36\n\x0cjob_progress\x18\x05 \x01(\x0b\x32\x1e.cozy.scheduler.v1.JobProgressH\x00\x12\x34\n\x0bmodel_event\x18\x06 \x01(\x0b\x32\x1d.cozy.scheduler.v1.ModelEventH\x00\x12:\n\x0e\x66n_unavailable\x18\x07 \x01(\x0b\x32 .cozy.scheduler.v1.FnUnavailableH\x00\x12\x34\n\x0b\x66n_degraded\x18\x08 \x01(\x0b\x32\x1d.cozy.scheduler.v1.FnDegradedH\x00\x12<\n\x0f\x61\x63tivity_update\x18\t \x01(\x0b\x32!.cozy.scheduler.v1.ActivityUpdateH\x00\x12\x44\n\x13hardware_unsuitable\x18\n \x01(\x0b\x32%.cozy.scheduler.v1.HardwareUnsuitableH\x00\x12\x36\n\x0cgoal_receipt\x18\x0b \x01(\x0b\x32\x1e.cozy.scheduler.v1.GoalReceiptH\x00\x12\x42\n\x12lifecycle_snapshot\x18\x0c \x01(\x0b\x32$.cozy.scheduler.v1.LifecycleSnapshotH\x00\x12\x32\n\nboot_phase\x18\r \x01(\x0b\x32\x1c.cozy.scheduler.v1.BootPhaseH\x00\x42\x05\n\x03msg\"\x8f\x03\n\x10SchedulerMessage\x12\x30\n\thello_ack\x18\x01 \x01(\x0b\x32\x1b.cozy.scheduler.v1.HelloAckH\x00\x12,\n\x07run_job\x18\x02 \x01(\x0b\x32\x19.cozy.scheduler.v1.RunJobH\x00\x12\x32\n\ncancel_job\x18\x03 \x01(\x0b\x32\x1c.cozy.scheduler.v1.CancelJobH\x00\x12.\n\x08model_op\x18\x04 \x01(\x0b\x32\x1a.cozy.scheduler.v1.ModelOpH\x00\x12)\n\x05\x64rain\x18\x05 \x01(\x0b\x32\x18.cozy.scheduler.v1.DrainH\x00\x12\x38\n\rtoken_refresh\x18\x06 \x01(\x0b\x32\x1f.cozy.scheduler.v1.TokenRefreshH\x00\x12\x38\n\rserve_posture\x18\x08 \x01(\x0b\x32\x1f.cozy.scheduler.v1.ServePostureH\x00\x42\x05\n\x03msgJ\x04\x08\x07\x10\x08R\x0brun_attempt\"\xb3\x03\n\x05Hello\x12<\n\x10protocol_version\x18\x01 \x01(\x0e\x32\".cozy.scheduler.v1.ProtocolVersion\x12\x11\n\tworker_id\x18\x02 \x01(\t\x12\x12\n\nrelease_id\x18\x03 \x01(\t\x12\x35\n\tresources\x18\x04 \x01(\x0b\x32\".cozy.scheduler.v1.WorkerResources\x12,\n\x05state\x18\x05 \x01(\x0b\x32\x1d.cozy.scheduler.v1.StateDelta\x12\x31\n\x06models\x18\x06 \x03(\x0b\x32!.cozy.scheduler.v1.ModelResidency\x12\x31\n\tin_flight\x18\x07 \x03(\x0b\x32\x1e.cozy.scheduler.v1.InFlightJob\x12\x1d\n\x15heartbeat_interval_ms\x18\x08 \x01(\x04\x12\x19\n\x11worker_session_id\x18\t \x01(\t\x12@\n\x12lifecycle_snapshot\x18\n \x01(\x0b\x32$.cozy.scheduler.v1.LifecycleSnapshot\"\x9b\x03\n\x0fWorkerResources\x12\x11\n\tgpu_count\x18\x01 \x01(\x05\x12\x18\n\x10vram_total_bytes\x18\x02 \x01(\x03\x12\x10\n\x08gpu_name\x18\x03 \x01(\t\x12\x0e\n\x06gpu_sm\x18\x04 \x01(\t\x12\x16\n\x0einstalled_libs\x18\x05 \x03(\t\x12\x14\n\x0cimage_digest\x18\x06 \x01(\t\x12\x12\n\ngit_commit\x18\x07 \x01(\t\x12\x13\n\x0binstance_id\x18\x08 \x01(\t\x12\x32\n\x0bhost_canary\x18\t \x01(\x0b\x32\x1d.cozy.scheduler.v1.HostCanary\x12\x15\n\rtorch_version\x18\n \x01(\t\x12\x1a\n\x12gen_worker_version\x18\x0b \x01(\t\x12\x16\n\x0e\x64river_version\x18\r \x01(\t\x12\x14\n\x0c\x63uda_version\x18\x0e \x01(\t\x12\x1f\n\x17\x63\x61pability_reason_class\x18\x0f \x01(\t\x12\x19\n\x11\x63\x61pability_detail\x18\x10 \x01(\tJ\x04\x08\x0c\x10\rR\x0bworker_mode\"\xa2\x02\n\x12HardwareUnsuitable\x12\x11\n\tworker_id\x18\x01 \x01(\t\x12\x12\n\nrelease_id\x18\x02 \x01(\t\x12\x14\n\x0creason_class\x18\x03 \x01(\t\x12\x0e\n\x06\x64\x65tail\x18\x04 \x01(\t\x12\x16\n\x0e\x64river_version\x18\x05 \x01(\t\x12\x10\n\x08gpu_name\x18\x06 \x01(\t\x12\x15\n\rtorch_version\x18\x07 \x01(\t\x12\x1a\n\x12torch_cuda_version\x18\x08 \x01(\t\x12\x1a\n\x12gen_worker_version\x18\t \x01(\t\x12\x14\n\x0cimage_digest\x18\n \x01(\t\x12\x13\n\x0binstance_id\x18\x0b \x01(\t\x12\x1b\n\x13reported_at_unix_ms\x18\x0c \x01(\x03\"\x88\x02\n\nHostCanary\x12\x13\n\x0bmemcpy_gbps\x18\x01 \x01(\x01\x12\x10\n\x08\x64\x32h_gbps\x18\x02 \x01(\x01\x12\x17\n\x0fpinned_alloc_ok\x18\x03 \x01(\x08\x12\x17\n\x0f\x63pu_single_mbps\x18\x04 \x01(\x01\x12\x16\n\x0e\x63pu_multi_mbps\x18\x05 \x01(\x01\x12\r\n\x05vcpus\x18\x06 \x01(\x05\x12\x14\n\x0cram_total_gb\x18\x07 \x01(\x01\x12\x13\n\x0b\x64uration_ms\x18\x08 \x01(\x03\x12\x14\n\x0cinterconnect\x18\t \x01(\t\x12\x11\n\tpeer_gbps\x18\n \x01(\x01\x12\x13\n\x0bpeer_access\x18\x0b \x01(\x08\x12\x11\n\ttopo_link\x18\x0c \x01(\t\"\x98\x01\n\x0eModelResidency\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12.\n\x04tier\x18\x02 \x01(\x0e\x32 .cozy.scheduler.v1.ResidencyTier\x12\x12\n\nvram_bytes\x18\x03 \x01(\x03\x12\x17\n\x0fsnapshot_digest\x18\x04 \x01(\t\x12\x1c\n\x14residency_generation\x18\x05 \x01(\x04\"2\n\x0bInFlightJob\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\"\x9f\x02\n\x08HelloAck\x12<\n\x10protocol_version\x18\x01 \x01(\x0e\x32\".cozy.scheduler.v1.ProtocolVersion\x12\x15\n\rfile_base_url\x18\x02 \x01(\t\x12\x37\n\x0bresolutions\x18\x03 \x03(\x0b\x32\".cozy.scheduler.v1.ModelResolution\x12>\n\x11\x64\x65sired_residency\x18\x04 \x01(\x0b\x32#.cozy.scheduler.v1.DesiredResidency\x12\x45\n\x15\x64\x65sired_state_command\x18\x05 \x01(\x0b\x32&.cozy.scheduler.v1.DesiredStateCommand\"\xaf\x02\n\x10\x44\x65siredResidency\x12\x12\n\ngeneration\x18\x01 \x01(\x04\x12\x11\n\tdisk_refs\x18\x02 \x03(\t\x12/\n\x03hot\x18\x03 \x03(\x0b\x32\".cozy.scheduler.v1.DesiredInstance\x12\x45\n\tsnapshots\x18\x04 \x03(\x0b\x32\x32.cozy.scheduler.v1.DesiredResidency.SnapshotsEntry\x12\x12\n\nrelease_id\x18\x05 \x01(\t\x12\x19\n\x11\x63onfig_generation\x18\x06 \x01(\x04\x1aM\n\x0eSnapshotsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.cozy.scheduler.v1.Snapshot:\x02\x38\x01\"\x99\x03\n\x13\x44\x65siredStateCommand\x12\x19\n\x11worker_session_id\x18\x01 \x01(\t\x12\x13\n\x0b\x63ommand_seq\x18\x02 \x01(\x04\x12\x0f\n\x07goal_id\x18\x03 \x01(\t\x12\x12\n\nrelease_id\x18\x04 \x01(\t\x12\x19\n\x11\x63onfig_generation\x18\x05 \x01(\x04\x12\x15\n\rconfig_digest\x18\x06 \x01(\x0c\x12\x19\n\x11issued_at_unix_ms\x18\x07 \x01(\x03\x12\x19\n\x11\x61\x63\x63\x65pt_by_unix_ms\x18\x08 \x01(\x03\x12\x1f\n\x17\x66irst_action_by_unix_ms\x18\t \x01(\x03\x12\x31\n\x07intents\x18\n \x03(\x0b\x32 .cozy.scheduler.v1.DesiredIntent\x12\x11\n\tmandatory\x18\x0b \x01(\x08\x12\x42\n\x16\x63hanged_config_classes\x18\x0c \x01(\x0b\x32\".cozy.scheduler.v1.ConfigClassMask\x12\x1a\n\x12parameter_snapshot\x18\r \x01(\x0c\"\x95\x03\n\rDesiredIntent\x12\x11\n\tintent_id\x18\x01 \x01(\t\x12\x32\n\x04kind\x18\x02 \x01(\x0e\x32$.cozy.scheduler.v1.DesiredIntentKind\x12\x34\n\x05\x63\x61use\x18\x03 \x01(\x0e\x32%.cozy.scheduler.v1.DesiredIntentCause\x12\x15\n\rfunction_name\x18\x04 \x01(\t\x12\x0b\n\x03ref\x18\x05 \x01(\t\x12\x17\n\x0fsnapshot_digest\x18\x06 \x01(\x0c\x12\x36\n\x0c\x64\x65sired_tier\x18\x07 \x01(\x0e\x32 .cozy.scheduler.v1.ResidencyTier\x12\x16\n\x0e\x62inding_digest\x18\x08 \x01(\x0c\x12\x18\n\x10parent_intent_id\x18\t \x01(\t\x12;\n\x10waiting_requests\x18\n \x03(\x0b\x32!.cozy.scheduler.v1.RequestAttempt\x12\x10\n\x08priority\x18\x0b \x01(\x05\x12\x11\n\tmandatory\x18\x0c \x01(\x08\"5\n\x0eRequestAttempt\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\"\xd0\x02\n\x0bGoalReceipt\x12\x19\n\x11worker_session_id\x18\x01 \x01(\t\x12\x13\n\x0b\x63ommand_seq\x18\x02 \x01(\x04\x12\x0f\n\x07goal_id\x18\x03 \x01(\t\x12\x12\n\nrelease_id\x18\x04 \x01(\t\x12\x34\n\x06status\x18\x05 \x01(\x0e\x32$.cozy.scheduler.v1.GoalReceiptStatus\x12\x39\n\nerror_code\x18\x06 \x01(\x0e\x32%.cozy.scheduler.v1.LifecycleErrorCode\x12\x36\n\nrejections\x18\x07 \x03(\x0b\x32\".cozy.scheduler.v1.IntentRejection\x12\x0e\n\x06\x64\x65tail\x18\x08 \x01(\t\x12\x1b\n\x13received_at_unix_ms\x18\t \x01(\x03\x12\x16\n\x0e\x63ommand_digest\x18\n \x01(\x0c\"o\n\x0fIntentRejection\x12\x11\n\tintent_id\x18\x01 \x01(\t\x12\x39\n\nerror_code\x18\x02 \x01(\x0e\x32%.cozy.scheduler.v1.LifecycleErrorCode\x12\x0e\n\x06\x64\x65tail\x18\x03 \x01(\t\"R\n\x11LifecycleProgress\x12\x0c\n\x04\x64one\x18\x01 \x01(\x01\x12\r\n\x05total\x18\x02 \x01(\x01\x12\x0c\n\x04unit\x18\x03 \x01(\t\x12\x12\n\nrate_per_s\x18\x04 \x01(\x01\"\x98\x05\n\x0bIntentState\x12\x19\n\x11worker_session_id\x18\x01 \x01(\t\x12\x11\n\tstate_seq\x18\x02 \x01(\x04\x12\x0f\n\x07goal_id\x18\x03 \x01(\t\x12\x11\n\tintent_id\x18\x04 \x01(\t\x12\x12\n\nrelease_id\x18\x05 \x01(\t\x12\x19\n\x11\x63onfig_generation\x18\x06 \x01(\x04\x12\x38\n\x06status\x18\x07 \x01(\x0e\x32(.cozy.scheduler.v1.LifecycleIntentStatus\x12\x36\n\x05stage\x18\x08 \x01(\x0e\x32\'.cozy.scheduler.v1.LifecycleIntentStage\x12\x36\n\x06reason\x18\t \x01(\x0e\x32&.cozy.scheduler.v1.LifecycleWaitReason\x12\x15\n\rsince_unix_ms\x18\n \x01(\x03\x12\x1a\n\x12updated_at_unix_ms\x18\x0b \x01(\x03\x12\x1d\n\x15next_retry_at_unix_ms\x18\x0c \x01(\x03\x12\x1b\n\x13\x64\x65\x61\x64line_at_unix_ms\x18\r \x01(\x03\x12\x19\n\x11\x62locker_intent_id\x18\x0e \x01(\t\x12:\n\x0f\x62locker_request\x18\x0f \x01(\x0b\x32!.cozy.scheduler.v1.RequestAttempt\x12\x36\n\x08progress\x18\x10 \x01(\x0b\x32$.cozy.scheduler.v1.LifecycleProgress\x12\x39\n\nerror_code\x18\x11 \x01(\x0e\x32%.cozy.scheduler.v1.LifecycleErrorCode\x12\x0e\n\x06\x64\x65tail\x18\x12 \x01(\t\x12\x15\n\ractual_digest\x18\x13 \x01(\x0c\"\x83\x01\n\rModelIdentity\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12\x17\n\x0fsnapshot_digest\x18\x02 \x01(\x0c\x12.\n\x04tier\x18\x03 \x01(\x0e\x32 .cozy.scheduler.v1.ResidencyTier\x12\x1c\n\x14residency_generation\x18\x04 \x01(\x04\"\xaa\x02\n\x12\x46unctionCapability\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12\x12\n\nrelease_id\x18\x02 \x01(\t\x12\x19\n\x11\x63onfig_generation\x18\x03 \x01(\x04\x12\x16\n\x0e\x62inding_digest\x18\x04 \x01(\x0c\x12\x0c\n\x04lane\x18\x05 \x01(\t\x12\x30\n\x06models\x18\x06 \x03(\x0b\x32 .cozy.scheduler.v1.ModelIdentity\x12%\n\x1d\x63ompile_target_incarnation_id\x18\x07 \x01(\t\x12\x39\n\x05state\x18\x08 \x01(\x0e\x32*.cozy.scheduler.v1.FunctionCapabilityState\x12\x14\n\x0cserving_tier\x18\t \x01(\t\"E\n\x0f\x43onfigClassMask\x12\x12\n\nparameters\x18\x01 \x01(\x08\x12\x10\n\x08\x62indings\x18\x02 \x01(\x08\x12\x0c\n\x04\x62oot\x18\x03 \x01(\x08\"\xf3\x02\n\x11\x43onfigApplication\x12\x12\n\nrelease_id\x18\x01 \x01(\t\x12\x19\n\x11target_generation\x18\x02 \x01(\x04\x12\x1b\n\x13received_generation\x18\x03 \x01(\x04\x12%\n\x1dparameter_snapshot_generation\x18\x04 \x01(\x04\x12 \n\x18\x62inding_ready_generation\x18\x05 \x01(\x04\x12\x17\n\x0f\x62oot_generation\x18\x06 \x01(\x04\x12\x38\n\x05state\x18\x07 \x01(\x0e\x32).cozy.scheduler.v1.ConfigApplicationState\x12;\n\x0fpending_classes\x18\x08 \x01(\x0b\x32\".cozy.scheduler.v1.ConfigClassMask\x12\x39\n\nerror_code\x18\t \x01(\x0e\x32%.cozy.scheduler.v1.LifecycleErrorCode\"\x89\x02\n\x0f\x44rainProjection\x12\x0f\n\x07goal_id\x18\x01 \x01(\t\x12\x11\n\tintent_id\x18\x02 \x01(\t\x12\x37\n\x06status\x18\x03 \x01(\x0e\x32\'.cozy.scheduler.v1.DrainLifecycleStatus\x12\x15\n\rsince_unix_ms\x18\x04 \x01(\x03\x12\x1a\n\x12updated_at_unix_ms\x18\x05 \x01(\x03\x12\x1b\n\x13\x64\x65\x61\x64line_at_unix_ms\x18\x06 \x01(\x03\x12\x39\n\nerror_code\x18\x07 \x01(\x0e\x32%.cozy.scheduler.v1.LifecycleErrorCode\x12\x0e\n\x06\x64\x65tail\x18\x08 \x01(\t\"\x8f\x03\n\x11LifecycleSnapshot\x12\x19\n\x11worker_session_id\x18\x01 \x01(\t\x12\x11\n\tstate_seq\x18\x02 \x01(\x04\x12/\n\x07intents\x18\x03 \x03(\x0b\x32\x1e.cozy.scheduler.v1.IntentState\x12;\n\x0c\x63\x61pabilities\x18\x04 \x03(\x0b\x32%.cozy.scheduler.v1.FunctionCapability\x12@\n\x12\x63onfig_application\x18\x05 \x01(\x0b\x32$.cozy.scheduler.v1.ConfigApplication\x12\x35\n\rgoal_receipts\x18\x06 \x03(\x0b\x32\x1e.cozy.scheduler.v1.GoalReceipt\x12\x31\n\x05\x64rain\x18\x07 \x01(\x0b\x32\".cozy.scheduler.v1.DrainProjection\x12\x14\n\x0c\x66ull_replace\x18\x08 \x01(\x08\x12\x1c\n\x14generated_at_unix_ms\x18\t \x01(\x03\"Y\n\x0f\x44\x65siredInstance\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12/\n\x06models\x18\x02 \x03(\x0b\x32\x1f.cozy.scheduler.v1.ModelBinding\"e\n\x0fModelResolution\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12\x14\n\x0cresolved_ref\x18\x02 \x01(\t\x12\x0c\n\x04\x63\x61st\x18\x03 \x01(\t\x12\x0c\n\x04lane\x18\x04 \x01(\t\x12\x13\n\x0blane_pinned\x18\x05 \x01(\x08\"\xab\x03\n\nStateDelta\x12-\n\x05phase\x18\x01 \x01(\x0e\x32\x1e.cozy.scheduler.v1.WorkerPhase\x12\x1b\n\x13\x61vailable_functions\x18\x02 \x03(\t\x12\x19\n\x11loading_functions\x18\x03 \x03(\t\x12\x17\n\x0f\x66ree_vram_bytes\x18\x04 \x01(\x03\x12\x17\n\x0f\x66inalizing_jobs\x18\x05 \x01(\x05\x12%\n\x1dobserved_residency_generation\x18\x06 \x01(\x04\x12\x39\n\x0f\x63ompile_targets\x18\x07 \x03(\x0b\x32 .cozy.scheduler.v1.CompileTarget\x12\x46\n\x16\x63ompiled_graph_lookups\x18\x08 \x03(\x0b\x32&.cozy.scheduler.v1.CompiledGraphLookup\x12\x36\n\ndisk_usage\x18\t \x01(\x0b\x32\".cozy.scheduler.v1.DiskUsageReport\x12\"\n\x1aobserved_config_generation\x18\n \x01(\x04\"\xac\x01\n\x10StorageTierUsage\x12,\n\x04tier\x18\x01 \x01(\x0e\x32\x1e.cozy.scheduler.v1.StorageTier\x12\x12\n\nmount_path\x18\x02 \x01(\t\x12\x13\n\x0btotal_bytes\x18\x03 \x01(\x03\x12\x12\n\nfree_bytes\x18\x04 \x01(\x03\x12\x12\n\nused_bytes\x18\x05 \x01(\x03\x12\x19\n\x11reclaimable_bytes\x18\x06 \x01(\x03\"b\n\x0f\x44iskUsageReport\x12\x32\n\x05tiers\x18\x01 \x03(\x0b\x32#.cozy.scheduler.v1.StorageTierUsage\x12\x1b\n\x13\x63\x61pacity_generation\x18\x02 \x01(\x04\"A\n\x13\x43ompiledGraphLookup\x12\x0e\n\x06\x66\x61mily\x18\x01 \x01(\t\x12\x1a\n\x12\x63ompiled_graph_key\x18\x02 \x01(\t\"\xe1\x02\n\rCompileTarget\x12\x16\n\x0eincarnation_id\x18\x01 \x01(\t\x12\x0e\n\x06\x66\x61mily\x18\x02 \x01(\t\x12\x1c\n\x14pipeline_weight_lane\x18\x03 \x01(\t\x12\x13\n\x0blora_bucket\x18\x04 \x01(\x05\x12\x17\n\x0f\x63ontract_digest\x18\x05 \x01(\t\x12\x1a\n\x12\x61\x63tive_compile_ref\x18\x06 \x01(\t\x12&\n\x1e\x61\x63tive_compile_snapshot_digest\x18\x07 \x01(\t\x12\x16\n\x0e\x66unction_names\x18\x08 \x03(\t\x12?\n\x0emodel_bindings\x18\t \x03(\x0b\x32\'.cozy.scheduler.v1.CompileTargetBinding\x12$\n\x1crequested_compiled_graph_key\x18\n \x01(\tJ\x04\x08\x0b\x10\x0cR\x13requested_cell_axes\"J\n\x14\x43ompileTargetBinding\x12\x0c\n\x04slot\x18\x01 \x01(\t\x12\x0b\n\x03ref\x18\x02 \x01(\t\x12\x17\n\x0fsnapshot_digest\x18\x03 \x01(\t\"\x9a\x06\n\x06RunJob\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\x12\x15\n\rfunction_name\x18\x03 \x01(\t\x12\x15\n\rinput_payload\x18\x04 \x01(\x0c\x12\x12\n\ntimeout_ms\x18\x05 \x01(\x03\x12\x0b\n\x03org\x18\x06 \x01(\t\x12\x12\n\ninvoker_id\x18\x07 \x01(\t\x12\x18\n\x10\x63\x61pability_token\x18\x08 \x01(\t\x12\x32\n\x0bmedia_bytes\x18\t \x01(\x0e\x32\x1d.cozy.scheduler.v1.MediaBytes\x12\x33\n\x07\x63ompute\x18\n \x01(\x0b\x32\".cozy.scheduler.v1.ResolvedCompute\x12/\n\x06models\x18\x0b \x03(\x0b\x32\x1f.cozy.scheduler.v1.ModelBinding\x12;\n\tsnapshots\x18\x0c \x03(\x0b\x32(.cozy.scheduler.v1.RunJob.SnapshotsEntry\x12\x45\n\x10required_compile\x18\r \x01(\x0b\x32+.cozy.scheduler.v1.RequiredCompileExecution\x12\x0c\n\x04lane\x18\x0e \x01(\t\x12\x33\n\x0cinput_assets\x18\x0f \x03(\x0b\x32\x1d.cozy.scheduler.v1.InputAsset\x12\x19\n\x11\x63onfig_generation\x18\x10 \x01(\x04\x12\x15\n\rconfig_params\x18\x11 \x01(\x0c\x12\x39\n\x0bintent_kind\x18\x12 \x01(\x0e\x32$.cozy.scheduler.v1.DesiredIntentKind\x12\x11\n\tintent_id\x18\x13 \x01(\t\x12\x0f\n\x07goal_id\x18\x14 \x01(\t\x12\x16\n\x0ephase_budget_s\x18\x15 \x01(\x03\x12\x15\n\rfile_base_url\x18\x16 \x01(\t\x1aM\n\x0eSnapshotsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.cozy.scheduler.v1.Snapshot:\x02\x38\x01\"w\n\nInputAsset\x12\x10\n\x08\x61sset_id\x18\x01 \x01(\t\x12\x12\n\nsource_ref\x18\x02 \x01(\t\x12\x0e\n\x06\x62lake3\x18\x03 \x01(\t\x12\x12\n\nsize_bytes\x18\x04 \x01(\x03\x12\x0c\n\x04kind\x18\x05 \x01(\t\x12\x11\n\tmime_type\x18\x06 \x01(\t\"\x96\x01\n\x18RequiredCompileExecution\x12\x1d\n\x15target_incarnation_id\x18\x01 \x01(\t\x12\x1a\n\x12\x63ompiled_graph_ref\x18\x02 \x01(\t\x12&\n\x1e\x63ompiled_graph_snapshot_digest\x18\x03 \x01(\t\x12\x17\n\x0f\x63ontract_digest\x18\x04 \x01(\t\"9\n\x0fResolvedCompute\x12\x13\n\x0b\x61\x63\x63\x65lerator\x18\x01 \x01(\t\x12\x11\n\tgpu_index\x18\x02 \x01(\x05\"\xa5\x02\n\x0cModelBinding\x12\x0c\n\x04slot\x18\x01 \x01(\t\x12\x0b\n\x03ref\x18\x02 \x01(\t\x12-\n\x05loras\x18\x03 \x03(\x0b\x32\x1e.cozy.scheduler.v1.LoraOverlay\x12\x1a\n\x12inference_defaults\x18\x04 \x01(\t\x12\x11\n\tobjective\x18\x06 \x01(\t\x12\x11\n\tdistilled\x18\x07 \x01(\x08\x12\x18\n\x10\x64istilled_status\x18\x08 \x01(\t\x12\r\n\x05model\x18\t \x01(\t\x12\x1c\n\x14\x62ind_contract_digest\x18\x0b \x01(\t\x12\x19\n\x11\x62ind_contract_url\x18\x0c \x01(\tJ\x04\x08\x05\x10\x06J\x04\x08\n\x10\x0bR\ncomponentsR\x0fmanifest_digest\"U\n\x0bLoraOverlay\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12\x0e\n\x06weight\x18\x02 \x01(\x01\x12\x1a\n\x12inference_defaults\x18\x03 \x01(\t\x12\r\n\x05model\x18\x04 \x01(\t\"J\n\x08Snapshot\x12\x0e\n\x06\x64igest\x18\x01 \x01(\t\x12.\n\x05\x66iles\x18\x02 \x03(\x0b\x32\x1f.cozy.scheduler.v1.SnapshotFile\"\xa4\x01\n\x0cSnapshotFile\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x12\n\nsize_bytes\x18\x02 \x01(\x03\x12\x0e\n\x06\x62lake3\x18\x03 \x01(\t\x12\x0b\n\x03url\x18\x04 \x01(\t\x12\x0e\n\x06\x64igest\x18\x05 \x01(\t\x12\x18\n\x10\x63hunk_size_bytes\x18\x06 \x01(\x03\x12+\n\x06\x63hunks\x18\x07 \x03(\x0b\x32\x1b.cozy.scheduler.v1.ChunkRef\"4\n\x08\x43hunkRef\x12\x0e\n\x06sha256\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12\x0b\n\x03len\x18\x03 \x01(\x03\"2\n\x0bJobAccepted\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\"\x9b\x02\n\tJobResult\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\x12,\n\x06status\x18\x03 \x01(\x0e\x32\x1c.cozy.scheduler.v1.JobStatus\x12\x10\n\x06inline\x18\x04 \x01(\x0cH\x00\x12\x12\n\x08\x62lob_ref\x18\x05 \x01(\tH\x00\x12\x14\n\x0csafe_message\x18\x06 \x01(\t\x12.\n\x07metrics\x18\x07 \x01(\x0b\x32\x1d.cozy.scheduler.v1.JobMetrics\x12\x32\n\x0b\x61\x64justments\x18\x08 \x03(\x0b\x32\x1d.cozy.scheduler.v1.Adjustment\x12\x11\n\ttraceback\x18\t \x01(\tB\x08\n\x06output\"O\n\nAdjustment\x12\r\n\x05\x66ield\x18\x01 \x01(\t\x12\x11\n\trequested\x18\x02 \x01(\t\x12\x0f\n\x07\x61pplied\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\"\x8d\x06\n\nJobMetrics\x12\x12\n\nruntime_ms\x18\x01 \x01(\x03\x12\x10\n\x08queue_ms\x18\x02 \x01(\x03\x12\x18\n\x10rss_at_end_bytes\x18\x03 \x01(\x03\x12\x17\n\x0fpeak_vram_bytes\x18\x04 \x01(\x03\x12\x1c\n\x14\x63oncurrency_at_start\x18\x05 \x01(\x05\x12\x1f\n\x17output_media_duration_s\x18\x06 \x01(\x01\x12\x14\n\x0cinput_tokens\x18\x07 \x01(\x03\x12\x1b\n\x13input_cached_tokens\x18\x08 \x01(\x03\x12\x15\n\routput_tokens\x18\t \x01(\x03\x12\x14\n\x0coutput_count\x18\n \x01(\x03\x12\x14\n\x0cslot_held_ms\x18\x0b \x01(\x03\x12\x18\n\x10\x66inalize_wall_ms\x18\x0c \x01(\x03\x12\x0c\n\x04lane\x18\r \x01(\t\x12\x46\n\rruntime_terms\x18\x0e \x03(\x0b\x32/.cozy.scheduler.v1.JobMetrics.RuntimeTermsEntry\x12<\n\x08stage_ms\x18\x0f \x03(\x0b\x32*.cozy.scheduler.v1.JobMetrics.StageMsEntry\x12\x14\n\x0cserving_mode\x18\x10 \x01(\t\x12!\n\x19served_compiled_graph_ref\x18\x11 \x01(\t\x12\x1d\n\x15served_eager_fallback\x18\x12 \x01(\x08\x12\x17\n\x0f\x66\x61llback_reason\x18\x13 \x01(\t\x12\n\n\x02sm\x18\x14 \x01(\t\x12\r\n\x05steps\x18\x15 \x01(\x03\x12\r\n\x05width\x18\x16 \x01(\x03\x12\x0e\n\x06height\x18\x17 \x01(\x03\x12\x33\n\x07posture\x18\x18 \x01(\x0b\x32\".cozy.scheduler.v1.MeasuredPosture\x1a\x33\n\x11RuntimeTermsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x01:\x02\x38\x01\x1a.\n\x0cStageMsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\x03:\x02\x38\x01\"\xdb\x02\n\x0fMeasuredPosture\x12\x16\n\x0e\x65xecution_lane\x18\x01 \x01(\t\x12\x19\n\x11\x61ttention_backend\x18\x02 \x01(\t\x12 \n\x18\x61ttention_backend_wanted\x18\x03 \x01(\t\x12\x15\n\rcompile_state\x18\x04 \x01(\t\x12\x1c\n\x14\x63ompile_state_wanted\x18\x05 \x01(\t\x12\x16\n\x0eresidency_mode\x18\x06 \x01(\t\x12\x34\n\x07\x61pplied\x18\x07 \x03(\x0b\x32#.cozy.scheduler.v1.AppliedTechnique\x12\x37\n\ncomponents\x18\x08 \x03(\x0b\x32#.cozy.scheduler.v1.ComponentPosture\x12\x37\n\tshortfall\x18\t \x01(\x0b\x32$.cozy.scheduler.v1.ResourceShortfall\"i\n\x10\x41ppliedTechnique\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x11\n\tcomponent\x18\x02 \x01(\t\x12\x0e\n\x06wanted\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x14\n\x0c\x65st_slowdown\x18\x05 \x01(\x01\"s\n\x10\x43omponentPosture\x12\x11\n\tcomponent\x18\x01 \x01(\t\x12\x15\n\rapplied_quant\x18\x02 \x01(\t\x12\x13\n\x0b\x62ound_quant\x18\x03 \x01(\t\x12\x11\n\tplacement\x18\x04 \x01(\t\x12\r\n\x05\x62ytes\x18\x05 \x01(\x03\"g\n\x11ResourceShortfall\x12\x10\n\x08resource\x18\x01 \x01(\t\x12\x11\n\tcomponent\x18\x02 \x01(\t\x12\x14\n\x0cneeded_bytes\x18\x03 \x01(\x03\x12\x17\n\x0f\x61vailable_bytes\x18\x04 \x01(\x03\"c\n\x0bJobProgress\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\x12\x0b\n\x03seq\x18\x03 \x01(\x03\x12\x0c\n\x04\x64\x61ta\x18\x04 \x01(\x0c\x12\x14\n\x0c\x63ontent_type\x18\x05 \x01(\t\"0\n\tCancelJob\x12\x12\n\nrequest_id\x18\x01 \x01(\t\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x03\"\xa6\x01\n\x07ModelOp\x12*\n\x02op\x18\x01 \x01(\x0e\x32\x1e.cozy.scheduler.v1.ModelOpKind\x12\x0b\n\x03ref\x18\x02 \x01(\t\x12-\n\x08snapshot\x18\x03 \x01(\x0b\x32\x1b.cozy.scheduler.v1.Snapshot\x12\x14\n\x0coperation_id\x18\x04 \x01(\t\x12\x1d\n\x15target_incarnation_id\x18\x05 \x01(\t\"\x9e\x04\n\nModelEvent\x12\x0b\n\x03ref\x18\x01 \x01(\t\x12,\n\x05state\x18\x02 \x01(\x0e\x32\x1d.cozy.scheduler.v1.ModelState\x12\x12\n\nvram_bytes\x18\x03 \x01(\x03\x12\r\n\x05\x65rror\x18\x04 \x01(\t\x12\x12\n\nbytes_done\x18\x05 \x01(\x03\x12\x13\n\x0b\x62ytes_total\x18\x06 \x01(\x03\x12\x13\n\x0b\x64uration_ms\x18\x07 \x01(\x03\x12\x12\n\ncache_hits\x18\x08 \x01(\x03\x12\x14\n\x0c\x63\x61\x63he_misses\x18\t \x01(\x03\x12\x10\n\x08warmup_s\x18\n \x01(\x01\x12\x1f\n\x17host_ram_required_bytes\x18\x0b \x01(\x03\x12\'\n\x1fhost_ram_available_before_bytes\x18\x0c \x01(\x03\x12&\n\x1ehost_ram_available_after_bytes\x18\r \x01(\x03\x12\x1d\n\x15host_ram_evicted_refs\x18\x0e \x03(\t\x12$\n\x1chost_ram_capacity_generation\x18\x0f \x01(\x04\x12\x17\n\x0fsnapshot_digest\x18\x10 \x01(\t\x12\x1c\n\x14residency_generation\x18\x11 \x01(\x04\x12\x14\n\x0coperation_id\x18\x12 \x01(\t\x12\x1d\n\x15target_incarnation_id\x18\x13 \x01(\t\x12\x15\n\rnetwork_bytes\x18\x14 \x01(\x03\"\xbe\x03\n\x0e\x41\x63tivityUpdate\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12\r\n\x05phase\x18\x02 \x01(\t\x12\x0c\n\x04step\x18\x03 \x01(\x03\x12\x13\n\x0btotal_steps\x18\x04 \x01(\x03\x12\x0b\n\x03seq\x18\x05 \x01(\x04\x12/\n\x05state\x18\x06 \x01(\x0e\x32 .cozy.scheduler.v1.ActivityState\x12\r\n\x05\x65rror\x18\x07 \x01(\t\x12\x0e\n\x06\x64\x65tail\x18\x08 \x01(\t\x12\x1a\n\x12updated_at_unix_ms\x18\t \x01(\x03\x12\x0f\n\x07\x63ounter\x18\n \x01(\t\x12\x14\n\x0c\x63ounter_unit\x18\x0b \x01(\t\x12\x14\n\x0c\x63ounter_done\x18\x0c \x01(\x01\x12\x15\n\rcounter_total\x18\r \x01(\x01\x12\x12\n\nrate_per_s\x18\x0e \x01(\x01\x12\x14\n\x0cself_stalled\x18\x0f \x01(\x08\x12\x16\n\x0estalled_for_ms\x18\x10 \x01(\x03\x12\x13\n\x0b\x64uration_ms\x18\x11 \x01(\x03\x12\x0e\n\x06\x66\x61mily\x18\x12 \x01(\t\x12\x1a\n\x12\x63ompiled_graph_key\x18\x13 \x01(\t\x12\x1c\n\x14graph_specialization\x18\x14 \x01(\t\"\xe2\x02\n\tBootPhase\x12\x0f\n\x07\x62oot_id\x18\x01 \x01(\t\x12\x0f\n\x07ordinal\x18\x02 \x01(\x03\x12\x16\n\x0eparent_ordinal\x18\x03 \x01(\x03\x12\r\n\x05phase\x18\x04 \x01(\t\x12\x10\n\x08terminal\x18\x05 \x01(\x08\x12\x1a\n\x12started_at_unix_ms\x18\x06 \x01(\x03\x12\x13\n\x0b\x64uration_ms\x18\x07 \x01(\x03\x12\x19\n\x11process_uptime_ms\x18\x08 \x01(\x03\x12\r\n\x05\x62ytes\x18\t \x01(\x03\x12\x0e\n\x06source\x18\n \x01(\t\x12\x0b\n\x03ref\x18\x0b \x01(\t\x12\x15\n\rartifact_kind\x18\x0c \x01(\t\x12\x14\n\x0c\x61rtifact_key\x18\r \x01(\t\x12\x10\n\x08\x66unction\x18\x0e \x01(\t\x12\x0f\n\x07outcome\x18\x0f \x01(\t\x12\x0e\n\x06reason\x18\x10 \x01(\t\x12\x0e\n\x06\x64\x65tail\x18\x11 \x01(\t\x12\x12\n\ncumulative\x18\x12 \x01(\x08\"\xad\x01\n\rFnUnavailable\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\x0e\n\x06\x64\x65tail\x18\x03 \x01(\t\x12\x38\n\x04\x61xes\x18\x04 \x03(\x0b\x32*.cozy.scheduler.v1.FnUnavailable.AxesEntry\x1a+\n\tAxesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"p\n\nFnDegraded\x12\x15\n\rfunction_name\x18\x01 \x01(\t\x12\x0e\n\x06wanted\x18\x02 \x01(\t\x12\x0b\n\x03ran\x18\x03 \x01(\t\x12\x0e\n\x06reason\x18\x04 \x01(\t\x12\x1e\n\x16\x65st_latency_multiplier\x18\x05 \x01(\x01\"\x1c\n\x05\x44rain\x12\x13\n\x0b\x64\x65\x61\x64line_ms\x18\x01 \x01(\x03\"6\n\x0cTokenRefresh\x12\r\n\x05token\x18\x01 \x01(\t\x12\x17\n\x0f\x65xpires_at_unix\x18\x02 \x01(\x03\"A\n\x0cServePosture\x12\x12\n\neager_only\x18\x01 \x01(\x08\x12\x0e\n\x06reason\x18\x02 \x01(\t\x12\r\n\x05\x61\x63tor\x18\x03 \x01(\t*Q\n\x0fProtocolVersion\x12 \n\x1cPROTOCOL_VERSION_UNSPECIFIED\x10\x00\x12\x1c\n\x18PROTOCOL_VERSION_CURRENT\x10\x01*y\n\rResidencyTier\x12\x1e\n\x1aRESIDENCY_TIER_UNSPECIFIED\x10\x00\x12\x17\n\x13RESIDENCY_TIER_DISK\x10\x01\x12\x16\n\x12RESIDENCY_TIER_RAM\x10\x02\x12\x17\n\x13RESIDENCY_TIER_VRAM\x10\x03*\x92\x02\n\x11\x44\x65siredIntentKind\x12#\n\x1f\x44\x45SIRED_INTENT_KIND_UNSPECIFIED\x10\x00\x12#\n\x1f\x44\x45SIRED_INTENT_KIND_MATERIALIZE\x10\x01\x12&\n\"DESIRED_INTENT_KIND_FUNCTION_READY\x10\x02\x12$\n DESIRED_INTENT_KIND_CONFIG_APPLY\x10\x03\x12%\n!DESIRED_INTENT_KIND_COMPILE_ADOPT\x10\x04\x12\x1d\n\x19\x44\x45SIRED_INTENT_KIND_DRAIN\x10\x05\x12\x1f\n\x1b\x44\x45SIRED_INTENT_KIND_RUN_JOB\x10\x06*\x95\x02\n\x12\x44\x65siredIntentCause\x12$\n DESIRED_INTENT_CAUSE_UNSPECIFIED\x10\x00\x12\"\n\x1e\x44\x45SIRED_INTENT_CAUSE_COLD_BOOT\x10\x01\x12 \n\x1c\x44\x45SIRED_INTENT_CAUSE_REQUEST\x10\x02\x12$\n DESIRED_INTENT_CAUSE_PREPOSITION\x10\x03\x12&\n\"DESIRED_INTENT_CAUSE_CONFIG_CHANGE\x10\x04\x12$\n DESIRED_INTENT_CAUSE_REPLACEMENT\x10\x05\x12\x1f\n\x1b\x44\x45SIRED_INTENT_CAUSE_RETIRE\x10\x06*|\n\x11GoalReceiptStatus\x12#\n\x1fGOAL_RECEIPT_STATUS_UNSPECIFIED\x10\x00\x12 \n\x1cGOAL_RECEIPT_STATUS_ACCEPTED\x10\x01\x12 \n\x1cGOAL_RECEIPT_STATUS_REJECTED\x10\x02*\xc9\x02\n\x15LifecycleIntentStatus\x12\'\n#LIFECYCLE_INTENT_STATUS_UNSPECIFIED\x10\x00\x12$\n LIFECYCLE_INTENT_STATUS_ACCEPTED\x10\x01\x12#\n\x1fLIFECYCLE_INTENT_STATUS_WAITING\x10\x02\x12#\n\x1fLIFECYCLE_INTENT_STATUS_RUNNING\x10\x03\x12%\n!LIFECYCLE_INTENT_STATUS_SUCCEEDED\x10\x04\x12\"\n\x1eLIFECYCLE_INTENT_STATUS_FAILED\x10\x05\x12$\n LIFECYCLE_INTENT_STATUS_CANCELED\x10\x06\x12&\n\"LIFECYCLE_INTENT_STATUS_SUPERSEDED\x10\x07*\xc9\x08\n\x14LifecycleIntentStage\x12&\n\"LIFECYCLE_INTENT_STAGE_UNSPECIFIED\x10\x00\x12%\n!LIFECYCLE_INTENT_STAGE_VALIDATING\x10\x01\x12+\n\'LIFECYCLE_INTENT_STAGE_WAIT_TENANT_IDLE\x10\x02\x12(\n$LIFECYCLE_INTENT_STAGE_WAIT_REF_LOCK\x10\x03\x12)\n%LIFECYCLE_INTENT_STAGE_WAIT_LOAD_LOCK\x10\x04\x12(\n$LIFECYCLE_INTENT_STAGE_WAIT_GPU_SLOT\x10\x05\x12(\n$LIFECYCLE_INTENT_STAGE_WAIT_SNAPSHOT\x10\x06\x12-\n)LIFECYCLE_INTENT_STAGE_WAIT_DISK_HEADROOM\x10\x07\x12(\n$LIFECYCLE_INTENT_STAGE_WAIT_HOST_RAM\x10\x08\x12-\n)LIFECYCLE_INTENT_STAGE_WAIT_NETWORK_RETRY\x10\t\x12+\n\'LIFECYCLE_INTENT_STAGE_WAIT_REPLACEMENT\x10\n\x12#\n\x1fLIFECYCLE_INTENT_STAGE_FETCHING\x10\x0b\x12$\n LIFECYCLE_INTENT_STAGE_VERIFYING\x10\x0c\x12\"\n\x1eLIFECYCLE_INTENT_STAGE_ON_DISK\x10\r\x12\'\n#LIFECYCLE_INTENT_STAGE_LOADING_HOST\x10\x0e\x12)\n%LIFECYCLE_INTENT_STAGE_LOADING_DEVICE\x10\x0f\x12\"\n\x1eLIFECYCLE_INTENT_STAGE_WARMING\x10\x10\x12$\n LIFECYCLE_INTENT_STAGE_COMPILING\x10\x11\x12#\n\x1fLIFECYCLE_INTENT_STAGE_ADOPTING\x10\x12\x12/\n+LIFECYCLE_INTENT_STAGE_CONFIG_MATERIALIZING\x10\x13\x12\x33\n/LIFECYCLE_INTENT_STAGE_CONFIG_BINDINGS_APPLYING\x10\x14\x12,\n(LIFECYCLE_INTENT_STAGE_CONFIG_BOOT_STALE\x10\x15\x12 \n\x1cLIFECYCLE_INTENT_STAGE_READY\x10\x16\x12#\n\x1fLIFECYCLE_INTENT_STAGE_DRAINING\x10\x17\x12%\n!LIFECYCLE_INTENT_STAGE_FINALIZING\x10\x18\x12#\n\x1fLIFECYCLE_INTENT_STAGE_FLUSHING\x10\x19*\xc0\x03\n\x13LifecycleWaitReason\x12%\n!LIFECYCLE_WAIT_REASON_UNSPECIFIED\x10\x00\x12%\n!LIFECYCLE_WAIT_REASON_TENANT_WORK\x10\x01\x12\"\n\x1eLIFECYCLE_WAIT_REASON_REF_LOCK\x10\x02\x12#\n\x1fLIFECYCLE_WAIT_REASON_LOAD_LOCK\x10\x03\x12\"\n\x1eLIFECYCLE_WAIT_REASON_GPU_SLOT\x10\x04\x12\"\n\x1eLIFECYCLE_WAIT_REASON_SNAPSHOT\x10\x05\x12\'\n#LIFECYCLE_WAIT_REASON_DISK_HEADROOM\x10\x06\x12\"\n\x1eLIFECYCLE_WAIT_REASON_HOST_RAM\x10\x07\x12\'\n#LIFECYCLE_WAIT_REASON_NETWORK_RETRY\x10\x08\x12%\n!LIFECYCLE_WAIT_REASON_REPLACEMENT\x10\t\x12-\n)LIFECYCLE_WAIT_REASON_SINGLE_FLIGHT_OWNER\x10\n*\xa6\x05\n\x12LifecycleErrorCode\x12$\n LIFECYCLE_ERROR_CODE_UNSPECIFIED\x10\x00\x12\x30\n,LIFECYCLE_ERROR_CODE_MISSING_MANDATORY_FIELD\x10\x01\x12\x30\n,LIFECYCLE_ERROR_CODE_WORKER_SESSION_MISMATCH\x10\x02\x12)\n%LIFECYCLE_ERROR_CODE_RELEASE_MISMATCH\x10\x03\x12/\n+LIFECYCLE_ERROR_CODE_COMMAND_SEQ_REGRESSION\x10\x04\x12-\n)LIFECYCLE_ERROR_CODE_COMMAND_SEQ_CONFLICT\x10\x05\x12*\n&LIFECYCLE_ERROR_CODE_CONFIG_REGRESSION\x10\x06\x12)\n%LIFECYCLE_ERROR_CODE_UNKNOWN_FUNCTION\x10\x07\x12\x32\n.LIFECYCLE_ERROR_CODE_SNAPSHOT_IDENTITY_MISSING\x10\x08\x12+\n\'LIFECYCLE_ERROR_CODE_UNSUPPORTED_INTENT\x10\t\x12\x32\n.LIFECYCLE_ERROR_CODE_UNKNOWN_MANDATORY_COMMAND\x10\n\x12\x31\n-LIFECYCLE_ERROR_CODE_PROTOCOL_UNREPORTED_WAIT\x10\x0b\x12\x35\n1LIFECYCLE_ERROR_CODE_CONFIG_SNAPSHOT_WRITE_FAILED\x10\x0c\x12%\n!LIFECYCLE_ERROR_CODE_DRAIN_FAILED\x10\r*\xe1\x01\n\x17\x46unctionCapabilityState\x12)\n%FUNCTION_CAPABILITY_STATE_UNSPECIFIED\x10\x00\x12#\n\x1f\x46UNCTION_CAPABILITY_STATE_READY\x10\x01\x12&\n\"FUNCTION_CAPABILITY_STATE_APPLYING\x10\x02\x12(\n$FUNCTION_CAPABILITY_STATE_BOOT_STALE\x10\x03\x12$\n FUNCTION_CAPABILITY_STATE_FAILED\x10\x04*\xdf\x01\n\x16\x43onfigApplicationState\x12(\n$CONFIG_APPLICATION_STATE_UNSPECIFIED\x10\x00\x12%\n!CONFIG_APPLICATION_STATE_APPLYING\x10\x01\x12&\n\"CONFIG_APPLICATION_STATE_CONVERGED\x10\x02\x12\'\n#CONFIG_APPLICATION_STATE_BOOT_STALE\x10\x03\x12#\n\x1f\x43ONFIG_APPLICATION_STATE_FAILED\x10\x04*\x9b\x02\n\x14\x44rainLifecycleStatus\x12&\n\"DRAIN_LIFECYCLE_STATUS_UNSPECIFIED\x10\x00\x12#\n\x1f\x44RAIN_LIFECYCLE_STATUS_ACCEPTED\x10\x01\x12#\n\x1f\x44RAIN_LIFECYCLE_STATUS_DRAINING\x10\x02\x12%\n!DRAIN_LIFECYCLE_STATUS_FINALIZING\x10\x03\x12#\n\x1f\x44RAIN_LIFECYCLE_STATUS_FLUSHING\x10\x04\x12\"\n\x1e\x44RAIN_LIFECYCLE_STATUS_DRAINED\x10\x05\x12!\n\x1d\x44RAIN_LIFECYCLE_STATUS_FAILED\x10\x06*v\n\x0bStorageTier\x12\x1c\n\x18STORAGE_TIER_UNSPECIFIED\x10\x00\x12\x1a\n\x16STORAGE_TIER_CONTAINER\x10\x01\x12\x17\n\x13STORAGE_TIER_VOLUME\x10\x02\x12\x14\n\x10STORAGE_TIER_NFS\x10\x03*\xd8\x01\n\x0bWorkerPhase\x12\x1c\n\x18WORKER_PHASE_UNSPECIFIED\x10\x00\x12\x18\n\x14WORKER_PHASE_BOOTING\x10\x01\x12#\n\x1fWORKER_PHASE_DOWNLOADING_MODELS\x10\x02\x12\"\n\x1eWORKER_PHASE_LOADING_PIPELINES\x10\x03\x12\x18\n\x14WORKER_PHASE_WARMING\x10\x04\x12\x16\n\x12WORKER_PHASE_READY\x10\x05\x12\x16\n\x12WORKER_PHASE_ERROR\x10\x06*V\n\nMediaBytes\x12\x1b\n\x17MEDIA_BYTES_UNSPECIFIED\x10\x00\x12\x13\n\x0fMEDIA_BYTES_URL\x10\x01\x12\x16\n\x12MEDIA_BYTES_INLINE\x10\x02*\x9b\x01\n\tJobStatus\x12\x1a\n\x16JOB_STATUS_UNSPECIFIED\x10\x00\x12\x11\n\rJOB_STATUS_OK\x10\x01\x12\x16\n\x12JOB_STATUS_INVALID\x10\x02\x12\x18\n\x14JOB_STATUS_RETRYABLE\x10\x03\x12\x14\n\x10JOB_STATUS_FATAL\x10\x04\x12\x17\n\x13JOB_STATUS_CANCELED\x10\x05*S\n\x0bModelOpKind\x12\x1d\n\x19MODEL_OP_KIND_UNSPECIFIED\x10\x00\x12%\n!MODEL_OP_KIND_ADOPT_COMPILE_CACHE\x10\x01*\x82\x02\n\nModelState\x12\x1b\n\x17MODEL_STATE_UNSPECIFIED\x10\x00\x12\x1b\n\x17MODEL_STATE_DOWNLOADING\x10\x01\x12\x17\n\x13MODEL_STATE_ON_DISK\x10\x02\x12\x16\n\x12MODEL_STATE_IN_RAM\x10\x03\x12\x17\n\x13MODEL_STATE_IN_VRAM\x10\x04\x12\x17\n\x13MODEL_STATE_EVICTED\x10\x05\x12\x16\n\x12MODEL_STATE_FAILED\x10\x06\x12\x17\n\x13MODEL_STATE_ADOPTED\x10\x07\x12&\n\"MODEL_STATE_HOST_CAPACITY_PROGRESS\x10\x08*\x84\x01\n\rActivityState\x12\x1e\n\x1a\x41\x43TIVITY_STATE_UNSPECIFIED\x10\x00\x12\x1a\n\x16\x41\x43TIVITY_STATE_RUNNING\x10\x01\x12\x1c\n\x18\x41\x43TIVITY_STATE_COMPLETED\x10\x02\x12\x19\n\x15\x41\x43TIVITY_STATE_FAILED\x10\x03\x32g\n\x0fWorkerScheduler\x12T\n\x07\x43onnect\x12 .cozy.scheduler.v1.WorkerMessage\x1a#.cozy.scheduler.v1.SchedulerMessage(\x01\x30\x01\x42YZWgithub.com/cozy-creator/tensorhub/internal/orchestrator/grpc/pb/schedulerv1;schedulerv1b\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -36,52 +36,50 @@ _globals['_DESIREDRESIDENCY_SNAPSHOTSENTRY']._serialized_options = b'8\001' _globals['_RUNJOB_SNAPSHOTSENTRY']._loaded_options = None _globals['_RUNJOB_SNAPSHOTSENTRY']._serialized_options = b'8\001' - _globals['_RESOLVEDPROVENANCE_COMPONENTSOURCESENTRY']._loaded_options = None - _globals['_RESOLVEDPROVENANCE_COMPONENTSOURCESENTRY']._serialized_options = b'8\001' _globals['_JOBMETRICS_RUNTIMETERMSENTRY']._loaded_options = None _globals['_JOBMETRICS_RUNTIMETERMSENTRY']._serialized_options = b'8\001' _globals['_JOBMETRICS_STAGEMSENTRY']._loaded_options = None _globals['_JOBMETRICS_STAGEMSENTRY']._serialized_options = b'8\001' _globals['_FNUNAVAILABLE_AXESENTRY']._loaded_options = None _globals['_FNUNAVAILABLE_AXESENTRY']._serialized_options = b'8\001' - _globals['_PROTOCOLVERSION']._serialized_start=14586 - _globals['_PROTOCOLVERSION']._serialized_end=14667 - _globals['_RESIDENCYTIER']._serialized_start=14669 - _globals['_RESIDENCYTIER']._serialized_end=14790 - _globals['_DESIREDINTENTKIND']._serialized_start=14793 - _globals['_DESIREDINTENTKIND']._serialized_end=15067 - _globals['_DESIREDINTENTCAUSE']._serialized_start=15070 - _globals['_DESIREDINTENTCAUSE']._serialized_end=15347 - _globals['_GOALRECEIPTSTATUS']._serialized_start=15349 - _globals['_GOALRECEIPTSTATUS']._serialized_end=15473 - _globals['_LIFECYCLEINTENTSTATUS']._serialized_start=15476 - _globals['_LIFECYCLEINTENTSTATUS']._serialized_end=15805 - _globals['_LIFECYCLEINTENTSTAGE']._serialized_start=15808 - _globals['_LIFECYCLEINTENTSTAGE']._serialized_end=16905 - _globals['_LIFECYCLEWAITREASON']._serialized_start=16908 - _globals['_LIFECYCLEWAITREASON']._serialized_end=17356 - _globals['_LIFECYCLEERRORCODE']._serialized_start=17359 - _globals['_LIFECYCLEERRORCODE']._serialized_end=18037 - _globals['_FUNCTIONCAPABILITYSTATE']._serialized_start=18040 - _globals['_FUNCTIONCAPABILITYSTATE']._serialized_end=18265 - _globals['_CONFIGAPPLICATIONSTATE']._serialized_start=18268 - _globals['_CONFIGAPPLICATIONSTATE']._serialized_end=18491 - _globals['_DRAINLIFECYCLESTATUS']._serialized_start=18494 - _globals['_DRAINLIFECYCLESTATUS']._serialized_end=18777 - _globals['_STORAGETIER']._serialized_start=18779 - _globals['_STORAGETIER']._serialized_end=18897 - _globals['_WORKERPHASE']._serialized_start=18900 - _globals['_WORKERPHASE']._serialized_end=19116 - _globals['_MEDIABYTES']._serialized_start=19118 - _globals['_MEDIABYTES']._serialized_end=19204 - _globals['_JOBSTATUS']._serialized_start=19207 - _globals['_JOBSTATUS']._serialized_end=19362 - _globals['_MODELOPKIND']._serialized_start=19364 - _globals['_MODELOPKIND']._serialized_end=19447 - _globals['_MODELSTATE']._serialized_start=19450 - _globals['_MODELSTATE']._serialized_end=19708 - _globals['_ACTIVITYSTATE']._serialized_start=19711 - _globals['_ACTIVITYSTATE']._serialized_end=19843 + _globals['_PROTOCOLVERSION']._serialized_start=14256 + _globals['_PROTOCOLVERSION']._serialized_end=14337 + _globals['_RESIDENCYTIER']._serialized_start=14339 + _globals['_RESIDENCYTIER']._serialized_end=14460 + _globals['_DESIREDINTENTKIND']._serialized_start=14463 + _globals['_DESIREDINTENTKIND']._serialized_end=14737 + _globals['_DESIREDINTENTCAUSE']._serialized_start=14740 + _globals['_DESIREDINTENTCAUSE']._serialized_end=15017 + _globals['_GOALRECEIPTSTATUS']._serialized_start=15019 + _globals['_GOALRECEIPTSTATUS']._serialized_end=15143 + _globals['_LIFECYCLEINTENTSTATUS']._serialized_start=15146 + _globals['_LIFECYCLEINTENTSTATUS']._serialized_end=15475 + _globals['_LIFECYCLEINTENTSTAGE']._serialized_start=15478 + _globals['_LIFECYCLEINTENTSTAGE']._serialized_end=16575 + _globals['_LIFECYCLEWAITREASON']._serialized_start=16578 + _globals['_LIFECYCLEWAITREASON']._serialized_end=17026 + _globals['_LIFECYCLEERRORCODE']._serialized_start=17029 + _globals['_LIFECYCLEERRORCODE']._serialized_end=17707 + _globals['_FUNCTIONCAPABILITYSTATE']._serialized_start=17710 + _globals['_FUNCTIONCAPABILITYSTATE']._serialized_end=17935 + _globals['_CONFIGAPPLICATIONSTATE']._serialized_start=17938 + _globals['_CONFIGAPPLICATIONSTATE']._serialized_end=18161 + _globals['_DRAINLIFECYCLESTATUS']._serialized_start=18164 + _globals['_DRAINLIFECYCLESTATUS']._serialized_end=18447 + _globals['_STORAGETIER']._serialized_start=18449 + _globals['_STORAGETIER']._serialized_end=18567 + _globals['_WORKERPHASE']._serialized_start=18570 + _globals['_WORKERPHASE']._serialized_end=18786 + _globals['_MEDIABYTES']._serialized_start=18788 + _globals['_MEDIABYTES']._serialized_end=18874 + _globals['_JOBSTATUS']._serialized_start=18877 + _globals['_JOBSTATUS']._serialized_end=19032 + _globals['_MODELOPKIND']._serialized_start=19034 + _globals['_MODELOPKIND']._serialized_end=19117 + _globals['_MODELSTATE']._serialized_start=19120 + _globals['_MODELSTATE']._serialized_end=19378 + _globals['_ACTIVITYSTATE']._serialized_start=19381 + _globals['_ACTIVITYSTATE']._serialized_end=19513 _globals['_WORKERMESSAGE']._serialized_start=46 _globals['_WORKERMESSAGE']._serialized_end=805 _globals['_SCHEDULERMESSAGE']._serialized_start=808 @@ -157,65 +155,59 @@ _globals['_RESOLVEDCOMPUTE']._serialized_start=9521 _globals['_RESOLVEDCOMPUTE']._serialized_end=9578 _globals['_MODELBINDING']._serialized_start=9581 - _globals['_MODELBINDING']._serialized_end=9819 - _globals['_LORAOVERLAY']._serialized_start=9821 - _globals['_LORAOVERLAY']._serialized_end=9906 - _globals['_SNAPSHOT']._serialized_start=9909 - _globals['_SNAPSHOT']._serialized_end=10042 - _globals['_RESOLVEDPROVENANCE']._serialized_start=10045 - _globals['_RESOLVEDPROVENANCE']._serialized_end=10297 - _globals['_RESOLVEDPROVENANCE_COMPONENTSOURCESENTRY']._serialized_start=10206 - _globals['_RESOLVEDPROVENANCE_COMPONENTSOURCESENTRY']._serialized_end=10297 - _globals['_COMPONENTORIGIN']._serialized_start=10299 - _globals['_COMPONENTORIGIN']._serialized_end=10367 - _globals['_SNAPSHOTFILE']._serialized_start=10370 - _globals['_SNAPSHOTFILE']._serialized_end=10534 - _globals['_CHUNKREF']._serialized_start=10536 - _globals['_CHUNKREF']._serialized_end=10588 - _globals['_JOBACCEPTED']._serialized_start=10590 - _globals['_JOBACCEPTED']._serialized_end=10640 - _globals['_JOBRESULT']._serialized_start=10643 - _globals['_JOBRESULT']._serialized_end=10926 - _globals['_ADJUSTMENT']._serialized_start=10928 - _globals['_ADJUSTMENT']._serialized_end=11007 - _globals['_JOBMETRICS']._serialized_start=11010 - _globals['_JOBMETRICS']._serialized_end=11791 - _globals['_JOBMETRICS_RUNTIMETERMSENTRY']._serialized_start=11692 - _globals['_JOBMETRICS_RUNTIMETERMSENTRY']._serialized_end=11743 - _globals['_JOBMETRICS_STAGEMSENTRY']._serialized_start=11745 - _globals['_JOBMETRICS_STAGEMSENTRY']._serialized_end=11791 - _globals['_MEASUREDPOSTURE']._serialized_start=11794 - _globals['_MEASUREDPOSTURE']._serialized_end=12141 - _globals['_APPLIEDTECHNIQUE']._serialized_start=12143 - _globals['_APPLIEDTECHNIQUE']._serialized_end=12248 - _globals['_COMPONENTPOSTURE']._serialized_start=12250 - _globals['_COMPONENTPOSTURE']._serialized_end=12365 - _globals['_RESOURCESHORTFALL']._serialized_start=12367 - _globals['_RESOURCESHORTFALL']._serialized_end=12470 - _globals['_JOBPROGRESS']._serialized_start=12472 - _globals['_JOBPROGRESS']._serialized_end=12571 - _globals['_CANCELJOB']._serialized_start=12573 - _globals['_CANCELJOB']._serialized_end=12621 - _globals['_MODELOP']._serialized_start=12624 - _globals['_MODELOP']._serialized_end=12790 - _globals['_MODELEVENT']._serialized_start=12793 - _globals['_MODELEVENT']._serialized_end=13335 - _globals['_ACTIVITYUPDATE']._serialized_start=13338 - _globals['_ACTIVITYUPDATE']._serialized_end=13784 - _globals['_BOOTPHASE']._serialized_start=13787 - _globals['_BOOTPHASE']._serialized_end=14141 - _globals['_FNUNAVAILABLE']._serialized_start=14144 - _globals['_FNUNAVAILABLE']._serialized_end=14317 - _globals['_FNUNAVAILABLE_AXESENTRY']._serialized_start=14274 - _globals['_FNUNAVAILABLE_AXESENTRY']._serialized_end=14317 - _globals['_FNDEGRADED']._serialized_start=14319 - _globals['_FNDEGRADED']._serialized_end=14431 - _globals['_DRAIN']._serialized_start=14433 - _globals['_DRAIN']._serialized_end=14461 - _globals['_TOKENREFRESH']._serialized_start=14463 - _globals['_TOKENREFRESH']._serialized_end=14517 - _globals['_SERVEPOSTURE']._serialized_start=14519 - _globals['_SERVEPOSTURE']._serialized_end=14584 - _globals['_WORKERSCHEDULER']._serialized_start=19845 - _globals['_WORKERSCHEDULER']._serialized_end=19948 + _globals['_MODELBINDING']._serialized_end=9874 + _globals['_LORAOVERLAY']._serialized_start=9876 + _globals['_LORAOVERLAY']._serialized_end=9961 + _globals['_SNAPSHOT']._serialized_start=9963 + _globals['_SNAPSHOT']._serialized_end=10037 + _globals['_SNAPSHOTFILE']._serialized_start=10040 + _globals['_SNAPSHOTFILE']._serialized_end=10204 + _globals['_CHUNKREF']._serialized_start=10206 + _globals['_CHUNKREF']._serialized_end=10258 + _globals['_JOBACCEPTED']._serialized_start=10260 + _globals['_JOBACCEPTED']._serialized_end=10310 + _globals['_JOBRESULT']._serialized_start=10313 + _globals['_JOBRESULT']._serialized_end=10596 + _globals['_ADJUSTMENT']._serialized_start=10598 + _globals['_ADJUSTMENT']._serialized_end=10677 + _globals['_JOBMETRICS']._serialized_start=10680 + _globals['_JOBMETRICS']._serialized_end=11461 + _globals['_JOBMETRICS_RUNTIMETERMSENTRY']._serialized_start=11362 + _globals['_JOBMETRICS_RUNTIMETERMSENTRY']._serialized_end=11413 + _globals['_JOBMETRICS_STAGEMSENTRY']._serialized_start=11415 + _globals['_JOBMETRICS_STAGEMSENTRY']._serialized_end=11461 + _globals['_MEASUREDPOSTURE']._serialized_start=11464 + _globals['_MEASUREDPOSTURE']._serialized_end=11811 + _globals['_APPLIEDTECHNIQUE']._serialized_start=11813 + _globals['_APPLIEDTECHNIQUE']._serialized_end=11918 + _globals['_COMPONENTPOSTURE']._serialized_start=11920 + _globals['_COMPONENTPOSTURE']._serialized_end=12035 + _globals['_RESOURCESHORTFALL']._serialized_start=12037 + _globals['_RESOURCESHORTFALL']._serialized_end=12140 + _globals['_JOBPROGRESS']._serialized_start=12142 + _globals['_JOBPROGRESS']._serialized_end=12241 + _globals['_CANCELJOB']._serialized_start=12243 + _globals['_CANCELJOB']._serialized_end=12291 + _globals['_MODELOP']._serialized_start=12294 + _globals['_MODELOP']._serialized_end=12460 + _globals['_MODELEVENT']._serialized_start=12463 + _globals['_MODELEVENT']._serialized_end=13005 + _globals['_ACTIVITYUPDATE']._serialized_start=13008 + _globals['_ACTIVITYUPDATE']._serialized_end=13454 + _globals['_BOOTPHASE']._serialized_start=13457 + _globals['_BOOTPHASE']._serialized_end=13811 + _globals['_FNUNAVAILABLE']._serialized_start=13814 + _globals['_FNUNAVAILABLE']._serialized_end=13987 + _globals['_FNUNAVAILABLE_AXESENTRY']._serialized_start=13944 + _globals['_FNUNAVAILABLE_AXESENTRY']._serialized_end=13987 + _globals['_FNDEGRADED']._serialized_start=13989 + _globals['_FNDEGRADED']._serialized_end=14101 + _globals['_DRAIN']._serialized_start=14103 + _globals['_DRAIN']._serialized_end=14131 + _globals['_TOKENREFRESH']._serialized_start=14133 + _globals['_TOKENREFRESH']._serialized_end=14187 + _globals['_SERVEPOSTURE']._serialized_start=14189 + _globals['_SERVEPOSTURE']._serialized_end=14254 + _globals['_WORKERSCHEDULER']._serialized_start=19515 + _globals['_WORKERSCHEDULER']._serialized_end=19618 # @@protoc_insertion_point(module_scope) diff --git a/src/gen_worker/pb/worker_scheduler_pb2.pyi b/src/gen_worker/pb/worker_scheduler_pb2.pyi index 45dfbc2e8..7c40a37a6 100644 --- a/src/gen_worker/pb/worker_scheduler_pb2.pyi +++ b/src/gen_worker/pb/worker_scheduler_pb2.pyi @@ -1019,7 +1019,7 @@ class ResolvedCompute(_message.Message): def __init__(self, accelerator: _Optional[str] = ..., gpu_index: _Optional[int] = ...) -> None: ... class ModelBinding(_message.Message): - __slots__ = ("slot", "ref", "loras", "inference_defaults", "objective", "distilled", "distilled_status", "model", "manifest_digest") + __slots__ = ("slot", "ref", "loras", "inference_defaults", "objective", "distilled", "distilled_status", "model", "bind_contract_digest", "bind_contract_url") SLOT_FIELD_NUMBER: _ClassVar[int] REF_FIELD_NUMBER: _ClassVar[int] LORAS_FIELD_NUMBER: _ClassVar[int] @@ -1028,7 +1028,8 @@ class ModelBinding(_message.Message): DISTILLED_FIELD_NUMBER: _ClassVar[int] DISTILLED_STATUS_FIELD_NUMBER: _ClassVar[int] MODEL_FIELD_NUMBER: _ClassVar[int] - MANIFEST_DIGEST_FIELD_NUMBER: _ClassVar[int] + BIND_CONTRACT_DIGEST_FIELD_NUMBER: _ClassVar[int] + BIND_CONTRACT_URL_FIELD_NUMBER: _ClassVar[int] slot: str ref: str loras: _containers.RepeatedCompositeFieldContainer[LoraOverlay] @@ -1037,8 +1038,9 @@ class ModelBinding(_message.Message): distilled: bool distilled_status: str model: str - manifest_digest: str - def __init__(self, slot: _Optional[str] = ..., ref: _Optional[str] = ..., loras: _Optional[_Iterable[_Union[LoraOverlay, _Mapping]]] = ..., inference_defaults: _Optional[str] = ..., objective: _Optional[str] = ..., distilled: _Optional[bool] = ..., distilled_status: _Optional[str] = ..., model: _Optional[str] = ..., manifest_digest: _Optional[str] = ...) -> None: ... + bind_contract_digest: str + bind_contract_url: str + def __init__(self, slot: _Optional[str] = ..., ref: _Optional[str] = ..., loras: _Optional[_Iterable[_Union[LoraOverlay, _Mapping]]] = ..., inference_defaults: _Optional[str] = ..., objective: _Optional[str] = ..., distilled: _Optional[bool] = ..., distilled_status: _Optional[str] = ..., model: _Optional[str] = ..., bind_contract_digest: _Optional[str] = ..., bind_contract_url: _Optional[str] = ...) -> None: ... class LoraOverlay(_message.Message): __slots__ = ("ref", "weight", "inference_defaults", "model") @@ -1053,43 +1055,12 @@ class LoraOverlay(_message.Message): def __init__(self, ref: _Optional[str] = ..., weight: _Optional[float] = ..., inference_defaults: _Optional[str] = ..., model: _Optional[str] = ...) -> None: ... class Snapshot(_message.Message): - __slots__ = ("digest", "files", "provenance") + __slots__ = ("digest", "files") DIGEST_FIELD_NUMBER: _ClassVar[int] FILES_FIELD_NUMBER: _ClassVar[int] - PROVENANCE_FIELD_NUMBER: _ClassVar[int] digest: str files: _containers.RepeatedCompositeFieldContainer[SnapshotFile] - provenance: ResolvedProvenance - def __init__(self, digest: _Optional[str] = ..., files: _Optional[_Iterable[_Union[SnapshotFile, _Mapping]]] = ..., provenance: _Optional[_Union[ResolvedProvenance, _Mapping]] = ...) -> None: ... - -class ResolvedProvenance(_message.Message): - __slots__ = ("repo", "release", "variant_id", "component_sources") - class ComponentSourcesEntry(_message.Message): - __slots__ = ("key", "value") - KEY_FIELD_NUMBER: _ClassVar[int] - VALUE_FIELD_NUMBER: _ClassVar[int] - key: str - value: ComponentOrigin - def __init__(self, key: _Optional[str] = ..., value: _Optional[_Union[ComponentOrigin, _Mapping]] = ...) -> None: ... - REPO_FIELD_NUMBER: _ClassVar[int] - RELEASE_FIELD_NUMBER: _ClassVar[int] - VARIANT_ID_FIELD_NUMBER: _ClassVar[int] - COMPONENT_SOURCES_FIELD_NUMBER: _ClassVar[int] - repo: str - release: str - variant_id: str - component_sources: _containers.MessageMap[str, ComponentOrigin] - def __init__(self, repo: _Optional[str] = ..., release: _Optional[str] = ..., variant_id: _Optional[str] = ..., component_sources: _Optional[_Mapping[str, ComponentOrigin]] = ...) -> None: ... - -class ComponentOrigin(_message.Message): - __slots__ = ("repo", "release", "variant_id") - REPO_FIELD_NUMBER: _ClassVar[int] - RELEASE_FIELD_NUMBER: _ClassVar[int] - VARIANT_ID_FIELD_NUMBER: _ClassVar[int] - repo: str - release: str - variant_id: str - def __init__(self, repo: _Optional[str] = ..., release: _Optional[str] = ..., variant_id: _Optional[str] = ...) -> None: ... + def __init__(self, digest: _Optional[str] = ..., files: _Optional[_Iterable[_Union[SnapshotFile, _Mapping]]] = ...) -> None: ... class SnapshotFile(_message.Message): __slots__ = ("path", "size_bytes", "blake3", "url", "digest", "chunk_size_bytes", "chunks") diff --git a/src/gen_worker/procsplit/actions.py b/src/gen_worker/procsplit/actions.py index 7ad5388df..c474d0abd 100644 --- a/src/gen_worker/procsplit/actions.py +++ b/src/gen_worker/procsplit/actions.py @@ -110,9 +110,9 @@ def _a( timeout_s=60.0, ), _a( - "release.compiled_graphs", + "bind.compiled_graphs", "GET", - r"^/v1/worker/releases/[A-Za-z0-9._+-]{1,128}/compiled-graphs$", + r"^/v1/worker/bind-contracts/sha256:[0-9a-f]{64}/compiled-graphs$", query=("lane", "sm"), timeout_s=30.0, ), diff --git a/src/gen_worker/release/derive.py b/src/gen_worker/release/derive.py index f368caf7b..6ddea1b2e 100644 --- a/src/gen_worker/release/derive.py +++ b/src/gen_worker/release/derive.py @@ -218,14 +218,6 @@ class ReleaseDeriveResult: #: BEFORE the release-wide union, which is what a pipeline log wants to #: read when two classes share a lane. class_lane_graphs: tuple[tuple[str, str, tuple[str, ...]], ...] = () - #: The component names the construction census states, sorted. - census_components: tuple[str, ...] = () - #: Why there is NO census — ``NO_PIPELINE_INDEX`` for a tree that is not a - #: diffusers pipeline and is therefore never streaming-served. Empty means - #: a census was emitted. There is no third value: a census that could not - #: be COMPUTED fails the build. - census_absent: str = "" - @property def eager_permanent(self) -> bool: return not self.lane_graphs @@ -947,11 +939,12 @@ def _construction_census( **ITS KEY IS IMAGE x CONFIG TREE** (th#2287's adjudication, 2026-08-21), not source x image like the rest of the release contract — and this function is already at that key, because it is the BIND-TIME - ``release derive --checkpoint`` run that censuses, and that run has exactly - one primary tree. The hub re-keys the emitted document onto the bind - (release x image digest x config digest) and never asks this side to. + ``release derive --checkpoint`` run that censuses. pgw#1653 runs this once + for every distinct tree named to that command; the hub keys each answer on + (release x image digest x config digest) and never asks this side to invent + or restate the digest. - **ONE census for the whole release, and the invariance is PROVEN here.** + **ONE census for this config tree, and lane invariance is PROVEN here.** A lane's only effect on construction is the dtype it casts wide floats to, and the census records those as ``census.LANE_DTYPE`` because the lane contract and ``engine._assert_lane_dtype`` already state that fact exactly. @@ -1017,6 +1010,43 @@ def _construction_census( return document +PRIMARY_CENSUS_OWNER = "$primary" + + +def _construction_censuses( + primary: Path, + trees: Mapping[str, Path], + lanes: tuple[tuple[str, Any], ...], +) -> list[dict[str, Any]]: + """One census per DISTINCT config tree, with every owner named. + + The owner labels are routing information inside the derive result, not + identity. The hub already computes each tree's config digest and keys the + Bind Contract on that digest. Multiple class/slot labels may resolve to the + same tree; they are one answer with several owners, never duplicated facts. + """ + + by_tree: dict[Path, set[str]] = {} + + def add(owner: str, tree: Path) -> None: + resolved = Path(tree).resolve() + by_tree.setdefault(resolved, set()).add(owner) + + add(PRIMARY_CENSUS_OWNER, primary) + for owner, tree in sorted(trees.items()): + add(str(owner), Path(tree)) + + rows = [ + { + "owners": sorted(owners), + "census": _construction_census(tree, lanes), + } + for tree, owners in by_tree.items() + ] + rows.sort(key=lambda row: tuple(row["owners"])) + return rows + + def _defaults_schema(model_type: Optional[type]) -> Optional[dict[str, Any]]: if model_type is None: @@ -1737,13 +1767,13 @@ def derive_release( warnings: list[str] = [] - # THE CONSTRUCTION CENSUS (pgw#1647), before any tracing — it is the cheaper - # question, and a tree that cannot be built from its own configs must not - # spend a trace first. ONE per release, over the PRIMARY tree, under the - # union of every declared lane of every subject class: the census is - # lane-invariant by construction and that invariance is checked here. - construction_census = _construction_census( - checkpoint_dir, + # THE CONSTRUCTION CENSUSES (pgw#1653), before any tracing — this is the + # cheaper question, and a tree that cannot be built from its own configs + # must not spend a trace first. Exactly one per DISTINCT config tree, under + # the union of every declared lane: a class/slot alias of the same tree is + # an additional owner on one row, never a duplicate fact. + construction_censuses = _construction_censuses( + checkpoint_dir, checkpoint_trees, tuple( ( lane_contract_handle(f"class {cls.__name__!r}", lane), @@ -1753,26 +1783,6 @@ def derive_release( for lane in model_declared_lanes(cls) ), ) - for cls in subjects: - owner = _checkpoint_tree(checkpoint_trees, checkpoint_dir, cls) - if owner != checkpoint_dir: - # LOUD, not silent, and not a refusal. The census describes the tree - # it was taken from, and th#2281 keys its storage by CONFIG DIGEST — - # so a class loading its own tree needs its own census row, which - # this document has no field for yet. Saying so is the honest - # answer; emitting the primary tree's census as if it described this - # class's would be the "second carrier" defect one level up. - warnings.append( - f"class {cls.__name__!r} loads its own checkpoint tree " - f"({owner}), and the construction census in this document is " - f"the PRIMARY tree's ({checkpoint_dir}). Nothing is MIS-KEYED: " - f"the census's key is image x config tree (th#2287), this run " - f"binds the primary tree, and the hub files the document under " - f"that bind. It is INCOMPLETE — no published census describes " - f"the auxiliary tree, so its serve-time fence replays only the " - f"census it builds itself. A per-tree census belongs to the " - f"BIND CONTRACT (th#2287 slice 2b), not to this document" - ) derivations = [ _derive_class( @@ -1821,12 +1831,12 @@ def derive_release( # that makes the lockstep matter more than the refusal does. "graphs": graphs_document.as_dict(), "lane_contracts": lane_contracts, - # pgw#1647 / th#2281. What the module IS, per declared lane — the - # complete tensor set incl. computed non-persistent buffers, the tied - # alias groups, the quantizer's swapped classes, eval mode. The hub - # stores it and forwards it; it interprets no torch semantics. The - # serve-time fence REPLAYS this instead of re-deriving trust. - "construction_census": construction_census, + # pgw#1653 / th#2290. What every DISTINCT config tree builds in this + # image. Owner labels let the hub join each row to the config digest it + # already computed; the digest is the Bind Contract identity and is + # never restated in this document. The singular primary-only field is + # deleted: absence of an auxiliary answer can no longer render green. + "construction_censuses": construction_censuses, "entrypoints": entrypoints, # pgw#1650: THE PER-CLASS BREAKDOWN, and the authoritative one. A # release derives EVERY compile-marking class (Paul, 2026-08-21), each @@ -1882,10 +1892,6 @@ def derive_release( (d.name, lane.contract, tuple(record.graph for record in lane.graphs)) for d in derivations for lane in d.lanes ), - census_components=tuple( - sorted(construction_census.get("components", {})) - ), - census_absent=str(construction_census.get("absent", "")), ) diff --git a/src/gen_worker/serving/__main__.py b/src/gen_worker/serving/__main__.py index fd1b3e1ad..f5771277a 100644 --- a/src/gen_worker/serving/__main__.py +++ b/src/gen_worker/serving/__main__.py @@ -31,13 +31,13 @@ def __init__( self.bearer = bearer self.timeout_s = timeout_s - def release_compiled_graphs( - self, release_id: str, lane: str, sm: str + def bind_compiled_graphs( + self, bind_contract_digest: str, lane: str, sm: str ) -> Mapping[str, Any]: query = urllib.parse.urlencode({"lane": lane, "sm": sm}) url = ( - f"{self.base_url}/v1/worker/releases/" - f"{urllib.parse.quote(release_id, safe='')}/compiled-graphs?{query}" + f"{self.base_url}/v1/worker/bind-contracts/" + f"{urllib.parse.quote(bind_contract_digest, safe='')}/compiled-graphs?{query}" ) request = urllib.request.Request(url) if self.bearer: @@ -50,8 +50,7 @@ def release_compiled_graphs( from .hub_store import ReleaseNotStamped raise ReleaseNotStamped( - f"release {release_id} carries no stamped compiled-graph " - f"document; serving eager" + f"Bind Contract {bind_contract_digest} is unavailable; serving eager" ) from exc raise SystemExit( f"adopt route answered {exc.code} from {url}: " @@ -106,13 +105,15 @@ def _parser() -> argparse.ArgumentParser: parser.add_argument("--hub-base-url", default="", help="hub base URL for the th#2133 adopt route") parser.add_argument("--release", default="", help="endpoint-release id (hub adopt)") + parser.add_argument("--bind-contract-digest", default="", + help="immutable Bind Contract selected for --checkpoint") parser.add_argument("--hub-token", default="", help="worker bearer for the adopt route (local/CI only; " "in production the parent holds the credential)") parser.add_argument("--via-broker", action="store_true", help="make the adopt ask through procsplit's action " "broker — the PRODUCTION path, allowlisted as " - "`release.compiled_graphs`") + "`bind.compiled_graphs`") parser.add_argument("--sm", default="", help="this GPU's sm (e.g. sm_89)") parser.add_argument( "--env-lockfile", default="", @@ -168,6 +169,8 @@ def _adoption_source( if not args.release: raise SystemExit("--release is required with --hub-base-url") + if not args.bind_contract_digest: + raise SystemExit("--bind-contract-digest is required with --hub-base-url") transport: Any = ( BrokerReleaseGraphTransport( base_url=args.hub_base_url, bearer=args.hub_token @@ -175,7 +178,9 @@ def _adoption_source( if args.via_broker else HttpReleaseGraphTransport(args.hub_base_url, args.hub_token) ) - store: Any = HubGraphStore(transport, args.release, args.lane, args.sm) + store: Any = HubGraphStore( + transport, args.release, args.bind_contract_digest, args.lane, args.sm + ) try: document = store.get_graphs(args.release) except ReleaseNotStamped as exc: diff --git a/src/gen_worker/serving/context.py b/src/gen_worker/serving/context.py index 44a8604cf..dc2679a97 100644 --- a/src/gen_worker/serving/context.py +++ b/src/gen_worker/serving/context.py @@ -132,12 +132,18 @@ class DeployBinding: lane_trees: Mapping[str, Path] = field(default_factory=dict) lane_verdicts: Mapping[str, str] = field(default_factory=dict) lane_bytes: Mapping[str, int] = field(default_factory=dict) + # pgw#1653: the exact Bind Contract selected for this config tree. The + # object owns its census and portable bind identity; the reporter records + # a typed mismatch against that identity before load refuses. + bind_contract: Any = None + bind_refusal_reporter: Optional[Callable[[Any, Any], None]] = None class LoaderEngine(Protocol): def build( - self, pipeline_cls: type, *, checkpoint_dir: Path, lane: Any + self, pipeline_cls: type, *, checkpoint_dir: Path, lane: Any, + expected_census: Any = None, ) -> Any: ... @@ -393,11 +399,7 @@ def load_pipeline(self, pipeline_cls: Type[P]) -> P: def load(self, pipeline_cls: Type[P]) -> P: """Build ``pipeline_cls`` with this checkpoint's weights resident.""" if self._engine is not None: - built: P = self._engine.build( - pipeline_cls, - checkpoint_dir=self.checkpoint_dir, - lane=self._lane, - ) + built: P = self._streaming_build(self._engine, pipeline_cls) return built from_pretrained = getattr(pipeline_cls, "from_pretrained", None) if from_pretrained is None: @@ -426,11 +428,7 @@ def load(self, pipeline_cls: Type[P]) -> P: if bound: engine = self._engine assert engine is not None - rebound: P = engine.build( - pipeline_cls, - checkpoint_dir=self.checkpoint_dir, - lane=self._lane, - ) + rebound: P = self._streaming_build(engine, pipeline_cls) return rebound raise ProjectedTreeNotStreamable( self.checkpoint_dir, @@ -465,6 +463,37 @@ def load(self, pipeline_cls: Type[P]) -> P: to(dtype) return self._placed(loaded) + def _streaming_build(self, engine: LoaderEngine, pipeline_cls: Type[P]) -> P: + contract = self._binding.bind_contract + if contract is None: + raise RuntimeError( + f"ctx.load({pipeline_cls.__name__}): no Bind Contract was " + "resolved for this checkpoint tree. The construction census " + "is a required serve input under pgw#1653; absence never " + "falls back to comparing the module with itself." + ) + try: + return cast(P, engine.build( + pipeline_cls, + checkpoint_dir=self.checkpoint_dir, + lane=self._lane, + expected_census=contract.census, + )) + except Exception as exc: + from .streaming.census import CensusMismatch + + if isinstance(exc, CensusMismatch): + reporter = self._binding.bind_refusal_reporter + if reporter is not None: + try: + reporter(contract, exc) + except Exception: + logger.exception( + "bind contract mismatch report failed; the load " + "still refuses against %s", contract.digest, + ) + raise + def _placed(self, pipeline: P) -> P: if not self._device: return pipeline diff --git a/src/gen_worker/serving/hub_store.py b/src/gen_worker/serving/hub_store.py index ca2e0e363..ca53f490f 100644 --- a/src/gen_worker/serving/hub_store.py +++ b/src/gen_worker/serving/hub_store.py @@ -1,6 +1,6 @@ -"""The hub-backed GraphStore (pgw#1372): th#2133's adopt route, thin. +"""The hub-backed GraphStore: the Bind Contract's graph references, thin. -One ask per boot — ``GET /v1/worker/releases//compiled-graphs +One ask per bind — ``GET /v1/worker/bind-contracts//compiled-graphs ?lane=&sm=`` — answers, for every graph in the release's lane, the artifact for THIS release's exact env + this sm (content digest, presigned transport manifest, the mint's requirements manifest) or a per-graph MISS. @@ -19,8 +19,9 @@ SPECULATED ``{document, artifacts, misses}`` shape before the route landed, and the route answers something else:: - {"object": "release_compiled_graphs", "release_id": "...", - "binding_generation": 0, "env_compile_stack": [["torch", "2.13.0"], ...], + {"object": "bind_compiled_graphs", "release_id": "...", + "bind_contract_digest": "sha256:...", + "env_compile_stack": [["torch", "2.13.0"], ...], "lane": "sdxl.diffusers@1+plain.bf16@1", "lane_stamped": true, "lane_contract": {"stamp": ..., "contract_digest": ..., "document": {...}, "requires": ...}, @@ -57,7 +58,7 @@ HTTP_TIMEOUT_S = 60.0 -ADOPT_PATH = "/v1/worker/releases/{release_id}/compiled-graphs" +ADOPT_PATH = "/v1/worker/bind-contracts/{bind_contract_digest}/compiled-graphs" HIT = "hit" @@ -69,8 +70,8 @@ class ReleaseNotStamped(Exception): class ReleaseGraphTransport(Protocol): """What the hub wiring must provide — and all it must provide.""" - def release_compiled_graphs( - self, release_id: str, lane: str, sm: str + def bind_compiled_graphs( + self, bind_contract_digest: str, lane: str, sm: str ) -> Mapping[str, Any]: """The one boot ask: the release's per-graph adopt answer.""" ... @@ -94,14 +95,14 @@ def __init__( self.bearer = bearer self.timeout_s = timeout_s - def release_compiled_graphs( - self, release_id: str, lane: str, sm: str + def bind_compiled_graphs( + self, bind_contract_digest: str, lane: str, sm: str ) -> Mapping[str, Any]: from ..procsplit import broker response = broker.request( "GET", - ADOPT_PATH.format(release_id=str(release_id)), + ADOPT_PATH.format(bind_contract_digest=str(bind_contract_digest)), base_url=self.base_url, bearer=self.bearer, params={"lane": str(lane), "sm": str(sm)}, @@ -109,13 +110,12 @@ def release_compiled_graphs( ) if response.status_code == 404: raise ReleaseNotStamped( - f"release {release_id} carries no stamped compiled-graph " - f"document; serving eager" + f"Bind Contract {bind_contract_digest} is unavailable; serving eager" ) if response.status_code != 200: raise StoreError( f"adopt route answered {response.status_code} for release " - f"{release_id} lane={lane} sm={sm}: {response.text[:400]}" + f"{bind_contract_digest} lane={lane} sm={sm}: {response.text[:400]}" ) payload = response.json() if not isinstance(payload, Mapping): @@ -140,10 +140,12 @@ def _graph_rows(answer: Mapping[str, Any]) -> Iterable[Mapping[str, Any]]: class HubGraphStore: def __init__( - self, transport: ReleaseGraphTransport, release_id: str, lane: str, sm: str + self, transport: ReleaseGraphTransport, release_id: str, + bind_contract_digest: str, lane: str, sm: str, ) -> None: self._transport = transport self._release_id = str(release_id) + self._bind_contract_digest = str(bind_contract_digest) self._lane = str(lane) self._sm = str(sm) self._answer: Optional[Mapping[str, Any]] = None @@ -152,8 +154,8 @@ def __init__( def _resolve(self) -> Mapping[str, Any]: if self._answer is None: - answer = self._transport.release_compiled_graphs( - self._release_id, self._lane, self._sm + answer = self._transport.bind_compiled_graphs( + self._bind_contract_digest, self._lane, self._sm ) if not isinstance(answer, Mapping): raise StoreError( @@ -203,6 +205,17 @@ def get_graphs(self, name: str) -> Optional[GraphSetDocument]: if self._document is not None: return self._document answer = self._resolve() + if str(answer.get("object") or "") != "bind_compiled_graphs": + raise StoreError( + f"Bind Contract {self._bind_contract_digest}: adopt route " + f"answered object={answer.get('object')!r}, not " + "'bind_compiled_graphs'" + ) + if str(answer.get("bind_contract_digest") or "") != self._bind_contract_digest: + raise StoreError( + f"Bind Contract {self._bind_contract_digest}: adopt route " + f"answered for {answer.get('bind_contract_digest')!r}" + ) raw_stack = answer.get("env_compile_stack") if not isinstance(raw_stack, (list, tuple)) or not raw_stack: raise StoreError( diff --git a/src/gen_worker/serving/mint.py b/src/gen_worker/serving/mint.py index 151657e7e..9aa1b7194 100644 --- a/src/gen_worker/serving/mint.py +++ b/src/gen_worker/serving/mint.py @@ -40,6 +40,11 @@ def hole_work_list(host: Any) -> Tuple[Any, ...]: return tuple(getattr(host, "holes", ()) or ()) +def adoption_session(host: Any) -> Any: + """The one graph session a mint owns, directly or through a host.""" + return getattr(host, "adoption", host) + + def entry_workers( holes: int, *, @@ -806,7 +811,8 @@ def worker() -> None: ) def _mint_one(self, record: Any, arm_lock: threading.Lock) -> MintedHole: - env = self.host.adoption.env + session = adoption_session(self.host) + env = session.env scratch = self.artifacts_dir / env.value / "build" / record.graph scratch.parent.mkdir(parents=True, exist_ok=True) position = self.artifacts_dir / env.value / f"{record.graph}.so" @@ -837,7 +843,7 @@ def _mint_one(self, record: Any, arm_lock: threading.Lock) -> MintedHole: armed = False try: with arm_lock: - self.host.adoption.arm(record, Path(fetched)) + session.arm(record, Path(fetched)) armed = True except Exception as exc: # noqa: BLE001 — published beats armed logger.warning( diff --git a/src/gen_worker/serving/serve_adoption.py b/src/gen_worker/serving/serve_adoption.py index a6049e01e..de6f2ece7 100644 --- a/src/gen_worker/serving/serve_adoption.py +++ b/src/gen_worker/serving/serve_adoption.py @@ -40,7 +40,7 @@ def __init__( transport: Any = None, stack: Optional[CompileStack] = None, loader: Optional[Callable[[Path, Any, Any], Any]] = None, - on_adopted: Optional[Callable[["ServeAdoption"], Any]] = None, + on_adopted: Optional[Callable[[Any, Any], Any]] = None, ) -> None: self.release_id = str(release_id) self.sm = str(sm) @@ -51,8 +51,10 @@ def __init__( self._loader = loader self._on_adopted = on_adopted self._lock = threading.Lock() - self._settled = False - self._triggered = False + self._sessions: Dict[Tuple[type, str, str], Any] = {} + self._stores: Dict[Tuple[str, str], Any] = {} + self._class_keys: Dict[type, Tuple[type, str, str]] = {} + self._triggered: set[Tuple[type, str, str]] = set() self.adoption: Any = None self.store: Any = None self.refusal: str = "" @@ -60,81 +62,96 @@ def __init__( self.refusal_permanent: bool = False self.contract: str = "" - def sink_for(self, model_cls: type, lane: Any) -> Optional[Callable[..., Any]]: + def sink_for( + self, model_cls: type, lane: Any, binding: Any, + ) -> Optional[Callable[..., Any]]: """``ctx.compile``'s sink for one model class — the adopt arm. - pgw#1650: a release derives EVERY compile-marking class, so this is - called once per class. Classes that share a lane share this session — - the document under that stamp carries all of their graphs and a mark - claims one by STRUCTURE. A class on a DIFFERENT lane gets no sink and - serves eager, LOUDLY: this session was built for one lane, and handing - it back for another silently adopts the wrong lane's graphs. Per-lane - sessions in one boot are owed (pgw#1650, serving half). + Sessions are keyed by (class, Bind Contract, lane), while classes that + share a bind reuse its graph store. A class bound to a second config + tree gets that Bind Contract's graphs and can never inherit the + primary's. """ - with self._lock: - if self._settled: - from .model import lane_handle + from .model import lane_handle - contract = lane_handle(lane) if lane is not None else "" - if self.contract and contract and contract != self.contract: - logger.warning( - "adopt: %s declares lane %s, but this boot adopted " - "lane %s — %s serves EAGER. One boot holds one lane " - "document.", - model_cls.__name__, contract, self.contract, - model_cls.__name__, - ) + contract = lane_handle(lane) if lane is not None else "" + bind = getattr(binding, "bind_contract", None) + bind_digest = str(getattr(bind, "digest", "") or "").strip() + key = (model_cls, bind_digest, contract) + with self._lock: + self._class_keys[model_cls] = key + if lane is None: + self._refuse("eager_permanent", "this model class declares no lane") + return None + if not bind_digest: + self._refuse( + "bind_contract_missing", + f"{model_cls.__name__} carries no Bind Contract", + ) + return None + bind_key = (bind_digest, contract) + with self._lock: + session = self._sessions.get(key) + if session is None: + try: + built = self._build(lane, bind_digest, bind_key) + except Exception as exc: # noqa: BLE001 — adoption never kills a boot + self._refuse(type(exc).__name__, str(exc)) return None - return self._sink() - self._settled = True - try: - self._build(lane) - except Exception as exc: # noqa: BLE001 — adoption never kills a boot - self._refuse(type(exc).__name__, str(exc)) - return None - return self._sink() + if built is None: + return None + store, session = built + self._stores[bind_key] = store + self._sessions[key] = session + if self.adoption is None: + self.adoption, self.store, self.contract = session, store, contract + return self._sink(session) - def _sink(self) -> Optional[Callable[..., Any]]: + def _sink(self, session: Any = None) -> Optional[Callable[..., Any]]: from .adapter_guard import sink as guarded_sink - if self.adoption is None: + selected = self.adoption if session is None else session + if selected is None: return None - return guarded_sink(self.adoption.adopt) + return guarded_sink(selected.adopt) def loaded(self, model_cls: type = type(None), lane: Any = None) -> None: """THE MINT'S TRIGGER: the author's ``load(ctx)`` has returned.""" with self._lock: - if self._triggered: + key = self._class_keys.get(model_cls) + if key is None or key in self._triggered: return - self._triggered = True - session = self.adoption + self._triggered.add(key) + session = self._sessions.get(key) + store = self._stores.get((key[1], key[2])) hook = self._on_adopted - contract = self.contract + contract = key[2] if session is None: - self._say_boot_end(armed=0, claimed=0, mint_running=False) + self._say_boot_end(session, contract, armed=0, claimed=0, mint_running=False) return - self._say_outcome(contract) + self._say_outcome(session, contract) armed, holes = len(session.adopted), len(session.holes) mint_running = False if hook is not None: try: - started = hook(self) + started = hook(store, session) running = getattr(started, "running", None) mint_running = bool(holes) if running is None else bool(running) except Exception: # noqa: BLE001 — the pod serves either way logger.exception("adopt: the post-load mint trigger raised") - self._say_boot_end(armed=armed, claimed=armed + holes, + self._say_boot_end(session, contract, armed=armed, claimed=armed + holes, mint_running=mint_running) def _say_boot_end( - self, *, armed: int, claimed: int, mint_running: bool + self, session: Any, contract: str, *, armed: int, claimed: int, + mint_running: bool, ) -> None: from ..compiled_graph_adopt import EagerPhase from .self_mint import KIND_SKIPPED if armed or mint_running: return - declared = self.adoption is not None or ( + declared = session is not None or ( bool(self.refusal_phase) and self.refusal_phase not in EAGER_BY_DESIGN_REFUSALS ) @@ -143,12 +160,12 @@ def _say_boot_end( why = ( f"the release declared {claimed} graph specialization(s) for this " f"(lane x sm) and none armed" - if self.adoption is not None + if session is not None else f"the adopt refused ({self.refusal})" ) activity_mod.emit_event( KIND_SKIPPED, - f"release={self.release_id} lane={self.contract or '(unresolved)'} " + f"release={self.release_id} lane={contract or '(unresolved)'} " f"sm={self.sm}: boot ended with ZERO armed graph specializations " f"and no mint in flight — {why}. This pod serves EAGER for the " f"rest of its life against a release that declared compile.", @@ -157,7 +174,12 @@ def _say_boot_end( total_steps=claimed, ) - def _build(self, lane: Any) -> None: + def _build( + self, + lane: Any, + bind_contract_digest: str, + bind_key: Tuple[str, str], + ) -> Optional[Tuple[Any, Any]]: from .._vendor.torchcg.adopt import AdoptSession from ..env_identity import installed_stack_drift from .mint_store import graph_store @@ -168,29 +190,33 @@ def _build(self, lane: Any) -> None: ) from .model import lane_handle - if lane is None: - self._refuse("eager_permanent", "this model class declares no lane") - return contract = lane_handle(lane) transport = ( self._transport if self._transport is not None else BrokerReleaseGraphTransport() ) - store = HubGraphStore(transport, self.release_id, contract, self.sm) + store = self._stores.get(bind_key) + upstream: Any = ( + getattr(store, "upstream", None) + if store is not None + else HubGraphStore( + transport, self.release_id, bind_contract_digest, contract, self.sm + ) + ) try: - document = store.get_graphs(self.release_id) + document = upstream.get_graphs(self.release_id) except ReleaseNotStamped as exc: self._refuse("release_not_stamped", str(exc)) - return + return None if document is None: self._refuse( "no_document", "the adopt route answered, and the answer rebuilt to no lane " "document for this (release x lane x sm)") - return + return None if getattr(document, "eager_permanent", False): self._refuse("eager_permanent", "the release document is eager-permanent") - return + return None stack = self._stack if self._stack is not None else dict(document.stack) for row in installed_stack_drift(dict(document.stack)): logger.warning("adopt: compile-stack drift vs this venv: %s", row) @@ -200,11 +226,11 @@ def _build(self, lane: Any) -> None: "this worker states no local CAS, so a fetched artifact could " "not be banked and a minted one could not be published — the " "adopt would be read-only against the fleet pool forever") - return - self.store = graph_store(self.cas_dir, store) - self.contract = contract - self.adoption = AdoptSession( - self.store, document, contract, self.sm, + return None + if store is None: + store = graph_store(self.cas_dir, upstream) + session = AdoptSession( + store, document, contract, self.sm, loader=self._loader, artifacts_dir=self.artifacts_dir, stack=stack, @@ -214,11 +240,11 @@ def _build(self, lane: Any) -> None: "document; what this boot arms is decided by the author's " "ctx.compile calls and reported after load", self.release_id, contract, self.sm, - len(self.adoption.lane.graphs), + len(session.lane.graphs), ) + return store, session - def _say_outcome(self, contract: str) -> None: - session = self.adoption + def _say_outcome(self, session: Any, contract: str) -> None: if session is None: return adopted, holes = len(session.adopted), len(session.holes) @@ -271,29 +297,32 @@ def _refuse(self, phase: str, detail: str) -> None: @property def holes(self) -> Tuple[Any, ...]: """The ordered mint work-list, in canonical document order.""" - return tuple(self.adoption.holes) if self.adoption is not None else () + return tuple(hole for session in self._sessions.values() for hole in session.holes) @property def layout_rungs(self) -> Tuple[Any, ...]: """Each armed graph's layout position, read off the artifact serving it.""" - if self.adoption is None: - return () - return layout_rung.rungs_of(self.adoption) + return tuple( + rung for session in self._sessions.values() + for rung in layout_rung.rungs_of(session) + ) def facts(self) -> Dict[str, Any]: """Counted, and never silent: `adopted`/`holes` are absent (not zero) when no session was ever built, and `refusal` says why.""" - if self.adoption is None: + if not self._sessions: return {"adopting": False, "refusal": self.refusal or "not_attempted"} - marks = tuple(self.adoption.unclaimed_marks) + sessions = tuple(self._sessions.values()) + marks = tuple(mark for session in sessions for mark in session.unclaimed_marks) rungs = self.layout_rungs return { "adopting": True, - "adopted": len(self.adoption.adopted), - "holes": len(self.adoption.holes), - "unclaimed": len(self.adoption.unclaimed), + "bind_sessions": len(sessions), + "adopted": sum(len(session.adopted) for session in sessions), + "holes": sum(len(session.holes) for session in sessions), + "unclaimed": sum(len(session.unclaimed) for session in sessions), "unmatched_marks": len(marks), "unmatched": [mark.describe() for mark in marks], - "ambiguous": len(self.adoption.ambiguous), + "ambiguous": sum(len(session.ambiguous) for session in sessions), # pgw#1645. Counted the same way everything else here is, and the # rows carry BOTH layouts: a pod on a lower rung must read as # EARNING rather than as a slow pod nobody can explain. diff --git a/src/gen_worker/serving/serve_loop.py b/src/gen_worker/serving/serve_loop.py index d9b932317..7aed52f10 100644 --- a/src/gen_worker/serving/serve_loop.py +++ b/src/gen_worker/serving/serve_loop.py @@ -220,7 +220,7 @@ def __init__( resolver: BindingResolver, engine: Optional[LoaderEngine] = None, lane_contract: str = "", - compile_sink_for: Optional[Callable[[type, Any], Any]] = None, + compile_sink_for: Optional[Callable[[type, Any, DeployBinding], Any]] = None, on_loaded: Optional[Callable[[type, Any], None]] = None, output_dir: Optional[Path] = None, context_kwargs: Optional[Mapping[str, Any]] = None, @@ -303,7 +303,7 @@ def make() -> _InstanceBackend: if lane is None and resolved is not None: lane = getattr(resolved.declared, "contract", None) sink = ( - self._compile_sink_for(model_cls, lane) + self._compile_sink_for(model_cls, lane, binding) if self._compile_sink_for is not None else None ) diff --git a/src/gen_worker/serving/streaming/__init__.py b/src/gen_worker/serving/streaming/__init__.py index 1fa19ae2c..572e03165 100644 --- a/src/gen_worker/serving/streaming/__init__.py +++ b/src/gen_worker/serving/streaming/__init__.py @@ -13,18 +13,16 @@ TensorRow, ) from .engine import LoadError, LoadReport, NameMismatch, StreamingLoader +from .fill_client import Destination from .skeleton import Skeleton, SkeletonError from .source import ( - BridgeWeightStore, NativeWeightStore, StreamedTensor, TensorStream, WeightStore, WeightStoreUnavailable, - native_available, store_for, ) -from .staging import StagingPool logger = logging.getLogger(__name__) @@ -57,12 +55,12 @@ def engine_for( __all__ = [ "engine_for", - "BridgeWeightStore", "CENSUS_KIND", "Census", "CensusError", "CensusMismatch", "ComponentCensus", + "Destination", "TensorRow", "LoadError", "LoadReport", @@ -70,12 +68,10 @@ def engine_for( "NativeWeightStore", "Skeleton", "SkeletonError", - "StagingPool", "StreamedTensor", "StreamingLoader", "TensorStream", "WeightStore", "WeightStoreUnavailable", - "native_available", "store_for", ] diff --git a/src/gen_worker/serving/streaming/engine.py b/src/gen_worker/serving/streaming/engine.py index 204c3495b..71aaf5c38 100644 --- a/src/gen_worker/serving/streaming/engine.py +++ b/src/gen_worker/serving/streaming/engine.py @@ -1,4 +1,4 @@ -"""``ctx.load``'s engine: chunk store -> pinned staging -> device, no files. +"""``ctx.load``'s engine: plan destinations, ask tensorfs to fill, fence. The load's postcondition is not asserted here. It is stated by the CONSTRUCTION CENSUS (:mod:`.census`, pgw#1647) and this engine REPLAYS it: after the fill and @@ -22,20 +22,22 @@ from dataclasses import dataclass from pathlib import Path from typing import ( - TYPE_CHECKING, Any, Dict, FrozenSet, List, Mapping, Optional, Sequence, Tuple, + TYPE_CHECKING, Any, Dict, FrozenSet, List, Mapping, Optional, Tuple, ) from . import census as _census from . import keymap as _keymap from . import skeleton as _skeleton -from .source import StreamedTensor, TensorStream, WeightStore, component_of -from .staging import DEFAULT_BUFFER_BYTES, DEFAULT_BUFFERS, StagingPool +from .fill_client import Destination, FillClient, client_for +from .source import WeightStore, component_of if TYPE_CHECKING: # pragma: no cover - typing only import torch logger = logging.getLogger(__name__) +DEFAULT_STAGING_BYTES = 64 * 1024 * 1024 + _TORCH_DTYPE: Mapping[str, str] = { "F64": "float64", "F32": "float32", @@ -79,12 +81,12 @@ class LoadReport: weights_streamed_bytes: int = 0 weights_stream_gbps: float = 0.0 - source: str = "bridge" - staging: str = "pageable" + source: str = "native" + staging: str = "destination" io: str = "buffered" containers: int = 0 tensors: int = 0 - windows: int = 0 + chunks: int = 0 seconds: float = 0.0 dtypes: Tuple[str, ...] = () cast_to_lane: int = 0 @@ -102,19 +104,6 @@ def attributes(self) -> Dict[str, object]: } -@dataclass(slots=True) -class _Placement: - - name: str - offset: int - nbytes: int - flat: "torch.Tensor" - - @property - def end(self) -> int: - return self.offset + self.nbytes - - @dataclass(slots=True) class _Slot: @@ -172,12 +161,6 @@ def _install(slot: _Slot, tensor: "torch.Tensor") -> None: slot.owner._buffers[slot.leaf] = tensor -def _flat_bytes(tensor: "torch.Tensor") -> "torch.Tensor": - import torch - - return tensor.view(-1).view(torch.uint8) - - class StreamingLoader: def __init__( @@ -186,16 +169,16 @@ def __init__( *, device: Any = "cuda", io: str = "buffered", - buffer_bytes: int = DEFAULT_BUFFER_BYTES, - buffers: int = DEFAULT_BUFFERS, + staging_bytes: int = DEFAULT_STAGING_BYTES, ) -> None: if io not in ("buffered", "direct"): raise LoadError(f"io must be 'buffered' or 'direct', not {io!r}") self._store = store self._device = device self._io = io - self._buffer_bytes = int(buffer_bytes) - self._buffers = int(buffers) + if staging_bytes <= 0: + raise LoadError("tensorfs staging must hold bytes") + self._staging_bytes = int(staging_bytes) self._planned: List[Tuple[str, str]] = [] self.last_report: Optional[LoadReport] = None #: The census the last load actually produced and the fence accepted. @@ -206,7 +189,14 @@ def last_census(self) -> Optional["_census.Census"]: """What the last load BUILT, as data — the fence's own answer.""" return self._census - def build(self, pipeline_cls: type, *, checkpoint_dir: Path, lane: Any) -> Any: + def build( + self, + pipeline_cls: type, + *, + checkpoint_dir: Path, + lane: Any, + expected_census: Optional[_census.Census] = None, + ) -> Any: """Meta skeleton, then weights streamed into it.""" import torch @@ -222,32 +212,36 @@ def build(self, pipeline_cls: type, *, checkpoint_dir: Path, lane: Any) -> Any: # release's census beside the resolved variant, `expected` is read from # it instead and the same predicate then also catches an image that # builds a different module than the one the release was derived from. - expected = self._expected_census(built) + expected = ( + expected_census + if expected_census is not None + else self._expected_census(built) + ) report = LoadReport( io=self._io, source=str(getattr(self._store, "KIND", type(self._store).__name__)), ) recasts: List[Tuple[_Slot, str, Any]] = [] - with StagingPool( - device, - buffer_bytes=self._buffer_bytes, - buffers=self._buffers, - ) as pool: - report.staging = pool.staging - for component, container in self._plan(built.modules): - self._stream_container( - built.modules[component], - component, - container, - pool=pool, - device=device, - report=report, - compute_dtype=compute_dtype, - recasts=recasts, - lane_exempt=_lane_exempt(built, component), - ) - report.containers = len(self._planned) + fill_client = client_for( + device.type, + device_index=int(device.index or 0), + staging_bytes=self._staging_bytes, + ) + report.staging = fill_client.staging + for component, container in self._plan(built.modules): + self._fill_container( + built.modules[component], + component, + container, + fill_client=fill_client, + device=device, + report=report, + compute_dtype=compute_dtype, + recasts=recasts, + lane_exempt=_lane_exempt(built, component), + ) + report.containers = len(self._planned) self._cast_to_lane(recasts, compute_dtype, report) # The prepare seam's TRAILING half, once per component and in one @@ -281,7 +275,7 @@ def build(self, pipeline_cls: type, *, checkpoint_dir: Path, lane: Any) -> Any: self._assert_lane_dtype(built.modules, compute_dtype, built) logger.info( "ctx.load: %s resident on %s — %.2f GiB streamed in %.2fs " - "(%.2f GB/s, staging=%s io=%s, %d tensors over %d windows)", + "(%.2f GB/s, staging=%s io=%s, %d tensors over %d chunks)", pipeline_cls.__name__, device, report.weights_streamed_bytes / (1 << 30), @@ -290,7 +284,7 @@ def build(self, pipeline_cls: type, *, checkpoint_dir: Path, lane: Any) -> Any: report.staging, report.io, report.tensors, - report.windows, + report.chunks, ) return built.pipeline @@ -326,13 +320,13 @@ def _plan(self, modules: Mapping[str, Any]) -> List[Tuple[str, str]]: self._planned = planned return planned - def _stream_container( + def _fill_container( self, module: Any, component: str, container: str, *, - pool: StagingPool, + fill_client: FillClient, device: Any, report: LoadReport, compute_dtype: Any = None, @@ -341,16 +335,16 @@ def _stream_container( ) -> None: import torch - stream: TensorStream = self._store.open( + stream = self._store.open( container, direct=self._io == "direct" ) - entries: Sequence[StreamedTensor] = stream.tensors + entries = stream.tensors if not entries: return slots = _slots(module) renames = _keymap.migration(module, (entry.name for entry in entries)) - placements: List[_Placement] = [] + destinations: List[Destination] = [] unexpected: List[str] = [] seen: set[str] = set() dtypes: set[str] = set(report.dtypes) @@ -366,18 +360,17 @@ def _stream_container( continue dtype = _torch_dtype(entry.dtype, f"{component}/{entry.name}") dtypes.add(entry.dtype.upper()) - destination = torch.empty( + tensor = torch.empty( tuple(int(dim) for dim in entry.shape), dtype=dtype, device=device ) - flat = _flat_bytes(destination) - if flat.numel() != entry.nbytes: + capacity = int(tensor.numel() * tensor.element_size()) + if capacity != entry.nbytes: raise LoadError( f"{component}/{entry.name}: the container says " f"{entry.nbytes} bytes, a {dtype} tensor of " - f"{tuple(entry.shape)} holds {flat.numel()}" + f"{tuple(entry.shape)} holds {capacity}" ) - pool.track(destination) - _install(slot, destination) + _install(slot, tensor) # pgw#1638: a tensor a QUANTIZER owns is out of the lane cast's # scope, whatever its dtype. Property 3 above already says a # quantized container is the lane's own bytes — but it said it of @@ -391,12 +384,15 @@ def _stream_container( and name not in lane_exempt): recasts.append((slot, f"{component}/{entry.name}", dtype)) seen.add(entry.name) - placements.append( - _Placement( + destinations.append( + Destination( name=entry.name, - offset=int(entry.offset), - nbytes=int(entry.nbytes), - flat=flat, + pointer=int(tensor.data_ptr()), + capacity=capacity, + source_offset=int(entry.offset), + shape=tuple(int(dim) for dim in entry.shape), + element_bytes=int(tensor.element_size()), + layout="torch.contiguous@1", ) ) @@ -419,55 +415,18 @@ def _stream_container( ) report.dtypes = tuple(sorted(dtypes)) - placements.sort(key=lambda placement: placement.offset) - report.tensors += len(placements) - report.windows += self._walk(stream, placements, pool=pool, report=report) - - def _walk( - self, - stream: TensorStream, - placements: List[_Placement], - *, - pool: StagingPool, - report: LoadReport, - ) -> int: - position = placements[0].offset - finish = placements[-1].end - window = pool.buffer_bytes - first = 0 - windows = 0 - - while position < finish: - count = min(window, finish - position) - slot = pool.acquire() - stream.readinto(position, count, slot.view[:count]) - windows += 1 - report.weights_streamed_bytes += count - - index = first - while index < len(placements) and placements[index].offset < position + count: - placement = placements[index] - if placement.end <= position: - index += 1 - first = index - continue - low = max(placement.offset, position) - high = min(placement.end, position + count) - pool.copy_out( - slot, - low - position, - placement.flat, - low - placement.offset, - high - low, + destinations.sort(key=lambda destination: destination.source_offset) + report.tensors += len(destinations) + for destination_data in destinations: + stats = fill_client.fill(stream, destination_data) + if int(stats.source_bytes) != destination_data.capacity: + raise LoadError( + f"{component}/{destination_data.name}: tensorfs filled " + f"{stats.source_bytes} source bytes into a " + f"{destination_data.capacity}-byte destination" ) - if placement.end <= position + count: - first = index + 1 - index += 1 - - pool.release(slot) - position += count - - return windows + report.weights_streamed_bytes += int(stats.source_bytes) + report.chunks += int(stats.chunks) @staticmethod def _cast_to_lane( @@ -565,6 +524,7 @@ def _lane_compute_dtype(lane: Any) -> Any: __all__ = [ "LaneDtypeUnmet", + "DEFAULT_STAGING_BYTES", "LoadError", "LoadReport", "NameMismatch", diff --git a/src/gen_worker/serving/streaming/fill_client.py b/src/gen_worker/serving/streaming/fill_client.py new file mode 100644 index 000000000..7e18f6277 --- /dev/null +++ b/src/gen_worker/serving/streaming/fill_client.py @@ -0,0 +1,165 @@ +"""The torch-free data seam into tensorfs's one fill implementation.""" + +from __future__ import annotations + +import importlib +from dataclasses import dataclass +from typing import Any, Optional, Protocol, Sequence, Tuple + + +@dataclass(frozen=True, slots=True) +class AddressSource: + """One contiguous source allocation, described without its owner type.""" + + pointer: int + capacity: int + + +@dataclass(frozen=True, slots=True) +class FileSource: + """One immutable file range, or a zero-filled hole when ``path`` is None.""" + + path: Optional[str] + offset: int + length: int + + +@dataclass(frozen=True, slots=True) +class Destination: + """Where one named tensor lands; every field is plain data.""" + + name: str + pointer: int + capacity: int + source_offset: int + shape: Tuple[int, ...] + element_bytes: int + layout: str + + +class FillClient(Protocol): + """Fill destinations without learning which allocator produced them.""" + + staging: str + + def fill(self, reader: Any, destination: Destination) -> Any: ... + + def fill_address( + self, source: AddressSource, destination: Destination + ) -> Any: ... + + def fill_files( + self, sources: Sequence[FileSource], destination: Destination + ) -> Any: ... + + +class HostFillClient: + """The host destination is its own staging allocation.""" + + staging = "destination" + + def fill(self, reader: Any, destination: Destination) -> Any: + return reader.fill_host_address( + destination.name, + destination.pointer, + destination.capacity, + layout=destination.layout, + ) + + @staticmethod + def _native() -> Any: + native = importlib.import_module("tensorfs.native") + NativeHostFillClient = getattr(native, "HostFillClient") + return NativeHostFillClient() + + def fill_address( + self, source: AddressSource, destination: Destination + ) -> Any: + return self._native().fill_address( + source.pointer, + source.capacity, + destination.pointer, + destination.capacity, + destination.shape, + destination.element_bytes, + layout=destination.layout, + ) + + def fill_files( + self, sources: Sequence[FileSource], destination: Destination + ) -> Any: + records = [(source.path, source.offset, source.length) for source in sources] + return self._native().fill_files( + records, + destination.pointer, + destination.capacity, + destination.shape, + destination.element_bytes, + layout=destination.layout, + ) + + +class CudaFillClient: + """One reusable tensorfs-owned pinned slab for all CUDA destinations.""" + + staging = "tensorfs-pinned" + + def __init__(self, staging_bytes: int, device: int) -> None: + native = importlib.import_module("tensorfs.native") + NativeCudaFillClient = getattr(native, "CudaFillClient") + self._client = NativeCudaFillClient(staging_bytes, device) + + def fill(self, reader: Any, destination: Destination) -> Any: + native_reader = getattr(reader, "native_reader", reader) + return self._client.fill( + native_reader, + destination.name, + destination.pointer, + destination.capacity, + layout=destination.layout, + ) + + def fill_address( + self, source: AddressSource, destination: Destination + ) -> Any: + return self._client.fill_address( + source.pointer, + source.capacity, + destination.pointer, + destination.capacity, + destination.shape, + destination.element_bytes, + layout=destination.layout, + ) + + def fill_files( + self, sources: Sequence[FileSource], destination: Destination + ) -> Any: + records = [(source.path, source.offset, source.length) for source in sources] + return self._client.fill_files( + records, + destination.pointer, + destination.capacity, + destination.shape, + destination.element_bytes, + layout=destination.layout, + ) + + +def client_for(device_type: str, *, device_index: int, staging_bytes: int) -> FillClient: + """Bind the one backend implied by the granted destination device.""" + + if device_type == "cpu": + return HostFillClient() + if device_type == "cuda": + return CudaFillClient(staging_bytes, device_index) + raise ValueError(f"tensorfs fill has no destination backend for {device_type!r}") + + +__all__ = [ + "AddressSource", + "Destination", + "FileSource", + "FillClient", + "client_for", +] diff --git a/src/gen_worker/serving/streaming/source.py b/src/gen_worker/serving/streaming/source.py index 88c284a5f..84d78cc38 100644 --- a/src/gen_worker/serving/streaming/source.py +++ b/src/gen_worker/serving/streaming/source.py @@ -2,19 +2,15 @@ from __future__ import annotations -import logging from pathlib import Path from typing import ( Any, Mapping, - Optional, Protocol, Sequence, runtime_checkable, ) -logger = logging.getLogger(__name__) - TENSOR_PLANNERS = frozenset({"safetensors-v1", "gguf-v1"}) TENSOR_SUFFIXES = (".safetensors", ".gguf") @@ -37,16 +33,21 @@ def nbytes(self) -> int: ... @runtime_checkable class TensorStream(Protocol): - """One tensor container, readable at arbitrary offsets into caller memory.""" + """One tensor container consumable by tensorfs's fill path.""" @property def tensors(self) -> Sequence[StreamedTensor]: ... @property def length(self) -> int: ... - def readinto(self, offset: int, length: int, buffer: Any) -> int: - """Copy ``[offset, offset+length)`` into a writable C-contiguous buffer — typically CUDA-pinned host memory, which the store neither knows nor cares about.""" - ... + def fill_host_address( + self, + name: str, + destination_ptr: int, + destination_bytes: int, + destination_offset: int = 0, + layout: str = "torch.contiguous@1", + ) -> Any: ... class WeightStore(Protocol): @@ -131,120 +132,14 @@ def open(self, container: str, *, direct: bool = False) -> TensorStream: return reader -class _BridgeStream: - - def __init__(self, reader: Any, container: str) -> None: - self._reader = reader - self._container = container - views = sorted(reader.values(), key=lambda view: view.offset) - self._tensors: list[StreamedTensor] = list(views) - self._length = max((v.offset + v.nbytes for v in views), default=0) - - @property - def tensors(self) -> Sequence[StreamedTensor]: - return tuple(self._tensors) - - @property - def length(self) -> int: - return self._length - - def readinto(self, offset: int, length: int, buffer: Any) -> int: - target = memoryview(buffer).cast("B") - if len(target) < length: - raise ValueError( - f"{self._container}: buffer holds {len(target)} bytes, " - f"the read needs {length}" - ) - at = 0 - for piece in self._reader._pieces(self._container, offset, length): - target[at : at + len(piece)] = piece - at += len(piece) - return at - - -class BridgeWeightStore: - """The interim byte source over ``_vendor.tensorfs``'s reader.""" - - KIND = "bridge" - - SUFFIXES = TENSOR_SUFFIXES - - def __init__(self, cas: Any, manifest: Any, *, verify: bool = False) -> None: - self._cas = cas - self._manifest = manifest - self._verify = verify - self._open: list[Any] = [] - self._entries = { - entry.path: entry - for entry in manifest.files - if entry.path.endswith(self.SUFFIXES) - } - - def containers(self) -> Sequence[str]: - return tuple(self._entries) - - def open(self, container: str, *, direct: bool = False) -> TensorStream: - from gen_worker._vendor.tensorfs.manifest import RepositoryManifest - from gen_worker._vendor.tensorfs.tensors import TensorReader - - if direct: - raise WeightStoreUnavailable( - "io=direct needs the native reader's O_DIRECT open " - "(tensorfs#115); the interim bridge is buffered only" - ) - entry = self._entries.get(container) - if entry is None: - raise WeightStoreUnavailable( - f"{container!r} is not a tensor container of this manifest" - ) - reader = TensorReader( - self._cas, - RepositoryManifest(files=(entry,)), - verify=self._verify, - ) - self._open.append(reader) - return _BridgeStream(reader, container) - - def close(self) -> None: - for reader in self._open: - reader.close() - self._open.clear() - - -def store_for( - checkpoint_dir: Path | str, *, native: Optional[bool] = None -) -> Optional[WeightStore]: +def store_for(checkpoint_dir: Path | str) -> WeightStore | None: """The byte source backing a projected checkpoint tree, or ``None``.""" from ...models import projection projected = projection.resolve_projection(checkpoint_dir) if projected is None: return None - if native is None: - native = native_available() - if native: - try: - return NativeWeightStore.from_manifest( - projected.cas.root, projected.manifest - ) - except Exception: - logger.warning( - "ctx.load: the native tensorfs store would not open over %s " - "— falling back to the GIL-bound bridge, which is ~10x " - "slower (tensorfs#115)", - projected.cas.root, - exc_info=True, - ) - return BridgeWeightStore(projected.cas, projected.manifest) - - -def native_available() -> bool: - """True when a tensorfs carrying the #115 stream surface is importable.""" - try: - _native_stream_reader() - except (WeightStoreUnavailable, AttributeError): - return False - return True + return NativeWeightStore.from_manifest(projected.cas.root, projected.manifest) def component_of(container: str) -> str: @@ -256,13 +151,11 @@ def component_of(container: str) -> str: __all__ = [ "TENSOR_PLANNERS", "TENSOR_SUFFIXES", - "BridgeWeightStore", "NativeWeightStore", "StreamedTensor", "TensorStream", "WeightStore", "WeightStoreUnavailable", "component_of", - "native_available", "store_for", ] diff --git a/src/gen_worker/serving/streaming/staging.py b/src/gen_worker/serving/streaming/staging.py deleted file mode 100644 index e7c24b7e9..000000000 --- a/src/gen_worker/serving/streaming/staging.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Double-buffered staging: store bytes -> pinned host memory -> device.""" - -from __future__ import annotations - -import logging -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, List, Optional - -if TYPE_CHECKING: # pragma: no cover - typing only - import torch - -logger = logging.getLogger(__name__) - -DEFAULT_BUFFER_BYTES = 64 * 1024 * 1024 -DEFAULT_BUFFERS = 4 - - -class StagingError(RuntimeError): - """The staging pool could not be built or used as declared.""" - - -def _writable_view(tensor: "torch.Tensor") -> memoryview: - import ctypes - - count = tensor.numel() - block = (ctypes.c_ubyte * count).from_address(tensor.data_ptr()) - return memoryview(block).cast("B") - - -@dataclass(slots=True) -class _Slot: - - index: int - tensor: "torch.Tensor" - view: memoryview - event: Optional[Any] = None - - -class StagingPool: - """A ring of host buffers and the stream the copies ride.""" - - def __init__( - self, - device: "torch.device", - *, - buffer_bytes: int = DEFAULT_BUFFER_BYTES, - buffers: int = DEFAULT_BUFFERS, - ) -> None: - import torch - - if buffers < 2: - raise StagingError( - f"a staging pool of {buffers} buffer(s) cannot double-buffer; " - "the read and the copy would serialize" - ) - if buffer_bytes <= 0: - raise StagingError("staging buffers must hold bytes") - - self.device = device - self.buffer_bytes = int(buffer_bytes) - self.pinned = device.type == "cuda" - self._torch = torch - self._stream: Optional[Any] = ( - torch.cuda.Stream(device=device) # type: ignore[no-untyped-call] - if self.pinned - else None - ) - self._slots: List[_Slot] = [] - for index in range(buffers): - host = torch.empty( - self.buffer_bytes, dtype=torch.uint8, pin_memory=self.pinned - ) - self._slots.append( - _Slot(index=index, tensor=host, view=_writable_view(host)) - ) - self._next = 0 - - def acquire(self) -> _Slot: - """The next buffer, once its previous copy has actually landed.""" - slot = self._slots[self._next] - self._next = (self._next + 1) % len(self._slots) - if slot.event is not None: - slot.event.synchronize() - slot.event = None - return slot - - def copy_out( - self, slot: _Slot, src_offset: int, dst: "torch.Tensor", dst_offset: int, count: int - ) -> None: - """Enqueue ``count`` bytes of ``slot`` into a flat uint8 destination.""" - source = slot.tensor[src_offset : src_offset + count] - target = dst[dst_offset : dst_offset + count] - if self._stream is None: - target.copy_(source) - return - with self._torch.cuda.stream(self._stream): - target.copy_(source, non_blocking=True) - - def release(self, slot: _Slot) -> None: - """Mark every copy enqueued out of this buffer, so reuse can wait.""" - if self._stream is None: - return - event = self._torch.cuda.Event() # type: ignore[no-untyped-call] - event.record(self._stream) - slot.event = event - - def track(self, tensor: "torch.Tensor") -> None: - """Tell the caching allocator this tensor is written on the copy stream, not the stream it was allocated on.""" - if self._stream is not None: - tensor.record_stream(self._stream) - - def finish(self) -> None: - """Block until every enqueued copy has landed, then make the compute stream see them.""" - if self._stream is None: - return - self._stream.synchronize() - self._torch.cuda.current_stream(self.device).wait_stream(self._stream) - - def close(self) -> None: - self._slots.clear() - - @property - def staging(self) -> str: - """The telemetry token: ``pinned`` or ``pageable``.""" - return "pinned" if self.pinned else "pageable" - - def __enter__(self) -> "StagingPool": - return self - - def __exit__(self, *_exc: object) -> None: - self.finish() - self.close() - - -__all__ = [ - "DEFAULT_BUFFERS", - "DEFAULT_BUFFER_BYTES", - "StagingError", - "StagingPool", -] diff --git a/src/gen_worker/wire_snapshots.py b/src/gen_worker/wire_snapshots.py index 782cb999a..625b8435c 100644 --- a/src/gen_worker/wire_snapshots.py +++ b/src/gen_worker/wire_snapshots.py @@ -1,17 +1,16 @@ -"""The wire's snapshot map is keyed by COMPOSED MANIFEST DIGEST: ModelBinding carries manifest_digest, and DesiredResidency.snapshots / RunJob.snapshots are keyed by it, so two resolutions of one repo with different component sources are two entries that cannot interfere. The worker's residency, event and GC key is still the canonical ref, so this module is the ONE place the two spellings meet — it resolves each binding's manifest_digest to its snapshot and files it under that binding's ref. A ref that two bindings claim at DIFFERENT digests is refused, never guessed.""" +"""Convert Tensorhub's ref-keyed snapshot wire map into worker objects. -from __future__ import annotations +``ModelBinding.manifest_digest`` never had a sender and is tombstoned by +th#2208. A snapshot map key is therefore the canonical ref on both sides of +the wire; there is no second spelling to reconcile here. +""" -from typing import Any, Dict, Iterable, Mapping +from __future__ import annotations -from .api.errors import ValidationError +from typing import Any, Dict, Mapping from .models.refs import WireRef -class AmbiguousManifestError(ValidationError): - """One ref arrived bound to two different composed manifests in a single message.""" - - def resolved_repo_from_snapshot(snap: Any) -> Any: """``pb.Snapshot`` -> the typed ``WorkerResolvedRepo`` the download layer speaks: the ONE wire-boundary conversion.""" from .models.hub_client import ( @@ -70,48 +69,22 @@ def snapshot_from_resolved_repo(resolved: Any) -> Any: def resolved_repos( wire: Mapping[str, Any], - bindings: Iterable[Any] = (), ) -> Dict[str, Any]: - """:func:`index_snapshots`, with every entry already converted to the typed ``WorkerResolvedRepo``.""" + """Return every ref with its typed ``WorkerResolvedRepo``.""" return { str(ref): resolved_repo_from_snapshot(snap) - for ref, snap in index_snapshots(wire, bindings).items() + for ref, snap in wire.items() } def index_snapshots( wire: Mapping[str, Any], - bindings: Iterable[Any] = (), ) -> Dict[WireRef, Any]: """Wire snapshot map -> the ref-keyed view the worker materializes from.""" - claimed: set[str] = set() - out: Dict[WireRef, Any] = {} - seen: Dict[str, str] = {} - for binding in bindings or (): - ref = str(getattr(binding, "ref", "") or "").strip() - digest = str(getattr(binding, "manifest_digest", "") or "").strip() - if not ref or not digest: - continue - prior = seen.get(ref) - if prior is not None and prior != digest: - raise AmbiguousManifestError( - f"ref {ref!r} is bound to two composed manifests in one " - f"message ({prior} and {digest}); the hub must send one " - f"manifest per ref per message" - ) - seen[ref] = digest - claimed.add(digest) - snapshot = wire.get(digest) - if snapshot is not None: - out[WireRef(ref)] = snapshot - for key, snapshot in wire.items(): - if key not in claimed: - out.setdefault(WireRef(key), snapshot) - return out + return {WireRef(ref): snapshot for ref, snapshot in wire.items()} __all__ = [ - "AmbiguousManifestError", "index_snapshots", "resolved_repo_from_snapshot", "resolved_repos", diff --git a/src/gen_worker/worker.py b/src/gen_worker/worker.py index f5086f5d8..7c907a253 100644 --- a/src/gen_worker/worker.py +++ b/src/gen_worker/worker.py @@ -190,9 +190,10 @@ class _Pick: slot: str ref: str - manifest_digest: str model: str inference_defaults: str + bind_contract_digest: str + bind_contract_url: str @dataclass(frozen=True, slots=True) @@ -212,17 +213,13 @@ def _picks_of(run: pb.RunJob) -> _DispatchPicks: by_ref: Dict[str, _Pick] = {} by_slot: Dict[str, str] = {} for binding in run.models: - digest = str(binding.manifest_digest).strip() - if not digest: - snapshot = run.snapshots.get(str(binding.ref)) - if snapshot is not None: - digest = str(snapshot.digest).strip() pick = _Pick( slot=str(binding.slot), ref=str(binding.ref), - manifest_digest=digest, model=str(binding.model).strip(), inference_defaults=str(binding.inference_defaults), + bind_contract_digest=str(binding.bind_contract_digest).strip(), + bind_contract_url=str(binding.bind_contract_url).strip(), ) by_ref[pick.ref] = pick by_slot[pick.slot] = pick.ref @@ -236,9 +233,10 @@ def _picks_of_bindings(bindings: Any) -> _DispatchPicks: pick = _Pick( slot=str(binding.slot), ref=str(binding.ref), - manifest_digest=str(binding.manifest_digest).strip(), model=str(binding.model).strip(), inference_defaults=str(binding.inference_defaults), + bind_contract_digest=str(binding.bind_contract_digest).strip(), + bind_contract_url=str(binding.bind_contract_url).strip(), ) by_ref[pick.ref] = pick by_slot[pick.slot] = pick.ref @@ -248,7 +246,7 @@ def _picks_of_bindings(bindings: Any) -> _DispatchPicks: def boot_picks( desired: Any, loaded: Any, config: "CheckpointConfig" ) -> Dict[str, _DispatchPicks]: - functions = sorted(getattr(loaded, "entrypoints", {}) or {}) + del loaded, config picks: Dict[str, _DispatchPicks] = {} for instance in getattr(desired, "hot", ()) or (): name = str(getattr(instance, "function_name", "")).strip() @@ -257,39 +255,28 @@ def boot_picks( table = _picks_of_bindings(getattr(instance, "models", ())) if table.by_slot: picks[name] = table - refs = [str(ref) for ref in config.refs] - if len(refs) != 1: - return picks - only_ref = refs[0] - snapshot = config.snapshots.get(WireRef(only_ref)) - digest = str(getattr(snapshot, "digest", "") or "").strip() - for name in functions: - if name in picks: - continue - spec = loaded.entrypoints[name] - slots = [slot for slot, _cls in spec.model_params] - if len(slots) != 1: - continue - pick = _Pick( - slot=slots[0], ref=only_ref, manifest_digest=digest, - model="", inference_defaults="", - ) - picks[name] = _DispatchPicks( - by_ref={only_ref: pick}, by_slot={slots[0]: only_ref} - ) return picks class HubBindingResolver: """The hub's half of the deploy state, per dispatch.""" - def __init__(self, snapshots_root: Optional[Path] = None) -> None: + def __init__( + self, + snapshots_root: Optional[Path] = None, + *, + release_id: str = "", + hub_base_url: str = "", + ) -> None: self.snapshots_root = ( Path(snapshots_root) if snapshots_root is not None else tensorhub_cas_dir() / SNAPSHOTS_DIR ) self._store: Optional[Any] = None + self._release_id = str(release_id or "").strip() + self._hub_base_url = str(hub_base_url or "").strip() + self._bind_contracts: Dict[str, Any] = {} def bind_store(self, store: Any) -> None: """Hand the resolver the store that materializes refs.""" @@ -329,10 +316,6 @@ def tree_for(self, model_cls: type, checkpoint_ref: str) -> Path: if tree.is_dir(): return tree tried.append(tree) - for tree in self._digest_trees(pick.manifest_digest): - if tree.is_dir(): - return tree - tried.append(tree) materialized = ( sorted(str(r) for r in self._store.disk_refs()) if self._store is not None else [] @@ -340,11 +323,41 @@ def tree_for(self, model_cls: type, checkpoint_ref: str) -> Path: raise CheckpointUnresolved( f"{model_cls.__name__}: checkpoint {checkpoint_ref!r} is not " f"materialized on this worker. Boot materialized " - f"{materialized or '[]'}; the dispatch's manifest_digest is " - f"{pick.manifest_digest or '(unset — the hub sends none)'}; " - f"tried {[str(p) for p in tried] or '[]'}" + f"{materialized or '[]'}; tried " + f"{[str(p) for p in tried] or '[]'}" ) + def _bind_contract(self, pick: _Pick) -> Any: + from .bind_contract import BindContractError, fetch + + digest = pick.bind_contract_digest + url = pick.bind_contract_url + if not digest or not url: + raise CheckpointUnresolved( + f"checkpoint {pick.ref!r} carries no complete Bind Contract " + "reference (digest and URL are both required); pgw#1653 " + "does not fall back to a self-census" + ) + cached = self._bind_contracts.get(digest) + if cached is None: + try: + cached = fetch(digest, url) + except BindContractError as exc: + raise CheckpointUnresolved(str(exc)) from exc + if self._release_id and cached.identity.release_id != self._release_id: + raise CheckpointUnresolved( + f"Bind Contract {digest} belongs to release " + f"{cached.identity.release_id!r}, not this worker's " + f"{self._release_id!r}" + ) + self._bind_contracts[digest] = cached + return cached + + def _report_bind_refusal(self, contract: Any, mismatch: Any) -> None: + from .bind_contract import report_refusal + + report_refusal(self._hub_base_url, contract, mismatch) + def resolve(self, model_cls: type, checkpoint_ref: str) -> DeployBinding: pick = self._pick(model_cls, checkpoint_ref) defaults: Mapping[str, Any] = {} @@ -377,6 +390,8 @@ def resolve(self, model_cls: type, checkpoint_ref: str) -> DeployBinding: checkpoint_dir=self.tree_for(model_cls, checkpoint_ref), model=pick.model or None, defaults=defaults, + bind_contract=self._bind_contract(pick), + bind_refusal_reporter=self._report_bind_refusal, ) def default_pick(self, model_cls: type, slot_name: str) -> str: @@ -456,7 +471,10 @@ def __init__( set_provider_index(build_provider_index_from_manifest(manifest)) self.loaded = harvest_entrypoints(list(user_module_names)) - self.resolver = HubBindingResolver() + self.resolver = HubBindingResolver( + release_id=str(settings.worker_release_id or ""), + hub_base_url=str(getattr(settings, "tensorhub_url", "") or ""), + ) budget = residency_budget(int(vram_budget_bytes)) self.residency = ResidencyManager(int(budget), SnapshotSizer(self.resolver)) hub_base = str(getattr(settings, "tensorhub_url", "") or "").strip() @@ -553,13 +571,13 @@ def _build_adoption(self) -> Any: artifacts = tensorhub_cache_dir() / "compiled-graphs" self._mint_box: List[Any] = [] - def _arm(adoption: Any) -> Any: + def _arm(store: Any, session: Any) -> Any: mint = production_mint( - store=adoption.store, artifacts_dir=artifacts, + store=store, artifacts_dir=artifacts, cas_dir=tensorhub_cas_dir(), sm=sm, ) self._mint_box.append(mint) - return mint.arm(adoption) + return mint.arm(session) return ServeAdoption( release_id, sm=sm, artifacts_dir=artifacts, @@ -989,7 +1007,7 @@ def _bind_context(ctx: Any, box: List[Any] = ctx_box) -> None: request_id=str(run.request_id), attempt=int(run.attempt), input_assets=manifest_from_run_job(run.input_assets), - snapshots=resolved_repos(run.snapshots, run.models), + snapshots=resolved_repos(run.snapshots), context=self._request_context_facts(run), on_context=_bind_context, ) diff --git a/tests/streaming_fixture.py b/tests/streaming_fixture.py index a0d9ccb9c..6e2d7777b 100644 --- a/tests/streaming_fixture.py +++ b/tests/streaming_fixture.py @@ -211,13 +211,31 @@ def tensors(self) -> Any: def length(self) -> int: return self._inner.length - def readinto(self, offset: int, length: int, buffer: Any) -> int: - self._log.append((self._container, int(offset), int(length))) - return self._inner.readinto(offset, length, buffer) + @property + def native_reader(self) -> Any: + return self._inner + + def fill_host_address( + self, + name: str, + destination_ptr: int, + destination_bytes: int, + destination_offset: int = 0, + layout: str = "torch.contiguous@1", + ) -> Any: + tensor = next(item for item in self.tensors if item.name == name) + self._log.append((self._container, int(tensor.offset), int(tensor.nbytes))) + return self._inner.fill_host_address( + name, + destination_ptr, + destination_bytes, + destination_offset, + layout, + ) class TracedStore: - """Records every byte range the engine asks the store for.""" + """Records every tensor range the engine asks tensorfs to fill.""" def __init__(self, inner: Any) -> None: self._inner = inner @@ -233,7 +251,7 @@ def open(self, container: str, *, direct: bool = False) -> Any: def assert_file_order(self) -> int: """One forward pass per container, never a seek backwards.""" - assert self.reads, "the engine read nothing" + assert self.reads, "the engine filled nothing" per_container: Dict[str, List[Tuple[int, int]]] = {} for container, offset, length in self.reads: per_container.setdefault(container, []).append((offset, length)) diff --git a/tests/test_adopt_flow.py b/tests/test_adopt_flow.py index abbc69133..ba0f19c0e 100644 --- a/tests/test_adopt_flow.py +++ b/tests/test_adopt_flow.py @@ -26,6 +26,7 @@ ) RELEASE = "rel-pgw1573" +BIND_DIGEST = "sha256:" + "b" * 64 @pytest.fixture() @@ -275,7 +276,8 @@ def _fleet_answer(document: Any, base_url: str, blobs: dict, }, }) return { - "object": "release_compiled_graphs", "release_id": RELEASE, + "object": "bind_compiled_graphs", "release_id": RELEASE, + "bind_contract_digest": BIND_DIGEST, "env_compile_stack": [[name, value] for name, value in STACK], "lane": LANE, "lane_stamped": True, "sm": SM, "empty": False, "hits": len(rows), "misses": 0, "graphs": rows, @@ -289,7 +291,7 @@ def _remote_store(cas: Path, base_url: str) -> Any: from gen_worker.serving.hub_store import HubGraphStore upstream = HubGraphStore( - HttpReleaseGraphTransport(base_url), RELEASE, LANE, SM) + HttpReleaseGraphTransport(base_url), RELEASE, BIND_DIGEST, LANE, SM) return graph_store(cas, upstream, cas.parent / "no-baked") diff --git a/tests/test_bind_contract_pgw1653.py b/tests/test_bind_contract_pgw1653.py new file mode 100644 index 000000000..29b4160ce --- /dev/null +++ b/tests/test_bind_contract_pgw1653.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +import pytest + +from gen_worker import bind_contract +from gen_worker.serving.context import DeployBinding, LoadContext +from gen_worker.serving.streaming.census import CensusMismatch, I5_TOTALITY + + +def _document() -> tuple[str, bytes]: + raw = json.dumps( + { + "v": 1, + "kind": "tensorhub.bind-contract@1", + "identity": { + "release_id": "release-a", + "derive_image_digest": "sha256:image", + "config_digest": "sha256:config", + }, + "release_contract_digest": "sha256:" + "1" * 64, + "construction_census": { + "v": 1, + "kind": "gen-worker.construction-census@1", + "pipeline_class": "TinyPipeline", + "components": { + "unet": { + "component": "unet", + "class": "TinyUNet", + "tensors": [], + "eval_mode": True, + } + }, + }, + "env_compile_stack": [], + "lanes": [{"stamp": "tiny@1"}], + "graphs": [{"lane": "tiny@1", "graph_hash": "cg-graph-v1-test"}], + }, + sort_keys=True, + separators=(",", ":"), + ).encode() + return "sha256:" + hashlib.sha256(raw).hexdigest(), raw + + +def test_fetch_verifies_the_CAS_address_before_decoding() -> None: + digest, raw = _document() + + class Response: + status = 200 + + def __enter__(self) -> "Response": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def read(self, _limit: int) -> bytes: + return raw + + requests: list[Any] = [] + + def open_request(request: Any, **_kwargs: Any) -> Response: + requests.append(request) + return Response() + + got = bind_contract.fetch( + digest, "https://objects.test/bind", token="worker-jwt", opener=open_request + ) + assert got.identity.config_digest == "sha256:config" + assert got.release_contract_digest == "sha256:" + "1" * 64 + assert got.census.pipeline_class == "TinyPipeline" + assert got.graphs[0]["graph_hash"] == "cg-graph-v1-test" + assert requests[0].get_header("Authorization") == "Bearer worker-jwt" + + wrong = "sha256:" + ("0" * 64) + with pytest.raises(bind_contract.BindContractError, match="fetched bytes hashing"): + bind_contract.fetch( + wrong, "https://objects.test/bind", token="worker-jwt", + opener=lambda *_a, **_k: Response(), + ) + + +def test_fetch_refuses_an_absent_worker_credential() -> None: + digest, _raw = _document() + with pytest.raises(bind_contract.BindContractError, match="no worker credential"): + bind_contract.fetch(digest, "https://hub.test/bind", token="") + + +def test_mismatch_report_is_attributed_to_the_bind_not_the_pod() -> None: + digest, raw = _document() + contract = bind_contract.decode(raw, digest=digest) + mismatch = CensusMismatch( + I5_TOTALITY, "unet", "weight", "shape moved", where="serve Tiny" + ) + payload = json.loads(bind_contract.refusal_payload(contract, mismatch)) + assert payload == { + "release_id": "release-a", + "derive_image_digest": "sha256:image", + "config_digest": "sha256:config", + "bind_contract_digest": digest, + "code": "bind_contract_census_mismatch", + "invariant": "I5_TOTALITY", + "component": "unet", + "tensor": "weight", + "detail": str(mismatch), + } + assert "pod" not in payload and "worker" not in payload + + +def test_serve_compares_against_remote_census_and_reports_before_refusing() -> None: + digest, raw = _document() + contract = bind_contract.decode(raw, digest=digest) + mismatch = CensusMismatch( + I5_TOTALITY, "unet", "weight", "shape moved", where="serve Tiny" + ) + reported: list[tuple[bind_contract.BindContract, CensusMismatch]] = [] + + class Engine: + def build( + self, pipeline_cls: type, *, checkpoint_dir: Path, + lane: Any, expected_census: Any = None, + ) -> Any: + assert expected_census is contract.census + raise mismatch + + binding = DeployBinding( + checkpoint_ref="org/model@release", + checkpoint_dir=Path("/unused"), + bind_contract=contract, + bind_refusal_reporter=lambda got, why: reported.append((got, why)), + ) + context: LoadContext[Any] = LoadContext(binding=binding, lane="plain") + + with pytest.raises(CensusMismatch) as caught: + context._streaming_build(Engine(), type("TinyPipeline", (), {})) + + assert caught.value is mismatch + assert reported == [(contract, mismatch)] diff --git a/tests/test_boot_end_verdict.py b/tests/test_boot_end_verdict.py index b5bf11c4a..c1a48c363 100644 --- a/tests/test_boot_end_verdict.py +++ b/tests/test_boot_end_verdict.py @@ -14,6 +14,11 @@ ) PHASE = EagerPhase.BOOT_ENDED_UNCOMPILED.value +BIND_DIGEST = "sha256:" + "b" * 64 + + +class _Model: + pass class FakeSession: @@ -72,14 +77,27 @@ def verdicts(seen: List[Any]) -> List[Any]: return [u for u in seen if u.kind == KIND_SKIPPED and u.phase == PHASE] +def register( + adoption: ServeAdoption, + session: Any, + contract: str = "h3.diffusers-bf16@1", +) -> None: + key = (_Model, BIND_DIGEST, contract) + adoption._class_keys[_Model] = key + if session is not None: + adoption._sessions[key] = session + adoption._stores[(BIND_DIGEST, contract)] = object() + adoption.adoption = session + adoption.contract = contract + + def test_declared_compile_and_ZERO_armed_specializations_FIRES( adoption: ServeAdoption, seen: List[Any] ) -> None: """The whole defect, in one boot: the release stamped a lane document, the document claimed graph specializations, not one of them armed, and nothing is going to fix it.""" - adoption.adoption = FakeSession(adopted=0, holes=0, unclaimed=3) - adoption.contract = "h3.diffusers-bf16@1" - adoption.loaded() + register(adoption, FakeSession(adopted=0, holes=0, unclaimed=3)) + adoption.loaded(_Model) rows = verdicts(seen) assert len(rows) == 1, f"expected the boot-end verdict, saw {phases(seen)}" @@ -92,10 +110,9 @@ def test_the_counts_ride_the_NUMERIC_columns_not_the_prose( ) -> None: """`0 of 6 armed` interpolated into `detail` is a metric nobody can group by.""" - adoption.adoption = FakeSession(adopted=0, holes=6) - adoption.contract = "h3.diffusers-bf16@1" - adoption._on_adopted = lambda _self: FakeMintStatus(running=False) - adoption.loaded() + register(adoption, FakeSession(adopted=0, holes=6)) + adoption._on_adopted = lambda _store, _session: FakeMintStatus(running=False) + adoption.loaded(_Model) row = verdicts(seen)[0] assert (row.step, row.total_steps) == (0, 6) @@ -106,10 +123,13 @@ def test_a_mint_that_NEVER_STARTED_is_terminal_and_FIRES( ) -> None: """`SelfMint.arm` answers `unavailable` on a pod with no compiler.""" - adoption.adoption = FakeSession(adopted=0, holes=2) - adoption.contract = "sdxl.diffusers@1+plain.bf16@1" - adoption._on_adopted = lambda _self: FakeMintStatus(running=False) - adoption.loaded() + register( + adoption, + FakeSession(adopted=0, holes=2), + "sdxl.diffusers@1+plain.bf16@1", + ) + adoption._on_adopted = lambda _store, _session: FakeMintStatus(running=False) + adoption.loaded(_Model) assert len(verdicts(seen)) == 1 @@ -120,7 +140,8 @@ def test_a_STAMPED_release_with_no_lane_document_for_this_sm_FIRES( """`no_document` is a release that DID declare compiled serving and simply has no graphs for this (lane x sm) — a pod serving eager under it is the headline shape, not an exemption.""" adoption._refuse("no_document", "no lane document for (rel-1 x lane x sm_90)") - adoption.loaded() + register(adoption, None) + adoption.loaded(_Model) assert len(verdicts(seen)) == 1 assert "no_document" in verdicts(seen)[0].detail @@ -131,7 +152,8 @@ def test_an_environment_mismatch_FIRES_because_the_release_still_declared( ) -> None: adoption._refuse("environment_mismatch", "closure abc != stamped def") - adoption.loaded() + register(adoption, None) + adoption.loaded(_Model) assert len(verdicts(seen)) == 1 @@ -141,10 +163,13 @@ def test_a_boot_that_ARMS_a_specialization_stays_SILENT( ) -> None: """One armed graph specialization is dispatchable.""" - adoption.adoption = FakeSession(adopted=1, holes=4) - adoption.contract = "sdxl.diffusers@1+plain.bf16@1" - adoption._on_adopted = lambda _self: FakeMintStatus(running=True) - adoption.loaded() + register( + adoption, + FakeSession(adopted=1, holes=4), + "sdxl.diffusers@1+plain.bf16@1", + ) + adoption._on_adopted = lambda _store, _session: FakeMintStatus(running=True) + adoption.loaded(_Model) assert verdicts(seen) == [] assert [u.phase for u in seen if u.kind == "boot_adopt_summary"] == ["minting"] @@ -155,10 +180,9 @@ def test_a_MINT_IN_FLIGHT_stays_SILENT_because_the_eager_window_has_an_END( ) -> None: """Boot 1 of any reuse proof: zero armed, holes registered, the background mint running.""" - adoption.adoption = FakeSession(adopted=0, holes=3) - adoption.contract = "h3.diffusers-bf16@1" - adoption._on_adopted = lambda _self: FakeMintStatus(running=True) - adoption.loaded() + register(adoption, FakeSession(adopted=0, holes=3)) + adoption._on_adopted = lambda _store, _session: FakeMintStatus(running=True) + adoption.loaded(_Model) assert verdicts(seen) == [] @@ -168,10 +192,9 @@ def test_an_UNANSWERING_hook_with_holes_stays_SILENT( ) -> None: """A wiring that returns nothing is UNKNOWN, and unknown resolves to silence when there are holes: a false alarm is worse than a missed one because it teaches the reader to stop reading.""" - adoption.adoption = FakeSession(adopted=0, holes=3) - adoption.contract = "h3.diffusers-bf16@1" - adoption._on_adopted = lambda _self: None - adoption.loaded() + register(adoption, FakeSession(adopted=0, holes=3)) + adoption._on_adopted = lambda _store, _session: None + adoption.loaded(_Model) assert verdicts(seen) == [] @@ -183,7 +206,8 @@ def test_an_EAGER_BY_DESIGN_release_stays_SILENT( """Not every eager boot is a defect.""" adoption._refuse(phase, "eager is this release's contract") - adoption.loaded() + register(adoption, None) + adoption.loaded(_Model) assert verdicts(seen) == [] @@ -203,10 +227,9 @@ def test_the_verdict_fires_ONCE_per_pod_not_once_per_instance( ) -> None: """`ServeLoop` makes instances lazily under residency leases, so `loaded` is called again for every new (class x checkpoint x lane).""" - adoption.adoption = FakeSession(adopted=0, holes=0, unclaimed=2) - adoption.contract = "h3.diffusers-bf16@1" + register(adoption, FakeSession(adopted=0, holes=0, unclaimed=2)) for _ in range(4): - adoption.loaded() + adoption.loaded(_Model) assert len(verdicts(seen)) == 1 diff --git a/tests/test_boot_materialize.py b/tests/test_boot_materialize.py index 1d2c41fb5..94fb5fd4d 100644 --- a/tests/test_boot_materialize.py +++ b/tests/test_boot_materialize.py @@ -259,8 +259,9 @@ async def run() -> None: asyncio.run(run()) pick = _Pick( - slot="model", ref=str(_REF), manifest_digest="", - model="", inference_defaults="", + slot="model", ref=str(_REF), model="", inference_defaults="", + bind_contract_digest="sha256:" + "0" * 64, + bind_contract_url="https://hub.invalid/bind", ) token = _DISPATCH.set( _DispatchPicks(by_ref={str(_REF): pick}, by_slot={"model": str(_REF)}) @@ -281,20 +282,14 @@ async def run() -> None: origin.close() -def test_the_dispatch_supplies_its_own_fetch_pointer(tmp_path: Path) -> None: - """`RunJob.snapshots` is keyed by ref and ships on every dispatch.""" - from gen_worker.worker import _picks_of +def test_the_dispatch_snapshot_map_has_one_ref_key() -> None: + """th#2208 deletes the unused second fetch pointer from ModelBinding.""" + from gen_worker.wire_snapshots import index_snapshots run = pb.RunJob() - binding = run.models.add() - binding.slot = "model" - binding.ref = str(_REF) run.snapshots[str(_REF)].digest = "b7f2c1a0" * 8 - picks = _picks_of(run) - assert picks.by_ref[str(_REF)].manifest_digest == "b7f2c1a0" * 8, ( - "the dispatch's own snapshot must supply the fetch pointer the wire's " - "digest field never carries") + assert index_snapshots(run.snapshots)[_REF].digest == "b7f2c1a0" * 8 @pytest.fixture diff --git a/tests/test_boot_warmup_pgw1584.py b/tests/test_boot_warmup_pgw1584.py index 336058908..0261de15b 100644 --- a/tests/test_boot_warmup_pgw1584.py +++ b/tests/test_boot_warmup_pgw1584.py @@ -417,7 +417,7 @@ def test_the_boot_picks_come_off_the_hubs_OWN_DesiredResidency_Hot() -> None: """Nothing is invented: `DesiredResidency.Hot` is `repeated DesiredInstance {function_name, models}` and `models` is the very `ModelBinding` a dispatch carries — slot, ref, the recognized `model` name...""" from gen_worker.worker import boot_picks - loaded = load_endpoint(FIXTURE_DIR) + loaded = object() desired = pb.DesiredResidency(generation=1, disk_refs=[DREAM, "org/other@1"]) desired.snapshots[DREAM].CopyFrom(pb.Snapshot(digest="sha256:beef")) instance = desired.hot.add() @@ -435,17 +435,14 @@ def test_the_boot_picks_come_off_the_hubs_OWN_DesiredResidency_Hot() -> None: assert table.by_ref[DREAM].inference_defaults == '{"steps": 4}' -def test_one_slot_and_one_ref_is_ARITHMETIC_but_anything_else_is_a_GUESS() -> None: - """The hub seeds `Hot` for dynamic-slot defaults and compile-cache prewarm; a static-binding release can arrive with it empty.""" +def test_no_hot_binding_means_no_boot_pick() -> None: + """A missing hub binding is absent, never synthesized from disk refs.""" from gen_worker.worker import boot_picks - loaded = load_endpoint(FIXTURE_DIR) + loaded = object() single = _config_for(DREAM) - picks = boot_picks(pb.DesiredResidency(), loaded, single) - assert picks["generate"].by_slot == {"model": DREAM} - assert picks["generate"].by_ref[DREAM].manifest_digest == "sha256:beef" - assert picks["generate"].by_ref[DREAM].model == "" + assert boot_picks(pb.DesiredResidency(), loaded, single) == {} two = pb.DesiredResidency(generation=1, disk_refs=[DREAM, "org/other@1"]) assert boot_picks(two, loaded, CheckpointConfig.from_wire(two)) == {} diff --git a/tests/test_checkpoint_binding.py b/tests/test_checkpoint_binding.py index 7b391e677..9e1f6049d 100644 --- a/tests/test_checkpoint_binding.py +++ b/tests/test_checkpoint_binding.py @@ -4,6 +4,7 @@ import json from pathlib import Path +from types import SimpleNamespace from typing import Any, Dict import pytest @@ -58,21 +59,35 @@ def _dispatch( pb.ModelBinding( slot="model", ref=ref, - manifest_digest=digest, model=model, inference_defaults="" if row is None else json.dumps(row), + bind_contract_digest="contract", + bind_contract_url="https://hub.invalid/bind", ) ] ) _DISPATCH.set(_picks_of(run)) - return HubBindingResolver(snapshots_root=root) + resolver = HubBindingResolver(snapshots_root=root) + resolver._bind_contracts["contract"] = SimpleNamespace( + digest="contract", + identity=SimpleNamespace(release_id="release"), + census=None, + ) + resolver.bind_store(SimpleNamespace( + disk_local_path=lambda candidate: tree if candidate == ref else None, + banked_snapshot=lambda candidate: None, + disk_refs=lambda: (ref,), + )) + return resolver def test_the_classification_column_is_on_the_wire_at_the_hubs_number() -> None: """These numbers are TENSORHUB's, and tensorhub is the only SENDER.""" fields = pb.ModelBinding.DESCRIPTOR.fields_by_name assert fields["model"].number == 9 - assert fields["manifest_digest"].number == 10 + assert fields["bind_contract_digest"].number == 11 + assert fields["bind_contract_url"].number == 12 + assert "manifest_digest" not in fields assert pb.LoraOverlay.DESCRIPTOR.fields_by_name["model"].number == 4 diff --git a/tests/test_checkpoint_juggle.py b/tests/test_checkpoint_juggle.py index 915a67503..9e6fae83f 100644 --- a/tests/test_checkpoint_juggle.py +++ b/tests/test_checkpoint_juggle.py @@ -4,7 +4,9 @@ import json import struct +from contextlib import nullcontext from pathlib import Path +from types import SimpleNamespace from typing import Any, Dict, List, Tuple import pytest @@ -20,6 +22,7 @@ from gen_worker.models.checkpoint_juggle import ( # noqa: E402 CheckpointCatalog, CheckpointImage, + CheckpointJuggler, JuggleRefusal, RegionInvalid, RegionValidity, @@ -181,7 +184,7 @@ def build_image( layout = layout_for(template) manifest = read_manifest(checkpoint_dir(tmp_path, name, module, **kw)) return layout, CheckpointImage( - name, layout, manifest, torch_mod=torch, varena_mod=None, engine=None + name, layout, manifest, torch_mod=torch, varena_mod=None ) @@ -323,3 +326,119 @@ def test_a_partial_switch_is_never_servable_under_either_identity(tmp_path: Path for identity in ("a", "b"): with pytest.raises(RegionInvalid): ledger.assert_servable(identity) + + +class _FillCapture: + staging = "tensorfs-pinned" + + def __init__(self) -> None: + self.addresses: List[Tuple[Any, Any]] = [] + self.files: List[Tuple[Any, Any]] = [] + + def fill_address(self, source: Any, destination: Any) -> Any: + self.addresses.append((source, destination)) + return SimpleNamespace(destination_bytes=destination.capacity) + + def fill_files(self, sources: Any, destination: Any) -> Any: + self.files.append((tuple(sources), destination)) + return SimpleNamespace(destination_bytes=destination.capacity) + + +def _fake_residency(layout: Any, compile_calls: List[int]) -> Any: + def compile_(value: Any) -> Any: + compile_calls.append(1) + return value + + fake_torch = SimpleNamespace( + no_grad=nullcontext, + cuda=SimpleNamespace(synchronize=lambda _device: None), + compile=compile_, + ) + return SimpleNamespace( + adopted=True, + layout=layout, + _torch=fake_torch, + _varena=None, + device=SimpleNamespace(index=0), + reservation=SimpleNamespace(base_ptr=0x40000000), + ring=SimpleNamespace(drain=lambda: None), + is_resident=lambda _name: True, + _host={}, + _triples={}, + _roots=[], + ) + + +def test_warm_swaps_reuse_tensorfs_at_stable_addresses_without_recompile( + tmp_path: Path, monkeypatch: Any +) -> None: + import gen_worker.models.checkpoint_juggle as checkpoint_juggle + + template = Net(seed=0) + mem = {"available": 64 << 30} + layout, catalog = make_catalog(tmp_path, template, mem, floor=4 << 30) + manifests = {} + for seed, name in ((30, "a"), (31, "b")): + manifest = read_manifest(checkpoint_dir(tmp_path, name, Net(seed=seed))) + manifests[name] = manifest + catalog.admit(name, manifest) + assert catalog.ensure_warm(name) is not None + + fills = _FillCapture() + monkeypatch.setattr(checkpoint_juggle, "CudaFillClient", lambda *_args: fills) + compile_calls: List[int] = [] + juggler = CheckpointJuggler( + _fake_residency(layout, compile_calls), + "a", + manifests["a"], + catalog=catalog, + ) + + juggler.switch_to("b") + juggler.switch_to("a") + + region_count = len(layout.regions) + first = [destination.pointer for _, destination in fills.addresses[:region_count]] + second = [destination.pointer for _, destination in fills.addresses[region_count:]] + assert first == second + assert first == [0x40000000 + region.offset for region in layout.regions] + assert compile_calls == [] + assert all(isinstance(source.pointer, int) for source, _ in fills.addresses) + assert all(isinstance(destination.shape, tuple) for _, destination in fills.addresses) + + +def test_cold_swap_is_file_records_through_the_same_tensorfs_client( + tmp_path: Path, monkeypatch: Any +) -> None: + import gen_worker.models.checkpoint_juggle as checkpoint_juggle + + template = Net(seed=0) + layout, catalog = make_catalog( + tmp_path, template, {"available": 0}, floor=4 << 30 + ) + manifests = { + name: read_manifest(checkpoint_dir(tmp_path, name, Net(seed=seed))) + for seed, name in ((40, "a"), (41, "b")) + } + catalog.admit("b", manifests["b"]) + + fills = _FillCapture() + monkeypatch.setattr(checkpoint_juggle, "CudaFillClient", lambda *_args: fills) + juggler = CheckpointJuggler( + _fake_residency(layout, []), + "a", + manifests["a"], + catalog=catalog, + ) + + report = juggler.switch_to("b") + + assert fills.addresses == [] + assert len(fills.files) == len(layout.regions) + for sources, destination in fills.files: + assert sum(source.length for source in sources) == destination.capacity + assert all( + source.path is None or isinstance(source.path, str) for source in sources + ) + assert report.tier == "disk-cold" + assert report.bytes_moved == sum(region.span for region in layout.regions) diff --git a/tests/test_lane_dtype_pgw1623.py b/tests/test_lane_dtype_pgw1623.py index c9fef1d24..5f82c9eb5 100644 --- a/tests/test_lane_dtype_pgw1623.py +++ b/tests/test_lane_dtype_pgw1623.py @@ -36,7 +36,7 @@ from gen_worker._vendor.tensorfs import LocalCAS, project_snapshot # noqa: E402 from gen_worker.models.projection import REF_PREFIX, SNAPSHOTS_DIR # noqa: E402 from gen_worker.serving.streaming import ( # noqa: E402 - BridgeWeightStore, + NativeWeightStore, StreamingLoader, ) from gen_worker.serving.streaming.engine import LaneDtypeUnmet # noqa: E402 @@ -114,8 +114,10 @@ def loaded(tmp_path_factory: pytest.TempPathFactory) -> Any: source = base / "source-model" pipeline_cls = _heterogeneous_source(source) tree = _project(base, source, key="c" * 64) + cas, manifest = _cas_manifest(tree) loader = StreamingLoader( - BridgeWeightStore(*_cas_manifest(tree)), device="cpu", buffer_bytes=4096 + NativeWeightStore.from_manifest(cas.root, manifest), + device="cpu", ) pipeline = loader.build(pipeline_cls, checkpoint_dir=tree, lane=Lane) return {"pipeline": pipeline, "report": loader.last_report, "tree": tree, diff --git a/tests/test_release_derive_census_pgw1647.py b/tests/test_release_derive_census_pgw1647.py index 29068260b..60a0d452f 100644 --- a/tests/test_release_derive_census_pgw1647.py +++ b/tests/test_release_derive_census_pgw1647.py @@ -91,7 +91,9 @@ def test_the_release_document_carries_the_census_in_th2281_s_ENVELOPE_pgw1647( lockfile=lockfile, ) document = json.loads(result.document) - row = document["construction_census"] + (tree_row,) = document["construction_censuses"] + assert tree_row["owners"] == ["$primary"] + row = tree_row["census"] assert row["v"] == census.CENSUS_VERSION assert row["kind"] == census.CENSUS_KIND assert row["pipeline_class"] == "StableDiffusionPipeline" @@ -109,14 +111,15 @@ def test_the_release_document_carries_the_census_in_th2281_s_ENVELOPE_pgw1647( assert parsed.tensor_count > 0 for component in parsed.components: assert component.eval_mode is True, component.component - assert result.census_components == ("text_encoder", "unet", "vae") - assert result.census_absent == "" + assert "construction_census" not in document, ( + "the deleted primary-only field survived the per-tree hardcut" + ) def test_the_census_is_LANE_INVARIANT_and_says_so_pgw1647( config_only_tree: Path, lockfile: Path ) -> None: - """ONE census per release, not one per lane — and the reason is in the row. + """ONE census per config tree, not one per lane — and the reason is in the row. A lane's only effect on construction is the dtype it casts wide floats to, and that fact already has a precise owner (the lane contract, and @@ -132,7 +135,8 @@ def test_the_census_is_LANE_INVARIANT_and_says_so_pgw1647( lockfile=lockfile, ) document = json.loads(result.document) - parsed = census.Census.from_document(document["construction_census"]) + parsed = census.Census.from_document( + document["construction_censuses"][0]["census"]) unet = parsed.by_component()["unet"] dtypes = {row.dtype for row in unet.tensors} @@ -239,7 +243,7 @@ def test_the_release_census_verifies_the_module_the_serve_path_builds_pgw1647( lockfile=lockfile, ) published = census.Census.from_document( - json.loads(result.document)["construction_census"]) + json.loads(result.document)["construction_censuses"][0]["census"]) import torch diff --git a/tests/test_release_derive_multi_class.py b/tests/test_release_derive_multi_class.py index 94e8b4ee7..c4f0f5d08 100644 --- a/tests/test_release_derive_multi_class.py +++ b/tests/test_release_derive_multi_class.py @@ -90,6 +90,22 @@ def test_a_release_derives_every_compile_marking_class( ) == 0 document = json.loads(out.read_bytes()) + # pgw#1653: BOTH distinct config trees state their own census. The old + # primary-only field warned about EditModel and then let absence look green. + assert "construction_census" not in document + census_rows = document["construction_censuses"] + assert len(census_rows) == 2, census_rows + by_owner = { + tuple(row["owners"]): row["census"] for row in census_rows + } + assert ("$primary",) in by_owner + assert ("EditModel",) in by_owner + assert by_owner[("$primary",)]["components"] + assert by_owner[("EditModel",)]["components"] + assert by_owner[("$primary",)] != by_owner[("EditModel",)], ( + "the wider EditModel tree produced the primary tree's census" + ) + # BOTH classes are the release, and the endpoint name says so. assert document["endpoint"].endswith(":EditModel+PrimaryModel") classes = {row["class"]: row for row in document["classes"]} diff --git a/tests/test_resolved_manifest_pgw1246.py b/tests/test_resolved_manifest_pgw1246.py index 2334d4dc1..9ccf7de30 100644 --- a/tests/test_resolved_manifest_pgw1246.py +++ b/tests/test_resolved_manifest_pgw1246.py @@ -6,7 +6,7 @@ from gen_worker.models import cozy_snapshot, download from gen_worker.models.refs import WireRef from gen_worker.pb import worker_scheduler_pb2 as pb -from gen_worker.wire_snapshots import AmbiguousManifestError, index_snapshots +from gen_worker.wire_snapshots import index_snapshots def _snap(digest: str, *paths: str) -> pb.Snapshot: @@ -16,16 +16,13 @@ def _snap(digest: str, *paths: str) -> pb.Snapshot: ) -def test_the_fetch_key_is_the_composed_digest_not_the_ref() -> None: - """Two slots on ONE repo with different component sources are two map entries with two digests.""" +def test_the_fetch_key_is_the_canonical_ref() -> None: + """th#2208 leaves one ref-keyed channel for snapshot fetch facts.""" wire = { - "d-a": _snap("d-a", "model_index.json", "vae/a.safetensors"), - "d-b": _snap("d-b", "model_index.json", "vae/b.safetensors"), + "acme/sdxl@prod": _snap("d-a", "model_index.json", "vae/a.safetensors"), + "acme/sdxl-r@prod": _snap("d-b", "model_index.json", "vae/b.safetensors"), } - out = index_snapshots(wire, [ - pb.ModelBinding(slot="pipeline", ref="acme/sdxl@prod", manifest_digest="d-a"), - pb.ModelBinding(slot="refiner", ref="acme/sdxl-r@prod", manifest_digest="d-b"), - ]) + out = index_snapshots(wire) assert out[WireRef("acme/sdxl@prod")].digest == "d-a" assert out[WireRef("acme/sdxl-r@prod")].digest == "d-b" assert [f.path for f in out[WireRef("acme/sdxl@prod")].files] == [ @@ -35,22 +32,10 @@ def test_the_fetch_key_is_the_composed_digest_not_the_ref() -> None: def test_an_artifact_with_no_composition_still_resolves_by_its_own_key() -> None: """LoRA overlays and payload source refs carry no ModelBinding, so the hub keys them by ref.""" wire = {"acme/lora@prod": _snap("dl", "adapter.safetensors")} - assert index_snapshots(wire, []) == { + assert index_snapshots(wire) == { WireRef("acme/lora@prod"): wire["acme/lora@prod"]} -def test_one_ref_bound_to_two_manifests_refuses_typed() -> None: - """The deciding twin of the first test: a worker that PICKED one of the two would have re-derived, by hand, the outbound mutation this deletes.""" - wire = {"d-a": _snap("d-a", "a"), "d-b": _snap("d-b", "b")} - with pytest.raises(AmbiguousManifestError) as err: - index_snapshots(wire, [ - pb.ModelBinding(slot="pipeline", ref="acme/sdxl@prod", manifest_digest="d-a"), - pb.ModelBinding(slot="refiner", ref="acme/sdxl@prod", manifest_digest="d-b"), - ]) - assert "acme/sdxl@prod" in str(err.value) - assert "d-a" in str(err.value) and "d-b" in str(err.value) - - def test_the_snapshot_directory_IS_the_composed_digest() -> None: """One key, one meaning: fetch identity, dispatch key and directory name.""" assert cozy_snapshot.snapshot_dir_key("sha256:cafe") == "sha256:cafe" @@ -65,4 +50,3 @@ def test_positive_component_selection_survives_and_negative_does_not() -> None: assert download.select_component_paths(paths, ()) == set(paths) with pytest.raises(TypeError): download.select_component_paths(paths, (), ("vae",)) # type: ignore[call-arg] - diff --git a/tests/test_self_mint_wiring.py b/tests/test_self_mint_wiring.py index 68a0454c6..3fd04d5a9 100644 --- a/tests/test_self_mint_wiring.py +++ b/tests/test_self_mint_wiring.py @@ -33,6 +33,7 @@ FIXTURE_DIR = Path(__file__).parent / "fixtures" / "serving_v2_endpoint" LANE = "sdxl.diffusers@1+plain.bf16@1" SM = "sm_89" +BIND_DIGEST = "sha256:" + "b" * 64 STACK: tuple[tuple[str, str], ...] = (("torch", torch.__version__),) ENV = EnvIdentity(stack=STACK, sm=SM) OVERRIDES: dict[str, Any] = {"steps": {"default": 2, "lo": 1, "hi": 8}} @@ -48,6 +49,7 @@ def binding(tmp_path: Path) -> DeployBinding: return DeployBinding( checkpoint_ref="ckpt:tiny@1", checkpoint_dir=root, model="sdxl", defaults=dict(OVERRIDES), + bind_contract=SimpleNamespace(digest=BIND_DIGEST), ) @@ -326,10 +328,14 @@ def __init__(self, answer: Any) -> None: self.answer = answer self.asks = 0 - def release_compiled_graphs(self, release_id: str, lane: str, sm: str) -> Any: + def bind_compiled_graphs( + self, bind_contract_digest: str, lane: str, sm: str + ) -> Any: self.asks += 1 if self.answer is None: - raise ReleaseNotStamped(f"release {release_id} is not stamped") + raise ReleaseNotStamped( + f"Bind Contract {bind_contract_digest} is unavailable" + ) return self.answer def fetch_blob(self, url: str) -> bytes: # pragma: no cover — all misses @@ -339,9 +345,9 @@ def fetch_blob(self, url: str) -> bytes: # pragma: no cover — all misses def all_miss_answer(document: GraphSetDocument) -> dict: lane = document.lanes[0] return { - "object": "release_compiled_graphs", + "object": "bind_compiled_graphs", "release_id": "release-1", - "binding_generation": 0, + "bind_contract_digest": BIND_DIGEST, "env_compile_stack": [list(row) for row in document.stack], "lane": lane.contract, "lane_stamped": True, @@ -377,25 +383,85 @@ def test_the_serve_loop_seam_adopts_from_the_hub_and_arms_the_mint( adoption = ServeAdoption( "release-1", sm=SM, artifacts_dir=tmp_path / "adopted", cas_dir=tmp_path / "podcas", transport=transport, stack=STACK, - loader=counting_loader([]), on_adopted=armed.append, + loader=counting_loader([]), + on_adopted=lambda store, session: armed.append((store, session)), ) loaded = load_endpoint(FIXTURE_DIR) (model_cls,) = loaded.models lane = loaded.lane(model_cls, LANE) - sink = adoption.sink_for(model_cls, lane) + sink = adoption.sink_for(model_cls, lane, binding) assert callable(sink), "the adopt sink is what ctx.compile arms through" assert transport.asks == 1 assert armed == [], "the mint must not be armed before the load runs" adoption.loaded(model_cls, lane) - assert armed == [adoption], "the post-load hook is the mint's trigger" + assert armed == [(adoption.store, adoption.adoption)], ( + "the post-load hook is the mint's trigger" + ) adoption.loaded(model_cls, lane) - assert armed == [adoption], "the trigger fires once, not once per load" - assert adoption.sink_for(model_cls, lane) is not None + assert armed == [(adoption.store, adoption.adoption)], ( + "the trigger fires once, not once per load" + ) + assert adoption.sink_for(model_cls, lane, binding) is not None assert transport.asks == 1 assert isinstance(adoption.store, TieredGraphStore) +def test_adoption_keys_graphs_by_bind_and_mint_sessions_by_class( + binding: DeployBinding, document: GraphSetDocument, tmp_path: Path, +) -> None: + """A second config tree never inherits the first tree's graph answer.""" + + second_digest = "sha256:" + "c" * 64 + + class PerBindTransport: + def __init__(self) -> None: + self.asks: List[str] = [] + + def bind_compiled_graphs( + self, bind_contract_digest: str, lane: str, sm: str + ) -> Any: + self.asks.append(bind_contract_digest) + answer = all_miss_answer(document) + answer["bind_contract_digest"] = bind_contract_digest + return answer + + def fetch_blob(self, url: str) -> bytes: # pragma: no cover + raise AssertionError(f"all-miss answer fetched {url}") + + class Primary: + pass + + class PrimaryTwin: + pass + + class Secondary: + pass + + transport = PerBindTransport() + adoption = ServeAdoption( + "release-1", + sm=SM, + artifacts_dir=tmp_path / "adopted", + cas_dir=tmp_path / "podcas", + transport=transport, + stack=STACK, + ) + lane = LANE + assert adoption.sink_for(Primary, lane, binding) is not None + assert adoption.sink_for(PrimaryTwin, lane, binding) is not None + secondary = DeployBinding( + checkpoint_ref=binding.checkpoint_ref, + checkpoint_dir=binding.checkpoint_dir, + bind_contract=SimpleNamespace(digest=second_digest), + ) + assert adoption.sink_for(Secondary, lane, secondary) is not None + + assert transport.asks == [BIND_DIGEST, second_digest] + assert len(adoption._sessions) == 3 + assert len(adoption._stores) == 2 + + def test_the_production_loop_adopts_on_first_load_and_mints_what_it_missed( document: GraphSetDocument, tmp_path: Path, ) -> None: @@ -411,7 +477,8 @@ class Resolver: def resolve(self, model_cls: type, checkpoint_ref: str) -> DeployBinding: return DeployBinding( checkpoint_ref=checkpoint_ref, checkpoint_dir=tree, - model="sdxl", defaults=dict(OVERRIDES)) + model="sdxl", defaults=dict(OVERRIDES), + bind_contract=SimpleNamespace(digest=BIND_DIGEST)) def default_pick(self, model_cls: type, slot_name: str) -> str: return "ckpt:tiny@1" @@ -421,10 +488,10 @@ def default_pick(self, model_cls: type, slot_name: str) -> str: mints: List[SelfMint] = [] compiler = FakeCompiler() - def arm_the_mint(adoption: ServeAdoption) -> None: - mint = a_mint(adoption.store, tmp_path, compiler, "loop") + def arm_the_mint(store: Any, session: Any) -> None: + mint = a_mint(store, tmp_path, compiler, "loop") mints.append(mint) - mint.arm(adoption) + mint.arm(session) adoption = ServeAdoption( "release-1", sm=SM, artifacts_dir=tmp_path / "adopted", @@ -475,7 +542,13 @@ def test_an_unstamped_release_is_an_eager_pod_with_a_stated_reason( transport=StubTransport(None), stack=STACK) loaded = load_endpoint(FIXTURE_DIR) (model_cls,) = loaded.models - assert adoption.sink_for(model_cls, loaded.lane(model_cls, LANE)) is None + stamped_binding = DeployBinding( + checkpoint_ref="ckpt:tiny@1", checkpoint_dir=tmp_path, + bind_contract=SimpleNamespace(digest=BIND_DIGEST), + ) + assert adoption.sink_for( + model_cls, loaded.lane(model_cls, LANE), stamped_binding + ) is None assert "release_not_stamped" in adoption.refusal assert adoption.facts() == {"adopting": False, "refusal": adoption.refusal} assert [e for e in wire if e[0] == "adopt_refused"] @@ -486,7 +559,9 @@ def test_a_minted_artifact_the_fleet_cannot_take_is_still_banked_and_stated( ) -> None: """The upstream publish leg has no worker-side caller at HEAD, so the hub store refuses.""" local = LocalGraphStore(LocalCAS(tmp_path / "cas")) - upstream = HubGraphStore(StubTransport(None), "release-1", LANE, SM) + upstream = HubGraphStore( + StubTransport(None), "release-1", BIND_DIGEST, LANE, SM + ) store = TieredGraphStore(local, upstream) artifact = elf(tmp_path / "one.so") @@ -538,7 +613,8 @@ class Resolver: def resolve(self, model_cls: type, checkpoint_ref: str) -> DeployBinding: return DeployBinding( checkpoint_ref=checkpoint_ref, checkpoint_dir=tree, - model="sdxl", defaults=dict(OVERRIDES)) + model="sdxl", defaults=dict(OVERRIDES), + bind_contract=SimpleNamespace(digest=BIND_DIGEST)) def default_pick(self, model_cls: type, slot_name: str) -> str: return "ckpt:tiny@1" @@ -547,10 +623,10 @@ def default_pick(self, model_cls: type, slot_name: str) -> str: mints: List[SelfMint] = [] def boot(tag: str, compiler: FakeCompiler) -> ServeAdoption: - def arm(adoption: ServeAdoption) -> None: - mint = a_mint(adoption.store, tmp_path, compiler, tag) + def arm(store: Any, session: Any) -> None: + mint = a_mint(store, tmp_path, compiler, tag) mints.append(mint) - mint.arm(adoption) + mint.arm(session) adoption = ServeAdoption( "release-1", sm=SM, artifacts_dir=tmp_path / f"adopted-{tag}", diff --git a/tests/test_serving_adopt_first.py b/tests/test_serving_adopt_first.py index 79421d411..f74c5e4b5 100644 --- a/tests/test_serving_adopt_first.py +++ b/tests/test_serving_adopt_first.py @@ -29,6 +29,7 @@ FIXTURE_DIR = Path(__file__).parent / "fixtures" / "serving_v2_endpoint" LANE = "sdxl.diffusers@1+plain.bf16@1" SM = "sm_89" +BIND_DIGEST = "sha256:" + "b" * 64 STACK: tuple[tuple[str, str], ...] = (("torch", torch.__version__),) ENV = EnvIdentity(stack=STACK, sm=SM) @@ -326,8 +327,8 @@ def __init__(self, answer: Mapping[str, Any], blobs: Mapping[str, bytes]) -> Non self.blobs = dict(blobs) self.asks = 0 - def release_compiled_graphs( - self, release_id: str, lane: str, sm: str + def bind_compiled_graphs( + self, bind_contract_digest: str, lane: str, sm: str ) -> Mapping[str, Any]: self.asks += 1 return self.answer @@ -378,9 +379,9 @@ def row(record: GraphRecord, status: str) -> dict[str, Any]: return base return { - "object": "release_compiled_graphs", + "object": "bind_compiled_graphs", "release_id": "release-1", - "binding_generation": 0, + "bind_contract_digest": BIND_DIGEST, "env_compile_stack": [list(row) for row in document.stack], "lane": lane.contract, "lane_stamped": True, @@ -405,7 +406,7 @@ def test_hub_store_partial_hit_verifies_digests_and_misses_clean( payload = b"presigned-compiled-bytes" answer = _adopt_answer(document, hit, hole, payload) transport = StubTransport(answer, {"https://presigned.example/hit": payload}) - store = HubGraphStore(transport, "release-1", LANE, SM) + store = HubGraphStore(transport, "release-1", BIND_DIGEST, LANE, SM) rebuilt = store.get_graphs("release-1") assert rebuilt is not None and rebuilt.stack == document.stack @@ -428,7 +429,9 @@ def test_hub_store_partial_hit_verifies_digests_and_misses_clean( assert hit_manifest is not None and hit_manifest.sm_compiled == SM transport.blobs["https://presigned.example/hit"] = b"tampered" - tampered_store = HubGraphStore(transport, "release-1", LANE, SM) + tampered_store = HubGraphStore( + transport, "release-1", BIND_DIGEST, LANE, SM + ) tampered_host = fresh_host(binding, tmp_path) tampered_host.setup( store=tampered_store, document=tampered_store.get_graphs("release-1"), @@ -450,22 +453,22 @@ def test_the_adopt_route_is_allowlisted_in_procsplit_pgw1372() -> None: action, query, _ = actions.authorize({ "method": "GET", - "path": "/v1/worker/releases/rel-123/compiled-graphs", + "path": f"/v1/worker/bind-contracts/{BIND_DIGEST}/compiled-graphs", "query": {"lane": LANE, "sm": SM}, }) - assert action.name == "release.compiled_graphs" + assert action.name == "bind.compiled_graphs" assert query == {"lane": LANE, "sm": SM} with pytest.raises(actions.ActionRefused, match="org_id"): actions.authorize({ "method": "GET", - "path": "/v1/worker/releases/rel-123/compiled-graphs", + "path": f"/v1/worker/bind-contracts/{BIND_DIGEST}/compiled-graphs", "query": {"lane": LANE, "sm": SM, "org_id": "someone-elses"}, }) with pytest.raises(actions.ActionRefused, match="not an allowlisted"): actions.authorize({ "method": "GET", - "path": "/v1/worker/releases/rel-123/extra/compiled-graphs", + "path": f"/v1/worker/bind-contracts/{BIND_DIGEST}/extra/compiled-graphs", "query": {"lane": LANE, "sm": SM}, }) diff --git a/tests/test_tensorfs_fill_client_pgw1648.py b/tests/test_tensorfs_fill_client_pgw1648.py new file mode 100644 index 000000000..461af3b5d --- /dev/null +++ b/tests/test_tensorfs_fill_client_pgw1648.py @@ -0,0 +1,87 @@ +"""pgw#1648: tensorfs owns bytes; pgw hands it destination data.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, get_args, get_origin, get_type_hints + +from gen_worker.serving.streaming.fill_client import ( + AddressSource, + Destination, + FileSource, + HostFillClient, +) + + +class _Reader: + def __init__(self) -> None: + self.call: tuple[Any, ...] = () + + def fill_host_address(self, *args: Any, **kwargs: Any) -> object: + self.call = (*args, kwargs) + return object() + + +def test_destination_map_crosses_as_plain_data_only() -> None: + destination = Destination( + name="layer.weight", + pointer=1234, + capacity=4096, + source_offset=8192, + shape=(16, 32), + element_bytes=2, + layout="torch.contiguous@1", + ) + annotations = get_type_hints(Destination) + assert set(annotations) == { + "name", + "pointer", + "capacity", + "source_offset", + "shape", + "element_bytes", + "layout", + } + assert all("torch" not in str(annotation).lower() for annotation in annotations.values()) + shape = annotations["shape"] + assert get_origin(shape) in (tuple, None) or tuple in get_args(shape) + + reader = _Reader() + result = HostFillClient().fill(reader, destination) + + assert result is not None + assert reader.call == ( + "layer.weight", + 1234, + 4096, + {"layout": "torch.contiguous@1"}, + ) + assert all(isinstance(value, (str, int, tuple, dict)) for value in reader.call) + + for seam_type in (AddressSource, FileSource): + assert all( + "torch" not in str(annotation).lower() + for annotation in get_type_hints(seam_type).values() + ) + + +def test_the_replaced_pgw_byte_plane_is_deleted() -> None: + root = Path(__file__).parents[1] / "src/gen_worker/serving/streaming" + assert not (root / "staging.py").exists() + sources = "\n".join(path.read_text(encoding="utf-8") for path in sorted(root.glob("*.py"))) + for dead in ( + "BridgeWeightStore", + "StagingPool", + "cudaMemcpyAsync", + "class _Placement", + "def _walk(", + ): + assert dead not in sources + + +def test_the_fill_seam_module_has_no_torch_import_or_type() -> None: + import gen_worker.serving.streaming.fill_client as fill_client + + source = Path(fill_client.__file__).read_text(encoding="utf-8") + assert "import torch" not in source + assert "torch.Tensor" not in source diff --git a/tests/test_weight_streaming.py b/tests/test_weight_streaming.py index f366625c3..82747c5c2 100644 --- a/tests/test_weight_streaming.py +++ b/tests/test_weight_streaming.py @@ -15,7 +15,7 @@ from cas_fixture import ingest_repository # noqa: E402 from gen_worker.models.projection import REF_PREFIX, SNAPSHOTS_DIR # noqa: E402 from gen_worker.serving.streaming import ( # noqa: E402 - BridgeWeightStore, + NativeWeightStore, NameMismatch, StreamingLoader, engine_for, @@ -38,9 +38,6 @@ write_bytes_now, ) -WINDOW = 4096 - - def _project(base: Path, source: Path, key: str) -> Path: cas = LocalCAS(base) manifest = ingest_repository(cas, source) @@ -70,6 +67,11 @@ def _cas_manifest(tree: Path) -> Tuple[Any, Any]: return projected.cas, projected.manifest +def _native_store(tree: Path) -> NativeWeightStore: + cas, manifest = _cas_manifest(tree) + return NativeWeightStore.from_manifest(cas.root, manifest) + + def test_the_fixture_can_actually_witness_a_scrambled_walk( article: dict[str, Any] ) -> None: @@ -95,9 +97,15 @@ def test_ctx_load_streams_store_to_memory_writing_nothing( stubs = [p for p in sorted(tree.rglob("*.safetensors")) if stub_at(p) is not None] assert stubs, f"{tree} projected no pointer stubs — nothing to stream" - store = TracedStore(BridgeWeightStore(*_cas_manifest(tree))) - loader = StreamingLoader(store, device="cpu", buffer_bytes=WINDOW, buffers=3) + store = TracedStore(_native_store(tree)) + loader = StreamingLoader(store, device="cpu") + # Warm library-level config/import caches before the process-I/O arm. The + # measured load below is still a fresh skeleton and a fresh destination + # map; only unrelated one-time interpreter writes are outside the fence. + StreamingLoader(_native_store(tree), device="cpu").build( + pipeline_cls, checkpoint_dir=tree, lane=Lane() + ) before = write_bytes_now() pipeline = loader.build(pipeline_cls, checkpoint_dir=tree, lane=Lane()) written = write_bytes_now() - before @@ -106,7 +114,7 @@ def test_ctx_load_streams_store_to_memory_writing_nothing( report = loader.last_report assert report is not None assert report.weights_streamed_bytes > 0 - assert report.staging == "pageable" + assert report.staging == "destination" assert report.io == "buffered" assert report.containers == 4 @@ -115,14 +123,14 @@ def test_ctx_load_streams_store_to_memory_writing_nothing( for component in ("unet", "vae", "text_encoder", "text_encoder_2"): assert on_meta(getattr(pipeline, component)) == () - windows = store.assert_file_order() - assert windows > 20, ( - f"only {windows} window(s) were read; a walk that fits in one window " - f"is ordered by accident and cannot witness a scrambled one" + fills = store.assert_file_order() + assert fills > 20, ( + f"only {fills} tensor(s) were filled; a one-tensor container " + f"cannot witness a scrambled walk" ) - assert report.windows == windows + assert report.tensors == fills - assert written < 1 << 20, ( + assert written == 0, ( f"the streamed load wrote {written} bytes; the whole point of the " f"2026-08-19 ruling is that it writes none" ) @@ -132,8 +140,8 @@ def test_every_container_is_read_end_to_end_exactly_once( article: dict[str, Any] ) -> None: """The windows tile each container's data range: no gap (a tensor read from nowhere) and no overlap (a byte paid for twice).""" - store = TracedStore(BridgeWeightStore(*_cas_manifest(article["tree"]))) - loader = StreamingLoader(store, device="cpu", buffer_bytes=WINDOW, buffers=3) + store = TracedStore(_native_store(article["tree"])) + loader = StreamingLoader(store, device="cpu") loader.build(article["pipeline_cls"], checkpoint_dir=article["tree"], lane=Lane()) per_container: dict[str, list[tuple[int, int]]] = {}