diff --git a/apis/inferenceclasses/definition.yaml b/apis/inferenceclasses/definition.yaml index b5c4d2d24..7487d37ca 100644 --- a/apis/inferenceclasses/definition.yaml +++ b/apis/inferenceclasses/definition.yaml @@ -44,10 +44,12 @@ spec: message: spec.provisioning.nebius is required when spec.provisioning.provider is Nebius. - rule: "self.provider != 'Vultr' || has(self.vultr)" message: spec.provisioning.vultr is required when spec.provisioning.provider is Vultr. + - rule: "self.provider != 'VultrBaremetal' || has(self.vultrBaremetal)" + message: spec.provisioning.vultrBaremetal is required when spec.provisioning.provider is VultrBaremetal. properties: provider: type: string - enum: [GKE, EKS, AKS, Nebius, Vultr] + enum: [GKE, EKS, AKS, Nebius, Vultr, VultrBaremetal] gke: type: object required: [machineType, accelerator] @@ -242,6 +244,41 @@ spec: type: integer minimum: 1 maximum: 16 + vultrBaremetal: + type: object + required: [plan, accelerator] + properties: + plan: + type: string + description: >- + Vultr bare metal plan ID (e.g. + vbm-256c-3072gb-8-mi355x-gpu). The plan + determines the GPU model and count; the + accelerator block below is informational. + minLength: 1 + maxLength: 63 + accelerator: + type: object + description: >- + GPU accelerator the plan carries. Provisioning + input only: the scheduler matches against + spec.devices, not this block. The type's vendor + prefix (amd-, nvidia-) selects the GPU node + taint and labels. + required: [type, count] + properties: + type: + type: string + description: >- + GPU accelerator type (e.g. amd-mi355x, + nvidia-h100). Reported on the consuming + InferenceCluster's status. + minLength: 1 + maxLength: 63 + count: + type: integer + minimum: 1 + maximum: 16 devices: type: array description: >- diff --git a/apis/inferenceclusters/definition.yaml b/apis/inferenceclusters/definition.yaml index 07088038b..f485f5b1b 100644 --- a/apis/inferenceclusters/definition.yaml +++ b/apis/inferenceclusters/definition.yaml @@ -32,6 +32,10 @@ spec: x-kubernetes-validations: - rule: "self.cluster.source != 'Nebius' || !has(self.nodePools) || self.nodePools.all(p, !has(p.fabric) || p.fabric.type != 'InfiniBand' || has(p.fabric.infiniband))" message: fabric.infiniband is required when fabric.type is InfiniBand and cluster.source is Nebius. + - rule: "self.cluster.source != 'VultrBaremetal' || !has(self.nodePools) || self.nodePools.all(p, !has(p.maxNodeCount))" + message: node pools cannot autoscale when cluster.source is VultrBaremetal; bare metal pools are fixed size. + - rule: "self.cluster.source != 'VultrBaremetal' || !has(self.stack) || self.stack == 'Standard'" + message: only the Standard stack is supported when cluster.source is VultrBaremetal. properties: cluster: type: object @@ -49,12 +53,14 @@ spec: message: spec.cluster.nebius is required when spec.cluster.source is Nebius. - rule: "self.source != 'Vultr' || has(self.vultr)" message: spec.cluster.vultr is required when spec.cluster.source is Vultr. + - rule: "self.source != 'VultrBaremetal' || has(self.vultrBaremetal)" + message: spec.cluster.vultrBaremetal is required when spec.cluster.source is VultrBaremetal. properties: source: type: string description: >- Cluster provisioning method. - enum: [GKE, EKS, AKS, Nebius, Vultr, Existing] + enum: [GKE, EKS, AKS, Nebius, Vultr, VultrBaremetal, Existing] existing: type: object description: >- @@ -310,6 +316,128 @@ spec: default: default minLength: 1 maxLength: 253 + vultrBaremetal: + type: object + description: >- + Vultr bare metal (k3s) cluster configuration. + Required when source is VultrBaremetal. Provisions + bare metal servers - one CPU-only management server + plus the GPU pools - and installs a k3s cluster + onto them over SSH. Bare metal has no autoscaling, + so pools are fixed size, and provisioning takes + tens of minutes. + required: [region, ssh] + properties: + region: + type: string + description: >- + Vultr region for all servers (e.g. ewr, ord). + Bare metal plan availability varies by region; + check with vultr-cli plans list --type vbm. + minLength: 1 + maxLength: 32 + management: + type: object + default: {} + description: >- + The CPU-only bare metal server that runs the + k3s server (the management plane). + properties: + plan: + type: string + default: vbm-6c-32gb-amd + description: >- + Vultr bare metal plan for the management + server. The default is a CPU-only plan + available in most regions that offer bare + metal. + minLength: 1 + maxLength: 63 + osId: + type: integer + default: 2284 + description: >- + Vultr operating system ID installed on + every server. Defaults to Ubuntu 24.04 LTS + x64; list IDs with vultr-cli os list. + ssh: + type: object + description: >- + SSH key pair used to reach the servers. The + public key is registered with Vultr and + installed on every server; the private key + drives the k3s install over SSH. + required: [secretRef] + properties: + secretRef: + type: object + description: >- + Secret holding the SSH key pair. The Secret + must exist in the modelplane-system + namespace. + required: [name] + properties: + name: + type: string + minLength: 1 + maxLength: 253 + privateKeyKey: + type: string + default: ssh-privatekey + description: >- + Key within the Secret that holds the + private key. + minLength: 1 + maxLength: 253 + publicKeyKey: + type: string + default: ssh-publickey + description: >- + Key within the Secret that holds the + public key. + minLength: 1 + maxLength: 253 + username: + type: string + default: root + description: >- + SSH user the servers accept the key for. + Vultr installs keys for root by default. + minLength: 1 + maxLength: 63 + k3s: + type: object + default: {} + description: The k3s release installed on the servers. + properties: + channel: + type: string + default: v1.34 + description: >- + k3s release channel. Defaults to the first + channel where Dynamic Resource Allocation + (how GPUs bind to pods) is generally + available. + minLength: 1 + maxLength: 32 + credentials: + type: object + description: >- + Vultr ProviderConfig or ClusterProviderConfig used to + authenticate to the Vultr API. Defaults to the + ClusterProviderConfig named default. + properties: + type: + type: string + default: ClusterProviderConfig + enum: + - ProviderConfig + - ClusterProviderConfig + name: + type: string + default: default + minLength: 1 + maxLength: 253 stack: type: string default: Standard diff --git a/apis/k3sclusters/composition.yaml b/apis/k3sclusters/composition.yaml new file mode 100644 index 000000000..506f32c18 --- /dev/null +++ b/apis/k3sclusters/composition.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: k3sclusters.infrastructure.modelplane.ai +spec: + compositeTypeRef: + apiVersion: infrastructure.modelplane.ai/v1alpha1 + kind: K3sCluster + mode: Pipeline + pipeline: + - functionRef: + name: modelplane-modelplanecompose-k3s-cluster + step: compose-k3s-cluster diff --git a/apis/k3sclusters/definition.yaml b/apis/k3sclusters/definition.yaml new file mode 100644 index 000000000..7d46e89ad --- /dev/null +++ b/apis/k3sclusters/definition.yaml @@ -0,0 +1,215 @@ +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: k3sclusters.infrastructure.modelplane.ai +spec: + group: infrastructure.modelplane.ai + names: + categories: + - crossplane + - modelplane + kind: K3sCluster + plural: k3sclusters + scope: Namespaced + versions: + - name: v1alpha1 + referenceable: true + additionalPrinterColumns: + - name: CONTROL-PLANE + type: string + jsonPath: .spec.controlPlane.host + schema: + openAPIV3Schema: + description: >- + A K3sCluster installs a k3s cluster onto existing machines over + SSH. The control plane machine runs the k3s server; each worker + joins as a k3s agent. It is provider-agnostic: any set of + reachable Linux hosts works, whether bare metal or virtual. It + outputs a Secret containing the cluster kubeconfig. The + kubeconfig embeds a static client certificate, so consumers need + nothing beyond it to reach the cluster. The control plane is a + single server; it is a single point of failure for the cluster + control plane. + properties: + spec: + description: K3sClusterSpec defines the desired state of K3sCluster. + required: + - controlPlane + - auth + properties: + controlPlane: + type: object + description: >- + The machine that runs the k3s server (the management + plane). Must be reachable over SSH from the control + plane running Modelplane. + required: + - host + properties: + host: + type: string + description: DNS name or IP address of the machine. + minLength: 1 + maxLength: 253 + port: + type: integer + default: 22 + description: SSH port. + minimum: 1 + maximum: 65535 + workers: + type: array + description: >- + Machines that join the cluster as k3s agents (the worker + plane). Labels and taints are applied at registration + time via k3s agent arguments. + maxItems: 64 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - host + properties: + name: + type: string + description: Unique name for this worker. + minLength: 1 + maxLength: 63 + host: + type: string + description: DNS name or IP address of the machine. + minLength: 1 + maxLength: 253 + port: + type: integer + default: 22 + description: SSH port. + minimum: 1 + maximum: 65535 + labels: + type: object + description: Node labels applied to this worker. + additionalProperties: + type: string + taints: + type: array + description: Node taints applied to this worker. + maxItems: 8 + items: + type: object + required: + - key + - effect + properties: + key: + type: string + minLength: 1 + maxLength: 253 + value: + type: string + maxLength: 63 + effect: + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + auth: + type: object + description: >- + SSH authentication used to reach every machine. All + machines must accept the same user and private key. + required: + - secretRef + properties: + username: + type: string + default: root + description: SSH user. Must be root or have passwordless sudo. + minLength: 1 + maxLength: 63 + secretRef: + type: object + description: >- + Secret holding the SSH private key, in the same + namespace as this K3sCluster. + required: + - name + properties: + name: + type: string + description: Name of the Secret. + minLength: 1 + maxLength: 253 + key: + type: string + default: ssh-privatekey + description: Key within the Secret that holds the private key. + minLength: 1 + maxLength: 253 + version: + type: object + description: >- + The k3s release to install. Defaults to the v1.34 + channel, the first where Dynamic Resource Allocation + (how GPUs bind to pods) is generally available. + x-kubernetes-validations: + - rule: "!(has(self.channel) && has(self.version))" + message: channel and version are mutually exclusive. + properties: + channel: + type: string + description: >- + k3s release channel (e.g. stable, v1.34). Installs + the channel's latest release. + minLength: 1 + maxLength: 32 + version: + type: string + description: >- + Exact k3s version to install (e.g. v1.34.1+k3s1). + minLength: 1 + maxLength: 32 + type: object + status: + description: K3sClusterStatus defines the observed state of K3sCluster. + properties: + secrets: + type: array + description: >- + Secrets produced by this cluster. Consumers use these to + authenticate to the cluster. All secrets are in the same + namespace as this K3sCluster. + items: + type: object + required: + - type + - name + - key + properties: + type: + type: string + description: >- + The type of credential this secret contains. + Kubeconfig contains a kubeconfig file with the + cluster endpoint, CA certificate, and a static + client certificate. + enum: + - Kubeconfig + name: + type: string + description: Name of the Secret. + maxLength: 253 + key: + type: string + description: >- + Key within the Secret that holds the credential data. + maxLength: 253 + type: object + required: + - spec + type: object + served: true diff --git a/apis/servingstacks/definition.yaml b/apis/servingstacks/definition.yaml index 61ebef272..0246dce7f 100644 --- a/apis/servingstacks/definition.yaml +++ b/apis/servingstacks/definition.yaml @@ -60,7 +60,24 @@ spec: - AKS - Nebius - Vultr + - VultrBaremetal - Existing + accelerators: + type: array + description: >- + Accelerator vendors present in the target cluster. + Filters the cloud's vendor-tagged components: only the + device stacks for the listed vendors are installed. + When omitted, no vendor filtering happens and every + component installs. Derived from the InferenceClasses + by the cluster composition; only clouds whose component + lists carry vendor tags (VultrBaremetal) are affected. + maxItems: 2 + items: + type: string + enum: + - AMD + - NVIDIA secrets: type: array description: >- diff --git a/apis/vultrbaremetalclusters/composition.yaml b/apis/vultrbaremetalclusters/composition.yaml new file mode 100644 index 000000000..a89bc2beb --- /dev/null +++ b/apis/vultrbaremetalclusters/composition.yaml @@ -0,0 +1,13 @@ +apiVersion: apiextensions.crossplane.io/v1 +kind: Composition +metadata: + name: vultrbaremetalclusters.infrastructure.modelplane.ai +spec: + compositeTypeRef: + apiVersion: infrastructure.modelplane.ai/v1alpha1 + kind: VultrBaremetalCluster + mode: Pipeline + pipeline: + - functionRef: + name: modelplane-modelplanecompose-vultr-baremetal-cluster + step: compose-vultr-baremetal-cluster diff --git a/apis/vultrbaremetalclusters/definition.yaml b/apis/vultrbaremetalclusters/definition.yaml new file mode 100644 index 000000000..1feacea81 --- /dev/null +++ b/apis/vultrbaremetalclusters/definition.yaml @@ -0,0 +1,242 @@ +apiVersion: apiextensions.crossplane.io/v2 +kind: CompositeResourceDefinition +metadata: + name: vultrbaremetalclusters.infrastructure.modelplane.ai +spec: + group: infrastructure.modelplane.ai + names: + categories: + - crossplane + - modelplane + - infrastructure + kind: VultrBaremetalCluster + plural: vultrbaremetalclusters + scope: Namespaced + versions: + - name: v1alpha1 + referenceable: true + additionalPrinterColumns: + - name: REGION + type: string + jsonPath: .spec.region + schema: + openAPIV3Schema: + description: >- + A VultrBaremetalCluster provisions Vultr bare metal servers and + installs a k3s cluster onto them. One CPU-only management server + runs the k3s server; each GPU pool's servers join as k3s agents. + Vultr bare metal has no autoscaling, so pools are fixed size. It + outputs a Secret containing the cluster kubeconfig. The + management server is a single point of failure for the cluster + control plane. Bare metal provisioning takes tens of minutes. + properties: + spec: + description: >- + VultrBaremetalClusterSpec defines the desired state of + VultrBaremetalCluster. + required: + - region + - ssh + - nodePools + properties: + region: + type: string + description: >- + Vultr region for all servers (e.g. ewr, ord). Bare metal + plan availability varies by region; check with + vultr-cli plans list --type vbm. + minLength: 1 + maxLength: 32 + management: + type: object + default: {} + description: >- + The CPU-only bare metal server that runs the k3s server + (the management plane). + properties: + plan: + type: string + default: vbm-6c-32gb-amd + description: >- + Vultr bare metal plan for the management server. The + default is a CPU-only plan available in most regions + that offer bare metal. + minLength: 1 + maxLength: 63 + osId: + type: integer + default: 2284 + description: >- + Vultr operating system ID installed on the server. + Defaults to Ubuntu 24.04 LTS x64; list IDs with + vultr-cli os list. + ssh: + type: object + description: >- + SSH key pair used to reach the servers. The public key + is registered with Vultr and installed on every server; + the private key drives the k3s install over SSH. + required: + - secretRef + properties: + secretRef: + type: object + description: >- + Secret holding the SSH key pair, in the same + namespace as this VultrBaremetalCluster. + required: + - name + properties: + name: + type: string + description: Name of the Secret. + minLength: 1 + maxLength: 253 + privateKeyKey: + type: string + default: ssh-privatekey + description: Key within the Secret that holds the private key. + minLength: 1 + maxLength: 253 + publicKeyKey: + type: string + default: ssh-publickey + description: Key within the Secret that holds the public key. + minLength: 1 + maxLength: 253 + username: + type: string + default: root + description: >- + SSH user the servers accept the key for. Vultr + installs keys for root by default. + minLength: 1 + maxLength: 63 + credentials: + type: object + description: >- + Vultr ProviderConfig or ClusterProviderConfig used to + authenticate to the Vultr API. Defaults to the + ClusterProviderConfig named default. + properties: + type: + type: string + default: ClusterProviderConfig + enum: + - ProviderConfig + - ClusterProviderConfig + name: + type: string + default: default + minLength: 1 + maxLength: 253 + k3s: + type: object + default: {} + description: The k3s release installed on the servers. + properties: + channel: + type: string + default: v1.34 + description: >- + k3s release channel. Defaults to the first channel + where Dynamic Resource Allocation (how GPUs bind to + pods) is generally available. + minLength: 1 + maxLength: 32 + nodePools: + type: array + description: >- + GPU bare metal pools that join the cluster as workers. + Fixed size; Vultr bare metal has no autoscaling. + minItems: 1 + maxItems: 8 + x-kubernetes-list-type: map + x-kubernetes-list-map-keys: + - name + items: + type: object + required: + - name + - plan + - gpu + properties: + name: + type: string + description: Unique name for this pool. + minLength: 1 + maxLength: 40 + plan: + type: string + description: >- + Vultr bare metal plan ID for the pool's servers + (e.g. vbm-256c-3072gb-8-mi355x-gpu). + minLength: 1 + maxLength: 63 + nodeCount: + type: integer + default: 1 + description: Number of servers in this pool. + minimum: 1 + maximum: 64 + osId: + type: integer + description: >- + Vultr operating system ID for the pool's servers. + Defaults to the management server's osId. + gpu: + type: object + description: GPU configuration. + required: + - acceleratorType + properties: + acceleratorType: + type: string + description: >- + GPU accelerator type (e.g. amd-mi355x). Used + to label the pool's nodes; the actual GPU and + count are determined by the plan. + minLength: 1 + maxLength: 63 + type: object + status: + description: >- + VultrBaremetalClusterStatus defines the observed state of + VultrBaremetalCluster. + properties: + secrets: + type: array + description: >- + Secrets produced by this cluster. Consumers use these to + authenticate to the cluster. All secrets are in the same + namespace as this VultrBaremetalCluster. + items: + type: object + required: + - type + - name + - key + properties: + type: + type: string + description: >- + The type of credential this secret contains. + Kubeconfig contains a kubeconfig file with the + cluster endpoint, CA certificate, and a static + client certificate. + enum: + - Kubeconfig + name: + type: string + description: Name of the Secret. + maxLength: 253 + key: + type: string + description: >- + Key within the Secret that holds the credential data. + maxLength: 253 + type: object + required: + - spec + type: object + served: true diff --git a/crossplane-project.yaml b/crossplane-project.yaml index fb8b8c3a7..afa3ca9e1 100644 --- a/crossplane-project.yaml +++ b/crossplane-project.yaml @@ -35,6 +35,10 @@ spec: tarball: name: compose-inference-cluster pathPrefix: _output/functions/compose-inference-cluster + - source: Tarball + tarball: + name: compose-k3s-cluster + pathPrefix: _output/functions/compose-k3s-cluster - source: Tarball tarball: name: compose-nebius-cluster @@ -75,6 +79,10 @@ spec: tarball: name: compose-vultr-cluster pathPrefix: _output/functions/compose-vultr-cluster + - source: Tarball + tarball: + name: compose-vultr-baremetal-cluster + pathPrefix: _output/functions/compose-vultr-baremetal-cluster dependencies: - type: crd git: @@ -159,3 +167,9 @@ spec: kind: Provider package: xpkg.upbound.io/upbound/provider-vultr version: v1.0.0 + - type: xpkg + xpkg: + apiVersion: pkg.crossplane.io/v1 + kind: Provider + package: xpkg.upbound.io/crossplane-contrib/provider-k3s + version: v0.4.0 diff --git a/docs/content/platform/providers.md b/docs/content/platform/providers.md index 6781af293..f487eaf5e 100644 --- a/docs/content/platform/providers.md +++ b/docs/content/platform/providers.md @@ -12,7 +12,7 @@ A provider can show up here in three ways: {{< hint "note" >}} - **Provisioning supported.** Modelplane creates and manages the whole cluster - from an `InferenceCluster`, selected through `provisioning.provider`. GKE, EKS, AKS, Nebius mk8s, Vultr VKE work this way today. + from an `InferenceCluster`, selected through `provisioning.provider`. GKE, EKS, AKS, Nebius mk8s, Vultr VKE, and Vultr Bare Metal (K3s) work this way today. - **Bring your own supported.** Register a cluster you already run with `source: Existing`. This works on any provider whose Kubernetes meets Modelplane's requirements (Dynamic Resource Allocation and a recent Kubernetes @@ -53,16 +53,19 @@ native provisioning. | Tencent Cloud (TKE) | {{< accel nvidia >}} | Planned | ✓ | {{< repolink "https://github.com/crossplane-contrib/provider-tencentcloud" "provider-tencentcloud" "community" >}} | | Voltage Park | {{< accel nvidia >}} | Planned | ✓ | none yet | | Vultr (VKE) | {{< accel nvidia >}} {{< accel amd >}} | ✓ | ✓ | {{< repolink "https://github.com/upbound/provider-vultr" "provider-vultr" "community" >}} | +| Vultr (Bare Metal, K3s) | {{< accel amd >}} {{< accel nvidia >}} | ✓ | ✓ | {{< repolink "https://github.com/upbound/provider-vultr" "provider-vultr" "community" >}} | {{< /table >}} {{< hint "note" >}} **On-premises and bare metal.** Bring an on-prem cluster the same way as any other: stand up Kubernetes on your own hardware (like NVIDIA DGX BasePOD or SuperPOD) with NVIDIA Base Command Manager, Run:ai, or your own tooling, then -register it with `source: Existing`. Provisioning it for you is on the roadmap -too. Modelplane can drive NVIDIA Base Command Manager or other bare-metal -Kubernetes provisioners through Crossplane, the same pattern it uses in the -cloud. +register it with `source: Existing`. On Vultr, Modelplane provisions bare metal +natively: `source: VultrBaremetal` creates the servers and installs a K3s +cluster onto them over SSH (a single K3s server, exposed through K3s's built-in +ServiceLB rather than a cloud load balancer). The K3s layer only needs machine +addresses and SSH credentials, so the same pattern extends to other bare-metal +providers through Crossplane. {{< /hint >}} diff --git a/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt b/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt index fc01f3c6d..84bd7bd05 100644 --- a/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt +++ b/docs/utils/vale/styles/config/vocabularies/Modelplane/accept.txt @@ -29,6 +29,11 @@ ModelReplica ModelReplicas ModelService ModelServices +K3sCluster +VultrBaremetalCluster +VultrBaremetal +VultrCluster +ServiceLB # Spec fields and config keys clusterSelector diff --git a/flake.nix b/flake.nix index 04806898c..e0b3bea4f 100644 --- a/flake.nix +++ b/flake.nix @@ -59,8 +59,10 @@ "compose-inference-class" "compose-inference-cluster" "compose-inference-gateway" + "compose-k3s-cluster" "compose-nebius-cluster" "compose-serving-stack" + "compose-vultr-baremetal-cluster" "compose-vultr-cluster" "compose-model-cache" "compose-model-deployment" diff --git a/functions/compose-inference-class/function/fn.py b/functions/compose-inference-class/function/fn.py index 829234cd7..3c099d4ea 100644 --- a/functions/compose-inference-class/function/fn.py +++ b/functions/compose-inference-class/function/fn.py @@ -16,15 +16,50 @@ InferenceClass is a data resource: it describes hardware (devices) and optionally how to provision it (provisioning). It has no composed -children. This function just marks the XR Ready. +children. This function marks the XR Ready, rejecting a class whose +devices name an accelerator vendor its provisioning provider's serving +stack cannot install - the earliest point the contradiction is visible, +before any cluster references the class. Classes without a provisioning +block are BYO: the cluster's operator manages the accelerator stack, so +any vendor is accepted. """ +from typing import Final + import grpc from crossplane.function import logging, resource, response from crossplane.function.proto.v1 import run_function_pb2 as fnv1 from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1 from models.ai.modelplane.inferenceclass import v1alpha1 +# The accelerator vendors each provider's serving stack can install. +# Kept in sync with ACCELERATOR_VENDORS in compose-serving-stack's +# stacks package, which is the authoritative table next to the +# component lists themselves. compose-inference-cluster carries the +# same table for the cluster-side pairing check. +_PROVIDER_ACCELERATOR_VENDORS: Final[dict[str, frozenset[str]]] = { + "GKE": frozenset({"NVIDIA"}), + "EKS": frozenset({"NVIDIA"}), + "AKS": frozenset({"NVIDIA"}), + "Nebius": frozenset({"NVIDIA"}), + "Vultr": frozenset({"NVIDIA"}), + "VultrBaremetal": frozenset({"AMD", "NVIDIA"}), +} + +CONDITION_REASON_UNSUPPORTED_DEVICES = "UnsupportedDevices" + + +def _accelerator_vendors(xr: v1alpha1.InferenceClass) -> set[str]: + """The accelerator vendors the class's devices name, from each + device's DRA driver (gpu.amd.com, gpu.nvidia.com).""" + vendors: set[str] = set() + for device in xr.spec.devices or []: + if "amd" in device.driver: + vendors.add("AMD") + elif "nvidia" in device.driver: + vendors.add("NVIDIA") + return vendors + class FunctionRunner(grpcv1.FunctionRunnerServiceServicer): """A FunctionRunner handles gRPC RunFunctionRequests.""" @@ -42,7 +77,30 @@ async def RunFunction( rsp = response.to(req) + xr = v1alpha1.InferenceClass(**resource.struct_to_dict(req.observed.composite.resource)) + resource.update_status(rsp.desired.composite, v1alpha1.Status()) + + provider = xr.spec.provisioning.provider if xr.spec.provisioning else None + supported = _PROVIDER_ACCELERATOR_VENDORS.get(provider) if provider else None + unsupported = sorted(_accelerator_vendors(xr) - supported) if supported is not None else [] + if unsupported: + msg = ( + f"{', '.join(unsupported)} devices are not supported on {provider}: " + f"its serving stack installs only {', '.join(sorted(supported or []))} accelerator stacks" + ) + response.set_conditions( + rsp, + resource.Condition( + typ="Accepted", + status="False", + reason=CONDITION_REASON_UNSUPPORTED_DEVICES, + message=msg, + ), + ) + response.warning(rsp, msg) + return rsp + response.set_conditions(rsp, resource.Condition(typ="Accepted", status="True", reason="Available")) rsp.desired.composite.ready = fnv1.READY_TRUE diff --git a/functions/compose-inference-class/tests/test_fn.py b/functions/compose-inference-class/tests/test_fn.py index d4c5c99c4..a2936e8ba 100644 --- a/functions/compose-inference-class/tests/test_fn.py +++ b/functions/compose-inference-class/tests/test_fn.py @@ -95,6 +95,124 @@ async def test_compose(self) -> None: ), ] + cases.append( + Case( + name="accepts an AMD class provisioned on VultrBaremetal", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct( + v1alpha1.InferenceClass( + metadata=metav1.ObjectMeta(name="gpu-mi355x"), + spec=v1alpha1.Spec( + provisioning=v1alpha1.Provisioning( + provider="VultrBaremetal", + vultrBaremetal=v1alpha1.VultrBaremetal( + plan="vbm-256c-3072gb-8-mi355x-gpu", + accelerator=v1alpha1.AcceleratorModel4(type="amd-mi355x", count=8), + ), + ), + devices=[ + v1alpha1.Device( + name="gpu", + claim="DRA", + driver="gpu.amd.com", + deviceClassName="gpu.amd.com", + count=8, + capacity={"memory": v1alpha1.Capacity(value="288Gi")}, + ), + ], + ), + ).model_dump(exclude_none=True, mode="json") + ), + ), + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct({"status": {}}), + ready=fnv1.READY_TRUE, + ), + ), + conditions=[ + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_TRUE, + reason="Available", + ), + ], + context=structpb.Struct(), + ), + ), + ) + + cases.append( + Case( + name="rejects an AMD class provisioned on an NVIDIA-only provider", + req=fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct( + v1alpha1.InferenceClass( + metadata=metav1.ObjectMeta(name="gpu-mi355x-vke"), + spec=v1alpha1.Spec( + provisioning=v1alpha1.Provisioning( + provider="Vultr", + vultr=v1alpha1.Vultr( + plan="vcg-mi355x-hypothetical", + accelerator=v1alpha1.AcceleratorModel3(type="amd-mi355x", count=1), + ), + ), + devices=[ + v1alpha1.Device( + name="gpu", + claim="DRA", + driver="gpu.amd.com", + deviceClassName="gpu.amd.com", + count=1, + capacity={"memory": v1alpha1.Capacity(value="288Gi")}, + ), + ], + ), + ).model_dump(exclude_none=True, mode="json") + ), + ), + ), + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct({"status": {}}), + ), + ), + conditions=[ + fnv1.Condition( + type="Accepted", + status=fnv1.STATUS_CONDITION_FALSE, + reason="UnsupportedDevices", + message=( + "AMD devices are not supported on Vultr: " + "its serving stack installs only NVIDIA accelerator stacks" + ), + ), + ], + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_WARNING, + message=( + "AMD devices are not supported on Vultr: " + "its serving stack installs only NVIDIA accelerator stacks" + ), + ), + ], + context=structpb.Struct(), + ), + ), + ) + for case in cases: with self.subTest(case.name): got = await self.runner.RunFunction(case.req, None) diff --git a/functions/compose-inference-cluster/function/fn.py b/functions/compose-inference-cluster/function/fn.py index b802716c4..ea9ee918c 100644 --- a/functions/compose-inference-cluster/function/fn.py +++ b/functions/compose-inference-cluster/function/fn.py @@ -43,6 +43,7 @@ from models.ai.modelplane.infrastructure.gkecluster import v1alpha1 as gkev1alpha1 from models.ai.modelplane.infrastructure.nebiuscluster import v1alpha1 as nebiusv1alpha1 from models.ai.modelplane.infrastructure.servingstack import v1alpha1 as ssv1alpha1 +from models.ai.modelplane.infrastructure.vultrbaremetalcluster import v1alpha1 as vbmv1alpha1 from models.ai.modelplane.infrastructure.vultrcluster import v1alpha1 as vultrv1alpha1 from models.io.crossplane.m.kubernetes.clusterproviderconfig import ( v1alpha1 as k8scpcv1alpha1, @@ -54,14 +55,30 @@ # Cluster source discriminator values from the XRD enum. The Literal # mirrors ServingStack spec.cloud, so passing a wrong or unsupported # cloud fails type checking; Final makes each constant a literal type. -Cloud = Literal["GKE", "EKS", "AKS", "Nebius", "Vultr", "Existing"] +Cloud = Literal["GKE", "EKS", "AKS", "Nebius", "Vultr", "VultrBaremetal", "Existing"] CLUSTER_SOURCE_GKE: Final = "GKE" CLUSTER_SOURCE_EKS: Final = "EKS" CLUSTER_SOURCE_AKS: Final = "AKS" CLUSTER_SOURCE_NEBIUS: Final = "Nebius" CLUSTER_SOURCE_VULTR: Final = "Vultr" +CLUSTER_SOURCE_VULTR_BAREMETAL: Final = "VultrBaremetal" CLUSTER_SOURCE_EXISTING: Final = "Existing" +# The accelerator vendors each cloud's serving stack can install. Kept +# in sync with ACCELERATOR_VENDORS in compose-serving-stack's stacks +# package, which is the authoritative table next to the component lists +# themselves. Existing is BYO: the cluster's operator manages the +# accelerator stack, so any vendor goes. +_CLOUD_ACCELERATOR_VENDORS: Final[dict[str, frozenset[str]]] = { + CLUSTER_SOURCE_GKE: frozenset({"NVIDIA"}), + CLUSTER_SOURCE_EKS: frozenset({"NVIDIA"}), + CLUSTER_SOURCE_AKS: frozenset({"NVIDIA"}), + CLUSTER_SOURCE_NEBIUS: frozenset({"NVIDIA"}), + CLUSTER_SOURCE_VULTR: frozenset({"NVIDIA"}), + CLUSTER_SOURCE_VULTR_BAREMETAL: frozenset({"AMD", "NVIDIA"}), + CLUSTER_SOURCE_EXISTING: frozenset({"AMD", "NVIDIA"}), +} + # Condition types and reasons for the InferenceCluster XR. CONDITION_TYPE_CLUSTER_READY = "ClusterReady" CONDITION_TYPE_BACKEND_READY = "BackendReady" @@ -73,6 +90,7 @@ CONDITION_REASON_BACKEND_HEALTHY = "BackendHealthy" CONDITION_REASON_INSTALLING = "Installing" CONDITION_REASON_INVALID_NODE_POOL = "InvalidNodePool" +CONDITION_REASON_UNSUPPORTED_DEVICES = "UnsupportedDevices" # Composed resource key for the backend XR. BACKEND_RESOURCE_KEY = "serving-stack" @@ -156,6 +174,31 @@ def compose(self) -> None: return source = cluster.source + + # A cloud's serving stack can only drive the accelerator vendors + # it has components for - an AMD-device class on an NVIDIA-only + # cloud would provision GPUs nothing can drive. Checked here + # rather than at admission: the class alone doesn't know which + # cluster will reference it, only the pairing does. + supported = _CLOUD_ACCELERATOR_VENDORS.get(source) + unsupported = sorted(set(self.accelerator_vendors()) - supported) if supported is not None else [] + if unsupported: + msg = ( + f"{', '.join(unsupported)} devices are not supported on {source}: " + f"its serving stack installs only {', '.join(sorted(supported or []))} accelerator stacks" + ) + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_UNSUPPORTED_DEVICES, + message=msg, + ), + ) + response.warning(self.rsp, msg) + return + if source == CLUSTER_SOURCE_GKE: self.compose_gke(cluster.gke) elif source == CLUSTER_SOURCE_EKS: @@ -166,6 +209,8 @@ def compose(self) -> None: self.compose_nebius(cluster.nebius) elif source == CLUSTER_SOURCE_VULTR: self.compose_vultr(cluster.vultr) + elif source == CLUSTER_SOURCE_VULTR_BAREMETAL: + self.compose_vultr_baremetal(cluster.vultrBaremetal) elif source == CLUSTER_SOURCE_EXISTING: self.compose_existing(cluster.existing) else: @@ -453,6 +498,65 @@ def compose_vultr(self, vultr: v1alpha1.Vultr | None) -> None: self.write_status(self.gpu_pools()) self.derive_conditions(cluster_ready=vultr_ready) + def compose_vultr_baremetal(self, vultr_baremetal: v1alpha1.VultrBaremetal | None) -> None: + """Compose an InferenceCluster backed by Modelplane-provisioned + Vultr bare metal servers running k3s. Composes the + VultrBaremetalCluster XR, waits for it to be ready, then wires + its kubeconfig into the backend. + + The k3s kubeconfig embeds a static client certificate, so the + kubeconfig alone is enough to reach the cluster and no identity + is layered on the ClusterProviderConfig. The backend gets the + GPU vendors the classes name, so the serving stack installs only + the matching GPU operators. + """ + if not vultr_baremetal: + response.warning(self.rsp, "VultrBaremetal configuration is required when source is VultrBaremetal") + return + + self.compose_vultr_baremetal_cluster(vultr_baremetal) + + ready = ( + resource.get_condition(self.req.observed.resources.get("vultr-baremetal-cluster"), "Ready").status == "True" + ) + kubeconfig = self.observed_vultr_baremetal_secret(_SECRET_TYPE_KUBECONFIG) + backend_exists = BACKEND_RESOURCE_KEY in self.req.observed.resources + + if ready and kubeconfig: + self.compose_cluster_provider_config(kubeconfig.name, kubeconfig.key) + + backend_secrets = self.resolve_vultr_baremetal_backend_secrets(ready=ready, backend_exists=backend_exists) + if backend_secrets or backend_exists: + if backend_secrets: + self.compose_serving_stack( + backend_secrets, CLUSTER_SOURCE_VULTR_BAREMETAL, accelerators=self.accelerator_vendors() + ) + self.compose_vultr_baremetal_usage() + + if ready: + self.rsp.desired.resources["vultr-baremetal-cluster"].ready = fnv1.READY_TRUE + if not backend_exists: + response.normal(self.rsp, "Vultr bare metal cluster ready, composing backend") + + self.write_status(self.gpu_pools()) + self.derive_conditions(cluster_ready=ready) + + def accelerator_vendors(self) -> list[Literal["AMD", "NVIDIA"]]: + """The accelerator vendors the resolved classes' devices name, + from each device's DRA driver (gpu.amd.com, gpu.nvidia.com). + Sorted so the composed ServingStack spec is deterministic.""" + vendors: set[Literal["AMD", "NVIDIA"]] = set() + for pool in self.xr.spec.nodePools or []: + cls = self.classes.get(pool.className) + if not cls: + continue + for device in cls.spec.devices or []: + if "amd" in device.driver: + vendors.add("AMD") + elif "nvidia" in device.driver: + vendors.add("NVIDIA") + return sorted(vendors) + def compose_existing(self, existing: v1alpha1.Existing | None) -> None: """Compose an InferenceCluster backed by a user-supplied cluster. No gating needed — the kubeconfig secret is provided by the user.""" @@ -486,19 +590,24 @@ def compose_serving_stack( self, backend_secrets: list[ssv1alpha1.Secret], cloud: Cloud, + accelerators: list[Literal["AMD", "NVIDIA"]] | None = None, ) -> None: """Compose a ServingStack XR with the given secrets. cloud names the cluster's source (this XR's spec.cluster.source) and selects the component list the serving stack installs, including cloud specifics like where the node image puts the - NVIDIA driver. + NVIDIA driver. accelerators, when set, filters the cloud's + vendor-tagged components to the vendors the classes actually + name. """ spec = ssv1alpha1.Spec( secrets=backend_secrets, stack=self.xr.spec.stack, cloud=cloud, ) + if accelerators: + spec.accelerators = accelerators resource.update( self.rsp.desired.resources[BACKEND_RESOURCE_KEY], ssv1alpha1.ServingStack( @@ -1098,6 +1207,137 @@ def observed_vultr_secret(self, secret_type: str) -> vultrv1alpha1.Secret | None return None return next((s for s in vultr_secrets if s.type == secret_type), None) + def compose_vultr_baremetal_cluster(self, vultr_baremetal: v1alpha1.VultrBaremetal) -> None: + """Compose a VultrBaremetalCluster XR. + + Combines the cluster-level config (region, management server, + SSH key pair, k3s release) with GPU pools derived from the + user's node pools + referenced classes. + """ + pools: list[vbmv1alpha1.NodePool] = [] + + for pool in self.xr.spec.nodePools or []: + cls = self.classes.get(pool.className) + if not cls or not cls.spec.provisioning or not cls.spec.provisioning.vultrBaremetal: + msg = f"InferenceClass {pool.className} has no VultrBaremetal provisioning block" + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_INVALID_NODE_POOL, + message=msg, + ), + ) + response.warning(self.rsp, msg) + return + prov = cls.spec.provisioning.vultrBaremetal + pools.append( + vbmv1alpha1.NodePool( + name=pool.name, + plan=prov.plan, + nodeCount=pool.nodeCount, + gpu=vbmv1alpha1.Gpu(acceleratorType=prov.accelerator.type), + ), + ) + + ssh = vultr_baremetal.ssh + spec = vbmv1alpha1.Spec( + region=vultr_baremetal.region, + ssh=vbmv1alpha1.Ssh( + secretRef=vbmv1alpha1.SecretRef( + name=ssh.secretRef.name, + privateKeyKey=ssh.secretRef.privateKeyKey, + publicKeyKey=ssh.secretRef.publicKeyKey, + ), + username=ssh.username, + ), + nodePools=pools, + ) + if vultr_baremetal.management: + spec.management = vbmv1alpha1.Management( + plan=vultr_baremetal.management.plan, + osId=vultr_baremetal.management.osId, + ) + if vultr_baremetal.k3s: + spec.k3s = vbmv1alpha1.K3s(channel=vultr_baremetal.k3s.channel) + if vultr_baremetal.credentials: + spec.credentials = vbmv1alpha1.Credentials( + type=vultr_baremetal.credentials.type, + name=vultr_baremetal.credentials.name, + ) + resource.update( + self.rsp.desired.resources["vultr-baremetal-cluster"], + vbmv1alpha1.VultrBaremetalCluster( + metadata=metav1.ObjectMeta( + name=_name(self.xr.metadata), + namespace=_NAMESPACE_SYSTEM, + ), + spec=spec, + ), + ) + + def compose_vultr_baremetal_usage(self) -> None: + """Block VultrBaremetalCluster deletion until the backend is deleted.""" + resource.update( + self.rsp.desired.resources["usage-vultr-baremetal-by-backend"], + usagev1beta1.Usage( + metadata=metav1.ObjectMeta(namespace=_NAMESPACE_SYSTEM), + spec=usagev1beta1.Spec( + of=usagev1beta1.Of( + apiVersion="infrastructure.modelplane.ai/v1alpha1", + kind="VultrBaremetalCluster", + resourceSelector=usagev1beta1.ResourceSelectorModel(matchControllerRef=True), + ), + by=usagev1beta1.By( + apiVersion="infrastructure.modelplane.ai/v1alpha1", + kind="ServingStack", + resourceSelector=usagev1beta1.ResourceSelector(matchControllerRef=True), + ), + replayDeletion=True, + ), + ), + ) + self.rsp.desired.resources["usage-vultr-baremetal-by-backend"].ready = fnv1.READY_TRUE + + def resolve_vultr_baremetal_backend_secrets( + self, *, ready: bool, backend_exists: bool + ) -> list[ssv1alpha1.Secret] | None: + """Resolve secrets for the backend from VultrBaremetalCluster + status. Falls back to the observed backend's spec.secrets if the + cluster's secrets aren't available but the backend already exists.""" + secrets = self.observed_vultr_baremetal_secrets() + + if ready and secrets: + return [ssv1alpha1.Secret(type=s.type, name=s.name, key=s.key) for s in secrets] + + if backend_exists: + observed = self.req.observed.resources.get(BACKEND_RESOURCE_KEY) + if observed: + d = resource.struct_to_dict(observed.resource) + observed_secrets = d.get("spec", {}).get("secrets", []) + if observed_secrets: + return [ssv1alpha1.Secret(type=s["type"], name=s["name"], key=s["key"]) for s in observed_secrets] + + return None + + def observed_vultr_baremetal_secrets(self) -> list[vbmv1alpha1.Secret] | None: + """Read the VultrBaremetalCluster's status.secrets from observed state.""" + observed = self.req.observed.resources.get("vultr-baremetal-cluster") + if not observed: + return None + cluster = vbmv1alpha1.VultrBaremetalCluster.model_validate(resource.struct_to_dict(observed.resource)) + if not cluster.status: + return None + return cluster.status.secrets + + def observed_vultr_baremetal_secret(self, secret_type: str) -> vbmv1alpha1.Secret | None: + """Read a specific secret from the observed VultrBaremetalCluster status.""" + secrets = self.observed_vultr_baremetal_secrets() + if not secrets: + return None + return next((s for s in secrets if s.type == secret_type), None) + def compose_eks_usage(self) -> None: """Block EKSCluster deletion until the backend is deleted.""" resource.update( diff --git a/functions/compose-inference-cluster/tests/test_fn.py b/functions/compose-inference-cluster/tests/test_fn.py index f0f260daf..38b8dd775 100644 --- a/functions/compose-inference-cluster/tests/test_fn.py +++ b/functions/compose-inference-cluster/tests/test_fn.py @@ -2524,6 +2524,359 @@ async def test_compose(self) -> None: # noqa: PLR0915 ) ) + # --- Case 16: VultrBaremetal first pass composes the + # VultrBaremetalCluster XR only. The AMD class flows into the + # pool and, later, into the ServingStack's accelerators. --- + inference_class_mi355x = { + "apiVersion": "modelplane.ai/v1alpha1", + "kind": "InferenceClass", + "metadata": {"name": "gpu-mi355x-vbm"}, + "spec": { + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.amd.com", + "deviceClassName": "gpu.amd.com", + "count": 8, + "capacity": {"memory": {"value": "288Gi"}}, + }, + ], + "provisioning": { + "provider": "VultrBaremetal", + "vultrBaremetal": { + "plan": "vbm-256c-3072gb-8-mi355x-gpu", + "accelerator": {"type": "amd-mi355x", "count": 8}, + }, + }, + }, + } + class_selector_mi355x = fnv1.ResourceSelector( + api_version="modelplane.ai/v1alpha1", + kind="InferenceClass", + match_name="gpu-mi355x-vbm", + ) + + req16 = fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct( + v1alpha1.InferenceCluster( + metadata=metav1.ObjectMeta( + name="test-cluster", + namespace="modelplane-system", + ), + spec=v1alpha1.Spec( + cluster=v1alpha1.Cluster( + source="VultrBaremetal", + vultrBaremetal=v1alpha1.VultrBaremetal( + region="ord", + ssh=v1alpha1.Ssh(secretRef=v1alpha1.SecretRefModel(name="bm-ssh")), + ), + ), + nodePools=[ + v1alpha1.NodePool( + name="mi355x-pool", + className="gpu-mi355x-vbm", + nodeCount=1, + ), + ], + ), + ).model_dump(exclude_none=True, mode="json"), + ), + ), + ), + ) + req16.required_resources["class-gpu-mi355x-vbm"].items.append( + fnv1.Resource(resource=resource.dict_to_struct(inference_class_mi355x)), + ) + + baremetal_status = { + "status": { + "providerConfigRef": { + "name": "test-cluster-cluster-kubeconfig-d0f89", + }, + "namespace": "modelplane-system", + "gpuPools": [ + { + "name": "mi355x-pool", + "nodes": 1, + "devices": [ + { + "name": "gpu", + "claim": "DRA", + "driver": "gpu.amd.com", + "deviceClassName": "gpu.amd.com", + "count": 8, + "capacity": {"memory": {"value": "288Gi"}}, + }, + ], + }, + ], + }, + } + + want16 = fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=fnv1.Resource(resource=resource.dict_to_struct(baremetal_status)), + resources={ + "vultr-baremetal-cluster": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "VultrBaremetalCluster", + "metadata": { + "name": "test-cluster", + "namespace": "modelplane-system", + }, + "spec": { + "region": "ord", + "ssh": { + "secretRef": { + "name": "bm-ssh", + "privateKeyKey": "ssh-privatekey", + "publicKeyKey": "ssh-publickey", + }, + "username": "root", + }, + "management": { + "plan": "vbm-6c-32gb-amd", + "osId": 2284, + }, + "k3s": {"channel": "v1.34"}, + "nodePools": [ + { + "name": "mi355x-pool", + "plan": "vbm-256c-3072gb-8-mi355x-gpu", + "nodeCount": 1, + "gpu": {"acceleratorType": "amd-mi355x"}, + }, + ], + }, + }, + ), + ), + }, + ), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Provisioning", + ), + fnv1.Condition( + type="BackendReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForCluster", + ), + ], + context=structpb.Struct(), + ) + want16.requirements.resources["class-gpu-mi355x-vbm"].CopyFrom(class_selector_mi355x) + + # --- Case 17: VultrBaremetal cluster ready - kubeconfig observed + # on the VultrBaremetalCluster status. The k3s kubeconfig embeds a + # static client certificate, so the ClusterProviderConfig carries + # no identity. The ServingStack gets cloud VultrBaremetal and the + # GPU vendors derived from the classes, so only the AMD GPU stack + # installs. --- + req17 = fnv1.RunFunctionRequest() + req17.CopyFrom(req16) + req17.observed.resources["vultr-baremetal-cluster"].CopyFrom( + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "VultrBaremetalCluster", + "metadata": {"name": "test-cluster", "namespace": "modelplane-system"}, + "spec": { + "region": "ord", + "ssh": {"secretRef": {"name": "bm-ssh"}}, + "nodePools": [ + { + "name": "mi355x-pool", + "plan": "vbm-256c-3072gb-8-mi355x-gpu", + "gpu": {"acceleratorType": "amd-mi355x"}, + }, + ], + }, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-bm-kubeconfig-abcde", + "key": "kubeconfig", + }, + ], + }, + } + ), + ), + ) + + want17 = fnv1.RunFunctionResponse() + want17.CopyFrom(want16) + want17.desired.resources["vultr-baremetal-cluster"].ready = fnv1.READY_TRUE + want17.desired.resources["cluster-provider-config-kubernetes"].CopyFrom( + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "kubernetes.m.crossplane.io/v1alpha1", + "kind": "ClusterProviderConfig", + "metadata": {"name": "test-cluster-cluster-kubeconfig-d0f89"}, + "spec": { + "credentials": { + "source": "Secret", + "secretRef": { + "namespace": "modelplane-system", + "name": "test-cluster-bm-kubeconfig-abcde", + "key": "kubeconfig", + }, + }, + }, + } + ), + ready=fnv1.READY_TRUE, + ), + ) + want17.desired.resources["serving-stack"].CopyFrom( + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "ServingStack", + "metadata": { + "name": "test-cluster-serving-stack-fd00b", + "namespace": "modelplane-system", + }, + "spec": { + "cloud": "VultrBaremetal", + "stack": "Standard", + "accelerators": ["AMD"], + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-bm-kubeconfig-abcde", + "key": "kubeconfig", + }, + ], + }, + } + ), + ), + ) + want17.desired.resources["usage-vultr-baremetal-by-backend"].CopyFrom( + fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "protection.crossplane.io/v1beta1", + "kind": "Usage", + "metadata": {"namespace": "modelplane-system"}, + "spec": { + "of": { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "VultrBaremetalCluster", + "resourceSelector": {"matchControllerRef": True}, + }, + "by": { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "ServingStack", + "resourceSelector": {"matchControllerRef": True}, + }, + "replayDeletion": True, + }, + } + ), + ready=fnv1.READY_TRUE, + ), + ) + del want17.conditions[:] + want17.conditions.extend( + [ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_TRUE, + reason="ClusterRunning", + ), + fnv1.Condition( + type="BackendReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="Installing", + ), + ] + ) + want17.results.append( + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Vultr bare metal cluster ready, composing backend", + ) + ) + + # --- Case 18: an AMD class referenced from an NVIDIA-only cloud + # gates with UnsupportedDevices. The class itself is valid (its + # provisioning could target VultrBaremetal too); only the pairing + # with a cloud whose stack is NVIDIA-only is rejected. --- + req18 = fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=fnv1.Resource( + resource=resource.dict_to_struct( + v1alpha1.InferenceCluster( + metadata=metav1.ObjectMeta( + name="test-cluster", + namespace="modelplane-system", + ), + spec=v1alpha1.Spec( + cluster=v1alpha1.Cluster( + source="Vultr", + vultr=v1alpha1.Vultr(region="ewr"), + ), + nodePools=[ + v1alpha1.NodePool( + name="mi355x-pool", + className="gpu-mi355x-vbm", + nodeCount=1, + ), + ], + ), + ).model_dump(exclude_none=True, mode="json"), + ), + ), + ), + ) + req18.required_resources["class-gpu-mi355x-vbm"].items.append( + fnv1.Resource(resource=resource.dict_to_struct(inference_class_mi355x)), + ) + + want18 = fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="UnsupportedDevices", + message="AMD devices are not supported on Vultr: its serving stack installs only NVIDIA accelerator stacks", + ), + ], + results=[ + fnv1.Result( + severity=fnv1.SEVERITY_WARNING, + message="AMD devices are not supported on Vultr: its serving stack installs only NVIDIA accelerator stacks", + ), + ], + context=structpb.Struct(), + ) + want18.requirements.resources["class-gpu-mi355x-vbm"].CopyFrom(class_selector_mi355x) + # Every compose path emits the ModelReplica guard requirement. for want in ( want1, @@ -2542,6 +2895,9 @@ async def test_compose(self) -> None: # noqa: PLR0915 want14, want_creds_vultr, want15, + want16, + want17, + want18, ): want.requirements.resources["model-replicas"].CopyFrom(_replicas_selector("test-cluster")) @@ -2724,6 +3080,21 @@ async def test_compose(self) -> None: # noqa: PLR0915 req=req15, want=want15, ), + Case( + name="VultrBaremetal first pass composes VultrBaremetalCluster XR only", + req=req16, + want=want16, + ), + Case( + name="VultrBaremetal ready composes CPC, ServingStack with AMD accelerators, and Usage", + req=req17, + want=want17, + ), + Case( + name="AMD class on an NVIDIA-only cloud gates with UnsupportedDevices", + req=req18, + want=want18, + ), *guard_cases, ] diff --git a/functions/compose-k3s-cluster/function/__init__.py b/functions/compose-k3s-cluster/function/__init__.py new file mode 100644 index 000000000..b53d39d12 --- /dev/null +++ b/functions/compose-k3s-cluster/function/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/functions/compose-k3s-cluster/function/fn.py b/functions/compose-k3s-cluster/function/fn.py new file mode 100644 index 000000000..9f9913c66 --- /dev/null +++ b/functions/compose-k3s-cluster/function/fn.py @@ -0,0 +1,273 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compose a k3s cluster onto existing machines over SSH. + +This function installs k3s on machines the caller already has: the control +plane machine runs the k3s server, and each worker joins as a k3s agent. +provider-k3s drives the installs over SSH using a ProviderConfig that +carries the SSH user and private key. + +The Cluster managed resource is composed first and publishes the cluster +kubeconfig as its connection secret. Node resources gate on the Cluster +being Ready: an agent can only join once the server is up and its join +token exists. Worker labels and taints are passed as k3s agent arguments, +so they are applied at node registration time. + +Unlike the managed-Kubernetes cluster functions, no GPU observer gates +readiness here. Nothing preinstalls a GPU stack on bare machines; the +serving stack installs one after the cluster is Ready, so cluster +readiness cannot wait for it. +""" + +import grpc +from crossplane.function import logging, resource, response +from crossplane.function.proto.v1 import run_function_pb2 as fnv1 +from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1 +from models.ai.modelplane.infrastructure.k3scluster import v1alpha1 +from models.io.crossplane.m.k3s.cluster import v1alpha1 as k3sclusterv1alpha1 +from models.io.crossplane.m.k3s.node import v1alpha1 as k3snodev1alpha1 +from models.io.crossplane.m.k3s.providerconfig import v1alpha1 as k3spcv1alpha1 +from models.io.k8s.apimachinery.pkg.apis.meta import v1 as metav1 + +# The k3s channel installed when the XR pins neither a channel nor a +# version: the first Kubernetes release where Dynamic Resource Allocation +# (how GPUs bind to pods) is generally available. +_DEFAULT_CHANNEL = "v1.34" + +# Secret type written to XR status. compose-inference-cluster reads this to +# wire the kubeconfig into a ClusterProviderConfig. +_SECRET_TYPE_KUBECONFIG = "Kubeconfig" + +# Key within the connection secret the Cluster resource writes. provider-k3s +# publishes the kubeconfig under this key once the server is installed. +_SECRET_KEY_KUBECONFIG = "kubeconfig" + + +def _name(meta: metav1.ObjectMeta | None) -> str: + """The object's name, always set on resources read from the API server.""" + if meta is None or meta.name is None: + raise ValueError("metadata.name is unexpectedly absent") + return meta.name + + +def _namespace(meta: metav1.ObjectMeta | None) -> str: + """The object's namespace, always set on resources read from the API server.""" + if meta is None or meta.namespace is None: + raise ValueError("metadata.namespace is unexpectedly absent") + return meta.namespace + + +def _kubeconfig_secret_name(xr: v1alpha1.K3sCluster) -> str: + """Derive the kubeconfig secret name from the XR.""" + return resource.child_name(_name(xr.metadata), "kubeconfig") + + +def _provider_config_name(xr: v1alpha1.K3sCluster) -> str: + """Derive the k3s ProviderConfig name from the XR.""" + return resource.child_name(_name(xr.metadata), "ssh") + + +def _cluster_name(xr: v1alpha1.K3sCluster) -> str: + """Derive the Cluster managed resource name from the XR. Set explicitly + so Node resources can reference it by name.""" + return resource.child_name(_name(xr.metadata), "cluster") + + +def _extra_args(worker: v1alpha1.Worker) -> str | None: + """k3s agent arguments applying the worker's labels and taints at node + registration time. Labels are sorted by key: the XR round-trips through + protobuf structs, which don't preserve map order.""" + args = [] + for key, value in sorted((worker.labels or {}).items()): + args.append(f"--node-label {key}={value}") + for taint in worker.taints or []: + spec = f"{taint.key}={taint.value}" if taint.value else taint.key + args.append(f"--node-taint {spec}:{taint.effect}") + return " ".join(args) if args else None + + +class FunctionRunner(grpcv1.FunctionRunnerServiceServicer): + """A FunctionRunner handles gRPC RunFunctionRequests.""" + + def __init__(self) -> None: + """Create a new FunctionRunner.""" + self.log = logging.get_logger() + + async def RunFunction( + self, req: fnv1.RunFunctionRequest, _: grpc.aio.ServicerContext | None + ) -> fnv1.RunFunctionResponse: # ty: ignore[invalid-method-override] # the generated grpc servicer base is untyped + """Run the function.""" + log = self.log.bind(tag=req.meta.tag) + log.info("Running function") + + rsp = response.to(req) + c = Composer(req, rsp) + c.compose() + return rsp + + +class Composer: + def __init__(self, req: fnv1.RunFunctionRequest, rsp: fnv1.RunFunctionResponse) -> None: + self.req = req + self.rsp = rsp + self.xr = v1alpha1.K3sCluster(**resource.struct_to_dict(req.observed.composite.resource)) + + def compose(self) -> None: + self.compose_provider_config() + self.compose_cluster() + if self._cluster_ready() or self._dependents_observed(): + self.compose_nodes() + self.write_status() + self.mark_readiness() + + def _cluster_ready(self) -> bool: + return resource.get_condition(self.req.observed.resources.get("cluster"), "Ready").status == "True" + + def _dependents_observed(self) -> bool: + """Whether the Ready-gated Nodes were composed on a previous + reconcile. The gate delays their first composition until the cluster + is Ready, but must not drop them from desired state when the Ready + condition transiently regresses - that would delete them, draining + the joined agents from the cluster.""" + return any(name.startswith("node-") for name in self.req.observed.resources) + + def _channel(self) -> str | None: + """The k3s channel to install, or None when an exact version is + pinned instead.""" + version = self.xr.spec.version + if version and version.version: + return None + if version and version.channel: + return version.channel + return _DEFAULT_CHANNEL + + def _version(self) -> str | None: + version = self.xr.spec.version + return version.version if version else None + + def _username(self) -> str: + return self.xr.spec.auth.username or "root" + + def compose_provider_config(self) -> None: + """Compose a k3s ProviderConfig carrying the SSH user and private + key provider-k3s uses to reach every machine.""" + resource.update( + self.rsp.desired.resources["provider-config-k3s"], + k3spcv1alpha1.ProviderConfig( + metadata=metav1.ObjectMeta( + name=_provider_config_name(self.xr), + namespace=_namespace(self.xr.metadata), + ), + spec=k3spcv1alpha1.Spec( + username=self._username(), + credentials=k3spcv1alpha1.Credentials( + source="Secret", + secretRef=k3spcv1alpha1.SecretRef( + namespace=_namespace(self.xr.metadata), + name=self.xr.spec.auth.secretRef.name, + key=self.xr.spec.auth.secretRef.key or "ssh-privatekey", + ), + ), + ), + ), + ) + + def compose_cluster(self) -> None: + """Compose the Cluster that installs the k3s server on the control + plane machine. Traefik is disabled - Envoy Gateway is the ingress - + while the default ServiceLB stays on: it is what gives LoadBalancer + Services an external IP on machines without a cloud load balancer.""" + cp = self.xr.spec.controlPlane + fp = k3sclusterv1alpha1.ForProvider( + host=cp.host, + port=cp.port, + tlsSAN=cp.host, + disableTraefik=True, + ) + # Only set the release field in use: an explicit None would still + # serialize (resource.update dumps with exclude_unset) and clobber + # the CRD's channel default. + if self._channel(): + fp.k3sChannel = self._channel() + if self._version(): + fp.k3sVersion = self._version() + cluster = k3sclusterv1alpha1.Cluster( + metadata=metav1.ObjectMeta(name=_cluster_name(self.xr)), + spec=k3sclusterv1alpha1.Spec( + providerConfigRef=k3sclusterv1alpha1.ProviderConfigRef( + kind="ProviderConfig", + name=_provider_config_name(self.xr), + ), + forProvider=fp, + writeConnectionSecretToRef=k3sclusterv1alpha1.WriteConnectionSecretToRef( + name=_kubeconfig_secret_name(self.xr), + ), + ), + ) + resource.update(self.rsp.desired.resources["cluster"], cluster) + + def compose_nodes(self) -> None: + """Compose a Node joining each worker as a k3s agent. Gated on the + cluster being Ready: an agent can only join once the server is up + and its join token exists.""" + for worker in self.xr.spec.workers or []: + fp = k3snodev1alpha1.ForProvider( + host=worker.host, + port=worker.port, + role="agent", + clusterRef=k3snodev1alpha1.ClusterRef(name=_cluster_name(self.xr)), + ) + if self._channel(): + fp.k3sChannel = self._channel() + if self._version(): + fp.k3sVersion = self._version() + extra_args = _extra_args(worker) + if extra_args: + fp.extraArgs = extra_args + node = k3snodev1alpha1.Node( + spec=k3snodev1alpha1.Spec( + providerConfigRef=k3snodev1alpha1.ProviderConfigRef( + kind="ProviderConfig", + name=_provider_config_name(self.xr), + ), + forProvider=fp, + ), + ) + resource.update(self.rsp.desired.resources[f"node-{worker.name}"], node) + + def write_status(self) -> None: + status = v1alpha1.Status( + secrets=[ + v1alpha1.Secret( + type=_SECRET_TYPE_KUBECONFIG, + name=_kubeconfig_secret_name(self.xr), + key=_SECRET_KEY_KUBECONFIG, + ), + ], + ) + resource.update_status(self.rsp.desired.composite, status) + + def mark_readiness(self) -> None: + """Mark composed resources as ready based on their observed + conditions. The ProviderConfig has no meaningful Ready condition and + is always marked ready. The cluster and each node are marked ready + only once their observed Ready condition is True, so the XR is Ready + only when the server runs and every agent has joined.""" + for r in self.rsp.desired.resources: + if r == "provider-config-k3s": + self.rsp.desired.resources[r].ready = fnv1.READY_TRUE + continue + if resource.get_condition(self.req.observed.resources.get(r), "Ready").status == "True": + self.rsp.desired.resources[r].ready = fnv1.READY_TRUE diff --git a/functions/compose-k3s-cluster/function/main.py b/functions/compose-k3s-cluster/function/main.py new file mode 100644 index 000000000..2e8441dac --- /dev/null +++ b/functions/compose-k3s-cluster/function/main.py @@ -0,0 +1,55 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The composition function's main CLI.""" + +import click +from crossplane.function import logging, runtime + +from function import fn + + +@click.command() +@click.option("--debug", "-d", is_flag=True, help="Emit debug logs.") +@click.option( + "--address", + default="0.0.0.0:9443", + show_default=True, + help="Address at which to listen for gRPC connections", +) +@click.option("--tls-certs-dir", help="Serve using mTLS certificates.", envvar="TLS_SERVER_CERTS_DIR") +@click.option( + "--insecure", + is_flag=True, + help="Run without mTLS credentials. If you supply this flag --tls-certs-dir will be ignored.", +) +def cli(debug: bool, address: str, tls_certs_dir: str, insecure: bool) -> None: + """A Crossplane composition function.""" + try: + level = logging.Level.INFO + if debug: + level = logging.Level.DEBUG + logging.configure(level=level) + runtime.serve( + fn.FunctionRunner(), + address, + creds=runtime.load_credentials(tls_certs_dir), + insecure=insecure, + ) + except Exception as e: + click.echo(f"Cannot run function: {e}") + + +if __name__ == "__main__": + cli() diff --git a/functions/compose-k3s-cluster/pyproject.toml b/functions/compose-k3s-cluster/pyproject.toml new file mode 100644 index 000000000..6678ead83 --- /dev/null +++ b/functions/compose-k3s-cluster/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["uv_build>=0.11.0,<0.12"] +build-backend = "uv_build" + +[project] +name = "compose-k3s-cluster" +version = "0.0.0" +description = "Compose a k3s cluster onto existing machines over SSH." +requires-python = ">=3.11,<3.14" +license = "Apache-2.0" +dependencies = [ + "crossplane-function-sdk-python>=0.14.0", + "click>=8.1.0", + "grpcio>=1.73.1", + "crossplane-models", +] + +[tool.uv.sources] +crossplane-models = { workspace = true } + +[project.scripts] +function = "function.main:cli" + +[tool.uv.build-backend] +module-name = "function" +module-root = "" diff --git a/functions/compose-k3s-cluster/tests/test_fn.py b/functions/compose-k3s-cluster/tests/test_fn.py new file mode 100644 index 000000000..3f348414b --- /dev/null +++ b/functions/compose-k3s-cluster/tests/test_fn.py @@ -0,0 +1,379 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the compose-k3s-cluster function.""" + +import dataclasses +import unittest + +from crossplane.function import logging, resource +from crossplane.function.proto.v1 import run_function_pb2 as fnv1 +from function import fn +from google.protobuf import duration_pb2 as durationpb +from google.protobuf import json_format +from google.protobuf import struct_pb2 as structpb +from models.ai.modelplane.infrastructure.k3scluster import v1alpha1 +from models.io.k8s.apimachinery.pkg.apis.meta import v1 as metav1 + + +@dataclasses.dataclass +class Case: + """A test case for compose-k3s-cluster.""" + + name: str + req: fnv1.RunFunctionRequest + want: fnv1.RunFunctionResponse + + +def setUpModule() -> None: + logging.configure(level=logging.Level.DISABLED) + + +# Names of the composed children. Derived like the function derives them - +# the hash suffix depends only on the parent and child names. +_PROVIDER_CONFIG_NAME = resource.child_name("test-cluster", "ssh") +_CLUSTER_NAME = resource.child_name("test-cluster", "cluster") +_KUBECONFIG_SECRET_NAME = resource.child_name("test-cluster", "kubeconfig") + +# A GPU worker with the labels and taint compose-vultr-baremetal-cluster +# would pass. +_GPU_WORKER = v1alpha1.Worker( + name="gpu-0", + host="203.0.113.20", + labels={"modelplane.ai/pool": "gpu", "modelplane.ai/gpu": "amd-mi355x"}, + taints=[v1alpha1.Taint(key="amd.com/gpu", value="true", effect="NoSchedule")], +) + +_GPU_WORKER_EXTRA_ARGS = ( + "--node-label modelplane.ai/gpu=amd-mi355x --node-label modelplane.ai/pool=gpu" + " --node-taint amd.com/gpu=true:NoSchedule" +) + + +def _xr( + workers: list[v1alpha1.Worker], + version: v1alpha1.Version | None = None, +) -> dict: + """A K3sCluster XR with the given workers, as a request dict.""" + return v1alpha1.K3sCluster( + metadata=metav1.ObjectMeta( + name="test-cluster", + namespace="modelplane-system", + ), + spec=v1alpha1.Spec( + controlPlane=v1alpha1.ControlPlane(host="203.0.113.10"), + workers=workers, + auth=v1alpha1.Auth(secretRef=v1alpha1.SecretRef(name="test-ssh")), + version=version, + ), + ).model_dump(exclude_none=True, mode="json") + + +def _req( + workers: list[v1alpha1.Worker], + version: v1alpha1.Version | None = None, + observed_resources: dict[str, fnv1.Resource] | None = None, +) -> fnv1.RunFunctionRequest: + return fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=fnv1.Resource(resource=resource.dict_to_struct(_xr(workers, version))), + resources=observed_resources or {}, + ), + ) + + +def _provider_config() -> dict: + """A k3s ProviderConfig golden carrying the SSH identity.""" + return { + "apiVersion": "k3s.m.crossplane.io/v1alpha1", + "kind": "ProviderConfig", + "metadata": { + "name": _PROVIDER_CONFIG_NAME, + "namespace": "modelplane-system", + }, + "spec": { + "username": "root", + "credentials": { + "source": "Secret", + "secretRef": { + "namespace": "modelplane-system", + "name": "test-ssh", + "key": "ssh-privatekey", + }, + }, + }, + } + + +def _cluster(release: dict | None = None) -> dict: + """A k3s Cluster golden installing the server on the control plane.""" + return { + "apiVersion": "k3s.m.crossplane.io/v1alpha1", + "kind": "Cluster", + "metadata": {"name": _CLUSTER_NAME}, + "spec": { + "providerConfigRef": {"kind": "ProviderConfig", "name": _PROVIDER_CONFIG_NAME}, + "forProvider": { + "host": "203.0.113.10", + "port": 22, + "tlsSAN": "203.0.113.10", + "disableTraefik": True, + **(release if release is not None else {"k3sChannel": "v1.34"}), + }, + "writeConnectionSecretToRef": {"name": _KUBECONFIG_SECRET_NAME}, + }, + } + + +def _node(host: str, release: dict | None = None, extra_args: str | None = None) -> dict: + """A k3s Node golden joining a worker as an agent.""" + fp = { + "host": host, + "port": 22, + "role": "agent", + "clusterRef": {"name": _CLUSTER_NAME}, + **(release if release is not None else {"k3sChannel": "v1.34"}), + } + if extra_args: + fp["extraArgs"] = extra_args + return { + "apiVersion": "k3s.m.crossplane.io/v1alpha1", + "kind": "Node", + "spec": { + "providerConfigRef": {"kind": "ProviderConfig", "name": _PROVIDER_CONFIG_NAME}, + "forProvider": fp, + }, + } + + +def _status() -> dict: + return { + "status": { + "secrets": [ + { + "type": "Kubeconfig", + "name": _KUBECONFIG_SECRET_NAME, + "key": "kubeconfig", + }, + ], + }, + } + + +def _observed_ready(desired: dict) -> fnv1.Resource: + """An observed variant of a desired resource with a Ready=True condition.""" + observed = { + **desired, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + return fnv1.Resource(resource=resource.dict_to_struct(observed)) + + +def _observed_unready(desired: dict) -> fnv1.Resource: + """An observed variant of a desired resource with a Ready=False condition.""" + observed = { + **desired, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "Unavailable", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + return fnv1.Resource(resource=resource.dict_to_struct(observed)) + + +class TestFunctionRunner(unittest.IsolatedAsyncioTestCase): + """Tests for FunctionRunner.RunFunction.""" + + maxDiff = None + + @classmethod + def setUpClass(cls) -> None: + cls.runner = fn.FunctionRunner() + + async def test_compose(self) -> None: + """The function composes a k3s cluster over SSH.""" + cases = [ + Case( + name="cluster composed first; nodes withheld until cluster Ready", + req=_req([_GPU_WORKER]), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=fnv1.Resource(resource=resource.dict_to_struct(_status())), + resources={ + "provider-config-k3s": fnv1.Resource( + resource=resource.dict_to_struct(_provider_config()), + ready=fnv1.READY_TRUE, + ), + "cluster": fnv1.Resource( + resource=resource.dict_to_struct(_cluster()), + ), + }, + ), + context=structpb.Struct(), + ), + ), + Case( + name="nodes composed once the cluster is Ready; labels and taints as agent args", + req=_req( + [_GPU_WORKER], + observed_resources={ + "cluster": _observed_ready(_cluster()), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=fnv1.Resource(resource=resource.dict_to_struct(_status())), + resources={ + "provider-config-k3s": fnv1.Resource( + resource=resource.dict_to_struct(_provider_config()), + ready=fnv1.READY_TRUE, + ), + "cluster": fnv1.Resource( + resource=resource.dict_to_struct(_cluster()), + ready=fnv1.READY_TRUE, + ), + "node-gpu-0": fnv1.Resource( + resource=resource.dict_to_struct( + _node("203.0.113.20", extra_args=_GPU_WORKER_EXTRA_ARGS), + ), + ), + }, + ), + context=structpb.Struct(), + ), + ), + Case( + name="nodes kept when the cluster Ready condition transiently regresses", + req=_req( + [_GPU_WORKER], + observed_resources={ + "cluster": _observed_unready(_cluster()), + "node-gpu-0": _observed_ready(_node("203.0.113.20", extra_args=_GPU_WORKER_EXTRA_ARGS)), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=fnv1.Resource(resource=resource.dict_to_struct(_status())), + resources={ + "provider-config-k3s": fnv1.Resource( + resource=resource.dict_to_struct(_provider_config()), + ready=fnv1.READY_TRUE, + ), + "cluster": fnv1.Resource( + resource=resource.dict_to_struct(_cluster()), + ), + "node-gpu-0": fnv1.Resource( + resource=resource.dict_to_struct( + _node("203.0.113.20", extra_args=_GPU_WORKER_EXTRA_ARGS), + ), + ready=fnv1.READY_TRUE, + ), + }, + ), + context=structpb.Struct(), + ), + ), + Case( + name="exact version pins k3sVersion; plain worker joins with no agent args", + req=_req( + [v1alpha1.Worker(name="w0", host="203.0.113.30")], + version=v1alpha1.Version(version="v1.34.1+k3s1"), + observed_resources={ + "cluster": _observed_ready(_cluster({"k3sVersion": "v1.34.1+k3s1"})), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=fnv1.Resource(resource=resource.dict_to_struct(_status())), + resources={ + "provider-config-k3s": fnv1.Resource( + resource=resource.dict_to_struct(_provider_config()), + ready=fnv1.READY_TRUE, + ), + "cluster": fnv1.Resource( + resource=resource.dict_to_struct(_cluster({"k3sVersion": "v1.34.1+k3s1"})), + ready=fnv1.READY_TRUE, + ), + "node-w0": fnv1.Resource( + resource=resource.dict_to_struct( + _node("203.0.113.30", {"k3sVersion": "v1.34.1+k3s1"}), + ), + ), + }, + ), + context=structpb.Struct(), + ), + ), + Case( + name="K3sCluster Ready only once the server and every agent are Ready", + req=_req( + [_GPU_WORKER], + observed_resources={ + "cluster": _observed_ready(_cluster()), + "node-gpu-0": _observed_ready(_node("203.0.113.20", extra_args=_GPU_WORKER_EXTRA_ARGS)), + }, + ), + want=fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State( + composite=fnv1.Resource(resource=resource.dict_to_struct(_status())), + resources={ + "provider-config-k3s": fnv1.Resource( + resource=resource.dict_to_struct(_provider_config()), + ready=fnv1.READY_TRUE, + ), + "cluster": fnv1.Resource( + resource=resource.dict_to_struct(_cluster()), + ready=fnv1.READY_TRUE, + ), + "node-gpu-0": fnv1.Resource( + resource=resource.dict_to_struct( + _node("203.0.113.20", extra_args=_GPU_WORKER_EXTRA_ARGS), + ), + ready=fnv1.READY_TRUE, + ), + }, + ), + context=structpb.Struct(), + ), + ), + ] + + for case in cases: + with self.subTest(case.name): + got = await self.runner.RunFunction(case.req, None) + self.assertEqual( + json_format.MessageToDict(case.want), + json_format.MessageToDict(got), + "-want, +got", + ) diff --git a/functions/compose-model-replica/function/backends/base.py b/functions/compose-model-replica/function/backends/base.py index 439efbe1a..b33b338fe 100644 --- a/functions/compose-model-replica/function/backends/base.py +++ b/functions/compose-model-replica/function/backends/base.py @@ -551,12 +551,18 @@ def engine_resources() -> dict: return {"claims": [{"name": _POD_CLAIM_NAME}]} -# Taint GPU node groups carry so non-GPU pods don't land on them. A pod that -# claims a GPU must tolerate it to schedule there. With GPUs bound via DRA (not -# the device plugin's extended resource), nothing injects this toleration for us -# - the ExtendedResourceToleration admission controller only acts on -# nvidia.com/gpu resource requests, which DRA pods don't make. -_GPU_TOLERATION = {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"} +# Taints GPU node groups carry so non-GPU pods don't land on them. A pod that +# claims a GPU must tolerate its pool's taint to schedule there. With GPUs +# bound via DRA (not the device plugin's extended resource), nothing injects +# these tolerations for us - the ExtendedResourceToleration admission +# controller only acts on nvidia.com/gpu resource requests, which DRA pods +# don't make. The pool carries one vendor's taint; tolerating the other +# vendor's too is harmless, so both are always added rather than threading +# the vendor through here. +_GPU_TOLERATIONS = [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, +] # Node label identifying the pool a node belongs to. compose-eks-cluster and # compose-gke-cluster stamp it on every node group they provision; the scheduler @@ -597,7 +603,7 @@ def place_pod(pod_spec: dict, replica: v1alpha1.ModelReplica, engine: v1alpha1.E pod_spec["resourceClaims"] = [ {"name": _POD_CLAIM_NAME, "resourceClaimTemplateName": claim_template_name(replica, engine, member)} ] - pod_spec.setdefault("tolerations", []).append(_GPU_TOLERATION) + pod_spec.setdefault("tolerations", []).extend(_GPU_TOLERATIONS) def resource_claim_template( diff --git a/functions/compose-model-replica/tests/test_backends.py b/functions/compose-model-replica/tests/test_backends.py index a05458223..dafadb94a 100644 --- a/functions/compose-model-replica/tests/test_backends.py +++ b/functions/compose-model-replica/tests/test_backends.py @@ -232,7 +232,10 @@ def _claim_template(count: int, *, replica: str = "r", engine: str = "main", rol "resourceClaimTemplateName": resource.child_name("r", "main", "standalone", "devices"), } ], - "tolerations": [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}], + "tolerations": [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], }, }, }, @@ -258,7 +261,10 @@ def _clique(manifest: dict, name: str) -> dict: def _pcs(leader_container: dict, worker_container: dict, *, worker_replicas: int = 1, copies: int = 1) -> dict: node_selector = {"modelplane.ai/pool": "frontier"} - tolerations = [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}] + tolerations = [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ] def pod_spec(container: dict, role: str) -> dict: return { @@ -619,7 +625,11 @@ def test_claimless_leader_gets_no_claim(self) -> None: self.assertNotIn("resources", leader["containers"][0]) self.assertEqual(leader["nodeSelector"], {"modelplane.ai/pool": "frontier"}) self.assertEqual( - leader["tolerations"], [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}] + leader["tolerations"], + [ + {"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + {"key": "amd.com/gpu", "operator": "Exists", "effect": "NoSchedule"}, + ], ) worker = _clique(manifest, "worker")["spec"]["podSpec"] diff --git a/functions/compose-model-replica/tests/test_fn.py b/functions/compose-model-replica/tests/test_fn.py index 32801c36c..f75f75547 100644 --- a/functions/compose-model-replica/tests/test_fn.py +++ b/functions/compose-model-replica/tests/test_fn.py @@ -217,6 +217,11 @@ async def test_compose(self) -> None: "operator": "Exists", "effect": "NoSchedule", }, + { + "key": "amd.com/gpu", + "operator": "Exists", + "effect": "NoSchedule", + }, ], }, }, diff --git a/functions/compose-serving-stack/function/fn.py b/functions/compose-serving-stack/function/fn.py index 46e7c7a4a..49bafea51 100644 --- a/functions/compose-serving-stack/function/fn.py +++ b/functions/compose-serving-stack/function/fn.py @@ -229,8 +229,13 @@ def compose(self) -> None: # The XRD requires and enums both fields, so the join raising means the # API and the stacks package disagree on a value, a broken Modelplane # build, not a cluster condition. Let it crash rather than dress it up - # as a fatal result. - components = stacks.join(self.xr.spec.cloud, self.xr.spec.stack or "Standard") + # as a fatal result. spec.accelerators, when set, filters the cloud's + # vendor-tagged accelerator components to the vendors actually present. + components = stacks.join( + self.xr.spec.cloud, + self.xr.spec.stack or "Standard", + accelerator_vendors=self.xr.spec.accelerators, + ) rendered = self.compose_components(components) rendered += self.compose_gateway() diff --git a/functions/compose-serving-stack/function/stacks/__init__.py b/functions/compose-serving-stack/function/stacks/__init__.py index eb8ede28a..876537255 100644 --- a/functions/compose-serving-stack/function/stacks/__init__.py +++ b/functions/compose-serving-stack/function/stacks/__init__.py @@ -22,11 +22,13 @@ """ from function.stacks import common, components, dynamo, standard -from function.stacks.clouds import existing, nebius, vultr +from function.stacks.clouds import existing, nebius, vultr, vultr_baremetal from function.stacks.clouds.generated.aicr import aks, eks, gke -from function.stacks.components import Chart, Cloud, Component, Manifests, Stack +from function.stacks.components import AcceleratorVendor, Chart, Cloud, Component, Manifests, Stack __all__ = [ + "ACCELERATOR_VENDORS", + "AcceleratorVendor", "Chart", "Cloud", "Component", @@ -47,6 +49,7 @@ "GKE": gke.COMPONENTS, "Nebius": nebius.COMPONENTS, "Vultr": vultr.COMPONENTS, + "VultrBaremetal": vultr_baremetal.COMPONENTS, "Existing": existing.COMPONENTS, } @@ -55,6 +58,24 @@ "Dynamo": dynamo.COMPONENTS, } +# The accelerator vendors each cloud's serving stack can install: the +# vendors its component list carries a device stack for. A +# single-vendor cloud keeps its accelerator components untagged (there +# is nothing to filter); a multi-vendor cloud tags them and the join +# filters by ServingStack spec.accelerators. Existing is BYO: the +# cluster's operator manages the accelerator stack, so any vendor goes. +# compose-inference-cluster mirrors this table to reject unsupported +# pairings with a condition before a ServingStack is ever composed. +ACCELERATOR_VENDORS: dict[Cloud, list[AcceleratorVendor]] = { + "EKS": ["NVIDIA"], + "AKS": ["NVIDIA"], + "GKE": ["NVIDIA"], + "Nebius": ["NVIDIA"], + "Vultr": ["NVIDIA"], + "VultrBaremetal": ["AMD", "NVIDIA"], + "Existing": ["AMD", "NVIDIA"], +} + def clouds() -> list[Cloud]: """The clouds a stack can be joined for.""" @@ -66,19 +87,34 @@ def stacks() -> list[Stack]: return list(_STACKS) -def join(cloud: Cloud, stack: Stack) -> list[Component]: +def join(cloud: Cloud, stack: Stack, accelerator_vendors: list[AcceleratorVendor] | None = None) -> list[Component]: """Join the component lists for a cloud and stack. Fails closed, at import or test time rather than on a cluster: on an - unknown cloud or stack, on a key two lists both produce, and on a + unknown cloud or stack, on a key two lists both produce, on a depends_on edge naming a component the join didn't produce - which catches a generator allowlist that dropped something another - component needs. + component needs - and on a component depending on another vendor's + accelerator stack, which vendor filtering could then remove from + under it. + + accelerator_vendors filters the vendor-tagged components: a tagged + component survives only when its vendor is listed, an untagged one + always does. None (the field unset on the XR) disables filtering, so + clouds that don't set it keep installing everything. The integrity + checks run on the unfiltered join, so a broken list fails every join + for its cloud, not just the vendor combination that trips it. """ if cloud not in _CLOUDS: raise ValueError(f"unknown cloud {cloud!r}; known: {', '.join(_CLOUDS)}") if stack not in _STACKS: raise ValueError(f"unknown stack {stack!r}; known: {', '.join(_STACKS)}") + for vendor in accelerator_vendors or []: + if vendor not in ACCELERATOR_VENDORS[cloud]: + raise ValueError( + f"{cloud}: the serving stack has no {vendor} accelerator stack; " + f"it installs {', '.join(ACCELERATOR_VENDORS[cloud])}" + ) joined = [*_CLOUDS[cloud], *common.COMPONENTS, *_STACKS[stack]] @@ -95,10 +131,20 @@ def join(cloud: Cloud, stack: Stack) -> list[Component]: if duplicates: raise ValueError(f"{cloud}/{stack}: duplicate composed-resource keys {duplicates}") - known = set(keys) + by_key = {c.key: c for c in joined} for c in joined: for dep in c.depends_on: - if dep not in known: + if dep not in by_key: raise ValueError(f"{cloud}/{stack}: {c.key} depends on {dep!r}, which the join did not produce") - - return joined + # A dependency tagged for another vendor would be filtered + # away while its dependent survives, leaving a dangling edge. + dep_vendor = by_key[dep].accelerator_vendor + if dep_vendor is not None and dep_vendor != c.accelerator_vendor: + raise ValueError( + f"{cloud}/{stack}: {c.key} ({c.accelerator_vendor or 'untagged'}) depends on " + f"{dep!r}, which is tagged {dep_vendor}" + ) + + if accelerator_vendors is None: + return joined + return [c for c in joined if c.accelerator_vendor is None or c.accelerator_vendor in accelerator_vendors] diff --git a/functions/compose-serving-stack/function/stacks/clouds/existing.py b/functions/compose-serving-stack/function/stacks/clouds/existing.py index 1ba050711..7c7d0a1de 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/existing.py +++ b/functions/compose-serving-stack/function/stacks/clouds/existing.py @@ -128,6 +128,7 @@ # (/): the node image puts the driver at the default root. Chart( key="nvidia-dra-driver-gpu", + accelerator_vendor="NVIDIA", release="mp-dra-driver-nvidia-gpu", namespace="nvidia-dra-driver", chart="dra-driver-nvidia-gpu", diff --git a/functions/compose-serving-stack/function/stacks/clouds/nebius.py b/functions/compose-serving-stack/function/stacks/clouds/nebius.py index 68d20efc9..cf1b9cc56 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/nebius.py +++ b/functions/compose-serving-stack/function/stacks/clouds/nebius.py @@ -127,6 +127,7 @@ # (/): the node image puts the driver at the default root. Chart( key="nvidia-dra-driver-gpu", + accelerator_vendor="NVIDIA", release="mp-dra-driver-nvidia-gpu", namespace="nvidia-dra-driver", chart="dra-driver-nvidia-gpu", diff --git a/functions/compose-serving-stack/function/stacks/clouds/vultr.py b/functions/compose-serving-stack/function/stacks/clouds/vultr.py index e49718742..39c5d234e 100644 --- a/functions/compose-serving-stack/function/stacks/clouds/vultr.py +++ b/functions/compose-serving-stack/function/stacks/clouds/vultr.py @@ -108,6 +108,7 @@ # (/): the node image puts the driver at the default root. Chart( key="nvidia-dra-driver-gpu", + accelerator_vendor="NVIDIA", release="mp-dra-driver-nvidia-gpu", namespace="nvidia-dra-driver", chart="dra-driver-nvidia-gpu", diff --git a/functions/compose-serving-stack/function/stacks/clouds/vultr_baremetal.py b/functions/compose-serving-stack/function/stacks/clouds/vultr_baremetal.py new file mode 100644 index 000000000..168815e9f --- /dev/null +++ b/functions/compose-serving-stack/function/stacks/clouds/vultr_baremetal.py @@ -0,0 +1,233 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The cloud half of the stack for Vultr bare metal (k3s). + +No generator covers this cloud, so Modelplane pins it by hand, in the +same shape a generator emits. Where a component also appears on another +cloud, this file states the same pin, so one review moves both. + +Unlike managed clusters, nothing preinstalls a GPU stack on bare metal: +the driver, the operator, and the DRA driver all install here. The GPU +components are vendor-tagged - Vultr sells both AMD Instinct and NVIDIA +bare metal GPU plans - and the join filters them to the vendors the +InferenceClasses actually name, so an AMD cluster doesn't run NVIDIA +machinery and vice versa. + +One node-feature-discovery serves both vendors: the AMD operator's NFD +subchart and the NVIDIA operator's are disabled so two instances never +race over the same label namespace. The AMD chart's default +NodeFeatureRule (feature.node.kubernetes.io/amd-gpu) still installs and +is honoured by the standalone NFD master. +""" + +from function.stacks.components import Chart, Component + +COMPONENTS: list[Component] = [ + Chart( + key="cert-manager", + release="mp-cert-manager", + namespace="cert-manager", + chart="cert-manager", + repository="https://charts.jetstack.io", + version="v1.20.2", + # envoy-gateway in common.py depends on this chart (a cross-half + # edge), so its Ready must mean healthy, not just deployed. + wait=True, + values={"crds": {"enabled": True, "keep": False}}, + ), + Chart( + key="kube-prometheus-stack", + release="mp-kube-prometheus-stack", + namespace="monitoring", + chart="kube-prometheus-stack", + repository="https://prometheus-community.github.io/helm-charts", + version="84.4.0", + # gpu-operator below depends on this chart, so its Ready must + # mean healthy, not just deployed. + wait=True, + values={ + "fullnameOverride": "prometheus", + "prometheus": { + "prometheusSpec": { + # Discover PodMonitors across all namespaces. + "podMonitorSelectorNilUsesHelmValues": False, + "podMonitorNamespaceSelector": {}, + # Scrape Envoy Gateway proxy pods for upstream + # request metrics (envoy_cluster_upstream_rq_active): + # in-flight requests at the proxy level. + "additionalScrapeConfigs": [ + { + "job_name": "envoy-gateway-proxy", + "kubernetes_sd_configs": [ + { + "role": "pod", + "namespaces": { + "names": ["envoy-gateway-system"], + }, + }, + ], + "relabel_configs": [ + { + "source_labels": [ + "__meta_kubernetes_pod_label_app_kubernetes_io_component", + ], + "action": "keep", + "regex": "proxy", + }, + { + "source_labels": ["__address__"], + "action": "replace", + "regex": "([^:]+)(?::\\d+)?", + "replacement": "$1:19001", + "target_label": "__address__", + }, + ], + "metrics_path": "/stats/prometheus", + }, + ], + }, + }, + # Disable components we don't need for observability. + "grafana": {"enabled": False}, + "alertmanager": {"enabled": False}, + }, + ), + # One NFD instance for both GPU vendors' operators, which both have + # their bundled NFD disabled. Same pin as the generated clouds. + Chart( + key="node-feature-discovery", + release="mp-node-feature-discovery", + namespace="node-feature-discovery", + chart="node-feature-discovery", + repository="https://kubernetes-sigs.github.io/node-feature-discovery/charts", + version="0.19.0", + wait=True, + values={ + "gc": {"enable": True, "tolerations": [{"operator": "Exists"}]}, + "master": {"enable": True, "tolerations": [{"operator": "Exists"}]}, + "topologyUpdater": { + "createCRDs": True, + "enable": False, + "kubeletStateDir": "", + "resources": {"limits": {"memory": "256Mi"}, "requests": {"cpu": "50m", "memory": "128Mi"}}, + "tolerations": [{"operator": "Exists"}], + }, + "worker": {"enable": True, "tolerations": [{"operator": "Exists"}]}, + }, + ), + # The AMD GPU operator, in DRA mode: the chart's default DeviceConfig + # (crds.defaultCR) enables the DRA driver and disables the device + # plugin - the two are mutually exclusive allocators - so the + # gpu.amd.com DeviceClass the chart registers is the sole path to the + # GPUs. KMM (a subchart) installs the amdgpu/ROCm driver on the bare + # Ubuntu nodes; the chart's default driver version tracks a ROCm 7.x + # release, which MI355X (gfx950) requires. The node labeller stays on + # for scheduling labels; NFD is the standalone instance above. + Chart( + key="amd-gpu-operator", + release="mp-gpu-operator-charts", + namespace="kube-amd-gpu", + chart="gpu-operator-charts", + repository="https://rocm.github.io/gpu-operator", + version="v1.5.1", + wait=True, + depends_on=["cert-manager", "node-feature-discovery"], + accelerator_vendor="AMD", + values={ + "node-feature-discovery": {"enabled": False}, + # Argo-based node remediation pulls a workflow controller the + # serving stack doesn't need. + "remediation": {"enabled": False, "installCRDs": False}, + "deviceConfig": { + "spec": { + "driver": {"enable": True}, + "devicePlugin": {"enableDevicePlugin": False}, + "draDriver": { + "enable": True, + "tolerations": [{"operator": "Exists"}], + }, + }, + }, + }, + ), + # The NVIDIA GPU operator. Bare metal deltas from the managed-cloud + # pin: the driver and container toolkit install here (no node image + # provides them), and the toolkit is pointed at k3s's containerd, + # which lives off the stock paths. The device plugin stays off: the + # DRA driver below is the sole allocator. + Chart( + key="gpu-operator", + release="mp-gpu-operator", + namespace="gpu-operator", + chart="gpu-operator", + repository="https://helm.ngc.nvidia.com/nvidia", + version="v26.3.3", + wait=True, + depends_on=["cert-manager", "node-feature-discovery", "kube-prometheus-stack"], + accelerator_vendor="NVIDIA", + values={ + "ccManager": {"enabled": False}, + "daemonsets": {"tolerations": [{"operator": "Exists"}]}, + "dcgm": {"enabled": True}, + "devicePlugin": {"enabled": False}, + "driver": { + "enabled": True, + "maxParallelUpgrades": 5, + "rdma": {"enabled": False}, + "useOpenKernelModules": True, + "version": "580.173.02", + }, + "fullnameOverride": "gpu-operator", + "gdrcopy": {"enabled": False}, + "gfd": {"enabled": True}, + "hostPaths": {"driverInstallDir": "/run/nvidia/driver"}, + "kataSandboxDevicePlugin": {"enabled": False}, + "migManager": {"enabled": False}, + "nfd": {"enabled": False}, + "toolkit": { + "enabled": True, + "env": [ + { + "name": "CONTAINERD_CONFIG", + "value": "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl", + }, + {"name": "CONTAINERD_SOCKET", "value": "/run/k3s/containerd/containerd.sock"}, + {"name": "CONTAINERD_RUNTIME_CLASS", "value": "nvidia"}, + {"name": "CONTAINERD_SET_AS_DEFAULT", "value": "true"}, + ], + }, + "validator": {"plugin": {"env": [{"name": "WITH_WORKLOAD", "value": "false"}]}}, + }, + ), + # Publishes each GPU node's devices as DRA ResourceSlices and + # registers the gpu.nvidia.com DeviceClass. The driver root points + # into the operator's install dir, not / as on managed clouds where + # the node image provides the driver. + Chart( + key="nvidia-dra-driver-gpu", + release="mp-dra-driver-nvidia-gpu", + namespace="nvidia-dra-driver", + chart="dra-driver-nvidia-gpu", + repository="oci://registry.k8s.io/dra-driver-nvidia/charts", + version="0.4.1", + depends_on=["gpu-operator"], + accelerator_vendor="NVIDIA", + values={ + "gpuResourcesEnabledOverride": True, + "nvidiaDriverRoot": "/run/nvidia/driver", + "resources": {"computeDomains": {"enabled": False}}, + }, + ), +] diff --git a/functions/compose-serving-stack/function/stacks/common.py b/functions/compose-serving-stack/function/stacks/common.py index 12e25b5e1..50b64eacb 100644 --- a/functions/compose-serving-stack/function/stacks/common.py +++ b/functions/compose-serving-stack/function/stacks/common.py @@ -181,10 +181,13 @@ def _crds(filename: str) -> list[dict[str, Any]]: # priority. GKE only admits such pods in a namespace whose # ResourceQuota permits those priority classes; without it the # daemonset gets FailedCreate and never publishes ResourceSlices. - # Laid down everywhere: it only grants headroom, so it's harmless on - # clusters that don't restrict them. + # Laid down everywhere the NVIDIA stack runs: it only grants + # headroom, so it's harmless on clusters that don't restrict them. + # The vendor tag keeps it off clusters where nothing creates the + # nvidia-dra-driver namespace, where the Object could never apply. Manifests( key="dra-driver-critical-pods-quota", + accelerator_vendor="NVIDIA", manifests=[ { "apiVersion": "v1", diff --git a/functions/compose-serving-stack/function/stacks/components.py b/functions/compose-serving-stack/function/stacks/components.py index 7208aaee7..68ea30e79 100644 --- a/functions/compose-serving-stack/function/stacks/components.py +++ b/functions/compose-serving-stack/function/stacks/components.py @@ -29,9 +29,16 @@ # The clouds and stacks the join can select - the values of ServingStack # spec.cloud and spec.stack, so a wrong or unsupported value fails type # checking at the caller. -Cloud = Literal["GKE", "EKS", "AKS", "Nebius", "Vultr", "Existing"] +Cloud = Literal["GKE", "EKS", "AKS", "Nebius", "Vultr", "VultrBaremetal", "Existing"] Stack = Literal["Standard", "Dynamo"] +# The accelerator vendors a component can be tagged with - the values +# of ServingStack spec.accelerators entries. A tagged component only +# survives a join filtered to a vendor list that includes its tag; an +# untagged component always installs. GPU vendors today; grows with +# whatever accelerator families land next (TPU, Trainium, ...). +AcceleratorVendor = Literal["AMD", "NVIDIA"] + @dataclass class Chart: @@ -56,6 +63,10 @@ class Chart: depends on, so the install gate orders on health rather than deploy - the generator derives it from the dependency edges, and the hand-written files state it where a cross-half edge lands on them. + + `accelerator_vendor` marks a component as part of one vendor's + accelerator stack, so the join can drop it on clusters without that + vendor's devices. Leave it unset for components every cluster needs. """ key: str @@ -67,6 +78,7 @@ class Chart: wait: bool = False depends_on: list[str] = field(default_factory=list) values: dict[str, Any] | None = None + accelerator_vendor: AcceleratorVendor | None = None @dataclass @@ -82,12 +94,15 @@ class Manifests: applied to every doc in the entry (see fn.py's _k8s_object): use it when readiness must reflect a controller-populated status field, and keep an entry to one doc when only that doc has one. + + `accelerator_vendor` behaves as on Chart. """ key: str manifests: list[dict[str, Any]] depends_on: list[str] = field(default_factory=list) ready: str | None = None + accelerator_vendor: AcceleratorVendor | None = None # A plain assignment rather than a `type` statement: the packages diff --git a/functions/compose-vultr-baremetal-cluster/function/__init__.py b/functions/compose-vultr-baremetal-cluster/function/__init__.py new file mode 100644 index 000000000..b53d39d12 --- /dev/null +++ b/functions/compose-vultr-baremetal-cluster/function/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/functions/compose-vultr-baremetal-cluster/function/fn.py b/functions/compose-vultr-baremetal-cluster/function/fn.py new file mode 100644 index 000000000..49dd20a64 --- /dev/null +++ b/functions/compose-vultr-baremetal-cluster/function/fn.py @@ -0,0 +1,376 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compose Vultr bare metal servers into a k3s cluster. + +This function provisions one CPU-only management server and a fixed number +of GPU servers per node pool, then composes a K3sCluster XR that installs +k3s onto them over SSH: the management server becomes the k3s server, the +GPU servers join as agents. + +The SSH key pair comes from a user-supplied Secret. The public key is +registered with Vultr as an SSHKey resource - every server selects it via +matchControllerRef, so Vultr installs it at provisioning time - and the +private key is handed to the K3sCluster for the installs. Nothing is +composed until the Secret resolves: servers provisioned without the key +would be unreachable. + +The K3sCluster is composed only once every server is active and reports +its main IP; bare metal provisioning takes tens of minutes. Server IPs +come from Vultr resource observations, which persist for the life of the +server, so the K3sCluster's hosts stay stable across reconciles. + +GPU pool workers are labelled for scheduling and tainted by GPU vendor +(amd.com/gpu or nvidia.com/gpu), derived from the accelerator type the +pool declares. Vultr's flagship bare metal GPU plans are AMD Instinct, +and the serving stack installs the AMD GPU operator on them. +""" + +import base64 + +import grpc +from crossplane.function import logging, request, resource, response +from crossplane.function.proto.v1 import run_function_pb2 as fnv1 +from crossplane.function.proto.v1 import run_function_pb2_grpc as grpcv1 +from models.ai.modelplane.infrastructure.k3scluster import v1alpha1 as k3sv1alpha1 +from models.ai.modelplane.infrastructure.vultrbaremetalcluster import v1alpha1 +from models.io.k8s.apimachinery.pkg.apis.meta import v1 as metav1 +from models.io.upbound.m.vultr.compute.baremetalserver import v1beta1 as bmv1beta1 +from models.io.upbound.m.vultr.compute.sshkey import v1beta1 as sshkeyv1beta1 + +# Labels written on worker nodes. compose-model-deployment reads these +# labels for GPU scheduling. Kept in sync with compose-vultr-cluster. +_LABEL_GPU = "modelplane.ai/gpu" +_LABEL_POOL = "modelplane.ai/pool" + +# Taint applied to GPU workers so only inference workloads that tolerate +# GPUs are scheduled on them. The key follows the pool's GPU vendor, +# derived from the accelerator type the InferenceClass declares: Vultr +# offers both AMD Instinct and NVIDIA bare metal GPU plans. +_GPU_TAINT_KEY_AMD = "amd.com/gpu" +_GPU_TAINT_KEY_NVIDIA = "nvidia.com/gpu" +_GPU_TAINT_VALUE = "true" +_GPU_TAINT_EFFECT = "NoSchedule" + +# Secret type written to XR status. compose-inference-cluster reads this to +# wire the kubeconfig into a ClusterProviderConfig. +_SECRET_TYPE_KUBECONFIG = "Kubeconfig" + +# Vultr reports this once a bare metal server is provisioned and running. +_SERVER_STATUS_ACTIVE = "active" + +# Cloud-init for every server. Vultr's Ubuntu images ship with UFW +# enabled and only SSH allowed, which lets the k3s install through (it +# runs over SSH) but firewalls everything the cluster itself needs: +# the API server (6443, agents joining and kubectl), the kubelet +# (10250), flannel's VXLAN overlay (8472/udp), and the gateway's HTTP +# ports served by k3s ServiceLB on the node IPs. The API server and +# kubelet authenticate with TLS client certificates; the VXLAN overlay +# rides the public network unauthenticated, which moving the data plane +# onto a Vultr VPC would fix. +_USER_DATA = """#cloud-config +runcmd: +- ufw allow 6443/tcp +- ufw allow 10250/tcp +- ufw allow 8472/udp +- ufw allow 80/tcp +- ufw allow 443/tcp +""" + +# Defaults the XRD also declares. Coalesced here so the function tolerates +# an XR that predates server-side defaulting. +_DEFAULT_MANAGEMENT_PLAN = "vbm-6c-32gb-amd" +_DEFAULT_OS_ID = 2284 +_DEFAULT_K3S_CHANNEL = "v1.34" +_DEFAULT_USERNAME = "root" +_DEFAULT_PRIVATE_KEY_KEY = "ssh-privatekey" +_DEFAULT_PUBLIC_KEY_KEY = "ssh-publickey" + +# Condition type and reason set while the SSH key Secret is unresolved. +CONDITION_TYPE_CLUSTER_READY = "ClusterReady" +CONDITION_REASON_WAITING_FOR_SSH_SECRET = "WaitingForSSHSecret" + +# Resource key of the management server; pool servers are keyed +# server--, which can't collide with it because a pool named +# management yields server-management-. +_MANAGEMENT_SERVER_KEY = "server-management" + + +def _gpu_taint_key(accelerator_type: str) -> str: + """The GPU taint key for a pool, by the vendor its accelerator type + names (e.g. nvidia-h100 vs amd-mi355x).""" + return _GPU_TAINT_KEY_NVIDIA if accelerator_type.startswith("nvidia") else _GPU_TAINT_KEY_AMD + + +def _name(meta: metav1.ObjectMeta | None) -> str: + """The object's name, always set on resources read from the API server.""" + if meta is None or meta.name is None: + raise ValueError("metadata.name is unexpectedly absent") + return meta.name + + +def _namespace(meta: metav1.ObjectMeta | None) -> str: + """The object's namespace, always set on resources read from the API server.""" + if meta is None or meta.namespace is None: + raise ValueError("metadata.namespace is unexpectedly absent") + return meta.namespace + + +class FunctionRunner(grpcv1.FunctionRunnerServiceServicer): + """A FunctionRunner handles gRPC RunFunctionRequests.""" + + def __init__(self) -> None: + """Create a new FunctionRunner.""" + self.log = logging.get_logger() + + async def RunFunction( + self, req: fnv1.RunFunctionRequest, _: grpc.aio.ServicerContext | None + ) -> fnv1.RunFunctionResponse: # ty: ignore[invalid-method-override] # the generated grpc servicer base is untyped + """Run the function.""" + log = self.log.bind(tag=req.meta.tag) + log.info("Running function") + + rsp = response.to(req) + c = Composer(req, rsp) + c.compose() + return rsp + + +class Composer: + def __init__(self, req: fnv1.RunFunctionRequest, rsp: fnv1.RunFunctionResponse) -> None: + self.req = req + self.rsp = rsp + self.xr = v1alpha1.VultrBaremetalCluster(**resource.struct_to_dict(req.observed.composite.resource)) + + def _cred_kind(self) -> str: + creds = self.xr.spec.credentials + return creds.type if creds and creds.type else "ClusterProviderConfig" + + def _cred_name(self) -> str: + creds = self.xr.spec.credentials + return creds.name if creds and creds.name else "default" + + def _management_plan(self) -> str: + management = self.xr.spec.management + return management.plan if management and management.plan else _DEFAULT_MANAGEMENT_PLAN + + def _management_os_id(self) -> int: + management = self.xr.spec.management + return management.osId if management and management.osId else _DEFAULT_OS_ID + + def _channel(self) -> str: + k3s = self.xr.spec.k3s + return k3s.channel if k3s and k3s.channel else _DEFAULT_K3S_CHANNEL + + def compose(self) -> None: + public_key = self.resolve_public_key() + if public_key is None: + return + + self.compose_ssh_key(public_key) + self.compose_servers() + self.compose_k3s_cluster() + self.write_status() + self.mark_readiness() + + def resolve_public_key(self) -> str | None: + """Declare and fetch the SSH key pair Secret, returning the public + key. Returns None if the Secret or key is missing, in which case + nothing is composed: servers provisioned without the key would be + unreachable over SSH.""" + ssh = self.xr.spec.ssh + response.require_resources( + self.rsp, + name="ssh-secret", + api_version="v1", + kind="Secret", + match_name=ssh.secretRef.name, + namespace=_namespace(self.xr.metadata), + ) + + secret = request.get_required_resource(self.req, "ssh-secret") + if secret is None: + msg = f"Waiting for SSH key Secret {ssh.secretRef.name}" + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_WAITING_FOR_SSH_SECRET, + message=msg, + ), + ) + response.normal(self.rsp, msg) + return None + + key = ssh.secretRef.publicKeyKey or _DEFAULT_PUBLIC_KEY_KEY + data = secret.get("data", {}).get(key) + if not data: + msg = f"SSH key Secret {ssh.secretRef.name} has no {key} key" + response.set_conditions( + self.rsp, + resource.Condition( + typ=CONDITION_TYPE_CLUSTER_READY, + status="False", + reason=CONDITION_REASON_WAITING_FOR_SSH_SECRET, + message=msg, + ), + ) + response.warning(self.rsp, msg) + return None + + return base64.b64decode(data).decode().strip() + + def compose_ssh_key(self, public_key: str) -> None: + """Register the public key with Vultr. Servers select it via + matchControllerRef, so Vultr installs it at provisioning time.""" + resource.update( + self.rsp.desired.resources["ssh-key"], + sshkeyv1beta1.SSHKey( + spec=sshkeyv1beta1.Spec( + providerConfigRef=sshkeyv1beta1.ProviderConfigRef( + kind=self._cred_kind(), + name=self._cred_name(), + ), + forProvider=sshkeyv1beta1.ForProvider( + name=_name(self.xr.metadata), + sshKey=public_key, + ), + ), + ), + ) + + def compose_servers(self) -> None: + """Compose the management server and each pool's servers.""" + name = _name(self.xr.metadata) + resource.update( + self.rsp.desired.resources[_MANAGEMENT_SERVER_KEY], + self._server(f"{name}-management", self._management_plan(), self._management_os_id()), + ) + for pool in self.xr.spec.nodePools: + os_id = pool.osId if pool.osId else self._management_os_id() + for i in range(pool.nodeCount or 1): + resource.update( + self.rsp.desired.resources[f"server-{pool.name}-{i}"], + self._server(f"{name}-{pool.name}-{i}", pool.plan, os_id), + ) + + def _server(self, label: str, plan: str, os_id: int) -> bmv1beta1.BareMetalServer: + return bmv1beta1.BareMetalServer( + spec=bmv1beta1.Spec( + providerConfigRef=bmv1beta1.ProviderConfigRef( + kind=self._cred_kind(), + name=self._cred_name(), + ), + forProvider=bmv1beta1.ForProvider( + label=label, + hostname=label, + plan=plan, + region=self.xr.spec.region, + osId=os_id, + sshKeyIdsSelector=bmv1beta1.SshKeyIdsSelector(matchControllerRef=True), + tags=[f"modelplane.ai/cluster={_name(self.xr.metadata)}"], + userData=_USER_DATA, + ), + ), + ) + + def _server_ip(self, key: str) -> str | None: + """The observed main IP of an active server, or None while it is + still provisioning.""" + observed = self.req.observed.resources.get(key) + if observed is None: + return None + server = bmv1beta1.BareMetalServer.model_validate(resource.struct_to_dict(observed.resource)) + at = server.status.atProvider if server.status else None + if at is None or at.status != _SERVER_STATUS_ACTIVE or not at.mainIp: + return None + return at.mainIp + + def compose_k3s_cluster(self) -> None: + """Compose the K3sCluster once every server is active and reports + its main IP. The management server becomes the k3s server; each GPU + server joins as an agent labelled for its pool and tainted for GPU + workloads.""" + management_ip = self._server_ip(_MANAGEMENT_SERVER_KEY) + if management_ip is None: + return + + workers: list[k3sv1alpha1.Worker] = [] + for pool in self.xr.spec.nodePools: + for i in range(pool.nodeCount or 1): + ip = self._server_ip(f"server-{pool.name}-{i}") + if ip is None: + return + workers.append( + k3sv1alpha1.Worker( + name=f"{pool.name}-{i}", + host=ip, + labels={ + _LABEL_POOL: pool.name, + _LABEL_GPU: pool.gpu.acceleratorType, + }, + taints=[ + k3sv1alpha1.Taint( + key=_gpu_taint_key(pool.gpu.acceleratorType), + value=_GPU_TAINT_VALUE, + effect=_GPU_TAINT_EFFECT, + ), + ], + ), + ) + + ssh = self.xr.spec.ssh + resource.update( + self.rsp.desired.resources["k3s-cluster"], + k3sv1alpha1.K3sCluster( + spec=k3sv1alpha1.Spec( + controlPlane=k3sv1alpha1.ControlPlane(host=management_ip), + workers=workers, + auth=k3sv1alpha1.Auth( + username=ssh.username or _DEFAULT_USERNAME, + secretRef=k3sv1alpha1.SecretRef( + name=ssh.secretRef.name, + key=ssh.secretRef.privateKeyKey or _DEFAULT_PRIVATE_KEY_KEY, + ), + ), + version=k3sv1alpha1.Version(channel=self._channel()), + ), + ), + ) + + def write_status(self) -> None: + """Relay the K3sCluster's published secrets. The kubeconfig secret + name derives from the K3sCluster's generated name, so it is read + from observation rather than derived here.""" + observed = self.req.observed.resources.get("k3s-cluster") + if observed is None: + return + k3s = k3sv1alpha1.K3sCluster.model_validate(resource.struct_to_dict(observed.resource)) + if not k3s.status or not k3s.status.secrets: + return + resource.update_status( + self.rsp.desired.composite, + v1alpha1.Status( + secrets=[v1alpha1.Secret(type=s.type, name=s.name, key=s.key) for s in k3s.status.secrets], + ), + ) + + def mark_readiness(self) -> None: + """Mark composed resources as ready based on their observed Ready + conditions. The XR is Ready only once the SSHKey and every server + are Ready and the K3sCluster reports the whole cluster up.""" + for r in self.rsp.desired.resources: + if resource.get_condition(self.req.observed.resources.get(r), "Ready").status == "True": + self.rsp.desired.resources[r].ready = fnv1.READY_TRUE diff --git a/functions/compose-vultr-baremetal-cluster/function/main.py b/functions/compose-vultr-baremetal-cluster/function/main.py new file mode 100644 index 000000000..2e8441dac --- /dev/null +++ b/functions/compose-vultr-baremetal-cluster/function/main.py @@ -0,0 +1,55 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The composition function's main CLI.""" + +import click +from crossplane.function import logging, runtime + +from function import fn + + +@click.command() +@click.option("--debug", "-d", is_flag=True, help="Emit debug logs.") +@click.option( + "--address", + default="0.0.0.0:9443", + show_default=True, + help="Address at which to listen for gRPC connections", +) +@click.option("--tls-certs-dir", help="Serve using mTLS certificates.", envvar="TLS_SERVER_CERTS_DIR") +@click.option( + "--insecure", + is_flag=True, + help="Run without mTLS credentials. If you supply this flag --tls-certs-dir will be ignored.", +) +def cli(debug: bool, address: str, tls_certs_dir: str, insecure: bool) -> None: + """A Crossplane composition function.""" + try: + level = logging.Level.INFO + if debug: + level = logging.Level.DEBUG + logging.configure(level=level) + runtime.serve( + fn.FunctionRunner(), + address, + creds=runtime.load_credentials(tls_certs_dir), + insecure=insecure, + ) + except Exception as e: + click.echo(f"Cannot run function: {e}") + + +if __name__ == "__main__": + cli() diff --git a/functions/compose-vultr-baremetal-cluster/pyproject.toml b/functions/compose-vultr-baremetal-cluster/pyproject.toml new file mode 100644 index 000000000..c8f261038 --- /dev/null +++ b/functions/compose-vultr-baremetal-cluster/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["uv_build>=0.11.0,<0.12"] +build-backend = "uv_build" + +[project] +name = "compose-vultr-baremetal-cluster" +version = "0.0.0" +description = "Compose Vultr bare metal servers into a k3s cluster." +requires-python = ">=3.11,<3.14" +license = "Apache-2.0" +dependencies = [ + "crossplane-function-sdk-python>=0.14.0", + "click>=8.1.0", + "grpcio>=1.73.1", + "crossplane-models", +] + +[tool.uv.sources] +crossplane-models = { workspace = true } + +[project.scripts] +function = "function.main:cli" + +[tool.uv.build-backend] +module-name = "function" +module-root = "" diff --git a/functions/compose-vultr-baremetal-cluster/tests/test_fn.py b/functions/compose-vultr-baremetal-cluster/tests/test_fn.py new file mode 100644 index 000000000..65aeef62e --- /dev/null +++ b/functions/compose-vultr-baremetal-cluster/tests/test_fn.py @@ -0,0 +1,531 @@ +# Copyright 2026 The Modelplane Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the compose-vultr-baremetal-cluster function.""" + +import base64 +import dataclasses +import unittest + +from crossplane.function import logging, resource +from crossplane.function.proto.v1 import run_function_pb2 as fnv1 +from function import fn +from google.protobuf import duration_pb2 as durationpb +from google.protobuf import json_format +from google.protobuf import struct_pb2 as structpb +from models.ai.modelplane.infrastructure.vultrbaremetalcluster import v1alpha1 +from models.io.k8s.apimachinery.pkg.apis.meta import v1 as metav1 + + +@dataclasses.dataclass +class Case: + """A test case for compose-vultr-baremetal-cluster.""" + + name: str + req: fnv1.RunFunctionRequest + want: fnv1.RunFunctionResponse + + +def setUpModule() -> None: + logging.configure(level=logging.Level.DISABLED) + + +_PUBLIC_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAA test@modelplane" + +# The GPU pool used across cases. +_GPU_POOL = v1alpha1.NodePool( + name="gpu", + plan="vbm-256c-3072gb-8-mi355x-gpu", + gpu=v1alpha1.Gpu(acceleratorType="amd-mi355x"), +) + + +def _xr(pools: list[v1alpha1.NodePool] | None = None) -> dict: + """A VultrBaremetalCluster XR as a request dict.""" + return v1alpha1.VultrBaremetalCluster( + metadata=metav1.ObjectMeta( + name="test-cluster", + namespace="modelplane-system", + ), + spec=v1alpha1.Spec( + region="ord", + ssh=v1alpha1.Ssh(secretRef=v1alpha1.SecretRef(name="test-ssh")), + nodePools=pools if pools is not None else [_GPU_POOL], + ), + ).model_dump(exclude_none=True, mode="json") + + +def _ssh_secret() -> fnv1.Resource: + """The observed SSH key pair Secret.""" + return fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "test-ssh", "namespace": "modelplane-system"}, + "data": { + "ssh-publickey": base64.b64encode(f"{_PUBLIC_KEY}\n".encode()).decode(), + "ssh-privatekey": base64.b64encode(b"private").decode(), + }, + }, + ), + ) + + +def _req( + observed_resources: dict[str, fnv1.Resource] | None = None, + *, + pools: list[v1alpha1.NodePool] | None = None, + with_secret: bool = True, + secret_data: dict[str, str] | None = None, +) -> fnv1.RunFunctionRequest: + req = fnv1.RunFunctionRequest( + observed=fnv1.State( + composite=fnv1.Resource(resource=resource.dict_to_struct(_xr(pools))), + resources=observed_resources or {}, + ), + ) + if with_secret: + secret = _ssh_secret() + if secret_data is not None: + secret = fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": {"name": "test-ssh", "namespace": "modelplane-system"}, + "data": secret_data, + }, + ), + ) + req.required_resources["ssh-secret"].items.append(secret) + return req + + +def _ssh_selector() -> fnv1.ResourceSelector: + """The requirement declared for the SSH key pair Secret.""" + return fnv1.ResourceSelector( + api_version="v1", + kind="Secret", + match_name="test-ssh", + namespace="modelplane-system", + ) + + +def _want( + resources: dict[str, fnv1.Resource], + composite: fnv1.Resource | None = None, +) -> fnv1.RunFunctionResponse: + want = fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State(composite=composite, resources=resources), + context=structpb.Struct(), + ) + want.requirements.resources["ssh-secret"].CopyFrom(_ssh_selector()) + return want + + +def _ssh_key() -> dict: + """An SSHKey golden registering the public key with Vultr.""" + return { + "apiVersion": "compute.vultr.m.upbound.io/v1beta1", + "kind": "SSHKey", + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, + "forProvider": { + "name": "test-cluster", + "sshKey": _PUBLIC_KEY, + }, + }, + } + + +def _server(label: str, plan: str) -> dict: + """A BareMetalServer golden.""" + return { + "apiVersion": "compute.vultr.m.upbound.io/v1beta1", + "kind": "BareMetalServer", + "spec": { + "providerConfigRef": {"kind": "ClusterProviderConfig", "name": "default"}, + "forProvider": { + "label": label, + "hostname": label, + "plan": plan, + "region": "ord", + "osId": 2284, + "sshKeyIdsSelector": {"matchControllerRef": True}, + "tags": ["modelplane.ai/cluster=test-cluster"], + "userData": fn._USER_DATA, + }, + }, + } + + +_MANAGEMENT_SERVER = _server("test-cluster-management", "vbm-6c-32gb-amd") +_GPU_SERVER = _server("test-cluster-gpu-0", "vbm-256c-3072gb-8-mi355x-gpu") + + +def _k3s_cluster() -> dict: + """A K3sCluster golden built from the servers' observed IPs.""" + return { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "K3sCluster", + "spec": { + "controlPlane": {"host": "203.0.113.10"}, + "workers": [ + { + "name": "gpu-0", + "host": "203.0.113.20", + "labels": { + "modelplane.ai/pool": "gpu", + "modelplane.ai/gpu": "amd-mi355x", + }, + "taints": [ + {"key": "amd.com/gpu", "value": "true", "effect": "NoSchedule"}, + ], + }, + ], + "auth": { + "username": "root", + "secretRef": {"name": "test-ssh", "key": "ssh-privatekey"}, + }, + "version": {"channel": "v1.34"}, + }, + } + + +def _observed_active(desired: dict, main_ip: str) -> fnv1.Resource: + """An observed server that is active with a main IP and Ready.""" + observed = { + **desired, + "status": { + "atProvider": {"mainIp": main_ip, "status": "active"}, + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + return fnv1.Resource(resource=resource.dict_to_struct(observed)) + + +def _observed_provisioning(desired: dict) -> fnv1.Resource: + """An observed server that is still provisioning: no IP, not Ready.""" + observed = { + **desired, + "status": { + "atProvider": {"status": "pending"}, + "conditions": [ + { + "type": "Ready", + "status": "False", + "reason": "Creating", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + return fnv1.Resource(resource=resource.dict_to_struct(observed)) + + +def _observed_ready(desired: dict) -> fnv1.Resource: + """An observed variant of a desired resource with a Ready=True condition.""" + observed = { + **desired, + "status": { + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + return fnv1.Resource(resource=resource.dict_to_struct(observed)) + + +def _observed_k3s_with_secrets(desired: dict) -> fnv1.Resource: + """An observed K3sCluster that is Ready and publishes its kubeconfig.""" + observed = { + **desired, + "status": { + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-abc12-kubeconfig-d34db", + "key": "kubeconfig", + }, + ], + "conditions": [ + { + "type": "Ready", + "status": "True", + "reason": "Available", + "lastTransitionTime": "2024-01-01T00:00:00Z", + }, + ], + }, + } + return fnv1.Resource(resource=resource.dict_to_struct(observed)) + + +class TestFunctionRunner(unittest.IsolatedAsyncioTestCase): + """Tests for FunctionRunner.RunFunction.""" + + maxDiff = None + + @classmethod + def setUpClass(cls) -> None: + cls.runner = fn.FunctionRunner() + + async def test_compose(self) -> None: + """The function composes bare metal servers into a k3s cluster.""" + cases = [ + Case( + name="nothing composed until the SSH key Secret resolves", + req=_req(with_secret=False), + want=self._waiting_want( + "Waiting for SSH key Secret test-ssh", + fnv1.Result( + severity=fnv1.SEVERITY_NORMAL, + message="Waiting for SSH key Secret test-ssh", + ), + ), + ), + Case( + name="nothing composed when the Secret lacks the public key", + req=_req(secret_data={"ssh-privatekey": base64.b64encode(b"private").decode()}), + want=self._waiting_want( + "SSH key Secret test-ssh has no ssh-publickey key", + fnv1.Result( + severity=fnv1.SEVERITY_WARNING, + message="SSH key Secret test-ssh has no ssh-publickey key", + ), + ), + ), + Case( + name="servers composed; K3sCluster withheld until every server is active", + req=_req(), + want=_want( + { + "ssh-key": fnv1.Resource(resource=resource.dict_to_struct(_ssh_key())), + "server-management": fnv1.Resource(resource=resource.dict_to_struct(_MANAGEMENT_SERVER)), + "server-gpu-0": fnv1.Resource(resource=resource.dict_to_struct(_GPU_SERVER)), + }, + ), + ), + Case( + name="K3sCluster withheld while a GPU server is still provisioning", + req=_req( + { + "ssh-key": _observed_ready(_ssh_key()), + "server-management": _observed_active(_MANAGEMENT_SERVER, "203.0.113.10"), + "server-gpu-0": _observed_provisioning(_GPU_SERVER), + }, + ), + want=_want( + { + "ssh-key": fnv1.Resource( + resource=resource.dict_to_struct(_ssh_key()), + ready=fnv1.READY_TRUE, + ), + "server-management": fnv1.Resource( + resource=resource.dict_to_struct(_MANAGEMENT_SERVER), + ready=fnv1.READY_TRUE, + ), + "server-gpu-0": fnv1.Resource(resource=resource.dict_to_struct(_GPU_SERVER)), + }, + ), + ), + Case( + name="K3sCluster composed from the servers' IPs once all are active", + req=_req( + { + "ssh-key": _observed_ready(_ssh_key()), + "server-management": _observed_active(_MANAGEMENT_SERVER, "203.0.113.10"), + "server-gpu-0": _observed_active(_GPU_SERVER, "203.0.113.20"), + }, + ), + want=_want( + { + "ssh-key": fnv1.Resource( + resource=resource.dict_to_struct(_ssh_key()), + ready=fnv1.READY_TRUE, + ), + "server-management": fnv1.Resource( + resource=resource.dict_to_struct(_MANAGEMENT_SERVER), + ready=fnv1.READY_TRUE, + ), + "server-gpu-0": fnv1.Resource( + resource=resource.dict_to_struct(_GPU_SERVER), + ready=fnv1.READY_TRUE, + ), + "k3s-cluster": fnv1.Resource(resource=resource.dict_to_struct(_k3s_cluster())), + }, + ), + ), + Case( + name="NVIDIA pool workers carry the nvidia.com/gpu taint", + req=_req( + { + "ssh-key": _observed_ready(_ssh_key()), + "server-management": _observed_active(_MANAGEMENT_SERVER, "203.0.113.10"), + "server-h100-0": _observed_active( + _server("test-cluster-h100-0", "vbm-64c-2048gb-8-h100-gpu"), + "203.0.113.30", + ), + }, + pools=[ + v1alpha1.NodePool( + name="h100", + plan="vbm-64c-2048gb-8-h100-gpu", + gpu=v1alpha1.Gpu(acceleratorType="nvidia-h100"), + ), + ], + ), + want=_want( + { + "ssh-key": fnv1.Resource( + resource=resource.dict_to_struct(_ssh_key()), + ready=fnv1.READY_TRUE, + ), + "server-management": fnv1.Resource( + resource=resource.dict_to_struct(_MANAGEMENT_SERVER), + ready=fnv1.READY_TRUE, + ), + "server-h100-0": fnv1.Resource( + resource=resource.dict_to_struct( + _server("test-cluster-h100-0", "vbm-64c-2048gb-8-h100-gpu"), + ), + ready=fnv1.READY_TRUE, + ), + "k3s-cluster": fnv1.Resource( + resource=resource.dict_to_struct( + { + "apiVersion": "infrastructure.modelplane.ai/v1alpha1", + "kind": "K3sCluster", + "spec": { + "controlPlane": {"host": "203.0.113.10"}, + "workers": [ + { + "name": "h100-0", + "host": "203.0.113.30", + "labels": { + "modelplane.ai/pool": "h100", + "modelplane.ai/gpu": "nvidia-h100", + }, + "taints": [ + { + "key": "nvidia.com/gpu", + "value": "true", + "effect": "NoSchedule", + }, + ], + }, + ], + "auth": { + "username": "root", + "secretRef": {"name": "test-ssh", "key": "ssh-privatekey"}, + }, + "version": {"channel": "v1.34"}, + }, + }, + ), + ), + }, + ), + ), + Case( + name="kubeconfig relayed once the K3sCluster publishes it", + req=_req( + { + "ssh-key": _observed_ready(_ssh_key()), + "server-management": _observed_active(_MANAGEMENT_SERVER, "203.0.113.10"), + "server-gpu-0": _observed_active(_GPU_SERVER, "203.0.113.20"), + "k3s-cluster": _observed_k3s_with_secrets(_k3s_cluster()), + }, + ), + want=_want( + { + "ssh-key": fnv1.Resource( + resource=resource.dict_to_struct(_ssh_key()), + ready=fnv1.READY_TRUE, + ), + "server-management": fnv1.Resource( + resource=resource.dict_to_struct(_MANAGEMENT_SERVER), + ready=fnv1.READY_TRUE, + ), + "server-gpu-0": fnv1.Resource( + resource=resource.dict_to_struct(_GPU_SERVER), + ready=fnv1.READY_TRUE, + ), + "k3s-cluster": fnv1.Resource( + resource=resource.dict_to_struct(_k3s_cluster()), + ready=fnv1.READY_TRUE, + ), + }, + composite=fnv1.Resource( + resource=resource.dict_to_struct( + { + "status": { + "secrets": [ + { + "type": "Kubeconfig", + "name": "test-cluster-abc12-kubeconfig-d34db", + "key": "kubeconfig", + }, + ], + }, + }, + ), + ), + ), + ), + ] + + for case in cases: + with self.subTest(case.name): + got = await self.runner.RunFunction(case.req, None) + self.assertEqual( + json_format.MessageToDict(case.want), + json_format.MessageToDict(got), + "-want, +got", + ) + + @staticmethod + def _waiting_want(message: str, result: fnv1.Result) -> fnv1.RunFunctionResponse: + """A response that only declares the Secret requirement and reports + why nothing was composed.""" + want = fnv1.RunFunctionResponse( + meta=fnv1.ResponseMeta(ttl=durationpb.Duration(seconds=60)), + desired=fnv1.State(), + conditions=[ + fnv1.Condition( + type="ClusterReady", + status=fnv1.STATUS_CONDITION_FALSE, + reason="WaitingForSSHSecret", + message=message, + ), + ], + results=[result], + context=structpb.Struct(), + ) + want.requirements.resources["ssh-secret"].CopyFrom(_ssh_selector()) + return want diff --git a/schemas/.lock.json b/schemas/.lock.json index 3f61a1f51..62637004b 100644 --- a/schemas/.lock.json +++ b/schemas/.lock.json @@ -1,7 +1,8 @@ { "packages": { - "fs://apis": "f0cff8a93ff8ea443b4004428e557a5e9a04107d149aae4bb64779402ddcffda", + "fs://apis": "d46a8e0c15978bb7efe447f51f30037b8c90d0ba78d1702cb8fb13626b200995", "git://https://github.com/crossplane/crossplane/cluster/crds": "90d8b72ad8b829f0bcd7d7d5a98eaa0d579f244a", + "xpkg://xpkg.upbound.io/crossplane-contrib/provider-k3s:v0.4.0": "sha256:896cdf265f92e5a3d3484d8b1897f9cf015cabb768da7fef13f4f003ae9b13b2", "xpkg://xpkg.upbound.io/upbound/provider-aws-ec2:v2.6.0": "sha256:acc26c8d2710e0306185b6c626a2f8c8fe0fdf89874e85efe7944a3668322865", "xpkg://xpkg.upbound.io/upbound/provider-aws-efs:v2.6.0": "sha256:00f1bbbb3c0f1948b6dd45c841a083d63e41850f5a559a15580915311f404911", "xpkg://xpkg.upbound.io/upbound/provider-aws-eks:v2.6.0": "sha256:5d144b19e188cb96c918aa7e4ccbc6759b8733ff09bd8ce412723c669aa3f763", diff --git a/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py b/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py index 208c6ecd5..fcac7bbe9 100644 --- a/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferenceclass/v1alpha1.py @@ -194,13 +194,33 @@ class Vultr(BaseModel): """ +class AcceleratorModel4(BaseModel): + count: conint(ge=1, le=16) + type: constr(min_length=1, max_length=63) + """ + GPU accelerator type (e.g. amd-mi355x, nvidia-h100). Reported on the consuming InferenceCluster's status. + """ + + +class VultrBaremetal(BaseModel): + accelerator: AcceleratorModel4 + """ + GPU accelerator the plan carries. Provisioning input only: the scheduler matches against spec.devices, not this block. The type's vendor prefix (amd-, nvidia-) selects the GPU node taint and labels. + """ + plan: constr(min_length=1, max_length=63) + """ + Vultr bare metal plan ID (e.g. vbm-256c-3072gb-8-mi355x-gpu). The plan determines the GPU model and count; the accelerator block below is informational. + """ + + class Provisioning(BaseModel): aks: Aks | None = None eks: Eks | None = None gke: Gke | None = None nebius: Nebius | None = None - provider: Literal['GKE', 'EKS', 'AKS', 'Nebius', 'Vultr'] + provider: Literal['GKE', 'EKS', 'AKS', 'Nebius', 'Vultr', 'VultrBaremetal'] vultr: Vultr | None = None + vultrBaremetal: VultrBaremetal | None = None class Spec(BaseModel): diff --git a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py index 122f93cc8..0dd4c037d 100644 --- a/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/inferencecluster/v1alpha1.py @@ -125,6 +125,70 @@ class Vultr(BaseModel): """ +class K3s(BaseModel): + channel: constr(min_length=1, max_length=32) | None = 'v1.34' + """ + k3s release channel. Defaults to the first channel where Dynamic Resource Allocation (how GPUs bind to pods) is generally available. + """ + + +class Management(BaseModel): + osId: int | None = 2284 + """ + Vultr operating system ID installed on every server. Defaults to Ubuntu 24.04 LTS x64; list IDs with vultr-cli os list. + """ + plan: constr(min_length=1, max_length=63) | None = 'vbm-6c-32gb-amd' + """ + Vultr bare metal plan for the management server. The default is a CPU-only plan available in most regions that offer bare metal. + """ + + +class SecretRefModel(BaseModel): + name: constr(min_length=1, max_length=253) + privateKeyKey: constr(min_length=1, max_length=253) | None = 'ssh-privatekey' + """ + Key within the Secret that holds the private key. + """ + publicKeyKey: constr(min_length=1, max_length=253) | None = 'ssh-publickey' + """ + Key within the Secret that holds the public key. + """ + + +class Ssh(BaseModel): + secretRef: SecretRefModel + """ + Secret holding the SSH key pair. The Secret must exist in the modelplane-system namespace. + """ + username: constr(min_length=1, max_length=63) | None = 'root' + """ + SSH user the servers accept the key for. Vultr installs keys for root by default. + """ + + +class VultrBaremetal(BaseModel): + credentials: Credentials | None = None + """ + Vultr ProviderConfig or ClusterProviderConfig used to authenticate to the Vultr API. Defaults to the ClusterProviderConfig named default. + """ + k3s: K3s | None = Field({}, validate_default=True) + """ + The k3s release installed on the servers. + """ + management: Management | None = Field({}, validate_default=True) + """ + The CPU-only bare metal server that runs the k3s server (the management plane). + """ + region: constr(min_length=1, max_length=32) + """ + Vultr region for all servers (e.g. ewr, ord). Bare metal plan availability varies by region; check with vultr-cli plans list --type vbm. + """ + ssh: Ssh + """ + SSH key pair used to reach the servers. The public key is registered with Vultr and installed on every server; the private key drives the k3s install over SSH. + """ + + class Cluster(BaseModel): aks: Aks | None = None """ @@ -146,7 +210,9 @@ class Cluster(BaseModel): """ Nebius mk8s cluster configuration. Required when source is Nebius; may be empty, since every field has a default. The cluster is created in the project the referenced ProviderConfig or ClusterProviderConfig sets as its projectID; Nebius projects are bound to a region, so the project also determines where the cluster runs. """ - source: Literal['GKE', 'EKS', 'AKS', 'Nebius', 'Vultr', 'Existing'] + source: Literal[ + 'GKE', 'EKS', 'AKS', 'Nebius', 'Vultr', 'VultrBaremetal', 'Existing' + ] """ Cluster provisioning method. """ @@ -154,6 +220,10 @@ class Cluster(BaseModel): """ Vultr Kubernetes Engine (VKE) cluster configuration. Required when source is Vultr. """ + vultrBaremetal: VultrBaremetal | None = None + """ + Vultr bare metal (k3s) cluster configuration. Required when source is VultrBaremetal. Provisions bare metal servers - one CPU-only management server plus the GPU pools - and installs a k3s cluster onto them over SSH. Bare metal has no autoscaling, so pools are fixed size, and provisioning takes tens of minutes. + """ class CompositionRef(BaseModel): diff --git a/schemas/python/models/ai/modelplane/infrastructure/k3scluster/__init__.py b/schemas/python/models/ai/modelplane/infrastructure/k3scluster/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/ai/modelplane/infrastructure/k3scluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/k3scluster/v1alpha1.py new file mode 100644 index 000000000..1cbb5b92e --- /dev/null +++ b/schemas/python/models/ai/modelplane/infrastructure/k3scluster/v1alpha1.py @@ -0,0 +1,216 @@ +# generated by datamodel-codegen: +# filename: workdir/infrastructure_modelplane_ai_v1alpha1_k3scluster.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel, Field, conint, constr + +from .....io.k8s.apimachinery.pkg.apis.meta import v1 + + +class SecretRef(BaseModel): + key: constr(min_length=1, max_length=253) | None = 'ssh-privatekey' + """ + Key within the Secret that holds the private key. + """ + name: constr(min_length=1, max_length=253) + """ + Name of the Secret. + """ + + +class Auth(BaseModel): + secretRef: SecretRef + """ + Secret holding the SSH private key, in the same namespace as this K3sCluster. + """ + username: constr(min_length=1, max_length=63) | None = 'root' + """ + SSH user. Must be root or have passwordless sudo. + """ + + +class ControlPlane(BaseModel): + host: constr(min_length=1, max_length=253) + """ + DNS name or IP address of the machine. + """ + port: conint(ge=1, le=65535) | None = 22 + """ + SSH port. + """ + + +class CompositionRef(BaseModel): + name: str + + +class CompositionRevisionRef(BaseModel): + name: str + + +class CompositionRevisionSelector(BaseModel): + matchLabels: dict[str, str] + + +class CompositionSelector(BaseModel): + matchLabels: dict[str, str] + + +class ResourceRef(BaseModel): + apiVersion: str + kind: str + name: str | None = None + + +class Crossplane(BaseModel): + compositionRef: CompositionRef | None = None + compositionRevisionRef: CompositionRevisionRef | None = None + compositionRevisionSelector: CompositionRevisionSelector | None = None + compositionSelector: CompositionSelector | None = None + compositionUpdatePolicy: Literal['Automatic', 'Manual'] | None = None + resourceRefs: list[ResourceRef] | None = None + + +class Version(BaseModel): + channel: constr(min_length=1, max_length=32) | None = None + """ + k3s release channel (e.g. stable, v1.34). Installs the channel's latest release. + """ + version: constr(min_length=1, max_length=32) | None = None + """ + Exact k3s version to install (e.g. v1.34.1+k3s1). + """ + + +class Taint(BaseModel): + effect: Literal['NoSchedule', 'PreferNoSchedule', 'NoExecute'] + key: constr(min_length=1, max_length=253) + value: constr(max_length=63) | None = None + + +class Worker(BaseModel): + host: constr(min_length=1, max_length=253) + """ + DNS name or IP address of the machine. + """ + labels: dict[str, str] | None = None + """ + Node labels applied to this worker. + """ + name: constr(min_length=1, max_length=63) + """ + Unique name for this worker. + """ + port: conint(ge=1, le=65535) | None = 22 + """ + SSH port. + """ + taints: list[Taint] | None = Field(None, max_length=8) + """ + Node taints applied to this worker. + """ + + +class Spec(BaseModel): + auth: Auth + """ + SSH authentication used to reach every machine. All machines must accept the same user and private key. + """ + controlPlane: ControlPlane + """ + The machine that runs the k3s server (the management plane). Must be reachable over SSH from the control plane running Modelplane. + """ + crossplane: Crossplane | None = None + """ + Configures how Crossplane will reconcile this composite resource + """ + version: Version | None = None + """ + The k3s release to install. Defaults to the v1.34 channel, the first where Dynamic Resource Allocation (how GPUs bind to pods) is generally available. + """ + workers: list[Worker] | None = Field(None, max_length=64) + """ + Machines that join the cluster as k3s agents (the worker plane). Labels and taints are applied at registration time via k3s agent arguments. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + message: str | None = None + observedGeneration: int | None = None + reason: str + status: str + type: str + + +class Secret(BaseModel): + key: constr(max_length=253) + """ + Key within the Secret that holds the credential data. + """ + name: constr(max_length=253) + """ + Name of the Secret. + """ + type: Literal['Kubeconfig'] = 'Kubeconfig' + """ + The type of credential this secret contains. Kubeconfig contains a kubeconfig file with the cluster endpoint, CA certificate, and a static client certificate. + """ + + +class Status(BaseModel): + conditions: list[Condition] | None = None + """ + Conditions of the resource. + """ + secrets: list[Secret] | None = None + """ + Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this K3sCluster. + """ + + +class K3sCluster(BaseModel): + apiVersion: Literal['infrastructure.modelplane.ai/v1alpha1'] | None = ( + 'infrastructure.modelplane.ai/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['K3sCluster'] | None = 'K3sCluster' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + K3sClusterSpec defines the desired state of K3sCluster. + """ + status: Status | None = None + """ + K3sClusterStatus defines the observed state of K3sCluster. + """ + + +class K3sClusterList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[K3sCluster] + """ + List of k3sclusters. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py index 83db695a5..d6b931527 100644 --- a/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py +++ b/schemas/python/models/ai/modelplane/infrastructure/servingstack/v1alpha1.py @@ -92,7 +92,11 @@ class Secret(BaseModel): class Spec(BaseModel): - cloud: Literal['GKE', 'EKS', 'AKS', 'Nebius', 'Vultr', 'Existing'] + accelerators: list[Literal['AMD', 'NVIDIA']] | None = Field(None, max_length=2) + """ + Accelerator vendors present in the target cluster. Filters the cloud's vendor-tagged components: only the device stacks for the listed vendors are installed. When omitted, no vendor filtering happens and every component installs. Derived from the InferenceClasses by the cluster composition; only clouds whose component lists carry vendor tags (VultrBaremetal) are affected. + """ + cloud: Literal['GKE', 'EKS', 'AKS', 'Nebius', 'Vultr', 'VultrBaremetal', 'Existing'] """ The cloud the target cluster runs on. Selects the fixed set of components and versions this stack installs there, which is resolved per cloud at build time and changes only with a Modelplane release. Mirrors InferenceCluster.spec.cluster.source; the cluster composition sets it. """ diff --git a/schemas/python/models/ai/modelplane/infrastructure/vultrbaremetalcluster/__init__.py b/schemas/python/models/ai/modelplane/infrastructure/vultrbaremetalcluster/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/ai/modelplane/infrastructure/vultrbaremetalcluster/v1alpha1.py b/schemas/python/models/ai/modelplane/infrastructure/vultrbaremetalcluster/v1alpha1.py new file mode 100644 index 000000000..0f4edc70c --- /dev/null +++ b/schemas/python/models/ai/modelplane/infrastructure/vultrbaremetalcluster/v1alpha1.py @@ -0,0 +1,232 @@ +# generated by datamodel-codegen: +# filename: workdir/infrastructure_modelplane_ai_v1alpha1_vultrbaremetalcluster.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel, Field, conint, constr + +from .....io.k8s.apimachinery.pkg.apis.meta import v1 + + +class Credentials(BaseModel): + name: constr(min_length=1, max_length=253) | None = 'default' + type: Literal['ProviderConfig', 'ClusterProviderConfig'] | None = ( + 'ClusterProviderConfig' + ) + + +class CompositionRef(BaseModel): + name: str + + +class CompositionRevisionRef(BaseModel): + name: str + + +class CompositionRevisionSelector(BaseModel): + matchLabels: dict[str, str] + + +class CompositionSelector(BaseModel): + matchLabels: dict[str, str] + + +class ResourceRef(BaseModel): + apiVersion: str + kind: str + name: str | None = None + + +class Crossplane(BaseModel): + compositionRef: CompositionRef | None = None + compositionRevisionRef: CompositionRevisionRef | None = None + compositionRevisionSelector: CompositionRevisionSelector | None = None + compositionSelector: CompositionSelector | None = None + compositionUpdatePolicy: Literal['Automatic', 'Manual'] | None = None + resourceRefs: list[ResourceRef] | None = None + + +class K3s(BaseModel): + channel: constr(min_length=1, max_length=32) | None = 'v1.34' + """ + k3s release channel. Defaults to the first channel where Dynamic Resource Allocation (how GPUs bind to pods) is generally available. + """ + + +class Management(BaseModel): + osId: int | None = 2284 + """ + Vultr operating system ID installed on the server. Defaults to Ubuntu 24.04 LTS x64; list IDs with vultr-cli os list. + """ + plan: constr(min_length=1, max_length=63) | None = 'vbm-6c-32gb-amd' + """ + Vultr bare metal plan for the management server. The default is a CPU-only plan available in most regions that offer bare metal. + """ + + +class Gpu(BaseModel): + acceleratorType: constr(min_length=1, max_length=63) + """ + GPU accelerator type (e.g. amd-mi355x). Used to label the pool's nodes; the actual GPU and count are determined by the plan. + """ + + +class NodePool(BaseModel): + gpu: Gpu + """ + GPU configuration. + """ + name: constr(min_length=1, max_length=40) + """ + Unique name for this pool. + """ + nodeCount: conint(ge=1, le=64) | None = 1 + """ + Number of servers in this pool. + """ + osId: int | None = None + """ + Vultr operating system ID for the pool's servers. Defaults to the management server's osId. + """ + plan: constr(min_length=1, max_length=63) + """ + Vultr bare metal plan ID for the pool's servers (e.g. vbm-256c-3072gb-8-mi355x-gpu). + """ + + +class SecretRef(BaseModel): + name: constr(min_length=1, max_length=253) + """ + Name of the Secret. + """ + privateKeyKey: constr(min_length=1, max_length=253) | None = 'ssh-privatekey' + """ + Key within the Secret that holds the private key. + """ + publicKeyKey: constr(min_length=1, max_length=253) | None = 'ssh-publickey' + """ + Key within the Secret that holds the public key. + """ + + +class Ssh(BaseModel): + secretRef: SecretRef + """ + Secret holding the SSH key pair, in the same namespace as this VultrBaremetalCluster. + """ + username: constr(min_length=1, max_length=63) | None = 'root' + """ + SSH user the servers accept the key for. Vultr installs keys for root by default. + """ + + +class Spec(BaseModel): + credentials: Credentials | None = None + """ + Vultr ProviderConfig or ClusterProviderConfig used to authenticate to the Vultr API. Defaults to the ClusterProviderConfig named default. + """ + crossplane: Crossplane | None = None + """ + Configures how Crossplane will reconcile this composite resource + """ + k3s: K3s | None = Field({}, validate_default=True) + """ + The k3s release installed on the servers. + """ + management: Management | None = Field({}, validate_default=True) + """ + The CPU-only bare metal server that runs the k3s server (the management plane). + """ + nodePools: list[NodePool] = Field(..., max_length=8, min_length=1) + """ + GPU bare metal pools that join the cluster as workers. Fixed size; Vultr bare metal has no autoscaling. + """ + region: constr(min_length=1, max_length=32) + """ + Vultr region for all servers (e.g. ewr, ord). Bare metal plan availability varies by region; check with vultr-cli plans list --type vbm. + """ + ssh: Ssh + """ + SSH key pair used to reach the servers. The public key is registered with Vultr and installed on every server; the private key drives the k3s install over SSH. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + message: str | None = None + observedGeneration: int | None = None + reason: str + status: str + type: str + + +class Secret(BaseModel): + key: constr(max_length=253) + """ + Key within the Secret that holds the credential data. + """ + name: constr(max_length=253) + """ + Name of the Secret. + """ + type: Literal['Kubeconfig'] = 'Kubeconfig' + """ + The type of credential this secret contains. Kubeconfig contains a kubeconfig file with the cluster endpoint, CA certificate, and a static client certificate. + """ + + +class Status(BaseModel): + conditions: list[Condition] | None = None + """ + Conditions of the resource. + """ + secrets: list[Secret] | None = None + """ + Secrets produced by this cluster. Consumers use these to authenticate to the cluster. All secrets are in the same namespace as this VultrBaremetalCluster. + """ + + +class VultrBaremetalCluster(BaseModel): + apiVersion: Literal['infrastructure.modelplane.ai/v1alpha1'] | None = ( + 'infrastructure.modelplane.ai/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['VultrBaremetalCluster'] | None = 'VultrBaremetalCluster' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + VultrBaremetalClusterSpec defines the desired state of VultrBaremetalCluster. + """ + status: Status | None = None + """ + VultrBaremetalClusterStatus defines the observed state of VultrBaremetalCluster. + """ + + +class VultrBaremetalClusterList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[VultrBaremetalCluster] + """ + List of vultrbaremetalclusters. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/io/crossplane/k3s/__init__.py b/schemas/python/models/io/crossplane/k3s/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/k3s/cluster/__init__.py b/schemas/python/models/io/crossplane/k3s/cluster/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/k3s/cluster/v1alpha1.py b/schemas/python/models/io/crossplane/k3s/cluster/v1alpha1.py new file mode 100644 index 000000000..7cba5c8b3 --- /dev/null +++ b/schemas/python/models/io/crossplane/k3s/cluster/v1alpha1.py @@ -0,0 +1,251 @@ +# generated by datamodel-codegen: +# filename: workdir/k3s_crossplane_io_v1alpha1_cluster.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel, Field + +from ....k8s.apimachinery.pkg.apis.meta import v1 + + +class ForProvider(BaseModel): + clusterInit: bool | None = None + """ + ClusterInit enables embedded etcd for HA multi-server setup. + """ + datastoreEndpoint: str | None = None + """ + DatastoreEndpoint is an external datastore URL for HA (MySQL/PostgreSQL). + """ + disableServiceLB: bool | None = None + """ + DisableServiceLB disables the default ServiceLB load balancer. + """ + disableTraefik: bool | None = None + """ + DisableTraefik disables the default Traefik ingress controller. + """ + extraArgs: str | None = None + """ + ExtraArgs are additional arguments passed to k3s server. + """ + host: str + """ + Host is the DNS name or IP address of the target machine. + """ + k3sChannel: str | None = 'stable' + """ + K3sChannel is the release channel (stable, latest, v1.28, etc.). + """ + k3sVersion: str | None = None + """ + K3sVersion is the specific k3s version to install (e.g., "v1.28.2+k3s1"). + """ + port: int | None = 22 + """ + Port is the SSH port. Defaults to 22. + """ + tlsSAN: str | None = None + """ + TLSSAN adds an additional hostname or IP as a TLS Subject Alternative Name. + """ + + +class Policy(BaseModel): + resolution: Literal['Required', 'Optional'] | None = 'Required' + """ + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. + """ + resolve: Literal['Always', 'IfNotPresent'] | None = None + """ + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. + """ + + +class ProviderConfigRef(BaseModel): + name: str + """ + Name of the referenced object. + """ + policy: Policy | None = None + """ + Policies for referencing. + """ + + +class WriteConnectionSecretToRef(BaseModel): + name: str + """ + Name of the secret. + """ + namespace: str + """ + Namespace of the secret. + """ + + +class Spec(BaseModel): + deletionPolicy: Literal['Orphan', 'Delete'] | None = 'Delete' + """ + DeletionPolicy specifies what will happen to the underlying external + when this managed resource is deleted - either "Delete" or "Orphan" the + external resource. + This field is planned to be deprecated in favor of the ManagementPolicies + field in a future release. Currently, both could be set independently and + non-default values would be honored if the feature flag is enabled. + See the design doc for more information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223 + """ + forProvider: ForProvider + """ + ClusterParameters are the configurable fields of a Cluster. + """ + managementPolicies: ( + list[Literal['Observe', 'Create', 'Update', 'Delete', 'LateInitialize', '*']] + | None + ) = ['*'] + """ + THIS IS A BETA FIELD. It is on by default but can be opted out + through a Crossplane feature flag. + ManagementPolicies specify the array of actions Crossplane is allowed to + take on the managed and external resources. + This field is planned to replace the DeletionPolicy field in a future + release. Currently, both could be set independently and non-default + values would be honored if the feature flag is enabled. If both are + custom, the DeletionPolicy field will be ignored. + See the design doc for more information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223 + and this one: https://github.com/crossplane/crossplane/blob/444267e84783136daa93568b364a5f01228cacbe/design/one-pager-ignore-changes.md + """ + providerConfigRef: ProviderConfigRef | None = Field( + {'name': 'default'}, validate_default=True + ) + """ + ProviderConfigReference specifies how the provider that will be used to + create, observe, update, and delete this managed resource should be + configured. + """ + writeConnectionSecretToRef: WriteConnectionSecretToRef | None = None + """ + WriteConnectionSecretToReference specifies the namespace and name of a + Secret to which any connection details for this managed resource should + be written. Connection details frequently include the endpoint, username, + and password required to connect to the managed resource. + """ + + +class AtProvider(BaseModel): + k3sVersion: str | None = None + """ + K3sVersion is the installed version reported by the server. + """ + ready: bool | None = None + """ + Ready indicates the k3s server is running. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + """ + LastTransitionTime is the last time this condition transitioned from one + status to another. + """ + message: str | None = None + """ + A Message containing details about this condition's last transition from + one status to another, if any. + """ + observedGeneration: int | None = None + """ + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + """ + reason: str + """ + A Reason for this condition's last transition from one status to another. + """ + status: str + """ + Status of this condition; is it currently True, False, or Unknown? + """ + type: str + """ + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. + """ + + +class Status(BaseModel): + atProvider: AtProvider | None = None + """ + ClusterObservation are the observable fields of a Cluster. + """ + conditions: list[Condition] | None = None + """ + Conditions of the resource. + """ + lastHandledReconcileAt: str | None = None + """ + LastHandledReconcileAt holds the value of the most recent + reconcile-requested-at annotation token that the controller has + processed. Users can compare this to the annotation to determine + whether a reconcile request has been handled. + """ + observedGeneration: int | None = None + """ + ObservedGeneration is the latest metadata.generation + which resulted in either a ready state, or stalled due to error + it can not recover from without human intervention. + """ + + +class Cluster(BaseModel): + apiVersion: Literal['k3s.crossplane.io/v1alpha1'] | None = ( + 'k3s.crossplane.io/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['Cluster'] | None = 'Cluster' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + A ClusterSpec defines the desired state of a Cluster. + """ + status: Status | None = None + """ + A ClusterStatus represents the observed state of a Cluster. + """ + + +class ClusterList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[Cluster] + """ + List of clusters. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/io/crossplane/k3s/node/__init__.py b/schemas/python/models/io/crossplane/k3s/node/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/k3s/node/v1alpha1.py b/schemas/python/models/io/crossplane/k3s/node/v1alpha1.py new file mode 100644 index 000000000..5b34b9e18 --- /dev/null +++ b/schemas/python/models/io/crossplane/k3s/node/v1alpha1.py @@ -0,0 +1,254 @@ +# generated by datamodel-codegen: +# filename: workdir/k3s_crossplane_io_v1alpha1_node.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel, Field + +from ....k8s.apimachinery.pkg.apis.meta import v1 + + +class Policy(BaseModel): + resolution: Literal['Required', 'Optional'] | None = 'Required' + """ + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. + """ + resolve: Literal['Always', 'IfNotPresent'] | None = None + """ + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. + """ + + +class ClusterRef(BaseModel): + name: str + """ + Name of the referenced object. + """ + policy: Policy | None = None + """ + Policies for referencing. + """ + + +class ForProvider(BaseModel): + clusterRef: ClusterRef + """ + ClusterRef is a reference to the Cluster resource this node joins. + """ + extraArgs: str | None = None + """ + ExtraArgs are additional arguments passed to k3s. + """ + host: str + """ + Host is the DNS name or IP address of the target machine. + """ + k3sChannel: str | None = 'stable' + """ + K3sChannel is the release channel. + """ + k3sVersion: str | None = None + """ + K3sVersion is the specific k3s version to install. + """ + port: int | None = 22 + """ + Port is the SSH port. Defaults to 22. + """ + role: Literal['agent', 'server'] + """ + Role is the role of this node: "agent" (worker) or "server" (additional control plane). + """ + tlsSAN: str | None = None + """ + TLSSAN adds an additional TLS SAN (only applicable for server role). + """ + + +class ProviderConfigRef(BaseModel): + name: str + """ + Name of the referenced object. + """ + policy: Policy | None = None + """ + Policies for referencing. + """ + + +class WriteConnectionSecretToRef(BaseModel): + name: str + """ + Name of the secret. + """ + namespace: str + """ + Namespace of the secret. + """ + + +class Spec(BaseModel): + deletionPolicy: Literal['Orphan', 'Delete'] | None = 'Delete' + """ + DeletionPolicy specifies what will happen to the underlying external + when this managed resource is deleted - either "Delete" or "Orphan" the + external resource. + This field is planned to be deprecated in favor of the ManagementPolicies + field in a future release. Currently, both could be set independently and + non-default values would be honored if the feature flag is enabled. + See the design doc for more information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223 + """ + forProvider: ForProvider + """ + NodeParameters are the configurable fields of a Node. + """ + managementPolicies: ( + list[Literal['Observe', 'Create', 'Update', 'Delete', 'LateInitialize', '*']] + | None + ) = ['*'] + """ + THIS IS A BETA FIELD. It is on by default but can be opted out + through a Crossplane feature flag. + ManagementPolicies specify the array of actions Crossplane is allowed to + take on the managed and external resources. + This field is planned to replace the DeletionPolicy field in a future + release. Currently, both could be set independently and non-default + values would be honored if the feature flag is enabled. If both are + custom, the DeletionPolicy field will be ignored. + See the design doc for more information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223 + and this one: https://github.com/crossplane/crossplane/blob/444267e84783136daa93568b364a5f01228cacbe/design/one-pager-ignore-changes.md + """ + providerConfigRef: ProviderConfigRef | None = Field( + {'name': 'default'}, validate_default=True + ) + """ + ProviderConfigReference specifies how the provider that will be used to + create, observe, update, and delete this managed resource should be + configured. + """ + writeConnectionSecretToRef: WriteConnectionSecretToRef | None = None + """ + WriteConnectionSecretToReference specifies the namespace and name of a + Secret to which any connection details for this managed resource should + be written. Connection details frequently include the endpoint, username, + and password required to connect to the managed resource. + """ + + +class AtProvider(BaseModel): + ready: bool | None = None + """ + Ready indicates the node has successfully joined the cluster. + """ + role: str | None = None + """ + Role is the observed role of the node. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + """ + LastTransitionTime is the last time this condition transitioned from one + status to another. + """ + message: str | None = None + """ + A Message containing details about this condition's last transition from + one status to another, if any. + """ + observedGeneration: int | None = None + """ + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + """ + reason: str + """ + A Reason for this condition's last transition from one status to another. + """ + status: str + """ + Status of this condition; is it currently True, False, or Unknown? + """ + type: str + """ + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. + """ + + +class Status(BaseModel): + atProvider: AtProvider | None = None + """ + NodeObservation are the observable fields of a Node. + """ + conditions: list[Condition] | None = None + """ + Conditions of the resource. + """ + lastHandledReconcileAt: str | None = None + """ + LastHandledReconcileAt holds the value of the most recent + reconcile-requested-at annotation token that the controller has + processed. Users can compare this to the annotation to determine + whether a reconcile request has been handled. + """ + observedGeneration: int | None = None + """ + ObservedGeneration is the latest metadata.generation + which resulted in either a ready state, or stalled due to error + it can not recover from without human intervention. + """ + + +class Node(BaseModel): + apiVersion: Literal['k3s.crossplane.io/v1alpha1'] | None = ( + 'k3s.crossplane.io/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['Node'] | None = 'Node' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + A NodeSpec defines the desired state of a Node. + """ + status: Status | None = None + """ + A NodeStatus represents the observed state of a Node. + """ + + +class NodeList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[Node] + """ + List of nodes. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/io/crossplane/k3s/providerconfig/__init__.py b/schemas/python/models/io/crossplane/k3s/providerconfig/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/k3s/providerconfig/v1alpha1.py b/schemas/python/models/io/crossplane/k3s/providerconfig/v1alpha1.py new file mode 100644 index 000000000..7d113fb84 --- /dev/null +++ b/schemas/python/models/io/crossplane/k3s/providerconfig/v1alpha1.py @@ -0,0 +1,162 @@ +# generated by datamodel-codegen: +# filename: workdir/k3s_crossplane_io_v1alpha1_providerconfig.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel + +from ....k8s.apimachinery.pkg.apis.meta import v1 + + +class Env(BaseModel): + name: str + """ + Name is the name of an environment variable. + """ + + +class Fs(BaseModel): + path: str + """ + Path is a filesystem path. + """ + + +class SecretRef(BaseModel): + key: str + """ + The key to select. + """ + name: str + """ + Name of the secret. + """ + namespace: str + """ + Namespace of the secret. + """ + + +class Credentials(BaseModel): + env: Env | None = None + """ + Env is a reference to an environment variable that contains credentials + that must be used to connect to the provider. + """ + fs: Fs | None = None + """ + Fs is a reference to a filesystem location that contains credentials that + must be used to connect to the provider. + """ + secretRef: SecretRef | None = None + """ + A SecretRef is a reference to a secret key that contains the credentials + that must be used to connect to the provider. + """ + source: Literal['None', 'Secret'] + """ + Source of the provider credentials. + """ + + +class Spec(BaseModel): + credentials: Credentials + """ + Credentials holds SSH authentication credentials. + The referenced secret should contain either: + - key "password" for password-based auth + - key "ssh-privatekey" for SSH key-based auth + """ + username: str + """ + Username is the SSH username. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + """ + LastTransitionTime is the last time this condition transitioned from one + status to another. + """ + message: str | None = None + """ + A Message containing details about this condition's last transition from + one status to another, if any. + """ + observedGeneration: int | None = None + """ + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + """ + reason: str + """ + A Reason for this condition's last transition from one status to another. + """ + status: str + """ + Status of this condition; is it currently True, False, or Unknown? + """ + type: str + """ + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. + """ + + +class Status(BaseModel): + conditions: list[Condition] | None = None + """ + Conditions of the resource. + """ + users: int | None = None + """ + Users of this provider configuration. + """ + + +class ProviderConfig(BaseModel): + apiVersion: Literal['k3s.crossplane.io/v1alpha1'] | None = ( + 'k3s.crossplane.io/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['ProviderConfig'] | None = 'ProviderConfig' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + ProviderConfigSpec defines the desired state of ProviderConfig. + """ + status: Status | None = None + """ + A ProviderConfigStatus defines the status of a Provider. + """ + + +class ProviderConfigList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[ProviderConfig] + """ + List of providerconfigs. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/io/crossplane/k3s/providerconfigusage/__init__.py b/schemas/python/models/io/crossplane/k3s/providerconfigusage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/k3s/providerconfigusage/v1alpha1.py b/schemas/python/models/io/crossplane/k3s/providerconfigusage/v1alpha1.py new file mode 100644 index 000000000..801169e4b --- /dev/null +++ b/schemas/python/models/io/crossplane/k3s/providerconfigusage/v1alpha1.py @@ -0,0 +1,101 @@ +# generated by datamodel-codegen: +# filename: workdir/k3s_crossplane_io_v1alpha1_providerconfigusage.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel + +from ....k8s.apimachinery.pkg.apis.meta import v1 + + +class Policy(BaseModel): + resolution: Literal['Required', 'Optional'] | None = 'Required' + """ + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. + """ + resolve: Literal['Always', 'IfNotPresent'] | None = None + """ + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. + """ + + +class ProviderConfigRef(BaseModel): + name: str + """ + Name of the referenced object. + """ + policy: Policy | None = None + """ + Policies for referencing. + """ + + +class ResourceRef(BaseModel): + apiVersion: str + """ + APIVersion of the referenced object. + """ + kind: str + """ + Kind of the referenced object. + """ + name: str + """ + Name of the referenced object. + """ + uid: str | None = None + """ + UID of the referenced object. + """ + + +class ProviderConfigUsage(BaseModel): + apiVersion: Literal['k3s.crossplane.io/v1alpha1'] | None = ( + 'k3s.crossplane.io/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['ProviderConfigUsage'] | None = 'ProviderConfigUsage' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + providerConfigRef: ProviderConfigRef + """ + ProviderConfigReference to the provider config being used. + """ + resourceRef: ResourceRef + """ + ResourceReference to the managed resource using the provider config. + """ + + +class ProviderConfigUsageList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[ProviderConfigUsage] + """ + List of providerconfigusages. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/io/crossplane/m/k3s/__init__.py b/schemas/python/models/io/crossplane/m/k3s/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/m/k3s/cluster/__init__.py b/schemas/python/models/io/crossplane/m/k3s/cluster/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/m/k3s/cluster/v1alpha1.py b/schemas/python/models/io/crossplane/m/k3s/cluster/v1alpha1.py new file mode 100644 index 000000000..bdad9a3b3 --- /dev/null +++ b/schemas/python/models/io/crossplane/m/k3s/cluster/v1alpha1.py @@ -0,0 +1,216 @@ +# generated by datamodel-codegen: +# filename: workdir/k3s_m_crossplane_io_v1alpha1_cluster.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel, Field + +from .....k8s.apimachinery.pkg.apis.meta import v1 + + +class ForProvider(BaseModel): + clusterInit: bool | None = None + """ + ClusterInit enables embedded etcd for HA multi-server setup. + """ + datastoreEndpoint: str | None = None + """ + DatastoreEndpoint is an external datastore URL for HA (MySQL/PostgreSQL). + """ + disableServiceLB: bool | None = None + """ + DisableServiceLB disables the default ServiceLB load balancer. + """ + disableTraefik: bool | None = None + """ + DisableTraefik disables the default Traefik ingress controller. + """ + extraArgs: str | None = None + """ + ExtraArgs are additional arguments passed to k3s server. + """ + host: str + """ + Host is the DNS name or IP address of the target machine. + """ + k3sChannel: str | None = 'stable' + """ + K3sChannel is the release channel (stable, latest, v1.28, etc.). + """ + k3sVersion: str | None = None + """ + K3sVersion is the specific k3s version to install (e.g., "v1.28.2+k3s1"). + """ + port: int | None = 22 + """ + Port is the SSH port. Defaults to 22. + """ + tlsSAN: str | None = None + """ + TLSSAN adds an additional hostname or IP as a TLS Subject Alternative Name. + """ + + +class ProviderConfigRef(BaseModel): + kind: str + """ + Kind of the referenced object. + """ + name: str + """ + Name of the referenced object. + """ + + +class WriteConnectionSecretToRef(BaseModel): + name: str + """ + Name of the secret. + """ + + +class Spec(BaseModel): + forProvider: ForProvider + """ + ClusterParameters are the configurable fields of a Cluster. + """ + managementPolicies: ( + list[Literal['Observe', 'Create', 'Update', 'Delete', 'LateInitialize', '*']] + | None + ) = ['*'] + """ + THIS IS A BETA FIELD. It is on by default but can be opted out + through a Crossplane feature flag. + ManagementPolicies specify the array of actions Crossplane is allowed to + take on the managed and external resources. + See the design doc for more information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223 + and this one: https://github.com/crossplane/crossplane/blob/444267e84783136daa93568b364a5f01228cacbe/design/one-pager-ignore-changes.md + """ + providerConfigRef: ProviderConfigRef | None = Field( + {'kind': 'ClusterProviderConfig', 'name': 'default'}, validate_default=True + ) + """ + ProviderConfigReference specifies how the provider that will be used to + create, observe, update, and delete this managed resource should be + configured. + """ + writeConnectionSecretToRef: WriteConnectionSecretToRef | None = None + """ + WriteConnectionSecretToReference specifies the namespace and name of a + Secret to which any connection details for this managed resource should + be written. Connection details frequently include the endpoint, username, + and password required to connect to the managed resource. + """ + + +class AtProvider(BaseModel): + k3sVersion: str | None = None + """ + K3sVersion is the installed version reported by the server. + """ + ready: bool | None = None + """ + Ready indicates the k3s server is running. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + """ + LastTransitionTime is the last time this condition transitioned from one + status to another. + """ + message: str | None = None + """ + A Message containing details about this condition's last transition from + one status to another, if any. + """ + observedGeneration: int | None = None + """ + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + """ + reason: str + """ + A Reason for this condition's last transition from one status to another. + """ + status: str + """ + Status of this condition; is it currently True, False, or Unknown? + """ + type: str + """ + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. + """ + + +class Status(BaseModel): + atProvider: AtProvider | None = None + """ + ClusterObservation are the observable fields of a Cluster. + """ + conditions: list[Condition] | None = None + """ + Conditions of the resource. + """ + lastHandledReconcileAt: str | None = None + """ + LastHandledReconcileAt holds the value of the most recent + reconcile-requested-at annotation token that the controller has + processed. Users can compare this to the annotation to determine + whether a reconcile request has been handled. + """ + observedGeneration: int | None = None + """ + ObservedGeneration is the latest metadata.generation + which resulted in either a ready state, or stalled due to error + it can not recover from without human intervention. + """ + + +class Cluster(BaseModel): + apiVersion: Literal['k3s.m.crossplane.io/v1alpha1'] | None = ( + 'k3s.m.crossplane.io/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['Cluster'] | None = 'Cluster' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + A ClusterSpec defines the desired state of a Cluster. + """ + status: Status | None = None + """ + A ClusterStatus represents the observed state of a Cluster. + """ + + +class ClusterList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[Cluster] + """ + List of clusters. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/io/crossplane/m/k3s/clusterproviderconfig/__init__.py b/schemas/python/models/io/crossplane/m/k3s/clusterproviderconfig/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/m/k3s/clusterproviderconfig/v1alpha1.py b/schemas/python/models/io/crossplane/m/k3s/clusterproviderconfig/v1alpha1.py new file mode 100644 index 000000000..b83a0f5fa --- /dev/null +++ b/schemas/python/models/io/crossplane/m/k3s/clusterproviderconfig/v1alpha1.py @@ -0,0 +1,162 @@ +# generated by datamodel-codegen: +# filename: workdir/k3s_m_crossplane_io_v1alpha1_clusterproviderconfig.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel + +from .....k8s.apimachinery.pkg.apis.meta import v1 + + +class Env(BaseModel): + name: str + """ + Name is the name of an environment variable. + """ + + +class Fs(BaseModel): + path: str + """ + Path is a filesystem path. + """ + + +class SecretRef(BaseModel): + key: str + """ + The key to select. + """ + name: str + """ + Name of the secret. + """ + namespace: str + """ + Namespace of the secret. + """ + + +class Credentials(BaseModel): + env: Env | None = None + """ + Env is a reference to an environment variable that contains credentials + that must be used to connect to the provider. + """ + fs: Fs | None = None + """ + Fs is a reference to a filesystem location that contains credentials that + must be used to connect to the provider. + """ + secretRef: SecretRef | None = None + """ + A SecretRef is a reference to a secret key that contains the credentials + that must be used to connect to the provider. + """ + source: Literal['None', 'Secret'] + """ + Source of the provider credentials. + """ + + +class Spec(BaseModel): + credentials: Credentials + """ + Credentials holds SSH authentication credentials. + The referenced secret should contain either: + - key "password" for password-based auth + - key "ssh-privatekey" for SSH key-based auth + """ + username: str + """ + Username is the SSH username. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + """ + LastTransitionTime is the last time this condition transitioned from one + status to another. + """ + message: str | None = None + """ + A Message containing details about this condition's last transition from + one status to another, if any. + """ + observedGeneration: int | None = None + """ + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + """ + reason: str + """ + A Reason for this condition's last transition from one status to another. + """ + status: str + """ + Status of this condition; is it currently True, False, or Unknown? + """ + type: str + """ + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. + """ + + +class Status(BaseModel): + conditions: list[Condition] | None = None + """ + Conditions of the resource. + """ + users: int | None = None + """ + Users of this provider configuration. + """ + + +class ClusterProviderConfig(BaseModel): + apiVersion: Literal['k3s.m.crossplane.io/v1alpha1'] | None = ( + 'k3s.m.crossplane.io/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['ClusterProviderConfig'] | None = 'ClusterProviderConfig' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + ProviderConfigSpec defines the desired state of ProviderConfig. + """ + status: Status | None = None + """ + A ProviderConfigStatus defines the status of a Provider. + """ + + +class ClusterProviderConfigList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[ClusterProviderConfig] + """ + List of clusterproviderconfigs. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/io/crossplane/m/k3s/node/__init__.py b/schemas/python/models/io/crossplane/m/k3s/node/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/m/k3s/node/v1alpha1.py b/schemas/python/models/io/crossplane/m/k3s/node/v1alpha1.py new file mode 100644 index 000000000..9462e21c3 --- /dev/null +++ b/schemas/python/models/io/crossplane/m/k3s/node/v1alpha1.py @@ -0,0 +1,236 @@ +# generated by datamodel-codegen: +# filename: workdir/k3s_m_crossplane_io_v1alpha1_node.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel, Field + +from .....k8s.apimachinery.pkg.apis.meta import v1 + + +class Policy(BaseModel): + resolution: Literal['Required', 'Optional'] | None = 'Required' + """ + Resolution specifies whether resolution of this reference is required. + The default is 'Required', which means the reconcile will fail if the + reference cannot be resolved. 'Optional' means this reference will be + a no-op if it cannot be resolved. + """ + resolve: Literal['Always', 'IfNotPresent'] | None = None + """ + Resolve specifies when this reference should be resolved. The default + is 'IfNotPresent', which will attempt to resolve the reference only when + the corresponding field is not present. Use 'Always' to resolve the + reference on every reconcile. + """ + + +class ClusterRef(BaseModel): + name: str + """ + Name of the referenced object. + """ + policy: Policy | None = None + """ + Policies for referencing. + """ + + +class ForProvider(BaseModel): + clusterRef: ClusterRef + """ + ClusterRef is a reference to the Cluster resource this node joins. + """ + extraArgs: str | None = None + """ + ExtraArgs are additional arguments passed to k3s. + """ + host: str + """ + Host is the DNS name or IP address of the target machine. + """ + k3sChannel: str | None = 'stable' + """ + K3sChannel is the release channel. + """ + k3sVersion: str | None = None + """ + K3sVersion is the specific k3s version to install. + """ + port: int | None = 22 + """ + Port is the SSH port. Defaults to 22. + """ + role: Literal['agent', 'server'] + """ + Role is the role of this node: "agent" (worker) or "server" (additional control plane). + """ + tlsSAN: str | None = None + """ + TLSSAN adds an additional TLS SAN (only applicable for server role). + """ + + +class ProviderConfigRef(BaseModel): + kind: str + """ + Kind of the referenced object. + """ + name: str + """ + Name of the referenced object. + """ + + +class WriteConnectionSecretToRef(BaseModel): + name: str + """ + Name of the secret. + """ + + +class Spec(BaseModel): + forProvider: ForProvider + """ + NodeParameters are the configurable fields of a Node. + """ + managementPolicies: ( + list[Literal['Observe', 'Create', 'Update', 'Delete', 'LateInitialize', '*']] + | None + ) = ['*'] + """ + THIS IS A BETA FIELD. It is on by default but can be opted out + through a Crossplane feature flag. + ManagementPolicies specify the array of actions Crossplane is allowed to + take on the managed and external resources. + See the design doc for more information: https://github.com/crossplane/crossplane/blob/499895a25d1a1a0ba1604944ef98ac7a1a71f197/design/design-doc-observe-only-resources.md?plain=1#L223 + and this one: https://github.com/crossplane/crossplane/blob/444267e84783136daa93568b364a5f01228cacbe/design/one-pager-ignore-changes.md + """ + providerConfigRef: ProviderConfigRef | None = Field( + {'kind': 'ClusterProviderConfig', 'name': 'default'}, validate_default=True + ) + """ + ProviderConfigReference specifies how the provider that will be used to + create, observe, update, and delete this managed resource should be + configured. + """ + writeConnectionSecretToRef: WriteConnectionSecretToRef | None = None + """ + WriteConnectionSecretToReference specifies the namespace and name of a + Secret to which any connection details for this managed resource should + be written. Connection details frequently include the endpoint, username, + and password required to connect to the managed resource. + """ + + +class AtProvider(BaseModel): + ready: bool | None = None + """ + Ready indicates the node has successfully joined the cluster. + """ + role: str | None = None + """ + Role is the observed role of the node. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + """ + LastTransitionTime is the last time this condition transitioned from one + status to another. + """ + message: str | None = None + """ + A Message containing details about this condition's last transition from + one status to another, if any. + """ + observedGeneration: int | None = None + """ + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + """ + reason: str + """ + A Reason for this condition's last transition from one status to another. + """ + status: str + """ + Status of this condition; is it currently True, False, or Unknown? + """ + type: str + """ + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. + """ + + +class Status(BaseModel): + atProvider: AtProvider | None = None + """ + NodeObservation are the observable fields of a Node. + """ + conditions: list[Condition] | None = None + """ + Conditions of the resource. + """ + lastHandledReconcileAt: str | None = None + """ + LastHandledReconcileAt holds the value of the most recent + reconcile-requested-at annotation token that the controller has + processed. Users can compare this to the annotation to determine + whether a reconcile request has been handled. + """ + observedGeneration: int | None = None + """ + ObservedGeneration is the latest metadata.generation + which resulted in either a ready state, or stalled due to error + it can not recover from without human intervention. + """ + + +class Node(BaseModel): + apiVersion: Literal['k3s.m.crossplane.io/v1alpha1'] | None = ( + 'k3s.m.crossplane.io/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['Node'] | None = 'Node' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + A NodeSpec defines the desired state of a Node. + """ + status: Status | None = None + """ + A NodeStatus represents the observed state of a Node. + """ + + +class NodeList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[Node] + """ + List of nodes. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/io/crossplane/m/k3s/providerconfig/__init__.py b/schemas/python/models/io/crossplane/m/k3s/providerconfig/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/m/k3s/providerconfig/v1alpha1.py b/schemas/python/models/io/crossplane/m/k3s/providerconfig/v1alpha1.py new file mode 100644 index 000000000..c56052523 --- /dev/null +++ b/schemas/python/models/io/crossplane/m/k3s/providerconfig/v1alpha1.py @@ -0,0 +1,162 @@ +# generated by datamodel-codegen: +# filename: workdir/k3s_m_crossplane_io_v1alpha1_providerconfig.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import AwareDatetime, BaseModel + +from .....k8s.apimachinery.pkg.apis.meta import v1 + + +class Env(BaseModel): + name: str + """ + Name is the name of an environment variable. + """ + + +class Fs(BaseModel): + path: str + """ + Path is a filesystem path. + """ + + +class SecretRef(BaseModel): + key: str + """ + The key to select. + """ + name: str + """ + Name of the secret. + """ + namespace: str + """ + Namespace of the secret. + """ + + +class Credentials(BaseModel): + env: Env | None = None + """ + Env is a reference to an environment variable that contains credentials + that must be used to connect to the provider. + """ + fs: Fs | None = None + """ + Fs is a reference to a filesystem location that contains credentials that + must be used to connect to the provider. + """ + secretRef: SecretRef | None = None + """ + A SecretRef is a reference to a secret key that contains the credentials + that must be used to connect to the provider. + """ + source: Literal['None', 'Secret'] + """ + Source of the provider credentials. + """ + + +class Spec(BaseModel): + credentials: Credentials + """ + Credentials holds SSH authentication credentials. + The referenced secret should contain either: + - key "password" for password-based auth + - key "ssh-privatekey" for SSH key-based auth + """ + username: str + """ + Username is the SSH username. + """ + + +class Condition(BaseModel): + lastTransitionTime: AwareDatetime + """ + LastTransitionTime is the last time this condition transitioned from one + status to another. + """ + message: str | None = None + """ + A Message containing details about this condition's last transition from + one status to another, if any. + """ + observedGeneration: int | None = None + """ + ObservedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + """ + reason: str + """ + A Reason for this condition's last transition from one status to another. + """ + status: str + """ + Status of this condition; is it currently True, False, or Unknown? + """ + type: str + """ + Type of this condition. At most one of each condition type may apply to + a resource at any point in time. + """ + + +class Status(BaseModel): + conditions: list[Condition] | None = None + """ + Conditions of the resource. + """ + users: int | None = None + """ + Users of this provider configuration. + """ + + +class ProviderConfig(BaseModel): + apiVersion: Literal['k3s.m.crossplane.io/v1alpha1'] | None = ( + 'k3s.m.crossplane.io/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['ProviderConfig'] | None = 'ProviderConfig' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + spec: Spec + """ + ProviderConfigSpec defines the desired state of ProviderConfig. + """ + status: Status | None = None + """ + A ProviderConfigStatus defines the status of a Provider. + """ + + +class ProviderConfigList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[ProviderConfig] + """ + List of providerconfigs. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/schemas/python/models/io/crossplane/m/k3s/providerconfigusage/__init__.py b/schemas/python/models/io/crossplane/m/k3s/providerconfigusage/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/schemas/python/models/io/crossplane/m/k3s/providerconfigusage/v1alpha1.py b/schemas/python/models/io/crossplane/m/k3s/providerconfigusage/v1alpha1.py new file mode 100644 index 000000000..757b5ff7d --- /dev/null +++ b/schemas/python/models/io/crossplane/m/k3s/providerconfigusage/v1alpha1.py @@ -0,0 +1,84 @@ +# generated by datamodel-codegen: +# filename: workdir/k3s_m_crossplane_io_v1alpha1_providerconfigusage.yaml + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel + +from .....k8s.apimachinery.pkg.apis.meta import v1 + + +class ProviderConfigRef(BaseModel): + kind: str + """ + Kind of the referenced object. + """ + name: str + """ + Name of the referenced object. + """ + + +class ResourceRef(BaseModel): + apiVersion: str + """ + APIVersion of the referenced object. + """ + kind: str + """ + Kind of the referenced object. + """ + name: str + """ + Name of the referenced object. + """ + uid: str | None = None + """ + UID of the referenced object. + """ + + +class ProviderConfigUsage(BaseModel): + apiVersion: Literal['k3s.m.crossplane.io/v1alpha1'] | None = ( + 'k3s.m.crossplane.io/v1alpha1' + ) + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + kind: Literal['ProviderConfigUsage'] | None = 'ProviderConfigUsage' + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ObjectMeta | None = None + """ + Standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + """ + providerConfigRef: ProviderConfigRef + """ + ProviderConfigReference to the provider config being used. + """ + resourceRef: ResourceRef + """ + ResourceReference to the managed resource using the provider config. + """ + + +class ProviderConfigUsageList(BaseModel): + apiVersion: str | None = None + """ + APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + """ + items: list[ProviderConfigUsage] + """ + List of providerconfigusages. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md + """ + kind: str | None = None + """ + Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ + metadata: v1.ListMeta | None = None + """ + Standard list metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + """ \ No newline at end of file diff --git a/uv.lock b/uv.lock index 8b1e07fcb..ce3baabdc 100644 --- a/uv.lock +++ b/uv.lock @@ -14,6 +14,7 @@ members = [ "compose-inference-class", "compose-inference-cluster", "compose-inference-gateway", + "compose-k3s-cluster", "compose-model-cache", "compose-model-deployment", "compose-model-endpoint", @@ -22,6 +23,7 @@ members = [ "compose-nebius-cluster", "compose-serving-stack", "compose-usages", + "compose-vultr-baremetal-cluster", "compose-vultr-cluster", "crossplane-models", "modelplane", @@ -189,6 +191,25 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0" }, ] +[[package]] +name = "compose-k3s-cluster" +version = "0.0.0" +source = { editable = "functions/compose-k3s-cluster" } +dependencies = [ + { name = "click" }, + { name = "crossplane-function-sdk-python" }, + { name = "crossplane-models" }, + { name = "grpcio" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.1.0" }, + { name = "crossplane-function-sdk-python", specifier = ">=0.14.0" }, + { name = "crossplane-models", editable = "schemas/python" }, + { name = "grpcio", specifier = ">=1.73.1" }, +] + [[package]] name = "compose-model-cache" version = "0.0.0" @@ -345,6 +366,25 @@ requires-dist = [ { name = "grpcio", specifier = ">=1.73.1" }, ] +[[package]] +name = "compose-vultr-baremetal-cluster" +version = "0.0.0" +source = { editable = "functions/compose-vultr-baremetal-cluster" } +dependencies = [ + { name = "click" }, + { name = "crossplane-function-sdk-python" }, + { name = "crossplane-models" }, + { name = "grpcio" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.1.0" }, + { name = "crossplane-function-sdk-python", specifier = ">=0.14.0" }, + { name = "crossplane-models", editable = "schemas/python" }, + { name = "grpcio", specifier = ">=1.73.1" }, +] + [[package]] name = "compose-vultr-cluster" version = "0.0.0"