From f9ef829bfe62c39e730cd8fecbd8bdbdc887393b Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Mon, 26 Oct 2026 03:48:05 -0700 Subject: [PATCH 1/9] feat: add the native NixOps4 integrations A caisson-nixops4 configuration, named nixops4 for the output it fills, publishes the root NixOps4 reads: rootFunction and getProviders over the root composite, a deferred caisson-nixops4-composite configuration declared under the name root. NixOps4 evaluates the root when it deploys, so the composite is finalized with the args NixOps4 passes: the output values and the deploying system. A composite holds its members as deferred caisson-nixops4-composite and caisson-nixops4-resource configurations beneath it and finalizes each with its part of those args. Composite and resource are two classes. A composite has NixOps4's providers option and no type; a resource has NixOps4's resource, type and providers options and no members. A component reads the outputs of its composite's members, and nothing else of them, through the members module argument. The NixOS adapter, the module nixos of the resource class, deploys the NixOS configuration declared beneath a resource as nixops4-nixos does, with the same SSH options, and runs the script with bash from the package set caisson selects for the deploying system. The default module of the resource class selects it and gives the NixOS configurations beneath the nixops4-apply NixOS module, which imports the apply module of nixops4-nixos. argsNeeded names the deploying system and the outputs of the members, and a resource lists the outputs a tool may cache in cacheableOutputs. caisson-core is pinned to the head of nix-caisson/caisson-core#57, where a function argsNeeded receives the name and parent of the entry. Co-Authored-By: Claude Opus 5.5 --- flake.lock | 6 +- .../caisson-nixops4-composite/default.nix | 309 +++++++++ .../caisson-nixops4-resource/default.nix | 276 ++++++++ lib-overlays/caisson-nixops4/default.nix | 360 +++++++++++ modules/caisson-nixops4-composite/core | 1 + modules/caisson-nixops4-resource/core | 1 + .../default/default.nix | 36 ++ .../nixos/default.nix | 158 +++++ modules/caisson-nixops4/core | 1 + modules/generic/core/default.nix | 3 + modules/nixos/nixops4-apply/default.nix | 36 ++ tests/dependencies/flake.lock | 604 +++++++++++++++++- tests/dependencies/flake.nix | 6 + tests/pinned-world/default.nix | 260 ++++++++ tests/unit/lib-overlays.nix | 9 + 15 files changed, 2052 insertions(+), 14 deletions(-) create mode 100644 lib-overlays/caisson-nixops4-composite/default.nix create mode 100644 lib-overlays/caisson-nixops4-resource/default.nix create mode 100644 lib-overlays/caisson-nixops4/default.nix create mode 120000 modules/caisson-nixops4-composite/core create mode 120000 modules/caisson-nixops4-resource/core create mode 100644 modules/caisson-nixops4-resource/default/default.nix create mode 100644 modules/caisson-nixops4-resource/nixos/default.nix create mode 120000 modules/caisson-nixops4/core create mode 100644 modules/nixos/nixops4-apply/default.nix diff --git a/flake.lock b/flake.lock index 21d0d659..6f584c22 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793243975, - "narHash": "sha256-OzlG2Nhl/kU2UUPq+ARH/GKUHmOqAxQu5Ug/+aT1ECc=", + "lastModified": 1793010712, + "narHash": "sha256-otbvrbV58kVW+yBVdSH+bt82O89iaz4Ul664iyKyq68=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "6868bca2682e96d56714bc9c9adcf0048fbe132f", + "rev": "ddcff6816293044822aa2c9418ca2fcf51643c1e", "type": "github" }, "original": { diff --git a/lib-overlays/caisson-nixops4-composite/default.nix b/lib-overlays/caisson-nixops4-composite/default.nix new file mode 100644 index 00000000..7dd2c29a --- /dev/null +++ b/lib-overlays/caisson-nixops4-composite/default.nix @@ -0,0 +1,309 @@ +# SPDX-License-Identifier: MIT +# +# The caisson-nixops4-composite integration: a composite of a NixOps4 +# deployment, a component that holds further components as its +# members. The root component NixOps4 deploys is the outermost +# composite, declared under the name `root` beneath a caisson-nixops4 +# configuration. +# +# A composite is a deferred configuration (caisson-core's +# `mkDeferredConfiguration`): it is evaluated when NixOps4 deploys, +# with the args its parent gives it, and NixOps4 evaluates it again +# each time the outputs of a resource become known. Its members are the +# caisson-nixops4-composite and caisson-nixops4-resource configurations +# declared beneath it, deferred children in turn, which it finalizes +# with their part of its args (`memberArgs` of caisson-nixops4). +# +# A composite declares providers, as NixOps4's components do, with +# NixOps4's own `providers` option, and its module reads its members' +# outputs through the `members` module argument. It has no `type` and +# no resource options: a composite that sets them is refused where it +# sets them, as an undeclared option. +# +# What a composite publishes to NixOps4 is its export: its members' +# exports, by name, as NixOps4's `_export` of a composite holds them. +{ + mkLibOverlay, + ... +}: +{ + + imports = [ + # The library of nixpkgs, which this overlay calls through the + # composed library, imported by key so it is composed wherever + # this is. + ((mkLibOverlay ../nixpkgs-lib) // { key = "nixpkgs-lib"; }) + # What an integration is written from, imported by key so it is + # composed wherever this integration is. + ((mkLibOverlay ../integrations) // { key = "integrations"; }) + # The declaration, whose helpers this integration builds on. + ((mkLibOverlay ../caisson-nixops4) // { key = "caisson-nixops4"; }) + ]; + + overlay = + final: prev: + let + nixops4 = final.caisson.caisson-nixops4; + context = "lib.caisson.caisson-nixops4-composite"; + compositeType = "caisson-nixops4-composite"; + resourceType = "caisson-nixops4-resource"; + + # The member path of a component, as NixOps4 writes it. + showPath = path: if path == [ ] then "the root" else builtins.concatStringsSep "." path; + + # The members of a composite, by name, from its full evaluation: + # the composites and resources declared beneath it, each the entry + # caisson-core holds for a deferred child. A name declared as both + # is refused, since NixOps4 holds one member under a name. + memberEntriesOf = + deploy: config: + let + composites = config.caisson.${compositeType}.configurations; + resources = config.caisson.${resourceType}.configurations; + both = builtins.attrNames (builtins.intersectAttrs composites resources); + in + if both == [ ] then + composites // resources + else + throw '' + ${context}: the composite at ${showPath deploy.path} declares + ${builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") both)} both as a composite and as a + resource; a member is one or the other. + ''; + + # The members finalized with the args the composite gives them, + # by name: each a manifest. + finalizeMembers = + deploy: entries: + builtins.mapAttrs ( + name: entry: + entry.finalize { + args = nixops4.memberArgs deploy name; + argsSource = "the composite at ${showPath deploy.path}"; + } + ) entries; + + # The output args of a composite's members, as `argsNeeded` + # declares them: by member name, a resource's outputs, each an + # arg cacheable where the resource declares it, or a composite's + # own output args. A member whose outputs cannot be named yet, + # because naming them waits for an output, is left out, and a tool + # asks again once it has that output. + outputArgsOf = + members: + builtins.mapAttrs ( + _: member: + let + named = + if member.type == resourceType then + builtins.mapAttrs ( + output: _: + final.caisson-core.mkArg { + cacheable = builtins.elem output member.outputs.cacheableOutputs; + description = "An output of the resource at ${showPath member.outputs.path}."; + } + ) member.outputs.export.resource.outputsSkeleton + else + outputArgsOf member.outputs.members; + attempt = builtins.tryEval (builtins.deepSeq (builtins.attrNames named) named); + in + if attempt.success then named else { } + ) members; + + # A composite configuration of the module class `class`, evaluated + # with the modules `baseModules` returns for the NixOps4 source + # and the args, and the `members` module argument `membersArg` + # returns. The native integration evaluates its class with + # NixOps4's provider declarations and the view of the members' + # outputs; the upstream-compatible one builds on this with + # NixOps4's class and its own modules. + compositeWith = + { + class, + type ? compositeType, + baseModules, + membersArg ? + { deploy, ... }: + nixops4.membersView deploy.path deploy.outputValues, + }: + args: + let + # The evaluator's call, on the lib of the view being + # evaluated: the base modules, the framework modules of the + # class, the selection and the configuration's module, with + # the composition's pinned sources as the `inputs` special + # argument and the deploying system as + # `resourceProviderSystem`, as NixOps4 passes it. + # `ecosystemArgs` is merged over the call last. + evaluate = + deploy: + { lib, manifest }: + let + selection = lib.caisson.integrations; + src = nixops4.ecosystemSrcOf { + explicit = args.ecosystemSrc or null; + inherit manifest; + }; + registry = lib.caisson.modules.${class} or { }; + frameworkModules = selection.frameworkModules class registry; + moduleImports = selection.moduleImportsOf class { inherit lib manifest; } args; + name = manifest.name or null; + configModule = selection.configModuleOf { + inherit lib name class; + configModule = args.configModule or null; + }; + # The members, finalized from the full evaluation; the + # childless view holds none. + finalized = + if manifest.childless then + { } + else + finalizeMembers deploy (memberEntriesOf deploy evaluated.config); + membersModule = selection.caissonModule { + key = "caisson.${type}.members"; + file = "caisson-${type}:members"; + module = { + _module.args.members = membersArg { + inherit deploy finalized manifest; + }; + }; + }; + moduleList = + baseModules { inherit src deploy; } + ++ [ membersModule ] + ++ frameworkModules + ++ moduleImports registry + ++ (if configModule == null then [ ] else [ configModule ]); + evaluated = lib.evalModules ( + { + inherit class; + specialArgs = { + inherit lib; + inputs = manifest.sources; + inherit (deploy) resourceProviderSystem; + } + // (if (args.specialArgs or null) != null then args.specialArgs else { }); + modules = moduleList; + } + // (if (args.ecosystemArgs or null) != null then args.ecosystemArgs else { }) + ); + in + { + value = evaluated; + ecosystems.nixops4 = nixops4.ecosystemAccountOf { + explicit = args.ecosystemSrc or null; + inherit manifest; + }; + modules = moduleList; + outputs = { + inherit (deploy) path; + # The members, by name, each finalized with its args. + members = finalized; + # What NixOps4 reads of the composite. + export.members = builtins.mapAttrs (_: member: member.outputs.export) finalized; + # The providers it declares, which `getProviders` builds + # for the root. + providerConfigs = evaluated.config.providers; + }; + }; + + configurationFor = + deploy: + final.caisson.integrations.mkModuleConfiguration { + selection = { + inherit class args; + }; + inherit type; + evaluate = evaluate deploy; + }; + in + final.caisson-core.mkDeferredConfiguration { + inherit type class; + # The deploying system, and the outputs of the members, which + # depend on the members the composite declares, so on the args + # known so far. + argsNeeded = + { + name, + parent, + args, + }: + let + manifest = configurationFor (nixops4.deployOf name args) { inherit name parent; }; + in + { + resourceProviderSystem = nixops4.systemArg; + outputValues = outputArgsOf manifest.outputs.members; + }; + configuration = + { + name, + parent, + args, + }: + configurationFor (nixops4.deployOf name args) { inherit name parent; }; + }; + + configuration = compositeWith { + class = compositeType; + baseModules = + { src, ... }: + [ + # NixOps4's `providers` option, whose resource types the + # composite writes into its members' `type`. + (nixops4.componentModule src "provider-declarations") + ]; + }; + + integration = final.caisson.integrations.mkIntegration { + name = compositeType; + class = compositeType; + # What these return is a deferred configuration, a function of + # `{ name, parent, args }`: the composite that declares it under + # `caisson.caisson-nixops4-composite.configurations.`, or + # the caisson-nixops4 configuration for `root`, finalizes it + # when NixOps4 deploys. + mkConfiguration = + { + # The composite's module: its providers and members. When + # absent, the configuration registered under the + # configuration's name, if any. + configModule ? null, + # The NixOps4 flake; resolved from the composition's + # declarations when absent. + ecosystemSrc ? null, + # The selection over the caisson-nixops4-composite class of + # the registry, replacing the default of the class. + moduleImports ? null, + # A selection added to that selection, whichever it is. + extraModuleImports ? null, + # Extra module arguments, merged over those the framework + # supplies. + specialArgs ? null, + }@args: + configuration args; + # The same arguments and `ecosystemArgs`, the evaluator's + # arguments merged over the composed call last. + mkConfigurationWithEcosystemArgs = + { + configModule ? null, + ecosystemSrc ? null, + moduleImports ? null, + extraModuleImports ? null, + specialArgs ? null, + ecosystemArgs ? null, + }@args: + configuration args; + # What the upstream-compatible integration builds on. + extra = { + inherit compositeWith showPath; + }; + }; + in + { + caisson = (prev.caisson or { }) // { + caisson-nixops4-composite = integration.namespace; + }; + }; + +} diff --git a/lib-overlays/caisson-nixops4-resource/default.nix b/lib-overlays/caisson-nixops4-resource/default.nix new file mode 100644 index 00000000..40822c35 --- /dev/null +++ b/lib-overlays/caisson-nixops4-resource/default.nix @@ -0,0 +1,276 @@ +# SPDX-License-Identifier: MIT +# +# The caisson-nixops4-resource integration: a resource of a NixOps4 +# deployment, one real thing a provider creates, updates and deletes. +# A resource is a member of a caisson-nixops4-composite configuration, +# declared beneath it, and a deferred configuration like the composite: +# the composite finalizes it with the resource's outputs, as NixOps4 +# passes them, the composite's view of its members and the deploying +# system. +# +# A resource is evaluated with NixOps4's own resource options +# (`nix/component/resource.nix`), its `type` option and its +# `providers` option. `type` is a resource type of a provider, which +# the composite usually writes from its `providers`. The resource +# reads its own outputs as `config.outputs`, and the outputs of the +# other members of its composite through the `members` module +# argument. It holds no NixOps4 members: `members` is not an option +# here, and setting it is refused where it is set. +# +# A resource may hold caisson configurations beneath it, such as a +# NixOS configuration with its home-manager configurations. NixOps4 +# does not see them. A module of the class turns them into the +# resource's inputs: the NixOS adapter, registered as `nixos` in the +# class and selected by its default module, deploys the NixOS +# configuration declared beneath a resource. +# +# The resource declares which of its outputs a tool that evaluates the +# deployment ahead of time may keep in a cache (`cacheableOutputs`); +# NixOps4 marks no output as secret, so none is cacheable unless +# listed. +{ + mkLibOverlay, + ... +}: +{ + + imports = [ + # The library of nixpkgs, which this overlay calls through the + # composed library, imported by key so it is composed wherever + # this is. + ((mkLibOverlay ../nixpkgs-lib) // { key = "nixpkgs-lib"; }) + # What an integration is written from, imported by key so it is + # composed wherever this integration is. + ((mkLibOverlay ../integrations) // { key = "integrations"; }) + # The declaration, whose helpers this integration builds on. + ((mkLibOverlay ../caisson-nixops4) // { key = "caisson-nixops4"; }) + ]; + + overlay = + final: prev: + let + nixops4 = final.caisson.caisson-nixops4; + resourceType = "caisson-nixops4-resource"; + + # The options of the class beside the options NixOps4 declares. + resourceOptions = + { lib, ... }: + { + options.cacheableOutputs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + The names of the outputs of this resource that a tool which + evaluates the deployment ahead of time may keep in a cache + only its user can read. NixOps4 marks no output as secret, + so an output is not cacheable unless it is listed here. + ''; + }; + }; + + # What NixOps4 gives a resource when it evaluates the root: its + # outputs, from the output values it passes, and its member path. + # The outputs of a resource NixOps4 has not applied yet are calls + # that abort the evaluation, recoverably, when read. + deployModule = + deploy: + final.caisson.integrations.caissonModule { + key = "caisson.${resourceType}.deploy"; + file = "caisson-${resourceType}:deploy"; + module = { + config.outputs = _: { config = deploy.outputValues; }; + config.absoluteMemberPath = deploy.path; + config._module.args.members = deploy.members; + }; + }; + + # A resource configuration of the module class `class`, evaluated + # with the modules `baseModules` returns for the NixOps4 source + # and the args. The native integration evaluates its class with + # NixOps4's resource, provider-value and provider-declaration + # modules; the upstream-compatible one builds on this with + # NixOps4's class and its own modules. + resourceWith = + { + class, + type ? resourceType, + baseModules, + }: + args: + let + # The evaluator's call, on the lib of the view being + # evaluated: the base modules, the framework modules of the + # class, the selection and the configuration's module, with + # the composition's pinned sources as the `inputs` special + # argument and the deploying system as + # `resourceProviderSystem`, as NixOps4 passes it. + # `ecosystemArgs` is merged over the call last. + evaluate = + deploy: + { lib, manifest }: + let + selection = lib.caisson.integrations; + src = nixops4.ecosystemSrcOf { + explicit = args.ecosystemSrc or null; + inherit manifest; + }; + registry = lib.caisson.modules.${class} or { }; + frameworkModules = selection.frameworkModules class registry; + moduleImports = selection.moduleImportsOf class { inherit lib manifest; } args; + name = manifest.name or null; + configModule = selection.configModuleOf { + inherit lib name class; + configModule = args.configModule or null; + }; + moduleList = + baseModules { inherit src deploy; } + ++ [ (deployModule deploy) ] + ++ frameworkModules + ++ moduleImports registry + ++ (if configModule == null then [ ] else [ configModule ]); + evaluated = lib.evalModules ( + { + inherit class; + specialArgs = { + inherit lib; + inputs = manifest.sources; + inherit (deploy) resourceProviderSystem; + } + // (if (args.specialArgs or null) != null then args.specialArgs else { }); + modules = moduleList; + } + // (if (args.ecosystemArgs or null) != null then args.ecosystemArgs else { }) + ); + in + { + value = evaluated; + ecosystems.nixops4 = nixops4.ecosystemAccountOf { + explicit = args.ecosystemSrc or null; + inherit manifest; + }; + modules = moduleList; + outputs = { + inherit (deploy) path; + # What NixOps4 reads of the resource. + export.resource = evaluated.config._resourceForNixOps; + cacheableOutputs = evaluated.config.cacheableOutputs or [ ]; + providerConfigs = evaluated.config.providers; + }; + }; + + configurationFor = + deploy: + final.caisson.integrations.mkModuleConfiguration { + selection = { + inherit class args; + }; + inherit type; + defaultPkgs = args.defaultPkgs or null; + evaluate = evaluate deploy; + }; + in + final.caisson-core.mkDeferredConfiguration { + inherit type class; + # The deploying system and the resource's own outputs, which + # its type declares. + argsNeeded = + { + name, + parent, + args, + }: + let + manifest = configurationFor (nixops4.deployOf name args) { inherit name parent; }; + in + { + resourceProviderSystem = nixops4.systemArg; + outputValues = builtins.mapAttrs ( + output: _: + final.caisson-core.mkArg { + cacheable = builtins.elem output manifest.outputs.cacheableOutputs; + description = "An output of this resource."; + } + ) manifest.outputs.export.resource.outputsSkeleton; + }; + configuration = + { + name, + parent, + args, + }: + configurationFor (nixops4.deployOf name args) { inherit name parent; }; + }; + + configuration = resourceWith { + class = resourceType; + baseModules = + { src, ... }: + [ + (nixops4.componentModule src "resource") + (nixops4.componentModule src "provider-values") + (nixops4.componentModule src "provider-declarations") + (final.caisson.integrations.caissonModule { + key = "caisson.${resourceType}.resourceOptions"; + file = "caisson-${resourceType}:resourceOptions"; + module = resourceOptions; + }) + ]; + }; + + integration = final.caisson.integrations.mkIntegration { + name = resourceType; + class = resourceType; + # What these return is a deferred configuration, a function of + # `{ name, parent, args }`: the composite that declares it under + # `caisson.caisson-nixops4-resource.configurations.` + # finalizes it when NixOps4 deploys. + mkConfiguration = + { + # The resource's module: its `type`, `inputs`, the options + # of the modules it selects, and the caisson configurations + # declared beneath it. When absent, the configuration + # registered under the configuration's name, if any. + configModule ? null, + # The package set the configurations beneath run on: a + # function that receives the package sets available, by + # package config name, and returns the set to run on. + defaultPkgs ? null, + # The NixOps4 flake; resolved from the composition's + # declarations when absent. + ecosystemSrc ? null, + # The selection over the caisson-nixops4-resource class of + # the registry, replacing the default of the class. + moduleImports ? null, + # A selection added to that selection, whichever it is. + extraModuleImports ? null, + # Extra module arguments, merged over those the framework + # supplies. + specialArgs ? null, + }@args: + configuration args; + # The same arguments and `ecosystemArgs`, the evaluator's + # arguments merged over the composed call last. + mkConfigurationWithEcosystemArgs = + { + configModule ? null, + defaultPkgs ? null, + ecosystemSrc ? null, + moduleImports ? null, + extraModuleImports ? null, + specialArgs ? null, + ecosystemArgs ? null, + }@args: + configuration args; + # What the upstream-compatible integration builds on. + extra = { + inherit resourceWith deployModule resourceOptions; + }; + }; + in + { + caisson = (prev.caisson or { }) // { + caisson-nixops4-resource = integration.namespace; + }; + }; + +} diff --git a/lib-overlays/caisson-nixops4/default.nix b/lib-overlays/caisson-nixops4/default.nix new file mode 100644 index 00000000..10a6c081 --- /dev/null +++ b/lib-overlays/caisson-nixops4/default.nix @@ -0,0 +1,360 @@ +# SPDX-License-Identifier: MIT +# +# The caisson-nixops4 integration, the native NixOps4 integration's +# declaration: a configuration of the `caisson-nixops4` class, evaluated +# when the flake is, which holds the root component of a NixOps4 +# deployment and publishes it as the `nixops4` output NixOps4's +# command reads. +# +# NixOps4 evaluates its root component when it deploys, again each time +# the outputs of a resource become known, and a component can read those +# outputs and compute its members from them. So the root component is a +# deferred child of the declaration (caisson-core's +# `mkDeferredConfiguration`): a `caisson-nixops4-composite` +# configuration declared under the name `root`, finalized by +# `rootFunction` with the output values NixOps4 passes. The composites +# and resources beneath it are deferred children of their composites, +# which finalize them with their part of those values. The integrations +# `caisson-nixops4-composite` and `caisson-nixops4-resource` are written +# beside this one, on the helpers it publishes under `extra`. +# +# The configuration named `nixops4`, for the output it fills, is +# published there: `_type`, `rootFunction` and `getProviders`, as +# NixOps4's `mkRoot` returns them. +{ + mkLibOverlay, + ... +}: +{ + + imports = [ + # The library of nixpkgs, which this overlay calls through the + # composed library, imported by key so it is composed wherever + # this is. + ((mkLibOverlay ../nixpkgs-lib) // { key = "nixpkgs-lib"; }) + # What an integration is written from, imported by key so it is + # composed wherever this integration is. + ((mkLibOverlay ../integrations) // { key = "integrations"; }) + ]; + + overlay = + final: prev: + let + + resolveEcosystemSrc = final.caisson.integrations.resolveEcosystemSrc ecosystem; + ecosystemAccountOf = final.caisson.integrations.ecosystemAccountOf ecosystem; + ecosystem = { + name = "nixops4"; + context = "caisson.caisson-nixops4"; + }; + context = "lib.caisson.caisson-nixops4"; + + # NixOps4's component modules are files of its source, which the + # composite and resource integrations evaluate in caisson's + # evaluations; its `local` provider is a module its flake + # publishes. + assertNixops4EcosystemSrc = + ecosystemSrc: + if + ecosystemSrc ? modules + && ecosystemSrc.modules ? nixops4Provider + && builtins.pathExists "${ecosystemSrc}/nix/component/resource.nix" + && builtins.pathExists "${ecosystemSrc}/nix/component/provider-values.nix" + && builtins.pathExists "${ecosystemSrc}/nix/component/provider-declarations.nix" + then + ecosystemSrc + else + throw "${context} requires `ecosystemSrc.modules.nixops4Provider` and NixOps4's component modules under `nix/component` (a NixOps4 flake)."; + + # The NixOps4 source of a configuration: `explicit` when given, + # else what the composition declares or pins under the name + # `nixops4`, as recorded on `manifest`. + ecosystemSrcOf = + { explicit, manifest }: + assertNixops4EcosystemSrc (resolveEcosystemSrc { + inherit explicit manifest; + }); + + # A component module of NixOps4, by its file name under + # `nix/component`. + componentModule = src: name: "${src}/nix/component/${name}.nix"; + + # The package set of a deploying system: the set the selection in + # force at `manifest` returns, for that system. + pkgsFor = + manifest: system: + let + what = "the deployment for ${system}"; + pkgSets = final.caisson.integrations.pkgSetsForSystem { inherit context what; } manifest system; + in + final.caisson.integrations.pkgSetOf { inherit context what; } manifest pkgSets; + + # What a component sees of the members of a composite, as the + # `members` module argument: for each member, by name, its + # `outputs`, its `members` in turn and its `absoluteMemberPath`, + # which NixOps4's `state` option reads. It is built from the + # output values of the composite (`outputValues`), which NixOps4 + # computes from the export of the root: by member name, a + # resource's outputs or a composite's own output values. A + # member's config is not reachable through it. + membersView = + path: outputValues: + builtins.mapAttrs (name: values: { + outputs = values; + members = membersView (path ++ [ name ]) values; + absoluteMemberPath = path ++ [ name ]; + }) outputValues; + + # The args a deferred member takes, as its composite gives them: + # `outputValues`, the member's part of the output values; + # `members`, the composite's view of its members (`membersView`); + # `resourceProviderSystem`, the system NixOps4 runs providers on; + # and `path`, the member path from the root. + deployOf = + what: args: + { + outputValues = args.outputValues or { }; + members = args.members or { }; + path = args.path or [ ]; + } + // { + resourceProviderSystem = + args.resourceProviderSystem or (throw "caisson-deferred: needs ${what}.resourceProviderSystem"); + }; + + # The args a composite, with args `deploy`, gives its member + # `name`. + memberArgs = deploy: name: { + outputValues = deploy.outputValues.${name} or { }; + members = membersView deploy.path deploy.outputValues; + inherit (deploy) resourceProviderSystem; + path = deploy.path ++ [ name ]; + }; + + # The args NixOps4 passes, as the declaration hands them to the + # root composite. + rootArgs = + { outputValues, resourceProviderSystem }: + { + inherit outputValues resourceProviderSystem; + members = { }; + path = [ ]; + }; + + # The arg `resourceProviderSystem`, which every component takes. + # A system name is no secret, so it is cacheable. + systemArg = final.caisson-core.mkArg { + cacheable = true; + type = "string"; + description = "The system NixOps4 runs providers and the programs of resources on."; + }; + + # The options of the declaration's class: none of its own, so + # only the framework's and what its modules declare. + declarationOptions = { ... }: { }; + + # The root of the deployment, as NixOps4's `mkRoot` returns it, + # over the deferred root composite `root` of the declaration. + # `rootFunction` finalizes it with the output values NixOps4 + # passes and returns its export. `getProviders` finalizes it with + # no output values for `system` and builds the providers its root + # declares, as the `getProviders` of NixOps4 does; providers + # declared beneath the root are not collected, as upstream + # collects none of them either. + rootOf = + { manifest, root }: + { + _type = "nixops4Component"; + rootFunction = + { + outputValues, + resourceProviderSystem, + ... + }: + (root.finalize { + args = rootArgs { inherit outputValues resourceProviderSystem; }; + argsSource = "NixOps4"; + }).outputs.export; + getProviders = + { system }: + let + finalized = root.finalize { + args = rootArgs { + outputValues = { }; + resourceProviderSystem = system; + }; + argsSource = "getProviders, with no output values"; + }; + serializable = builtins.mapAttrs (_: provider: { + inherit (provider) executable args; + }) finalized.outputs.providerConfigs; + in + (pkgsFor manifest system).writeText "nixops-root-providers" '' + Store path contents subject to change + ${builtins.toJSON serializable} + ''; + # The deferred root composite, for a tool that walks the + # deployment ahead of time. + caisson.deferredRoot = root; + }; + + # The evaluator's call, on the lib of the view being evaluated: + # the options of the class, the framework modules of the class, the + # selection and the configuration's module, with the composition's + # pinned sources as the `inputs` special argument. `ecosystemArgs`, + # which only the twin's pattern admits, is merged over the call + # last. + evaluate = + args: + { lib, manifest }: + let + selection = lib.caisson.integrations; + registry = lib.caisson.modules.caisson-nixops4 or { }; + frameworkModules = selection.frameworkModules "caisson-nixops4" registry; + moduleImports = selection.moduleImportsOf "caisson-nixops4" { inherit lib manifest; } args; + name = manifest.name or null; + configModule = selection.configModuleOf { + inherit lib name; + class = "caisson-nixops4"; + configModule = args.configModule or null; + }; + moduleList = [ + (selection.caissonModule { + key = "caisson.caisson-nixops4.declarationOptions"; + file = "caisson-caisson-nixops4:declarationOptions"; + module = declarationOptions; + }) + ] + ++ frameworkModules + ++ moduleImports registry + ++ (if configModule == null then [ ] else [ configModule ]); + evaluated = lib.evalModules ( + { + class = "caisson-nixops4"; + specialArgs = { + inherit lib; + inputs = manifest.sources; + } + // (if (args.specialArgs or null) != null then args.specialArgs else { }); + modules = moduleList; + } + // (if (args.ecosystemArgs or null) != null then args.ecosystemArgs else { }) + ); + composites = evaluated.config.caisson.caisson-nixops4-composite.configurations; + # The root composite, the composite declared under the name + # `root`: a `nixops4` output holds one root. + root = + if builtins.attrNames composites == [ "root" ] then + composites.root + else + throw '' + ${context}: a nixops4 configuration holds one root composite, declared as + `caisson.caisson-nixops4-composite.configurations.root`, and this one declares + ${ + if composites == { } then + "none" + else + builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") (builtins.attrNames composites)) + }. + ''; + in + { + value = evaluated; + ecosystems.nixops4 = ecosystemAccountOf { + explicit = args.ecosystemSrc or null; + inherit manifest; + }; + modules = moduleList; + outputs.root = rootOf { inherit manifest root; }; + }; + + # A nixops4 configuration is published as `nixops4`, an output that + # holds a single value: the root of the deployment. + exportsTo = { + attrset = "nixops4"; + single = true; + value = manifest: manifest.outputs.root; + }; + + configuration = + args: + final.caisson.integrations.mkModuleConfiguration { + selection = { + class = "caisson-nixops4"; + inherit args; + }; + type = "caisson-nixops4"; + defaultPkgs = args.defaultPkgs or null; + inherit exportsTo; + evaluate = evaluate args; + }; + + integration = final.caisson.integrations.mkIntegration { + name = "caisson-nixops4"; + class = "caisson-nixops4"; + # What these return is a configuration, a function of + # `{ name, parent }`: the parent that declares it under + # `caisson.caisson-nixops4.configurations.` finalizes it. + mkConfiguration = + { + # The declaration's module: the root composite, declared as + # `caisson.caisson-nixops4-composite.configurations.root`. + # When absent, the configuration registered under the + # configuration's name, if any. + configModule ? null, + # The package set the deployment runs on: a function that + # receives the package sets available, by package config + # name, and returns the set to run on. + defaultPkgs ? null, + # The NixOps4 flake; resolved from the composition's + # declarations when absent. + ecosystemSrc ? null, + # The selection over the caisson-nixops4 class of the + # registry, replacing the default of the class. + moduleImports ? null, + # A selection added to that selection, whichever it is. + extraModuleImports ? null, + # Extra module arguments, merged over those the framework + # supplies. + specialArgs ? null, + }@args: + configuration args; + # The same arguments and `ecosystemArgs`, the evaluator's + # arguments merged over the composed call last. + mkConfigurationWithEcosystemArgs = + { + configModule ? null, + defaultPkgs ? null, + ecosystemSrc ? null, + moduleImports ? null, + extraModuleImports ? null, + specialArgs ? null, + ecosystemArgs ? null, + }@args: + configuration args; + # What the composite and resource integrations build on. + extra = { + inherit + exportsTo + ecosystemSrcOf + componentModule + pkgsFor + membersView + deployOf + memberArgs + rootArgs + systemArg + ; + # Where the NixOps4 source of a configuration came from, as a + # manifest records it under `ecosystems.nixops4`. + ecosystemAccountOf = { explicit, manifest }: ecosystemAccountOf { inherit explicit manifest; }; + }; + }; + in + { + caisson = (prev.caisson or { }) // { + caisson-nixops4 = integration.namespace; + }; + }; + +} diff --git a/modules/caisson-nixops4-composite/core b/modules/caisson-nixops4-composite/core new file mode 120000 index 00000000..3715354a --- /dev/null +++ b/modules/caisson-nixops4-composite/core @@ -0,0 +1 @@ +../generic/core \ No newline at end of file diff --git a/modules/caisson-nixops4-resource/core b/modules/caisson-nixops4-resource/core new file mode 120000 index 00000000..3715354a --- /dev/null +++ b/modules/caisson-nixops4-resource/core @@ -0,0 +1 @@ +../generic/core \ No newline at end of file diff --git a/modules/caisson-nixops4-resource/default/default.nix b/modules/caisson-nixops4-resource/default/default.nix new file mode 100644 index 00000000..5120e61f --- /dev/null +++ b/modules/caisson-nixops4-resource/default/default.nix @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: MIT +# +# The default module of the caisson-nixops4-resource class. +# +# It selects the NixOS adapter, registered as `nixos` in the class, +# which deploys the NixOS configuration declared beneath a resource. +# A resource that holds no NixOS configuration is not changed by it. +# +# NixOps4 runs `bin/apply` from the toplevel of the NixOS +# configuration it deploys. This module adds caisson's NixOS module +# `nixops4-apply`, which builds that script, to the modules that the +# nearest NixOS configurations beneath a resource import by default. +# It reaches a NixOS configuration inside a structural group or a disk +# layout beneath the resource. A NixOS configuration declared inside +# that one is not deployed by the resource and does not get it. +# +# A NixOS configuration that passes `moduleImports` selects its modules +# itself, and then it lists `nixops4-apply` itself, as +# `caisson/nixops4-apply` in a flake that lists caisson in `projects`. +# +# `closure-lib` is the library of caisson, where the adapter and +# `nixops4-apply` are registered under those names. +{ closure-lib, ... }: +{ ... }: +{ + imports = [ + # The option under `caisson.forNearestChildren` is declared by the + # core module of the class. + closure-lib.caisson.modules.caisson-nixops4-resource.core + closure-lib.caisson.modules.caisson-nixops4-resource.nixos + ]; + + caisson.forNearestChildren.extraDefaultModuleImports.nixos = _lib: [ + closure-lib.caisson.modules.nixos.nixops4-apply + ]; +} diff --git a/modules/caisson-nixops4-resource/nixos/default.nix b/modules/caisson-nixops4-resource/nixos/default.nix new file mode 100644 index 00000000..b853d14c --- /dev/null +++ b/modules/caisson-nixops4-resource/nixos/default.nix @@ -0,0 +1,158 @@ +# SPDX-License-Identifier: MIT +# +# The NixOS adapter of the caisson-nixops4-resource class: it deploys +# the NixOS configuration declared beneath a resource, as +# nixops4-nixos deploys the NixOS configuration of its resource. +# +# When the resource holds a NixOS configuration, the adapter makes the +# resource an `exec` resource of NixOps4's `local` provider, unless the +# resource sets `type` itself. Its inputs copy the toplevel of the +# configuration to the host with `nix copy` and run its +# `bin/apply switch` over SSH. The SSH settings are the options +# nixops4-nixos declares, under the same names: `ssh.user`, `ssh.host`, +# `ssh.hostPublicKey`, `ssh.opts`, `copy.substituteOnDestination` and +# `sudo.enable`. The host defaults to the name of the resource. The +# shell that runs the script is `bash` from the package set caisson +# selects for the resource, for the system NixOps4 runs providers on. +# +# A resource holds one NixOS configuration to deploy, evaluated for one +# system; the adapter refuses more. +# +# The default module of the class selects this module. A resource that +# selects its modules itself lists it as +# `caisson/caisson-nixops4-resource/nixos` in a flake that lists caisson +# in `projects`. +{ ... }: +{ + config, + lib, + providers, + resourceProviderSystem, + ... +}: +let + inherit (lib) mkOption types; + manifest = lib.caisson-core.evalManifest; + context = "caisson's NixOS adapter of the caisson-nixops4-resource class"; + declared = config.caisson.nixos.configurations; + # Whether the resource holds a NixOS configuration. The names are + # known in the childless view too, so a resource without one is left + # alone there and in its full evaluation. + holdsMachine = builtins.attrNames declared != [ ]; + # The NixOS configuration the resource deploys: the one declared + # beneath it, evaluated for one system. + machine = + let + names = builtins.attrNames declared; + only = declared.${builtins.head names}; + evaluations = builtins.attrValues only; + in + if builtins.length names != 1 then + throw '' + ${context}: the resource `${manifest.name}` declares the NixOS configurations + ${builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") names)}, and a resource deploys one. + Declare each as the NixOS configuration of a resource of its own. + '' + else if (only._type or null) == "caisson-manifest" then + only + else if builtins.length evaluations == 1 then + builtins.head evaluations + else + throw '' + ${context}: the NixOS configuration `${builtins.head names}` of the resource + `${manifest.name}` is evaluated for ${builtins.concatStringsSep ", " (builtins.attrNames only)}, + and a resource deploys one evaluation. Declare it where a single system is in force. + ''; + nixos = machine.value; + target = (if config.ssh.user == null then "" else "${config.ssh.user}@") + config.ssh.host; + # The host key is the only key the SSH client accepts for the host, + # as nixops4-nixos has it. + knownHosts = builtins.toFile "known_hosts" '' + ${config.ssh.host} ${config.ssh.hostPublicKey} + ''; + src = lib.caisson.caisson-nixops4.ecosystemSrcOf { + explicit = null; + inherit manifest; + }; + bash = lib.getExe (lib.caisson.caisson-nixops4.pkgsFor manifest resourceProviderSystem).bash; +in +{ + _file = "caisson:caisson-nixops4-resource/nixos"; + options = { + nixos.configuration = mkOption { + type = types.raw; + readOnly = true; + description = '' + The NixOS configuration this resource deploys: the one declared + beneath it, as caisson evaluated it. + ''; + }; + ssh.user = mkOption { + type = types.nullOr types.str; + default = "root"; + description = '' + The user name of the SSH connection. When `null`, the SSH client + chooses it from its configuration. + ''; + }; + ssh.host = mkOption { + type = types.str; + default = manifest.name; + defaultText = lib.literalMD "the name of the resource"; + description = "The host the SSH client connects to."; + }; + ssh.hostPublicKey = mkOption { + type = types.str; + description = '' + The public key of the host. It is the only key the SSH client + accepts for the host. + ''; + }; + ssh.opts = mkOption { + type = types.str; + default = ""; + description = "Further options of the SSH client, as one shell word."; + }; + copy.substituteOnDestination = mkOption { + type = types.bool; + default = true; + description = '' + Whether the host tries its substituters for the paths it receives + before they are copied to it. + ''; + }; + sudo.enable = mkOption { + type = types.bool; + default = config.ssh.user != "root"; + defaultText = lib.literalMD ''`false` iff `ssh.user` is `"root"`''; + description = "Whether the configuration is applied with `sudo`."; + }; + }; + config = lib.mkIf holdsMachine { + # NixOps4's `local` provider, whose `exec` type deploys the + # configuration. A resource that declares `providers.local` or sets + # `type` replaces them. + providers.local = lib.mkDefault src.modules.nixops4Provider.local; + type = lib.mkDefault providers.local.exec; + nixos.configuration = nixos; + inputs = { + executable = bash; + args = [ + "-c" + # ConnectTimeout and ConnectionAttempts keep a host that is not + # up from hanging the deployment. Password and + # keyboard-interactive authentication are off because stdin is + # not connected. StrictHostKeyChecking with the configured key + # alone refuses a host the configuration does not name. + '' + set -euo pipefail + export NIX_SSHOPTS="-o ConnectTimeout=10 -o ConnectionAttempts=12 -o KbdInteractiveAuthentication=no -o PasswordAuthentication=no -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${knownHosts} "${lib.escapeShellArg config.ssh.opts} + nix copy --to "ssh-ng://$0" "$1" --no-check-sigs --extra-experimental-features nix-command${lib.optionalString config.copy.substituteOnDestination " --substitute-on-destination"} + ssh $NIX_SSHOPTS "$0" "${lib.optionalString config.sudo.enable "sudo "}$1/bin/apply switch" + '' + target + nixos.config.system.build.toplevel + ]; + }; + }; +} diff --git a/modules/caisson-nixops4/core b/modules/caisson-nixops4/core new file mode 120000 index 00000000..3715354a --- /dev/null +++ b/modules/caisson-nixops4/core @@ -0,0 +1 @@ +../generic/core \ No newline at end of file diff --git a/modules/generic/core/default.nix b/modules/generic/core/default.nix index 71c61c43..c8692ea1 100644 --- a/modules/generic/core/default.nix +++ b/modules/generic/core/default.nix @@ -33,6 +33,9 @@ "caisson-disko" "caisson-facter" "caisson-microvm" + "caisson-nixops4" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "colmena" "darwin" "disko" diff --git a/modules/nixos/nixops4-apply/default.nix b/modules/nixos/nixops4-apply/default.nix new file mode 100644 index 00000000..f309a593 --- /dev/null +++ b/modules/nixos/nixops4-apply/default.nix @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: MIT +# +# A NixOS module for a NixOS configuration that NixOps4 deploys. It +# imports the `apply` NixOS module of nixops4-nixos, which adds +# `bin/apply` to the toplevel: the script that sets the system profile +# and switches to the configuration. NixOps4 runs `bin/apply switch` on +# the host after it copies the toplevel there. Where nixpkgs declares +# `system.apply` itself, the module of nixops4-nixos adds nothing. +# +# The default module of the caisson-nixops4-resource class adds this +# module to the nearest NixOS configurations beneath a resource, the +# configurations the resource deploys. +# +# The nixops4-nixos source is the one the composition declares or pins +# under the name `nixops4-nixos`, read from the library of the +# evaluation. +{ ... }: +{ lib, ... }: +let + src = + let + found = lib.caisson-core.ecosystemSrc "nixops4-nixos"; + in + if found == null then + throw '' + caisson's `nixops4-apply` NixOS module needs the nixops4-nixos source. + Declare `defaultEcosystemSrc.nixops4-nixos` in the mkLib call, or pin a + source named `nixops4-nixos` in the `sources` passed to mkLib. + '' + else + found; +in +{ + _file = "caisson:nixops4-apply"; + imports = [ src.modules.nixos.apply ]; +} diff --git a/tests/dependencies/flake.lock b/tests/dependencies/flake.lock index 77798be2..8d23d640 100644 --- a/tests/dependencies/flake.lock +++ b/tests/dependencies/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793243975, - "narHash": "sha256-OzlG2Nhl/kU2UUPq+ARH/GKUHmOqAxQu5Ug/+aT1ECc=", + "lastModified": 1793010712, + "narHash": "sha256-otbvrbV58kVW+yBVdSH+bt82O89iaz4Ul664iyKyq68=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "6868bca2682e96d56714bc9c9adcf0048fbe132f", + "rev": "ddcff6816293044822aa2c9418ca2fcf51643c1e", "type": "github" }, "original": { @@ -39,6 +39,23 @@ "type": "github" } }, + "crane": { + "flake": false, + "locked": { + "lastModified": 1758758545, + "narHash": "sha256-NU5WaEdfwF6i8faJ2Yh+jcK9vVFrofLcwlD/mP65JrI=", + "owner": "ipetkov", + "repo": "crane", + "rev": "95d528a5f54eaba0d12102249ce42f4d01f4e364", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "ref": "v0.21.1", + "repo": "crane", + "type": "github" + } + }, "devshell": { "inputs": { "nixpkgs": [ @@ -80,6 +97,30 @@ "type": "github" } }, + "dream2nix": { + "inputs": { + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "nixpkgs" + ], + "purescript-overlay": "purescript-overlay", + "pyproject-nix": "pyproject-nix" + }, + "locked": { + "lastModified": 1779999259, + "narHash": "sha256-aLAG+TG7stjp0AOI2uvn4agxrrws2Q+fXMDS8ZuHYmo=", + "owner": "nix-community", + "repo": "dream2nix", + "rev": "dd8a3ebbbfcfafc139dba1323a6c443ec034c9f5", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "dream2nix", + "type": "github" + } + }, "flake-compat": { "flake": false, "locked": { @@ -113,6 +154,54 @@ } }, "flake-compat_3": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_4": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_5": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_6": { "flake": false, "locked": { "lastModified": 1777699697, @@ -128,7 +217,7 @@ "url": "https://git.lix.systems/lix-project/flake-compat.git" } }, - "flake-compat_4": { + "flake-compat_7": { "flake": false, "locked": { "lastModified": 1767039857, @@ -201,6 +290,51 @@ } }, "flake-parts_4": { + "inputs": { + "nixpkgs-lib": [ + "nixops4", + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1788450739, + "narHash": "sha256-glZLQlzIn1fXH6PazR2iUmTo7kzzyYSshrWhLS9TqCU=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "31729ca8cbdb4fa927b34e5f4353e6a83f39e993", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_5": { + "inputs": { + "nixpkgs-lib": [ + "nixops4", + "nix-bindings-rust", + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1733312601, + "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_6": { "inputs": { "nixpkgs-lib": [ "nixvim", @@ -221,7 +355,7 @@ "type": "github" } }, - "flake-parts_5": { + "flake-parts_7": { "inputs": { "nixpkgs-lib": [ "system-manager", @@ -243,7 +377,7 @@ "type": "github" } }, - "flake-parts_6": { + "flake-parts_8": { "inputs": { "nixpkgs-lib": [ "terranix", @@ -300,6 +434,71 @@ "type": "github" } }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [ + "nixops4", + "nix" + ], + "nixpkgs": [ + "nixops4", + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1788267358, + "narHash": "sha256-nt+lUqYVpc9Y6JeMd2WmXzCDojasdadKo0mWcluvY2Y=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "27555e2624241fb116b49095df4caaee85a25691", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "git-hooks-nix_2": { + "inputs": { + "flake-compat": [ + "nixops4", + "nix-bindings-rust", + "nix" + ], + "gitignore": [ + "nixops4", + "nix-bindings-rust", + "nix" + ], + "nixpkgs": [ + "nixops4", + "nix-bindings-rust", + "nix", + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixops4", + "nix-bindings-rust", + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1734279981, + "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "gitignore": { "inputs": { "nixpkgs": [ @@ -440,6 +639,22 @@ "type": "github" } }, + "mk-naked-shell": { + "flake": false, + "locked": { + "lastModified": 1681286841, + "narHash": "sha256-3XlJrwlR0nBiREnuogoa5i1b4+w/XPe0z8bbrJASw0g=", + "owner": "90-008", + "repo": "mk-naked-shell", + "rev": "7612f828dd6f22b7fb332cc69440e839d7ffe6bd", + "type": "github" + }, + "original": { + "owner": "90-008", + "repo": "mk-naked-shell", + "type": "github" + } + }, "mnw": { "locked": { "lastModified": 1784154424, @@ -455,6 +670,90 @@ "type": "github" } }, + "nix": { + "inputs": { + "flake-compat": "flake-compat_3", + "flake-parts": "flake-parts_4", + "git-hooks-nix": "git-hooks-nix", + "nixpkgs": [ + "nixops4", + "nixpkgs" + ], + "nixpkgs-23-11": "nixpkgs-23-11", + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1790895260, + "narHash": "sha256-GGeEfw8mghwKHhsjDzmYXoP97JLoggLxF2nns4MRBG8=", + "owner": "NixOS", + "repo": "nix", + "rev": "f83a7f39e6f8c9aa254de32e51e37733fb07d7e6", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nix", + "type": "github" + } + }, + "nix-bindings-rust": { + "inputs": { + "flake-parts": [ + "nixops4", + "flake-parts" + ], + "nix": "nix_2", + "nix-cargo-integration": [ + "nixops4", + "nix-cargo-integration" + ], + "nixpkgs": [ + "nixops4", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1778917363, + "narHash": "sha256-p2X0V5FbS7pa13lSSi+hvH1LMwXBypRZ217NSWXmPK4=", + "owner": "nixops4", + "repo": "nix-bindings-rust", + "rev": "49468a3da7fe52b22ab9ef73d147cc4ec573ed8e", + "type": "github" + }, + "original": { + "owner": "nixops4", + "ref": "0.2.1", + "repo": "nix-bindings-rust", + "type": "github" + } + }, + "nix-cargo-integration": { + "inputs": { + "crane": "crane", + "dream2nix": "dream2nix", + "mk-naked-shell": "mk-naked-shell", + "nixpkgs": [ + "nixops4", + "nixpkgs" + ], + "parts": "parts", + "rust-overlay": "rust-overlay", + "treefmt": "treefmt" + }, + "locked": { + "lastModified": 1790856477, + "narHash": "sha256-C4RZUuKpBb2OS+MzuJizybH4kauwqDumj8BoEdmZwOg=", + "owner": "yusdacra", + "repo": "nix-cargo-integration", + "rev": "5630c7aa777ca213a7ea78f3b6a7e165ab46a30f", + "type": "github" + }, + "original": { + "owner": "yusdacra", + "repo": "nix-cargo-integration", + "type": "github" + } + }, "nix-darwin": { "inputs": { "nixpkgs": [ @@ -595,6 +894,33 @@ "type": "github" } }, + "nix_2": { + "inputs": { + "flake-compat": "flake-compat_4", + "flake-parts": "flake-parts_5", + "git-hooks-nix": "git-hooks-nix_2", + "nixpkgs": [ + "nixops4", + "nix-bindings-rust", + "nixpkgs" + ], + "nixpkgs-23-11": "nixpkgs-23-11_2", + "nixpkgs-regression": "nixpkgs-regression_2" + }, + "locked": { + "lastModified": 1777584486, + "narHash": "sha256-/2wsg4bAs8t6QlSOHAiz2qeO5OXpYQqFYPdp2R3AxyI=", + "owner": "NixOS", + "repo": "nix", + "rev": "2d26f28966311c3b734c03f50565d15615dd4c2b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nix", + "type": "github" + } + }, "nixidy": { "inputs": { "flake-utils": "flake-utils_2", @@ -617,6 +943,58 @@ "type": "github" } }, + "nixops4": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "nix": "nix", + "nix-bindings-rust": "nix-bindings-rust", + "nix-cargo-integration": "nix-cargo-integration", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1791020006, + "narHash": "sha256-codUYzTTiynymi0RA8b4fX8dtUiHfVtNZAx56y7XI6I=", + "owner": "nixops4", + "repo": "nixops4", + "rev": "41858b44eda2a2d9d0f0610bf5b314688a7505d7", + "type": "github" + }, + "original": { + "owner": "nixops4", + "repo": "nixops4", + "type": "github" + } + }, + "nixops4-nixos": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "nixops4-nixos": [ + "nixops4-nixos" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1791211623, + "narHash": "sha256-OkYw3jYX5tx6+WZMV7L8VXhU4pg6y+JrsY4aLML6jBQ=", + "owner": "nixops4", + "repo": "nixops4-nixos", + "rev": "4c7ceab7a80bafcd2ce90337a6c386cb0ef0ec43", + "type": "github" + }, + "original": { + "owner": "nixops4", + "repo": "nixops4-nixos", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1789785513, @@ -633,6 +1011,38 @@ "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, + "nixpkgs-23-11_2": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-lib": { "locked": { "lastModified": 1785031560, @@ -678,9 +1088,41 @@ "type": "github" } }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-regression_2": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, "nixvim": { "inputs": { - "flake-parts": "flake-parts_4", + "flake-parts": "flake-parts_6", "nixpkgs": [ "nixpkgs" ], @@ -702,7 +1144,7 @@ }, "nvf": { "inputs": { - "flake-compat": "flake-compat_3", + "flake-compat": "flake-compat_6", "mnw": "mnw", "nixpkgs": [ "nixpkgs" @@ -722,6 +1164,28 @@ "type": "github" } }, + "parts": { + "inputs": { + "nixpkgs-lib": [ + "nixops4", + "nix-cargo-integration", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1788450739, + "narHash": "sha256-glZLQlzIn1fXH6PazR2iUmTo7kzzyYSshrWhLS9TqCU=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "31729ca8cbdb4fa927b34e5f4353e6a83f39e993", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "pre-commit-hooks-nix": { "inputs": { "flake-compat": [ @@ -750,6 +1214,54 @@ "type": "github" } }, + "purescript-overlay": { + "inputs": { + "flake-compat": "flake-compat_5", + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "dream2nix", + "nixpkgs" + ], + "slimlock": "slimlock" + }, + "locked": { + "lastModified": 1728546539, + "narHash": "sha256-Sws7w0tlnjD+Bjck1nv29NjC5DbL6nH5auL9Ex9Iz2A=", + "owner": "thomashoneyman", + "repo": "purescript-overlay", + "rev": "4ad4c15d07bd899d7346b331f377606631eb0ee4", + "type": "github" + }, + "original": { + "owner": "thomashoneyman", + "repo": "purescript-overlay", + "type": "github" + } + }, + "pyproject-nix": { + "inputs": { + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "dream2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1763017646, + "narHash": "sha256-Z+R2lveIp6Skn1VPH3taQIuMhABg1IizJd8oVdmdHsQ=", + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "rev": "47bd6f296502842643078d66128f7b5e5370790c", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "type": "github" + } + }, "root": { "inputs": { "caisson-core": "caisson-core", @@ -765,6 +1277,8 @@ "nix-unit": "nix-unit", "nix-vm-test": "nix-vm-test", "nixidy": "nixidy", + "nixops4": "nixops4", + "nixops4-nixos": "nixops4-nixos", "nixpkgs": "nixpkgs", "nixpkgs-lib": "nixpkgs-lib_3", "nixvim": "nixvim", @@ -774,6 +1288,52 @@ "treefmt-nix": "treefmt-nix_3" } }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1790843958, + "narHash": "sha256-jtYAaAXT1gxEtzmeYmOGu0IvKPle5J39BOFbgl6os+A=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "b2ccad3b67335d368eef49ec48b9c8166858e73d", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "slimlock": { + "inputs": { + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "dream2nix", + "purescript-overlay", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688756706, + "narHash": "sha256-xzkkMv3neJJJ89zo3o2ojp7nFeaZc2G0fYwNXNJRFlo=", + "owner": "thomashoneyman", + "repo": "slimlock", + "rev": "cf72723f59e2340d24881fd7bf61cb113b4c407c", + "type": "github" + }, + "original": { + "owner": "thomashoneyman", + "repo": "slimlock", + "type": "github" + } + }, "spectrum": { "flake": false, "locked": { @@ -808,7 +1368,7 @@ }, "system-manager": { "inputs": { - "flake-compat": "flake-compat_4", + "flake-compat": "flake-compat_7", "nixpkgs": [ "nixpkgs" ], @@ -922,7 +1482,7 @@ }, "terranix": { "inputs": { - "flake-parts": "flake-parts_6", + "flake-parts": "flake-parts_8", "import-tree": "import-tree_2", "nixpkgs": [ "nixpkgs" @@ -943,6 +1503,28 @@ "type": "github" } }, + "treefmt": { + "inputs": { + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1786901030, + "narHash": "sha256-WSFCsDSE5ffgD2MqzkM2CYjeFiKhRF/dJUN8uedb6YE=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "27b3b12a8e6375f28ebe122f07d230ca5459bbfa", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + }, "treefmt-nix": { "inputs": { "nixpkgs": [ @@ -1011,7 +1593,7 @@ "system-manager", "flake-compat" ], - "flake-parts": "flake-parts_5", + "flake-parts": "flake-parts_7", "nixpkgs": [ "system-manager", "nixpkgs" diff --git a/tests/dependencies/flake.nix b/tests/dependencies/flake.nix index 7d8af0fb..fd82a307 100644 --- a/tests/dependencies/flake.nix +++ b/tests/dependencies/flake.nix @@ -37,6 +37,12 @@ inputs.nix-darwin.inputs.nixpkgs.follows = "nixpkgs"; inputs.microvm.url = "github:microvm-nix/microvm.nix"; inputs.microvm.inputs.nixpkgs.follows = "nixpkgs"; + inputs.nixops4.url = "github:nixops4/nixops4"; + inputs.nixops4.inputs.nixpkgs.follows = "nixpkgs"; + inputs.nixops4.inputs.flake-parts.follows = "flake-parts"; + inputs.nixops4-nixos.url = "github:nixops4/nixops4-nixos"; + inputs.nixops4-nixos.inputs.nixpkgs.follows = "nixpkgs"; + inputs.nixops4-nixos.inputs.flake-parts.follows = "flake-parts"; # The VMs of other distributions the system-manager VM test boots # (tests/system-manager). inputs.nix-vm-test.url = "github:numtide/nix-vm-test"; diff --git a/tests/pinned-world/default.nix b/tests/pinned-world/default.nix index 95072ca7..c61e8f17 100644 --- a/tests/pinned-world/default.nix +++ b/tests/pinned-world/default.nix @@ -70,6 +70,9 @@ let registered."caisson/facter" registered."caisson/caisson-microvm" registered."caisson/microvm" + registered."caisson/caisson-nixops4" + registered."caisson/caisson-nixops4-composite" + registered."caisson/caisson-nixops4-resource" registered."caisson/darwin" registered."caisson/terranix" registered."caisson/system-manager" @@ -94,6 +97,9 @@ let "caisson-disko" "caisson-facter" "caisson-microvm" + "caisson-nixops4" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "callConsumerFlake" "callFlake" "colmena" @@ -229,6 +235,8 @@ let hjem terranix microvm + nixops4 + nixops4-nixos ; }; projects = { @@ -370,6 +378,95 @@ let }; colmenaTop = colmenaTopWith { }; + # A top holding a caisson-nixops4 configuration named `nixops4`, + # whose root composite declares a resource for each machine given, by + # name, holding a NixOS configuration of that name. `root` is a + # further module of the root composite, `resources` a further module + # of each resource, by name, `defaultPkgs` the selection of the + # caisson-nixops4 configuration and `machineArgs` further arguments of + # each machine's constructor. + nixops4TopWith = + { + root ? { }, + resources ? { }, + defaultPkgs ? null, + machineArgs ? { }, + }: + machines: + hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4.configurations.nixops4 = lib.caisson.caisson-nixops4.mkConfiguration { + inherit defaultPkgs; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.root = + lib.caisson.caisson-nixops4-composite.mkConfiguration + { + configModule = + { lib, ... }: + { + imports = [ root ]; + caisson.caisson-nixops4-resource.configurations = builtins.mapAttrs ( + name: nixosModule: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + configModule = + { lib, ... }: + { + imports = [ (resources.${name} or { }) ]; + caisson.nixos.configurations.${name} = lib.caisson.nixos.mkConfiguration ( + { configModule = nixosModule; } // machineArgs + ); + }; + } + ) machines; + }; + }; + }; + }; + }; + }; + + # The export of a root as NixOps4's command reads it: the root + # function called with the deploying system and the outputs of each + # resource, here `stub` for every output the resource declares, at + # the fixpoint, as the command closes it. + nixops4Export = + root: + let + outputsOf = builtins.mapAttrs ( + _: member: + if member ? resource then + builtins.mapAttrs (_: _: "stub") member.resource.outputsSkeleton + else + outputsOf member.members + ); + fixpoint = root.rootFunction { + resourceProviderSystem = "x86_64-linux"; + outputValues = outputsOf fixpoint.members; + }; + in + fixpoint; + + # The root composite of a top made by `nixops4TopWith`, finalized as + # NixOps4 finalizes it, with `stub` outputs: its manifest. + nixops4RootManifest = + top: + top.nixops4.caisson.deferredRoot.finalize { + args = { + resourceProviderSystem = "x86_64-linux"; + outputValues = builtins.mapAttrs ( + _: member: builtins.mapAttrs (_: _: "stub") member.resource.outputsSkeleton + ) (nixops4Export top.nixops4).members; + }; + }; + + # A host key for the machines NixOps4 deploys in the probes. + nixops4HostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAiszi43aOWWV7voNgQ1Ifa7LGKwGJfOuiLM1n42h2Y8"; + # What a microvm.nix guest needs beside what microvm.nix's guest # module gives it, which boots it and gives it its file systems. microvmGuestBase = { @@ -637,6 +734,9 @@ let "caisson/facter" "caisson/caisson-microvm" "caisson/microvm" + "caisson/caisson-nixops4" + "caisson/caisson-nixops4-composite" + "caisson/caisson-nixops4-resource" "caisson/darwin" "caisson/terranix" "caisson/system-manager" @@ -744,6 +844,9 @@ let "facter" "caisson-microvm" "microvm" + "caisson-nixops4" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "darwin" "terranix" "system-manager" @@ -2043,6 +2146,163 @@ let } ]; + # A caisson-nixops4 configuration named `nixops4` fills the top's + # `nixops4`, the root NixOps4's command reads. A resource holding a + # NixOS configuration becomes, through the NixOS adapter, an `exec` + # resource of NixOps4's `local` provider that copies the toplevel of + # that configuration and runs its `bin/apply switch` over SSH, to + # the name of the resource as the host when nothing sets one. The + # toplevel carries the `bin/apply` the `nixops4-apply` module + # builds. + nixops4RootEvaluatesEndToEnd = + let + top = nixops4TopWith { + resources.probe.ssh.hostPublicKey = nixops4HostKey; + } { probe = minimalNixosBase; }; + export = nixops4Export top.nixops4; + resource = export.members.probe.resource; + args = resource.inputs.args; + machine = (nixops4RootManifest top).outputs.members.probe.value.config.nixos.configuration; + in + top.nixops4._type == "nixops4Component" + && builtins.attrNames export.members == [ "probe" ] + && resource.type == "exec" + && resource.inputs.executable == pkgs.lib.getExe pkgs.bash + && builtins.elemAt args 2 == "root@probe" + && builtins.elemAt args 3 == "${machine.config.system.build.toplevel}" + && pkgs.lib.hasInfix "$1/bin/apply switch" (builtins.elemAt args 1) + && pkgs.lib.hasInfix "$out/bin/apply" machine.config.system.activatableSystemBuilderCommands + && pkgs.lib.hasInfix nixops4HostKey ( + builtins.readFile ( + builtins.head (builtins.match ".*UserKnownHostsFile=([^ ]*) .*" (builtins.elemAt args 1)) + ) + ); + + # A resource reads the outputs of another member of its composite + # through `members`, and NixOps4 evaluates the root again with the + # outputs it learns. The SSH settings change the command line as + # they do in nixops4-nixos. + nixops4ResourceReadsSiblingOutputs = + let + top = nixops4TopWith { + root = + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.address = + lib.caisson.caisson-nixops4-resource.mkConfiguration + { + configModule = + { providers, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + }; + }; + resources.probe = + { members, ... }: + { + ssh.host = members.address.outputs.stdout; + ssh.hostPublicKey = nixops4HostKey; + ssh.user = "deploy"; + ssh.opts = "-p 2222"; + copy.substituteOnDestination = false; + }; + } { probe = minimalNixosBase; }; + export = nixops4Export top.nixops4; + args = export.members.probe.resource.inputs.args; + script = builtins.elemAt args 1; + in + builtins.attrNames export.members == [ + "address" + "probe" + ] + && builtins.elemAt args 2 == "deploy@stub" + && pkgs.lib.hasInfix "sudo $1/bin/apply switch" script + && pkgs.lib.hasInfix "'-p 2222'" script + && !(pkgs.lib.hasInfix "--substitute-on-destination" script); + + # The shell that runs the deployment comes from the package set + # caisson selects for the resource, inherited from the + # caisson-nixops4 configuration, for the deploying system: here a + # selection whose `bash` is another shell. + nixops4DeploysWithTheSelectedPackageSet = + let + top = nixops4TopWith { + resources.probe.ssh.hostPublicKey = nixops4HostKey; + defaultPkgs = pkgSets: pkgSets.default // { bash = pkgSets.default.dash; }; + } { probe = minimalNixosBase; }; + in + (nixops4Export top.nixops4).members.probe.resource.inputs.executable == pkgs.lib.getExe pkgs.dash; + + # Composite and resource are two classes: a composite has no `type` + # and a resource no `members`, and setting either is an error where + # it is set rather than a member silently dropped. + nixops4CompositeAndResourceAreTwoClasses = + let + evaluates = top: (builtins.tryEval (builtins.deepSeq (nixops4Export top.nixops4) true)).success; + resourceWith = + extra: + nixops4TopWith { + resources.probe = { + ssh.hostPublicKey = nixops4HostKey; + } + // extra; + } { probe = minimalNixosBase; }; + in + evaluates (nixops4TopWith { } { }) + && !(evaluates (nixops4TopWith { root.type = "exec"; } { })) + && evaluates (resourceWith { }) + && !(evaluates (resourceWith { + members.extra = { }; + })); + + # The root composite is a deferred child of the caisson-nixops4 + # configuration: the manifest lists its entry, and nothing beneath + # it is evaluated at flake time. The args it needs name the + # deploying system and, once that is known, the outputs of its + # resources, which are not cacheable unless a resource lists them. + # Finalized, it records which args it was given, not their values. + nixops4RootIsDeferred = + let + top = nixops4TopWith { + resources.probe = { + ssh.hostPublicKey = nixops4HostKey; + cacheableOutputs = [ "stdout" ]; + }; + } { probe = minimalNixosBase; }; + declaration = top.caisson.manifest.children.caisson-nixops4.nixops4; + entry = declaration.children.caisson-nixops4-composite.root; + needed = core.argsNeededFor entry { resourceProviderSystem = "x86_64-linux"; }; + in + core.isDeferred entry + && entry.type == "caisson-nixops4-composite" + && entry.parent.childless + && + core.argNamesOf needed == [ + [ + "outputValues" + "probe" + "stdout" + ] + [ "resourceProviderSystem" ] + ] + && needed.outputValues.probe.stdout.cacheable + && (nixops4RootManifest top).deferred.argNames == core.argNamesOf needed; + + # A provider is evaluated inside the component that declares it. + # `getProviders` evaluates the root without outputs and builds the + # providers the root declares, as NixOps4's `getProviders` does. + nixops4ProvidersBuildWithoutOutputs = + let + top = nixops4TopWith { + root.providers.local = inputs.nixops4.modules.nixops4Provider.local; + } { }; + built = top.nixops4.getProviders { system = "x86_64-linux"; }; + in + pkgs.lib.isDerivation built && pkgs.lib.hasPrefix "nixops-root-providers" built.name; + # A node of the upstream-compatible integration builds the same # system as colmena builds from the same hive. colmenaUpstreamHiveBuildsWhatColmenaBuilds = diff --git a/tests/unit/lib-overlays.nix b/tests/unit/lib-overlays.nix index 9c1dcdad..538cdbfb 100644 --- a/tests/unit/lib-overlays.nix +++ b/tests/unit/lib-overlays.nix @@ -126,6 +126,9 @@ let "facter" "caisson-microvm" "microvm" + "caisson-nixops4" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "terranix" "system-manager" "treefmt" @@ -5690,6 +5693,9 @@ in "facter" "caisson-microvm" "microvm" + "caisson-nixops4" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "treefmt" "nvf" "nixvim" @@ -5709,6 +5715,9 @@ in "caisson-facter" "facter" "microvm" + "caisson-nixops4" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "treefmt" "nvf" "nixvim" From e840c4762fe15af9c42b38048773402758d9a057 Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Mon, 26 Oct 2026 04:08:27 -0700 Subject: [PATCH 2/9] feat: add the upstream-compatible nixops4 integration A nixops4 configuration, named nixops4, takes the root modules of a NixOps4 deployment as NixOps4 users write them for mkRoot, and caisson evaluates every component, with NixOps4's class name nixops4Component and NixOps4's resource, type and providers options. A composite collects members. and decides each member's integration by upstream's test, with a probe evaluation over NixOps4's base modules: a member whose resourceType is defined is a caisson-nixops4-resource configuration, any other a caisson-nixops4-composite configuration. A member that sets type and declares members is refused. The members module argument of a composite holds its members' configs, as upstream's does, so every composite sets unsafeCrossChildReferences for the class of its members. A member that imports nixops4-nixos gets its nixos.module evaluated as a caisson NixOS configuration beneath it: the module nixos of the nixops4Component class forces nixpkgs to a value whose lib.nixosSystem returns that configuration, since nixos.configuration is read-only, and takes the shell from caisson's package set. cacheableOutputs is the allow-list of outputs a tool may cache, by member path. The eval-weight baseline is re-measured: the core module of every class declares the configurations of every integration, so the four integrations of the NixOps4 work add to caisson's overhead in every evaluation. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 + docs/development/repository-structure.md | 4 +- .../caisson-nixops4-composite/default.nix | 18 +- lib-overlays/caisson-nixops4/default.nix | 34 ++ lib-overlays/nixops4/default.nix | 346 ++++++++++++++++++ .../nixos/default.nix | 28 +- modules/generic/core/default.nix | 1 + modules/nixops4/core | 1 + modules/nixops4Component/core | 1 + modules/nixops4Component/default/default.nix | 32 ++ modules/nixops4Component/nixos/default.nix | 53 +++ tests/eval-weight/baseline.json | 24 +- tests/pinned-world/default.nix | 195 +++++++++- 13 files changed, 684 insertions(+), 57 deletions(-) create mode 100644 lib-overlays/nixops4/default.nix create mode 120000 modules/nixops4/core create mode 120000 modules/nixops4Component/core create mode 100644 modules/nixops4Component/default/default.nix create mode 100644 modules/nixops4Component/nixos/default.nix diff --git a/README.md b/README.md index 86ff1c0c..7e1db0f8 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,10 @@ source by. | `facter` | none | a `facter.json` as nixos-facter writes it, with the NixOS configurations that run on it declared inside it; a machine gets it as `hardware.facter.reportPath` | | `caisson-microvm` | `microvm` | microvm.nix guests declared inside the NixOS host that runs them, each guest a NixOS configuration declared inside it that sets its own `microvm.*` options and gets microvm.nix's guest module by default; the host selects caisson's `microvm-host` NixOS module, which defines microvm.nix's `microvm.vms.` from them; the microvm integration we recommend for new configurations | | `microvm` | `microvm` | a microvm.nix guest written as microvm.nix reads it, a NixOS configuration that sets its `microvm.*` options itself, declared inside the NixOS host that runs it; for porting an existing guest | +| `caisson-nixops4` | `nixops4` | a NixOps4 deployment, holding its root composite under the name `root`; the configuration named `nixops4` is published as the `nixops4` output the NixOps4 command reads; the NixOps4 integration we recommend for new configurations | +| `caisson-nixops4-composite` | `nixops4` | a composite of a NixOps4 deployment, evaluated when NixOps4 deploys, with its members, composites and resources, declared beneath it; it declares providers and reads the outputs of its members | +| `caisson-nixops4-resource` | `nixops4` | a resource of a NixOps4 deployment, evaluated when NixOps4 deploys, holding the caisson configurations it deploys; a NixOS configuration declared inside it is deployed over SSH as nixops4-nixos deploys one | +| `nixops4` | `nixops4` | the root modules of a NixOps4 deployment, written as NixOps4 users write them for `mkRoot`, each component evaluated by caisson and each nixops4-nixos machine a caisson NixOS configuration; for porting an existing deployment | | `treefmt` | `treefmt-nix` | the formatters of a tree, as treefmt-nix modules; the configuration named `formatter` is published as the `formatter.` that `nix fmt` runs | | `nvf` | `nvf` | a Neovim written as nvf modules, published as `nvfConfigurations..`, in the shape `neovimConfiguration` returns, kept in a separate Nix profile that the machine, home-manager configuration or hjem configuration it is declared inside activates | | `nixvim` | `nixvim` | a Neovim written as nixvim modules, published as `nixvimConfigurations..`, kept in a profile as an nvf configuration is | diff --git a/docs/development/repository-structure.md b/docs/development/repository-structure.md index 551724d7..8ddf0015 100644 --- a/docs/development/repository-structure.md +++ b/docs/development/repository-structure.md @@ -28,7 +28,9 @@ The layer users reach for: the integrations (`structural`, `flake-parts`, `nixpkgs`, `nixos`, `nixos-minimal`, `home-manager`, `home-manager-minimal`, `darwin`, `hjem`, `caisson-colmena`, `colmena`, `caisson-disko`, `disko`, `caisson-facter`, `facter`, -`caisson-microvm`, `microvm`, `terranix`, `system-manager`, `treefmt`, +`caisson-microvm`, `microvm`, `caisson-nixops4`, +`caisson-nixops4-composite`, `caisson-nixops4-resource`, `nixops4`, +`terranix`, `system-manager`, `treefmt`, `nvf`, `nixvim`, `topology`, `kubenix`, `nixidy`), each a library overlay contributing its `lib.caisson` namespace, registering its module diff --git a/lib-overlays/caisson-nixops4-composite/default.nix b/lib-overlays/caisson-nixops4-composite/default.nix index 7dd2c29a..0813b578 100644 --- a/lib-overlays/caisson-nixops4-composite/default.nix +++ b/lib-overlays/caisson-nixops4-composite/default.nix @@ -72,13 +72,13 @@ ''; # The members finalized with the args the composite gives them, - # by name: each a manifest. + # `memberArgs deploy name`, by name: each a manifest. finalizeMembers = - deploy: entries: + memberArgs: deploy: entries: builtins.mapAttrs ( name: entry: entry.finalize { - args = nixops4.memberArgs deploy name; + args = memberArgs deploy name; argsSource = "the composite at ${showPath deploy.path}"; } ) entries; @@ -113,10 +113,11 @@ # A composite configuration of the module class `class`, evaluated # with the modules `baseModules` returns for the NixOps4 source # and the args, and the `members` module argument `membersArg` - # returns. The native integration evaluates its class with - # NixOps4's provider declarations and the view of the members' - # outputs; the upstream-compatible one builds on this with - # NixOps4's class and its own modules. + # returns. It gives each member the args `memberArgs` returns. The + # native integration evaluates its class with NixOps4's provider + # declarations, and the view of the members' outputs as `members` + # in the composite and in each member; the upstream-compatible one + # builds on this with NixOps4's class and its own modules. compositeWith = { class, @@ -125,6 +126,7 @@ membersArg ? { deploy, ... }: nixops4.membersView deploy.path deploy.outputValues, + memberArgs ? nixops4.memberArgs, }: args: let @@ -158,7 +160,7 @@ if manifest.childless then { } else - finalizeMembers deploy (memberEntriesOf deploy evaluated.config); + finalizeMembers memberArgs deploy (memberEntriesOf deploy evaluated.config); membersModule = selection.caissonModule { key = "caisson.${type}.members"; file = "caisson-${type}:members"; diff --git a/lib-overlays/caisson-nixops4/default.nix b/lib-overlays/caisson-nixops4/default.nix index 10a6c081..01aac931 100644 --- a/lib-overlays/caisson-nixops4/default.nix +++ b/lib-overlays/caisson-nixops4/default.nix @@ -141,6 +141,38 @@ path = [ ]; }; + # The NixOS configuration a resource named `resource` deploys, from + # the NixOS configurations declared beneath it (`declared`): the + # only one, evaluated for one system. `context` names the module + # that asks, for the errors. + machineOf = + { + context, + resource, + declared, + }: + let + names = builtins.attrNames declared; + only = declared.${builtins.head names}; + evaluations = builtins.attrValues only; + in + if builtins.length names != 1 then + throw '' + ${context}: the resource `${resource}` declares the NixOS configurations + ${builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") names)}, and a resource deploys one. + Declare each as the NixOS configuration of a resource of its own. + '' + else if (only._type or null) == "caisson-manifest" then + only + else if builtins.length evaluations == 1 then + builtins.head evaluations + else + throw '' + ${context}: the NixOS configuration `${builtins.head names}` of the resource + `${resource}` is evaluated for ${builtins.concatStringsSep ", " (builtins.attrNames only)}, + and a resource deploys one evaluation. Declare it where a single system is in force. + ''; + # The arg `resourceProviderSystem`, which every component takes. # A system name is no secret, so it is cacheable. systemArg = final.caisson-core.mkArg { @@ -343,7 +375,9 @@ deployOf memberArgs rootArgs + rootOf systemArg + machineOf ; # Where the NixOps4 source of a configuration came from, as a # manifest records it under `ecosystems.nixops4`. diff --git a/lib-overlays/nixops4/default.nix b/lib-overlays/nixops4/default.nix new file mode 100644 index 00000000..c0fcf7a1 --- /dev/null +++ b/lib-overlays/nixops4/default.nix @@ -0,0 +1,346 @@ +# SPDX-License-Identifier: MIT +# +# The nixops4 integration, the upstream-compatible one: it takes the +# modules of a root component as NixOps4 users write them for NixOps4's +# `mkRoot`, of NixOps4's class `nixops4Component`, and caisson evaluates +# every component of the deployment. The native integrations, +# caisson-nixops4 and its composite and resource, take caisson's form +# instead. +# +# A nixops4 configuration, named `nixops4` for the output it fills, +# declares the root composite beneath it under the name `root` from +# those modules, and publishes the root as the caisson-nixops4 +# configuration does. Every component is evaluated with NixOps4's +# class name, so a module that sets `_class = "nixops4Component"` +# loads, and with NixOps4's resource, type and providers options. +# +# Upstream declares `members` and the resource options on every +# component and decides which kind a component is when its export is +# read: a component whose `resourceType` is defined is a resource. A +# composite here collects the definitions of each member under +# `members.` and decides the same way, with a probe evaluation +# of the member's modules over NixOps4's own base modules. A member +# becomes a caisson-nixops4-resource or a caisson-nixops4-composite +# configuration declared beneath the composite, evaluated with the +# same modules. A member that sets `type` and also has members is +# refused; upstream drops the members. +# +# The `members` module argument is the one upstream gives: in a +# composite, the configs of its members, which a composite reads to +# wire one member to another (`members.db.outputs.address`); in a +# resource, nothing. A composite reads its members' configs through its +# full evaluation, so every composite sets `unsafeCrossChildReferences` +# for the class of its members. +# +# A member that imports nixops4-nixos's module gets its `nixos.module` +# evaluated as a caisson NixOS configuration beneath it, rather than by +# the `nixosSystem` call of nixops4-nixos: see the module `nixos` of the +# `nixops4Component` class. +{ + mkLibOverlay, + ... +}: +{ + + imports = [ + # The library of nixpkgs, which this overlay calls through the + # composed library, imported by key so it is composed wherever + # this is. + ((mkLibOverlay ../nixpkgs-lib) // { key = "nixpkgs-lib"; }) + # What an integration is written from, imported by key so it is + # composed wherever this integration is. + ((mkLibOverlay ../integrations) // { key = "integrations"; }) + # The native integrations, whose evaluators this one builds on. + ((mkLibOverlay ../caisson-nixops4) // { key = "caisson-nixops4"; }) + ((mkLibOverlay ../caisson-nixops4-composite) // { key = "caisson-nixops4-composite"; }) + ((mkLibOverlay ../caisson-nixops4-resource) // { key = "caisson-nixops4-resource"; }) + ]; + + overlay = + final: prev: + let + native = final.caisson.caisson-nixops4; + composites = final.caisson.caisson-nixops4-composite; + resources = final.caisson.caisson-nixops4-resource; + context = "lib.caisson.nixops4"; + upstreamClass = "nixops4Component"; + + # The options every upstream component has beside `members`: + # NixOps4's resource, type and providers options. + componentBase = src: [ + (native.componentModule src "resource") + (native.componentModule src "provider-values") + (native.componentModule src "provider-declarations") + ]; + + # The modules of a component, as the configuration's module. + modulesModule = + modules: + final.caisson.integrations.caissonModule { + key = "caisson.nixops4.component"; + file = "caisson-nixops4:component"; + module = { + imports = modules; + }; + }; + + # Which caisson integration the member `name` of the composite at + # `path` becomes, by upstream's test: a resource when its + # `resourceType` is defined, a composite otherwise. The probe + # evaluates the member's modules over NixOps4's base modules, + # with the special arguments the member receives. + kindOf = + { + lib, + src, + specialArgs, + path, + }: + name: module: + let + probe = lib.evalModules { + class = upstreamClass; + inherit specialArgs; + modules = [ + "${src}/nix/component/base-modules.nix" + module + ]; + }; + isResource = probe.options.resourceType.isDefined; + hasMembers = builtins.attrNames probe.config.members != [ ]; + in + if isResource && hasMembers then + throw '' + ${context}: the member ${composites.showPath (path ++ [ name ])} sets `type`, which makes it a + resource, and declares members, which a resource does not hold. + NixOps4 would drop them; declare them in a composite instead. + '' + else if isResource then + "caisson-nixops4-resource" + else + "caisson-nixops4-composite"; + + # The module of an upstream composite that turns `members.` + # into the configurations declared beneath it. + membersModule = + ctx: + { src, deploy }: + final.caisson.integrations.caissonModule { + key = "caisson.nixops4.members"; + file = "caisson-nixops4:members"; + module = + { config, lib, ... }: + let + kinds = builtins.mapAttrs (kindOf { + inherit lib src; + inherit (deploy) path; + specialArgs = ctx.specialArgs // { + inherit lib; + inherit (deploy) resourceProviderSystem; + }; + }) config.members; + ofKind = kind: lib.filterAttrs (name: _: kinds.${name} == kind) config.members; + in + { + options.members = lib.mkOption { + type = lib.types.lazyAttrsOf lib.types.deferredModule; + default = { }; + description = '' + The member components, by name, as NixOps4 declares them: a + member that sets `type` is a resource, any other a composite. + ''; + }; + # A composite reads its members' configs, through its full + # evaluation. + config.caisson.forDirectChildren.unsafeCrossChildReferences.${upstreamClass} = true; + config.caisson.caisson-nixops4-composite.configurations = builtins.mapAttrs ( + _: module: compositeOf ctx [ module ] + ) (ofKind "caisson-nixops4-composite"); + config.caisson.caisson-nixops4-resource.configurations = builtins.mapAttrs ( + _: module: resourceOf ctx [ module ] + ) (ofKind "caisson-nixops4-resource"); + }; + }; + + # The outputs of the resource at `path` that a tool may cache, from + # the allow-list of the nixops4 configuration. + cacheableModule = + ctx: deploy: + final.caisson.integrations.caissonModule { + key = "caisson.nixops4.cacheableOutputs"; + file = "caisson-nixops4:cacheableOutputs"; + module = + { lib, ... }: + { + options.cacheableOutputs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = ctx.cacheableOutputs.${builtins.concatStringsSep "." deploy.path} or [ ]; + readOnly = true; + description = '' + The outputs of this resource a tool may cache, from + `cacheableOutputs` of the nixops4 configuration. + ''; + }; + }; + }; + + # The arguments the native constructors receive for a component + # with `modules`. + componentArgs = ctx: modules: { + configModule = modulesModule modules; + inherit (ctx) ecosystemSrc; + specialArgs = ctx.specialArgs; + }; + + # An upstream composite: NixOps4's class and options, `members` + # collected and declared beneath as configurations, and the + # configs of the members as its `members` argument. A member's + # own `members` argument holds the members it declares itself. + compositeOf = + ctx: modules: + composites.compositeWith { + class = upstreamClass; + baseModules = + { src, deploy }: + componentBase src + ++ [ + (membersModule ctx { inherit src deploy; }) + ]; + membersArg = { finalized, ... }: builtins.mapAttrs (_: member: member.value.config) finalized; + memberArgs = deploy: name: native.memberArgs deploy name // { members = { }; }; + } (componentArgs ctx modules); + + # An upstream resource: NixOps4's class and options, and the + # outputs the configuration allows a tool to cache. + resourceOf = + ctx: modules: + resources.resourceWith { + class = upstreamClass; + baseModules = { src, deploy }: componentBase src ++ [ (cacheableModule ctx deploy) ]; + } (componentArgs ctx modules); + + # The evaluator's call, on the lib of the view being evaluated: + # the framework modules of the class, the selection, the + # configuration's module and the root composite made from + # `modules`, with the composition's pinned sources as the `inputs` + # special argument. `ecosystemArgs`, which only the twin's pattern + # admits, is merged over the call last. + evaluate = + args: + { lib, manifest }: + let + selection = lib.caisson.integrations; + registry = lib.caisson.modules.nixops4 or { }; + frameworkModules = selection.frameworkModules "nixops4" registry; + moduleImports = selection.moduleImportsOf "nixops4" { inherit lib manifest; } args; + ctx = { + ecosystemSrc = args.ecosystemSrc or null; + specialArgs = { + inputs = manifest.sources; + } + // (if (args.specialArgs or null) != null then args.specialArgs else { }); + cacheableOutputs = args.cacheableOutputs or { }; + }; + rootModule = selection.caissonModule { + key = "caisson.nixops4.root"; + file = "caisson-nixops4:root"; + module = { + caisson.caisson-nixops4-composite.configurations.root = compositeOf ctx (args.modules or [ ]); + }; + }; + moduleList = frameworkModules ++ moduleImports registry ++ [ rootModule ]; + evaluated = lib.evalModules ( + { + class = "nixops4"; + specialArgs = { + inherit lib; + inputs = manifest.sources; + }; + modules = moduleList; + } + // (if (args.ecosystemArgs or null) != null then args.ecosystemArgs else { }) + ); + in + { + value = evaluated; + ecosystems.nixops4 = native.ecosystemAccountOf { + explicit = args.ecosystemSrc or null; + inherit manifest; + }; + modules = moduleList; + outputs.root = native.rootOf { + inherit manifest; + root = evaluated.config.caisson.caisson-nixops4-composite.configurations.root; + }; + }; + + configuration = + args: + final.caisson.integrations.mkModuleConfiguration { + selection = { + class = "nixops4"; + inherit args; + }; + type = "nixops4"; + defaultPkgs = args.defaultPkgs or null; + inherit (native) exportsTo; + evaluate = evaluate args; + }; + + integration = final.caisson.integrations.mkIntegration { + name = "nixops4"; + class = "nixops4"; + # What these return is a configuration, a function of + # `{ name, parent }`: the parent that declares it under + # `caisson.nixops4.configurations.` finalizes it. + mkConfiguration = + { + # The modules of the root component, as NixOps4's `mkRoot` + # takes them. + modules ? [ ], + # Special arguments of every component, as `mkRoot` takes + # them for the root. + specialArgs ? null, + # The outputs a tool that evaluates the deployment ahead of + # time may cache, by member path, the names from the root + # joined with `.`: `{ "web.address" = [ "stdout" ]; }`. + # NixOps4 marks no output as secret, so none is cacheable + # unless listed. + cacheableOutputs ? { }, + # The package set the deployment runs on: a function that + # receives the package sets available, by package config + # name, and returns the set to run on. + defaultPkgs ? null, + # The NixOps4 flake; resolved from the composition's + # declarations when absent. + ecosystemSrc ? null, + # The selection over the nixops4 class of the registry, + # replacing the default of the class. + moduleImports ? null, + # A selection added to that selection, whichever it is. + extraModuleImports ? null, + }@args: + configuration args; + # The same arguments and `ecosystemArgs`, the evaluator's + # arguments merged over the composed call last. + mkConfigurationWithEcosystemArgs = + { + modules ? [ ], + specialArgs ? null, + cacheableOutputs ? { }, + defaultPkgs ? null, + ecosystemSrc ? null, + moduleImports ? null, + extraModuleImports ? null, + ecosystemArgs ? null, + }@args: + configuration args; + }; + in + { + caisson = (prev.caisson or { }) // { + nixops4 = integration.namespace; + }; + }; + +} diff --git a/modules/caisson-nixops4-resource/nixos/default.nix b/modules/caisson-nixops4-resource/nixos/default.nix index b853d14c..b3aab040 100644 --- a/modules/caisson-nixops4-resource/nixos/default.nix +++ b/modules/caisson-nixops4-resource/nixos/default.nix @@ -41,29 +41,11 @@ let holdsMachine = builtins.attrNames declared != [ ]; # The NixOS configuration the resource deploys: the one declared # beneath it, evaluated for one system. - machine = - let - names = builtins.attrNames declared; - only = declared.${builtins.head names}; - evaluations = builtins.attrValues only; - in - if builtins.length names != 1 then - throw '' - ${context}: the resource `${manifest.name}` declares the NixOS configurations - ${builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") names)}, and a resource deploys one. - Declare each as the NixOS configuration of a resource of its own. - '' - else if (only._type or null) == "caisson-manifest" then - only - else if builtins.length evaluations == 1 then - builtins.head evaluations - else - throw '' - ${context}: the NixOS configuration `${builtins.head names}` of the resource - `${manifest.name}` is evaluated for ${builtins.concatStringsSep ", " (builtins.attrNames only)}, - and a resource deploys one evaluation. Declare it where a single system is in force. - ''; - nixos = machine.value; + nixos = + (lib.caisson.caisson-nixops4.machineOf { + inherit context declared; + resource = manifest.name; + }).value; target = (if config.ssh.user == null then "" else "${config.ssh.user}@") + config.ssh.host; # The host key is the only key the SSH client accepts for the host, # as nixops4-nixos has it. diff --git a/modules/generic/core/default.nix b/modules/generic/core/default.nix index c8692ea1..2f7c7743 100644 --- a/modules/generic/core/default.nix +++ b/modules/generic/core/default.nix @@ -46,6 +46,7 @@ "kubenix" "microvm" "nixidy" + "nixops4" "nixos" "nixpkgs" "nixvim" diff --git a/modules/nixops4/core b/modules/nixops4/core new file mode 120000 index 00000000..3715354a --- /dev/null +++ b/modules/nixops4/core @@ -0,0 +1 @@ +../generic/core \ No newline at end of file diff --git a/modules/nixops4Component/core b/modules/nixops4Component/core new file mode 120000 index 00000000..3715354a --- /dev/null +++ b/modules/nixops4Component/core @@ -0,0 +1 @@ +../generic/core \ No newline at end of file diff --git a/modules/nixops4Component/default/default.nix b/modules/nixops4Component/default/default.nix new file mode 100644 index 00000000..41c4953f --- /dev/null +++ b/modules/nixops4Component/default/default.nix @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: MIT +# +# The default module of the nixops4Component class, the class the +# nixops4 integration evaluates components with. +# +# It selects the module `nixos` of the class, which evaluates the +# `nixos.module` of a member that imports nixops4-nixos as a caisson +# NixOS configuration beneath the member. +# +# NixOps4 runs `bin/apply` from the toplevel of the NixOS +# configuration it deploys. This module adds caisson's NixOS module +# `nixops4-apply`, which builds that script, to the modules that the +# nearest NixOS configurations beneath a component import by default, +# as nixops4-nixos adds its `apply` module to the configuration it +# evaluates. +# +# `closure-lib` is the library of caisson, where these modules are +# registered under those names. +{ closure-lib, ... }: +{ ... }: +{ + imports = [ + # The option under `caisson.forNearestChildren` is declared by the + # core module of the class. + closure-lib.caisson.modules.nixops4Component.core + closure-lib.caisson.modules.nixops4Component.nixos + ]; + + caisson.forNearestChildren.extraDefaultModuleImports.nixos = _lib: [ + closure-lib.caisson.modules.nixos.nixops4-apply + ]; +} diff --git a/modules/nixops4Component/nixos/default.nix b/modules/nixops4Component/nixos/default.nix new file mode 100644 index 00000000..da6e9263 --- /dev/null +++ b/modules/nixops4Component/nixos/default.nix @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: MIT +# +# The module of the nixops4Component class, the class the nixops4 +# integration evaluates components with, for a member that imports the +# NixOS resource module of nixops4-nixos +# (`nixops4-nixos.modules.nixops4Component.nixos`). +# +# That module declares the options a NixOS member is written with +# (`nixpkgs`, `nixos.module`, `nixos.specialArgs`, `ssh.*`, +# `copy.substituteOnDestination`, `sudo.enable`), evaluates +# `nixos.module` with `config.nixpkgs.lib.nixosSystem`, and takes the +# shell its deploy script runs in from its own nixpkgs. In caisson the +# member's `nixos.module` is a caisson NixOS configuration declared +# beneath the member, under the member's name, with +# `nixos.specialArgs` as its special arguments. This module declares +# it, makes `nixpkgs.lib.nixosSystem` return it in place of evaluating +# one, and takes the shell from the package set caisson selects for the +# member, for the system NixOps4 runs providers on. A member that does +# not import nixops4-nixos is not changed. +# +# The default module of the class selects this module. +{ ... }: +{ + config, + lib, + options, + resourceProviderSystem, + ... +}: +let + manifest = lib.caisson-core.evalManifest; + usesNixos = options ? nixos && options.nixos ? module; + machine = lib.caisson.caisson-nixops4.machineOf { + context = "caisson's `nixos` module of the nixops4Component class"; + resource = manifest.name; + declared = config.caisson.nixos.configurations; + }; + bash = lib.getExe (lib.caisson.caisson-nixops4.pkgsFor manifest resourceProviderSystem).bash; +in +{ + _file = "caisson:nixops4Component/nixos"; + config = lib.optionalAttrs usesNixos { + caisson.nixos.configurations.${manifest.name} = lib.caisson.nixos.mkConfiguration { + configModule = config.nixos.module; + specialArgs = config.nixos.specialArgs; + }; + # nixops4-nixos builds `nixos.configuration` with this call. + nixpkgs = lib.mkForce { + lib.nixosSystem = _: machine.value; + }; + inputs.executable = lib.mkForce bash; + }; +} diff --git a/tests/eval-weight/baseline.json b/tests/eval-weight/baseline.json index be5026ac..666d5c70 100644 --- a/tests/eval-weight/baseline.json +++ b/tests/eval-weight/baseline.json @@ -1,22 +1,22 @@ { "caisson-overhead": { - "nrThunks": 62007, - "nrValues": 77486, - "nrEnvs": 45988, - "nrFunctionCalls": 38520, - "nrPrimOpCalls": 22785, - "totalAllocBytes": 3889344, + "nrThunks": 70319, + "nrValues": 91203, + "nrEnvs": 53548, + "nrFunctionCalls": 44802, + "nrPrimOpCalls": 25801, + "totalAllocBytes": 4618224, "nixpkgsEvals": 0, "nixpkgsLibEvals": 0, "moduleSystemEvals": 11 }, "minimal-consumer-total": { - "nrThunks": 253742, - "nrValues": 734160, - "nrEnvs": 139119, - "nrFunctionCalls": 113586, - "nrPrimOpCalls": 73028, - "totalAllocBytes": 46965344, + "nrThunks": 262054, + "nrValues": 747877, + "nrEnvs": 146679, + "nrFunctionCalls": 119868, + "nrPrimOpCalls": 76044, + "totalAllocBytes": 47694224, "nixpkgsEvals": 1, "nixpkgsLibEvals": 2, "moduleSystemEvals": 28 diff --git a/tests/pinned-world/default.nix b/tests/pinned-world/default.nix index c61e8f17..8131cb25 100644 --- a/tests/pinned-world/default.nix +++ b/tests/pinned-world/default.nix @@ -73,6 +73,7 @@ let registered."caisson/caisson-nixops4" registered."caisson/caisson-nixops4-composite" registered."caisson/caisson-nixops4-resource" + registered."caisson/nixops4" registered."caisson/darwin" registered."caisson/terranix" registered."caisson/system-manager" @@ -132,6 +133,7 @@ let "mkPkgOverlays" "modules" "nixidy" + "nixops4" "nixos" "nixos-minimal" "nixpkgs" @@ -437,30 +439,76 @@ let nixops4Export = root: let - outputsOf = builtins.mapAttrs ( - _: member: - if member ? resource then - builtins.mapAttrs (_: _: "stub") member.resource.outputsSkeleton - else - outputsOf member.members - ); fixpoint = root.rootFunction { resourceProviderSystem = "x86_64-linux"; - outputValues = outputsOf fixpoint.members; + outputValues = nixops4StubOutputs fixpoint.members; }; in fixpoint; - # The root composite of a top made by `nixops4TopWith`, finalized as - # NixOps4 finalizes it, with `stub` outputs: its manifest. + # The output values NixOps4 passes for the members of an export, with + # `stub` for every output a resource declares. + nixops4StubOutputs = builtins.mapAttrs ( + _: member: + if member ? resource then + builtins.mapAttrs (_: _: "stub") member.resource.outputsSkeleton + else + nixops4StubOutputs member.members + ); + + # The root composite of a top's `nixops4`, finalized as NixOps4 + # finalizes it, with `stub` outputs: its manifest. nixops4RootManifest = top: top.nixops4.caisson.deferredRoot.finalize { args = { resourceProviderSystem = "x86_64-linux"; - outputValues = builtins.mapAttrs ( - _: member: builtins.mapAttrs (_: _: "stub") member.resource.outputsSkeleton - ) (nixops4Export top.nixops4).members; + outputValues = nixops4StubOutputs (nixops4Export top.nixops4).members; + }; + }; + + # A top holding a nixops4 configuration named `nixops4` made from the + # root modules given, as NixOps4's `mkRoot` takes them, with the + # allow-list `cacheableOutputs`. + nixops4UpstreamTopWith = + { + cacheableOutputs ? { }, + }: + modules: + hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.nixops4.configurations.nixops4 = lib.caisson.nixops4.mkConfiguration { + inherit modules cacheableOutputs; + }; + }; + }; + + # Root modules as a NixOps4 user writes them: a resource whose output + # another member reads, a NixOS machine through nixops4-nixos, and a + # nested composite holding a resource. + nixops4UpstreamRoot = + { providers, members, ... }: + { + _class = "nixops4Component"; + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members.address = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + members.probe = { + imports = [ inputs.nixops4-nixos.modules.nixops4Component.nixos ]; + type = providers.local.exec; + nixpkgs = inputs.nixpkgs; + nixos.module = minimalNixosBase; + ssh.host = members.address.outputs.stdout; + ssh.hostPublicKey = nixops4HostKey; + }; + members.group.members.inner = { + type = providers.local.exec; + inputs.executable = "/bin/false"; }; }; @@ -737,6 +785,7 @@ let "caisson/caisson-nixops4" "caisson/caisson-nixops4-composite" "caisson/caisson-nixops4-resource" + "caisson/nixops4" "caisson/darwin" "caisson/terranix" "caisson/system-manager" @@ -847,6 +896,7 @@ let "caisson-nixops4" "caisson-nixops4-composite" "caisson-nixops4-resource" + "nixops4" "darwin" "terranix" "system-manager" @@ -2303,6 +2353,125 @@ let in pkgs.lib.isDerivation built && pkgs.lib.hasPrefix "nixops-root-providers" built.name; + # The upstream-compatible integration takes root modules as a + # NixOps4 user writes them, `_class` included, and NixOps4 reads + # the same export from it: a member that sets `type` is a resource, + # any other a composite, to any depth. A member reads another + # member's outputs through `members`. + nixops4UpstreamRootEvaluates = + let + top = nixops4UpstreamTopWith { } [ nixops4UpstreamRoot ]; + export = nixops4Export top.nixops4; + probe = export.members.probe.resource; + in + top.nixops4._type == "nixops4Component" + && + builtins.attrNames export.members == [ + "address" + "group" + "probe" + ] + && export.members.address.resource.type == "exec" + && builtins.attrNames export.members.group.members == [ "inner" ] + && export.members.group.members.inner.resource.inputs.executable == "/bin/false" + && builtins.elemAt probe.inputs.args 2 == "root@stub"; + + # A member that imports nixops4-nixos gets its `nixos.module` + # evaluated as a caisson NixOS configuration beneath it, with the + # `bin/apply` NixOps4 runs, and builds the system NixOS builds from + # the same module and the `apply` module nixops4-nixos adds, named + # for the member as caisson names it. The shell comes from the + # package set caisson selects. + nixops4UpstreamNixosIsACaissonConfiguration = + let + top = nixops4UpstreamTopWith { } [ nixops4UpstreamRoot ]; + probe = (nixops4Export top.nixops4).members.probe.resource; + member = (nixops4RootManifest top).outputs.members.probe; + machine = member.children.system.x86_64-linux.children.nixos.probe; + upstream = import "${inputs.nixpkgs}/nixos/lib/eval-config.nix" { + system = null; + modules = [ + minimalNixosBase + inputs.nixops4-nixos.modules.nixos.apply + { + nixpkgs.pkgs = pkgs; + networking.hostName = "probe"; + } + ]; + }; + in + machine.type == "nixos" + && probe.inputs.executable == pkgs.lib.getExe pkgs.bash + && builtins.elemAt probe.inputs.args 3 == "${machine.value.config.system.build.toplevel}" + && pkgs.lib.hasInfix "$out/bin/apply" machine.value.config.system.activatableSystemBuilderCommands + && + machine.value.config.system.build.toplevel.drvPath == upstream.config.system.build.toplevel.drvPath; + + # The shell of a nixops4-nixos member comes from the package set + # caisson selects for the nixops4 configuration, not from the + # nixpkgs of nixops4-nixos: here a selection whose `bash` is + # another shell. + nixops4UpstreamDeploysWithTheSelectedPackageSet = + let + top = hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.nixops4.configurations.nixops4 = lib.caisson.nixops4.mkConfiguration { + modules = [ nixops4UpstreamRoot ]; + defaultPkgs = pkgSets: pkgSets.default // { bash = pkgSets.default.dash; }; + }; + }; + }; + in + (nixops4Export top.nixops4).members.probe.resource.inputs.executable == pkgs.lib.getExe pkgs.dash; + + # Every upstream composite reads its members' configs through its + # full evaluation: each member records the composite as the level + # whose children it reads. + nixops4UpstreamMembersReadTheFullComposite = + let + root = nixops4RootManifest (nixops4UpstreamTopWith { } [ nixops4UpstreamRoot ]); + probe = root.outputs.members.probe; + in + probe.crossChildLevel != null + && !probe.crossChildLevel.childless + && probe.crossChildLevel.name == "root"; + + # A member that sets `type` and declares members is refused, where + # NixOps4 drops the members. + nixops4UpstreamRefusesAResourceWithMembers = + let + evaluates = top: (builtins.tryEval (builtins.deepSeq (nixops4Export top.nixops4) true)).success; + rootWith = + extra: + { providers, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members.probe = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + } + // extra; + }; + in + evaluates (nixops4UpstreamTopWith { } [ (rootWith { }) ]) + && !(evaluates (nixops4UpstreamTopWith { } [ (rootWith { members.extra = { }; }) ])); + + # The allow-list of the nixops4 configuration says which outputs a + # tool may cache, by member path; every other output is not + # cacheable. + nixops4UpstreamCacheableOutputsAreAllowListed = + let + top = nixops4UpstreamTopWith { + cacheableOutputs."group.inner" = [ "stdout" ]; + } [ nixops4UpstreamRoot ]; + entry = top.caisson.manifest.children.nixops4.nixops4.children.caisson-nixops4-composite.root; + needed = core.argsNeededFor entry { resourceProviderSystem = "x86_64-linux"; }; + in + needed.outputValues.group.inner.stdout.cacheable && !needed.outputValues.address.stdout.cacheable; + # A node of the upstream-compatible integration builds the same # system as colmena builds from the same hive. colmenaUpstreamHiveBuildsWhatColmenaBuilds = From d463b263772883e54426ac5055906d9c84ec211a Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Mon, 26 Oct 2026 04:25:22 -0700 Subject: [PATCH 3/9] feat(nixops4): discover recorded outputs from the state resources A root composite, native or upstream-compatible, takes stateResources, the member paths of the resources that hold the state of the deployment, as nixops4 state dump takes them. It then gives discoverArgs: for each system a package config builds a set for, a program that runs nixops4 state dump for each of them in the working directory and prints the recorded outputs as outputValues, with the deploying system. A finalized composite publishes deferredChildArgs, the args it gives a member, so a tool can finalize a member's entry itself. caisson-core is pinned to the head of nix-caisson/caisson-core#57, where discoverArgs may be derivations by system or a function of the entry's name and parent. Co-Authored-By: Claude Opus 5.5 --- flake.lock | 6 +- .../caisson-nixops4-composite/default.nix | 25 +++++++ lib-overlays/caisson-nixops4/default.nix | 51 ++++++++++++++ lib-overlays/nixops4/default.nix | 14 +++- tests/dependencies/flake.lock | 6 +- tests/pinned-world/default.nix | 66 +++++++++++++------ 6 files changed, 140 insertions(+), 28 deletions(-) diff --git a/flake.lock b/flake.lock index 6f584c22..d3b3e85f 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793010712, - "narHash": "sha256-otbvrbV58kVW+yBVdSH+bt82O89iaz4Ul664iyKyq68=", + "lastModified": 1793013687, + "narHash": "sha256-hjBo9BHN9eHL3FDe+7qs8P4hDTE8Mx2Gx7/x0xnu4bs=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "ddcff6816293044822aa2c9418ca2fcf51643c1e", + "rev": "ea612fa6368360c430e56dd33869bc5ef10f2e74", "type": "github" }, "original": { diff --git a/lib-overlays/caisson-nixops4-composite/default.nix b/lib-overlays/caisson-nixops4-composite/default.nix index 0813b578..11e323f3 100644 --- a/lib-overlays/caisson-nixops4-composite/default.nix +++ b/lib-overlays/caisson-nixops4-composite/default.nix @@ -201,6 +201,9 @@ inherit (deploy) path; # The members, by name, each finalized with its args. members = finalized; + # The args the composite gives its member of a name, for + # a tool that finalizes a member's entry itself. + deferredChildArgs = memberArgs deploy; # What NixOps4 reads of the composite. export.members = builtins.mapAttrs (_: member: member.outputs.export) finalized; # The providers it declares, which `getProviders` builds @@ -221,6 +224,21 @@ in final.caisson-core.mkDeferredConfiguration { inherit type class; + # A program that reads the outputs NixOps4 recorded, for the + # composite that lists the resources holding its state. + discoverArgs = + if (args.stateResources or [ ]) == [ ] then + null + else + { parent, ... }: + nixops4.discoverArgsOf { + manifest = parent; + src = nixops4.ecosystemSrcOf { + explicit = args.ecosystemSrc or null; + manifest = parent; + }; + inherit (args) stateResources; + }; # The deploying system, and the outputs of the members, which # depend on the members the composite declares, so on the args # known so far. @@ -282,6 +300,12 @@ # Extra module arguments, merged over those the framework # supplies. specialArgs ? null, + # For the root composite: the member paths of the resources + # that hold the state of the deployment, as + # `nixops4 state dump` takes them, such as `default.state`. + # A tool reads the outputs NixOps4 recorded from them + # (`discoverArgs`). + stateResources ? [ ], }@args: configuration args; # The same arguments and `ecosystemArgs`, the evaluator's @@ -293,6 +317,7 @@ moduleImports ? null, extraModuleImports ? null, specialArgs ? null, + stateResources ? [ ], ecosystemArgs ? null, }@args: configuration args; diff --git a/lib-overlays/caisson-nixops4/default.nix b/lib-overlays/caisson-nixops4/default.nix index 01aac931..1d33ac51 100644 --- a/lib-overlays/caisson-nixops4/default.nix +++ b/lib-overlays/caisson-nixops4/default.nix @@ -173,6 +173,56 @@ and a resource deploys one evaluation. Declare it where a single system is in force. ''; + # The `discoverArgs` of a root composite, by system: a program + # that prints the args NixOps4 recorded, for every system the + # package configs at `manifest` build a set for. It runs + # `nixops4 state dump` for each of `stateResources` in the working + # directory, which NixOps4 finds the flake from, passes its own + # arguments to NixOps4, such as `--override-input`, and prints + # `{ outputValues, resourceProviderSystem }`. The state of a + # resource at `a.state` covers the composite `a`: its resources' + # recorded outputs, and its nested composites in turn. Only + # resources with a state have recorded outputs. + discoverArgsOf = + { + manifest, + src, + stateResources, + }: + let + systems = final.unique ( + builtins.concatMap (config: builtins.attrNames ((config.children or { }).nixpkgs or { })) ( + builtins.attrValues (manifest.pkgSets or { }) + ) + ); + programFor = + system: + let + pkgs = pkgsFor manifest system; + in + pkgs.writeShellApplication { + name = "caisson-nixops4-discover"; + runtimeInputs = [ + src.packages.${system}.nixops4 + pkgs.jq + ]; + text = '' + merged='{}' + for state in ${final.escapeShellArgs stateResources}; do + dump=$(nixops4 "$@" state dump "$state") + merged=$(jq -cn --argjson merged "$merged" --argjson dump "$dump" --arg state "$state" ' + def outputValues: + ((.resources // {}) | map_values(.output_properties)) + + ((.deployments // {}) | map_values(outputValues)); + $merged * ({} | setpath($state | split(".") | .[:-1]; $dump | outputValues)) + ') + done + jq -cn --argjson merged "$merged" '{ outputValues: $merged, resourceProviderSystem: "${system}" }' + ''; + }; + in + final.genAttrs systems programFor; + # The arg `resourceProviderSystem`, which every component takes. # A system name is no secret, so it is cacheable. systemArg = final.caisson-core.mkArg { @@ -378,6 +428,7 @@ rootOf systemArg machineOf + discoverArgsOf ; # Where the NixOps4 source of a configuration came from, as a # manifest records it under `ecosystems.nixops4`. diff --git a/lib-overlays/nixops4/default.nix b/lib-overlays/nixops4/default.nix index c0fcf7a1..d9750d8f 100644 --- a/lib-overlays/nixops4/default.nix +++ b/lib-overlays/nixops4/default.nix @@ -140,6 +140,8 @@ }; }) config.members; ofKind = kind: lib.filterAttrs (name: _: kinds.${name} == kind) config.members; + # The state resources are the root's to list. + memberCtx = builtins.removeAttrs ctx [ "stateResources" ]; in { options.members = lib.mkOption { @@ -154,10 +156,10 @@ # evaluation. config.caisson.forDirectChildren.unsafeCrossChildReferences.${upstreamClass} = true; config.caisson.caisson-nixops4-composite.configurations = builtins.mapAttrs ( - _: module: compositeOf ctx [ module ] + _: module: compositeOf memberCtx [ module ] ) (ofKind "caisson-nixops4-composite"); config.caisson.caisson-nixops4-resource.configurations = builtins.mapAttrs ( - _: module: resourceOf ctx [ module ] + _: module: resourceOf memberCtx [ module ] ) (ofKind "caisson-nixops4-resource"); }; }; @@ -190,6 +192,7 @@ configModule = modulesModule modules; inherit (ctx) ecosystemSrc; specialArgs = ctx.specialArgs; + stateResources = ctx.stateResources or [ ]; }; # An upstream composite: NixOps4's class and options, `members` @@ -240,6 +243,7 @@ } // (if (args.specialArgs or null) != null then args.specialArgs else { }); cacheableOutputs = args.cacheableOutputs or { }; + stateResources = args.stateResources or [ ]; }; rootModule = selection.caissonModule { key = "caisson.nixops4.root"; @@ -307,6 +311,11 @@ # NixOps4 marks no output as secret, so none is cacheable # unless listed. cacheableOutputs ? { }, + # The member paths of the resources that hold the state of + # the deployment, as `nixops4 state dump` takes them, such + # as `default.state`. A tool reads the outputs NixOps4 + # recorded from them. + stateResources ? [ ], # The package set the deployment runs on: a function that # receives the package sets available, by package config # name, and returns the set to run on. @@ -328,6 +337,7 @@ modules ? [ ], specialArgs ? null, cacheableOutputs ? { }, + stateResources ? [ ], defaultPkgs ? null, ecosystemSrc ? null, moduleImports ? null, diff --git a/tests/dependencies/flake.lock b/tests/dependencies/flake.lock index 8d23d640..22e320c0 100644 --- a/tests/dependencies/flake.lock +++ b/tests/dependencies/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793010712, - "narHash": "sha256-otbvrbV58kVW+yBVdSH+bt82O89iaz4Ul664iyKyq68=", + "lastModified": 1793013687, + "narHash": "sha256-hjBo9BHN9eHL3FDe+7qs8P4hDTE8Mx2Gx7/x0xnu4bs=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "ddcff6816293044822aa2c9418ca2fcf51643c1e", + "rev": "ea612fa6368360c430e56dd33869bc5ef10f2e74", "type": "github" }, "original": { diff --git a/tests/pinned-world/default.nix b/tests/pinned-world/default.nix index 8131cb25..f07d1252 100644 --- a/tests/pinned-world/default.nix +++ b/tests/pinned-world/default.nix @@ -390,6 +390,7 @@ let nixops4TopWith = { root ? { }, + rootArgs ? { }, resources ? { }, defaultPkgs ? null, machineArgs ? { }, @@ -407,26 +408,29 @@ let { caisson.caisson-nixops4-composite.configurations.root = lib.caisson.caisson-nixops4-composite.mkConfiguration - { - configModule = - { lib, ... }: - { - imports = [ root ]; - caisson.caisson-nixops4-resource.configurations = builtins.mapAttrs ( - name: nixosModule: - lib.caisson.caisson-nixops4-resource.mkConfiguration { - configModule = - { lib, ... }: - { - imports = [ (resources.${name} or { }) ]; - caisson.nixos.configurations.${name} = lib.caisson.nixos.mkConfiguration ( - { configModule = nixosModule; } // machineArgs - ); - }; - } - ) machines; - }; - }; + ( + rootArgs + // { + configModule = + { lib, ... }: + { + imports = [ root ]; + caisson.caisson-nixops4-resource.configurations = builtins.mapAttrs ( + name: nixosModule: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + configModule = + { lib, ... }: + { + imports = [ (resources.${name} or { }) ]; + caisson.nixos.configurations.${name} = lib.caisson.nixos.mkConfiguration ( + { configModule = nixosModule; } // machineArgs + ); + }; + } + ) machines; + }; + } + ); }; }; }; @@ -2341,6 +2345,28 @@ let && needed.outputValues.probe.stdout.cacheable && (nixops4RootManifest top).deferred.argNames == core.argNamesOf needed; + # A root composite that lists the resources holding its state gives + # a program, for each system a package config builds a set for, + # that reads the outputs NixOps4 recorded; one that lists none gives + # nothing. A finalized composite tells a tool the args it gives a + # member (`deferredChildArgs`), so the tool can finalize the + # member's entry itself. + nixops4RootDiscoversRecordedOutputs = + let + entryOf = + top: top.caisson.manifest.children.caisson-nixops4.nixops4.children.caisson-nixops4-composite.root; + stateful = nixops4TopWith { + rootArgs.stateResources = [ "state" ]; + resources.probe.ssh.hostPublicKey = nixops4HostKey; + } { probe = minimalNixosBase; }; + discover = core.discoverArgsFor (entryOf stateful); + memberArgs = (nixops4RootManifest stateful).outputs.deferredChildArgs "probe"; + in + core.discoverArgsFor (entryOf (nixops4TopWith { } { })) == null + && discover.x86_64-linux.name == "caisson-nixops4-discover" + && memberArgs.path == [ "probe" ] + && memberArgs.outputValues.stdout == "stub"; + # A provider is evaluated inside the component that declares it. # `getProviders` evaluates the root without outputs and builds the # providers the root declares, as NixOps4's `getProviders` does. From cc10238eb40d7fd468320f15d38898a3d96e9bab Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Mon, 26 Oct 2026 17:36:51 -0700 Subject: [PATCH 4/9] refactor(nixops4): build the discovery program for the system that asks caisson-core now calls discoverArgs with the system of the tool that runs it, so a root composite builds one discovery program, for that system, rather than one for every system its package configs build a set for. caisson-core is pinned to the head of nix-caisson/caisson-core#57, where argsNeeded and discoverArgs are always functions. Co-Authored-By: Claude Opus 5.5 --- flake.lock | 6 +- .../caisson-nixops4-composite/default.nix | 7 +- lib-overlays/caisson-nixops4/default.nix | 65 ++++++++----------- tests/dependencies/flake.lock | 6 +- tests/pinned-world/default.nix | 11 ++-- 5 files changed, 46 insertions(+), 49 deletions(-) diff --git a/flake.lock b/flake.lock index d3b3e85f..fa7616dd 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793013687, - "narHash": "sha256-hjBo9BHN9eHL3FDe+7qs8P4hDTE8Mx2Gx7/x0xnu4bs=", + "lastModified": 1793061250, + "narHash": "sha256-J0tA+OML7AyShhcGBBEFMGFJkV6kPFIRGPjgIFc5o1E=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "ea612fa6368360c430e56dd33869bc5ef10f2e74", + "rev": "aa397aa6eedfc756252454b1984b3c5b0c4c48c6", "type": "github" }, "original": { diff --git a/lib-overlays/caisson-nixops4-composite/default.nix b/lib-overlays/caisson-nixops4-composite/default.nix index 11e323f3..fe0ea861 100644 --- a/lib-overlays/caisson-nixops4-composite/default.nix +++ b/lib-overlays/caisson-nixops4-composite/default.nix @@ -227,10 +227,14 @@ # A program that reads the outputs NixOps4 recorded, for the # composite that lists the resources holding its state. discoverArgs = + { + name, + parent, + system, + }: if (args.stateResources or [ ]) == [ ] then null else - { parent, ... }: nixops4.discoverArgsOf { manifest = parent; src = nixops4.ecosystemSrcOf { @@ -238,6 +242,7 @@ manifest = parent; }; inherit (args) stateResources; + inherit system; }; # The deploying system, and the outputs of the members, which # depend on the members the composite declares, so on the args diff --git a/lib-overlays/caisson-nixops4/default.nix b/lib-overlays/caisson-nixops4/default.nix index 1d33ac51..72d982b2 100644 --- a/lib-overlays/caisson-nixops4/default.nix +++ b/lib-overlays/caisson-nixops4/default.nix @@ -173,12 +173,12 @@ and a resource deploys one evaluation. Declare it where a single system is in force. ''; - # The `discoverArgs` of a root composite, by system: a program - # that prints the args NixOps4 recorded, for every system the - # package configs at `manifest` build a set for. It runs - # `nixops4 state dump` for each of `stateResources` in the working - # directory, which NixOps4 finds the flake from, passes its own - # arguments to NixOps4, such as `--override-input`, and prints + # The discovery program of a root composite for `system`, the + # system of the tool that runs it, on the package set selected at + # `manifest` for that system: it prints the args NixOps4 recorded. + # It runs `nixops4 state dump` for each of `stateResources` in the + # working directory, which NixOps4 finds the flake from, passes its + # own arguments to NixOps4, such as `--override-input`, and prints # `{ outputValues, resourceProviderSystem }`. The state of a # resource at `a.state` covers the composite `a`: its resources' # recorded outputs, and its nested composites in turn. Only @@ -188,40 +188,31 @@ manifest, src, stateResources, + system, }: let - systems = final.unique ( - builtins.concatMap (config: builtins.attrNames ((config.children or { }).nixpkgs or { })) ( - builtins.attrValues (manifest.pkgSets or { }) - ) - ); - programFor = - system: - let - pkgs = pkgsFor manifest system; - in - pkgs.writeShellApplication { - name = "caisson-nixops4-discover"; - runtimeInputs = [ - src.packages.${system}.nixops4 - pkgs.jq - ]; - text = '' - merged='{}' - for state in ${final.escapeShellArgs stateResources}; do - dump=$(nixops4 "$@" state dump "$state") - merged=$(jq -cn --argjson merged "$merged" --argjson dump "$dump" --arg state "$state" ' - def outputValues: - ((.resources // {}) | map_values(.output_properties)) - + ((.deployments // {}) | map_values(outputValues)); - $merged * ({} | setpath($state | split(".") | .[:-1]; $dump | outputValues)) - ') - done - jq -cn --argjson merged "$merged" '{ outputValues: $merged, resourceProviderSystem: "${system}" }' - ''; - }; + pkgs = pkgsFor manifest system; in - final.genAttrs systems programFor; + pkgs.writeShellApplication { + name = "caisson-nixops4-discover"; + runtimeInputs = [ + src.packages.${system}.nixops4 + pkgs.jq + ]; + text = '' + merged='{}' + for state in ${final.escapeShellArgs stateResources}; do + dump=$(nixops4 "$@" state dump "$state") + merged=$(jq -cn --argjson merged "$merged" --argjson dump "$dump" --arg state "$state" ' + def outputValues: + ((.resources // {}) | map_values(.output_properties)) + + ((.deployments // {}) | map_values(outputValues)); + $merged * ({} | setpath($state | split(".") | .[:-1]; $dump | outputValues)) + ') + done + jq -cn --argjson merged "$merged" '{ outputValues: $merged, resourceProviderSystem: "${system}" }' + ''; + }; # The arg `resourceProviderSystem`, which every component takes. # A system name is no secret, so it is cacheable. diff --git a/tests/dependencies/flake.lock b/tests/dependencies/flake.lock index 22e320c0..43d943af 100644 --- a/tests/dependencies/flake.lock +++ b/tests/dependencies/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793013687, - "narHash": "sha256-hjBo9BHN9eHL3FDe+7qs8P4hDTE8Mx2Gx7/x0xnu4bs=", + "lastModified": 1793061250, + "narHash": "sha256-J0tA+OML7AyShhcGBBEFMGFJkV6kPFIRGPjgIFc5o1E=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "ea612fa6368360c430e56dd33869bc5ef10f2e74", + "rev": "aa397aa6eedfc756252454b1984b3c5b0c4c48c6", "type": "github" }, "original": { diff --git a/tests/pinned-world/default.nix b/tests/pinned-world/default.nix index f07d1252..38d58553 100644 --- a/tests/pinned-world/default.nix +++ b/tests/pinned-world/default.nix @@ -2346,8 +2346,8 @@ let && (nixops4RootManifest top).deferred.argNames == core.argNamesOf needed; # A root composite that lists the resources holding its state gives - # a program, for each system a package config builds a set for, - # that reads the outputs NixOps4 recorded; one that lists none gives + # a program, built for the system of the tool that asks, that + # reads the outputs NixOps4 recorded; one that lists none gives # nothing. A finalized composite tells a tool the args it gives a # member (`deferredChildArgs`), so the tool can finalize the # member's entry itself. @@ -2359,11 +2359,12 @@ let rootArgs.stateResources = [ "state" ]; resources.probe.ssh.hostPublicKey = nixops4HostKey; } { probe = minimalNixosBase; }; - discover = core.discoverArgsFor (entryOf stateful); + discover = core.discoverArgsFor (entryOf stateful) "x86_64-linux"; memberArgs = (nixops4RootManifest stateful).outputs.deferredChildArgs "probe"; in - core.discoverArgsFor (entryOf (nixops4TopWith { } { })) == null - && discover.x86_64-linux.name == "caisson-nixops4-discover" + core.discoverArgsFor (entryOf (nixops4TopWith { } { })) "x86_64-linux" == null + && discover.name == "caisson-nixops4-discover" + && discover.system == "x86_64-linux" && memberArgs.path == [ "probe" ] && memberArgs.outputValues.stdout == "stub"; From 2db09302c7c3575244ebb88ee50c17ebfe5d65ef Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Mon, 26 Oct 2026 18:27:50 -0700 Subject: [PATCH 5/9] refactor(nixops4): name the composite and resource builders mkCompositeConfiguration and mkResourceConfiguration They return what a mkConfiguration returns, built on mkModuleConfiguration; the With of compositeWith and resourceWith said nothing. Co-Authored-By: Claude Opus 5.5 --- lib-overlays/caisson-nixops4-composite/default.nix | 6 +++--- lib-overlays/caisson-nixops4-resource/default.nix | 6 +++--- lib-overlays/nixops4/default.nix | 4 ++-- tests/pinned-world/default.nix | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib-overlays/caisson-nixops4-composite/default.nix b/lib-overlays/caisson-nixops4-composite/default.nix index fe0ea861..9fb41467 100644 --- a/lib-overlays/caisson-nixops4-composite/default.nix +++ b/lib-overlays/caisson-nixops4-composite/default.nix @@ -118,7 +118,7 @@ # declarations, and the view of the members' outputs as `members` # in the composite and in each member; the upstream-compatible one # builds on this with NixOps4's class and its own modules. - compositeWith = + mkCompositeConfiguration = { class, type ? compositeType, @@ -269,7 +269,7 @@ configurationFor (nixops4.deployOf name args) { inherit name parent; }; }; - configuration = compositeWith { + configuration = mkCompositeConfiguration { class = compositeType; baseModules = { src, ... }: @@ -328,7 +328,7 @@ configuration args; # What the upstream-compatible integration builds on. extra = { - inherit compositeWith showPath; + inherit mkCompositeConfiguration showPath; }; }; in diff --git a/lib-overlays/caisson-nixops4-resource/default.nix b/lib-overlays/caisson-nixops4-resource/default.nix index 40822c35..ff3b22aa 100644 --- a/lib-overlays/caisson-nixops4-resource/default.nix +++ b/lib-overlays/caisson-nixops4-resource/default.nix @@ -90,7 +90,7 @@ # NixOps4's resource, provider-value and provider-declaration # modules; the upstream-compatible one builds on this with # NixOps4's class and its own modules. - resourceWith = + mkResourceConfiguration = { class, type ? resourceType, @@ -201,7 +201,7 @@ configurationFor (nixops4.deployOf name args) { inherit name parent; }; }; - configuration = resourceWith { + configuration = mkResourceConfiguration { class = resourceType; baseModules = { src, ... }: @@ -263,7 +263,7 @@ configuration args; # What the upstream-compatible integration builds on. extra = { - inherit resourceWith deployModule resourceOptions; + inherit mkResourceConfiguration deployModule resourceOptions; }; }; in diff --git a/lib-overlays/nixops4/default.nix b/lib-overlays/nixops4/default.nix index d9750d8f..7908b440 100644 --- a/lib-overlays/nixops4/default.nix +++ b/lib-overlays/nixops4/default.nix @@ -201,7 +201,7 @@ # own `members` argument holds the members it declares itself. compositeOf = ctx: modules: - composites.compositeWith { + composites.mkCompositeConfiguration { class = upstreamClass; baseModules = { src, deploy }: @@ -217,7 +217,7 @@ # outputs the configuration allows a tool to cache. resourceOf = ctx: modules: - resources.resourceWith { + resources.mkResourceConfiguration { class = upstreamClass; baseModules = { src, deploy }: componentBase src ++ [ (cacheableModule ctx deploy) ]; } (componentArgs ctx modules); diff --git a/tests/pinned-world/default.nix b/tests/pinned-world/default.nix index 38d58553..1663f9ec 100644 --- a/tests/pinned-world/default.nix +++ b/tests/pinned-world/default.nix @@ -2296,7 +2296,7 @@ let nixops4CompositeAndResourceAreTwoClasses = let evaluates = top: (builtins.tryEval (builtins.deepSeq (nixops4Export top.nixops4) true)).success; - resourceWith = + resourceTop = extra: nixops4TopWith { resources.probe = { @@ -2307,8 +2307,8 @@ let in evaluates (nixops4TopWith { } { }) && !(evaluates (nixops4TopWith { root.type = "exec"; } { })) - && evaluates (resourceWith { }) - && !(evaluates (resourceWith { + && evaluates (resourceTop { }) + && !(evaluates (resourceTop { members.extra = { }; })); From d0c7067b1c05dc4a06616b235eac3c0524c58b63 Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Mon, 26 Oct 2026 22:58:28 -0700 Subject: [PATCH 6/9] feat(nixops4)!: declare the root composite under the name nixops4 and finalize members with their part of its args The root of a deployment is a deferred caisson-nixops4-composite configuration declared under the name `nixops4`, for the output it fills, and the upstream-compatible nixops4 configuration is such a root of class nixops4Component. The caisson-nixops4 declaration integration is removed. Its helpers are published by the resource integration. A composite evaluates with its args, `outputValues` and `resourceProviderSystem`, finalizes each member once with that member's part of them, and returns the members' manifests as its children. Its manifest names the args it takes in `argsNeeded`: the deploying system and, by member, the outputs each resource names, cacheable where the resource lists them. The root gives `discoverArgs`, which finds the resources that hold state from their `state` option. A native composite's `members` holds each member's outputs and `absoluteMemberPath`, from the args and the declared names. An upstream composite declares `members` again as deferred modules, and reading it gives the members' configs, so `members.x.outputs.y`, `config.members.x.outputs.y` and `state = members.x` work. The integrations builder passes `argsNeeded`, `discoverArgs` and `children` through from an integration's evaluate, publishes the entry of a deferred configuration from the `exportsTo` its record holds, and passes no registries up from an entry. caisson-core is pinned to 2293708 on nix-caisson/caisson-core#57, where `finalize { args, argsSource }` fills the args from the manifest's own `argsNeeded`. Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +- docs/development/repository-structure.md | 4 +- flake.lock | 6 +- .../caisson-nixops4-composite/default.nix | 393 ++++++++------- .../caisson-nixops4-resource/default.nix | 344 +++++++++---- lib-overlays/caisson-nixops4/default.nix | 436 ----------------- lib-overlays/integrations/default.nix | 66 ++- lib-overlays/nixops4/default.nix | 300 ++++++------ .../nixos/default.nix | 7 +- modules/caisson-nixops4/core | 1 - .../generic/core/caisson/configurations.nix | 11 +- modules/generic/core/default.nix | 1 - modules/nixops4/core | 1 - modules/nixops4Component/nixos/default.nix | 5 +- tests/dependencies/flake.lock | 6 +- tests/pinned-world/default.nix | 460 ++++++++++++++---- tests/unit/lib-overlays.nix | 3 - 17 files changed, 1069 insertions(+), 980 deletions(-) delete mode 100644 lib-overlays/caisson-nixops4/default.nix delete mode 120000 modules/caisson-nixops4/core delete mode 120000 modules/nixops4/core diff --git a/README.md b/README.md index 7e1db0f8..9f221b55 100644 --- a/README.md +++ b/README.md @@ -134,10 +134,9 @@ source by. | `facter` | none | a `facter.json` as nixos-facter writes it, with the NixOS configurations that run on it declared inside it; a machine gets it as `hardware.facter.reportPath` | | `caisson-microvm` | `microvm` | microvm.nix guests declared inside the NixOS host that runs them, each guest a NixOS configuration declared inside it that sets its own `microvm.*` options and gets microvm.nix's guest module by default; the host selects caisson's `microvm-host` NixOS module, which defines microvm.nix's `microvm.vms.` from them; the microvm integration we recommend for new configurations | | `microvm` | `microvm` | a microvm.nix guest written as microvm.nix reads it, a NixOS configuration that sets its `microvm.*` options itself, declared inside the NixOS host that runs it; for porting an existing guest | -| `caisson-nixops4` | `nixops4` | a NixOps4 deployment, holding its root composite under the name `root`; the configuration named `nixops4` is published as the `nixops4` output the NixOps4 command reads; the NixOps4 integration we recommend for new configurations | -| `caisson-nixops4-composite` | `nixops4` | a composite of a NixOps4 deployment, evaluated when NixOps4 deploys, with its members, composites and resources, declared beneath it; it declares providers and reads the outputs of its members | +| `caisson-nixops4-composite` | `nixops4` | a composite of a NixOps4 deployment, evaluated when NixOps4 deploys, with its members, composites and resources, declared beneath it; it declares providers and reads the outputs of its members; the composite declared under the name `nixops4` where no composite holds it is the root, published as the `nixops4` output the NixOps4 command reads; the NixOps4 integration we recommend for new configurations | | `caisson-nixops4-resource` | `nixops4` | a resource of a NixOps4 deployment, evaluated when NixOps4 deploys, holding the caisson configurations it deploys; a NixOS configuration declared inside it is deployed over SSH as nixops4-nixos deploys one | -| `nixops4` | `nixops4` | the root modules of a NixOps4 deployment, written as NixOps4 users write them for `mkRoot`, each component evaluated by caisson and each nixops4-nixos machine a caisson NixOS configuration; for porting an existing deployment | +| `nixops4` | `nixops4` | the root modules of a NixOps4 deployment, written as NixOps4 users write them for `mkRoot`, each component evaluated by caisson and each nixops4-nixos machine a caisson NixOS configuration; the configuration named `nixops4` is published as the `nixops4` output; for porting an existing deployment | | `treefmt` | `treefmt-nix` | the formatters of a tree, as treefmt-nix modules; the configuration named `formatter` is published as the `formatter.` that `nix fmt` runs | | `nvf` | `nvf` | a Neovim written as nvf modules, published as `nvfConfigurations..`, in the shape `neovimConfiguration` returns, kept in a separate Nix profile that the machine, home-manager configuration or hjem configuration it is declared inside activates | | `nixvim` | `nixvim` | a Neovim written as nixvim modules, published as `nixvimConfigurations..`, kept in a profile as an nvf configuration is | diff --git a/docs/development/repository-structure.md b/docs/development/repository-structure.md index 8ddf0015..8972f35c 100644 --- a/docs/development/repository-structure.md +++ b/docs/development/repository-structure.md @@ -28,8 +28,8 @@ The layer users reach for: the integrations (`structural`, `flake-parts`, `nixpkgs`, `nixos`, `nixos-minimal`, `home-manager`, `home-manager-minimal`, `darwin`, `hjem`, `caisson-colmena`, `colmena`, `caisson-disko`, `disko`, `caisson-facter`, `facter`, -`caisson-microvm`, `microvm`, `caisson-nixops4`, -`caisson-nixops4-composite`, `caisson-nixops4-resource`, `nixops4`, +`caisson-microvm`, `microvm`, `caisson-nixops4-composite`, +`caisson-nixops4-resource`, `nixops4`, `terranix`, `system-manager`, `treefmt`, `nvf`, `nixvim`, `topology`, `kubenix`, `nixidy`), each a library overlay diff --git a/flake.lock b/flake.lock index fa7616dd..fbd55d5f 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793061250, - "narHash": "sha256-J0tA+OML7AyShhcGBBEFMGFJkV6kPFIRGPjgIFc5o1E=", + "lastModified": 1793078671, + "narHash": "sha256-0ZawlBBGyrn9LhAhyQh5yLePUrFZBftEOdQCkpckCGY=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "aa397aa6eedfc756252454b1984b3c5b0c4c48c6", + "rev": "229370819fa078ae1f0bc5ed6b5d08312958f6ee", "type": "github" }, "original": { diff --git a/lib-overlays/caisson-nixops4-composite/default.nix b/lib-overlays/caisson-nixops4-composite/default.nix index 9fb41467..1b361ead 100644 --- a/lib-overlays/caisson-nixops4-composite/default.nix +++ b/lib-overlays/caisson-nixops4-composite/default.nix @@ -3,25 +3,34 @@ # The caisson-nixops4-composite integration: a composite of a NixOps4 # deployment, a component that holds further components as its # members. The root component NixOps4 deploys is the outermost -# composite, declared under the name `root` beneath a caisson-nixops4 -# configuration. +# composite, declared under the name `nixops4`, for the output it +# fills. # # A composite is a deferred configuration (caisson-core's # `mkDeferredConfiguration`): it is evaluated when NixOps4 deploys, -# with the args its parent gives it, and NixOps4 evaluates it again -# each time the outputs of a resource become known. Its members are the -# caisson-nixops4-composite and caisson-nixops4-resource configurations -# declared beneath it, deferred children in turn, which it finalizes -# with their part of its args (`memberArgs` of caisson-nixops4). +# with its args, and NixOps4 evaluates it again each time the outputs +# of a resource become known. Its args are `outputValues`, the outputs +# of its members by member name as NixOps4 passes them, and +# `resourceProviderSystem`, the system NixOps4 runs providers on. Its +# members are the caisson-nixops4-composite and +# caisson-nixops4-resource configurations declared beneath it. It +# finalizes each member with that member's part of its args, so its +# manifest holds the members' manifests as its children. # # A composite declares providers, as NixOps4's components do, with # NixOps4's own `providers` option, and its module reads its members' -# outputs through the `members` module argument. It has no `type` and -# no resource options: a composite that sets them is refused where it -# sets them, as an undeclared option. +# outputs through the `members` module argument: for each member, its +# `outputs` and its `absoluteMemberPath`, both from the args and the +# names the members are declared under. A member's config is not +# reachable through it. A composite has no `type` and no resource +# options: a composite that sets them is refused where it sets them, as +# an undeclared option. # # What a composite publishes to NixOps4 is its export: its members' # exports, by name, as NixOps4's `_export` of a composite holds them. +# The root composite is published as the `nixops4` output NixOps4's +# command reads: `_type`, `rootFunction` and `getProviders`, as +# NixOps4's `mkRoot` returns them. { mkLibOverlay, ... @@ -36,142 +45,190 @@ # What an integration is written from, imported by key so it is # composed wherever this integration is. ((mkLibOverlay ../integrations) // { key = "integrations"; }) - # The declaration, whose helpers this integration builds on. - ((mkLibOverlay ../caisson-nixops4) // { key = "caisson-nixops4"; }) + # The resource integration, whose helpers this integration builds + # on. + ((mkLibOverlay ../caisson-nixops4-resource) // { key = "caisson-nixops4-resource"; }) ]; overlay = final: prev: let - nixops4 = final.caisson.caisson-nixops4; + resources = final.caisson.caisson-nixops4-resource; + inherit (resources) showPath; context = "lib.caisson.caisson-nixops4-composite"; compositeType = "caisson-nixops4-composite"; resourceType = "caisson-nixops4-resource"; - # The member path of a component, as NixOps4 writes it. - showPath = path: if path == [ ] then "the root" else builtins.concatStringsSep "." path; - - # The members of a composite, by name, from its full evaluation: - # the composites and resources declared beneath it, each the entry - # caisson-core holds for a deferred child. A name declared as both - # is refused, since NixOps4 holds one member under a name. + # The members of a composite at `path`, by name, from its + # evaluation: the composites and resources declared beneath it, + # each the entry caisson-core holds for a deferred child. Their + # names are readable in the childless view too, and the entries + # only in the full evaluation. A name declared as both is refused, + # since NixOps4 holds one member under a name. memberEntriesOf = - deploy: config: + path: config: let composites = config.caisson.${compositeType}.configurations; - resources = config.caisson.${resourceType}.configurations; - both = builtins.attrNames (builtins.intersectAttrs composites resources); + members = config.caisson.${resourceType}.configurations; + both = builtins.attrNames (builtins.intersectAttrs composites members); in if both == [ ] then - composites // resources + composites // members else throw '' - ${context}: the composite at ${showPath deploy.path} declares + ${context}: the composite at ${showPath path} declares ${builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") both)} both as a composite and as a resource; a member is one or the other. ''; - # The members finalized with the args the composite gives them, - # `memberArgs deploy name`, by name: each a manifest. - finalizeMembers = - memberArgs: deploy: entries: - builtins.mapAttrs ( - name: entry: - entry.finalize { - args = memberArgs deploy name; - argsSource = "the composite at ${showPath deploy.path}"; - } - ) entries; + # The `members` module argument of a native composite: for each + # member, its outputs from the composite's args and its member + # path from its name. + nativeMembersArg = + { + args, + path, + entries, + ... + }: + builtins.mapAttrs (member: _: { + outputs = args.outputValues.${member}; + absoluteMemberPath = path ++ [ member ]; + }) entries; - # The output args of a composite's members, as `argsNeeded` - # declares them: by member name, a resource's outputs, each an - # arg cacheable where the resource declares it, or a composite's - # own output args. A member whose outputs cannot be named yet, - # because naming them waits for an output, is left out, and a tool - # asks again once it has that output. - outputArgsOf = - members: - builtins.mapAttrs ( - _: member: + # The root of a deployment, as NixOps4's `mkRoot` returns it, over + # the entry of the root composite. `rootFunction` finalizes the + # root with the args NixOps4 passes and returns its export. + # `getProviders` finalizes it with the deploying system alone and + # builds the providers the root declares, as the `getProviders` + # of NixOps4 does; reading an output there fails, as it does + # upstream. Providers declared beneath the root are not + # collected, as upstream collects none of them either. + rootOf = entry: { + _type = "nixops4Component"; + rootFunction = + { + outputValues, + resourceProviderSystem, + ... + }: + (entry.finalize { + args = { inherit outputValues resourceProviderSystem; }; + argsSource = _: "NixOps4"; + }).outputs.export; + getProviders = + { system }: let - named = - if member.type == resourceType then - builtins.mapAttrs ( - output: _: - final.caisson-core.mkArg { - cacheable = builtins.elem output member.outputs.cacheableOutputs; - description = "An output of the resource at ${showPath member.outputs.path}."; - } - ) member.outputs.export.resource.outputsSkeleton - else - outputArgsOf member.outputs.members; - attempt = builtins.tryEval (builtins.deepSeq (builtins.attrNames named) named); + finalized = entry.finalize { + args.resourceProviderSystem = system; + argsSource = _: "getProviders for ${system}"; + }; + serializable = builtins.mapAttrs (_: provider: { + inherit (provider) executable args; + }) finalized.outputs.providerConfigs; in - if attempt.success then named else { } - ) members; + (resources.pkgsFor finalized system).writeText "nixops-root-providers" '' + Store path contents subject to change + ${builtins.toJSON serializable} + ''; + }; + + # How a composite is published. A composite whose parent is not a + # composite is a root, published as `nixops4`, an output that + # holds a single value. A composite beneath another is a member + # and is not published. + published = { + exportsTo = { + attrset = "nixops4"; + single = true; + only = + entry: + !(builtins.elem (entry.parent.type or null) [ + compositeType + "nixops4" + ]); + value = rootOf; + }; + }; # A composite configuration of the module class `class`, evaluated - # with the modules `baseModules` returns for the NixOps4 source - # and the args, and the `members` module argument `membersArg` - # returns. It gives each member the args `memberArgs` returns. The - # native integration evaluates its class with NixOps4's provider - # declarations, and the view of the members' outputs as `members` - # in the composite and in each member; the upstream-compatible one - # builds on this with NixOps4's class and its own modules. + # with the modules `baseModules` returns for the NixOps4 source, + # the member path and the finalized members, and the `members` + # module argument `membersArg` returns. The native integration + # evaluates its class with NixOps4's provider declarations and + # `nativeMembersArg`; the upstream-compatible one builds on this + # with NixOps4's class and its own modules. mkCompositeConfiguration = { class, type ? compositeType, baseModules, - membersArg ? - { deploy, ... }: - nixops4.membersView deploy.path deploy.outputValues, - memberArgs ? nixops4.memberArgs, + membersArg ? nativeMembersArg, }: - args: + settings: let - # The evaluator's call, on the lib of the view being - # evaluated: the base modules, the framework modules of the - # class, the selection and the configuration's module, with - # the composition's pinned sources as the `inputs` special - # argument and the deploying system as + # The evaluator's call with the args `args`, on the lib of the + # view being evaluated: the base modules, the framework + # modules of the class, the selection and the configuration's + # module, with the composition's pinned sources as the + # `inputs` special argument and the deploying system as # `resourceProviderSystem`, as NixOps4 passes it. # `ecosystemArgs` is merged over the call last. evaluate = - deploy: + args: { lib, manifest }: let selection = lib.caisson.integrations; - src = nixops4.ecosystemSrcOf { - explicit = args.ecosystemSrc or null; + path = resources.absoluteMemberPathOf manifest; + src = resources.ecosystemSrcOf { + explicit = settings.ecosystemSrc or null; inherit manifest; }; registry = lib.caisson.modules.${class} or { }; frameworkModules = selection.frameworkModules class registry; - moduleImports = selection.moduleImportsOf class { inherit lib manifest; } args; + moduleImports = selection.moduleImportsOf class { inherit lib manifest; } settings; name = manifest.name or null; configModule = selection.configModuleOf { inherit lib name class; - configModule = args.configModule or null; + configModule = settings.configModule or null; }; - # The members, finalized from the full evaluation; the - # childless view holds none. + entries = memberEntriesOf path evaluated.config; + # The members, each finalized once with its part of the + # composite's args, from the full evaluation. The + # childless view finalizes none. finalized = if manifest.childless then { } else - finalizeMembers memberArgs deploy (memberEntriesOf deploy evaluated.config); + builtins.mapAttrs ( + member: entry: + entry.finalize { + args = { + outputValues = args.outputValues.${member}; + inherit (args) resourceProviderSystem; + }; + argsSource = _: "the composite at ${showPath path}"; + } + ) entries; + ofType = memberType: final.filterAttrs (_: member: member.type == memberType) finalized; membersModule = selection.caissonModule { key = "caisson.${type}.members"; file = "caisson-${type}:members"; - module = { - _module.args.members = membersArg { - inherit deploy finalized manifest; + module = + { config, ... }: + { + _module.args.members = membersArg { + inherit + args + path + entries + config + ; + }; }; - }; }; moduleList = - baseModules { inherit src deploy; } + baseModules { inherit src path finalized; } ++ [ membersModule ] ++ frameworkModules ++ moduleImports registry @@ -182,91 +239,97 @@ specialArgs = { inherit lib; inputs = manifest.sources; - inherit (deploy) resourceProviderSystem; + inherit (args) resourceProviderSystem; } - // (if (args.specialArgs or null) != null then args.specialArgs else { }); + // (if (settings.specialArgs or null) != null then settings.specialArgs else { }); modules = moduleList; } - // (if (args.ecosystemArgs or null) != null then args.ecosystemArgs else { }) + // (if (settings.ecosystemArgs or null) != null then settings.ecosystemArgs else { }) + ); + # The member paths of the resources in the tree of this + # composite that hold state, each once. + stateResources = final.unique ( + builtins.concatMap ( + member: + if member.type == compositeType then + member.outputs.stateResources + else + let + state = member.outputs.export.resource.state; + in + if state == null then [ ] else [ state ] + ) (builtins.attrValues finalized) ); in { value = evaluated; - ecosystems.nixops4 = nixops4.ecosystemAccountOf { - explicit = args.ecosystemSrc or null; + ecosystems.nixops4 = resources.ecosystemAccountOf { + explicit = settings.ecosystemSrc or null; inherit manifest; }; modules = moduleList; + # The finalized members, by integration and then name, so + # the manifest holds their manifests. + children = final.filterAttrs (_: members: members != { }) { + ${compositeType} = ofType compositeType; + ${resourceType} = ofType resourceType; + }; outputs = { - inherit (deploy) path; - # The members, by name, each finalized with its args. - members = finalized; - # The args the composite gives its member of a name, for - # a tool that finalizes a member's entry itself. - deferredChildArgs = memberArgs deploy; + inherit path stateResources; # What NixOps4 reads of the composite. export.members = builtins.mapAttrs (_: member: member.outputs.export) finalized; # The providers it declares, which `getProviders` builds # for the root. providerConfigs = evaluated.config.providers; }; - }; - - configurationFor = - deploy: - final.caisson.integrations.mkModuleConfiguration { - selection = { - inherit class args; + # The deploying system, and the outputs of the members, by + # member name, as each member names them. + argsNeeded = { + resourceProviderSystem = resources.systemArg; + outputValues = builtins.mapAttrs (_: member: member.argsNeeded.outputValues) finalized; }; - inherit type; - evaluate = evaluate deploy; - }; + } + // ( + # The root gives a program that reads the outputs NixOps4 + # recorded, when a resource of the deployment holds state. + if path == [ ] then + { + discoverArgs = + { system }: + if stateResources == [ ] then + null + else + resources.discoverArgsOf { + inherit manifest src system; + stateResources = builtins.map (builtins.concatStringsSep ".") stateResources; + }; + } + else + { } + ); in final.caisson-core.mkDeferredConfiguration { inherit type class; - # A program that reads the outputs NixOps4 recorded, for the - # composite that lists the resources holding its state. - discoverArgs = - { - name, - parent, - system, - }: - if (args.stateResources or [ ]) == [ ] then - null - else - nixops4.discoverArgsOf { - manifest = parent; - src = nixops4.ecosystemSrcOf { - explicit = args.ecosystemSrc or null; - manifest = parent; - }; - inherit (args) stateResources; - inherit system; - }; - # The deploying system, and the outputs of the members, which - # depend on the members the composite declares, so on the args - # known so far. - argsNeeded = - { - name, - parent, - args, - }: - let - manifest = configurationFor (nixops4.deployOf name args) { inherit name parent; }; - in - { - resourceProviderSystem = nixops4.systemArg; - outputValues = outputArgsOf manifest.outputs.members; - }; + record = published // { + exportsToPosition = final.caisson.integrations.positionOf ( + builtins.unsafeGetAttrPos "exportsTo" published + ); + }; configuration = { name, parent, args, }: - configurationFor (nixops4.deployOf name args) { inherit name parent; }; + final.caisson.integrations.mkModuleConfiguration { + selection = { + inherit class; + args = settings; + }; + inherit type; + defaultPkgs = settings.defaultPkgs or null; + evaluate = evaluate args; + } { inherit name parent; }; }; configuration = mkCompositeConfiguration { @@ -276,7 +339,7 @@ [ # NixOps4's `providers` option, whose resource types the # composite writes into its members' `type`. - (nixops4.componentModule src "provider-declarations") + (resources.componentModule src "provider-declarations") ]; }; @@ -284,16 +347,22 @@ name = compositeType; class = compositeType; # What these return is a deferred configuration, a function of - # `{ name, parent, args }`: the composite that declares it under - # `caisson.caisson-nixops4-composite.configurations.`, or - # the caisson-nixops4 configuration for `root`, finalizes it - # when NixOps4 deploys. + # `{ name, parent, args }`. Declared under the name `nixops4` + # where no composite holds it, it is the root, finalized when + # NixOps4 calls `rootFunction`. Declared under + # `caisson.caisson-nixops4-composite.configurations.` of + # a composite, it is a member, finalized by that composite. mkConfiguration = { # The composite's module: its providers and members. When # absent, the configuration registered under the # configuration's name, if any. configModule ? null, + # The package set the deployment runs on, for the composite + # and its members: a function that receives the package + # sets available, by package config name, and returns the + # set to run on. + defaultPkgs ? null, # The NixOps4 flake; resolved from the composition's # declarations when absent. ecosystemSrc ? null, @@ -305,30 +374,24 @@ # Extra module arguments, merged over those the framework # supplies. specialArgs ? null, - # For the root composite: the member paths of the resources - # that hold the state of the deployment, as - # `nixops4 state dump` takes them, such as `default.state`. - # A tool reads the outputs NixOps4 recorded from them - # (`discoverArgs`). - stateResources ? [ ], - }@args: - configuration args; + }@settings: + configuration settings; # The same arguments and `ecosystemArgs`, the evaluator's # arguments merged over the composed call last. mkConfigurationWithEcosystemArgs = { configModule ? null, + defaultPkgs ? null, ecosystemSrc ? null, moduleImports ? null, extraModuleImports ? null, specialArgs ? null, - stateResources ? [ ], ecosystemArgs ? null, - }@args: - configuration args; + }@settings: + configuration settings; # What the upstream-compatible integration builds on. extra = { - inherit mkCompositeConfiguration showPath; + inherit mkCompositeConfiguration; }; }; in diff --git a/lib-overlays/caisson-nixops4-resource/default.nix b/lib-overlays/caisson-nixops4-resource/default.nix index ff3b22aa..048b5ae8 100644 --- a/lib-overlays/caisson-nixops4-resource/default.nix +++ b/lib-overlays/caisson-nixops4-resource/default.nix @@ -3,19 +3,21 @@ # The caisson-nixops4-resource integration: a resource of a NixOps4 # deployment, one real thing a provider creates, updates and deletes. # A resource is a member of a caisson-nixops4-composite configuration, -# declared beneath it, and a deferred configuration like the composite: -# the composite finalizes it with the resource's outputs, as NixOps4 -# passes them, the composite's view of its members and the deploying -# system. +# declared beneath it, and a deferred configuration like the composite +# (caisson-core's `mkDeferredConfiguration`). Its args are +# `outputValues`, its outputs as NixOps4 passes them, and +# `resourceProviderSystem`, the system NixOps4 runs providers on. The +# composite finalizes it with its part of the composite's args. # # A resource is evaluated with NixOps4's own resource options # (`nix/component/resource.nix`), its `type` option and its # `providers` option. `type` is a resource type of a provider, which # the composite usually writes from its `providers`. The resource -# reads its own outputs as `config.outputs`, and the outputs of the -# other members of its composite through the `members` module -# argument. It holds no NixOps4 members: `members` is not an option -# here, and setting it is refused where it is set. +# reads its own outputs as `config.outputs`. It reads the outputs of +# the other members of its composite through what the composite writes +# into it, from the composite's `members` module argument. It holds no +# NixOps4 members: `members` is not an option here, and setting it is +# refused where it is set. # # A resource may hold caisson configurations beneath it, such as a # NixOS configuration with its home-manager configurations. NixOps4 @@ -28,6 +30,10 @@ # deployment ahead of time may keep in a cache (`cacheableOutputs`); # NixOps4 marks no output as secret, so none is cacheable unless # listed. +# +# The helpers the composite integration, the upstream-compatible +# integration and the NixOS adapters build on are published under +# `extra`, as `lib.caisson.caisson-nixops4-resource.`. { mkLibOverlay, ... @@ -42,15 +48,166 @@ # What an integration is written from, imported by key so it is # composed wherever this integration is. ((mkLibOverlay ../integrations) // { key = "integrations"; }) - # The declaration, whose helpers this integration builds on. - ((mkLibOverlay ../caisson-nixops4) // { key = "caisson-nixops4"; }) ]; overlay = final: prev: let - nixops4 = final.caisson.caisson-nixops4; resourceType = "caisson-nixops4-resource"; + context = "lib.caisson.${resourceType}"; + + resolveEcosystemSrc = final.caisson.integrations.resolveEcosystemSrc ecosystem; + ecosystemAccountOf = final.caisson.integrations.ecosystemAccountOf ecosystem; + ecosystem = { + name = "nixops4"; + context = "caisson.${resourceType}"; + }; + + # NixOps4's component modules are files of its source, which the + # composite and resource integrations evaluate in caisson's + # evaluations; its `local` provider is a module its flake + # publishes. + assertNixops4EcosystemSrc = + ecosystemSrc: + if + ecosystemSrc ? modules + && ecosystemSrc.modules ? nixops4Provider + && builtins.pathExists "${ecosystemSrc}/nix/component/resource.nix" + && builtins.pathExists "${ecosystemSrc}/nix/component/provider-values.nix" + && builtins.pathExists "${ecosystemSrc}/nix/component/provider-declarations.nix" + then + ecosystemSrc + else + throw "${context} requires `ecosystemSrc.modules.nixops4Provider` and NixOps4's component modules under `nix/component` (a NixOps4 flake)."; + + # The NixOps4 source of a configuration: `explicit` when given, + # else what the composition declares or pins under the name + # `nixops4`, as recorded on `manifest`. + ecosystemSrcOf = + { explicit, manifest }: + assertNixops4EcosystemSrc (resolveEcosystemSrc { + inherit explicit manifest; + }); + + # A component module of NixOps4, by its file name under + # `nix/component`. + componentModule = src: name: "${src}/nix/component/${name}.nix"; + + # The package set of a deploying system: the set the selection in + # force at `manifest` returns, for that system. + pkgsFor = + manifest: system: + let + what = "the deployment for ${system}"; + pkgSets = final.caisson.integrations.pkgSetsForSystem { inherit context what; } manifest system; + in + final.caisson.integrations.pkgSetOf { inherit context what; } manifest pkgSets; + + # The member path of a component, as NixOps4 writes it. + showPath = path: if path == [ ] then "the root" else builtins.concatStringsSep "." path; + + # The integrations whose configurations are composites: the + # native composite and the root of the upstream-compatible + # integration. + compositeTypes = [ + "caisson-nixops4-composite" + "nixops4" + ]; + + # The member path of the component `manifest` describes, from the + # root, as NixOps4's `absoluteMemberPath` holds it: empty for a + # component whose parent is not a composite, which is the root, + # else the path of the parent, the childless manifest of the + # composite, and the name of the component. It is built from the + # names the components are declared under. + absoluteMemberPathOf = + manifest: + if builtins.elem (manifest.parent.type or null) compositeTypes then + manifest.parent.outputs.path ++ [ manifest.name ] + else + [ ]; + + # The NixOS configuration a resource named `resource` deploys, from + # the NixOS configurations declared beneath it (`declared`): the + # only one, evaluated for one system. `context` names the module + # that asks, for the errors. + machineOf = + { + context, + resource, + declared, + }: + let + names = builtins.attrNames declared; + only = declared.${builtins.head names}; + evaluations = builtins.attrValues only; + in + if builtins.length names != 1 then + throw '' + ${context}: the resource `${resource}` declares the NixOS configurations + ${builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") names)}, and a resource deploys one. + Declare each as the NixOS configuration of a resource of its own. + '' + else if (only._type or null) == "caisson-manifest" then + only + else if builtins.length evaluations == 1 then + builtins.head evaluations + else + throw '' + ${context}: the NixOS configuration `${builtins.head names}` of the resource + `${resource}` is evaluated for ${builtins.concatStringsSep ", " (builtins.attrNames only)}, + and a resource deploys one evaluation. Declare it where a single system is in force. + ''; + + # The discovery program of a root composite for `system`, the + # system of the tool that runs it, on the package set selected at + # `manifest` for that system: it prints the args NixOps4 recorded. + # It runs `nixops4 state dump` for each of `stateResources`, the + # member paths of the resources that hold state joined with `.`, + # in the working directory, which NixOps4 finds the flake from. It + # passes its own arguments to NixOps4, such as `--override-input`, + # and prints `{ outputValues, resourceProviderSystem }`. The state + # of a resource at `a.state` covers the composite `a`: its + # resources' recorded outputs, and its nested composites in turn. + # Only resources with a state have recorded outputs. + discoverArgsOf = + { + manifest, + src, + stateResources, + system, + }: + let + pkgs = pkgsFor manifest system; + in + pkgs.writeShellApplication { + name = "caisson-nixops4-discover"; + runtimeInputs = [ + src.packages.${system}.nixops4 + pkgs.jq + ]; + text = '' + merged='{}' + for state in ${final.escapeShellArgs stateResources}; do + dump=$(nixops4 "$@" state dump "$state") + merged=$(jq -cn --argjson merged "$merged" --argjson dump "$dump" --arg state "$state" ' + def outputValues: + ((.resources // {}) | map_values(.output_properties)) + + ((.deployments // {}) | map_values(outputValues)); + $merged * ({} | setpath($state | split(".") | .[:-1]; $dump | outputValues)) + ') + done + jq -cn --argjson merged "$merged" '{ outputValues: $merged, resourceProviderSystem: "${system}" }' + ''; + }; + + # The arg `resourceProviderSystem`, which every component takes. + # A system name is no secret, so it is cacheable. + systemArg = final.caisson-core.mkArg { + cacheable = true; + type = "string"; + description = "The system NixOps4 runs providers and the programs of resources on."; + }; # The options of the class beside the options NixOps4 declares. resourceOptions = @@ -68,63 +225,62 @@ }; }; - # What NixOps4 gives a resource when it evaluates the root: its - # outputs, from the output values it passes, and its member path. - # The outputs of a resource NixOps4 has not applied yet are calls - # that abort the evaluation, recoverably, when read. - deployModule = - deploy: - final.caisson.integrations.caissonModule { - key = "caisson.${resourceType}.deploy"; - file = "caisson-${resourceType}:deploy"; - module = { - config.outputs = _: { config = deploy.outputValues; }; - config.absoluteMemberPath = deploy.path; - config._module.args.members = deploy.members; - }; - }; - # A resource configuration of the module class `class`, evaluated # with the modules `baseModules` returns for the NixOps4 source - # and the args. The native integration evaluates its class with - # NixOps4's resource, provider-value and provider-declaration - # modules; the upstream-compatible one builds on this with - # NixOps4's class and its own modules. + # and the member path, and the settings the constructor received. + # The native integration evaluates its class with NixOps4's + # resource, provider-value and provider-declaration modules; the + # upstream-compatible one builds on this with NixOps4's class and + # its own modules. mkResourceConfiguration = { class, type ? resourceType, baseModules, }: - args: + settings: let - # The evaluator's call, on the lib of the view being - # evaluated: the base modules, the framework modules of the - # class, the selection and the configuration's module, with - # the composition's pinned sources as the `inputs` special + # The evaluator's call with the args `args`, on the lib of the + # view being evaluated: the base modules, the outputs and the + # member path, the framework modules of the class, the + # selection and the configuration's module, with the + # composition's pinned sources as the `inputs` special # argument and the deploying system as # `resourceProviderSystem`, as NixOps4 passes it. # `ecosystemArgs` is merged over the call last. evaluate = - deploy: + args: { lib, manifest }: let selection = lib.caisson.integrations; - src = nixops4.ecosystemSrcOf { - explicit = args.ecosystemSrc or null; + path = absoluteMemberPathOf manifest; + src = ecosystemSrcOf { + explicit = settings.ecosystemSrc or null; inherit manifest; }; registry = lib.caisson.modules.${class} or { }; frameworkModules = selection.frameworkModules class registry; - moduleImports = selection.moduleImportsOf class { inherit lib manifest; } args; + moduleImports = selection.moduleImportsOf class { inherit lib manifest; } settings; name = manifest.name or null; configModule = selection.configModuleOf { inherit lib name class; - configModule = args.configModule or null; + configModule = settings.configModule or null; + }; + # What NixOps4 gives a resource: its outputs, from the + # output values it passes, and its member path. An output + # NixOps4 has not passed fails when it is read, naming + # itself. + outputsModule = selection.caissonModule { + key = "caisson.${type}.outputs"; + file = "caisson-${type}:outputs"; + module = { + config.outputs = _: { config = args.outputValues; }; + config.absoluteMemberPath = path; + }; }; moduleList = - baseModules { inherit src deploy; } - ++ [ (deployModule deploy) ] + baseModules { inherit src path; } + ++ [ outputsModule ] ++ frameworkModules ++ moduleImports registry ++ (if configModule == null then [ ] else [ configModule ]); @@ -134,71 +290,60 @@ specialArgs = { inherit lib; inputs = manifest.sources; - inherit (deploy) resourceProviderSystem; + inherit (args) resourceProviderSystem; } - // (if (args.specialArgs or null) != null then args.specialArgs else { }); + // (if (settings.specialArgs or null) != null then settings.specialArgs else { }); modules = moduleList; } - // (if (args.ecosystemArgs or null) != null then args.ecosystemArgs else { }) + // (if (settings.ecosystemArgs or null) != null then settings.ecosystemArgs else { }) ); + cacheableOutputs = evaluated.config.cacheableOutputs or [ ]; in { value = evaluated; - ecosystems.nixops4 = nixops4.ecosystemAccountOf { - explicit = args.ecosystemSrc or null; + ecosystems.nixops4 = ecosystemAccountOf { + explicit = settings.ecosystemSrc or null; inherit manifest; }; modules = moduleList; outputs = { - inherit (deploy) path; + inherit path cacheableOutputs; # What NixOps4 reads of the resource. export.resource = evaluated.config._resourceForNixOps; - cacheableOutputs = evaluated.config.cacheableOutputs or [ ]; providerConfigs = evaluated.config.providers; }; - }; - - configurationFor = - deploy: - final.caisson.integrations.mkModuleConfiguration { - selection = { - inherit class args; + # The deploying system and the resource's own outputs, + # which its type names. An output is cacheable when the + # resource lists it. + argsNeeded = { + resourceProviderSystem = systemArg; + outputValues = builtins.mapAttrs ( + output: _: + final.caisson-core.mkArg { + cacheable = builtins.elem output cacheableOutputs; + description = "An output of the resource at ${showPath path}."; + } + ) evaluated.config.outputsSkeleton; }; - inherit type; - defaultPkgs = args.defaultPkgs or null; - evaluate = evaluate deploy; }; in final.caisson-core.mkDeferredConfiguration { inherit type class; - # The deploying system and the resource's own outputs, which - # its type declares. - argsNeeded = - { - name, - parent, - args, - }: - let - manifest = configurationFor (nixops4.deployOf name args) { inherit name parent; }; - in - { - resourceProviderSystem = nixops4.systemArg; - outputValues = builtins.mapAttrs ( - output: _: - final.caisson-core.mkArg { - cacheable = builtins.elem output manifest.outputs.cacheableOutputs; - description = "An output of this resource."; - } - ) manifest.outputs.export.resource.outputsSkeleton; - }; configuration = { name, parent, args, }: - configurationFor (nixops4.deployOf name args) { inherit name parent; }; + final.caisson.integrations.mkModuleConfiguration { + selection = { + inherit class; + args = settings; + }; + inherit type; + defaultPkgs = settings.defaultPkgs or null; + evaluate = evaluate args; + } { inherit name parent; }; }; configuration = mkResourceConfiguration { @@ -206,9 +351,9 @@ baseModules = { src, ... }: [ - (nixops4.componentModule src "resource") - (nixops4.componentModule src "provider-values") - (nixops4.componentModule src "provider-declarations") + (componentModule src "resource") + (componentModule src "provider-values") + (componentModule src "provider-declarations") (final.caisson.integrations.caissonModule { key = "caisson.${resourceType}.resourceOptions"; file = "caisson-${resourceType}:resourceOptions"; @@ -223,7 +368,7 @@ # What these return is a deferred configuration, a function of # `{ name, parent, args }`: the composite that declares it under # `caisson.caisson-nixops4-resource.configurations.` - # finalizes it when NixOps4 deploys. + # finalizes it with its part of the composite's args. mkConfiguration = { # The resource's module: its `type`, `inputs`, the options @@ -246,8 +391,8 @@ # Extra module arguments, merged over those the framework # supplies. specialArgs ? null, - }@args: - configuration args; + }@settings: + configuration settings; # The same arguments and `ecosystemArgs`, the evaluator's # arguments merged over the composed call last. mkConfigurationWithEcosystemArgs = @@ -259,11 +404,26 @@ extraModuleImports ? null, specialArgs ? null, ecosystemArgs ? null, - }@args: - configuration args; - # What the upstream-compatible integration builds on. + }@settings: + configuration settings; + # What the composite integration, the upstream-compatible + # integration and the NixOS adapters build on. extra = { - inherit mkResourceConfiguration deployModule resourceOptions; + inherit + mkResourceConfiguration + resourceOptions + ecosystemSrcOf + componentModule + pkgsFor + showPath + absoluteMemberPathOf + machineOf + discoverArgsOf + systemArg + ; + # Where the NixOps4 source of a configuration came from, as a + # manifest records it under `ecosystems.nixops4`. + ecosystemAccountOf = { explicit, manifest }: ecosystemAccountOf { inherit explicit manifest; }; }; }; in diff --git a/lib-overlays/caisson-nixops4/default.nix b/lib-overlays/caisson-nixops4/default.nix deleted file mode 100644 index 72d982b2..00000000 --- a/lib-overlays/caisson-nixops4/default.nix +++ /dev/null @@ -1,436 +0,0 @@ -# SPDX-License-Identifier: MIT -# -# The caisson-nixops4 integration, the native NixOps4 integration's -# declaration: a configuration of the `caisson-nixops4` class, evaluated -# when the flake is, which holds the root component of a NixOps4 -# deployment and publishes it as the `nixops4` output NixOps4's -# command reads. -# -# NixOps4 evaluates its root component when it deploys, again each time -# the outputs of a resource become known, and a component can read those -# outputs and compute its members from them. So the root component is a -# deferred child of the declaration (caisson-core's -# `mkDeferredConfiguration`): a `caisson-nixops4-composite` -# configuration declared under the name `root`, finalized by -# `rootFunction` with the output values NixOps4 passes. The composites -# and resources beneath it are deferred children of their composites, -# which finalize them with their part of those values. The integrations -# `caisson-nixops4-composite` and `caisson-nixops4-resource` are written -# beside this one, on the helpers it publishes under `extra`. -# -# The configuration named `nixops4`, for the output it fills, is -# published there: `_type`, `rootFunction` and `getProviders`, as -# NixOps4's `mkRoot` returns them. -{ - mkLibOverlay, - ... -}: -{ - - imports = [ - # The library of nixpkgs, which this overlay calls through the - # composed library, imported by key so it is composed wherever - # this is. - ((mkLibOverlay ../nixpkgs-lib) // { key = "nixpkgs-lib"; }) - # What an integration is written from, imported by key so it is - # composed wherever this integration is. - ((mkLibOverlay ../integrations) // { key = "integrations"; }) - ]; - - overlay = - final: prev: - let - - resolveEcosystemSrc = final.caisson.integrations.resolveEcosystemSrc ecosystem; - ecosystemAccountOf = final.caisson.integrations.ecosystemAccountOf ecosystem; - ecosystem = { - name = "nixops4"; - context = "caisson.caisson-nixops4"; - }; - context = "lib.caisson.caisson-nixops4"; - - # NixOps4's component modules are files of its source, which the - # composite and resource integrations evaluate in caisson's - # evaluations; its `local` provider is a module its flake - # publishes. - assertNixops4EcosystemSrc = - ecosystemSrc: - if - ecosystemSrc ? modules - && ecosystemSrc.modules ? nixops4Provider - && builtins.pathExists "${ecosystemSrc}/nix/component/resource.nix" - && builtins.pathExists "${ecosystemSrc}/nix/component/provider-values.nix" - && builtins.pathExists "${ecosystemSrc}/nix/component/provider-declarations.nix" - then - ecosystemSrc - else - throw "${context} requires `ecosystemSrc.modules.nixops4Provider` and NixOps4's component modules under `nix/component` (a NixOps4 flake)."; - - # The NixOps4 source of a configuration: `explicit` when given, - # else what the composition declares or pins under the name - # `nixops4`, as recorded on `manifest`. - ecosystemSrcOf = - { explicit, manifest }: - assertNixops4EcosystemSrc (resolveEcosystemSrc { - inherit explicit manifest; - }); - - # A component module of NixOps4, by its file name under - # `nix/component`. - componentModule = src: name: "${src}/nix/component/${name}.nix"; - - # The package set of a deploying system: the set the selection in - # force at `manifest` returns, for that system. - pkgsFor = - manifest: system: - let - what = "the deployment for ${system}"; - pkgSets = final.caisson.integrations.pkgSetsForSystem { inherit context what; } manifest system; - in - final.caisson.integrations.pkgSetOf { inherit context what; } manifest pkgSets; - - # What a component sees of the members of a composite, as the - # `members` module argument: for each member, by name, its - # `outputs`, its `members` in turn and its `absoluteMemberPath`, - # which NixOps4's `state` option reads. It is built from the - # output values of the composite (`outputValues`), which NixOps4 - # computes from the export of the root: by member name, a - # resource's outputs or a composite's own output values. A - # member's config is not reachable through it. - membersView = - path: outputValues: - builtins.mapAttrs (name: values: { - outputs = values; - members = membersView (path ++ [ name ]) values; - absoluteMemberPath = path ++ [ name ]; - }) outputValues; - - # The args a deferred member takes, as its composite gives them: - # `outputValues`, the member's part of the output values; - # `members`, the composite's view of its members (`membersView`); - # `resourceProviderSystem`, the system NixOps4 runs providers on; - # and `path`, the member path from the root. - deployOf = - what: args: - { - outputValues = args.outputValues or { }; - members = args.members or { }; - path = args.path or [ ]; - } - // { - resourceProviderSystem = - args.resourceProviderSystem or (throw "caisson-deferred: needs ${what}.resourceProviderSystem"); - }; - - # The args a composite, with args `deploy`, gives its member - # `name`. - memberArgs = deploy: name: { - outputValues = deploy.outputValues.${name} or { }; - members = membersView deploy.path deploy.outputValues; - inherit (deploy) resourceProviderSystem; - path = deploy.path ++ [ name ]; - }; - - # The args NixOps4 passes, as the declaration hands them to the - # root composite. - rootArgs = - { outputValues, resourceProviderSystem }: - { - inherit outputValues resourceProviderSystem; - members = { }; - path = [ ]; - }; - - # The NixOS configuration a resource named `resource` deploys, from - # the NixOS configurations declared beneath it (`declared`): the - # only one, evaluated for one system. `context` names the module - # that asks, for the errors. - machineOf = - { - context, - resource, - declared, - }: - let - names = builtins.attrNames declared; - only = declared.${builtins.head names}; - evaluations = builtins.attrValues only; - in - if builtins.length names != 1 then - throw '' - ${context}: the resource `${resource}` declares the NixOS configurations - ${builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") names)}, and a resource deploys one. - Declare each as the NixOS configuration of a resource of its own. - '' - else if (only._type or null) == "caisson-manifest" then - only - else if builtins.length evaluations == 1 then - builtins.head evaluations - else - throw '' - ${context}: the NixOS configuration `${builtins.head names}` of the resource - `${resource}` is evaluated for ${builtins.concatStringsSep ", " (builtins.attrNames only)}, - and a resource deploys one evaluation. Declare it where a single system is in force. - ''; - - # The discovery program of a root composite for `system`, the - # system of the tool that runs it, on the package set selected at - # `manifest` for that system: it prints the args NixOps4 recorded. - # It runs `nixops4 state dump` for each of `stateResources` in the - # working directory, which NixOps4 finds the flake from, passes its - # own arguments to NixOps4, such as `--override-input`, and prints - # `{ outputValues, resourceProviderSystem }`. The state of a - # resource at `a.state` covers the composite `a`: its resources' - # recorded outputs, and its nested composites in turn. Only - # resources with a state have recorded outputs. - discoverArgsOf = - { - manifest, - src, - stateResources, - system, - }: - let - pkgs = pkgsFor manifest system; - in - pkgs.writeShellApplication { - name = "caisson-nixops4-discover"; - runtimeInputs = [ - src.packages.${system}.nixops4 - pkgs.jq - ]; - text = '' - merged='{}' - for state in ${final.escapeShellArgs stateResources}; do - dump=$(nixops4 "$@" state dump "$state") - merged=$(jq -cn --argjson merged "$merged" --argjson dump "$dump" --arg state "$state" ' - def outputValues: - ((.resources // {}) | map_values(.output_properties)) - + ((.deployments // {}) | map_values(outputValues)); - $merged * ({} | setpath($state | split(".") | .[:-1]; $dump | outputValues)) - ') - done - jq -cn --argjson merged "$merged" '{ outputValues: $merged, resourceProviderSystem: "${system}" }' - ''; - }; - - # The arg `resourceProviderSystem`, which every component takes. - # A system name is no secret, so it is cacheable. - systemArg = final.caisson-core.mkArg { - cacheable = true; - type = "string"; - description = "The system NixOps4 runs providers and the programs of resources on."; - }; - - # The options of the declaration's class: none of its own, so - # only the framework's and what its modules declare. - declarationOptions = { ... }: { }; - - # The root of the deployment, as NixOps4's `mkRoot` returns it, - # over the deferred root composite `root` of the declaration. - # `rootFunction` finalizes it with the output values NixOps4 - # passes and returns its export. `getProviders` finalizes it with - # no output values for `system` and builds the providers its root - # declares, as the `getProviders` of NixOps4 does; providers - # declared beneath the root are not collected, as upstream - # collects none of them either. - rootOf = - { manifest, root }: - { - _type = "nixops4Component"; - rootFunction = - { - outputValues, - resourceProviderSystem, - ... - }: - (root.finalize { - args = rootArgs { inherit outputValues resourceProviderSystem; }; - argsSource = "NixOps4"; - }).outputs.export; - getProviders = - { system }: - let - finalized = root.finalize { - args = rootArgs { - outputValues = { }; - resourceProviderSystem = system; - }; - argsSource = "getProviders, with no output values"; - }; - serializable = builtins.mapAttrs (_: provider: { - inherit (provider) executable args; - }) finalized.outputs.providerConfigs; - in - (pkgsFor manifest system).writeText "nixops-root-providers" '' - Store path contents subject to change - ${builtins.toJSON serializable} - ''; - # The deferred root composite, for a tool that walks the - # deployment ahead of time. - caisson.deferredRoot = root; - }; - - # The evaluator's call, on the lib of the view being evaluated: - # the options of the class, the framework modules of the class, the - # selection and the configuration's module, with the composition's - # pinned sources as the `inputs` special argument. `ecosystemArgs`, - # which only the twin's pattern admits, is merged over the call - # last. - evaluate = - args: - { lib, manifest }: - let - selection = lib.caisson.integrations; - registry = lib.caisson.modules.caisson-nixops4 or { }; - frameworkModules = selection.frameworkModules "caisson-nixops4" registry; - moduleImports = selection.moduleImportsOf "caisson-nixops4" { inherit lib manifest; } args; - name = manifest.name or null; - configModule = selection.configModuleOf { - inherit lib name; - class = "caisson-nixops4"; - configModule = args.configModule or null; - }; - moduleList = [ - (selection.caissonModule { - key = "caisson.caisson-nixops4.declarationOptions"; - file = "caisson-caisson-nixops4:declarationOptions"; - module = declarationOptions; - }) - ] - ++ frameworkModules - ++ moduleImports registry - ++ (if configModule == null then [ ] else [ configModule ]); - evaluated = lib.evalModules ( - { - class = "caisson-nixops4"; - specialArgs = { - inherit lib; - inputs = manifest.sources; - } - // (if (args.specialArgs or null) != null then args.specialArgs else { }); - modules = moduleList; - } - // (if (args.ecosystemArgs or null) != null then args.ecosystemArgs else { }) - ); - composites = evaluated.config.caisson.caisson-nixops4-composite.configurations; - # The root composite, the composite declared under the name - # `root`: a `nixops4` output holds one root. - root = - if builtins.attrNames composites == [ "root" ] then - composites.root - else - throw '' - ${context}: a nixops4 configuration holds one root composite, declared as - `caisson.caisson-nixops4-composite.configurations.root`, and this one declares - ${ - if composites == { } then - "none" - else - builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") (builtins.attrNames composites)) - }. - ''; - in - { - value = evaluated; - ecosystems.nixops4 = ecosystemAccountOf { - explicit = args.ecosystemSrc or null; - inherit manifest; - }; - modules = moduleList; - outputs.root = rootOf { inherit manifest root; }; - }; - - # A nixops4 configuration is published as `nixops4`, an output that - # holds a single value: the root of the deployment. - exportsTo = { - attrset = "nixops4"; - single = true; - value = manifest: manifest.outputs.root; - }; - - configuration = - args: - final.caisson.integrations.mkModuleConfiguration { - selection = { - class = "caisson-nixops4"; - inherit args; - }; - type = "caisson-nixops4"; - defaultPkgs = args.defaultPkgs or null; - inherit exportsTo; - evaluate = evaluate args; - }; - - integration = final.caisson.integrations.mkIntegration { - name = "caisson-nixops4"; - class = "caisson-nixops4"; - # What these return is a configuration, a function of - # `{ name, parent }`: the parent that declares it under - # `caisson.caisson-nixops4.configurations.` finalizes it. - mkConfiguration = - { - # The declaration's module: the root composite, declared as - # `caisson.caisson-nixops4-composite.configurations.root`. - # When absent, the configuration registered under the - # configuration's name, if any. - configModule ? null, - # The package set the deployment runs on: a function that - # receives the package sets available, by package config - # name, and returns the set to run on. - defaultPkgs ? null, - # The NixOps4 flake; resolved from the composition's - # declarations when absent. - ecosystemSrc ? null, - # The selection over the caisson-nixops4 class of the - # registry, replacing the default of the class. - moduleImports ? null, - # A selection added to that selection, whichever it is. - extraModuleImports ? null, - # Extra module arguments, merged over those the framework - # supplies. - specialArgs ? null, - }@args: - configuration args; - # The same arguments and `ecosystemArgs`, the evaluator's - # arguments merged over the composed call last. - mkConfigurationWithEcosystemArgs = - { - configModule ? null, - defaultPkgs ? null, - ecosystemSrc ? null, - moduleImports ? null, - extraModuleImports ? null, - specialArgs ? null, - ecosystemArgs ? null, - }@args: - configuration args; - # What the composite and resource integrations build on. - extra = { - inherit - exportsTo - ecosystemSrcOf - componentModule - pkgsFor - membersView - deployOf - memberArgs - rootArgs - rootOf - systemArg - machineOf - discoverArgsOf - ; - # Where the NixOps4 source of a configuration came from, as a - # manifest records it under `ecosystems.nixops4`. - ecosystemAccountOf = { explicit, manifest }: ecosystemAccountOf { inherit explicit manifest; }; - }; - }; - in - { - caisson = (prev.caisson or { }) // { - caisson-nixops4 = integration.namespace; - }; - }; - -} diff --git a/lib-overlays/integrations/default.nix b/lib-overlays/integrations/default.nix index 4b42767d..05fb4ec9 100644 --- a/lib-overlays/integrations/default.nix +++ b/lib-overlays/integrations/default.nix @@ -722,7 +722,20 @@ # The configurations its modules declare under # `caisson..configurations`, of any integration, # are its children, and `caisson.exports`, which carries what - # they pass up, is its `exports` output. + # they pass up, is its `exports` output. An integration whose + # evaluation finalizes its children itself returns them as + # `children`, by integration and then name, and they are its + # children in place of those values. A deferred configuration + # (`caisson-core.mkDeferredConfiguration`) finalizes the deferred + # configurations declared beneath it this way, with their part of + # its args. + # + # `evaluate` may also return `argsNeeded`, the args a deferred + # configuration takes, and `discoverArgs`, the function of + # `{ system }` that returns the program which finds them, or + # null. The manifest carries both, as caisson-core describes + # them. A configuration whose `evaluate` returns neither takes no + # args. # # `defaultPkgs` is the selection of a package set the configuration # was constructed with, null when it was given none. It is @@ -739,7 +752,7 @@ # returning `value`, the name to publish it under, and # optionally `description`, a sentence saying what it did. It may # carry `single`, for an output that holds the one configuration - # named `default`, and `bySystem`, for an output keyed by the + # named for that output, and `bySystem`, for an output keyed by the # system of each evaluation first (`packages..`). It # may carry `only`, a function of the manifest that says whether # the configuration is published there at all; a configuration it @@ -823,7 +836,8 @@ outputs = (evaluated.outputs or { }) // { exports = config.caisson.exports; }; - children = view.lib.caisson.integrations.childrenOf { inherit options config; }; + children = + evaluated.children or (view.lib.caisson.integrations.childrenOf { inherit options config; }); # What the modules give the configurations beneath # (`caisson.forChildren`). forChildren = { @@ -889,7 +903,12 @@ } # How the evaluation selected its modules. // (if selection == null then { } else { moduleImports = selectionAccountOf selection view; }); - }; + } + # The args of a deferred configuration and the program that + # finds them, where the integration's `evaluate` returns + # them. + // (if evaluated ? argsNeeded then { inherit (evaluated) argsNeeded; } else { }) + // (if evaluated ? discoverArgs then { inherit (evaluated) discoverArgs; } else { }); }; # The body of both entry points, from the caisson arguments a @@ -1568,7 +1587,10 @@ # that is not published under a name (structural, flake-parts). # An integration may give one export or a list of them; an export # with `only` applies to the configurations whose manifest it - # holds true for. + # holds true for. The entry of a deferred configuration + # (`caisson-core.deferChild`) is published in the same way, from + # the `exportsTo` its `record` holds: `value` and `only` receive + # the entry, which holds its name, type, class and parent. exportsOf = manifest: let @@ -1703,10 +1725,7 @@ name = system; }; beneath = - # A deferred child passes nothing up at flake time. - if final.caisson-core.isDeferred finalized then - [ ] - else if isManifest finalized then + if isManifest finalized || final.caisson-core.isDeferred finalized then [ { prefix = [ (segmentOf name) ]; @@ -1734,16 +1753,24 @@ manifest, }: builtins.map (entryOf level declared prefix manifest) (exportsOf manifest) - ++ builtins.map ( - entry: - entry - // { - path = prefix ++ entry.path; - namePath = declared ++ entry.namePath; - notes = entry.notes ++ level.notes; - code = entry.code ++ level.code; - } - ) (manifest.outputs.exports.configurations or [ ]) + # The entry of a deferred child is published as its + # `exportsTo` says, and nothing beneath it is evaluated at + # flake time, so it passes nothing else up. + ++ ( + if final.caisson-core.isDeferred manifest then + [ ] + else + builtins.map ( + entry: + entry + // { + path = prefix ++ entry.path; + namePath = declared ++ entry.namePath; + notes = entry.notes ++ level.notes; + code = entry.code ++ level.code; + } + ) (manifest.outputs.exports.configurations or [ ]) + ) ) beneath ) (builtins.attrNames selected.${integration}) ) (builtins.attrNames selected); @@ -1935,6 +1962,7 @@ ecosystemUnit inEcosystem entriesOf + positionOf configurationsModuleFor integrationsIn integrationsDeclaringIn diff --git a/lib-overlays/nixops4/default.nix b/lib-overlays/nixops4/default.nix index 7908b440..f1d6a3d9 100644 --- a/lib-overlays/nixops4/default.nix +++ b/lib-overlays/nixops4/default.nix @@ -4,33 +4,34 @@ # modules of a root component as NixOps4 users write them for NixOps4's # `mkRoot`, of NixOps4's class `nixops4Component`, and caisson evaluates # every component of the deployment. The native integrations, -# caisson-nixops4 and its composite and resource, take caisson's form -# instead. +# caisson-nixops4-composite and caisson-nixops4-resource, take +# caisson's form instead. # -# A nixops4 configuration, named `nixops4` for the output it fills, -# declares the root composite beneath it under the name `root` from -# those modules, and publishes the root as the caisson-nixops4 -# configuration does. Every component is evaluated with NixOps4's -# class name, so a module that sets `_class = "nixops4Component"` -# loads, and with NixOps4's resource, type and providers options. +# A nixops4 configuration is the root composite of the deployment, a +# deferred configuration as a native composite is. Declared under the +# name `nixops4`, for the output it fills, it is published as the +# native root is. Every component is evaluated with NixOps4's class +# name, so a module that sets `_class = "nixops4Component"` loads, and +# with NixOps4's resource, type and providers options. # # Upstream declares `members` and the resource options on every # component and decides which kind a component is when its export is # read: a component whose `resourceType` is defined is a resource. A -# composite here collects the definitions of each member under -# `members.` and decides the same way, with a probe evaluation -# of the member's modules over NixOps4's own base modules. A member -# becomes a caisson-nixops4-resource or a caisson-nixops4-composite -# configuration declared beneath the composite, evaluated with the -# same modules. A member that sets `type` and also has members is -# refused; upstream drops the members. +# composite here declares `members` again, as `lazyAttrsOf +# deferredModule`, and decides the same way for each member, with a +# probe evaluation of the member's modules over NixOps4's own base +# modules. A member becomes a caisson-nixops4-resource or a +# caisson-nixops4-composite configuration declared beneath the +# composite, evaluated with the same modules. A member that sets `type` +# and also has members is refused; upstream drops the members. # -# The `members` module argument is the one upstream gives: in a -# composite, the configs of its members, which a composite reads to -# wire one member to another (`members.db.outputs.address`); in a -# resource, nothing. A composite reads its members' configs through its -# full evaluation, so every composite sets `unsafeCrossChildReferences` -# for the class of its members. +# The `members` module argument and `config.members` are what upstream +# gives: in a composite, the configs of its members, which a composite +# reads to wire one member to another (`members.db.outputs.address`) +# and to name the resource that holds the state of another +# (`state = members.state`); in a resource, nothing. A composite reads +# its members' configs through its full evaluation, so every composite +# sets `unsafeCrossChildReferences` for the class of its members. # # A member that imports nixops4-nixos's module gets its `nixos.module` # evaluated as a caisson NixOS configuration beneath it, rather than by @@ -51,7 +52,6 @@ # composed wherever this integration is. ((mkLibOverlay ../integrations) // { key = "integrations"; }) # The native integrations, whose evaluators this one builds on. - ((mkLibOverlay ../caisson-nixops4) // { key = "caisson-nixops4"; }) ((mkLibOverlay ../caisson-nixops4-composite) // { key = "caisson-nixops4-composite"; }) ((mkLibOverlay ../caisson-nixops4-resource) // { key = "caisson-nixops4-resource"; }) ]; @@ -59,18 +59,19 @@ overlay = final: prev: let - native = final.caisson.caisson-nixops4; composites = final.caisson.caisson-nixops4-composite; resources = final.caisson.caisson-nixops4-resource; context = "lib.caisson.nixops4"; upstreamClass = "nixops4Component"; + compositeType = "caisson-nixops4-composite"; + resourceType = "caisson-nixops4-resource"; # The options every upstream component has beside `members`: # NixOps4's resource, type and providers options. componentBase = src: [ - (native.componentModule src "resource") - (native.componentModule src "provider-values") - (native.componentModule src "provider-declarations") + (resources.componentModule src "resource") + (resources.componentModule src "provider-values") + (resources.componentModule src "provider-declarations") ]; # The modules of a component, as the configuration's module. @@ -111,63 +112,91 @@ in if isResource && hasMembers then throw '' - ${context}: the member ${composites.showPath (path ++ [ name ])} sets `type`, which makes it a + ${context}: the member ${resources.showPath (path ++ [ name ])} sets `type`, which makes it a resource, and declares members, which a resource does not hold. NixOps4 would drop them; declare them in a composite instead. '' else if isResource then - "caisson-nixops4-resource" + resourceType else - "caisson-nixops4-composite"; + compositeType; - # The module of an upstream composite that turns `members.` - # into the configurations declared beneath it. + # The module of an upstream composite at `path` that declares + # `members` and turns the modules defined there into the + # configurations declared beneath it. The members are computed + # from the definitions of `members`, since its value is the + # members' configs: in the full evaluation those of the members + # the composite finalized (`finalized`), and in the childless view + # those of the full evaluation, which the composite reads through + # `unsafeCrossChildReferences`. membersModule = ctx: - { src, deploy }: + { + src, + path, + finalized, + }: final.caisson.integrations.caissonModule { - key = "caisson.nixops4.members"; - file = "caisson-nixops4:members"; + key = "caisson.nixops4.memberComponents"; + file = "caisson-nixops4:memberComponents"; module = - { config, lib, ... }: + { + lib, + options, + inputs, + resourceProviderSystem, + ... + }: let + manifest = lib.caisson-core.evalManifest; + declared = options.members.type.merge options.members.loc options.members.definitionsWithLocations; kinds = builtins.mapAttrs (kindOf { - inherit lib src; - inherit (deploy) path; - specialArgs = ctx.specialArgs // { - inherit lib; - inherit (deploy) resourceProviderSystem; + inherit lib src path; + specialArgs = { + inherit inputs; + } + // ctx.specialArgs + // { + inherit lib resourceProviderSystem; }; - }) config.members; - ofKind = kind: lib.filterAttrs (name: _: kinds.${name} == kind) config.members; - # The state resources are the root's to list. - memberCtx = builtins.removeAttrs ctx [ "stateResources" ]; + }) declared; + ofKind = kind: lib.filterAttrs (name: _: kinds.${name} == kind) declared; + members = + if manifest.childless then + let + full = manifest.crossChildFull; + in + (full.children.${compositeType} or { }) // (full.children.${resourceType} or { }) + else + finalized; in { options.members = lib.mkOption { type = lib.types.lazyAttrsOf lib.types.deferredModule; default = { }; + apply = _: builtins.mapAttrs (_: member: member.value.config) members; description = '' The member components, by name, as NixOps4 declares them: a member that sets `type` is a resource, any other a composite. + Read back, each is the config of that member. ''; }; # A composite reads its members' configs, through its full # evaluation. config.caisson.forDirectChildren.unsafeCrossChildReferences.${upstreamClass} = true; - config.caisson.caisson-nixops4-composite.configurations = builtins.mapAttrs ( - _: module: compositeOf memberCtx [ module ] - ) (ofKind "caisson-nixops4-composite"); - config.caisson.caisson-nixops4-resource.configurations = builtins.mapAttrs ( - _: module: resourceOf memberCtx [ module ] - ) (ofKind "caisson-nixops4-resource"); + config.caisson.${compositeType}.configurations = builtins.mapAttrs ( + _: module: compositeOf ctx [ module ] + ) (ofKind compositeType); + config.caisson.${resourceType}.configurations = builtins.mapAttrs ( + _: module: resourceOf ctx [ module ] + ) (ofKind resourceType); }; }; # The outputs of the resource at `path` that a tool may cache, from # the allow-list of the nixops4 configuration. cacheableModule = - ctx: deploy: + ctx: path: final.caisson.integrations.caissonModule { key = "caisson.nixops4.cacheableOutputs"; file = "caisson-nixops4:cacheableOutputs"; @@ -176,7 +205,7 @@ { options.cacheableOutputs = lib.mkOption { type = lib.types.listOf lib.types.str; - default = ctx.cacheableOutputs.${builtins.concatStringsSep "." deploy.path} or [ ]; + default = ctx.cacheableOutputs.${builtins.concatStringsSep "." path} or [ ]; readOnly = true; description = '' The outputs of this resource a tool may cache, from @@ -186,32 +215,51 @@ }; }; + # A resource holds no members, so its `members` module argument is + # empty, as it is upstream. + resourceMembersModule = final.caisson.integrations.caissonModule { + key = "caisson.nixops4.resourceMembers"; + file = "caisson-nixops4:resourceMembers"; + module = { + _module.args.members = { }; + }; + }; + # The arguments the native constructors receive for a component # with `modules`. componentArgs = ctx: modules: { configModule = modulesModule modules; - inherit (ctx) ecosystemSrc; - specialArgs = ctx.specialArgs; - stateResources = ctx.stateResources or [ ]; + inherit (ctx) ecosystemSrc specialArgs; }; - # An upstream composite: NixOps4's class and options, `members` - # collected and declared beneath as configurations, and the - # configs of the members as its `members` argument. A member's - # own `members` argument holds the members it declares itself. - compositeOf = + # An upstream composite of the integration `type`: NixOps4's class + # and options, `members` declared and its definitions declared + # beneath as configurations, and `config.members` as its `members` + # argument. `settings` are further arguments of the native + # constructor. + compositeWith = + { + type ? compositeType, + settings ? { }, + }: ctx: modules: composites.mkCompositeConfiguration { + inherit type; class = upstreamClass; baseModules = - { src, deploy }: + { + src, + path, + finalized, + }: componentBase src ++ [ - (membersModule ctx { inherit src deploy; }) + (membersModule ctx { inherit src path finalized; }) ]; - membersArg = { finalized, ... }: builtins.mapAttrs (_: member: member.value.config) finalized; - memberArgs = deploy: name: native.memberArgs deploy name // { members = { }; }; - } (componentArgs ctx modules); + membersArg = { config, ... }: config.members; + } (componentArgs ctx modules // settings); + + compositeOf = compositeWith { }; # An upstream resource: NixOps4's class and options, and the # outputs the configuration allows a tool to cache. @@ -219,84 +267,43 @@ ctx: modules: resources.mkResourceConfiguration { class = upstreamClass; - baseModules = { src, deploy }: componentBase src ++ [ (cacheableModule ctx deploy) ]; + baseModules = + { src, path }: + componentBase src + ++ [ + (cacheableModule ctx path) + resourceMembersModule + ]; } (componentArgs ctx modules); - # The evaluator's call, on the lib of the view being evaluated: - # the framework modules of the class, the selection, the - # configuration's module and the root composite made from - # `modules`, with the composition's pinned sources as the `inputs` - # special argument. `ecosystemArgs`, which only the twin's pattern - # admits, is merged over the call last. - evaluate = - args: - { lib, manifest }: - let - selection = lib.caisson.integrations; - registry = lib.caisson.modules.nixops4 or { }; - frameworkModules = selection.frameworkModules "nixops4" registry; - moduleImports = selection.moduleImportsOf "nixops4" { inherit lib manifest; } args; - ctx = { - ecosystemSrc = args.ecosystemSrc or null; - specialArgs = { - inputs = manifest.sources; - } - // (if (args.specialArgs or null) != null then args.specialArgs else { }); - cacheableOutputs = args.cacheableOutputs or { }; - stateResources = args.stateResources or [ ]; - }; - rootModule = selection.caissonModule { - key = "caisson.nixops4.root"; - file = "caisson-nixops4:root"; - module = { - caisson.caisson-nixops4-composite.configurations.root = compositeOf ctx (args.modules or [ ]); - }; - }; - moduleList = frameworkModules ++ moduleImports registry ++ [ rootModule ]; - evaluated = lib.evalModules ( - { - class = "nixops4"; - specialArgs = { - inherit lib; - inputs = manifest.sources; - }; - modules = moduleList; - } - // (if (args.ecosystemArgs or null) != null then args.ecosystemArgs else { }) - ); - in - { - value = evaluated; - ecosystems.nixops4 = native.ecosystemAccountOf { - explicit = args.ecosystemSrc or null; - inherit manifest; - }; - modules = moduleList; - outputs.root = native.rootOf { - inherit manifest; - root = evaluated.config.caisson.caisson-nixops4-composite.configurations.root; - }; - }; - + # The root composite made from `modules`, as `mkRoot` takes them, + # of the integration `nixops4`, with the allow-list, the package + # set and the selection of the configuration. configuration = - args: - final.caisson.integrations.mkModuleConfiguration { - selection = { - class = "nixops4"; - inherit args; - }; - type = "nixops4"; - defaultPkgs = args.defaultPkgs or null; - inherit (native) exportsTo; - evaluate = evaluate args; - }; + settings: + compositeWith + { + type = "nixops4"; + settings = builtins.intersectAttrs { + defaultPkgs = null; + moduleImports = null; + extraModuleImports = null; + ecosystemArgs = null; + } settings; + } + { + ecosystemSrc = settings.ecosystemSrc or null; + specialArgs = if (settings.specialArgs or null) != null then settings.specialArgs else { }; + cacheableOutputs = settings.cacheableOutputs or { }; + } + (settings.modules or [ ]); integration = final.caisson.integrations.mkIntegration { name = "nixops4"; - class = "nixops4"; - # What these return is a configuration, a function of - # `{ name, parent }`: the parent that declares it under - # `caisson.nixops4.configurations.` finalizes it. + class = upstreamClass; + # What these return is a deferred configuration, a function of + # `{ name, parent, args }`: declared under the name `nixops4`, + # it is the root, finalized when NixOps4 calls `rootFunction`. mkConfiguration = { # The modules of the root component, as NixOps4's `mkRoot` @@ -311,11 +318,6 @@ # NixOps4 marks no output as secret, so none is cacheable # unless listed. cacheableOutputs ? { }, - # The member paths of the resources that hold the state of - # the deployment, as `nixops4 state dump` takes them, such - # as `default.state`. A tool reads the outputs NixOps4 - # recorded from them. - stateResources ? [ ], # The package set the deployment runs on: a function that # receives the package sets available, by package config # name, and returns the set to run on. @@ -323,13 +325,14 @@ # The NixOps4 flake; resolved from the composition's # declarations when absent. ecosystemSrc ? null, - # The selection over the nixops4 class of the registry, - # replacing the default of the class. + # The selection over the nixops4Component class of the + # registry for the root, replacing the default of the + # class. moduleImports ? null, # A selection added to that selection, whichever it is. extraModuleImports ? null, - }@args: - configuration args; + }@settings: + configuration settings; # The same arguments and `ecosystemArgs`, the evaluator's # arguments merged over the composed call last. mkConfigurationWithEcosystemArgs = @@ -337,14 +340,13 @@ modules ? [ ], specialArgs ? null, cacheableOutputs ? { }, - stateResources ? [ ], defaultPkgs ? null, ecosystemSrc ? null, moduleImports ? null, extraModuleImports ? null, ecosystemArgs ? null, - }@args: - configuration args; + }@settings: + configuration settings; }; in { diff --git a/modules/caisson-nixops4-resource/nixos/default.nix b/modules/caisson-nixops4-resource/nixos/default.nix index b3aab040..a7d79d70 100644 --- a/modules/caisson-nixops4-resource/nixos/default.nix +++ b/modules/caisson-nixops4-resource/nixos/default.nix @@ -42,7 +42,7 @@ let # The NixOS configuration the resource deploys: the one declared # beneath it, evaluated for one system. nixos = - (lib.caisson.caisson-nixops4.machineOf { + (lib.caisson.caisson-nixops4-resource.machineOf { inherit context declared; resource = manifest.name; }).value; @@ -52,11 +52,12 @@ let knownHosts = builtins.toFile "known_hosts" '' ${config.ssh.host} ${config.ssh.hostPublicKey} ''; - src = lib.caisson.caisson-nixops4.ecosystemSrcOf { + src = lib.caisson.caisson-nixops4-resource.ecosystemSrcOf { explicit = null; inherit manifest; }; - bash = lib.getExe (lib.caisson.caisson-nixops4.pkgsFor manifest resourceProviderSystem).bash; + bash = lib.getExe (lib.caisson.caisson-nixops4-resource.pkgsFor manifest resourceProviderSystem) + .bash; in { _file = "caisson:caisson-nixops4-resource/nixos"; diff --git a/modules/caisson-nixops4/core b/modules/caisson-nixops4/core deleted file mode 120000 index 3715354a..00000000 --- a/modules/caisson-nixops4/core +++ /dev/null @@ -1 +0,0 @@ -../generic/core \ No newline at end of file diff --git a/modules/generic/core/caisson/configurations.nix b/modules/generic/core/caisson/configurations.nix index c3be1377..0979f6a4 100644 --- a/modules/generic/core/caisson/configurations.nix +++ b/modules/generic/core/caisson/configurations.nix @@ -38,11 +38,18 @@ let ); # The manifests passed up: a configuration evaluated per system - # stands for each of its evaluations. + # stands for each of its evaluations. The entry of a deferred + # configuration is no manifest and exports no registries. exported = builtins.concatMap ( integration: builtins.concatMap ( - finalized: if isManifest finalized then [ finalized ] else builtins.attrValues finalized + finalized: + if isManifest finalized then + [ finalized ] + else if lib.caisson-core.isDeferred finalized then + [ ] + else + builtins.attrValues finalized ) (builtins.attrValues selected.${integration}) ) (builtins.attrNames selected); diff --git a/modules/generic/core/default.nix b/modules/generic/core/default.nix index 2f7c7743..4eddecd4 100644 --- a/modules/generic/core/default.nix +++ b/modules/generic/core/default.nix @@ -33,7 +33,6 @@ "caisson-disko" "caisson-facter" "caisson-microvm" - "caisson-nixops4" "caisson-nixops4-composite" "caisson-nixops4-resource" "colmena" diff --git a/modules/nixops4/core b/modules/nixops4/core deleted file mode 120000 index 3715354a..00000000 --- a/modules/nixops4/core +++ /dev/null @@ -1 +0,0 @@ -../generic/core \ No newline at end of file diff --git a/modules/nixops4Component/nixos/default.nix b/modules/nixops4Component/nixos/default.nix index da6e9263..58168a1b 100644 --- a/modules/nixops4Component/nixos/default.nix +++ b/modules/nixops4Component/nixos/default.nix @@ -30,12 +30,13 @@ let manifest = lib.caisson-core.evalManifest; usesNixos = options ? nixos && options.nixos ? module; - machine = lib.caisson.caisson-nixops4.machineOf { + machine = lib.caisson.caisson-nixops4-resource.machineOf { context = "caisson's `nixos` module of the nixops4Component class"; resource = manifest.name; declared = config.caisson.nixos.configurations; }; - bash = lib.getExe (lib.caisson.caisson-nixops4.pkgsFor manifest resourceProviderSystem).bash; + bash = lib.getExe (lib.caisson.caisson-nixops4-resource.pkgsFor manifest resourceProviderSystem) + .bash; in { _file = "caisson:nixops4Component/nixos"; diff --git a/tests/dependencies/flake.lock b/tests/dependencies/flake.lock index 43d943af..78760aab 100644 --- a/tests/dependencies/flake.lock +++ b/tests/dependencies/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793061250, - "narHash": "sha256-J0tA+OML7AyShhcGBBEFMGFJkV6kPFIRGPjgIFc5o1E=", + "lastModified": 1793078671, + "narHash": "sha256-0ZawlBBGyrn9LhAhyQh5yLePUrFZBftEOdQCkpckCGY=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "aa397aa6eedfc756252454b1984b3c5b0c4c48c6", + "rev": "229370819fa078ae1f0bc5ed6b5d08312958f6ee", "type": "github" }, "original": { diff --git a/tests/pinned-world/default.nix b/tests/pinned-world/default.nix index 1663f9ec..1d08f005 100644 --- a/tests/pinned-world/default.nix +++ b/tests/pinned-world/default.nix @@ -70,7 +70,6 @@ let registered."caisson/facter" registered."caisson/caisson-microvm" registered."caisson/microvm" - registered."caisson/caisson-nixops4" registered."caisson/caisson-nixops4-composite" registered."caisson/caisson-nixops4-resource" registered."caisson/nixops4" @@ -98,7 +97,6 @@ let "caisson-disko" "caisson-facter" "caisson-microvm" - "caisson-nixops4" "caisson-nixops4-composite" "caisson-nixops4-resource" "callConsumerFlake" @@ -380,18 +378,20 @@ let }; colmenaTop = colmenaTopWith { }; - # A top holding a caisson-nixops4 configuration named `nixops4`, - # whose root composite declares a resource for each machine given, by - # name, holding a NixOS configuration of that name. `root` is a - # further module of the root composite, `resources` a further module - # of each resource, by name, `defaultPkgs` the selection of the - # caisson-nixops4 configuration and `machineArgs` further arguments of - # each machine's constructor. + # A top holding the root composite of a NixOps4 deployment, declared + # under the name `nixops4`, which declares a resource for each machine + # given, by name, holding a NixOS configuration of that name. `root` + # is a further module of the root composite, `resources` a further + # module of each resource, by name, and `wiring` a function of the + # `members` argument of the root composite returning more such + # modules, by name, as a composite wires one member to another. + # `defaultPkgs` is the selection of the root composite and + # `machineArgs` further arguments of each machine's constructor. nixops4TopWith = { root ? { }, - rootArgs ? { }, resources ? { }, + wiring ? _: { }, defaultPkgs ? null, machineArgs ? { }, }: @@ -401,41 +401,44 @@ let configModule = { lib, ... }: { - caisson.caisson-nixops4.configurations.nixops4 = lib.caisson.caisson-nixops4.mkConfiguration { - inherit defaultPkgs; - configModule = - { lib, ... }: + caisson.caisson-nixops4-composite.configurations.nixops4 = + lib.caisson.caisson-nixops4-composite.mkConfiguration { - caisson.caisson-nixops4-composite.configurations.root = - lib.caisson.caisson-nixops4-composite.mkConfiguration - ( - rootArgs - // { + inherit defaultPkgs; + configModule = + { lib, members, ... }: + { + imports = [ root ]; + caisson.caisson-nixops4-resource.configurations = builtins.mapAttrs ( + name: nixosModule: + lib.caisson.caisson-nixops4-resource.mkConfiguration { configModule = { lib, ... }: { - imports = [ root ]; - caisson.caisson-nixops4-resource.configurations = builtins.mapAttrs ( - name: nixosModule: - lib.caisson.caisson-nixops4-resource.mkConfiguration { - configModule = - { lib, ... }: - { - imports = [ (resources.${name} or { }) ]; - caisson.nixos.configurations.${name} = lib.caisson.nixos.mkConfiguration ( - { configModule = nixosModule; } // machineArgs - ); - }; - } - ) machines; + imports = [ + (resources.${name} or { }) + ((wiring members).${name} or { }) + ]; + caisson.nixos.configurations.${name} = lib.caisson.nixos.mkConfiguration ( + { configModule = nixosModule; } // machineArgs + ); }; } - ); + ) machines; + }; }; - }; }; }; + # The entry the top holds for the root composite named `nixops4`, of + # the native or the upstream-compatible integration. + nixops4Entry = + top: + let + children = top.caisson.manifest.children; + in + (children.caisson-nixops4-composite or children.nixops4).nixops4; + # The export of a root as NixOps4's command reads it: the root # function called with the deploying system and the outputs of each # resource, here `stub` for every output the resource declares, at @@ -464,11 +467,12 @@ let # finalizes it, with `stub` outputs: its manifest. nixops4RootManifest = top: - top.nixops4.caisson.deferredRoot.finalize { + (nixops4Entry top).finalize { args = { resourceProviderSystem = "x86_64-linux"; outputValues = nixops4StubOutputs (nixops4Export top.nixops4).members; }; + argsSource = _: "the probe"; }; # A top holding a nixops4 configuration named `nixops4` made from the @@ -519,6 +523,86 @@ let # A host key for the machines NixOps4 deploys in the probes. nixops4HostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAiszi43aOWWV7voNgQ1Ifa7LGKwGJfOuiLM1n42h2Y8"; + # A further module of a native root composite: a resource `address` + # of NixOps4's `local` provider, whose `stdout` another member reads. + nixops4AddressRoot = + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.address = + lib.caisson.caisson-nixops4-resource.mkConfiguration + { + configModule = + { providers, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + }; + }; + + # The paths of the args an `argsNeeded` names, each a list of names. + nixops4ArgPathsOf = + let + walk = + path: value: + if core.isArg value then + [ path ] + else + builtins.concatMap (name: walk (path ++ [ name ]) value.${name}) (builtins.attrNames value); + in + walk [ ]; + + # A deferred configuration of a tool that is not NixOps4, declared + # beside a NixOps4 deployment: it takes the cacheable arg `who` and + # greets it. Its entry is published as `greetings.`, a function + # of `who`, and its discovery program is named `greeter-discover`. + greeterConfiguration = + lib: + lib.caisson-core.mkDeferredConfiguration { + type = "greeter"; + record.exportsTo = { + attrset = "greetings"; + value = + entry: who: + (entry.finalize { + args = { inherit who; }; + argsSource = _: "the probe"; + }).outputs.greeting; + }; + configuration = + { + name, + parent, + args, + }: + lib.caisson.integrations.mkModuleConfiguration { + type = "greeter"; + evaluate = + { lib, ... }: + let + evaluated = lib.evalModules { + specialArgs = { inherit lib; }; + modules = lib.caisson.integrations.frameworkModules "structural" { } ++ [ + ( + { lib, ... }: + { + options.greeting = lib.mkOption { type = lib.types.str; }; + config.greeting = "hello ${args.who}"; + } + ) + ]; + }; + in + { + value = evaluated; + outputs.greeting = evaluated.config.greeting; + argsNeeded.who = lib.caisson-core.mkArg { cacheable = true; }; + discoverArgs = { system }: pkgs.writeShellScript "greeter-discover" "echo '{\"who\":\"world\"}'"; + }; + } { inherit name parent; }; + }; + # What a microvm.nix guest needs beside what microvm.nix's guest # module gives it, which boots it and gives it its file systems. microvmGuestBase = { @@ -786,9 +870,8 @@ let "caisson/facter" "caisson/caisson-microvm" "caisson/microvm" - "caisson/caisson-nixops4" - "caisson/caisson-nixops4-composite" "caisson/caisson-nixops4-resource" + "caisson/caisson-nixops4-composite" "caisson/nixops4" "caisson/darwin" "caisson/terranix" @@ -897,7 +980,6 @@ let "facter" "caisson-microvm" "microvm" - "caisson-nixops4" "caisson-nixops4-composite" "caisson-nixops4-resource" "nixops4" @@ -2200,8 +2282,8 @@ let } ]; - # A caisson-nixops4 configuration named `nixops4` fills the top's - # `nixops4`, the root NixOps4's command reads. A resource holding a + # The root composite, declared under the name `nixops4`, fills the + # `nixops4` of the top, the root NixOps4's command reads. A resource holding a # NixOS configuration becomes, through the NixOS adapter, an `exec` # resource of NixOps4's `local` provider that copies the toplevel of # that configuration and runs its `bin/apply switch` over SSH, to @@ -2216,7 +2298,8 @@ let export = nixops4Export top.nixops4; resource = export.members.probe.resource; args = resource.inputs.args; - machine = (nixops4RootManifest top).outputs.members.probe.value.config.nixos.configuration; + machine = + (nixops4RootManifest top).children.caisson-nixops4-resource.probe.value.config.nixos.configuration; in top.nixops4._type == "nixops4Component" && builtins.attrNames export.members == [ "probe" ] @@ -2232,37 +2315,23 @@ let ) ); - # A resource reads the outputs of another member of its composite + # A composite wires a resource to the outputs of another member # through `members`, and NixOps4 evaluates the root again with the # outputs it learns. The SSH settings change the command line as # they do in nixops4-nixos. nixops4ResourceReadsSiblingOutputs = let top = nixops4TopWith { - root = - { lib, ... }: - { - caisson.caisson-nixops4-resource.configurations.address = - lib.caisson.caisson-nixops4-resource.mkConfiguration - { - configModule = - { providers, ... }: - { - providers.local = inputs.nixops4.modules.nixops4Provider.local; - type = providers.local.exec; - inputs.executable = "/bin/true"; - }; - }; - }; - resources.probe = - { members, ... }: - { + root = nixops4AddressRoot; + wiring = members: { + probe = { ssh.host = members.address.outputs.stdout; ssh.hostPublicKey = nixops4HostKey; ssh.user = "deploy"; ssh.opts = "-p 2222"; copy.substituteOnDestination = false; }; + }; } { probe = minimalNixosBase; }; export = nixops4Export top.nixops4; args = export.members.probe.resource.inputs.args; @@ -2278,9 +2347,9 @@ let && !(pkgs.lib.hasInfix "--substitute-on-destination" script); # The shell that runs the deployment comes from the package set - # caisson selects for the resource, inherited from the - # caisson-nixops4 configuration, for the deploying system: here a - # selection whose `bash` is another shell. + # caisson selects for the resource, inherited from the root + # composite, for the deploying system: here a selection whose + # `bash` is another shell. nixops4DeploysWithTheSelectedPackageSet = let top = nixops4TopWith { @@ -2312,12 +2381,12 @@ let members.extra = { }; })); - # The root composite is a deferred child of the caisson-nixops4 - # configuration: the manifest lists its entry, and nothing beneath - # it is evaluated at flake time. The args it needs name the - # deploying system and, once that is known, the outputs of its + # The root composite is a deferred configuration declared at the + # top: the manifest of the top holds its entry, and nothing beneath + # it is evaluated at flake time. Finalized, its manifest names the + # args it takes: the deploying system and the outputs of its # resources, which are not cacheable unless a resource lists them. - # Finalized, it records which args it was given, not their values. + # Each manifest records where its args came from, not their values. nixops4RootIsDeferred = let top = nixops4TopWith { @@ -2326,15 +2395,15 @@ let cacheableOutputs = [ "stdout" ]; }; } { probe = minimalNixosBase; }; - declaration = top.caisson.manifest.children.caisson-nixops4.nixops4; - entry = declaration.children.caisson-nixops4-composite.root; - needed = core.argsNeededFor entry { resourceProviderSystem = "x86_64-linux"; }; + entry = nixops4Entry top; + root = nixops4RootManifest top; + needed = root.argsNeeded; in core.isDeferred entry && entry.type == "caisson-nixops4-composite" && entry.parent.childless && - core.argNamesOf needed == [ + nixops4ArgPathsOf needed == [ [ "outputValues" "probe" @@ -2343,30 +2412,186 @@ let [ "resourceProviderSystem" ] ] && needed.outputValues.probe.stdout.cacheable - && (nixops4RootManifest top).deferred.argNames == core.argNamesOf needed; - - # A root composite that lists the resources holding its state gives - # a program, built for the system of the tool that asks, that - # reads the outputs NixOps4 recorded; one that lists none gives - # nothing. A finalized composite tells a tool the args it gives a - # member (`deferredChildArgs`), so the tool can finalize the - # member's entry itself. + && needed.resourceProviderSystem.cacheable + && + root.argsSource [ + "outputValues" + "probe" + "stdout" + ] == "the probe" + && + root.children.caisson-nixops4-resource.probe.argsSource [ + "outputValues" + "stdout" + ] == "the composite at the root"; + + # An output NixOps4 has not passed fails where it is read, with the + # text `caisson-deferred: needs outputValues.address.stdout`, and + # the members that do not read it evaluate. + nixops4MissingOutputFailsWhereRead = + let + top = nixops4TopWith { + root = nixops4AddressRoot; + wiring = members: { + probe = { + ssh.host = members.address.outputs.stdout; + ssh.hostPublicKey = nixops4HostKey; + }; + }; + } { probe = minimalNixosBase; }; + export = top.nixops4.rootFunction { + resourceProviderSystem = "x86_64-linux"; + outputValues.probe.stdout = "stub"; + }; + in + export.members.address.resource.type == "exec" + && !(builtins.tryEval (builtins.elemAt export.members.probe.resource.inputs.args 2)).success; + + # A resource whose type depends on the output of another member + # names its own outputs once that output is known. Given the + # output, it evaluates, with the type the output selects. Without + # it, the outputs of the other members are still named. + nixops4TypeDependsOnAnotherOutput = + let + top = nixops4TopWith { + root = + { lib, members, ... }: + { + imports = [ nixops4AddressRoot ]; + caisson.caisson-nixops4-resource.configurations.dependent = + lib.caisson.caisson-nixops4-resource.mkConfiguration + { + configModule = + { providers, ... }: + let + file = members.address.outputs.stdout == "file"; + in + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = if file then providers.local.file else providers.local.exec; + inputs = + if file then + { + name = "/run/probe"; + contents = ""; + } + else + { executable = "/bin/true"; }; + }; + }; + }; + } { }; + exportWith = + stdout: + top.nixops4.rootFunction { + resourceProviderSystem = "x86_64-linux"; + outputValues = { + address = { inherit stdout; }; + dependent = { }; + }; + }; + withoutOutputs = (nixops4Entry top).finalize { + args.resourceProviderSystem = "x86_64-linux"; + argsSource = _: "the probe"; + }; + in + (exportWith "file").members.dependent.resource.type == "file" + && (exportWith "file").members.dependent.resource.outputsSkeleton == { } + && builtins.attrNames (exportWith "exec").members.dependent.resource.outputsSkeleton == [ "stdout" ] + && builtins.attrNames withoutOutputs.argsNeeded.outputValues.address == [ "stdout" ] + && !(builtins.tryEval withoutOutputs.argsNeeded.outputValues.dependent).success; + + # The `members` argument of a native composite holds, for each + # member, its outputs and its member path, from the args and the + # names the members are declared under. A member's config is not + # reachable through it. + nixops4NativeMembersHoldOutputsAndPaths = + let + top = nixops4TopWith { + root = + { lib, members, ... }: + { + imports = [ nixops4AddressRoot ]; + options.seen = lib.mkOption { type = lib.types.raw; }; + config.seen = members; + }; + } { }; + seen = (nixops4RootManifest top).value.config.seen; + in + builtins.attrNames seen.address == [ + "absoluteMemberPath" + "outputs" + ] + && seen.address.absoluteMemberPath == [ "address" ] + && seen.address.outputs.stdout == "stub"; + + # A deployment that holds a resource with a state gives a program, + # built for the system of the tool that asks, that reads the + # outputs NixOps4 recorded. The program finds the resources that + # hold state from the `state` option of each resource. A deployment + # with no state gives none. nixops4RootDiscoversRecordedOutputs = let - entryOf = - top: top.caisson.manifest.children.caisson-nixops4.nixops4.children.caisson-nixops4-composite.root; stateful = nixops4TopWith { - rootArgs.stateResources = [ "state" ]; + root = + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.state = + lib.caisson.caisson-nixops4-resource.mkConfiguration + { + configModule = + { providers, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = providers.local.state_file; + inputs.name = "state.json"; + }; + }; + }; resources.probe.ssh.hostPublicKey = nixops4HostKey; + wiring = members: { probe.state = members.state; }; } { probe = minimalNixosBase; }; - discover = core.discoverArgsFor (entryOf stateful) "x86_64-linux"; - memberArgs = (nixops4RootManifest stateful).outputs.deferredChildArgs "probe"; + root = nixops4RootManifest stateful; + discover = root.discoverArgs { system = "x86_64-linux"; }; in - core.discoverArgsFor (entryOf (nixops4TopWith { } { })) "x86_64-linux" == null + (nixops4RootManifest (nixops4TopWith { } { })).discoverArgs { system = "x86_64-linux"; } == null + && root.outputs.stateResources == [ [ "state" ] ] + && (nixops4Export stateful.nixops4).members.probe.resource.state == [ "state" ] && discover.name == "caisson-nixops4-discover" - && discover.system == "x86_64-linux" - && memberArgs.path == [ "probe" ] - && memberArgs.outputValues.stdout == "stub"; + && discover.system == "x86_64-linux"; + + # A deferred configuration of another tool beside the root of a + # NixOps4 deployment in one flake has its own entry, publication, + # args and discovery program, and each entry finalizes without the + # other. + nixops4BesideAnotherDeferredTool = + let + top = hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + imports = [ (lib.caisson.integrations.configurationsModuleFor [ "greeter" ]) ]; + caisson.greeter.configurations.hello = greeterConfiguration lib; + caisson.caisson-nixops4-composite.configurations.nixops4 = + lib.caisson.caisson-nixops4-composite.mkConfiguration + { configModule = nixops4AddressRoot; }; + }; + }; + greeter = top.caisson.manifest.children.greeter.hello; + greeted = greeter.finalize { + args.who = "caisson"; + argsSource = _: "the probe"; + }; + in + core.isDeferred greeter + && greeter.type == "greeter" + && top.greetings.hello "world" == "hello world" + && greeted.outputs.greeting == "hello caisson" + && greeted.argsNeeded.who.cacheable + && (greeted.discoverArgs { system = "x86_64-linux"; }).name == "greeter-discover" + && (nixops4RootManifest top).discoverArgs { system = "x86_64-linux"; } == null + && (nixops4Export top.nixops4).members.address.resource.type == "exec"; # A provider is evaluated inside the component that declares it. # `getProviders` evaluates the root without outputs and builds the @@ -2413,7 +2638,7 @@ let let top = nixops4UpstreamTopWith { } [ nixops4UpstreamRoot ]; probe = (nixops4Export top.nixops4).members.probe.resource; - member = (nixops4RootManifest top).outputs.members.probe; + member = (nixops4RootManifest top).children.caisson-nixops4-resource.probe; machine = member.children.system.x86_64-linux.children.nixos.probe; upstream = import "${inputs.nixpkgs}/nixos/lib/eval-config.nix" { system = null; @@ -2460,11 +2685,57 @@ let nixops4UpstreamMembersReadTheFullComposite = let root = nixops4RootManifest (nixops4UpstreamTopWith { } [ nixops4UpstreamRoot ]); - probe = root.outputs.members.probe; + probe = root.children.caisson-nixops4-resource.probe; in probe.crossChildLevel != null && !probe.crossChildLevel.childless - && probe.crossChildLevel.name == "root"; + && probe.crossChildLevel.name == "nixops4"; + + # Upstream code reads a member through `members` and through + # `config.members`: its outputs, and the member itself as the + # resource that holds the state of another, which NixOps4's `state` + # option accepts by its `absoluteMemberPath`. + nixops4UpstreamReadsMembersAsUpstreamDoes = + let + top = nixops4UpstreamTopWith { } [ + ( + { + providers, + members, + config, + ... + }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members.state = { + type = providers.local.state_file; + inputs.name = "state.json"; + }; + members.address = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + members.memo = { + type = providers.local.memo; + state = members.state; + inputs.initialize_with = config.members.address.outputs.stdout; + }; + members.group.members.echo = { + type = providers.local.exec; + inputs.executable = "/bin/echo"; + inputs.args = [ members.address.outputs.stdout ]; + }; + } + ) + ]; + export = nixops4Export top.nixops4; + root = nixops4RootManifest top; + in + export.members.memo.resource.state == [ "state" ] + && export.members.memo.resource.inputs.initialize_with == "stub" + && export.members.group.members.echo.resource.inputs.args == [ "stub" ] + && root.children.caisson-nixops4-resource.memo.value.config.absoluteMemberPath == [ "memo" ] + && root.outputs.stateResources == [ [ "state" ] ]; # A member that sets `type` and declares members is refused, where # NixOps4 drops the members. @@ -2494,8 +2765,7 @@ let top = nixops4UpstreamTopWith { cacheableOutputs."group.inner" = [ "stdout" ]; } [ nixops4UpstreamRoot ]; - entry = top.caisson.manifest.children.nixops4.nixops4.children.caisson-nixops4-composite.root; - needed = core.argsNeededFor entry { resourceProviderSystem = "x86_64-linux"; }; + needed = (nixops4RootManifest top).argsNeeded; in needed.outputValues.group.inner.stdout.cacheable && !needed.outputValues.address.stdout.cacheable; diff --git a/tests/unit/lib-overlays.nix b/tests/unit/lib-overlays.nix index 538cdbfb..e36831a0 100644 --- a/tests/unit/lib-overlays.nix +++ b/tests/unit/lib-overlays.nix @@ -126,7 +126,6 @@ let "facter" "caisson-microvm" "microvm" - "caisson-nixops4" "caisson-nixops4-composite" "caisson-nixops4-resource" "terranix" @@ -5693,7 +5692,6 @@ in "facter" "caisson-microvm" "microvm" - "caisson-nixops4" "caisson-nixops4-composite" "caisson-nixops4-resource" "treefmt" @@ -5715,7 +5713,6 @@ in "caisson-facter" "facter" "microvm" - "caisson-nixops4" "caisson-nixops4-composite" "caisson-nixops4-resource" "treefmt" From 415767f73b457197584c50ab4b610ddb81f5f7eb Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Tue, 27 Oct 2026 02:05:05 -0700 Subject: [PATCH 7/9] fix(nixops4): build the discovery program for one state resource, read dumps from the root, and name the errors of members and outputs named from outputs The discovery program looped with `for state in `, which shellcheck refuses (SC2043) when a root has one state resource, so the program did not build. It loops over a bash array now. It also nested the outputs of a state resource inside a composite twice: `nixops4 state dump` keys the resources from the root, and the program prefixed the path of the composite again. The new `nixops4-discovery` check builds the program of two roots against a stand-in for NixOps4 and compares what it prints from recorded dumps with the paths the root names in `argsNeeded`. A composite that decided its members from `members` or their outputs, and a resource that named its outputs from its own outputs, ended in an infinite recursion. A composite now takes the names of its members from an evaluation in which `members` fails with an error naming the composite, and a resource takes the names of its outputs from one in which its outputs fail with an error naming the resource. The pinned-world suite gains probes for nested composites, a machine in a nested composite, these errors, one member under a name, one root per flake, a root inside a machine, entries a parent does not finalize, `getProviders`, and nested upstream composites. The `deferred-errors` check reads the text of the errors, `caisson-deferred: needs ` included, with the real evaluator. Co-Authored-By: Claude Opus 5.5 --- configs/flake/caisson/partitions/checks.nix | 28 + docs/development/testing-architecture.md | 2 + docs/development/testing.md | 19 + .../caisson-nixops4-composite/default.nix | 96 ++- .../caisson-nixops4-resource/default.nix | 87 ++- tests/deferred-errors/cases.nix | 290 +++++++++ tests/deferred-errors/check.sh | 47 ++ tests/nixops4-discovery/default.nix | 174 ++++++ tests/nixops4-discovery/group-state.json | 23 + tests/nixops4-discovery/root-state.json | 18 + tests/pinned-world/default.nix | 565 ++++++++++++++++++ 11 files changed, 1285 insertions(+), 64 deletions(-) create mode 100644 tests/deferred-errors/cases.nix create mode 100644 tests/deferred-errors/check.sh create mode 100644 tests/nixops4-discovery/default.nix create mode 100644 tests/nixops4-discovery/group-state.json create mode 100644 tests/nixops4-discovery/root-state.json diff --git a/configs/flake/caisson/partitions/checks.nix b/configs/flake/caisson/partitions/checks.nix index 18e2b26f..f824d884 100644 --- a/configs/flake/caisson/partitions/checks.nix +++ b/configs/flake/caisson/partitions/checks.nix @@ -234,6 +234,14 @@ caisson = self; }; }; + # The discovery program of a NixOps4 root, built and run + # against recorded state dumps, with a stand-in for + # NixOps4 that prints them (tests/nixops4-discovery). + nixops4-discovery = import (self.outPath + "/tests/nixops4-discovery") { + inputs = inputs // { + caisson = self; + }; + }; # A machine booted in a VM that activates the # home-manager configurations, the hjem configurations # and the profiles declared inside it, and a standalone @@ -264,6 +272,26 @@ --impure --extra-experimental-features nix-command touch $out ''; + # The errors of deferred configurations and the NixOps4 + # integrations, read as text by the real evaluator, as + # a tool that finalizes a deferred configuration reads + # them. It runs as the argument-error check does, on + # the same library. + deferred-errors = + pkgs.runCommand "deferred-errors" + { + nativeBuildInputs = [ pkgs.nix ]; + } + '' + export HOME="$(realpath .)" + unset NIX_STORE + export NIX_STORE_DIR=${builtins.storeDir} + export NIX_REMOTE="$HOME/storedata" + bash ${self.outPath}/tests/deferred-errors/check.sh ${lib.escapeShellArg argumentErrorsLib} \ + ${inputs.nixops4.outPath} \ + --impure --extra-experimental-features nix-command + touch $out + ''; minimal-consumer-all-outputs = builtins.seq minimalConsumerOutputs.flakeModule ( builtins.seq minimalConsumerOutputs.lib ( pkgs.runCommand "minimal-consumer-all-outputs" { } "touch $out" diff --git a/docs/development/testing-architecture.md b/docs/development/testing-architecture.md index 869cfa87..51e97107 100644 --- a/docs/development/testing-architecture.md +++ b/docs/development/testing-architecture.md @@ -320,4 +320,6 @@ The checks partition merges outputs from all test and example flakes into the | `system-manager-vm-` | `checks.nix` + `tests/system-manager/vm.nix` | A system-manager machine, activated through sequences of changes in a VM of Ubuntu 22.04 or Debian 12 booted from the cloud image of the distribution, applies its profiles, its home-manager configurations and its hjem configuration, and a home-manager configuration activated by hand for another version warns of the drift | | `containerTests.x86_64-linux.system-manager-` (not a check) | `checks.nix` + `tests/system-manager/container.nix` | The same sequences in a container of each distribution of `tests/system-manager/container-distros.nix`; the container needs `auto-allocate-uids`, so the `system-manager-container` job of the check workflow builds each by name | | `argument-errors` | `checks.nix` + `tests/argument-errors/` | The argument error of every entry point comes from Nix itself, raised at the call site with no frame of caisson | +| `deferred-errors` | `checks.nix` + `tests/deferred-errors/` | The errors of deferred configurations and the NixOps4 integrations carry their text: `caisson-deferred: needs ` with the member path from the root, and the errors that refuse members or outputs named from outputs, a name declared as composite and resource, and a second root | +| `nixops4-discovery` | `checks.nix` + `tests/nixops4-discovery/` | The discovery program of a NixOps4 root builds, with one state resource and with two, and prints, from recorded state dumps, the output values at the paths the root names in `argsNeeded` | | `eval-weight` | `checks.nix` + `tests/eval-weight/` | Framework evaluation cost held to committed ceilings ([guide](https://nix-caisson.github.io/docs/eval-weight.html)) | diff --git a/docs/development/testing.md b/docs/development/testing.md index 6465a6b0..0b34f69b 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -76,6 +76,25 @@ plumbing works. 'let f = builtins.getFlake "path:'"$PWD"'"; in f.lib' --impure ``` +## Deferred Configuration Errors (Message Text) + +- **Scope:** The text of the errors of deferred configurations and the + NixOps4 integrations. A tool tells an arg it lacks from a failure by + `caisson-deferred: needs `, and `builtins.tryEval` in the + pinned-world suite sees only that an evaluation failed. +- **Location:** `tests/deferred-errors/check.sh` and its cases in + `tests/deferred-errors/cases.nix`. +- **Mechanism:** the script runs `nix eval` on each case and looks for + the line of the message the case names. The `deferred-errors` check + runs it in the sandbox over the library of the `argument-errors` + check and the NixOps4 source. +- **Execution against the flake's library, from a checkout:** + ```bash + tests/deferred-errors/check.sh \ + 'let f = builtins.getFlake "path:'"$PWD"'"; in f.lib' \ + --impure + ``` + ## Integration Testing (Test Flakes) - **Scope:** Module composition, end-to-end evaluation, and build success. diff --git a/lib-overlays/caisson-nixops4-composite/default.nix b/lib-overlays/caisson-nixops4-composite/default.nix index 1b361ead..85f33c65 100644 --- a/lib-overlays/caisson-nixops4-composite/default.nix +++ b/lib-overlays/caisson-nixops4-composite/default.nix @@ -192,7 +192,12 @@ inherit lib name class; configModule = settings.configModule or null; }; - entries = memberEntriesOf path evaluated.config; + # The members, by name. The names come from `naming`, and + # each entry comes from the evaluation, so the composite's + # module may read `members` wherever it does not decide + # which members there are. + declared = memberEntriesOf path evaluated.config; + entries = builtins.mapAttrs (member: _: declared.${member}) (memberEntriesOf path naming.config); # The members, each finalized once with its part of the # composite's args, from the full evaluation. The # childless view finalizes none. @@ -211,41 +216,70 @@ } ) entries; ofType = memberType: final.filterAttrs (_: member: member.type == memberType) finalized; - membersModule = selection.caissonModule { - key = "caisson.${type}.members"; - file = "caisson-${type}:members"; - module = - { config, ... }: - { - _module.args.members = membersArg { - inherit - args - path - entries - config - ; - }; - }; - }; - moduleList = + # The modules of an evaluation in which the `members` + # module argument is what `members` returns for the + # evaluation's config, and the base modules receive + # `finalized`. + modulesWith = + { members, finalized }: baseModules { inherit src path finalized; } - ++ [ membersModule ] + ++ [ + (selection.caissonModule { + key = "caisson.${type}.members"; + file = "caisson-${type}:members"; + module = + { config, ... }: + { + _module.args.members = members config; + }; + }) + ] ++ frameworkModules ++ moduleImports registry ++ (if configModule == null then [ ] else [ configModule ]); - evaluated = lib.evalModules ( - { - inherit class; - specialArgs = { - inherit lib; - inputs = manifest.sources; - inherit (args) resourceProviderSystem; + evaluationOf = + modules: + lib.evalModules ( + { + inherit class modules; + specialArgs = { + inherit lib; + inputs = manifest.sources; + inherit (args) resourceProviderSystem; + } + // (if (settings.specialArgs or null) != null then settings.specialArgs else { }); } - // (if (settings.specialArgs or null) != null then settings.specialArgs else { }); - modules = moduleList; - } - // (if (settings.ecosystemArgs or null) != null then settings.ecosystemArgs else { }) - ); + // (if (settings.ecosystemArgs or null) != null then settings.ecosystemArgs else { }) + ); + moduleList = modulesWith { + inherit finalized; + members = + config: + membersArg { + inherit + args + path + entries + config + ; + }; + }; + evaluated = evaluationOf moduleList; + # The evaluation that decides which members there are. The + # members and their outputs fail there when they are read, + # with an error that names the composite, since the names + # of the members must be known before any member is + # finalized. + naming = evaluationOf (modulesWith { + members = _: membersUnknown; + finalized = membersUnknown; + }); + membersUnknown = throw '' + ${context}: the composite at ${showPath path} reads its members, or their + outputs, where it decides which members it has. Which members a composite + has, and whether each is a composite or a resource, must not depend on its + members or on their outputs. + ''; # The member paths of the resources in the tree of this # composite that hold state, each once. stateResources = final.unique ( diff --git a/lib-overlays/caisson-nixops4-resource/default.nix b/lib-overlays/caisson-nixops4-resource/default.nix index 048b5ae8..c49c5c36 100644 --- a/lib-overlays/caisson-nixops4-resource/default.nix +++ b/lib-overlays/caisson-nixops4-resource/default.nix @@ -166,10 +166,12 @@ # member paths of the resources that hold state joined with `.`, # in the working directory, which NixOps4 finds the flake from. It # passes its own arguments to NixOps4, such as `--override-input`, - # and prints `{ outputValues, resourceProviderSystem }`. The state - # of a resource at `a.state` covers the composite `a`: its - # resources' recorded outputs, and its nested composites in turn. - # Only resources with a state have recorded outputs. + # and prints `{ outputValues, resourceProviderSystem }`. A dump + # holds the resources that record their outputs in that state, at + # their member paths from the root: the resources of the root + # under `resources`, and those of a composite `a` under + # `deployments.a`, in the same shape. Only resources with a state + # have recorded outputs. discoverArgsOf = { manifest, @@ -188,13 +190,14 @@ ]; text = '' merged='{}' - for state in ${final.escapeShellArgs stateResources}; do + states=(${final.escapeShellArgs stateResources}) + for state in "''${states[@]}"; do dump=$(nixops4 "$@" state dump "$state") - merged=$(jq -cn --argjson merged "$merged" --argjson dump "$dump" --arg state "$state" ' + merged=$(jq -cn --argjson merged "$merged" --argjson dump "$dump" ' def outputValues: ((.resources // {}) | map_values(.output_properties)) + ((.deployments // {}) | map_values(outputValues)); - $merged * ({} | setpath($state | split(".") | .[:-1]; $dump | outputValues)) + $merged * ($dump | outputValues) ') done jq -cn --argjson merged "$merged" '{ outputValues: $merged, resourceProviderSystem: "${system}" }' @@ -266,36 +269,54 @@ inherit lib name class; configModule = settings.configModule or null; }; - # What NixOps4 gives a resource: its outputs, from the - # output values it passes, and its member path. An output - # NixOps4 has not passed fails when it is read, naming - # itself. - outputsModule = selection.caissonModule { - key = "caisson.${type}.outputs"; - file = "caisson-${type}:outputs"; - module = { - config.outputs = _: { config = args.outputValues; }; - config.absoluteMemberPath = path; - }; - }; - moduleList = + # The modules of an evaluation in which the resource's + # outputs are `outputs`, with what NixOps4 gives a + # resource beside them, its member path. + modulesWith = + outputs: baseModules { inherit src path; } - ++ [ outputsModule ] + ++ [ + (selection.caissonModule { + key = "caisson.${type}.outputs"; + file = "caisson-${type}:outputs"; + module = { + config.outputs = _: { config = outputs; }; + config.absoluteMemberPath = path; + }; + }) + ] ++ frameworkModules ++ moduleImports registry ++ (if configModule == null then [ ] else [ configModule ]); - evaluated = lib.evalModules ( - { - inherit class; - specialArgs = { - inherit lib; - inputs = manifest.sources; - inherit (args) resourceProviderSystem; + evaluationOf = + modules: + lib.evalModules ( + { + inherit class modules; + specialArgs = { + inherit lib; + inputs = manifest.sources; + inherit (args) resourceProviderSystem; + } + // (if (settings.specialArgs or null) != null then settings.specialArgs else { }); } - // (if (settings.specialArgs or null) != null then settings.specialArgs else { }); - modules = moduleList; - } - // (if (settings.ecosystemArgs or null) != null then settings.ecosystemArgs else { }) + // (if (settings.ecosystemArgs or null) != null then settings.ecosystemArgs else { }) + ); + # The outputs are the output values NixOps4 passes. An + # output NixOps4 has not passed fails when it is read, + # naming itself. + moduleList = modulesWith args.outputValues; + evaluated = evaluationOf moduleList; + # The evaluation that names the outputs, for `argsNeeded`. + # Reading an output fails there with an error that names + # the resource, since NixOps4 needs the names of the + # outputs before it has their values. + naming = evaluationOf ( + modulesWith (throw '' + ${context}: the resource at ${showPath path} reads its own outputs in its + `outputsSkeleton`, which names its outputs. NixOps4 needs the names of the + outputs of a resource before it has their values. + '') ); cacheableOutputs = evaluated.config.cacheableOutputs or [ ]; in @@ -323,7 +344,7 @@ cacheable = builtins.elem output cacheableOutputs; description = "An output of the resource at ${showPath path}."; } - ) evaluated.config.outputsSkeleton; + ) naming.config.outputsSkeleton; }; }; in diff --git a/tests/deferred-errors/cases.nix b/tests/deferred-errors/cases.nix new file mode 100644 index 00000000..334735b8 --- /dev/null +++ b/tests/deferred-errors/cases.nix @@ -0,0 +1,290 @@ +# SPDX-License-Identifier: MIT +# +# The cases of tests/deferred-errors/check.sh: deferred configurations +# that fail, each with a line of the message it must fail with. `lib` +# is caisson's composed library, and `nixops4` the NixOps4 source, +# whose component modules the NixOps4 integrations evaluate. The +# deployments use a provider written here, `probe`, whose resource +# type `exec` has the output `stdout`, so nothing of NixOps4's flake +# outputs is evaluated. +# +# Each case is `{ value; message; }`: forcing `value` must fail, and the +# error must contain `message` as written. +{ lib, nixops4 }: +let + system = "x86_64-linux"; + + # The NixOps4 source as the integrations check it: its component + # modules, and a module collection that holds `nixops4Provider`. + ecosystemSrc = { + outPath = nixops4; + modules.nixops4Provider = { }; + }; + + provider = + { lib, ... }: + { + executable = "/bin/probe"; + resourceTypes.exec = { + description = "A resource with the output `stdout`."; + requireState = false; + inputs.options.value = lib.mkOption { + type = lib.types.str; + default = ""; + }; + outputs.options.stdout = lib.mkOption { type = lib.types.str; }; + }; + }; + + # A native composite with the module `module`. + composite = + lib: module: + lib.caisson.caisson-nixops4-composite.mkConfiguration { + ecosystemSrc = _sources: ecosystemSrc; + configModule = module; + }; + + # A native resource of the type `exec` with the module `module`. + resource = + lib: module: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + ecosystemSrc = _sources: ecosystemSrc; + configModule = + { providers, ... }: + { + imports = [ module ]; + providers.probe = provider; + type = providers.probe.exec; + }; + }; + + # A top whose root composite `nixops4` has the module `module`. + topWith = + module: + lib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.nixops4 = composite lib module; + }; + }; + + # A top whose root is an upstream-compatible nixops4 configuration + # made from `modules`. + upstreamTopWith = + modules: + lib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.nixops4.configurations.nixops4 = lib.caisson.nixops4.mkConfiguration { + ecosystemSrc = _sources: ecosystemSrc; + inherit modules; + }; + }; + }; + + # The root of a top, finalized with the deploying system alone, as + # `getProviders` finalizes it: every output fails when it is read. + rootOf = + top: + let + children = top.caisson.manifest.children; + in + (children.caisson-nixops4-composite or children.nixops4).nixops4.finalize { + args.resourceProviderSystem = system; + argsSource = _: "the check"; + }; + + # A composite module whose resource `reader` reads the output + # `stdout` of its member `source`. + readerModule = + { lib, members, ... }: + { + caisson.caisson-nixops4-resource.configurations = { + source = resource lib { }; + reader = resource lib { inputs.value = members.source.outputs.stdout; }; + }; + }; +in +{ + # An output NixOps4 has not passed names its member path from the + # root. + missingOutput = { + value = (rootOf (topWith readerModule)).outputs.export; + message = "caisson-deferred: needs outputValues.source.stdout"; + }; + + # The member path of an output inside nested composites holds each + # composite. + missingNestedOutput = { + value = + (rootOf ( + topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.a = composite lib ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.b = composite lib readerModule; + } + ); + } + ) + )).outputs.export; + message = "caisson-deferred: needs outputValues.a.b.source.stdout"; + }; + + # The same holds for an upstream composite inside another. + missingUpstreamNestedOutput = { + value = + (rootOf (upstreamTopWith [ + ( + { providers, members, ... }: + { + providers.probe = provider; + members.g.members.h.members.source.type = providers.probe.exec; + members.g.members.reader = { + type = providers.probe.exec; + inputs.value = members.g.members.h.members.source.outputs.stdout; + }; + } + ) + ])).outputs.export; + message = "caisson-deferred: needs outputValues.g.h.source.stdout"; + }; + + # Which members a composite has may not depend on their outputs. + membersFromOutputs = { + value = + (rootOf ( + topWith ( + { lib, members, ... }: + { + caisson.caisson-nixops4-resource.configurations = { + source = resource lib { }; + } + // lib.optionalAttrs (members.source.outputs.stdout == "more") { + extra = resource lib { }; + }; + } + ) + )).argsNeeded; + message = "the composite at the root reads its members, or their"; + }; + + # The error names the composite that reads them, here one beneath the + # root. + nestedMembersFromOutputs = { + value = + (rootOf ( + topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.a = composite lib ( + { lib, members, ... }: + { + caisson.caisson-nixops4-resource.configurations = lib.optionalAttrs (members ? source) { + source = resource lib { }; + }; + } + ); + } + ) + )).argsNeeded; + message = "the composite at a reads its members, or their"; + }; + + # Whether an upstream member is a composite or a resource may not + # depend on the outputs of another member. + upstreamKindFromOutputs = { + value = + (rootOf (upstreamTopWith [ + ( + { providers, members, ... }: + { + providers.probe = provider; + members.source.type = providers.probe.exec; + members.dynamic = + if members.source.outputs.stdout == "resource" then + { type = providers.probe.exec; } + else + { members.inner.type = providers.probe.exec; }; + } + ) + ])).argsNeeded; + message = "the composite at the root reads its members, or their"; + }; + + # A resource may not name its outputs from its own outputs. + skeletonFromOwnOutputs = { + value = + (rootOf ( + topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.selfish = resource lib ( + { config, ... }: + { + outputsSkeleton = lib.mkForce ( + { + stdout = { }; + } + // lib.optionalAttrs (config.outputs.stdout == "more") { extra = { }; } + ); + } + ); + } + ) + )).argsNeeded; + message = "the resource at selfish reads its own outputs in its"; + }; + + # A member is a composite or a resource. + compositeAndResource = { + value = + (rootOf ( + topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.x = resource lib { }; + caisson.caisson-nixops4-composite.configurations.x = composite lib { }; + } + ) + )).argsNeeded; + message = "`x` both as a composite and as a"; + }; + + # A flake publishes one root as `nixops4`. + twoRoots = { + value = + (lib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.nixops4 = composite lib { }; + caisson.nixops4.configurations.nixops4 = lib.caisson.nixops4.mkConfiguration { + ecosystemSrc = _sources: ecosystemSrc; + }; + }; + }).nixops4; + message = "nixops4: this output holds a single value, the configuration"; + }; + + # A root is declared under the name of the output it fills. + rootNamedOtherwise = { + value = + (lib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.deploy = composite lib { }; + }; + }).nixops4; + message = "caisson-nixops4-composite deploy"; + }; +} diff --git a/tests/deferred-errors/check.sh b/tests/deferred-errors/check.sh new file mode 100644 index 00000000..2d437976 --- /dev/null +++ b/tests/deferred-errors/check.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# The errors of deferred configurations and the NixOps4 integrations, +# read as text. A tool reads the error of a deferred configuration to +# tell an arg it lacks (`caisson-deferred: needs `) from a +# failure, and the integrations refuse what they cannot evaluate with +# errors that name the configuration. `builtins.tryEval` hides the +# text of an error, so this runs the real evaluator on each case of +# cases.nix and looks for the line the case names in what it prints. +# +# check.sh LIB_EXPR NIXOPS4_SRC [nix eval flag...] +# +# LIB_EXPR is a Nix expression for the composed library and NIXOPS4_SRC +# the path of the NixOps4 source; the flags follow `nix eval`. The +# flake check composes the library from store paths, as the check of +# tests/argument-errors does. +set -euo pipefail + +lib_expr="$1" +nixops4="$2" +shift 2 +flags=("$@") + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cases="(import $here/cases.nix { lib = $lib_expr; nixops4 = \"$nixops4\"; })" + +names="$(nix eval "${flags[@]}" --raw --expr "builtins.concatStringsSep \"\n\" (builtins.attrNames $cases)")" + +failures=0 +while IFS= read -r name; do + message="$(nix eval "${flags[@]}" --raw --expr "$cases.$name.message")" + # `nix eval` prints an error inside a value in place of failing, so + # the value is forced whole first. + if out="$(nix eval "${flags[@]}" --expr "let value = $cases.$name.value; in builtins.deepSeq value true" 2>&1)"; then + printf 'FAIL %s: evaluated without an error:\n%s\n\n' "$name" "$out" + failures=$((failures + 1)) + elif grep -qF -- "$message" <<< "$out"; then + printf 'ok %s\n' "$name" + else + printf 'FAIL %s: the error does not contain\n %s\n%s\n\n' "$name" "$message" "$out" + failures=$((failures + 1)) + fi +done <<< "$names" + +if [ "$failures" -ne 0 ]; then + printf '%d case(s) failed\n' "$failures" + exit 1 +fi diff --git a/tests/nixops4-discovery/default.nix b/tests/nixops4-discovery/default.nix new file mode 100644 index 00000000..5a09af3d --- /dev/null +++ b/tests/nixops4-discovery/default.nix @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: MIT +# +# The discovery program of a NixOps4 root, built and run. `inputs` is +# the dependency pool of the checks partition with `caisson` set to +# the tree itself. +# +# Each deployment below holds resources that record their outputs in a +# state resource. Its root composite takes NixOps4 from a source whose +# `nixops4` package is a stand-in: `nixops4 state dump ` prints +# the dump recorded in this directory for the state resource at that +# path. The dumps are trimmed from the dumps of a real `nixops4 apply`. +# Building the program runs shellcheck on it. Running it must print the +# output values at the paths the root names in `argsNeeded`, with the +# values the dumps record. +{ inputs }: +let + suite = import ../pinned-world { inherit inputs; }; + inherit (suite.fixtures) + hiveLib + pkgs + nixops4ArgPathsOf + nixops4Entry + ; + system = "x86_64-linux"; + local = inputs.nixops4.modules.nixops4Provider.local; + + # The stand-in for NixOps4: it answers `state dump ` with the + # dump recorded for the state resource at that path. + nixops4 = pkgs.writeShellScriptBin "nixops4" '' + case "''${*: -1}" in + state) cat ${./root-state.json} ;; + group.gstate) cat ${./group-state.json} ;; + *) echo "no dump for ''${*: -1}" >&2; exit 1 ;; + esac + ''; + ecosystemSrc = inputs.nixops4 // { + packages = inputs.nixops4.packages // { + ${system} = inputs.nixops4.packages.${system} // { + inherit nixops4; + }; + }; + }; + + # A top whose root composite `nixops4` holds the composite `group`, + # with the state resource `gstate` and the resource `inner`, which + # records its outputs there. With `rootState`, the root also holds the + # state resource `state` and the resource `a`, which records its + # outputs there. + topWith = + { rootState }: + hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + let + resource = + module: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + configModule = + { providers, ... }: + { + imports = [ module ]; + providers.local = local; + }; + }; + stateFile = + name: + resource ( + { providers, ... }: + { + type = providers.local.state_file; + inputs.name = name; + } + ); + exec = + state: + resource ( + { providers, ... }: + { + type = providers.local.exec; + inputs.executable = "/bin/sh"; + inherit state; + } + ); + in + { + caisson.caisson-nixops4-composite.configurations.nixops4 = + lib.caisson.caisson-nixops4-composite.mkConfiguration + { + ecosystemSrc = _sources: ecosystemSrc; + configModule = + { lib, members, ... }: + { + caisson.caisson-nixops4-composite.configurations.group = + lib.caisson.caisson-nixops4-composite.mkConfiguration + { + configModule = + { members, ... }: + { + caisson.caisson-nixops4-resource.configurations = { + gstate = stateFile "group-state.json"; + inner = exec members.gstate; + }; + }; + }; + caisson.caisson-nixops4-resource.configurations = lib.optionalAttrs rootState { + state = stateFile "state.json"; + a = exec members.state; + }; + }; + }; + }; + }; + + # The paths of the values in a tree of attribute sets, each a list of + # names. + valuePathsOf = + let + walk = + path: value: + if builtins.isAttrs value then + builtins.concatMap (name: walk (path ++ [ name ]) value.${name}) (builtins.attrNames value) + else + [ path ]; + in + walk [ ]; + + # The discovery program of a top's root and the output values it must + # print. The values are those the dumps record, and their paths are + # the paths of the outputs the root names in `argsNeeded`. + caseOf = + name: rootState: outputValues: + let + root = + (nixops4Entry (topWith { + inherit rootState; + })).finalize + { + args.resourceProviderSystem = system; + argsSource = _: "the discovery check"; + }; + expected = { + inherit outputValues; + resourceProviderSystem = system; + }; + in + assert nixops4ArgPathsOf root.argsNeeded.outputValues == valuePathsOf outputValues; + { + inherit name; + program = root.discoverArgs { inherit system; }; + expected = pkgs.writeText "${name}.json" (builtins.toJSON expected); + }; + + cases = [ + # One state resource, inside a composite. + (caseOf "nested" false { group.inner.stdout = "nested-state-value"; }) + # Two state resources, one at the root and one inside a composite. + (caseOf "both" true { + a.stdout = "alpha-42"; + group.inner.stdout = "nested-state-value"; + }) + ]; +in +pkgs.runCommand "nixops4-discovery" { nativeBuildInputs = [ pkgs.jq ]; } ( + pkgs.lib.concatMapStrings (case: '' + ${pkgs.lib.getExe case.program} > ${case.name}.json + jq -S . ${case.expected} > ${case.name}.expected + jq -S . ${case.name}.json > ${case.name}.printed + diff -u ${case.name}.expected ${case.name}.printed + '') cases + + '' + touch $out + '' +) diff --git a/tests/nixops4-discovery/group-state.json b/tests/nixops4-discovery/group-state.json new file mode 100644 index 00000000..cef6af7c --- /dev/null +++ b/tests/nixops4-discovery/group-state.json @@ -0,0 +1,23 @@ +{ + "_type": "nixopsState", + "resources": {}, + "deployments": { + "group": { + "resources": { + "inner": { + "type": "exec", + "input_properties": { + "args": ["-c", "printf %s nested-state-value"], + "executable": "/bin/sh", + "once": false, + "stdin": null + }, + "output_properties": { + "stdout": "nested-state-value" + } + } + }, + "deployments": {} + } + } +} diff --git a/tests/nixops4-discovery/root-state.json b/tests/nixops4-discovery/root-state.json new file mode 100644 index 00000000..1b960268 --- /dev/null +++ b/tests/nixops4-discovery/root-state.json @@ -0,0 +1,18 @@ +{ + "_type": "nixopsState", + "resources": { + "a": { + "type": "exec", + "input_properties": { + "args": ["-c", "printf %s alpha-42"], + "executable": "/bin/sh", + "once": false, + "stdin": null + }, + "output_properties": { + "stdout": "alpha-42" + } + } + }, + "deployments": {} +} diff --git a/tests/pinned-world/default.nix b/tests/pinned-world/default.nix index 1d08f005..8a86a637 100644 --- a/tests/pinned-world/default.nix +++ b/tests/pinned-world/default.nix @@ -553,6 +553,65 @@ let in walk [ ]; + # A native composite whose module is `module`, under `lib`. + nixops4Composite = + lib: module: + lib.caisson.caisson-nixops4-composite.mkConfiguration { + configModule = module; + }; + + # A native resource of the `exec` type of NixOps4's `local` provider, + # with the further module `module`. + nixops4Exec = + lib: module: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + configModule = + { providers, ... }: + { + imports = [ module ]; + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + }; + + # A native resource of the `state_file` type of NixOps4's `local` + # provider, a resource that holds the state of others. + nixops4StateFile = + lib: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + configModule = + { providers, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = providers.local.state_file; + inputs.name = "state.json"; + }; + }; + + # A composite module that records its `members` argument as `seen`. + nixops4SeesMembers = + { lib, members, ... }: + { + options.seen = lib.mkOption { type = lib.types.raw; }; + config.seen = members; + }; + + # A top whose root composite `nixops4` has the module `module`. + nixops4RootTopWith = + module: + hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.nixops4 = nixops4Composite lib module; + }; + }; + + # Whether forcing a value fails with an error a probe can catch. + nixops4Fails = value: !(builtins.tryEval (builtins.deepSeq value true)).success; + # A deferred configuration of a tool that is not NixOps4, declared # beside a NixOps4 deployment: it takes the cacheable arg `who` and # greets it. Its entry is published as `greetings.`, a function @@ -2769,6 +2828,503 @@ let in needed.outputValues.group.inner.stdout.cacheable && !needed.outputValues.address.stdout.cacheable; + # Native composites nest to any depth. Each composite finalizes its + # members with their part of its args: its member path, the + # `members` argument of each composite, the `argsSource` of each + # member, the args the root names and the export all follow the + # nesting. The `members` argument holds, for a composite member, the + # outputs of its members by name. The root collects the state + # resources of the whole tree, each once, and only the root gives a + # discovery program. + nixops4NestedCompositesFollowTheNesting = + let + top = nixops4RootTopWith ( + { lib, members, ... }: + { + imports = [ nixops4SeesMembers ]; + caisson.caisson-nixops4-resource.configurations.state = nixops4StateFile lib; + caisson.caisson-nixops4-resource.configurations.m = nixops4Exec lib { state = members.state; }; + caisson.caisson-nixops4-composite.configurations.a = nixops4Composite lib ( + { lib, members, ... }: + { + imports = [ nixops4SeesMembers ]; + caisson.caisson-nixops4-resource.configurations.s = nixops4StateFile lib; + caisson.caisson-nixops4-resource.configurations.w = nixops4Exec lib { state = members.s; }; + caisson.caisson-nixops4-resource.configurations.w2 = nixops4Exec lib { state = members.s; }; + caisson.caisson-nixops4-composite.configurations.b = nixops4Composite lib ( + { lib, ... }: + { + imports = [ nixops4SeesMembers ]; + caisson.caisson-nixops4-resource.configurations.r = nixops4Exec lib { + cacheableOutputs = [ "stdout" ]; + }; + } + ); + } + ); + } + ); + root = nixops4RootManifest top; + export = nixops4Export top.nixops4; + a = root.children.caisson-nixops4-composite.a; + b = a.children.caisson-nixops4-composite.b; + r = b.children.caisson-nixops4-resource.r; + discover = root.discoverArgs { system = "x86_64-linux"; }; + in + [ + root.outputs.path + a.outputs.path + b.outputs.path + r.value.config.absoluteMemberPath + ] == [ + [ ] + [ "a" ] + [ + "a" + "b" + ] + [ + "a" + "b" + "r" + ] + ] + && root.value.config.seen.a.absoluteMemberPath == [ "a" ] + && root.value.config.seen.a.outputs.b.r.stdout == "stub" + && root.value.config.seen.a.outputs.w.stdout == "stub" + && a.value.config.seen.b.outputs.r.stdout == "stub" + && + b.value.config.seen.r == { + absoluteMemberPath = [ + "a" + "b" + "r" + ]; + outputs.stdout = "stub"; + } + && a.argsSource [ ] == "the composite at the root" + && b.argsSource [ ] == "the composite at a" + && r.argsSource [ ] == "the composite at a.b" + && + nixops4ArgPathsOf root.argsNeeded == [ + [ + "outputValues" + "a" + "b" + "r" + "stdout" + ] + [ + "outputValues" + "a" + "w" + "stdout" + ] + [ + "outputValues" + "a" + "w2" + "stdout" + ] + [ + "outputValues" + "m" + "stdout" + ] + [ "resourceProviderSystem" ] + ] + && root.argsNeeded.outputValues.a.b.r.stdout.cacheable + && !root.argsNeeded.outputValues.a.w.stdout.cacheable + && export.members.a.members.b.members.r.resource.type == "exec" + && + export.members.a.members.w.resource.state == [ + "a" + "s" + ] + && + root.outputs.stateResources == [ + [ + "a" + "s" + ] + [ "state" ] + ] + && a.discoverArgs { system = "x86_64-linux"; } == null + && pkgs.lib.hasInfix "states=(a.s state)" discover.text; + + # A NixOS configuration declared beneath a resource inside a nested + # composite is deployed by that resource, as one beneath a resource + # of the root is. + nixops4MachineInsideANestedComposite = + let + top = nixops4RootTopWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.site = nixops4Composite lib ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.web = + lib.caisson.caisson-nixops4-resource.mkConfiguration + { + configModule = + { lib, ... }: + { + ssh.hostPublicKey = nixops4HostKey; + caisson.nixos.configurations.web = lib.caisson.nixos.mkConfiguration { + configModule = minimalNixosBase; + }; + }; + }; + } + ); + } + ); + args = (nixops4Export top.nixops4).members.site.members.web.resource.inputs.args; + web = + (nixops4RootManifest top) + .children.caisson-nixops4-composite.site.children.caisson-nixops4-resource.web; + machine = web.children.system.x86_64-linux.children.nixos.web; + in + web.value.config.absoluteMemberPath == [ + "site" + "web" + ] + && builtins.elemAt args 2 == "root@web" + && builtins.elemAt args 3 == "${machine.value.config.system.build.toplevel}"; + + # Which members a composite has, and whether each is a composite or + # a resource, may not depend on its members or on their outputs: + # the composite that reads them there is refused with an error that + # names it, in place of an infinite recursion. Reading `members` + # anywhere else stays allowed. tests/deferred-errors reads the text. + nixops4MembersDecidedFromOutputsAreRefused = + let + argsNeededOf = top: (nixops4RootManifest top).argsNeeded; + native = nixops4RootTopWith ( + { lib, members, ... }: + { + imports = [ nixops4AddressRoot ]; + caisson.caisson-nixops4-resource.configurations = lib.optionalAttrs ( + members.address.outputs.stdout == "more" + ) { extra = nixops4Exec lib { }; }; + } + ); + nested = nixops4RootTopWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.a = nixops4Composite lib ( + { lib, members, ... }: + { + caisson.caisson-nixops4-resource.configurations = lib.optionalAttrs (members ? x) { + x = nixops4Exec lib { }; + }; + } + ); + } + ); + upstreamNames = nixops4UpstreamTopWith { } [ + ( + { providers, config, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members = { + address = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + } + // (if config.members.address.outputs.stdout == "more" then { extra = { }; } else { }); + } + ) + ]; + upstreamKind = nixops4UpstreamTopWith { } [ + ( + { providers, members, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members.address = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + members.dynamic = + if members.address.outputs.stdout == "resource" then + { type = providers.local.exec; } + else + { members.inner.type = providers.local.exec; }; + } + ) + ]; + in + nixops4Fails (argsNeededOf native) + && nixops4Fails (argsNeededOf nested) + && nixops4Fails (argsNeededOf upstreamNames) + && nixops4Fails (argsNeededOf upstreamKind) + && !(nixops4Fails (argsNeededOf (nixops4TopWith { root = nixops4AddressRoot; } { }))); + + # A resource whose `outputsSkeleton` reads its own outputs is + # refused with an error that names it, in place of an infinite + # recursion. tests/deferred-errors reads the text. + nixops4OutputsNamedFromOwnOutputsAreRefused = + let + top = nixops4RootTopWith ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.selfish = nixops4Exec lib ( + { config, ... }: + { + outputsSkeleton = lib.mkForce ( + { + stdout = { }; + } + // lib.optionalAttrs (config.outputs.stdout == "more") { extra = { }; } + ); + } + ); + } + ); + in + nixops4Fails (nixops4RootManifest top).argsNeeded; + + # A name declared both as a composite and as a resource is refused, + # since NixOps4 holds one member under a name. + nixops4OneMemberUnderAName = + nixops4Fails + (nixops4RootManifest ( + nixops4RootTopWith ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.x = nixops4Exec lib { }; + caisson.caisson-nixops4-composite.configurations.x = nixops4Composite lib { }; + } + ) + )).argsNeeded; + + # A flake publishes one root as `nixops4`: two roots, declared side + # by side or in two groups, are refused, and so is a root declared + # under another name. tests/deferred-errors reads the text. + nixops4OneRootPerFlake = + let + topWith = + module: + hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = module; + }; + rootIn = + lib: + lib.caisson.structural.mkConfiguration { + configModule.caisson.caisson-nixops4-composite.configurations.nixops4 = + nixops4Composite lib nixops4AddressRoot; + }; + in + nixops4Fails + (topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.nixops4 = nixops4Composite lib nixops4AddressRoot; + caisson.nixops4.configurations.nixops4 = lib.caisson.nixops4.mkConfiguration { }; + } + )).nixops4._type + && + nixops4Fails + (topWith ( + { lib, ... }: + { + caisson.structural.configurations.one = rootIn lib; + caisson.structural.configurations.two = rootIn lib; + } + )).nixops4._type + && + nixops4Fails + (topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.deploy = nixops4Composite lib nixops4AddressRoot; + } + )).nixops4._type; + + # A root declared inside a NixOS configuration, which is evaluated + # per system, is published as `nixops4` where one system is in + # force. Its entry stands beneath that system. + nixops4RootInsideAMachine = + let + top = hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.nixos.configurations.host = lib.caisson.nixos.mkConfiguration { + configModule = { + imports = [ minimalNixosBase ]; + caisson.caisson-nixops4-composite.configurations.nixops4 = nixops4Composite lib nixops4AddressRoot; + }; + }; + }; + }; + entry = + top.caisson.manifest.children.system.x86_64-linux.children.nixos.host.children.caisson-nixops4-composite.nixops4; + in + core.isDeferred entry + && entry.parent.type == "nixos" + && (nixops4Export top.nixops4).members.address.resource.type == "exec"; + + # A deferred configuration whose parent does not finalize it stays + # an entry in the children of that parent, for a tool to finalize: + # a resource declared at the top, and a composite declared beneath + # a resource. Neither has a composite above it, so each is at the + # root of its member paths, and neither is published. + nixops4EntriesAParentDoesNotFinalize = + let + top = hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.lone = nixops4Exec lib { }; + caisson.caisson-nixops4-composite.configurations.nixops4 = nixops4Composite lib ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.holder = nixops4Exec lib { + caisson.caisson-nixops4-composite.configurations.inner = nixops4Composite lib nixops4AddressRoot; + }; + } + ); + }; + }; + finalizeWith = + entry: outputValues: + entry.finalize { + args = { + resourceProviderSystem = "x86_64-linux"; + inherit outputValues; + }; + argsSource = _: "the probe"; + }; + lone = top.caisson.manifest.children.caisson-nixops4-resource.lone; + inner = + (nixops4RootManifest top) + .children.caisson-nixops4-resource.holder.children.caisson-nixops4-composite.inner; + loneFinalized = finalizeWith lone { stdout = "out"; }; + innerFinalized = finalizeWith inner { address.stdout = "out"; }; + in + core.isDeferred lone + && core.isDeferred inner + && loneFinalized.outputs.path == [ ] + && loneFinalized.outputs.export.resource.type == "exec" + && innerFinalized.outputs.path == [ ] + && innerFinalized.outputs.export.members.address.resource.type == "exec" + && + builtins.attrNames top == [ + "caisson" + "libOverlays" + "modules" + "nixops4" + "pkgOverlays" + ]; + + # `getProviders` finalizes the root without outputs, so a provider + # of the root configured from an output fails, as it does upstream. + # It builds the providers of the root alone, not those a composite + # beneath declares, as upstream collects them. + nixops4ProvidersOfTheRootAlone = + let + providersOf = + top: + builtins.fromJSON ( + builtins.elemAt (pkgs.lib.splitString "\n" + (top.nixops4.getProviders { system = "x86_64-linux"; }).text + ) 1 + ); + fromOutputs = nixops4RootTopWith ( + { members, ... }: + { + imports = [ nixops4AddressRoot ]; + providers.reader = { + executable = "/bin/reader"; + args = [ members.address.outputs.stdout ]; + resourceTypes = { }; + }; + } + ); + nested = nixops4RootTopWith ( + { lib, ... }: + { + providers.root = { + executable = "/bin/root"; + resourceTypes = { }; + }; + caisson.caisson-nixops4-composite.configurations.sub = nixops4Composite lib { + providers.nested = { + executable = "/bin/nested"; + resourceTypes = { }; + }; + }; + } + ); + in + nixops4Fails (providersOf fromOutputs) + && + providersOf nested == { + root = { + executable = "/bin/root"; + args = [ ]; + }; + }; + + # Upstream composites nest to any depth, as native ones do: the + # member paths, the args the root names, the allow-list by member + # path, and a member reading the output of a member two levels + # down. + nixops4UpstreamCompositesFollowTheNesting = + let + top = nixops4UpstreamTopWith { cacheableOutputs."g.h.r" = [ "stdout" ]; } [ + ( + { providers, members, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members.g.members.h.members.r = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + members.g.members.reader = { + type = providers.local.exec; + inputs.executable = "/bin/echo"; + inputs.args = [ members.g.members.h.members.r.outputs.stdout ]; + }; + } + ) + ]; + root = nixops4RootManifest top; + r = + root.children.caisson-nixops4-composite.g.children.caisson-nixops4-composite.h.children.caisson-nixops4-resource.r; + in + nixops4ArgPathsOf root.argsNeeded == [ + [ + "outputValues" + "g" + "h" + "r" + "stdout" + ] + [ + "outputValues" + "g" + "reader" + "stdout" + ] + [ "resourceProviderSystem" ] + ] + && root.argsNeeded.outputValues.g.h.r.stdout.cacheable + && !root.argsNeeded.outputValues.g.reader.stdout.cacheable + && + r.value.config.absoluteMemberPath == [ + "g" + "h" + "r" + ] + && r.crossChildLevel.name == "h" + && (nixops4Export top.nixops4).members.g.members.reader.resource.inputs.args == [ "stub" ]; + # A node of the upstream-compatible integration builds the same # system as colmena builds from the same hive. colmenaUpstreamHiveBuildsWhatColmenaBuilds = @@ -5122,6 +5678,15 @@ let in { inherit results failures; + # What the checks that build from the same world read. + fixtures = { + inherit + hiveLib + pkgs + nixops4ArgPathsOf + nixops4Entry + ; + }; ok = failures == [ ]; summary = if failures == [ ] then From 18d50bbf2b214fccaec00a55f7295c3d682542e8 Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Tue, 27 Oct 2026 02:08:29 -0700 Subject: [PATCH 8/9] test(nixops4): pin caisson-core with named errors for args that are not a set caisson-core b9de3d0 names the path of args given as a non-attrset where argsNeeded declares a set. The deferred-errors check reads that message for output values given as a string. Co-Authored-By: Claude Opus 5.5 --- flake.lock | 6 +++--- tests/deferred-errors/cases.nix | 15 +++++++++++++++ tests/dependencies/flake.lock | 6 +++--- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/flake.lock b/flake.lock index fbd55d5f..079f1d51 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793078671, - "narHash": "sha256-0ZawlBBGyrn9LhAhyQh5yLePUrFZBftEOdQCkpckCGY=", + "lastModified": 1793091679, + "narHash": "sha256-OzlG2Nhl/kU2UUPq+ARH/GKUHmOqAxQu5Ug/+aT1ECc=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "229370819fa078ae1f0bc5ed6b5d08312958f6ee", + "rev": "b9de3d04fa9caaa8ab5ace9830741a4e65bce7f4", "type": "github" }, "original": { diff --git a/tests/deferred-errors/cases.nix b/tests/deferred-errors/cases.nix index 334735b8..8407355a 100644 --- a/tests/deferred-errors/cases.nix +++ b/tests/deferred-errors/cases.nix @@ -117,6 +117,21 @@ in message = "caisson-deferred: needs outputValues.source.stdout"; }; + # Output values given as a string where the root declares an attrset + # of members name the path they were given at, rather than failing + # with Nix's own `expected a set`. + outputValuesNotASet = { + value = + ((topWith readerModule).caisson.manifest.children.caisson-nixops4-composite.nixops4.finalize { + args = { + resourceProviderSystem = system; + outputValues = "none"; + }; + argsSource = _: "the check"; + }).outputs.export; + message = "caisson-deferred: the args given at outputValues are a string"; + }; + # The member path of an output inside nested composites holds each # composite. missingNestedOutput = { diff --git a/tests/dependencies/flake.lock b/tests/dependencies/flake.lock index 78760aab..94ea1651 100644 --- a/tests/dependencies/flake.lock +++ b/tests/dependencies/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793078671, - "narHash": "sha256-0ZawlBBGyrn9LhAhyQh5yLePUrFZBftEOdQCkpckCGY=", + "lastModified": 1793091679, + "narHash": "sha256-OzlG2Nhl/kU2UUPq+ARH/GKUHmOqAxQu5Ug/+aT1ECc=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "229370819fa078ae1f0bc5ed6b5d08312958f6ee", + "rev": "b9de3d04fa9caaa8ab5ace9830741a4e65bce7f4", "type": "github" }, "original": { From f6411636983c5a7c87972b1c597ce3c53cb6d3aa Mon Sep 17 00:00:00 2001 From: Chris Hodapp Date: Wed, 28 Oct 2026 20:21:35 -0700 Subject: [PATCH 9/9] maint: pin caisson-core main Co-Authored-By: Claude Opus 5.5 --- flake.lock | 4 ++-- tests/dependencies/flake.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/flake.lock b/flake.lock index 079f1d51..21d0d659 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793091679, + "lastModified": 1793243975, "narHash": "sha256-OzlG2Nhl/kU2UUPq+ARH/GKUHmOqAxQu5Ug/+aT1ECc=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "b9de3d04fa9caaa8ab5ace9830741a4e65bce7f4", + "rev": "6868bca2682e96d56714bc9c9adcf0048fbe132f", "type": "github" }, "original": { diff --git a/tests/dependencies/flake.lock b/tests/dependencies/flake.lock index 94ea1651..ff5cf93a 100644 --- a/tests/dependencies/flake.lock +++ b/tests/dependencies/flake.lock @@ -2,11 +2,11 @@ "nodes": { "caisson-core": { "locked": { - "lastModified": 1793091679, + "lastModified": 1793243975, "narHash": "sha256-OzlG2Nhl/kU2UUPq+ARH/GKUHmOqAxQu5Ug/+aT1ECc=", "owner": "nix-caisson", "repo": "caisson-core", - "rev": "b9de3d04fa9caaa8ab5ace9830741a4e65bce7f4", + "rev": "6868bca2682e96d56714bc9c9adcf0048fbe132f", "type": "github" }, "original": {