diff --git a/README.md b/README.md index 86ff1c0c..9f221b55 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,9 @@ source by. | `facter` | none | a `facter.json` as nixos-facter writes it, with the NixOS configurations that run on it declared inside it; a machine gets it as `hardware.facter.reportPath` | | `caisson-microvm` | `microvm` | microvm.nix guests declared inside the NixOS host that runs them, each guest a NixOS configuration declared inside it that sets its own `microvm.*` options and gets microvm.nix's guest module by default; the host selects caisson's `microvm-host` NixOS module, which defines microvm.nix's `microvm.vms.` from them; the microvm integration we recommend for new configurations | | `microvm` | `microvm` | a microvm.nix guest written as microvm.nix reads it, a NixOS configuration that sets its `microvm.*` options itself, declared inside the NixOS host that runs it; for porting an existing guest | +| `caisson-nixops4-composite` | `nixops4` | a composite of a NixOps4 deployment, evaluated when NixOps4 deploys, with its members, composites and resources, declared beneath it; it declares providers and reads the outputs of its members; the composite declared under the name `nixops4` where no composite holds it is the root, published as the `nixops4` output the NixOps4 command reads; the NixOps4 integration we recommend for new configurations | +| `caisson-nixops4-resource` | `nixops4` | a resource of a NixOps4 deployment, evaluated when NixOps4 deploys, holding the caisson configurations it deploys; a NixOS configuration declared inside it is deployed over SSH as nixops4-nixos deploys one | +| `nixops4` | `nixops4` | the root modules of a NixOps4 deployment, written as NixOps4 users write them for `mkRoot`, each component evaluated by caisson and each nixops4-nixos machine a caisson NixOS configuration; the configuration named `nixops4` is published as the `nixops4` output; for porting an existing deployment | | `treefmt` | `treefmt-nix` | the formatters of a tree, as treefmt-nix modules; the configuration named `formatter` is published as the `formatter.` that `nix fmt` runs | | `nvf` | `nvf` | a Neovim written as nvf modules, published as `nvfConfigurations..`, in the shape `neovimConfiguration` returns, kept in a separate Nix profile that the machine, home-manager configuration or hjem configuration it is declared inside activates | | `nixvim` | `nixvim` | a Neovim written as nixvim modules, published as `nixvimConfigurations..`, kept in a profile as an nvf configuration is | diff --git a/configs/flake/caisson/partitions/checks.nix b/configs/flake/caisson/partitions/checks.nix index 18e2b26f..f824d884 100644 --- a/configs/flake/caisson/partitions/checks.nix +++ b/configs/flake/caisson/partitions/checks.nix @@ -234,6 +234,14 @@ caisson = self; }; }; + # The discovery program of a NixOps4 root, built and run + # against recorded state dumps, with a stand-in for + # NixOps4 that prints them (tests/nixops4-discovery). + nixops4-discovery = import (self.outPath + "/tests/nixops4-discovery") { + inputs = inputs // { + caisson = self; + }; + }; # A machine booted in a VM that activates the # home-manager configurations, the hjem configurations # and the profiles declared inside it, and a standalone @@ -264,6 +272,26 @@ --impure --extra-experimental-features nix-command touch $out ''; + # The errors of deferred configurations and the NixOps4 + # integrations, read as text by the real evaluator, as + # a tool that finalizes a deferred configuration reads + # them. It runs as the argument-error check does, on + # the same library. + deferred-errors = + pkgs.runCommand "deferred-errors" + { + nativeBuildInputs = [ pkgs.nix ]; + } + '' + export HOME="$(realpath .)" + unset NIX_STORE + export NIX_STORE_DIR=${builtins.storeDir} + export NIX_REMOTE="$HOME/storedata" + bash ${self.outPath}/tests/deferred-errors/check.sh ${lib.escapeShellArg argumentErrorsLib} \ + ${inputs.nixops4.outPath} \ + --impure --extra-experimental-features nix-command + touch $out + ''; minimal-consumer-all-outputs = builtins.seq minimalConsumerOutputs.flakeModule ( builtins.seq minimalConsumerOutputs.lib ( pkgs.runCommand "minimal-consumer-all-outputs" { } "touch $out" diff --git a/docs/development/repository-structure.md b/docs/development/repository-structure.md index 551724d7..8972f35c 100644 --- a/docs/development/repository-structure.md +++ b/docs/development/repository-structure.md @@ -28,7 +28,9 @@ The layer users reach for: the integrations (`structural`, `flake-parts`, `nixpkgs`, `nixos`, `nixos-minimal`, `home-manager`, `home-manager-minimal`, `darwin`, `hjem`, `caisson-colmena`, `colmena`, `caisson-disko`, `disko`, `caisson-facter`, `facter`, -`caisson-microvm`, `microvm`, `terranix`, `system-manager`, `treefmt`, +`caisson-microvm`, `microvm`, `caisson-nixops4-composite`, +`caisson-nixops4-resource`, `nixops4`, +`terranix`, `system-manager`, `treefmt`, `nvf`, `nixvim`, `topology`, `kubenix`, `nixidy`), each a library overlay contributing its `lib.caisson` namespace, registering its module diff --git a/docs/development/testing-architecture.md b/docs/development/testing-architecture.md index 869cfa87..51e97107 100644 --- a/docs/development/testing-architecture.md +++ b/docs/development/testing-architecture.md @@ -320,4 +320,6 @@ The checks partition merges outputs from all test and example flakes into the | `system-manager-vm-` | `checks.nix` + `tests/system-manager/vm.nix` | A system-manager machine, activated through sequences of changes in a VM of Ubuntu 22.04 or Debian 12 booted from the cloud image of the distribution, applies its profiles, its home-manager configurations and its hjem configuration, and a home-manager configuration activated by hand for another version warns of the drift | | `containerTests.x86_64-linux.system-manager-` (not a check) | `checks.nix` + `tests/system-manager/container.nix` | The same sequences in a container of each distribution of `tests/system-manager/container-distros.nix`; the container needs `auto-allocate-uids`, so the `system-manager-container` job of the check workflow builds each by name | | `argument-errors` | `checks.nix` + `tests/argument-errors/` | The argument error of every entry point comes from Nix itself, raised at the call site with no frame of caisson | +| `deferred-errors` | `checks.nix` + `tests/deferred-errors/` | The errors of deferred configurations and the NixOps4 integrations carry their text: `caisson-deferred: needs ` with the member path from the root, and the errors that refuse members or outputs named from outputs, a name declared as composite and resource, and a second root | +| `nixops4-discovery` | `checks.nix` + `tests/nixops4-discovery/` | The discovery program of a NixOps4 root builds, with one state resource and with two, and prints, from recorded state dumps, the output values at the paths the root names in `argsNeeded` | | `eval-weight` | `checks.nix` + `tests/eval-weight/` | Framework evaluation cost held to committed ceilings ([guide](https://nix-caisson.github.io/docs/eval-weight.html)) | diff --git a/docs/development/testing.md b/docs/development/testing.md index 6465a6b0..0b34f69b 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -76,6 +76,25 @@ plumbing works. 'let f = builtins.getFlake "path:'"$PWD"'"; in f.lib' --impure ``` +## Deferred Configuration Errors (Message Text) + +- **Scope:** The text of the errors of deferred configurations and the + NixOps4 integrations. A tool tells an arg it lacks from a failure by + `caisson-deferred: needs `, and `builtins.tryEval` in the + pinned-world suite sees only that an evaluation failed. +- **Location:** `tests/deferred-errors/check.sh` and its cases in + `tests/deferred-errors/cases.nix`. +- **Mechanism:** the script runs `nix eval` on each case and looks for + the line of the message the case names. The `deferred-errors` check + runs it in the sandbox over the library of the `argument-errors` + check and the NixOps4 source. +- **Execution against the flake's library, from a checkout:** + ```bash + tests/deferred-errors/check.sh \ + 'let f = builtins.getFlake "path:'"$PWD"'"; in f.lib' \ + --impure + ``` + ## Integration Testing (Test Flakes) - **Scope:** Module composition, end-to-end evaluation, and build success. diff --git a/lib-overlays/caisson-nixops4-composite/default.nix b/lib-overlays/caisson-nixops4-composite/default.nix new file mode 100644 index 00000000..85f33c65 --- /dev/null +++ b/lib-overlays/caisson-nixops4-composite/default.nix @@ -0,0 +1,438 @@ +# SPDX-License-Identifier: MIT +# +# The caisson-nixops4-composite integration: a composite of a NixOps4 +# deployment, a component that holds further components as its +# members. The root component NixOps4 deploys is the outermost +# composite, declared under the name `nixops4`, for the output it +# fills. +# +# A composite is a deferred configuration (caisson-core's +# `mkDeferredConfiguration`): it is evaluated when NixOps4 deploys, +# with its args, and NixOps4 evaluates it again each time the outputs +# of a resource become known. Its args are `outputValues`, the outputs +# of its members by member name as NixOps4 passes them, and +# `resourceProviderSystem`, the system NixOps4 runs providers on. Its +# members are the caisson-nixops4-composite and +# caisson-nixops4-resource configurations declared beneath it. It +# finalizes each member with that member's part of its args, so its +# manifest holds the members' manifests as its children. +# +# A composite declares providers, as NixOps4's components do, with +# NixOps4's own `providers` option, and its module reads its members' +# outputs through the `members` module argument: for each member, its +# `outputs` and its `absoluteMemberPath`, both from the args and the +# names the members are declared under. A member's config is not +# reachable through it. A composite has no `type` and no resource +# options: a composite that sets them is refused where it sets them, as +# an undeclared option. +# +# What a composite publishes to NixOps4 is its export: its members' +# exports, by name, as NixOps4's `_export` of a composite holds them. +# The root composite is published as the `nixops4` output NixOps4's +# command reads: `_type`, `rootFunction` and `getProviders`, as +# NixOps4's `mkRoot` returns them. +{ + mkLibOverlay, + ... +}: +{ + + imports = [ + # The library of nixpkgs, which this overlay calls through the + # composed library, imported by key so it is composed wherever + # this is. + ((mkLibOverlay ../nixpkgs-lib) // { key = "nixpkgs-lib"; }) + # What an integration is written from, imported by key so it is + # composed wherever this integration is. + ((mkLibOverlay ../integrations) // { key = "integrations"; }) + # The resource integration, whose helpers this integration builds + # on. + ((mkLibOverlay ../caisson-nixops4-resource) // { key = "caisson-nixops4-resource"; }) + ]; + + overlay = + final: prev: + let + resources = final.caisson.caisson-nixops4-resource; + inherit (resources) showPath; + context = "lib.caisson.caisson-nixops4-composite"; + compositeType = "caisson-nixops4-composite"; + resourceType = "caisson-nixops4-resource"; + + # The members of a composite at `path`, by name, from its + # evaluation: the composites and resources declared beneath it, + # each the entry caisson-core holds for a deferred child. Their + # names are readable in the childless view too, and the entries + # only in the full evaluation. A name declared as both is refused, + # since NixOps4 holds one member under a name. + memberEntriesOf = + path: config: + let + composites = config.caisson.${compositeType}.configurations; + members = config.caisson.${resourceType}.configurations; + both = builtins.attrNames (builtins.intersectAttrs composites members); + in + if both == [ ] then + composites // members + else + throw '' + ${context}: the composite at ${showPath path} declares + ${builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") both)} both as a composite and as a + resource; a member is one or the other. + ''; + + # The `members` module argument of a native composite: for each + # member, its outputs from the composite's args and its member + # path from its name. + nativeMembersArg = + { + args, + path, + entries, + ... + }: + builtins.mapAttrs (member: _: { + outputs = args.outputValues.${member}; + absoluteMemberPath = path ++ [ member ]; + }) entries; + + # The root of a deployment, as NixOps4's `mkRoot` returns it, over + # the entry of the root composite. `rootFunction` finalizes the + # root with the args NixOps4 passes and returns its export. + # `getProviders` finalizes it with the deploying system alone and + # builds the providers the root declares, as the `getProviders` + # of NixOps4 does; reading an output there fails, as it does + # upstream. Providers declared beneath the root are not + # collected, as upstream collects none of them either. + rootOf = entry: { + _type = "nixops4Component"; + rootFunction = + { + outputValues, + resourceProviderSystem, + ... + }: + (entry.finalize { + args = { inherit outputValues resourceProviderSystem; }; + argsSource = _: "NixOps4"; + }).outputs.export; + getProviders = + { system }: + let + finalized = entry.finalize { + args.resourceProviderSystem = system; + argsSource = _: "getProviders for ${system}"; + }; + serializable = builtins.mapAttrs (_: provider: { + inherit (provider) executable args; + }) finalized.outputs.providerConfigs; + in + (resources.pkgsFor finalized system).writeText "nixops-root-providers" '' + Store path contents subject to change + ${builtins.toJSON serializable} + ''; + }; + + # How a composite is published. A composite whose parent is not a + # composite is a root, published as `nixops4`, an output that + # holds a single value. A composite beneath another is a member + # and is not published. + published = { + exportsTo = { + attrset = "nixops4"; + single = true; + only = + entry: + !(builtins.elem (entry.parent.type or null) [ + compositeType + "nixops4" + ]); + value = rootOf; + }; + }; + + # A composite configuration of the module class `class`, evaluated + # with the modules `baseModules` returns for the NixOps4 source, + # the member path and the finalized members, and the `members` + # module argument `membersArg` returns. The native integration + # evaluates its class with NixOps4's provider declarations and + # `nativeMembersArg`; the upstream-compatible one builds on this + # with NixOps4's class and its own modules. + mkCompositeConfiguration = + { + class, + type ? compositeType, + baseModules, + membersArg ? nativeMembersArg, + }: + settings: + let + # The evaluator's call with the args `args`, on the lib of the + # view being evaluated: the base modules, the framework + # modules of the class, the selection and the configuration's + # module, with the composition's pinned sources as the + # `inputs` special argument and the deploying system as + # `resourceProviderSystem`, as NixOps4 passes it. + # `ecosystemArgs` is merged over the call last. + evaluate = + args: + { lib, manifest }: + let + selection = lib.caisson.integrations; + path = resources.absoluteMemberPathOf manifest; + src = resources.ecosystemSrcOf { + explicit = settings.ecosystemSrc or null; + inherit manifest; + }; + registry = lib.caisson.modules.${class} or { }; + frameworkModules = selection.frameworkModules class registry; + moduleImports = selection.moduleImportsOf class { inherit lib manifest; } settings; + name = manifest.name or null; + configModule = selection.configModuleOf { + inherit lib name class; + configModule = settings.configModule or null; + }; + # The members, by name. The names come from `naming`, and + # each entry comes from the evaluation, so the composite's + # module may read `members` wherever it does not decide + # which members there are. + declared = memberEntriesOf path evaluated.config; + entries = builtins.mapAttrs (member: _: declared.${member}) (memberEntriesOf path naming.config); + # The members, each finalized once with its part of the + # composite's args, from the full evaluation. The + # childless view finalizes none. + finalized = + if manifest.childless then + { } + else + builtins.mapAttrs ( + member: entry: + entry.finalize { + args = { + outputValues = args.outputValues.${member}; + inherit (args) resourceProviderSystem; + }; + argsSource = _: "the composite at ${showPath path}"; + } + ) entries; + ofType = memberType: final.filterAttrs (_: member: member.type == memberType) finalized; + # The modules of an evaluation in which the `members` + # module argument is what `members` returns for the + # evaluation's config, and the base modules receive + # `finalized`. + modulesWith = + { members, finalized }: + baseModules { inherit src path finalized; } + ++ [ + (selection.caissonModule { + key = "caisson.${type}.members"; + file = "caisson-${type}:members"; + module = + { config, ... }: + { + _module.args.members = members config; + }; + }) + ] + ++ frameworkModules + ++ moduleImports registry + ++ (if configModule == null then [ ] else [ configModule ]); + evaluationOf = + modules: + lib.evalModules ( + { + inherit class modules; + specialArgs = { + inherit lib; + inputs = manifest.sources; + inherit (args) resourceProviderSystem; + } + // (if (settings.specialArgs or null) != null then settings.specialArgs else { }); + } + // (if (settings.ecosystemArgs or null) != null then settings.ecosystemArgs else { }) + ); + moduleList = modulesWith { + inherit finalized; + members = + config: + membersArg { + inherit + args + path + entries + config + ; + }; + }; + evaluated = evaluationOf moduleList; + # The evaluation that decides which members there are. The + # members and their outputs fail there when they are read, + # with an error that names the composite, since the names + # of the members must be known before any member is + # finalized. + naming = evaluationOf (modulesWith { + members = _: membersUnknown; + finalized = membersUnknown; + }); + membersUnknown = throw '' + ${context}: the composite at ${showPath path} reads its members, or their + outputs, where it decides which members it has. Which members a composite + has, and whether each is a composite or a resource, must not depend on its + members or on their outputs. + ''; + # The member paths of the resources in the tree of this + # composite that hold state, each once. + stateResources = final.unique ( + builtins.concatMap ( + member: + if member.type == compositeType then + member.outputs.stateResources + else + let + state = member.outputs.export.resource.state; + in + if state == null then [ ] else [ state ] + ) (builtins.attrValues finalized) + ); + in + { + value = evaluated; + ecosystems.nixops4 = resources.ecosystemAccountOf { + explicit = settings.ecosystemSrc or null; + inherit manifest; + }; + modules = moduleList; + # The finalized members, by integration and then name, so + # the manifest holds their manifests. + children = final.filterAttrs (_: members: members != { }) { + ${compositeType} = ofType compositeType; + ${resourceType} = ofType resourceType; + }; + outputs = { + inherit path stateResources; + # What NixOps4 reads of the composite. + export.members = builtins.mapAttrs (_: member: member.outputs.export) finalized; + # The providers it declares, which `getProviders` builds + # for the root. + providerConfigs = evaluated.config.providers; + }; + # The deploying system, and the outputs of the members, by + # member name, as each member names them. + argsNeeded = { + resourceProviderSystem = resources.systemArg; + outputValues = builtins.mapAttrs (_: member: member.argsNeeded.outputValues) finalized; + }; + } + // ( + # The root gives a program that reads the outputs NixOps4 + # recorded, when a resource of the deployment holds state. + if path == [ ] then + { + discoverArgs = + { system }: + if stateResources == [ ] then + null + else + resources.discoverArgsOf { + inherit manifest src system; + stateResources = builtins.map (builtins.concatStringsSep ".") stateResources; + }; + } + else + { } + ); + in + final.caisson-core.mkDeferredConfiguration { + inherit type class; + record = published // { + exportsToPosition = final.caisson.integrations.positionOf ( + builtins.unsafeGetAttrPos "exportsTo" published + ); + }; + configuration = + { + name, + parent, + args, + }: + final.caisson.integrations.mkModuleConfiguration { + selection = { + inherit class; + args = settings; + }; + inherit type; + defaultPkgs = settings.defaultPkgs or null; + evaluate = evaluate args; + } { inherit name parent; }; + }; + + configuration = mkCompositeConfiguration { + class = compositeType; + baseModules = + { src, ... }: + [ + # NixOps4's `providers` option, whose resource types the + # composite writes into its members' `type`. + (resources.componentModule src "provider-declarations") + ]; + }; + + integration = final.caisson.integrations.mkIntegration { + name = compositeType; + class = compositeType; + # What these return is a deferred configuration, a function of + # `{ name, parent, args }`. Declared under the name `nixops4` + # where no composite holds it, it is the root, finalized when + # NixOps4 calls `rootFunction`. Declared under + # `caisson.caisson-nixops4-composite.configurations.` of + # a composite, it is a member, finalized by that composite. + mkConfiguration = + { + # The composite's module: its providers and members. When + # absent, the configuration registered under the + # configuration's name, if any. + configModule ? null, + # The package set the deployment runs on, for the composite + # and its members: a function that receives the package + # sets available, by package config name, and returns the + # set to run on. + defaultPkgs ? null, + # The NixOps4 flake; resolved from the composition's + # declarations when absent. + ecosystemSrc ? null, + # The selection over the caisson-nixops4-composite class of + # the registry, replacing the default of the class. + moduleImports ? null, + # A selection added to that selection, whichever it is. + extraModuleImports ? null, + # Extra module arguments, merged over those the framework + # supplies. + specialArgs ? null, + }@settings: + configuration settings; + # The same arguments and `ecosystemArgs`, the evaluator's + # arguments merged over the composed call last. + mkConfigurationWithEcosystemArgs = + { + configModule ? null, + defaultPkgs ? null, + ecosystemSrc ? null, + moduleImports ? null, + extraModuleImports ? null, + specialArgs ? null, + ecosystemArgs ? null, + }@settings: + configuration settings; + # What the upstream-compatible integration builds on. + extra = { + inherit mkCompositeConfiguration; + }; + }; + in + { + caisson = (prev.caisson or { }) // { + caisson-nixops4-composite = integration.namespace; + }; + }; + +} diff --git a/lib-overlays/caisson-nixops4-resource/default.nix b/lib-overlays/caisson-nixops4-resource/default.nix new file mode 100644 index 00000000..c49c5c36 --- /dev/null +++ b/lib-overlays/caisson-nixops4-resource/default.nix @@ -0,0 +1,457 @@ +# SPDX-License-Identifier: MIT +# +# The caisson-nixops4-resource integration: a resource of a NixOps4 +# deployment, one real thing a provider creates, updates and deletes. +# A resource is a member of a caisson-nixops4-composite configuration, +# declared beneath it, and a deferred configuration like the composite +# (caisson-core's `mkDeferredConfiguration`). Its args are +# `outputValues`, its outputs as NixOps4 passes them, and +# `resourceProviderSystem`, the system NixOps4 runs providers on. The +# composite finalizes it with its part of the composite's args. +# +# A resource is evaluated with NixOps4's own resource options +# (`nix/component/resource.nix`), its `type` option and its +# `providers` option. `type` is a resource type of a provider, which +# the composite usually writes from its `providers`. The resource +# reads its own outputs as `config.outputs`. It reads the outputs of +# the other members of its composite through what the composite writes +# into it, from the composite's `members` module argument. It holds no +# NixOps4 members: `members` is not an option here, and setting it is +# refused where it is set. +# +# A resource may hold caisson configurations beneath it, such as a +# NixOS configuration with its home-manager configurations. NixOps4 +# does not see them. A module of the class turns them into the +# resource's inputs: the NixOS adapter, registered as `nixos` in the +# class and selected by its default module, deploys the NixOS +# configuration declared beneath a resource. +# +# The resource declares which of its outputs a tool that evaluates the +# deployment ahead of time may keep in a cache (`cacheableOutputs`); +# NixOps4 marks no output as secret, so none is cacheable unless +# listed. +# +# The helpers the composite integration, the upstream-compatible +# integration and the NixOS adapters build on are published under +# `extra`, as `lib.caisson.caisson-nixops4-resource.`. +{ + mkLibOverlay, + ... +}: +{ + + imports = [ + # The library of nixpkgs, which this overlay calls through the + # composed library, imported by key so it is composed wherever + # this is. + ((mkLibOverlay ../nixpkgs-lib) // { key = "nixpkgs-lib"; }) + # What an integration is written from, imported by key so it is + # composed wherever this integration is. + ((mkLibOverlay ../integrations) // { key = "integrations"; }) + ]; + + overlay = + final: prev: + let + resourceType = "caisson-nixops4-resource"; + context = "lib.caisson.${resourceType}"; + + resolveEcosystemSrc = final.caisson.integrations.resolveEcosystemSrc ecosystem; + ecosystemAccountOf = final.caisson.integrations.ecosystemAccountOf ecosystem; + ecosystem = { + name = "nixops4"; + context = "caisson.${resourceType}"; + }; + + # NixOps4's component modules are files of its source, which the + # composite and resource integrations evaluate in caisson's + # evaluations; its `local` provider is a module its flake + # publishes. + assertNixops4EcosystemSrc = + ecosystemSrc: + if + ecosystemSrc ? modules + && ecosystemSrc.modules ? nixops4Provider + && builtins.pathExists "${ecosystemSrc}/nix/component/resource.nix" + && builtins.pathExists "${ecosystemSrc}/nix/component/provider-values.nix" + && builtins.pathExists "${ecosystemSrc}/nix/component/provider-declarations.nix" + then + ecosystemSrc + else + throw "${context} requires `ecosystemSrc.modules.nixops4Provider` and NixOps4's component modules under `nix/component` (a NixOps4 flake)."; + + # The NixOps4 source of a configuration: `explicit` when given, + # else what the composition declares or pins under the name + # `nixops4`, as recorded on `manifest`. + ecosystemSrcOf = + { explicit, manifest }: + assertNixops4EcosystemSrc (resolveEcosystemSrc { + inherit explicit manifest; + }); + + # A component module of NixOps4, by its file name under + # `nix/component`. + componentModule = src: name: "${src}/nix/component/${name}.nix"; + + # The package set of a deploying system: the set the selection in + # force at `manifest` returns, for that system. + pkgsFor = + manifest: system: + let + what = "the deployment for ${system}"; + pkgSets = final.caisson.integrations.pkgSetsForSystem { inherit context what; } manifest system; + in + final.caisson.integrations.pkgSetOf { inherit context what; } manifest pkgSets; + + # The member path of a component, as NixOps4 writes it. + showPath = path: if path == [ ] then "the root" else builtins.concatStringsSep "." path; + + # The integrations whose configurations are composites: the + # native composite and the root of the upstream-compatible + # integration. + compositeTypes = [ + "caisson-nixops4-composite" + "nixops4" + ]; + + # The member path of the component `manifest` describes, from the + # root, as NixOps4's `absoluteMemberPath` holds it: empty for a + # component whose parent is not a composite, which is the root, + # else the path of the parent, the childless manifest of the + # composite, and the name of the component. It is built from the + # names the components are declared under. + absoluteMemberPathOf = + manifest: + if builtins.elem (manifest.parent.type or null) compositeTypes then + manifest.parent.outputs.path ++ [ manifest.name ] + else + [ ]; + + # The NixOS configuration a resource named `resource` deploys, from + # the NixOS configurations declared beneath it (`declared`): the + # only one, evaluated for one system. `context` names the module + # that asks, for the errors. + machineOf = + { + context, + resource, + declared, + }: + let + names = builtins.attrNames declared; + only = declared.${builtins.head names}; + evaluations = builtins.attrValues only; + in + if builtins.length names != 1 then + throw '' + ${context}: the resource `${resource}` declares the NixOS configurations + ${builtins.concatStringsSep ", " (builtins.map (n: "`${n}`") names)}, and a resource deploys one. + Declare each as the NixOS configuration of a resource of its own. + '' + else if (only._type or null) == "caisson-manifest" then + only + else if builtins.length evaluations == 1 then + builtins.head evaluations + else + throw '' + ${context}: the NixOS configuration `${builtins.head names}` of the resource + `${resource}` is evaluated for ${builtins.concatStringsSep ", " (builtins.attrNames only)}, + and a resource deploys one evaluation. Declare it where a single system is in force. + ''; + + # The discovery program of a root composite for `system`, the + # system of the tool that runs it, on the package set selected at + # `manifest` for that system: it prints the args NixOps4 recorded. + # It runs `nixops4 state dump` for each of `stateResources`, the + # member paths of the resources that hold state joined with `.`, + # in the working directory, which NixOps4 finds the flake from. It + # passes its own arguments to NixOps4, such as `--override-input`, + # and prints `{ outputValues, resourceProviderSystem }`. A dump + # holds the resources that record their outputs in that state, at + # their member paths from the root: the resources of the root + # under `resources`, and those of a composite `a` under + # `deployments.a`, in the same shape. Only resources with a state + # have recorded outputs. + discoverArgsOf = + { + manifest, + src, + stateResources, + system, + }: + let + pkgs = pkgsFor manifest system; + in + pkgs.writeShellApplication { + name = "caisson-nixops4-discover"; + runtimeInputs = [ + src.packages.${system}.nixops4 + pkgs.jq + ]; + text = '' + merged='{}' + states=(${final.escapeShellArgs stateResources}) + for state in "''${states[@]}"; do + dump=$(nixops4 "$@" state dump "$state") + merged=$(jq -cn --argjson merged "$merged" --argjson dump "$dump" ' + def outputValues: + ((.resources // {}) | map_values(.output_properties)) + + ((.deployments // {}) | map_values(outputValues)); + $merged * ($dump | outputValues) + ') + done + jq -cn --argjson merged "$merged" '{ outputValues: $merged, resourceProviderSystem: "${system}" }' + ''; + }; + + # The arg `resourceProviderSystem`, which every component takes. + # A system name is no secret, so it is cacheable. + systemArg = final.caisson-core.mkArg { + cacheable = true; + type = "string"; + description = "The system NixOps4 runs providers and the programs of resources on."; + }; + + # The options of the class beside the options NixOps4 declares. + resourceOptions = + { lib, ... }: + { + options.cacheableOutputs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + The names of the outputs of this resource that a tool which + evaluates the deployment ahead of time may keep in a cache + only its user can read. NixOps4 marks no output as secret, + so an output is not cacheable unless it is listed here. + ''; + }; + }; + + # A resource configuration of the module class `class`, evaluated + # with the modules `baseModules` returns for the NixOps4 source + # and the member path, and the settings the constructor received. + # The native integration evaluates its class with NixOps4's + # resource, provider-value and provider-declaration modules; the + # upstream-compatible one builds on this with NixOps4's class and + # its own modules. + mkResourceConfiguration = + { + class, + type ? resourceType, + baseModules, + }: + settings: + let + # The evaluator's call with the args `args`, on the lib of the + # view being evaluated: the base modules, the outputs and the + # member path, the framework modules of the class, the + # selection and the configuration's module, with the + # composition's pinned sources as the `inputs` special + # argument and the deploying system as + # `resourceProviderSystem`, as NixOps4 passes it. + # `ecosystemArgs` is merged over the call last. + evaluate = + args: + { lib, manifest }: + let + selection = lib.caisson.integrations; + path = absoluteMemberPathOf manifest; + src = ecosystemSrcOf { + explicit = settings.ecosystemSrc or null; + inherit manifest; + }; + registry = lib.caisson.modules.${class} or { }; + frameworkModules = selection.frameworkModules class registry; + moduleImports = selection.moduleImportsOf class { inherit lib manifest; } settings; + name = manifest.name or null; + configModule = selection.configModuleOf { + inherit lib name class; + configModule = settings.configModule or null; + }; + # The modules of an evaluation in which the resource's + # outputs are `outputs`, with what NixOps4 gives a + # resource beside them, its member path. + modulesWith = + outputs: + baseModules { inherit src path; } + ++ [ + (selection.caissonModule { + key = "caisson.${type}.outputs"; + file = "caisson-${type}:outputs"; + module = { + config.outputs = _: { config = outputs; }; + config.absoluteMemberPath = path; + }; + }) + ] + ++ frameworkModules + ++ moduleImports registry + ++ (if configModule == null then [ ] else [ configModule ]); + evaluationOf = + modules: + lib.evalModules ( + { + inherit class modules; + specialArgs = { + inherit lib; + inputs = manifest.sources; + inherit (args) resourceProviderSystem; + } + // (if (settings.specialArgs or null) != null then settings.specialArgs else { }); + } + // (if (settings.ecosystemArgs or null) != null then settings.ecosystemArgs else { }) + ); + # The outputs are the output values NixOps4 passes. An + # output NixOps4 has not passed fails when it is read, + # naming itself. + moduleList = modulesWith args.outputValues; + evaluated = evaluationOf moduleList; + # The evaluation that names the outputs, for `argsNeeded`. + # Reading an output fails there with an error that names + # the resource, since NixOps4 needs the names of the + # outputs before it has their values. + naming = evaluationOf ( + modulesWith (throw '' + ${context}: the resource at ${showPath path} reads its own outputs in its + `outputsSkeleton`, which names its outputs. NixOps4 needs the names of the + outputs of a resource before it has their values. + '') + ); + cacheableOutputs = evaluated.config.cacheableOutputs or [ ]; + in + { + value = evaluated; + ecosystems.nixops4 = ecosystemAccountOf { + explicit = settings.ecosystemSrc or null; + inherit manifest; + }; + modules = moduleList; + outputs = { + inherit path cacheableOutputs; + # What NixOps4 reads of the resource. + export.resource = evaluated.config._resourceForNixOps; + providerConfigs = evaluated.config.providers; + }; + # The deploying system and the resource's own outputs, + # which its type names. An output is cacheable when the + # resource lists it. + argsNeeded = { + resourceProviderSystem = systemArg; + outputValues = builtins.mapAttrs ( + output: _: + final.caisson-core.mkArg { + cacheable = builtins.elem output cacheableOutputs; + description = "An output of the resource at ${showPath path}."; + } + ) naming.config.outputsSkeleton; + }; + }; + in + final.caisson-core.mkDeferredConfiguration { + inherit type class; + configuration = + { + name, + parent, + args, + }: + final.caisson.integrations.mkModuleConfiguration { + selection = { + inherit class; + args = settings; + }; + inherit type; + defaultPkgs = settings.defaultPkgs or null; + evaluate = evaluate args; + } { inherit name parent; }; + }; + + configuration = mkResourceConfiguration { + class = resourceType; + baseModules = + { src, ... }: + [ + (componentModule src "resource") + (componentModule src "provider-values") + (componentModule src "provider-declarations") + (final.caisson.integrations.caissonModule { + key = "caisson.${resourceType}.resourceOptions"; + file = "caisson-${resourceType}:resourceOptions"; + module = resourceOptions; + }) + ]; + }; + + integration = final.caisson.integrations.mkIntegration { + name = resourceType; + class = resourceType; + # What these return is a deferred configuration, a function of + # `{ name, parent, args }`: the composite that declares it under + # `caisson.caisson-nixops4-resource.configurations.` + # finalizes it with its part of the composite's args. + mkConfiguration = + { + # The resource's module: its `type`, `inputs`, the options + # of the modules it selects, and the caisson configurations + # declared beneath it. When absent, the configuration + # registered under the configuration's name, if any. + configModule ? null, + # The package set the configurations beneath run on: a + # function that receives the package sets available, by + # package config name, and returns the set to run on. + defaultPkgs ? null, + # The NixOps4 flake; resolved from the composition's + # declarations when absent. + ecosystemSrc ? null, + # The selection over the caisson-nixops4-resource class of + # the registry, replacing the default of the class. + moduleImports ? null, + # A selection added to that selection, whichever it is. + extraModuleImports ? null, + # Extra module arguments, merged over those the framework + # supplies. + specialArgs ? null, + }@settings: + configuration settings; + # The same arguments and `ecosystemArgs`, the evaluator's + # arguments merged over the composed call last. + mkConfigurationWithEcosystemArgs = + { + configModule ? null, + defaultPkgs ? null, + ecosystemSrc ? null, + moduleImports ? null, + extraModuleImports ? null, + specialArgs ? null, + ecosystemArgs ? null, + }@settings: + configuration settings; + # What the composite integration, the upstream-compatible + # integration and the NixOS adapters build on. + extra = { + inherit + mkResourceConfiguration + resourceOptions + ecosystemSrcOf + componentModule + pkgsFor + showPath + absoluteMemberPathOf + machineOf + discoverArgsOf + systemArg + ; + # Where the NixOps4 source of a configuration came from, as a + # manifest records it under `ecosystems.nixops4`. + ecosystemAccountOf = { explicit, manifest }: ecosystemAccountOf { inherit explicit manifest; }; + }; + }; + in + { + caisson = (prev.caisson or { }) // { + caisson-nixops4-resource = integration.namespace; + }; + }; + +} diff --git a/lib-overlays/integrations/default.nix b/lib-overlays/integrations/default.nix index 4b42767d..05fb4ec9 100644 --- a/lib-overlays/integrations/default.nix +++ b/lib-overlays/integrations/default.nix @@ -722,7 +722,20 @@ # The configurations its modules declare under # `caisson..configurations`, of any integration, # are its children, and `caisson.exports`, which carries what - # they pass up, is its `exports` output. + # they pass up, is its `exports` output. An integration whose + # evaluation finalizes its children itself returns them as + # `children`, by integration and then name, and they are its + # children in place of those values. A deferred configuration + # (`caisson-core.mkDeferredConfiguration`) finalizes the deferred + # configurations declared beneath it this way, with their part of + # its args. + # + # `evaluate` may also return `argsNeeded`, the args a deferred + # configuration takes, and `discoverArgs`, the function of + # `{ system }` that returns the program which finds them, or + # null. The manifest carries both, as caisson-core describes + # them. A configuration whose `evaluate` returns neither takes no + # args. # # `defaultPkgs` is the selection of a package set the configuration # was constructed with, null when it was given none. It is @@ -739,7 +752,7 @@ # returning `value`, the name to publish it under, and # optionally `description`, a sentence saying what it did. It may # carry `single`, for an output that holds the one configuration - # named `default`, and `bySystem`, for an output keyed by the + # named for that output, and `bySystem`, for an output keyed by the # system of each evaluation first (`packages..`). It # may carry `only`, a function of the manifest that says whether # the configuration is published there at all; a configuration it @@ -823,7 +836,8 @@ outputs = (evaluated.outputs or { }) // { exports = config.caisson.exports; }; - children = view.lib.caisson.integrations.childrenOf { inherit options config; }; + children = + evaluated.children or (view.lib.caisson.integrations.childrenOf { inherit options config; }); # What the modules give the configurations beneath # (`caisson.forChildren`). forChildren = { @@ -889,7 +903,12 @@ } # How the evaluation selected its modules. // (if selection == null then { } else { moduleImports = selectionAccountOf selection view; }); - }; + } + # The args of a deferred configuration and the program that + # finds them, where the integration's `evaluate` returns + # them. + // (if evaluated ? argsNeeded then { inherit (evaluated) argsNeeded; } else { }) + // (if evaluated ? discoverArgs then { inherit (evaluated) discoverArgs; } else { }); }; # The body of both entry points, from the caisson arguments a @@ -1568,7 +1587,10 @@ # that is not published under a name (structural, flake-parts). # An integration may give one export or a list of them; an export # with `only` applies to the configurations whose manifest it - # holds true for. + # holds true for. The entry of a deferred configuration + # (`caisson-core.deferChild`) is published in the same way, from + # the `exportsTo` its `record` holds: `value` and `only` receive + # the entry, which holds its name, type, class and parent. exportsOf = manifest: let @@ -1703,10 +1725,7 @@ name = system; }; beneath = - # A deferred child passes nothing up at flake time. - if final.caisson-core.isDeferred finalized then - [ ] - else if isManifest finalized then + if isManifest finalized || final.caisson-core.isDeferred finalized then [ { prefix = [ (segmentOf name) ]; @@ -1734,16 +1753,24 @@ manifest, }: builtins.map (entryOf level declared prefix manifest) (exportsOf manifest) - ++ builtins.map ( - entry: - entry - // { - path = prefix ++ entry.path; - namePath = declared ++ entry.namePath; - notes = entry.notes ++ level.notes; - code = entry.code ++ level.code; - } - ) (manifest.outputs.exports.configurations or [ ]) + # The entry of a deferred child is published as its + # `exportsTo` says, and nothing beneath it is evaluated at + # flake time, so it passes nothing else up. + ++ ( + if final.caisson-core.isDeferred manifest then + [ ] + else + builtins.map ( + entry: + entry + // { + path = prefix ++ entry.path; + namePath = declared ++ entry.namePath; + notes = entry.notes ++ level.notes; + code = entry.code ++ level.code; + } + ) (manifest.outputs.exports.configurations or [ ]) + ) ) beneath ) (builtins.attrNames selected.${integration}) ) (builtins.attrNames selected); @@ -1935,6 +1962,7 @@ ecosystemUnit inEcosystem entriesOf + positionOf configurationsModuleFor integrationsIn integrationsDeclaringIn diff --git a/lib-overlays/nixops4/default.nix b/lib-overlays/nixops4/default.nix new file mode 100644 index 00000000..f1d6a3d9 --- /dev/null +++ b/lib-overlays/nixops4/default.nix @@ -0,0 +1,358 @@ +# SPDX-License-Identifier: MIT +# +# The nixops4 integration, the upstream-compatible one: it takes the +# modules of a root component as NixOps4 users write them for NixOps4's +# `mkRoot`, of NixOps4's class `nixops4Component`, and caisson evaluates +# every component of the deployment. The native integrations, +# caisson-nixops4-composite and caisson-nixops4-resource, take +# caisson's form instead. +# +# A nixops4 configuration is the root composite of the deployment, a +# deferred configuration as a native composite is. Declared under the +# name `nixops4`, for the output it fills, it is published as the +# native root is. Every component is evaluated with NixOps4's class +# name, so a module that sets `_class = "nixops4Component"` loads, and +# with NixOps4's resource, type and providers options. +# +# Upstream declares `members` and the resource options on every +# component and decides which kind a component is when its export is +# read: a component whose `resourceType` is defined is a resource. A +# composite here declares `members` again, as `lazyAttrsOf +# deferredModule`, and decides the same way for each member, with a +# probe evaluation of the member's modules over NixOps4's own base +# modules. A member becomes a caisson-nixops4-resource or a +# caisson-nixops4-composite configuration declared beneath the +# composite, evaluated with the same modules. A member that sets `type` +# and also has members is refused; upstream drops the members. +# +# The `members` module argument and `config.members` are what upstream +# gives: in a composite, the configs of its members, which a composite +# reads to wire one member to another (`members.db.outputs.address`) +# and to name the resource that holds the state of another +# (`state = members.state`); in a resource, nothing. A composite reads +# its members' configs through its full evaluation, so every composite +# sets `unsafeCrossChildReferences` for the class of its members. +# +# A member that imports nixops4-nixos's module gets its `nixos.module` +# evaluated as a caisson NixOS configuration beneath it, rather than by +# the `nixosSystem` call of nixops4-nixos: see the module `nixos` of the +# `nixops4Component` class. +{ + mkLibOverlay, + ... +}: +{ + + imports = [ + # The library of nixpkgs, which this overlay calls through the + # composed library, imported by key so it is composed wherever + # this is. + ((mkLibOverlay ../nixpkgs-lib) // { key = "nixpkgs-lib"; }) + # What an integration is written from, imported by key so it is + # composed wherever this integration is. + ((mkLibOverlay ../integrations) // { key = "integrations"; }) + # The native integrations, whose evaluators this one builds on. + ((mkLibOverlay ../caisson-nixops4-composite) // { key = "caisson-nixops4-composite"; }) + ((mkLibOverlay ../caisson-nixops4-resource) // { key = "caisson-nixops4-resource"; }) + ]; + + overlay = + final: prev: + let + composites = final.caisson.caisson-nixops4-composite; + resources = final.caisson.caisson-nixops4-resource; + context = "lib.caisson.nixops4"; + upstreamClass = "nixops4Component"; + compositeType = "caisson-nixops4-composite"; + resourceType = "caisson-nixops4-resource"; + + # The options every upstream component has beside `members`: + # NixOps4's resource, type and providers options. + componentBase = src: [ + (resources.componentModule src "resource") + (resources.componentModule src "provider-values") + (resources.componentModule src "provider-declarations") + ]; + + # The modules of a component, as the configuration's module. + modulesModule = + modules: + final.caisson.integrations.caissonModule { + key = "caisson.nixops4.component"; + file = "caisson-nixops4:component"; + module = { + imports = modules; + }; + }; + + # Which caisson integration the member `name` of the composite at + # `path` becomes, by upstream's test: a resource when its + # `resourceType` is defined, a composite otherwise. The probe + # evaluates the member's modules over NixOps4's base modules, + # with the special arguments the member receives. + kindOf = + { + lib, + src, + specialArgs, + path, + }: + name: module: + let + probe = lib.evalModules { + class = upstreamClass; + inherit specialArgs; + modules = [ + "${src}/nix/component/base-modules.nix" + module + ]; + }; + isResource = probe.options.resourceType.isDefined; + hasMembers = builtins.attrNames probe.config.members != [ ]; + in + if isResource && hasMembers then + throw '' + ${context}: the member ${resources.showPath (path ++ [ name ])} sets `type`, which makes it a + resource, and declares members, which a resource does not hold. + NixOps4 would drop them; declare them in a composite instead. + '' + else if isResource then + resourceType + else + compositeType; + + # The module of an upstream composite at `path` that declares + # `members` and turns the modules defined there into the + # configurations declared beneath it. The members are computed + # from the definitions of `members`, since its value is the + # members' configs: in the full evaluation those of the members + # the composite finalized (`finalized`), and in the childless view + # those of the full evaluation, which the composite reads through + # `unsafeCrossChildReferences`. + membersModule = + ctx: + { + src, + path, + finalized, + }: + final.caisson.integrations.caissonModule { + key = "caisson.nixops4.memberComponents"; + file = "caisson-nixops4:memberComponents"; + module = + { + lib, + options, + inputs, + resourceProviderSystem, + ... + }: + let + manifest = lib.caisson-core.evalManifest; + declared = options.members.type.merge options.members.loc options.members.definitionsWithLocations; + kinds = builtins.mapAttrs (kindOf { + inherit lib src path; + specialArgs = { + inherit inputs; + } + // ctx.specialArgs + // { + inherit lib resourceProviderSystem; + }; + }) declared; + ofKind = kind: lib.filterAttrs (name: _: kinds.${name} == kind) declared; + members = + if manifest.childless then + let + full = manifest.crossChildFull; + in + (full.children.${compositeType} or { }) // (full.children.${resourceType} or { }) + else + finalized; + in + { + options.members = lib.mkOption { + type = lib.types.lazyAttrsOf lib.types.deferredModule; + default = { }; + apply = _: builtins.mapAttrs (_: member: member.value.config) members; + description = '' + The member components, by name, as NixOps4 declares them: a + member that sets `type` is a resource, any other a composite. + Read back, each is the config of that member. + ''; + }; + # A composite reads its members' configs, through its full + # evaluation. + config.caisson.forDirectChildren.unsafeCrossChildReferences.${upstreamClass} = true; + config.caisson.${compositeType}.configurations = builtins.mapAttrs ( + _: module: compositeOf ctx [ module ] + ) (ofKind compositeType); + config.caisson.${resourceType}.configurations = builtins.mapAttrs ( + _: module: resourceOf ctx [ module ] + ) (ofKind resourceType); + }; + }; + + # The outputs of the resource at `path` that a tool may cache, from + # the allow-list of the nixops4 configuration. + cacheableModule = + ctx: path: + final.caisson.integrations.caissonModule { + key = "caisson.nixops4.cacheableOutputs"; + file = "caisson-nixops4:cacheableOutputs"; + module = + { lib, ... }: + { + options.cacheableOutputs = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = ctx.cacheableOutputs.${builtins.concatStringsSep "." path} or [ ]; + readOnly = true; + description = '' + The outputs of this resource a tool may cache, from + `cacheableOutputs` of the nixops4 configuration. + ''; + }; + }; + }; + + # A resource holds no members, so its `members` module argument is + # empty, as it is upstream. + resourceMembersModule = final.caisson.integrations.caissonModule { + key = "caisson.nixops4.resourceMembers"; + file = "caisson-nixops4:resourceMembers"; + module = { + _module.args.members = { }; + }; + }; + + # The arguments the native constructors receive for a component + # with `modules`. + componentArgs = ctx: modules: { + configModule = modulesModule modules; + inherit (ctx) ecosystemSrc specialArgs; + }; + + # An upstream composite of the integration `type`: NixOps4's class + # and options, `members` declared and its definitions declared + # beneath as configurations, and `config.members` as its `members` + # argument. `settings` are further arguments of the native + # constructor. + compositeWith = + { + type ? compositeType, + settings ? { }, + }: + ctx: modules: + composites.mkCompositeConfiguration { + inherit type; + class = upstreamClass; + baseModules = + { + src, + path, + finalized, + }: + componentBase src + ++ [ + (membersModule ctx { inherit src path finalized; }) + ]; + membersArg = { config, ... }: config.members; + } (componentArgs ctx modules // settings); + + compositeOf = compositeWith { }; + + # An upstream resource: NixOps4's class and options, and the + # outputs the configuration allows a tool to cache. + resourceOf = + ctx: modules: + resources.mkResourceConfiguration { + class = upstreamClass; + baseModules = + { src, path }: + componentBase src + ++ [ + (cacheableModule ctx path) + resourceMembersModule + ]; + } (componentArgs ctx modules); + + # The root composite made from `modules`, as `mkRoot` takes them, + # of the integration `nixops4`, with the allow-list, the package + # set and the selection of the configuration. + configuration = + settings: + compositeWith + { + type = "nixops4"; + settings = builtins.intersectAttrs { + defaultPkgs = null; + moduleImports = null; + extraModuleImports = null; + ecosystemArgs = null; + } settings; + } + { + ecosystemSrc = settings.ecosystemSrc or null; + specialArgs = if (settings.specialArgs or null) != null then settings.specialArgs else { }; + cacheableOutputs = settings.cacheableOutputs or { }; + } + (settings.modules or [ ]); + + integration = final.caisson.integrations.mkIntegration { + name = "nixops4"; + class = upstreamClass; + # What these return is a deferred configuration, a function of + # `{ name, parent, args }`: declared under the name `nixops4`, + # it is the root, finalized when NixOps4 calls `rootFunction`. + mkConfiguration = + { + # The modules of the root component, as NixOps4's `mkRoot` + # takes them. + modules ? [ ], + # Special arguments of every component, as `mkRoot` takes + # them for the root. + specialArgs ? null, + # The outputs a tool that evaluates the deployment ahead of + # time may cache, by member path, the names from the root + # joined with `.`: `{ "web.address" = [ "stdout" ]; }`. + # NixOps4 marks no output as secret, so none is cacheable + # unless listed. + cacheableOutputs ? { }, + # The package set the deployment runs on: a function that + # receives the package sets available, by package config + # name, and returns the set to run on. + defaultPkgs ? null, + # The NixOps4 flake; resolved from the composition's + # declarations when absent. + ecosystemSrc ? null, + # The selection over the nixops4Component class of the + # registry for the root, replacing the default of the + # class. + moduleImports ? null, + # A selection added to that selection, whichever it is. + extraModuleImports ? null, + }@settings: + configuration settings; + # The same arguments and `ecosystemArgs`, the evaluator's + # arguments merged over the composed call last. + mkConfigurationWithEcosystemArgs = + { + modules ? [ ], + specialArgs ? null, + cacheableOutputs ? { }, + defaultPkgs ? null, + ecosystemSrc ? null, + moduleImports ? null, + extraModuleImports ? null, + ecosystemArgs ? null, + }@settings: + configuration settings; + }; + in + { + caisson = (prev.caisson or { }) // { + nixops4 = integration.namespace; + }; + }; + +} diff --git a/modules/caisson-nixops4-composite/core b/modules/caisson-nixops4-composite/core new file mode 120000 index 00000000..3715354a --- /dev/null +++ b/modules/caisson-nixops4-composite/core @@ -0,0 +1 @@ +../generic/core \ No newline at end of file diff --git a/modules/caisson-nixops4-resource/core b/modules/caisson-nixops4-resource/core new file mode 120000 index 00000000..3715354a --- /dev/null +++ b/modules/caisson-nixops4-resource/core @@ -0,0 +1 @@ +../generic/core \ No newline at end of file diff --git a/modules/caisson-nixops4-resource/default/default.nix b/modules/caisson-nixops4-resource/default/default.nix new file mode 100644 index 00000000..5120e61f --- /dev/null +++ b/modules/caisson-nixops4-resource/default/default.nix @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: MIT +# +# The default module of the caisson-nixops4-resource class. +# +# It selects the NixOS adapter, registered as `nixos` in the class, +# which deploys the NixOS configuration declared beneath a resource. +# A resource that holds no NixOS configuration is not changed by it. +# +# NixOps4 runs `bin/apply` from the toplevel of the NixOS +# configuration it deploys. This module adds caisson's NixOS module +# `nixops4-apply`, which builds that script, to the modules that the +# nearest NixOS configurations beneath a resource import by default. +# It reaches a NixOS configuration inside a structural group or a disk +# layout beneath the resource. A NixOS configuration declared inside +# that one is not deployed by the resource and does not get it. +# +# A NixOS configuration that passes `moduleImports` selects its modules +# itself, and then it lists `nixops4-apply` itself, as +# `caisson/nixops4-apply` in a flake that lists caisson in `projects`. +# +# `closure-lib` is the library of caisson, where the adapter and +# `nixops4-apply` are registered under those names. +{ closure-lib, ... }: +{ ... }: +{ + imports = [ + # The option under `caisson.forNearestChildren` is declared by the + # core module of the class. + closure-lib.caisson.modules.caisson-nixops4-resource.core + closure-lib.caisson.modules.caisson-nixops4-resource.nixos + ]; + + caisson.forNearestChildren.extraDefaultModuleImports.nixos = _lib: [ + closure-lib.caisson.modules.nixos.nixops4-apply + ]; +} diff --git a/modules/caisson-nixops4-resource/nixos/default.nix b/modules/caisson-nixops4-resource/nixos/default.nix new file mode 100644 index 00000000..a7d79d70 --- /dev/null +++ b/modules/caisson-nixops4-resource/nixos/default.nix @@ -0,0 +1,141 @@ +# SPDX-License-Identifier: MIT +# +# The NixOS adapter of the caisson-nixops4-resource class: it deploys +# the NixOS configuration declared beneath a resource, as +# nixops4-nixos deploys the NixOS configuration of its resource. +# +# When the resource holds a NixOS configuration, the adapter makes the +# resource an `exec` resource of NixOps4's `local` provider, unless the +# resource sets `type` itself. Its inputs copy the toplevel of the +# configuration to the host with `nix copy` and run its +# `bin/apply switch` over SSH. The SSH settings are the options +# nixops4-nixos declares, under the same names: `ssh.user`, `ssh.host`, +# `ssh.hostPublicKey`, `ssh.opts`, `copy.substituteOnDestination` and +# `sudo.enable`. The host defaults to the name of the resource. The +# shell that runs the script is `bash` from the package set caisson +# selects for the resource, for the system NixOps4 runs providers on. +# +# A resource holds one NixOS configuration to deploy, evaluated for one +# system; the adapter refuses more. +# +# The default module of the class selects this module. A resource that +# selects its modules itself lists it as +# `caisson/caisson-nixops4-resource/nixos` in a flake that lists caisson +# in `projects`. +{ ... }: +{ + config, + lib, + providers, + resourceProviderSystem, + ... +}: +let + inherit (lib) mkOption types; + manifest = lib.caisson-core.evalManifest; + context = "caisson's NixOS adapter of the caisson-nixops4-resource class"; + declared = config.caisson.nixos.configurations; + # Whether the resource holds a NixOS configuration. The names are + # known in the childless view too, so a resource without one is left + # alone there and in its full evaluation. + holdsMachine = builtins.attrNames declared != [ ]; + # The NixOS configuration the resource deploys: the one declared + # beneath it, evaluated for one system. + nixos = + (lib.caisson.caisson-nixops4-resource.machineOf { + inherit context declared; + resource = manifest.name; + }).value; + target = (if config.ssh.user == null then "" else "${config.ssh.user}@") + config.ssh.host; + # The host key is the only key the SSH client accepts for the host, + # as nixops4-nixos has it. + knownHosts = builtins.toFile "known_hosts" '' + ${config.ssh.host} ${config.ssh.hostPublicKey} + ''; + src = lib.caisson.caisson-nixops4-resource.ecosystemSrcOf { + explicit = null; + inherit manifest; + }; + bash = lib.getExe (lib.caisson.caisson-nixops4-resource.pkgsFor manifest resourceProviderSystem) + .bash; +in +{ + _file = "caisson:caisson-nixops4-resource/nixos"; + options = { + nixos.configuration = mkOption { + type = types.raw; + readOnly = true; + description = '' + The NixOS configuration this resource deploys: the one declared + beneath it, as caisson evaluated it. + ''; + }; + ssh.user = mkOption { + type = types.nullOr types.str; + default = "root"; + description = '' + The user name of the SSH connection. When `null`, the SSH client + chooses it from its configuration. + ''; + }; + ssh.host = mkOption { + type = types.str; + default = manifest.name; + defaultText = lib.literalMD "the name of the resource"; + description = "The host the SSH client connects to."; + }; + ssh.hostPublicKey = mkOption { + type = types.str; + description = '' + The public key of the host. It is the only key the SSH client + accepts for the host. + ''; + }; + ssh.opts = mkOption { + type = types.str; + default = ""; + description = "Further options of the SSH client, as one shell word."; + }; + copy.substituteOnDestination = mkOption { + type = types.bool; + default = true; + description = '' + Whether the host tries its substituters for the paths it receives + before they are copied to it. + ''; + }; + sudo.enable = mkOption { + type = types.bool; + default = config.ssh.user != "root"; + defaultText = lib.literalMD ''`false` iff `ssh.user` is `"root"`''; + description = "Whether the configuration is applied with `sudo`."; + }; + }; + config = lib.mkIf holdsMachine { + # NixOps4's `local` provider, whose `exec` type deploys the + # configuration. A resource that declares `providers.local` or sets + # `type` replaces them. + providers.local = lib.mkDefault src.modules.nixops4Provider.local; + type = lib.mkDefault providers.local.exec; + nixos.configuration = nixos; + inputs = { + executable = bash; + args = [ + "-c" + # ConnectTimeout and ConnectionAttempts keep a host that is not + # up from hanging the deployment. Password and + # keyboard-interactive authentication are off because stdin is + # not connected. StrictHostKeyChecking with the configured key + # alone refuses a host the configuration does not name. + '' + set -euo pipefail + export NIX_SSHOPTS="-o ConnectTimeout=10 -o ConnectionAttempts=12 -o KbdInteractiveAuthentication=no -o PasswordAuthentication=no -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${knownHosts} "${lib.escapeShellArg config.ssh.opts} + nix copy --to "ssh-ng://$0" "$1" --no-check-sigs --extra-experimental-features nix-command${lib.optionalString config.copy.substituteOnDestination " --substitute-on-destination"} + ssh $NIX_SSHOPTS "$0" "${lib.optionalString config.sudo.enable "sudo "}$1/bin/apply switch" + '' + target + nixos.config.system.build.toplevel + ]; + }; + }; +} diff --git a/modules/generic/core/caisson/configurations.nix b/modules/generic/core/caisson/configurations.nix index c3be1377..0979f6a4 100644 --- a/modules/generic/core/caisson/configurations.nix +++ b/modules/generic/core/caisson/configurations.nix @@ -38,11 +38,18 @@ let ); # The manifests passed up: a configuration evaluated per system - # stands for each of its evaluations. + # stands for each of its evaluations. The entry of a deferred + # configuration is no manifest and exports no registries. exported = builtins.concatMap ( integration: builtins.concatMap ( - finalized: if isManifest finalized then [ finalized ] else builtins.attrValues finalized + finalized: + if isManifest finalized then + [ finalized ] + else if lib.caisson-core.isDeferred finalized then + [ ] + else + builtins.attrValues finalized ) (builtins.attrValues selected.${integration}) ) (builtins.attrNames selected); diff --git a/modules/generic/core/default.nix b/modules/generic/core/default.nix index 71c61c43..4eddecd4 100644 --- a/modules/generic/core/default.nix +++ b/modules/generic/core/default.nix @@ -33,6 +33,8 @@ "caisson-disko" "caisson-facter" "caisson-microvm" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "colmena" "darwin" "disko" @@ -43,6 +45,7 @@ "kubenix" "microvm" "nixidy" + "nixops4" "nixos" "nixpkgs" "nixvim" diff --git a/modules/nixops4Component/core b/modules/nixops4Component/core new file mode 120000 index 00000000..3715354a --- /dev/null +++ b/modules/nixops4Component/core @@ -0,0 +1 @@ +../generic/core \ No newline at end of file diff --git a/modules/nixops4Component/default/default.nix b/modules/nixops4Component/default/default.nix new file mode 100644 index 00000000..41c4953f --- /dev/null +++ b/modules/nixops4Component/default/default.nix @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: MIT +# +# The default module of the nixops4Component class, the class the +# nixops4 integration evaluates components with. +# +# It selects the module `nixos` of the class, which evaluates the +# `nixos.module` of a member that imports nixops4-nixos as a caisson +# NixOS configuration beneath the member. +# +# NixOps4 runs `bin/apply` from the toplevel of the NixOS +# configuration it deploys. This module adds caisson's NixOS module +# `nixops4-apply`, which builds that script, to the modules that the +# nearest NixOS configurations beneath a component import by default, +# as nixops4-nixos adds its `apply` module to the configuration it +# evaluates. +# +# `closure-lib` is the library of caisson, where these modules are +# registered under those names. +{ closure-lib, ... }: +{ ... }: +{ + imports = [ + # The option under `caisson.forNearestChildren` is declared by the + # core module of the class. + closure-lib.caisson.modules.nixops4Component.core + closure-lib.caisson.modules.nixops4Component.nixos + ]; + + caisson.forNearestChildren.extraDefaultModuleImports.nixos = _lib: [ + closure-lib.caisson.modules.nixos.nixops4-apply + ]; +} diff --git a/modules/nixops4Component/nixos/default.nix b/modules/nixops4Component/nixos/default.nix new file mode 100644 index 00000000..58168a1b --- /dev/null +++ b/modules/nixops4Component/nixos/default.nix @@ -0,0 +1,54 @@ +# SPDX-License-Identifier: MIT +# +# The module of the nixops4Component class, the class the nixops4 +# integration evaluates components with, for a member that imports the +# NixOS resource module of nixops4-nixos +# (`nixops4-nixos.modules.nixops4Component.nixos`). +# +# That module declares the options a NixOS member is written with +# (`nixpkgs`, `nixos.module`, `nixos.specialArgs`, `ssh.*`, +# `copy.substituteOnDestination`, `sudo.enable`), evaluates +# `nixos.module` with `config.nixpkgs.lib.nixosSystem`, and takes the +# shell its deploy script runs in from its own nixpkgs. In caisson the +# member's `nixos.module` is a caisson NixOS configuration declared +# beneath the member, under the member's name, with +# `nixos.specialArgs` as its special arguments. This module declares +# it, makes `nixpkgs.lib.nixosSystem` return it in place of evaluating +# one, and takes the shell from the package set caisson selects for the +# member, for the system NixOps4 runs providers on. A member that does +# not import nixops4-nixos is not changed. +# +# The default module of the class selects this module. +{ ... }: +{ + config, + lib, + options, + resourceProviderSystem, + ... +}: +let + manifest = lib.caisson-core.evalManifest; + usesNixos = options ? nixos && options.nixos ? module; + machine = lib.caisson.caisson-nixops4-resource.machineOf { + context = "caisson's `nixos` module of the nixops4Component class"; + resource = manifest.name; + declared = config.caisson.nixos.configurations; + }; + bash = lib.getExe (lib.caisson.caisson-nixops4-resource.pkgsFor manifest resourceProviderSystem) + .bash; +in +{ + _file = "caisson:nixops4Component/nixos"; + config = lib.optionalAttrs usesNixos { + caisson.nixos.configurations.${manifest.name} = lib.caisson.nixos.mkConfiguration { + configModule = config.nixos.module; + specialArgs = config.nixos.specialArgs; + }; + # nixops4-nixos builds `nixos.configuration` with this call. + nixpkgs = lib.mkForce { + lib.nixosSystem = _: machine.value; + }; + inputs.executable = lib.mkForce bash; + }; +} diff --git a/modules/nixos/nixops4-apply/default.nix b/modules/nixos/nixops4-apply/default.nix new file mode 100644 index 00000000..f309a593 --- /dev/null +++ b/modules/nixos/nixops4-apply/default.nix @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: MIT +# +# A NixOS module for a NixOS configuration that NixOps4 deploys. It +# imports the `apply` NixOS module of nixops4-nixos, which adds +# `bin/apply` to the toplevel: the script that sets the system profile +# and switches to the configuration. NixOps4 runs `bin/apply switch` on +# the host after it copies the toplevel there. Where nixpkgs declares +# `system.apply` itself, the module of nixops4-nixos adds nothing. +# +# The default module of the caisson-nixops4-resource class adds this +# module to the nearest NixOS configurations beneath a resource, the +# configurations the resource deploys. +# +# The nixops4-nixos source is the one the composition declares or pins +# under the name `nixops4-nixos`, read from the library of the +# evaluation. +{ ... }: +{ lib, ... }: +let + src = + let + found = lib.caisson-core.ecosystemSrc "nixops4-nixos"; + in + if found == null then + throw '' + caisson's `nixops4-apply` NixOS module needs the nixops4-nixos source. + Declare `defaultEcosystemSrc.nixops4-nixos` in the mkLib call, or pin a + source named `nixops4-nixos` in the `sources` passed to mkLib. + '' + else + found; +in +{ + _file = "caisson:nixops4-apply"; + imports = [ src.modules.nixos.apply ]; +} diff --git a/tests/deferred-errors/cases.nix b/tests/deferred-errors/cases.nix new file mode 100644 index 00000000..8407355a --- /dev/null +++ b/tests/deferred-errors/cases.nix @@ -0,0 +1,305 @@ +# SPDX-License-Identifier: MIT +# +# The cases of tests/deferred-errors/check.sh: deferred configurations +# that fail, each with a line of the message it must fail with. `lib` +# is caisson's composed library, and `nixops4` the NixOps4 source, +# whose component modules the NixOps4 integrations evaluate. The +# deployments use a provider written here, `probe`, whose resource +# type `exec` has the output `stdout`, so nothing of NixOps4's flake +# outputs is evaluated. +# +# Each case is `{ value; message; }`: forcing `value` must fail, and the +# error must contain `message` as written. +{ lib, nixops4 }: +let + system = "x86_64-linux"; + + # The NixOps4 source as the integrations check it: its component + # modules, and a module collection that holds `nixops4Provider`. + ecosystemSrc = { + outPath = nixops4; + modules.nixops4Provider = { }; + }; + + provider = + { lib, ... }: + { + executable = "/bin/probe"; + resourceTypes.exec = { + description = "A resource with the output `stdout`."; + requireState = false; + inputs.options.value = lib.mkOption { + type = lib.types.str; + default = ""; + }; + outputs.options.stdout = lib.mkOption { type = lib.types.str; }; + }; + }; + + # A native composite with the module `module`. + composite = + lib: module: + lib.caisson.caisson-nixops4-composite.mkConfiguration { + ecosystemSrc = _sources: ecosystemSrc; + configModule = module; + }; + + # A native resource of the type `exec` with the module `module`. + resource = + lib: module: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + ecosystemSrc = _sources: ecosystemSrc; + configModule = + { providers, ... }: + { + imports = [ module ]; + providers.probe = provider; + type = providers.probe.exec; + }; + }; + + # A top whose root composite `nixops4` has the module `module`. + topWith = + module: + lib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.nixops4 = composite lib module; + }; + }; + + # A top whose root is an upstream-compatible nixops4 configuration + # made from `modules`. + upstreamTopWith = + modules: + lib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.nixops4.configurations.nixops4 = lib.caisson.nixops4.mkConfiguration { + ecosystemSrc = _sources: ecosystemSrc; + inherit modules; + }; + }; + }; + + # The root of a top, finalized with the deploying system alone, as + # `getProviders` finalizes it: every output fails when it is read. + rootOf = + top: + let + children = top.caisson.manifest.children; + in + (children.caisson-nixops4-composite or children.nixops4).nixops4.finalize { + args.resourceProviderSystem = system; + argsSource = _: "the check"; + }; + + # A composite module whose resource `reader` reads the output + # `stdout` of its member `source`. + readerModule = + { lib, members, ... }: + { + caisson.caisson-nixops4-resource.configurations = { + source = resource lib { }; + reader = resource lib { inputs.value = members.source.outputs.stdout; }; + }; + }; +in +{ + # An output NixOps4 has not passed names its member path from the + # root. + missingOutput = { + value = (rootOf (topWith readerModule)).outputs.export; + message = "caisson-deferred: needs outputValues.source.stdout"; + }; + + # Output values given as a string where the root declares an attrset + # of members name the path they were given at, rather than failing + # with Nix's own `expected a set`. + outputValuesNotASet = { + value = + ((topWith readerModule).caisson.manifest.children.caisson-nixops4-composite.nixops4.finalize { + args = { + resourceProviderSystem = system; + outputValues = "none"; + }; + argsSource = _: "the check"; + }).outputs.export; + message = "caisson-deferred: the args given at outputValues are a string"; + }; + + # The member path of an output inside nested composites holds each + # composite. + missingNestedOutput = { + value = + (rootOf ( + topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.a = composite lib ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.b = composite lib readerModule; + } + ); + } + ) + )).outputs.export; + message = "caisson-deferred: needs outputValues.a.b.source.stdout"; + }; + + # The same holds for an upstream composite inside another. + missingUpstreamNestedOutput = { + value = + (rootOf (upstreamTopWith [ + ( + { providers, members, ... }: + { + providers.probe = provider; + members.g.members.h.members.source.type = providers.probe.exec; + members.g.members.reader = { + type = providers.probe.exec; + inputs.value = members.g.members.h.members.source.outputs.stdout; + }; + } + ) + ])).outputs.export; + message = "caisson-deferred: needs outputValues.g.h.source.stdout"; + }; + + # Which members a composite has may not depend on their outputs. + membersFromOutputs = { + value = + (rootOf ( + topWith ( + { lib, members, ... }: + { + caisson.caisson-nixops4-resource.configurations = { + source = resource lib { }; + } + // lib.optionalAttrs (members.source.outputs.stdout == "more") { + extra = resource lib { }; + }; + } + ) + )).argsNeeded; + message = "the composite at the root reads its members, or their"; + }; + + # The error names the composite that reads them, here one beneath the + # root. + nestedMembersFromOutputs = { + value = + (rootOf ( + topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.a = composite lib ( + { lib, members, ... }: + { + caisson.caisson-nixops4-resource.configurations = lib.optionalAttrs (members ? source) { + source = resource lib { }; + }; + } + ); + } + ) + )).argsNeeded; + message = "the composite at a reads its members, or their"; + }; + + # Whether an upstream member is a composite or a resource may not + # depend on the outputs of another member. + upstreamKindFromOutputs = { + value = + (rootOf (upstreamTopWith [ + ( + { providers, members, ... }: + { + providers.probe = provider; + members.source.type = providers.probe.exec; + members.dynamic = + if members.source.outputs.stdout == "resource" then + { type = providers.probe.exec; } + else + { members.inner.type = providers.probe.exec; }; + } + ) + ])).argsNeeded; + message = "the composite at the root reads its members, or their"; + }; + + # A resource may not name its outputs from its own outputs. + skeletonFromOwnOutputs = { + value = + (rootOf ( + topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.selfish = resource lib ( + { config, ... }: + { + outputsSkeleton = lib.mkForce ( + { + stdout = { }; + } + // lib.optionalAttrs (config.outputs.stdout == "more") { extra = { }; } + ); + } + ); + } + ) + )).argsNeeded; + message = "the resource at selfish reads its own outputs in its"; + }; + + # A member is a composite or a resource. + compositeAndResource = { + value = + (rootOf ( + topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.x = resource lib { }; + caisson.caisson-nixops4-composite.configurations.x = composite lib { }; + } + ) + )).argsNeeded; + message = "`x` both as a composite and as a"; + }; + + # A flake publishes one root as `nixops4`. + twoRoots = { + value = + (lib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.nixops4 = composite lib { }; + caisson.nixops4.configurations.nixops4 = lib.caisson.nixops4.mkConfiguration { + ecosystemSrc = _sources: ecosystemSrc; + }; + }; + }).nixops4; + message = "nixops4: this output holds a single value, the configuration"; + }; + + # A root is declared under the name of the output it fills. + rootNamedOtherwise = { + value = + (lib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.deploy = composite lib { }; + }; + }).nixops4; + message = "caisson-nixops4-composite deploy"; + }; +} diff --git a/tests/deferred-errors/check.sh b/tests/deferred-errors/check.sh new file mode 100644 index 00000000..2d437976 --- /dev/null +++ b/tests/deferred-errors/check.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# The errors of deferred configurations and the NixOps4 integrations, +# read as text. A tool reads the error of a deferred configuration to +# tell an arg it lacks (`caisson-deferred: needs `) from a +# failure, and the integrations refuse what they cannot evaluate with +# errors that name the configuration. `builtins.tryEval` hides the +# text of an error, so this runs the real evaluator on each case of +# cases.nix and looks for the line the case names in what it prints. +# +# check.sh LIB_EXPR NIXOPS4_SRC [nix eval flag...] +# +# LIB_EXPR is a Nix expression for the composed library and NIXOPS4_SRC +# the path of the NixOps4 source; the flags follow `nix eval`. The +# flake check composes the library from store paths, as the check of +# tests/argument-errors does. +set -euo pipefail + +lib_expr="$1" +nixops4="$2" +shift 2 +flags=("$@") + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cases="(import $here/cases.nix { lib = $lib_expr; nixops4 = \"$nixops4\"; })" + +names="$(nix eval "${flags[@]}" --raw --expr "builtins.concatStringsSep \"\n\" (builtins.attrNames $cases)")" + +failures=0 +while IFS= read -r name; do + message="$(nix eval "${flags[@]}" --raw --expr "$cases.$name.message")" + # `nix eval` prints an error inside a value in place of failing, so + # the value is forced whole first. + if out="$(nix eval "${flags[@]}" --expr "let value = $cases.$name.value; in builtins.deepSeq value true" 2>&1)"; then + printf 'FAIL %s: evaluated without an error:\n%s\n\n' "$name" "$out" + failures=$((failures + 1)) + elif grep -qF -- "$message" <<< "$out"; then + printf 'ok %s\n' "$name" + else + printf 'FAIL %s: the error does not contain\n %s\n%s\n\n' "$name" "$message" "$out" + failures=$((failures + 1)) + fi +done <<< "$names" + +if [ "$failures" -ne 0 ]; then + printf '%d case(s) failed\n' "$failures" + exit 1 +fi diff --git a/tests/dependencies/flake.lock b/tests/dependencies/flake.lock index 77798be2..ff5cf93a 100644 --- a/tests/dependencies/flake.lock +++ b/tests/dependencies/flake.lock @@ -39,6 +39,23 @@ "type": "github" } }, + "crane": { + "flake": false, + "locked": { + "lastModified": 1758758545, + "narHash": "sha256-NU5WaEdfwF6i8faJ2Yh+jcK9vVFrofLcwlD/mP65JrI=", + "owner": "ipetkov", + "repo": "crane", + "rev": "95d528a5f54eaba0d12102249ce42f4d01f4e364", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "ref": "v0.21.1", + "repo": "crane", + "type": "github" + } + }, "devshell": { "inputs": { "nixpkgs": [ @@ -80,6 +97,30 @@ "type": "github" } }, + "dream2nix": { + "inputs": { + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "nixpkgs" + ], + "purescript-overlay": "purescript-overlay", + "pyproject-nix": "pyproject-nix" + }, + "locked": { + "lastModified": 1779999259, + "narHash": "sha256-aLAG+TG7stjp0AOI2uvn4agxrrws2Q+fXMDS8ZuHYmo=", + "owner": "nix-community", + "repo": "dream2nix", + "rev": "dd8a3ebbbfcfafc139dba1323a6c443ec034c9f5", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "dream2nix", + "type": "github" + } + }, "flake-compat": { "flake": false, "locked": { @@ -113,6 +154,54 @@ } }, "flake-compat_3": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_4": { + "flake": false, + "locked": { + "lastModified": 1767039857, + "narHash": "sha256-vNpUSpF5Nuw8xvDLj2KCwwksIbjua2LZCqhV1LNRDns=", + "owner": "NixOS", + "repo": "flake-compat", + "rev": "5edf11c44bc78a0d334f6334cdaf7d60d732daab", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_5": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_6": { "flake": false, "locked": { "lastModified": 1777699697, @@ -128,7 +217,7 @@ "url": "https://git.lix.systems/lix-project/flake-compat.git" } }, - "flake-compat_4": { + "flake-compat_7": { "flake": false, "locked": { "lastModified": 1767039857, @@ -201,6 +290,51 @@ } }, "flake-parts_4": { + "inputs": { + "nixpkgs-lib": [ + "nixops4", + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1788450739, + "narHash": "sha256-glZLQlzIn1fXH6PazR2iUmTo7kzzyYSshrWhLS9TqCU=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "31729ca8cbdb4fa927b34e5f4353e6a83f39e993", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_5": { + "inputs": { + "nixpkgs-lib": [ + "nixops4", + "nix-bindings-rust", + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1733312601, + "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_6": { "inputs": { "nixpkgs-lib": [ "nixvim", @@ -221,7 +355,7 @@ "type": "github" } }, - "flake-parts_5": { + "flake-parts_7": { "inputs": { "nixpkgs-lib": [ "system-manager", @@ -243,7 +377,7 @@ "type": "github" } }, - "flake-parts_6": { + "flake-parts_8": { "inputs": { "nixpkgs-lib": [ "terranix", @@ -300,6 +434,71 @@ "type": "github" } }, + "git-hooks-nix": { + "inputs": { + "flake-compat": [ + "nixops4", + "nix" + ], + "nixpkgs": [ + "nixops4", + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1788267358, + "narHash": "sha256-nt+lUqYVpc9Y6JeMd2WmXzCDojasdadKo0mWcluvY2Y=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "27555e2624241fb116b49095df4caaee85a25691", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "git-hooks-nix_2": { + "inputs": { + "flake-compat": [ + "nixops4", + "nix-bindings-rust", + "nix" + ], + "gitignore": [ + "nixops4", + "nix-bindings-rust", + "nix" + ], + "nixpkgs": [ + "nixops4", + "nix-bindings-rust", + "nix", + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixops4", + "nix-bindings-rust", + "nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1734279981, + "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, "gitignore": { "inputs": { "nixpkgs": [ @@ -440,6 +639,22 @@ "type": "github" } }, + "mk-naked-shell": { + "flake": false, + "locked": { + "lastModified": 1681286841, + "narHash": "sha256-3XlJrwlR0nBiREnuogoa5i1b4+w/XPe0z8bbrJASw0g=", + "owner": "90-008", + "repo": "mk-naked-shell", + "rev": "7612f828dd6f22b7fb332cc69440e839d7ffe6bd", + "type": "github" + }, + "original": { + "owner": "90-008", + "repo": "mk-naked-shell", + "type": "github" + } + }, "mnw": { "locked": { "lastModified": 1784154424, @@ -455,6 +670,90 @@ "type": "github" } }, + "nix": { + "inputs": { + "flake-compat": "flake-compat_3", + "flake-parts": "flake-parts_4", + "git-hooks-nix": "git-hooks-nix", + "nixpkgs": [ + "nixops4", + "nixpkgs" + ], + "nixpkgs-23-11": "nixpkgs-23-11", + "nixpkgs-regression": "nixpkgs-regression" + }, + "locked": { + "lastModified": 1790895260, + "narHash": "sha256-GGeEfw8mghwKHhsjDzmYXoP97JLoggLxF2nns4MRBG8=", + "owner": "NixOS", + "repo": "nix", + "rev": "f83a7f39e6f8c9aa254de32e51e37733fb07d7e6", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nix", + "type": "github" + } + }, + "nix-bindings-rust": { + "inputs": { + "flake-parts": [ + "nixops4", + "flake-parts" + ], + "nix": "nix_2", + "nix-cargo-integration": [ + "nixops4", + "nix-cargo-integration" + ], + "nixpkgs": [ + "nixops4", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1778917363, + "narHash": "sha256-p2X0V5FbS7pa13lSSi+hvH1LMwXBypRZ217NSWXmPK4=", + "owner": "nixops4", + "repo": "nix-bindings-rust", + "rev": "49468a3da7fe52b22ab9ef73d147cc4ec573ed8e", + "type": "github" + }, + "original": { + "owner": "nixops4", + "ref": "0.2.1", + "repo": "nix-bindings-rust", + "type": "github" + } + }, + "nix-cargo-integration": { + "inputs": { + "crane": "crane", + "dream2nix": "dream2nix", + "mk-naked-shell": "mk-naked-shell", + "nixpkgs": [ + "nixops4", + "nixpkgs" + ], + "parts": "parts", + "rust-overlay": "rust-overlay", + "treefmt": "treefmt" + }, + "locked": { + "lastModified": 1790856477, + "narHash": "sha256-C4RZUuKpBb2OS+MzuJizybH4kauwqDumj8BoEdmZwOg=", + "owner": "yusdacra", + "repo": "nix-cargo-integration", + "rev": "5630c7aa777ca213a7ea78f3b6a7e165ab46a30f", + "type": "github" + }, + "original": { + "owner": "yusdacra", + "repo": "nix-cargo-integration", + "type": "github" + } + }, "nix-darwin": { "inputs": { "nixpkgs": [ @@ -595,6 +894,33 @@ "type": "github" } }, + "nix_2": { + "inputs": { + "flake-compat": "flake-compat_4", + "flake-parts": "flake-parts_5", + "git-hooks-nix": "git-hooks-nix_2", + "nixpkgs": [ + "nixops4", + "nix-bindings-rust", + "nixpkgs" + ], + "nixpkgs-23-11": "nixpkgs-23-11_2", + "nixpkgs-regression": "nixpkgs-regression_2" + }, + "locked": { + "lastModified": 1777584486, + "narHash": "sha256-/2wsg4bAs8t6QlSOHAiz2qeO5OXpYQqFYPdp2R3AxyI=", + "owner": "NixOS", + "repo": "nix", + "rev": "2d26f28966311c3b734c03f50565d15615dd4c2b", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nix", + "type": "github" + } + }, "nixidy": { "inputs": { "flake-utils": "flake-utils_2", @@ -617,6 +943,58 @@ "type": "github" } }, + "nixops4": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "nix": "nix", + "nix-bindings-rust": "nix-bindings-rust", + "nix-cargo-integration": "nix-cargo-integration", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1791020006, + "narHash": "sha256-codUYzTTiynymi0RA8b4fX8dtUiHfVtNZAx56y7XI6I=", + "owner": "nixops4", + "repo": "nixops4", + "rev": "41858b44eda2a2d9d0f0610bf5b314688a7505d7", + "type": "github" + }, + "original": { + "owner": "nixops4", + "repo": "nixops4", + "type": "github" + } + }, + "nixops4-nixos": { + "inputs": { + "flake-parts": [ + "flake-parts" + ], + "nixops4-nixos": [ + "nixops4-nixos" + ], + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1791211623, + "narHash": "sha256-OkYw3jYX5tx6+WZMV7L8VXhU4pg6y+JrsY4aLML6jBQ=", + "owner": "nixops4", + "repo": "nixops4-nixos", + "rev": "4c7ceab7a80bafcd2ce90337a6c386cb0ef0ec43", + "type": "github" + }, + "original": { + "owner": "nixops4", + "repo": "nixops4-nixos", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1789785513, @@ -633,6 +1011,38 @@ "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, + "nixpkgs-23-11_2": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-lib": { "locked": { "lastModified": 1785031560, @@ -678,9 +1088,41 @@ "type": "github" } }, + "nixpkgs-regression": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, + "nixpkgs-regression_2": { + "locked": { + "lastModified": 1643052045, + "narHash": "sha256-uGJ0VXIhWKGXxkeNnq4TvV3CIOkUJ3PAoLZ3HMzNVMw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "215d4d0fd80ca5163643b03a33fde804a29cc1e2", + "type": "github" + } + }, "nixvim": { "inputs": { - "flake-parts": "flake-parts_4", + "flake-parts": "flake-parts_6", "nixpkgs": [ "nixpkgs" ], @@ -702,7 +1144,7 @@ }, "nvf": { "inputs": { - "flake-compat": "flake-compat_3", + "flake-compat": "flake-compat_6", "mnw": "mnw", "nixpkgs": [ "nixpkgs" @@ -722,6 +1164,28 @@ "type": "github" } }, + "parts": { + "inputs": { + "nixpkgs-lib": [ + "nixops4", + "nix-cargo-integration", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1788450739, + "narHash": "sha256-glZLQlzIn1fXH6PazR2iUmTo7kzzyYSshrWhLS9TqCU=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "31729ca8cbdb4fa927b34e5f4353e6a83f39e993", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "pre-commit-hooks-nix": { "inputs": { "flake-compat": [ @@ -750,6 +1214,54 @@ "type": "github" } }, + "purescript-overlay": { + "inputs": { + "flake-compat": "flake-compat_5", + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "dream2nix", + "nixpkgs" + ], + "slimlock": "slimlock" + }, + "locked": { + "lastModified": 1728546539, + "narHash": "sha256-Sws7w0tlnjD+Bjck1nv29NjC5DbL6nH5auL9Ex9Iz2A=", + "owner": "thomashoneyman", + "repo": "purescript-overlay", + "rev": "4ad4c15d07bd899d7346b331f377606631eb0ee4", + "type": "github" + }, + "original": { + "owner": "thomashoneyman", + "repo": "purescript-overlay", + "type": "github" + } + }, + "pyproject-nix": { + "inputs": { + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "dream2nix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1763017646, + "narHash": "sha256-Z+R2lveIp6Skn1VPH3taQIuMhABg1IizJd8oVdmdHsQ=", + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "rev": "47bd6f296502842643078d66128f7b5e5370790c", + "type": "github" + }, + "original": { + "owner": "pyproject-nix", + "repo": "pyproject.nix", + "type": "github" + } + }, "root": { "inputs": { "caisson-core": "caisson-core", @@ -765,6 +1277,8 @@ "nix-unit": "nix-unit", "nix-vm-test": "nix-vm-test", "nixidy": "nixidy", + "nixops4": "nixops4", + "nixops4-nixos": "nixops4-nixos", "nixpkgs": "nixpkgs", "nixpkgs-lib": "nixpkgs-lib_3", "nixvim": "nixvim", @@ -774,6 +1288,52 @@ "treefmt-nix": "treefmt-nix_3" } }, + "rust-overlay": { + "inputs": { + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1790843958, + "narHash": "sha256-jtYAaAXT1gxEtzmeYmOGu0IvKPle5J39BOFbgl6os+A=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "b2ccad3b67335d368eef49ec48b9c8166858e73d", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "slimlock": { + "inputs": { + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "dream2nix", + "purescript-overlay", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1688756706, + "narHash": "sha256-xzkkMv3neJJJ89zo3o2ojp7nFeaZc2G0fYwNXNJRFlo=", + "owner": "thomashoneyman", + "repo": "slimlock", + "rev": "cf72723f59e2340d24881fd7bf61cb113b4c407c", + "type": "github" + }, + "original": { + "owner": "thomashoneyman", + "repo": "slimlock", + "type": "github" + } + }, "spectrum": { "flake": false, "locked": { @@ -808,7 +1368,7 @@ }, "system-manager": { "inputs": { - "flake-compat": "flake-compat_4", + "flake-compat": "flake-compat_7", "nixpkgs": [ "nixpkgs" ], @@ -922,7 +1482,7 @@ }, "terranix": { "inputs": { - "flake-parts": "flake-parts_6", + "flake-parts": "flake-parts_8", "import-tree": "import-tree_2", "nixpkgs": [ "nixpkgs" @@ -943,6 +1503,28 @@ "type": "github" } }, + "treefmt": { + "inputs": { + "nixpkgs": [ + "nixops4", + "nix-cargo-integration", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1786901030, + "narHash": "sha256-WSFCsDSE5ffgD2MqzkM2CYjeFiKhRF/dJUN8uedb6YE=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "27b3b12a8e6375f28ebe122f07d230ca5459bbfa", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + }, "treefmt-nix": { "inputs": { "nixpkgs": [ @@ -1011,7 +1593,7 @@ "system-manager", "flake-compat" ], - "flake-parts": "flake-parts_5", + "flake-parts": "flake-parts_7", "nixpkgs": [ "system-manager", "nixpkgs" diff --git a/tests/dependencies/flake.nix b/tests/dependencies/flake.nix index 7d8af0fb..fd82a307 100644 --- a/tests/dependencies/flake.nix +++ b/tests/dependencies/flake.nix @@ -37,6 +37,12 @@ inputs.nix-darwin.inputs.nixpkgs.follows = "nixpkgs"; inputs.microvm.url = "github:microvm-nix/microvm.nix"; inputs.microvm.inputs.nixpkgs.follows = "nixpkgs"; + inputs.nixops4.url = "github:nixops4/nixops4"; + inputs.nixops4.inputs.nixpkgs.follows = "nixpkgs"; + inputs.nixops4.inputs.flake-parts.follows = "flake-parts"; + inputs.nixops4-nixos.url = "github:nixops4/nixops4-nixos"; + inputs.nixops4-nixos.inputs.nixpkgs.follows = "nixpkgs"; + inputs.nixops4-nixos.inputs.flake-parts.follows = "flake-parts"; # The VMs of other distributions the system-manager VM test boots # (tests/system-manager). inputs.nix-vm-test.url = "github:numtide/nix-vm-test"; diff --git a/tests/eval-weight/baseline.json b/tests/eval-weight/baseline.json index be5026ac..666d5c70 100644 --- a/tests/eval-weight/baseline.json +++ b/tests/eval-weight/baseline.json @@ -1,22 +1,22 @@ { "caisson-overhead": { - "nrThunks": 62007, - "nrValues": 77486, - "nrEnvs": 45988, - "nrFunctionCalls": 38520, - "nrPrimOpCalls": 22785, - "totalAllocBytes": 3889344, + "nrThunks": 70319, + "nrValues": 91203, + "nrEnvs": 53548, + "nrFunctionCalls": 44802, + "nrPrimOpCalls": 25801, + "totalAllocBytes": 4618224, "nixpkgsEvals": 0, "nixpkgsLibEvals": 0, "moduleSystemEvals": 11 }, "minimal-consumer-total": { - "nrThunks": 253742, - "nrValues": 734160, - "nrEnvs": 139119, - "nrFunctionCalls": 113586, - "nrPrimOpCalls": 73028, - "totalAllocBytes": 46965344, + "nrThunks": 262054, + "nrValues": 747877, + "nrEnvs": 146679, + "nrFunctionCalls": 119868, + "nrPrimOpCalls": 76044, + "totalAllocBytes": 47694224, "nixpkgsEvals": 1, "nixpkgsLibEvals": 2, "moduleSystemEvals": 28 diff --git a/tests/nixops4-discovery/default.nix b/tests/nixops4-discovery/default.nix new file mode 100644 index 00000000..5a09af3d --- /dev/null +++ b/tests/nixops4-discovery/default.nix @@ -0,0 +1,174 @@ +# SPDX-License-Identifier: MIT +# +# The discovery program of a NixOps4 root, built and run. `inputs` is +# the dependency pool of the checks partition with `caisson` set to +# the tree itself. +# +# Each deployment below holds resources that record their outputs in a +# state resource. Its root composite takes NixOps4 from a source whose +# `nixops4` package is a stand-in: `nixops4 state dump ` prints +# the dump recorded in this directory for the state resource at that +# path. The dumps are trimmed from the dumps of a real `nixops4 apply`. +# Building the program runs shellcheck on it. Running it must print the +# output values at the paths the root names in `argsNeeded`, with the +# values the dumps record. +{ inputs }: +let + suite = import ../pinned-world { inherit inputs; }; + inherit (suite.fixtures) + hiveLib + pkgs + nixops4ArgPathsOf + nixops4Entry + ; + system = "x86_64-linux"; + local = inputs.nixops4.modules.nixops4Provider.local; + + # The stand-in for NixOps4: it answers `state dump ` with the + # dump recorded for the state resource at that path. + nixops4 = pkgs.writeShellScriptBin "nixops4" '' + case "''${*: -1}" in + state) cat ${./root-state.json} ;; + group.gstate) cat ${./group-state.json} ;; + *) echo "no dump for ''${*: -1}" >&2; exit 1 ;; + esac + ''; + ecosystemSrc = inputs.nixops4 // { + packages = inputs.nixops4.packages // { + ${system} = inputs.nixops4.packages.${system} // { + inherit nixops4; + }; + }; + }; + + # A top whose root composite `nixops4` holds the composite `group`, + # with the state resource `gstate` and the resource `inner`, which + # records its outputs there. With `rootState`, the root also holds the + # state resource `state` and the resource `a`, which records its + # outputs there. + topWith = + { rootState }: + hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + let + resource = + module: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + configModule = + { providers, ... }: + { + imports = [ module ]; + providers.local = local; + }; + }; + stateFile = + name: + resource ( + { providers, ... }: + { + type = providers.local.state_file; + inputs.name = name; + } + ); + exec = + state: + resource ( + { providers, ... }: + { + type = providers.local.exec; + inputs.executable = "/bin/sh"; + inherit state; + } + ); + in + { + caisson.caisson-nixops4-composite.configurations.nixops4 = + lib.caisson.caisson-nixops4-composite.mkConfiguration + { + ecosystemSrc = _sources: ecosystemSrc; + configModule = + { lib, members, ... }: + { + caisson.caisson-nixops4-composite.configurations.group = + lib.caisson.caisson-nixops4-composite.mkConfiguration + { + configModule = + { members, ... }: + { + caisson.caisson-nixops4-resource.configurations = { + gstate = stateFile "group-state.json"; + inner = exec members.gstate; + }; + }; + }; + caisson.caisson-nixops4-resource.configurations = lib.optionalAttrs rootState { + state = stateFile "state.json"; + a = exec members.state; + }; + }; + }; + }; + }; + + # The paths of the values in a tree of attribute sets, each a list of + # names. + valuePathsOf = + let + walk = + path: value: + if builtins.isAttrs value then + builtins.concatMap (name: walk (path ++ [ name ]) value.${name}) (builtins.attrNames value) + else + [ path ]; + in + walk [ ]; + + # The discovery program of a top's root and the output values it must + # print. The values are those the dumps record, and their paths are + # the paths of the outputs the root names in `argsNeeded`. + caseOf = + name: rootState: outputValues: + let + root = + (nixops4Entry (topWith { + inherit rootState; + })).finalize + { + args.resourceProviderSystem = system; + argsSource = _: "the discovery check"; + }; + expected = { + inherit outputValues; + resourceProviderSystem = system; + }; + in + assert nixops4ArgPathsOf root.argsNeeded.outputValues == valuePathsOf outputValues; + { + inherit name; + program = root.discoverArgs { inherit system; }; + expected = pkgs.writeText "${name}.json" (builtins.toJSON expected); + }; + + cases = [ + # One state resource, inside a composite. + (caseOf "nested" false { group.inner.stdout = "nested-state-value"; }) + # Two state resources, one at the root and one inside a composite. + (caseOf "both" true { + a.stdout = "alpha-42"; + group.inner.stdout = "nested-state-value"; + }) + ]; +in +pkgs.runCommand "nixops4-discovery" { nativeBuildInputs = [ pkgs.jq ]; } ( + pkgs.lib.concatMapStrings (case: '' + ${pkgs.lib.getExe case.program} > ${case.name}.json + jq -S . ${case.expected} > ${case.name}.expected + jq -S . ${case.name}.json > ${case.name}.printed + diff -u ${case.name}.expected ${case.name}.printed + '') cases + + '' + touch $out + '' +) diff --git a/tests/nixops4-discovery/group-state.json b/tests/nixops4-discovery/group-state.json new file mode 100644 index 00000000..cef6af7c --- /dev/null +++ b/tests/nixops4-discovery/group-state.json @@ -0,0 +1,23 @@ +{ + "_type": "nixopsState", + "resources": {}, + "deployments": { + "group": { + "resources": { + "inner": { + "type": "exec", + "input_properties": { + "args": ["-c", "printf %s nested-state-value"], + "executable": "/bin/sh", + "once": false, + "stdin": null + }, + "output_properties": { + "stdout": "nested-state-value" + } + } + }, + "deployments": {} + } + } +} diff --git a/tests/nixops4-discovery/root-state.json b/tests/nixops4-discovery/root-state.json new file mode 100644 index 00000000..1b960268 --- /dev/null +++ b/tests/nixops4-discovery/root-state.json @@ -0,0 +1,18 @@ +{ + "_type": "nixopsState", + "resources": { + "a": { + "type": "exec", + "input_properties": { + "args": ["-c", "printf %s alpha-42"], + "executable": "/bin/sh", + "once": false, + "stdin": null + }, + "output_properties": { + "stdout": "alpha-42" + } + } + }, + "deployments": {} +} diff --git a/tests/pinned-world/default.nix b/tests/pinned-world/default.nix index 95072ca7..8a86a637 100644 --- a/tests/pinned-world/default.nix +++ b/tests/pinned-world/default.nix @@ -70,6 +70,9 @@ let registered."caisson/facter" registered."caisson/caisson-microvm" registered."caisson/microvm" + registered."caisson/caisson-nixops4-composite" + registered."caisson/caisson-nixops4-resource" + registered."caisson/nixops4" registered."caisson/darwin" registered."caisson/terranix" registered."caisson/system-manager" @@ -94,6 +97,8 @@ let "caisson-disko" "caisson-facter" "caisson-microvm" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "callConsumerFlake" "callFlake" "colmena" @@ -126,6 +131,7 @@ let "mkPkgOverlays" "modules" "nixidy" + "nixops4" "nixos" "nixos-minimal" "nixpkgs" @@ -229,6 +235,8 @@ let hjem terranix microvm + nixops4 + nixops4-nixos ; }; projects = { @@ -370,6 +378,290 @@ let }; colmenaTop = colmenaTopWith { }; + # A top holding the root composite of a NixOps4 deployment, declared + # under the name `nixops4`, which declares a resource for each machine + # given, by name, holding a NixOS configuration of that name. `root` + # is a further module of the root composite, `resources` a further + # module of each resource, by name, and `wiring` a function of the + # `members` argument of the root composite returning more such + # modules, by name, as a composite wires one member to another. + # `defaultPkgs` is the selection of the root composite and + # `machineArgs` further arguments of each machine's constructor. + nixops4TopWith = + { + root ? { }, + resources ? { }, + wiring ? _: { }, + defaultPkgs ? null, + machineArgs ? { }, + }: + machines: + hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.nixops4 = + lib.caisson.caisson-nixops4-composite.mkConfiguration + { + inherit defaultPkgs; + configModule = + { lib, members, ... }: + { + imports = [ root ]; + caisson.caisson-nixops4-resource.configurations = builtins.mapAttrs ( + name: nixosModule: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + configModule = + { lib, ... }: + { + imports = [ + (resources.${name} or { }) + ((wiring members).${name} or { }) + ]; + caisson.nixos.configurations.${name} = lib.caisson.nixos.mkConfiguration ( + { configModule = nixosModule; } // machineArgs + ); + }; + } + ) machines; + }; + }; + }; + }; + + # The entry the top holds for the root composite named `nixops4`, of + # the native or the upstream-compatible integration. + nixops4Entry = + top: + let + children = top.caisson.manifest.children; + in + (children.caisson-nixops4-composite or children.nixops4).nixops4; + + # The export of a root as NixOps4's command reads it: the root + # function called with the deploying system and the outputs of each + # resource, here `stub` for every output the resource declares, at + # the fixpoint, as the command closes it. + nixops4Export = + root: + let + fixpoint = root.rootFunction { + resourceProviderSystem = "x86_64-linux"; + outputValues = nixops4StubOutputs fixpoint.members; + }; + in + fixpoint; + + # The output values NixOps4 passes for the members of an export, with + # `stub` for every output a resource declares. + nixops4StubOutputs = builtins.mapAttrs ( + _: member: + if member ? resource then + builtins.mapAttrs (_: _: "stub") member.resource.outputsSkeleton + else + nixops4StubOutputs member.members + ); + + # The root composite of a top's `nixops4`, finalized as NixOps4 + # finalizes it, with `stub` outputs: its manifest. + nixops4RootManifest = + top: + (nixops4Entry top).finalize { + args = { + resourceProviderSystem = "x86_64-linux"; + outputValues = nixops4StubOutputs (nixops4Export top.nixops4).members; + }; + argsSource = _: "the probe"; + }; + + # A top holding a nixops4 configuration named `nixops4` made from the + # root modules given, as NixOps4's `mkRoot` takes them, with the + # allow-list `cacheableOutputs`. + nixops4UpstreamTopWith = + { + cacheableOutputs ? { }, + }: + modules: + hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.nixops4.configurations.nixops4 = lib.caisson.nixops4.mkConfiguration { + inherit modules cacheableOutputs; + }; + }; + }; + + # Root modules as a NixOps4 user writes them: a resource whose output + # another member reads, a NixOS machine through nixops4-nixos, and a + # nested composite holding a resource. + nixops4UpstreamRoot = + { providers, members, ... }: + { + _class = "nixops4Component"; + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members.address = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + members.probe = { + imports = [ inputs.nixops4-nixos.modules.nixops4Component.nixos ]; + type = providers.local.exec; + nixpkgs = inputs.nixpkgs; + nixos.module = minimalNixosBase; + ssh.host = members.address.outputs.stdout; + ssh.hostPublicKey = nixops4HostKey; + }; + members.group.members.inner = { + type = providers.local.exec; + inputs.executable = "/bin/false"; + }; + }; + + # A host key for the machines NixOps4 deploys in the probes. + nixops4HostKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAiszi43aOWWV7voNgQ1Ifa7LGKwGJfOuiLM1n42h2Y8"; + + # A further module of a native root composite: a resource `address` + # of NixOps4's `local` provider, whose `stdout` another member reads. + nixops4AddressRoot = + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.address = + lib.caisson.caisson-nixops4-resource.mkConfiguration + { + configModule = + { providers, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + }; + }; + + # The paths of the args an `argsNeeded` names, each a list of names. + nixops4ArgPathsOf = + let + walk = + path: value: + if core.isArg value then + [ path ] + else + builtins.concatMap (name: walk (path ++ [ name ]) value.${name}) (builtins.attrNames value); + in + walk [ ]; + + # A native composite whose module is `module`, under `lib`. + nixops4Composite = + lib: module: + lib.caisson.caisson-nixops4-composite.mkConfiguration { + configModule = module; + }; + + # A native resource of the `exec` type of NixOps4's `local` provider, + # with the further module `module`. + nixops4Exec = + lib: module: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + configModule = + { providers, ... }: + { + imports = [ module ]; + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + }; + + # A native resource of the `state_file` type of NixOps4's `local` + # provider, a resource that holds the state of others. + nixops4StateFile = + lib: + lib.caisson.caisson-nixops4-resource.mkConfiguration { + configModule = + { providers, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = providers.local.state_file; + inputs.name = "state.json"; + }; + }; + + # A composite module that records its `members` argument as `seen`. + nixops4SeesMembers = + { lib, members, ... }: + { + options.seen = lib.mkOption { type = lib.types.raw; }; + config.seen = members; + }; + + # A top whose root composite `nixops4` has the module `module`. + nixops4RootTopWith = + module: + hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.nixops4 = nixops4Composite lib module; + }; + }; + + # Whether forcing a value fails with an error a probe can catch. + nixops4Fails = value: !(builtins.tryEval (builtins.deepSeq value true)).success; + + # A deferred configuration of a tool that is not NixOps4, declared + # beside a NixOps4 deployment: it takes the cacheable arg `who` and + # greets it. Its entry is published as `greetings.`, a function + # of `who`, and its discovery program is named `greeter-discover`. + greeterConfiguration = + lib: + lib.caisson-core.mkDeferredConfiguration { + type = "greeter"; + record.exportsTo = { + attrset = "greetings"; + value = + entry: who: + (entry.finalize { + args = { inherit who; }; + argsSource = _: "the probe"; + }).outputs.greeting; + }; + configuration = + { + name, + parent, + args, + }: + lib.caisson.integrations.mkModuleConfiguration { + type = "greeter"; + evaluate = + { lib, ... }: + let + evaluated = lib.evalModules { + specialArgs = { inherit lib; }; + modules = lib.caisson.integrations.frameworkModules "structural" { } ++ [ + ( + { lib, ... }: + { + options.greeting = lib.mkOption { type = lib.types.str; }; + config.greeting = "hello ${args.who}"; + } + ) + ]; + }; + in + { + value = evaluated; + outputs.greeting = evaluated.config.greeting; + argsNeeded.who = lib.caisson-core.mkArg { cacheable = true; }; + discoverArgs = { system }: pkgs.writeShellScript "greeter-discover" "echo '{\"who\":\"world\"}'"; + }; + } { inherit name parent; }; + }; + # What a microvm.nix guest needs beside what microvm.nix's guest # module gives it, which boots it and gives it its file systems. microvmGuestBase = { @@ -637,6 +929,9 @@ let "caisson/facter" "caisson/caisson-microvm" "caisson/microvm" + "caisson/caisson-nixops4-resource" + "caisson/caisson-nixops4-composite" + "caisson/nixops4" "caisson/darwin" "caisson/terranix" "caisson/system-manager" @@ -744,6 +1039,9 @@ let "facter" "caisson-microvm" "microvm" + "caisson-nixops4-composite" + "caisson-nixops4-resource" + "nixops4" "darwin" "terranix" "system-manager" @@ -2043,6 +2341,990 @@ let } ]; + # The root composite, declared under the name `nixops4`, fills the + # `nixops4` of the top, the root NixOps4's command reads. A resource holding a + # NixOS configuration becomes, through the NixOS adapter, an `exec` + # resource of NixOps4's `local` provider that copies the toplevel of + # that configuration and runs its `bin/apply switch` over SSH, to + # the name of the resource as the host when nothing sets one. The + # toplevel carries the `bin/apply` the `nixops4-apply` module + # builds. + nixops4RootEvaluatesEndToEnd = + let + top = nixops4TopWith { + resources.probe.ssh.hostPublicKey = nixops4HostKey; + } { probe = minimalNixosBase; }; + export = nixops4Export top.nixops4; + resource = export.members.probe.resource; + args = resource.inputs.args; + machine = + (nixops4RootManifest top).children.caisson-nixops4-resource.probe.value.config.nixos.configuration; + in + top.nixops4._type == "nixops4Component" + && builtins.attrNames export.members == [ "probe" ] + && resource.type == "exec" + && resource.inputs.executable == pkgs.lib.getExe pkgs.bash + && builtins.elemAt args 2 == "root@probe" + && builtins.elemAt args 3 == "${machine.config.system.build.toplevel}" + && pkgs.lib.hasInfix "$1/bin/apply switch" (builtins.elemAt args 1) + && pkgs.lib.hasInfix "$out/bin/apply" machine.config.system.activatableSystemBuilderCommands + && pkgs.lib.hasInfix nixops4HostKey ( + builtins.readFile ( + builtins.head (builtins.match ".*UserKnownHostsFile=([^ ]*) .*" (builtins.elemAt args 1)) + ) + ); + + # A composite wires a resource to the outputs of another member + # through `members`, and NixOps4 evaluates the root again with the + # outputs it learns. The SSH settings change the command line as + # they do in nixops4-nixos. + nixops4ResourceReadsSiblingOutputs = + let + top = nixops4TopWith { + root = nixops4AddressRoot; + wiring = members: { + probe = { + ssh.host = members.address.outputs.stdout; + ssh.hostPublicKey = nixops4HostKey; + ssh.user = "deploy"; + ssh.opts = "-p 2222"; + copy.substituteOnDestination = false; + }; + }; + } { probe = minimalNixosBase; }; + export = nixops4Export top.nixops4; + args = export.members.probe.resource.inputs.args; + script = builtins.elemAt args 1; + in + builtins.attrNames export.members == [ + "address" + "probe" + ] + && builtins.elemAt args 2 == "deploy@stub" + && pkgs.lib.hasInfix "sudo $1/bin/apply switch" script + && pkgs.lib.hasInfix "'-p 2222'" script + && !(pkgs.lib.hasInfix "--substitute-on-destination" script); + + # The shell that runs the deployment comes from the package set + # caisson selects for the resource, inherited from the root + # composite, for the deploying system: here a selection whose + # `bash` is another shell. + nixops4DeploysWithTheSelectedPackageSet = + let + top = nixops4TopWith { + resources.probe.ssh.hostPublicKey = nixops4HostKey; + defaultPkgs = pkgSets: pkgSets.default // { bash = pkgSets.default.dash; }; + } { probe = minimalNixosBase; }; + in + (nixops4Export top.nixops4).members.probe.resource.inputs.executable == pkgs.lib.getExe pkgs.dash; + + # Composite and resource are two classes: a composite has no `type` + # and a resource no `members`, and setting either is an error where + # it is set rather than a member silently dropped. + nixops4CompositeAndResourceAreTwoClasses = + let + evaluates = top: (builtins.tryEval (builtins.deepSeq (nixops4Export top.nixops4) true)).success; + resourceTop = + extra: + nixops4TopWith { + resources.probe = { + ssh.hostPublicKey = nixops4HostKey; + } + // extra; + } { probe = minimalNixosBase; }; + in + evaluates (nixops4TopWith { } { }) + && !(evaluates (nixops4TopWith { root.type = "exec"; } { })) + && evaluates (resourceTop { }) + && !(evaluates (resourceTop { + members.extra = { }; + })); + + # The root composite is a deferred configuration declared at the + # top: the manifest of the top holds its entry, and nothing beneath + # it is evaluated at flake time. Finalized, its manifest names the + # args it takes: the deploying system and the outputs of its + # resources, which are not cacheable unless a resource lists them. + # Each manifest records where its args came from, not their values. + nixops4RootIsDeferred = + let + top = nixops4TopWith { + resources.probe = { + ssh.hostPublicKey = nixops4HostKey; + cacheableOutputs = [ "stdout" ]; + }; + } { probe = minimalNixosBase; }; + entry = nixops4Entry top; + root = nixops4RootManifest top; + needed = root.argsNeeded; + in + core.isDeferred entry + && entry.type == "caisson-nixops4-composite" + && entry.parent.childless + && + nixops4ArgPathsOf needed == [ + [ + "outputValues" + "probe" + "stdout" + ] + [ "resourceProviderSystem" ] + ] + && needed.outputValues.probe.stdout.cacheable + && needed.resourceProviderSystem.cacheable + && + root.argsSource [ + "outputValues" + "probe" + "stdout" + ] == "the probe" + && + root.children.caisson-nixops4-resource.probe.argsSource [ + "outputValues" + "stdout" + ] == "the composite at the root"; + + # An output NixOps4 has not passed fails where it is read, with the + # text `caisson-deferred: needs outputValues.address.stdout`, and + # the members that do not read it evaluate. + nixops4MissingOutputFailsWhereRead = + let + top = nixops4TopWith { + root = nixops4AddressRoot; + wiring = members: { + probe = { + ssh.host = members.address.outputs.stdout; + ssh.hostPublicKey = nixops4HostKey; + }; + }; + } { probe = minimalNixosBase; }; + export = top.nixops4.rootFunction { + resourceProviderSystem = "x86_64-linux"; + outputValues.probe.stdout = "stub"; + }; + in + export.members.address.resource.type == "exec" + && !(builtins.tryEval (builtins.elemAt export.members.probe.resource.inputs.args 2)).success; + + # A resource whose type depends on the output of another member + # names its own outputs once that output is known. Given the + # output, it evaluates, with the type the output selects. Without + # it, the outputs of the other members are still named. + nixops4TypeDependsOnAnotherOutput = + let + top = nixops4TopWith { + root = + { lib, members, ... }: + { + imports = [ nixops4AddressRoot ]; + caisson.caisson-nixops4-resource.configurations.dependent = + lib.caisson.caisson-nixops4-resource.mkConfiguration + { + configModule = + { providers, ... }: + let + file = members.address.outputs.stdout == "file"; + in + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = if file then providers.local.file else providers.local.exec; + inputs = + if file then + { + name = "/run/probe"; + contents = ""; + } + else + { executable = "/bin/true"; }; + }; + }; + }; + } { }; + exportWith = + stdout: + top.nixops4.rootFunction { + resourceProviderSystem = "x86_64-linux"; + outputValues = { + address = { inherit stdout; }; + dependent = { }; + }; + }; + withoutOutputs = (nixops4Entry top).finalize { + args.resourceProviderSystem = "x86_64-linux"; + argsSource = _: "the probe"; + }; + in + (exportWith "file").members.dependent.resource.type == "file" + && (exportWith "file").members.dependent.resource.outputsSkeleton == { } + && builtins.attrNames (exportWith "exec").members.dependent.resource.outputsSkeleton == [ "stdout" ] + && builtins.attrNames withoutOutputs.argsNeeded.outputValues.address == [ "stdout" ] + && !(builtins.tryEval withoutOutputs.argsNeeded.outputValues.dependent).success; + + # The `members` argument of a native composite holds, for each + # member, its outputs and its member path, from the args and the + # names the members are declared under. A member's config is not + # reachable through it. + nixops4NativeMembersHoldOutputsAndPaths = + let + top = nixops4TopWith { + root = + { lib, members, ... }: + { + imports = [ nixops4AddressRoot ]; + options.seen = lib.mkOption { type = lib.types.raw; }; + config.seen = members; + }; + } { }; + seen = (nixops4RootManifest top).value.config.seen; + in + builtins.attrNames seen.address == [ + "absoluteMemberPath" + "outputs" + ] + && seen.address.absoluteMemberPath == [ "address" ] + && seen.address.outputs.stdout == "stub"; + + # A deployment that holds a resource with a state gives a program, + # built for the system of the tool that asks, that reads the + # outputs NixOps4 recorded. The program finds the resources that + # hold state from the `state` option of each resource. A deployment + # with no state gives none. + nixops4RootDiscoversRecordedOutputs = + let + stateful = nixops4TopWith { + root = + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.state = + lib.caisson.caisson-nixops4-resource.mkConfiguration + { + configModule = + { providers, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + type = providers.local.state_file; + inputs.name = "state.json"; + }; + }; + }; + resources.probe.ssh.hostPublicKey = nixops4HostKey; + wiring = members: { probe.state = members.state; }; + } { probe = minimalNixosBase; }; + root = nixops4RootManifest stateful; + discover = root.discoverArgs { system = "x86_64-linux"; }; + in + (nixops4RootManifest (nixops4TopWith { } { })).discoverArgs { system = "x86_64-linux"; } == null + && root.outputs.stateResources == [ [ "state" ] ] + && (nixops4Export stateful.nixops4).members.probe.resource.state == [ "state" ] + && discover.name == "caisson-nixops4-discover" + && discover.system == "x86_64-linux"; + + # A deferred configuration of another tool beside the root of a + # NixOps4 deployment in one flake has its own entry, publication, + # args and discovery program, and each entry finalizes without the + # other. + nixops4BesideAnotherDeferredTool = + let + top = hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + imports = [ (lib.caisson.integrations.configurationsModuleFor [ "greeter" ]) ]; + caisson.greeter.configurations.hello = greeterConfiguration lib; + caisson.caisson-nixops4-composite.configurations.nixops4 = + lib.caisson.caisson-nixops4-composite.mkConfiguration + { configModule = nixops4AddressRoot; }; + }; + }; + greeter = top.caisson.manifest.children.greeter.hello; + greeted = greeter.finalize { + args.who = "caisson"; + argsSource = _: "the probe"; + }; + in + core.isDeferred greeter + && greeter.type == "greeter" + && top.greetings.hello "world" == "hello world" + && greeted.outputs.greeting == "hello caisson" + && greeted.argsNeeded.who.cacheable + && (greeted.discoverArgs { system = "x86_64-linux"; }).name == "greeter-discover" + && (nixops4RootManifest top).discoverArgs { system = "x86_64-linux"; } == null + && (nixops4Export top.nixops4).members.address.resource.type == "exec"; + + # A provider is evaluated inside the component that declares it. + # `getProviders` evaluates the root without outputs and builds the + # providers the root declares, as NixOps4's `getProviders` does. + nixops4ProvidersBuildWithoutOutputs = + let + top = nixops4TopWith { + root.providers.local = inputs.nixops4.modules.nixops4Provider.local; + } { }; + built = top.nixops4.getProviders { system = "x86_64-linux"; }; + in + pkgs.lib.isDerivation built && pkgs.lib.hasPrefix "nixops-root-providers" built.name; + + # The upstream-compatible integration takes root modules as a + # NixOps4 user writes them, `_class` included, and NixOps4 reads + # the same export from it: a member that sets `type` is a resource, + # any other a composite, to any depth. A member reads another + # member's outputs through `members`. + nixops4UpstreamRootEvaluates = + let + top = nixops4UpstreamTopWith { } [ nixops4UpstreamRoot ]; + export = nixops4Export top.nixops4; + probe = export.members.probe.resource; + in + top.nixops4._type == "nixops4Component" + && + builtins.attrNames export.members == [ + "address" + "group" + "probe" + ] + && export.members.address.resource.type == "exec" + && builtins.attrNames export.members.group.members == [ "inner" ] + && export.members.group.members.inner.resource.inputs.executable == "/bin/false" + && builtins.elemAt probe.inputs.args 2 == "root@stub"; + + # A member that imports nixops4-nixos gets its `nixos.module` + # evaluated as a caisson NixOS configuration beneath it, with the + # `bin/apply` NixOps4 runs, and builds the system NixOS builds from + # the same module and the `apply` module nixops4-nixos adds, named + # for the member as caisson names it. The shell comes from the + # package set caisson selects. + nixops4UpstreamNixosIsACaissonConfiguration = + let + top = nixops4UpstreamTopWith { } [ nixops4UpstreamRoot ]; + probe = (nixops4Export top.nixops4).members.probe.resource; + member = (nixops4RootManifest top).children.caisson-nixops4-resource.probe; + machine = member.children.system.x86_64-linux.children.nixos.probe; + upstream = import "${inputs.nixpkgs}/nixos/lib/eval-config.nix" { + system = null; + modules = [ + minimalNixosBase + inputs.nixops4-nixos.modules.nixos.apply + { + nixpkgs.pkgs = pkgs; + networking.hostName = "probe"; + } + ]; + }; + in + machine.type == "nixos" + && probe.inputs.executable == pkgs.lib.getExe pkgs.bash + && builtins.elemAt probe.inputs.args 3 == "${machine.value.config.system.build.toplevel}" + && pkgs.lib.hasInfix "$out/bin/apply" machine.value.config.system.activatableSystemBuilderCommands + && + machine.value.config.system.build.toplevel.drvPath == upstream.config.system.build.toplevel.drvPath; + + # The shell of a nixops4-nixos member comes from the package set + # caisson selects for the nixops4 configuration, not from the + # nixpkgs of nixops4-nixos: here a selection whose `bash` is + # another shell. + nixops4UpstreamDeploysWithTheSelectedPackageSet = + let + top = hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.nixops4.configurations.nixops4 = lib.caisson.nixops4.mkConfiguration { + modules = [ nixops4UpstreamRoot ]; + defaultPkgs = pkgSets: pkgSets.default // { bash = pkgSets.default.dash; }; + }; + }; + }; + in + (nixops4Export top.nixops4).members.probe.resource.inputs.executable == pkgs.lib.getExe pkgs.dash; + + # Every upstream composite reads its members' configs through its + # full evaluation: each member records the composite as the level + # whose children it reads. + nixops4UpstreamMembersReadTheFullComposite = + let + root = nixops4RootManifest (nixops4UpstreamTopWith { } [ nixops4UpstreamRoot ]); + probe = root.children.caisson-nixops4-resource.probe; + in + probe.crossChildLevel != null + && !probe.crossChildLevel.childless + && probe.crossChildLevel.name == "nixops4"; + + # Upstream code reads a member through `members` and through + # `config.members`: its outputs, and the member itself as the + # resource that holds the state of another, which NixOps4's `state` + # option accepts by its `absoluteMemberPath`. + nixops4UpstreamReadsMembersAsUpstreamDoes = + let + top = nixops4UpstreamTopWith { } [ + ( + { + providers, + members, + config, + ... + }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members.state = { + type = providers.local.state_file; + inputs.name = "state.json"; + }; + members.address = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + members.memo = { + type = providers.local.memo; + state = members.state; + inputs.initialize_with = config.members.address.outputs.stdout; + }; + members.group.members.echo = { + type = providers.local.exec; + inputs.executable = "/bin/echo"; + inputs.args = [ members.address.outputs.stdout ]; + }; + } + ) + ]; + export = nixops4Export top.nixops4; + root = nixops4RootManifest top; + in + export.members.memo.resource.state == [ "state" ] + && export.members.memo.resource.inputs.initialize_with == "stub" + && export.members.group.members.echo.resource.inputs.args == [ "stub" ] + && root.children.caisson-nixops4-resource.memo.value.config.absoluteMemberPath == [ "memo" ] + && root.outputs.stateResources == [ [ "state" ] ]; + + # A member that sets `type` and declares members is refused, where + # NixOps4 drops the members. + nixops4UpstreamRefusesAResourceWithMembers = + let + evaluates = top: (builtins.tryEval (builtins.deepSeq (nixops4Export top.nixops4) true)).success; + rootWith = + extra: + { providers, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members.probe = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + } + // extra; + }; + in + evaluates (nixops4UpstreamTopWith { } [ (rootWith { }) ]) + && !(evaluates (nixops4UpstreamTopWith { } [ (rootWith { members.extra = { }; }) ])); + + # The allow-list of the nixops4 configuration says which outputs a + # tool may cache, by member path; every other output is not + # cacheable. + nixops4UpstreamCacheableOutputsAreAllowListed = + let + top = nixops4UpstreamTopWith { + cacheableOutputs."group.inner" = [ "stdout" ]; + } [ nixops4UpstreamRoot ]; + needed = (nixops4RootManifest top).argsNeeded; + in + needed.outputValues.group.inner.stdout.cacheable && !needed.outputValues.address.stdout.cacheable; + + # Native composites nest to any depth. Each composite finalizes its + # members with their part of its args: its member path, the + # `members` argument of each composite, the `argsSource` of each + # member, the args the root names and the export all follow the + # nesting. The `members` argument holds, for a composite member, the + # outputs of its members by name. The root collects the state + # resources of the whole tree, each once, and only the root gives a + # discovery program. + nixops4NestedCompositesFollowTheNesting = + let + top = nixops4RootTopWith ( + { lib, members, ... }: + { + imports = [ nixops4SeesMembers ]; + caisson.caisson-nixops4-resource.configurations.state = nixops4StateFile lib; + caisson.caisson-nixops4-resource.configurations.m = nixops4Exec lib { state = members.state; }; + caisson.caisson-nixops4-composite.configurations.a = nixops4Composite lib ( + { lib, members, ... }: + { + imports = [ nixops4SeesMembers ]; + caisson.caisson-nixops4-resource.configurations.s = nixops4StateFile lib; + caisson.caisson-nixops4-resource.configurations.w = nixops4Exec lib { state = members.s; }; + caisson.caisson-nixops4-resource.configurations.w2 = nixops4Exec lib { state = members.s; }; + caisson.caisson-nixops4-composite.configurations.b = nixops4Composite lib ( + { lib, ... }: + { + imports = [ nixops4SeesMembers ]; + caisson.caisson-nixops4-resource.configurations.r = nixops4Exec lib { + cacheableOutputs = [ "stdout" ]; + }; + } + ); + } + ); + } + ); + root = nixops4RootManifest top; + export = nixops4Export top.nixops4; + a = root.children.caisson-nixops4-composite.a; + b = a.children.caisson-nixops4-composite.b; + r = b.children.caisson-nixops4-resource.r; + discover = root.discoverArgs { system = "x86_64-linux"; }; + in + [ + root.outputs.path + a.outputs.path + b.outputs.path + r.value.config.absoluteMemberPath + ] == [ + [ ] + [ "a" ] + [ + "a" + "b" + ] + [ + "a" + "b" + "r" + ] + ] + && root.value.config.seen.a.absoluteMemberPath == [ "a" ] + && root.value.config.seen.a.outputs.b.r.stdout == "stub" + && root.value.config.seen.a.outputs.w.stdout == "stub" + && a.value.config.seen.b.outputs.r.stdout == "stub" + && + b.value.config.seen.r == { + absoluteMemberPath = [ + "a" + "b" + "r" + ]; + outputs.stdout = "stub"; + } + && a.argsSource [ ] == "the composite at the root" + && b.argsSource [ ] == "the composite at a" + && r.argsSource [ ] == "the composite at a.b" + && + nixops4ArgPathsOf root.argsNeeded == [ + [ + "outputValues" + "a" + "b" + "r" + "stdout" + ] + [ + "outputValues" + "a" + "w" + "stdout" + ] + [ + "outputValues" + "a" + "w2" + "stdout" + ] + [ + "outputValues" + "m" + "stdout" + ] + [ "resourceProviderSystem" ] + ] + && root.argsNeeded.outputValues.a.b.r.stdout.cacheable + && !root.argsNeeded.outputValues.a.w.stdout.cacheable + && export.members.a.members.b.members.r.resource.type == "exec" + && + export.members.a.members.w.resource.state == [ + "a" + "s" + ] + && + root.outputs.stateResources == [ + [ + "a" + "s" + ] + [ "state" ] + ] + && a.discoverArgs { system = "x86_64-linux"; } == null + && pkgs.lib.hasInfix "states=(a.s state)" discover.text; + + # A NixOS configuration declared beneath a resource inside a nested + # composite is deployed by that resource, as one beneath a resource + # of the root is. + nixops4MachineInsideANestedComposite = + let + top = nixops4RootTopWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.site = nixops4Composite lib ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.web = + lib.caisson.caisson-nixops4-resource.mkConfiguration + { + configModule = + { lib, ... }: + { + ssh.hostPublicKey = nixops4HostKey; + caisson.nixos.configurations.web = lib.caisson.nixos.mkConfiguration { + configModule = minimalNixosBase; + }; + }; + }; + } + ); + } + ); + args = (nixops4Export top.nixops4).members.site.members.web.resource.inputs.args; + web = + (nixops4RootManifest top) + .children.caisson-nixops4-composite.site.children.caisson-nixops4-resource.web; + machine = web.children.system.x86_64-linux.children.nixos.web; + in + web.value.config.absoluteMemberPath == [ + "site" + "web" + ] + && builtins.elemAt args 2 == "root@web" + && builtins.elemAt args 3 == "${machine.value.config.system.build.toplevel}"; + + # Which members a composite has, and whether each is a composite or + # a resource, may not depend on its members or on their outputs: + # the composite that reads them there is refused with an error that + # names it, in place of an infinite recursion. Reading `members` + # anywhere else stays allowed. tests/deferred-errors reads the text. + nixops4MembersDecidedFromOutputsAreRefused = + let + argsNeededOf = top: (nixops4RootManifest top).argsNeeded; + native = nixops4RootTopWith ( + { lib, members, ... }: + { + imports = [ nixops4AddressRoot ]; + caisson.caisson-nixops4-resource.configurations = lib.optionalAttrs ( + members.address.outputs.stdout == "more" + ) { extra = nixops4Exec lib { }; }; + } + ); + nested = nixops4RootTopWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.a = nixops4Composite lib ( + { lib, members, ... }: + { + caisson.caisson-nixops4-resource.configurations = lib.optionalAttrs (members ? x) { + x = nixops4Exec lib { }; + }; + } + ); + } + ); + upstreamNames = nixops4UpstreamTopWith { } [ + ( + { providers, config, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members = { + address = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + } + // (if config.members.address.outputs.stdout == "more" then { extra = { }; } else { }); + } + ) + ]; + upstreamKind = nixops4UpstreamTopWith { } [ + ( + { providers, members, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members.address = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + members.dynamic = + if members.address.outputs.stdout == "resource" then + { type = providers.local.exec; } + else + { members.inner.type = providers.local.exec; }; + } + ) + ]; + in + nixops4Fails (argsNeededOf native) + && nixops4Fails (argsNeededOf nested) + && nixops4Fails (argsNeededOf upstreamNames) + && nixops4Fails (argsNeededOf upstreamKind) + && !(nixops4Fails (argsNeededOf (nixops4TopWith { root = nixops4AddressRoot; } { }))); + + # A resource whose `outputsSkeleton` reads its own outputs is + # refused with an error that names it, in place of an infinite + # recursion. tests/deferred-errors reads the text. + nixops4OutputsNamedFromOwnOutputsAreRefused = + let + top = nixops4RootTopWith ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.selfish = nixops4Exec lib ( + { config, ... }: + { + outputsSkeleton = lib.mkForce ( + { + stdout = { }; + } + // lib.optionalAttrs (config.outputs.stdout == "more") { extra = { }; } + ); + } + ); + } + ); + in + nixops4Fails (nixops4RootManifest top).argsNeeded; + + # A name declared both as a composite and as a resource is refused, + # since NixOps4 holds one member under a name. + nixops4OneMemberUnderAName = + nixops4Fails + (nixops4RootManifest ( + nixops4RootTopWith ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.x = nixops4Exec lib { }; + caisson.caisson-nixops4-composite.configurations.x = nixops4Composite lib { }; + } + ) + )).argsNeeded; + + # A flake publishes one root as `nixops4`: two roots, declared side + # by side or in two groups, are refused, and so is a root declared + # under another name. tests/deferred-errors reads the text. + nixops4OneRootPerFlake = + let + topWith = + module: + hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = module; + }; + rootIn = + lib: + lib.caisson.structural.mkConfiguration { + configModule.caisson.caisson-nixops4-composite.configurations.nixops4 = + nixops4Composite lib nixops4AddressRoot; + }; + in + nixops4Fails + (topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.nixops4 = nixops4Composite lib nixops4AddressRoot; + caisson.nixops4.configurations.nixops4 = lib.caisson.nixops4.mkConfiguration { }; + } + )).nixops4._type + && + nixops4Fails + (topWith ( + { lib, ... }: + { + caisson.structural.configurations.one = rootIn lib; + caisson.structural.configurations.two = rootIn lib; + } + )).nixops4._type + && + nixops4Fails + (topWith ( + { lib, ... }: + { + caisson.caisson-nixops4-composite.configurations.deploy = nixops4Composite lib nixops4AddressRoot; + } + )).nixops4._type; + + # A root declared inside a NixOS configuration, which is evaluated + # per system, is published as `nixops4` where one system is in + # force. Its entry stands beneath that system. + nixops4RootInsideAMachine = + let + top = hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.nixos.configurations.host = lib.caisson.nixos.mkConfiguration { + configModule = { + imports = [ minimalNixosBase ]; + caisson.caisson-nixops4-composite.configurations.nixops4 = nixops4Composite lib nixops4AddressRoot; + }; + }; + }; + }; + entry = + top.caisson.manifest.children.system.x86_64-linux.children.nixos.host.children.caisson-nixops4-composite.nixops4; + in + core.isDeferred entry + && entry.parent.type == "nixos" + && (nixops4Export top.nixops4).members.address.resource.type == "exec"; + + # A deferred configuration whose parent does not finalize it stays + # an entry in the children of that parent, for a tool to finalize: + # a resource declared at the top, and a composite declared beneath + # a resource. Neither has a composite above it, so each is at the + # root of its member paths, and neither is published. + nixops4EntriesAParentDoesNotFinalize = + let + top = hiveLib.caisson.structural.mkTopConfiguration { + moduleImports = _modules: [ ]; + configModule = + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.lone = nixops4Exec lib { }; + caisson.caisson-nixops4-composite.configurations.nixops4 = nixops4Composite lib ( + { lib, ... }: + { + caisson.caisson-nixops4-resource.configurations.holder = nixops4Exec lib { + caisson.caisson-nixops4-composite.configurations.inner = nixops4Composite lib nixops4AddressRoot; + }; + } + ); + }; + }; + finalizeWith = + entry: outputValues: + entry.finalize { + args = { + resourceProviderSystem = "x86_64-linux"; + inherit outputValues; + }; + argsSource = _: "the probe"; + }; + lone = top.caisson.manifest.children.caisson-nixops4-resource.lone; + inner = + (nixops4RootManifest top) + .children.caisson-nixops4-resource.holder.children.caisson-nixops4-composite.inner; + loneFinalized = finalizeWith lone { stdout = "out"; }; + innerFinalized = finalizeWith inner { address.stdout = "out"; }; + in + core.isDeferred lone + && core.isDeferred inner + && loneFinalized.outputs.path == [ ] + && loneFinalized.outputs.export.resource.type == "exec" + && innerFinalized.outputs.path == [ ] + && innerFinalized.outputs.export.members.address.resource.type == "exec" + && + builtins.attrNames top == [ + "caisson" + "libOverlays" + "modules" + "nixops4" + "pkgOverlays" + ]; + + # `getProviders` finalizes the root without outputs, so a provider + # of the root configured from an output fails, as it does upstream. + # It builds the providers of the root alone, not those a composite + # beneath declares, as upstream collects them. + nixops4ProvidersOfTheRootAlone = + let + providersOf = + top: + builtins.fromJSON ( + builtins.elemAt (pkgs.lib.splitString "\n" + (top.nixops4.getProviders { system = "x86_64-linux"; }).text + ) 1 + ); + fromOutputs = nixops4RootTopWith ( + { members, ... }: + { + imports = [ nixops4AddressRoot ]; + providers.reader = { + executable = "/bin/reader"; + args = [ members.address.outputs.stdout ]; + resourceTypes = { }; + }; + } + ); + nested = nixops4RootTopWith ( + { lib, ... }: + { + providers.root = { + executable = "/bin/root"; + resourceTypes = { }; + }; + caisson.caisson-nixops4-composite.configurations.sub = nixops4Composite lib { + providers.nested = { + executable = "/bin/nested"; + resourceTypes = { }; + }; + }; + } + ); + in + nixops4Fails (providersOf fromOutputs) + && + providersOf nested == { + root = { + executable = "/bin/root"; + args = [ ]; + }; + }; + + # Upstream composites nest to any depth, as native ones do: the + # member paths, the args the root names, the allow-list by member + # path, and a member reading the output of a member two levels + # down. + nixops4UpstreamCompositesFollowTheNesting = + let + top = nixops4UpstreamTopWith { cacheableOutputs."g.h.r" = [ "stdout" ]; } [ + ( + { providers, members, ... }: + { + providers.local = inputs.nixops4.modules.nixops4Provider.local; + members.g.members.h.members.r = { + type = providers.local.exec; + inputs.executable = "/bin/true"; + }; + members.g.members.reader = { + type = providers.local.exec; + inputs.executable = "/bin/echo"; + inputs.args = [ members.g.members.h.members.r.outputs.stdout ]; + }; + } + ) + ]; + root = nixops4RootManifest top; + r = + root.children.caisson-nixops4-composite.g.children.caisson-nixops4-composite.h.children.caisson-nixops4-resource.r; + in + nixops4ArgPathsOf root.argsNeeded == [ + [ + "outputValues" + "g" + "h" + "r" + "stdout" + ] + [ + "outputValues" + "g" + "reader" + "stdout" + ] + [ "resourceProviderSystem" ] + ] + && root.argsNeeded.outputValues.g.h.r.stdout.cacheable + && !root.argsNeeded.outputValues.g.reader.stdout.cacheable + && + r.value.config.absoluteMemberPath == [ + "g" + "h" + "r" + ] + && r.crossChildLevel.name == "h" + && (nixops4Export top.nixops4).members.g.members.reader.resource.inputs.args == [ "stub" ]; + # A node of the upstream-compatible integration builds the same # system as colmena builds from the same hive. colmenaUpstreamHiveBuildsWhatColmenaBuilds = @@ -4396,6 +5678,15 @@ let in { inherit results failures; + # What the checks that build from the same world read. + fixtures = { + inherit + hiveLib + pkgs + nixops4ArgPathsOf + nixops4Entry + ; + }; ok = failures == [ ]; summary = if failures == [ ] then diff --git a/tests/unit/lib-overlays.nix b/tests/unit/lib-overlays.nix index 9c1dcdad..e36831a0 100644 --- a/tests/unit/lib-overlays.nix +++ b/tests/unit/lib-overlays.nix @@ -126,6 +126,8 @@ let "facter" "caisson-microvm" "microvm" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "terranix" "system-manager" "treefmt" @@ -5690,6 +5692,8 @@ in "facter" "caisson-microvm" "microvm" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "treefmt" "nvf" "nixvim" @@ -5709,6 +5713,8 @@ in "caisson-facter" "facter" "microvm" + "caisson-nixops4-composite" + "caisson-nixops4-resource" "treefmt" "nvf" "nixvim"