diff --git a/.site-sync/manifest.json b/.site-sync/manifest.json index ac6f13f..026617d 100644 --- a/.site-sync/manifest.json +++ b/.site-sync/manifest.json @@ -28,6 +28,7 @@ "common-operation-script/substrate/README.md": "develop-business-logic/smart-contract-development/substrate", "common-operation-script/substrate/chaincode-environment-variables.md": "deployment/chaincode-environment-variables", "common-operation-script/substrate/decimals.md": "develop-business-logic/smart-contract-development/decimals", + "common-operation-script/substrate/defflow.md": "develop-business-logic/smart-contract-development/defflow", "common-operation-script/substrate/elpspath.md": "develop-business-logic/smart-contract-development/ellipse-path", "common-operation-script/substrate/substrate-indexes.md": "develop-business-logic/smart-contract-development/data-structures", "common-operation-script/substrate/substrate-utilities.md": "develop-business-logic/smart-contract-development/common-utilities", @@ -37,6 +38,7 @@ "connectors/connectorhub.md": "luther-connectors-setup/connector-hub", "connectors/lutherauth/README.md": "luther-connectors-setup/application-module-configuration/lutherauth", "connectors/lutherauth/behind-a-cdn.md": "luther-connectors-setup/application-module-configuration/lutherauth/behind-a-cdn", + "connectors/lutherauth/csrf.md": "luther-connectors-setup/application-module-configuration/lutherauth/csrf", "connectors/lutherauth/network-access.md": "luther-connectors-setup/application-module-configuration/lutherauth/network-access", "connectors/lutherauth/service-tokens.md": "luther-connectors-setup/application-module-configuration/lutherauth/service-tokens", "connectors/lutherauth/sessions.md": "luther-connectors-setup/application-module-configuration/lutherauth/sessions", @@ -46,6 +48,8 @@ "middleware/entity-update-guide.md": "luther-connectors-setup/api-update-guide", "middleware/martin/README.md": "deployment/testing/martin-end-to-end-testing", "middleware/martin/proxy.md": "deployment/testing/martin-proxy", + "operations/cloud-cost-optimization.md": "deployment/cloud-cost-optimization", + "operations/fabric-3-upgrade.md": "deployment/fabric-3-upgrade", "operations/fabric-snapshot-archiver.md": "deployment/fabric-snapshot-archiver", "operations/peer-storage.md": "deployment/peer-storage", "release-notes.md": "deployment/release-notes", diff --git a/common-operation-script/ellipse-dev-language/function-reference.md b/common-operation-script/ellipse-dev-language/function-reference.md index f92eccc..c9cd52a 100644 --- a/common-operation-script/ellipse-dev-language/function-reference.md +++ b/common-operation-script/ellipse-dev-language/function-reference.md @@ -1,12 +1,12 @@ --- -description: Every ELPS builtin, macro and library function in elps v1.95.0, from its docstring. +description: Every ELPS builtin, macro and library function in elps v1.98.0, from its docstring. --- # Function Reference - + -This page lists every function, macro, special operator and variable of ELPS v1.95.0, the release the examples on this site run. It is generated from the docstrings in [luthersystems/elps](https://github.com/luthersystems/elps/tree/v1.95.0), the same text `elps doc ` prints. For examples and guidance, see the [Standard Library](standard-library.md) and the [Language Guide](language-guide.md). +This page lists every function, macro, special operator and variable of ELPS v1.98.0, the release the examples on this site run. It is generated from the docstrings in [luthersystems/elps](https://github.com/luthersystems/elps/tree/v1.98.0), the same text `elps doc ` prints. For examples and guidance, see the [Standard Library](standard-library.md) and the [Language Guide](language-guide.md). Functions of the `lisp` package need no prefix. Call a function of another package with its package name, such as `string:join`. @@ -56,7 +56,7 @@ Paths are given as positional ELPS values — one argument per path step, no str ? reads without copying. ?set, ?del and ?nil copy the document's maps, lists and vectors; their "!" forms mutate in place. In-place edits of list elements are refused; use a copying form or a vector instead. An out-of-range integer index leaves the document unchanged on a write (and returns nil on a read). Negative indexes count from the end. -Tagged values and quote wrappers are rebuilt recursively by copying writes and traversed by cycle checks, but cannot be indexed into. Other value types are opaque leaves, including symbols, bytes, functions and natives. Reads may return them and writes may store them. A further path step into a leaf raises an error naming its type and location. Copies share opaque leaves by reference; mutation of their storage outside elpspath is visible through both documents. Containers must be acyclic, and arrays must have exactly one dimension; every operation checks this throughout the document, including off-path containers and the contents of tagged values and quote wrappers. New values pass the same container check. Function bodies and captured environments remain opaque and shared; they are not walked. +Tagged values and quote wrappers are rebuilt recursively by copying writes and traversed by cycle checks, but cannot be indexed into. Other value types are opaque leaves, including symbols, bytes, functions and natives. Reads may return them and writes may store them. A further path step into a leaf raises an error naming its type and location. Copies share opaque leaves by reference; mutation of their storage outside elpspath is visible through both documents. Writes need containers that are acyclic and arrays with exactly one dimension, and check this throughout the document, including off-path containers and the contents of tagged values and quote wrappers, because a write may copy or rebuild any of them. New values pass the same check. A read (?) with a path checks only what that path steps into: it never walks beyond its path or into the value it returns, so a fault elsewhere cannot change its answer, and its cost follows the path rather than the document. A read with no path checks the whole document, as writes do. Function bodies and captured environments remain opaque and shared; they are not walked. An iterator retains its per-element error handling: reads substitute nil for a failed element path, and writes leave that element unchanged. @@ -106,7 +106,9 @@ Delete value at a path specified by positional args, mutating the original. Out-of-range integer indexes are no-ops. List element mutation is refused. -(?del! obj "foo") => obj with foo removed (mutated) (?del! obj "items" 1) => obj with items\[1\] removed (mutated) (?del! records '\* "cache") => remove cache key from all elements +(?del! obj "foo") => obj with foo removed (mutated) (?del! obj "items" 1) => obj with items\[1\] removed (mutated) (?del! records '\* "cache") => remove cache key from all elements (?del! scores '\*) => every element of scores is nil + +A path that ends in '\* stores nil in each element and keeps the length, as ?del does. The sequence must be a vector. Accepts at most 1024 nested iterator steps; deeper traversal raises a depth error. @@ -132,7 +134,9 @@ Set value at a path to nil, mutating the original. The key is kept. Out-of-range integer indexes are no-ops. List element mutation is refused. -(?nil! obj "foo") => obj with foo=nil (mutated) (?nil! rows '\* "cached") => nil out cached on all elements +(?nil! obj "foo") => obj with foo=nil (mutated) (?nil! rows '\* "cached") => nil out cached on all elements (?nil! scores '\*) => every element of scores is nil + +A path that ends in '\* stores nil in each element. The sequence must be a vector. Accepts at most 1024 nested iterator steps; deeper traversal raises a depth error. @@ -162,7 +166,9 @@ The last argument is the new value; all preceding arguments are path steps. Retu The new value is stored BY REFERENCE, not copied: after the call it is reachable and mutable through the result, and a later write through either name is visible through the other. -(?set! obj "foo" "bar" "new") => obj with foo.bar="new" (mutated) (?set! obj "items" 0 "x") => obj with items\[0\]="x" (mutated) (?set! data '\* "active" true) => set active=true on all elements +(?set! obj "foo" "bar" "new") => obj with foo.bar="new" (mutated) (?set! obj "items" 0 "x") => obj with items\[0\]="x" (mutated) (?set! data '\* "active" true) => set active=true on all elements (?set! scores '\* 0) => every element of scores is 0 + +A path that ends in '\* stores the value in each element. The sequence must be a vector. Accepts at most 1024 nested iterator steps; deeper traversal raises a depth error. @@ -203,7 +209,7 @@ A selector may not SPAN LINES, and that raises for the same reason: The jq-string grammar this shares cuts a bracket-led selector at its first newline and DISCARDS the rest, which would convert the selector above to the single step 0 -- a live path to the whole element. (apply ?set ...) through it would overwrite the record instead of its "password" field, silently. -Converting on each call is slower than one string-path operation, since it parses AND builds a list. How much the caching is worth depends on document size -- every operation first walks the whole document to validate it, so on a large document the parse is noise and on a small one it dominates. +Converting on each call is slower than one string-path operation, since it parses AND builds a list. How much the caching is worth depends on the operation: a write first walks the whole document to validate it, so on a large document the parse is noise, but a read (?) walks only its path, so for reads the parse can dominate at any document size. ## `golang` @@ -833,9 +839,9 @@ Returns the value associated with key in a sorted-map, or nil if the key is not ### `get-default` -*macro*: `(get-default map key default)` +*operator*: `(get-default map key default)` -Looks up key in a sorted-map, returning the associated value if found. If the key is not present or map is nil, evaluates and returns default. String and symbol keys with the same name are interchangeable, including in JSON-decoded maps. The default expression is evaluated only when needed (lazy evaluation). +Looks up key in a sorted-map, returning the associated value if found. If the key is not present or map is nil, evaluates and returns default. String and symbol keys with the same name are interchangeable, including in JSON-decoded maps. The default expression is evaluated only when needed (lazy evaluation), in tail position. The map and key expressions are evaluated once each, in that order. ### `handler-bind` @@ -977,9 +983,15 @@ Performs a single macro expansion step on quoted-form and returns the result. Us ### `macroexpand-all` -*function*: `(macroexpand-all quoted-form)` +*function*: `(macroexpand-all quoted-form &key lower)` + +Expands every macro call in quoted-form, not just the head: the head is expanded until it is no longer a macro, then each nested form in code position is expanded the same way. Knows the binding shape of every special form, so a local variable or function that shadows a macro name is left alone, macrolet macros are expanded, and quoted data (quote, the non-unquoted parts of quasiquote, condition types) is never entered. defun and defmacro are kept as written with their bodies expanded. The input is not modified. Nesting deeper than the evaluator's nesting limit, or a head that keeps expanding past the macro expansion limit, is an error. With :lower true, get-default (today the only special operator with a lowering) is expanded as the macro it replaced expanded it, for tools that model the macro's expansion and not the operator. + +### `macroexpand-all-lowered` + +*function*: `(macroexpand-all-lowered quoted-form)` -Expands every macro call in quoted-form, not just the head: the head is expanded until it is no longer a macro, then each nested form in code position is expanded the same way. Knows the binding shape of every special form, so a local variable or function that shadows a macro name is left alone, macrolet macros are expanded, and quoted data (quote, the non-unquoted parts of quasiquote, condition types) is never entered. defun and defmacro are kept as written with their bodies expanded. The input is not modified. Nesting deeper than the evaluator's nesting limit, or a head that keeps expanding past the macro expansion limit, is an error. +Is (macroexpand-all quoted-form :lower true) with one argument, so a caller that counts evaluation steps pays the same as for macroexpand-all without :lower. ### `macrolet` @@ -1263,7 +1275,7 @@ Converts value to a bytes value. Accepts strings (encoded as UTF-8) and bytes (r *function*: `(to-float value)` -Converts value to a float. Accepts strings (parsed), floats (returned as-is), and integers (widened). +Converts value to a float. Accepts floats (returned as-is), integers (widened) and strings. A string must be a decimal number: an optional sign, digits with an optional point, and an optional exponent, as in "-12.75", ".5" or "1e5". The strings "nan", "inf" and "infinity", hexadecimal floats and digits separated by "\_" are errors, as is text with spaces. A number too large for a float is an error. A number too small for one converts to zero. ### `to-int` @@ -1517,7 +1529,7 @@ Compiles a regular expression pattern string and returns a native regexp object. *function*: `(regexp:regexp-match? re text)` -Returns true if the regexp matches text. re may be a compiled regexp or a pattern string (compiled on each call). text may be a string or bytes value. +Returns true if the regexp matches text. re may be a compiled regexp or a pattern string (compiled on each call). text may be a string or bytes value. A match costs one step for each KiB of text, times one plus the size of the compiled program in units of 256 instructions. ### `regexp:regexp-pattern` @@ -2085,7 +2097,7 @@ Returns a new time value equal to datetime plus duration. The first argument mus *function*: `(time:time-from start end)` -Returns the duration between start and end as a native duration value (end - start). Both arguments must be native time values. The result can be negative if end is before start. +Returns the duration between start and end as a native duration value (end - start). Both arguments must be native time values. The result can be negative if end is before start. A duration holds at most about 292 years in either direction. Returns an error if the difference is larger. ### `time:time<` diff --git a/common-operation-script/substrate/defflow.md b/common-operation-script/substrate/defflow.md new file mode 100644 index 0000000..5e3fb85 --- /dev/null +++ b/common-operation-script/substrate/defflow.md @@ -0,0 +1,344 @@ +--- +description: (Experimental) Write a business process that waits for inputs, connector replies and other flows across many transactions as one straight-line defflow body. +--- + +# Durable Flows (defflow) + +> **Experimental.** `defflow` is an experimental feature. Its forms, options, +> endpoints and stored records can change in a later substrate release, and an +> upgrade can need changes to your flows. Test each substrate upgrade with your +> flows before you upgrade production, and read the +> [release notes](../../release-notes.md) for each release. Contact Luther +> before you use a flow in production. + +`defflow` defines a durable flow: a process that waits for inputs, connector +replies or other flows across many transactions. You write the body as +straight-line code. Substrate splits the body at each wait (an await). Each +block between two awaits runs in its own transaction, and the flow saves its +local variables on the ledger between blocks. + +Use a flow when one business process spans more than one transaction. Without +`defflow`, you must write one endpoint per step and keep the state of the +process in your own records. With `defflow`, substrate generates the endpoints, +stores the state, checks who may send each input, and handles timeouts and +retries. + +`defflow` is in substrate v2.252.0 and later. A libschema validator as +`:schema` or `:result-schema` needs v2.253.0. `let` patterns and `match` +around an await need v2.254.0. + +| Use a flow for | Example | +|---|---| +| A wait for a person | An underwriter approves a loan | +| A wait for an external system | A credit score from a connector | +| A deadline | Escalate if no decision arrives in 72 hours | +| A process that starts other processes | A claim starts one payment flow per payee | + +## A first flow + +This flow gets a credit score from a connector. If the score is low, it waits +for an underwriter's decision. + +```lisp +(in-package 'loanflow) +(use-package 'router) +(use-package 'defflow) + +(defflow loan (ctx app) + (:connector-msp "Org1MSP" :start-from ("BankMSP") :admin-from ("SchedulerMSP") + :connector-timeout "1h") + (let ([score (get (await-connector 'credit-score (credit-event app) :retries 2) + "score")]) + (when (< score 650) + (let ([d (await-input 'underwriter :from ("UnderwriterMSP") + :timeout "72h")]) + (unless (equal? (get d "decision") "approve") + (done (sorted-map "decision" "denied"))))) + (sorted-map "decision" "disbursed"))) +``` + +The form is `(defflow NAME (CTX PARAMS...) (OPTIONS...) BODY...)`. + +- `CTX` names the flow context. Read it with the `flow:` accessors (see + [Flow context](#flow-context)). Do not save it, return it or assign it. +- `PARAMS` are the parameters of the start endpoint. +- The value of the last form is the result of the run. + +Without retries or timeouts, this flow runs in up to three transactions: + +1. `loan_start` (from `BankMSP`) runs to the first await and sends the + credit score request to the connector. +2. The connector reply runs the block that checks the score. A low score + waits at `underwriter`. +3. `loan_underwriter` (from `UnderwriterMSP`) runs the last block, which ends + the run. + +Each connector retry and each expired deadline adds one transaction. + +## A full example + +The [sandbox](https://github.com/luthersystems/sandbox) reference application +runs an insurance claim as a flow. The claim waits for a claimant input, then +for an identity check and a policy lookup through ConnectorHub. + +| File | Content | +|---|---| +| [`phylum/claim.lisp`](https://github.com/luthersystems/sandbox/blob/main/phylum/claim.lisp) | The claim flow | +| [`phylum/claim_test.lisp`](https://github.com/luthersystems/sandbox/blob/main/phylum/claim_test.lisp) | Tests that use the `flow-test` package | +| [`phylum/flows/claim.mmd`](https://github.com/luthersystems/sandbox/blob/main/phylum/flows/claim.mmd) | The flow's state graph, drawn by `shirotester flow-graph` | +| [`phylum/README.md`](https://github.com/luthersystems/sandbox/blob/main/phylum/README.md#the-claim-flow) | A walk through the flow and its endpoints | + +## Awaits + +Each await has a quoted label. A label must be unique in its flow. The run +stores the label it waits at. + +| Form | Waits for | Main options | +|---|---|---| +| `(await-input 'L)` | A call to the flow's `L` input endpoint | `:from` (required: the MSPs that may send the input, for example `("A" "B")`), `:allow` (a predicate for more checks), `:key` (a lookup key), `:schema` (JSON Schema or a [libschema validator](../ellipse-dev-language/schema.md)), `:timeout` | +| `(await-connector 'L EVENT)` | The ConnectorHub reply to `EVENT` | `:retries`, `:timeout` | +| `(await-flow 'L FLOW INPUT)` | A child run of `FLOW` to end | `:timeout` | + +The value of an await is the input, the connector reply or the child's result. + +`:timeout` is a Go duration string, for example `"90s"`, `"1h30m"` or +`"120h"`. Every `await-connector` needs its own `:timeout` or the flow's +`:connector-timeout`. A connector reply is not delivered again if its +transaction fails, so a connector wait without a deadline could wait forever. + +### Errors at an await + +Add `:on-error HANDLER` to an await to handle its error. The handler gets a map +with `"kind"`, `"message"` and `"data"`. The value of the handler is the value +of the await. + +| Kind | Cause | +|---|---| +| `connector-error` | The connector failed, after all `:retries` | +| `timeout` | The await's deadline passed | +| `child-failed` | A child run failed | +| `cancelled` | An admin cancelled the run | + +Without `:on-error`, the error fails the run. You can also catch the error +with `handler-bind` or `ignore-errors` around the await. + +```lisp +(await-connector 'payment (payment-event app) + :timeout "10m" + :on-error (lambda (e) + (if (equal? (get e "kind") "timeout") + (await-connector 'status (status-event app) :timeout "10m") + (fail (get e "message"))))) +``` + +An inline `:on-error` lambda can await, call `done` and call `fail`. It is the +only `lambda` that can await. Its labels get the await's label as a prefix, so +the `status` await above is `payment/status`. + +### Other body forms + +| Form | Effect | +|---|---| +| `(done VALUE)` | Ends the run in state `done`. A map value is the result | +| `(fail REASON)` | Ends the run in state `failed` | +| `(start-flow 'L FLOW INPUT)` | Starts a detached run of `FLOW` and returns its run ID. The parent does not wait for it | + +A run is in one of three states: `waiting`, `done` or `failed`. + +`while` loops, `let` and `let*` patterns +([Destructuring in let](../ellipse-dev-language/let-destructuring.md)), +`match` ([Pattern Matching](../ellipse-dev-language/pattern-matching.md)), +`handler-bind`, `ignore-errors`, `flet` and `labels` work across awaits. An +await inside `dotimes`, `with-cleanup` or a `lambda` (other than an inline +`:on-error` handler) is a compile error. Use +`while` for a loop that awaits. + +## Values across an await + +A run saves only its local variables. Everything else comes from the phylum +again when the run resumes. + +- A write to a global variable in a flow body is lost at the next await. Keep + the state of the process in locals. +- The saved locals of one run must encode to 16 KiB or less. A larger set + fails the block and names the largest locals. Keep large data in your own + records and save only the keys in the flow. +- A per-transaction value, such as `(flow:now ctx)` or `(flow:sender ctx)`, + changes after each await. Read it again after the await. Do not keep it in a + local across an await. + +## Flow context + +Read the flow context with these accessors from package `flow`. + +| Accessor | Value | Same in every block | +|---|---|---| +| `(flow:run-id ctx)` | Run ID | Yes | +| `(flow:name ctx)` | Flow name | Yes | +| `(flow:caller ctx)` | MSP that started the run | Yes | +| `(flow:started-at ctx)` | Start time, RFC 3339 | Yes | +| `(flow:parent ctx)` | Parent run ID, or nil | Yes | +| `(flow:now ctx)` | This transaction's timestamp | No | +| `(flow:sender ctx)` | MSP that sent this transaction | No | +| `(flow:step ctx)` | The label this block resumed at | No | +| `(flow:step-key ctx)` | The idempotency key of the next await | No | + +`(flow:step-key ctx)` gives a key that is the same when a transaction is +retried. Put it in a connector event, so that the external system can drop a +duplicate request. + +## Options + +| Option | Meaning | +|---|---| +| `:start-from` | The MSPs that may start a run, for example `("BankMSP")` | +| `:public-start` | `true`: any MSP may start a run | +| `:child-of` | Only a run of the named parent flow may start this flow | +| `:connector-msp` | The MSP that must send connector replies. Required when the flow has an `await-connector` | +| `:connector-timeout` | The deadline of each `await-connector` without its own `:timeout` | +| `:admin-from` | The MSPs that may call the admin endpoints and fire deadlines. Required when any await has a deadline | +| `:get-from` | The MSPs that may read a run. The default is `:start-from` and each literal `:from` MSP. Required when there is no default: with `:child-of`, a computed `:from` or an `:allow` | +| `:store` | `:public`, or a private data collection name. The default is the `private` collection | +| `:start-idempotent` | `true` (the default): the start endpoint takes a required `request_key` first. `false`: the key is an optional last parameter | +| `:audit-identity` | `true`: each history row records the sender MSP | +| `:endpoints` | Renames endpoints, for example `(:start "create_claim" :get "get_claim" claimant "add_claimant")`. `nil` in place of a name drops that endpoint | +| `:expose` | Saved locals that `_get` returns under `"vars"` | +| `:result-schema` | A JSON Schema or libschema validator that the result must match | +| `:version` | The flow's version. Raise it when the flow's awaits, labels or saved locals change (see [Upgrades](#upgrades)) | + +Use exactly one of `:start-from`, `:public-start` and `:child-of`. + +## Endpoints + +Substrate registers these endpoints for each flow. The names replace `-` and +`/` with `_`. The `:endpoints` option can rename them. + +| Endpoint | Parameters | Caller | +|---|---|---| +| `NAME_start` | `request_key PARAMS...` | `:start-from` | +| `NAME_get` | `run_id` | `:get-from` | +| `NAME_history` | `run_id &optional bookmark` | `:get-from` | +| `NAME_L` (one per `await-input`) | `run_id input gen &optional request_key` | The await's `:from` | +| `NAME_L_by_key` (with `:key`) | `key input gen &optional request_key` | The await's `:from` | +| `NAME_expire` | `run_id gen` | `:admin-from` | +| `NAME_cancel` | `run_id gen &optional reason` | `:admin-from` | +| `NAME_fail` | `run_id gen &optional reason` | `:admin-from` | + +- A start runs once for each caller and `request_key`. A retry with the same + key and parameters returns the first run. +- `gen` is the run's wait generation. Each endpoint returns it. An input names + the generation it is for, so an input for an old wait is refused as + `stale-generation`. +- Start, input and admin calls return only `run_id`, `flow`, `state`, `gen` + and `label`. Read the result and the exposed locals with `NAME_get`. + +A refused call answers with an error code: `forbidden`, `bad-request`, +`not-found`, `not-waiting`, `stale-generation`, `deadline-passed`, `not-due`, +`start-rejected`, `request-key-reused` or `schema`. + +## Deadlines and the scheduler + +A deadline does not fire by itself. A scheduler from an `:admin-from` MSP must +query `$flow_tick` for the due items and call `NAME_expire` for each one. +ConnectorHub (v0.0.26 or later) has this scheduler. Without ConnectorHub, see +[Use defflow without ConnectorHub](#use-defflow-without-connectorhub). Turn it on in the +ConnectorHub configuration: + +```yaml +scheduler: + interval: 30s +``` + +Add ConnectorHub's MSP to each flow's `:admin-from`, and set `interval` +shorter than your shortest flow timeout. Configure the scheduler on one +ConnectorHub per network, or turn on leader election. See +[Connector Hub](../../connectors/connectorhub.md). + +## Use defflow without ConnectorHub + +A flow needs ConnectorHub only for `await-connector`. A flow that uses only +`await-input`, `await-flow` and `start-flow` runs without ConnectorHub. Your +own service (for example the oracle) calls the start and input endpoints. + +| Feature | Needs ConnectorHub | +|---|---| +| `await-input`, `await-flow`, `start-flow`, `done`, `fail` | No | +| `await-connector` and `:connector-msp` | Yes | +| `:timeout` on an await | No. A scheduler must fire the deadlines (see below) | + +A deadline needs a scheduler. Without ConnectorHub, your own service can be the +scheduler. It must use an identity of an `:admin-from` MSP, and at a fixed +interval: + +1. Query `$flow_tick`. Pass `max_items` (1 to 100, default 10) as transient + data. The answer lists the due items as + `{"due" [{"flow" "run_id" "gen" "kind" "due_at"} ...] "count" N "next" CURSOR}`. +2. For each item, call `NAME_expire` with `run_id` and `gen`, each in its own + transaction. +3. If `next` is set, query again with `next` as the transient `after`. + +Run only one scheduler for each network. Two schedulers expire the same runs, +and the duplicate calls fail. + +## Child flows + +`await-flow` starts a child run and waits for it to end. `start-flow` starts a +child run and does not wait. + +- A child flow must have `:child-of` set to the parent flow, or + `:public-start true`. A flow with `:child-of` also needs `:get-from`. +- Runs nest at most 4 deep. One transaction starts at most 8 child runs. +- A parent that ends, or leaves an `await-flow`, cancels the child it waits + for. A `start-flow` child is detached: it keeps running when the parent ends. + +## Upgrades + +A run stays on the phylum version it started on, until it ends. A new phylum +version does not change the code of a running run, so a new version can remove +or change any label while runs wait there. + +To move waiting runs to a new version, add a `defflow-migration` form to the +new phylum. It names the flow versions and, for each label, how to compute the +new version's locals from the old ones: + +```lisp +(defflow-migration loan :from "1" :to "2" + (at 'approval :when (< amount 1000000) + (let* ([amount-cents (* amount 100)] + [risk-score (score-risk applicant)])))) +``` + +In this example, version 1 saved the locals `amount` and `applicant` at the +`approval` await. Version 2 needs `amount-cents` and `risk-score`. A run whose +`:when` test is false stays on version 1. + +Only the runs that wait at a label that an `at` clause lists move. The runs +move in bulk when you run `shiroclient init` for the new version. With +`:lazy true` on the form, a run also moves when an event reaches it. Locals that +no clause binds keep their values by name. Test a migration with the +`flow-test` package before you deploy it. + +A flow's manifest (its labels, awaits, saved locals and options) is fixed for +each `:version`. If you change the manifest, raise `:version`. An install that +changes the manifest of a stored version is refused. + +You cannot disable a phylum version while a run waits on it. + +## Test a flow + +| Tool | Use | +|---|---| +| `flow-test` package | Test helpers for `*_test.lisp`: `start`, `input`, `reply`, `tick`, `advance`, `run`, `history` and more. Each call runs as its own simulated transaction | +| `shirotester flow-test --phylum DIR SCENARIO.yaml` | Runs YAML scenarios against the phylum in memory | +| `shirotester flow-graph --format mermaid` | Draws each flow's state graph | +| `shirotester lint` | Reports flow errors, for example a connector await without a deadline or a write to a global in a flow body | + +## Before you deploy + +- Flow records, step counts and endpoint answers are part of consensus. + Upgrade every peer of a channel together before you install a phylum with a + flow. +- Give the `:connector-msp` identity to ConnectorHub only. Any identity of that + MSP can answer a connector request. +- Do not call flow endpoints through another chaincode + (`cc:invoke-chaincode`). Call them directly. diff --git a/connectors/lutherauth/README.md b/connectors/lutherauth/README.md index 26c2871..4991609 100644 --- a/connectors/lutherauth/README.md +++ b/connectors/lutherauth/README.md @@ -6,7 +6,7 @@ description: The Luther authentication service, its service tokens, token checks LutherAuth (the `luthersystems/auth` image) signs in users and services, and issues the Luther access tokens (JWTs) that Luther services check. It signs in users with OAuth 2.0 and OpenID Connect identity providers (IdPs) such as Auth0, Amazon Cognito, Azure AD B2C or another OpenID Connect provider, and with magic links sent by email. It signs in services with the OAuth 2.0 client credentials grant. -The current release is LutherAuth v4.13.0 ([`luthersystems/auth:v4.13.0`](https://hub.docker.com/r/luthersystems/auth/tags) on Docker Hub). Go services check its tokens with [lutherauth-sdk-go](https://github.com/luthersystems/lutherauth-sdk-go) ([v0.0.12](https://github.com/luthersystems/lutherauth-sdk-go/releases/tag/v0.0.12)). +The current release is LutherAuth v4.15.0 ([`luthersystems/auth:v4.15.0`](https://hub.docker.com/r/luthersystems/auth/tags) on Docker Hub). Go services check its tokens with [lutherauth-sdk-go](https://github.com/luthersystems/lutherauth-sdk-go) ([v0.0.12](https://github.com/luthersystems/lutherauth-sdk-go/releases/tag/v0.0.12)). | Task | Page | |---|---| @@ -16,6 +16,7 @@ The current release is LutherAuth v4.13.0 ([`luthersystems/auth:v4.13.0`](https: | Upgrade an existing deployment from v4.7 or later, and stop using the fake IdP | [Upgrade LutherAuth](upgrade-v4-9-0.md) | | Set login and session lifetimes, sign users out at LutherAuth and at the IdP, and end every session of a user | [Sessions, Sign-Out and Revocation](sessions.md) | | Put LutherAuth behind a CDN such as CloudFront | [Behind a CDN](behind-a-cdn.md) | +| Protect cookie requests against CSRF, and keep the access token in an HttpOnly cookie | [CSRF Protection and Cookie Mode](csrf.md) | Every setting on these pages is an environment variable: `AUTH_` followed by the setting name in upper snake case. For example, `enable-client-credentials` is `AUTH_ENABLE_CLIENT_CREDENTIALS`. diff --git a/connectors/lutherauth/behind-a-cdn.md b/connectors/lutherauth/behind-a-cdn.md index b414231..1c4c3b9 100644 --- a/connectors/lutherauth/behind-a-cdn.md +++ b/connectors/lutherauth/behind-a-cdn.md @@ -26,12 +26,20 @@ LutherAuth sends `Cache-Control: no-store` on every response from v4.12.0. The e | Error caching | `0` seconds for every 4xx and 5xx code (CloudFront: a custom error response with `error_caching_min_ttl = 0`) | | Cookies | Forward all cookies to the origin | | Query strings | Forward all. `oauth_login` reads `oauth_callback_url` and `frontend_redirect_url` | -| Headers | Forward `Authorization`, `Origin`, `Content-Type`, `X-CSRF-Token` and `Sec-Fetch-Mode` | +| Headers | Forward `Authorization`, `Origin`, `Content-Type`, `X-CSRF-Token` and `Sec-Fetch-Mode`. Also forward `X-CSRF-Token` on the routes of each service that checks it. See [CSRF Protection and Cookie Mode](csrf.md) | | `Set-Cookie` from the origin | Pass it through unchanged | | Methods | Allow `GET`, `POST` and `OPTIONS` | Apply the same rule to any API behind the same CDN that returns data for one user or sets cookies. Cache a response only when the origin sends a `Cache-Control` header that allows it. With CloudFront, a default TTL above 0 caches every response that has no `Cache-Control` header. +### Headers on CloudFront + +CloudFront sends the origin only the headers that its cache policy or origin request policy lists. It drops every other header with no error. + +- A dropped header fails as an authentication error at the service. If CloudFront drops `X-CSRF-Token`, a service with the CSRF check on refuses every cookie request. +- Each header in a cache policy is part of the cache key. This is not a guard for cookie requests. A `GET` with the access token in a cookie carries neither `Authorization` nor `X-CSRF-Token`, so two users can get the same cache key. Do not cache a route that needs authentication, and never cache `/v1/auth/*`. +- A cache policy takes at most 10 headers (a CloudFront quota). Before you add a header to a full policy, get a quota increase or move a header to an origin request policy. Otherwise the change fails with `TooManyHeadersInCachePolicy`. + --- ## Check the CDN diff --git a/connectors/lutherauth/csrf.md b/connectors/lutherauth/csrf.md new file mode 100644 index 0000000..8b687a2 --- /dev/null +++ b/connectors/lutherauth/csrf.md @@ -0,0 +1,196 @@ +--- +description: Turn on the LutherAuth CSRF token cookie, send it from the frontend, check it in your services, and keep the access token in an HttpOnly cookie. +--- + +# CSRF Protection and Cookie Mode + +A browser sends cookies with every request to their host, also a request that another site starts. If your services accept the LutherAuth access token from a cookie, protect them against cross-site request forgery (CSRF). LutherAuth supports the double-submit pattern: + +1. At login, LutherAuth sets a CSRF token cookie that scripts on your page can read. +2. Your frontend reads the cookie and sends its value in the `X-CSRF-Token` header. +3. Your service compares the header with the `nonce` claim of the access token. + +A page on another site cannot read your cookies, so it cannot set the header. LutherAuth sets the cookie. It does not check the header. The check is the job of each service. + +Use LutherAuth v4.15.0 or later with refresh tokens. In earlier versions, the CSRF token cookie expired after the access token lifetime (30 minutes by default in refresh mode). After that, every checked call failed. + +--- + +## Settings + +| Setting | Default | Effect | +|---|---|---| +| `AUTH_OAUTH_PROTECT_CSRF` | `false` | `true` makes each OAuth login set the CSRF token cookie. With `AUTH_ENABLE_REFRESH=true`, each `POST /v1/auth/refresh` sets it again | +| `AUTH_JWT_REFRESH_COOKIE_ACCESS_TOKEN` | `false` | `true` turns on cookie mode: `POST /v1/auth/refresh` puts the access token in an HttpOnly cookie, not in the response body. See [Cookie mode](#cookie-mode) | +| `AUTH_COOKIE_PREFIX` | empty | A prefix for every LutherAuth cookie name. With `ui`, the CSRF token cookie is `ui_csrf_token` | +| `AUTH_SECURE_COOKIES` | `true` | Sends the cookies over HTTPS only. Keep it on outside local development | + +--- + +## Cookies + +| Cookie | Set by | Attributes | Lifetime | +|---|---|---|---| +| `_csrf_token` (`csrf_token` with no prefix) | Each OAuth login, and each refresh in refresh mode | Not HttpOnly, so your scripts can read it. `SameSite=Strict`, `Path=/`, `Secure` with `AUTH_SECURE_COOKIES`, no `Domain` | Refresh mode: the refresh session (`AUTH_REFRESH_TIMEOUT`, 120h by default). Each refresh sets it again. Legacy login: `AUTH_JWT_TIMEOUT` | +| `_authorization` | The legacy cookie login (`/v1/auth/oauth_return`), a magic link login with refresh off, and each refresh in cookie mode | HttpOnly, `SameSite=Strict`, `Path=/`, `Secure` with `AUTH_SECURE_COOKIES` | Cookie mode: `AUTH_JWT_REFRESH_INTERVAL` (30m by default). Legacy login and magic link: `AUTH_JWT_TIMEOUT` | + +The value of the CSRF token cookie is equal to the `nonce` claim of every access token of that login. A refreshed access token keeps the `nonce` of its login, so the cookie stays valid for the whole session. + +The cookies have no `Domain` attribute. Only pages on the same host as LutherAuth can read the CSRF token cookie. Serve your frontend and LutherAuth from one host, for example behind one CDN. + +A successful sign-out (`POST /v1/auth/oauth_delete`) deletes the CSRF token cookie with the other LutherAuth cookies. + +### Magic link logins + +A magic link login sets no CSRF token cookie. In refresh mode, the first `POST /v1/auth/refresh` after the login sets it. Have the frontend refresh before its first state-changing call. With refresh off, a magic link login sets the access token in the `_authorization` cookie, but no CSRF token cookie. Its access token has a `nonce` claim, so a service that checks the header refuses its requests. Use refresh mode for magic link logins before you turn on the check. + +### Tokens without a nonce + +[Service tokens](service-tokens.md) (client credentials and minted tokens) have no `nonce` claim. The check does not apply to them. + +### Callers that send a login token as a bearer token + +The check applies to every token with a `nonce` claim, also one sent in the `Authorization: Bearer` header. A script, a test or a backend that sends a user's login token as a bearer token must also send `X-CSRF-Token` with the token's `nonce` claim. A non-browser caller should use a [service token](service-tokens.md) instead. + +--- + +## What the frontend sends + +On each state-changing request (`POST`, `PUT`, `PATCH`, `DELETE`), read the CSRF token cookie and send its value in the `X-CSRF-Token` header: + +```js +function csrfToken(prefix) { + const name = (prefix ? prefix + "_" : "") + "csrf_token="; + const c = document.cookie.split("; ").find((v) => v.startsWith(name)); + return c ? decodeURIComponent(c.slice(name.length)) : ""; +} + +await fetch("/v1/my-service/orders", { + method: "POST", + credentials: "same-origin", + headers: { + "Content-Type": "application/json", + "X-CSRF-Token": csrfToken("ui"), + }, + body: JSON.stringify(order), +}); +``` + +Read the cookie for each request. Do not keep a copy. Each login and each refresh can set a new value. + +--- + +## What the service checks + +For each state-changing request that carries a LutherAuth access token with a `nonce` claim: + +1. Check the access token as usual (signature, issuer, audience, expiry). +2. Read the `X-CSRF-Token` header. +3. Compare it with the `nonce` claim in constant time. +4. Refuse the request if the header is missing or different. + +A Go service that uses [lutherauth-sdk-go](https://github.com/luthersystems/lutherauth-sdk-go) passes a CSRF getter as the second argument of `claims.NewJWKClaims` (the `nil` in [Verify Tokens in Go](verify-tokens.md#set-the-policy)). + +The first argument gets the access token. `claims.NewAuthTokenGetter()` reads only the `Authorization: Bearer` header. To accept the access token from the cookie (the legacy login, a magic link with refresh off, or cookie mode), use `claims.NewCookieTokenGetter`, or a getter that tries both: + +```go +bearerToken := claims.NewAuthTokenGetter() +cookieToken := claims.NewCookieTokenGetter("ui_authorization", false) // _authorization +authToken := func(ctx context.Context) (string, error) { + if t, err := bearerToken(ctx); err == nil { + return t, nil + } + return cookieToken(ctx) +} + +claimGetter := claims.NewJWKClaims( + authToken, + claims.NewRawTokenGetter("X-CSRF-Token", false), // the CSRF getter + logger, + jwk.WithIssuerPolicy(policy), +) +``` + +With a CSRF getter, the SDK compares the header with the `nonce` claim in constant time. A missing or different header gets gRPC `Unauthenticated`. A token without a `nonce` claim is not checked. + +The SDK reads the cookie and the header from the incoming gRPC metadata, under the keys `cookie` and `x-csrf-token`. grpc-gateway does not put them there by default: its default matcher drops `X-CSRF-Token` and passes `Cookie` as `grpcgateway-cookie`. Add an incoming header matcher: + +```go +mux := runtime.NewServeMux( + runtime.WithIncomingHeaderMatcher(func(key string) (string, bool) { + switch k := strings.ToLower(key); k { + case "cookie", "x-csrf-token": + return k, true + } + return runtime.DefaultHeaderMatcher(key) + }), +) +``` + +The SDK checks every call that uses this getter, also a `GET`. Use it for the state-changing methods only, or have the frontend send the header on every request. + +Check the header also when the request carries an `Authorization` header. A browser can send both the cookie and a header. + +--- + +## Cookie mode + +By default, `POST /v1/auth/refresh` returns the access token in the response body, and the frontend sends it in the `Authorization` header. With `AUTH_JWT_REFRESH_COOKIE_ACCESS_TOKEN=true`, the refresh answers `200` with no body, and sets the access token in the HttpOnly `_authorization` cookie. Scripts on the page cannot read the token, so an injected script cannot copy it. + +The browser then sends the access token with every request to the host. Turn on CSRF protection with cookie mode, and turn on the check in each service. + +Keep cookie mode off while any browser app on the same host uses the legacy login (`AUTH_REFRESH_ALLOW_LEGACY_LOGIN=true`). A refresh writes its short-lived access token over the legacy app's `_authorization` cookie. See [Sessions, Sign-Out and Revocation](sessions.md). + +--- + +## Turn it on + +Every LutherAuth user login gives an access token with a `nonce` claim, also with CSRF protection off. A service that checks before the cookie and the header exist refuses every browser call. Turn on each part in this order, and check each step in a test environment first. + +| Step | Do this | +|---|---| +| 1. LutherAuth | Upgrade to v4.15.0 or later if you use refresh tokens. Set `AUTH_OAUTH_PROTECT_CSRF=true` | +| 2. CDN | Forward `X-CSRF-Token` to LutherAuth and to each service. See [Behind a CDN](behind-a-cdn.md) | +| 3. Frontend | Send the `X-CSRF-Token` header on each state-changing request | +| 4. Services | Turn on the check in each service | +| 5. Cookie mode (optional) | Set `AUTH_JWT_REFRESH_COOKIE_ACCESS_TOKEN=true`, when no app on the host uses the legacy login | + +A session that started before step 1 has no CSRF token cookie. In refresh mode, its next refresh sets the cookie. A legacy login session gets the cookie only at its next login, so it fails a checked call until then. Wait `AUTH_JWT_TIMEOUT` after step 1 before step 4, or ask those users to log in again. + +A CDN drops a header that it does not forward, with no error. The request then fails at the service as an authentication error. Make sure that step 2 is done before step 4. + +To turn it off, go in the opposite order: first the services, then the frontend and LutherAuth. + +--- + +## Check it + +1. Log in in a browser. +2. In the browser developer tools, confirm the CSRF token cookie: not HttpOnly, `SameSite=Strict`. +3. Decode the access token and confirm that its `nonce` claim is equal to the cookie value. +4. Send a state-changing request through the CDN with the session cookies and `X-CSRF-Token` set to the cookie value. It must succeed. This proves that the service reads the access token from the cookie and gets the header. +5. Send the same request without `X-CSRF-Token`. The service must refuse it. +6. Send the access token as `Authorization: Bearer`, with no cookies and no `X-CSRF-Token`. The service must refuse it. Add the header, and it must succeed. +7. In refresh mode, wait longer than `AUTH_JWT_REFRESH_INTERVAL`, refresh, and repeat step 4. It must still succeed. + +```bash +# Step 4: expect success +curl -s -o /dev/null -w '%{http_code}\n' -X POST \ + -H 'Cookie: ui_authorization=; ui_csrf_token=' \ + -H 'X-CSRF-Token: ' \ + https:///v1/my-service/orders + +# Step 5: expect a refusal +curl -s -o /dev/null -w '%{http_code}\n' -X POST \ + -H 'Cookie: ui_authorization=; ui_csrf_token=' \ + https:///v1/my-service/orders + +# Step 6: expect a refusal, then success with -H 'X-CSRF-Token: ' +curl -s -o /dev/null -w '%{http_code}\n' -X POST \ + -H 'Authorization: Bearer ' \ + https:///v1/my-service/orders +``` + +If step 4 fails with an authentication error, the service does not read the cookie or does not get the header. Check the token getter, the header matcher and the CDN before you go on. Otherwise step 5 proves nothing. + +Use a test user. Do not paste a real user's token into a shared shell history or a ticket. diff --git a/connectors/lutherauth/sessions.md b/connectors/lutherauth/sessions.md index d697961..3b584c1 100644 --- a/connectors/lutherauth/sessions.md +++ b/connectors/lutherauth/sessions.md @@ -9,7 +9,9 @@ A LutherAuth login gives the browser two kinds of token: | Token | Where it lives | Lifetime | Checked by | |---|---|---|---| | Refresh token | HttpOnly cookie on `/v1/auth`, with `AUTH_ENABLE_REFRESH=true` | `AUTH_REFRESH_TIMEOUT` | LutherAuth, on each refresh, against its Redis store | -| Access token | Returned by `POST /v1/auth/refresh`, or an `authorization` cookie for the legacy login | `AUTH_JWT_REFRESH_INTERVAL` (30m by default), or `AUTH_JWT_TIMEOUT` for the legacy login | Each service, offline, against the JWKS | +| Access token | Returned in the body of `POST /v1/auth/refresh` | `AUTH_JWT_REFRESH_INTERVAL` (30m by default) | Each service, offline, against the JWKS | +| Access token in cookie mode | HttpOnly `_authorization` cookie, set by `POST /v1/auth/refresh` with `AUTH_JWT_REFRESH_COOKIE_ACCESS_TOKEN=true`. See [CSRF Protection and Cookie Mode](csrf.md) | `AUTH_JWT_REFRESH_INTERVAL` | Each service, offline, against the JWKS | +| Access token of the legacy login | HttpOnly `_authorization` cookie | `AUTH_JWT_TIMEOUT` | Each service, offline, against the JWKS | Each refresh rotates the refresh token. A refresh token that is used a second time (a copied cookie, for example) revokes its whole family, and the user must log in again. @@ -78,6 +80,8 @@ Allow connections to the cache only from the nodes or pods that run LutherAuth ( It takes an optional `redirect_url`. The host of the URL must match `AUTH_OAUTH_ENDPOINT_DELETE_COOKIE_VALID_REDIRECT_REGEXP`. A GET is refused with `405`. A bad `redirect_url` is refused with `400` (v4.12.0 and later). +From v4.15.0, if the revoke fails two times (for example, Redis is down), sign-out answers `503` and does not redirect. The session is still live. Sign-out deletes the access cookie, but keeps the refresh, CSRF token and ID token cookies, so the user can sign out again when Redis is back. Treat a `503` as a failed sign-out: retry later, or run `revoke-user` for the user. A sign-out with no refresh cookie, or with an expired or invalid one, has no session to revoke. It redirects as usual. + --- ## End the IdP session too @@ -101,6 +105,7 @@ On the IdP, register each `redirect_url` as an allowed post-logout URL (Auth0: A |---|---| | A page load with `redirect_url` (a form post, `Sec-Fetch-Mode: navigate`, or no `Sec-Fetch-Mode`) | `303` to the IdP logout URL, with `client_id`, the return URL and, with the hint on, `id_token_hint`. The IdP ends its session and sends the browser to `redirect_url` | | A `fetch` or XHR, or a request without `redirect_url` | `200` with `{"idp_logout_url": "..."}`. A `fetch` cannot follow a redirect to another site, so the app sets `window.location` to this URL | +| Any request, when the refresh token revoke fails two times (v4.15.0 and later) | `503`, with no redirect and no `idp_logout_url`. The access cookie is deleted. The refresh, CSRF token and ID token cookies stay. This applies to a form post too, so the browser shows the error page. See [Sign out](#sign-out) | `idp_logout_url` never carries the ID token, because any script on the page can read the response body. If your IdP needs `id_token_hint`, sign out with a form post that has `redirect_url`: @@ -140,9 +145,9 @@ A refresh never asks the IdP again. A user that you disable at the IdP stays sig kubectl exec -n deploy/ -- /opt/app revoke-user --sub '' ``` -- Every refresh token of the user issued up to one minute after the command is refused at its next use. The user must log in again. The minute covers a refresh that is in progress and a small clock difference between replicas. +- Every refresh session of the user that started (logged in) up to one minute after the command is refused at its next refresh. The user must log in again. The minute covers a refresh that is in progress and a small clock difference between replicas. From v4.15.0, a rotated refresh token keeps the time of its login, so a rotation in progress cannot carry a session past the revoke. - An access token already issued stays valid until it expires. -- The `sub` must match exactly, and it depends on the login: the IdP's `sub` for OAuth, the email address for a magic link. Revoke each `sub` that the person has. +- The `sub` must match exactly. The match is case-sensitive, and nothing is trimmed. It depends on the login: the IdP's `sub` for OAuth (`/` for Cognito), the email address as typed for a magic link. Revoke each `sub` that the person has. The command reports success also when no session has that `sub`. - It needs `AUTH_ENABLE_REFRESH=true` and the Redis refresh store. It refuses to run without them. --- diff --git a/connectors/lutherauth/upgrade-v4-9-0.md b/connectors/lutherauth/upgrade-v4-9-0.md index 1c4a83c..8d15036 100644 --- a/connectors/lutherauth/upgrade-v4-9-0.md +++ b/connectors/lutherauth/upgrade-v4-9-0.md @@ -4,7 +4,7 @@ description: Upgrade a LutherAuth deployment from v4.7 or later to the latest re # Upgrade LutherAuth -This guide upgrades a LutherAuth deployment from v4.7 or later to the latest release, **v4.13.0**. You can upgrade in one step. Since v4.9.0, LutherAuth has the client credentials grant and the `mint-token` command for [service tokens](service-tokens.md), which replace the fake IdP. +This guide upgrades a LutherAuth deployment from v4.7 or later to the latest release, **v4.15.0**. You can upgrade in one step. Since v4.9.0, LutherAuth has the client credentials grant and the `mint-token` command for [service tokens](service-tokens.md), which replace the fake IdP.

Do not deploy v4.12.0. With refresh tokens on and a Redis user that is limited to the f_* keys (the recommended permissions), every refresh on v4.12.0 fails with 500. v4.13.0 fixes it. See Known Issues.

@@ -57,6 +57,9 @@ Read every row after your current version. **Action** says what to do before you | v4.12.0 | Every response, except the JWKS, sends `Cache-Control: no-store`. A bad login, callback or sign-out request gets `400` instead of `500`. New `revoke-user` command (no action). See [Behind a CDN](behind-a-cdn.md) and [Revoke a user](sessions.md#revoke-a-user). | | v4.12.0 | Known issue: with refresh tokens on and a Redis user limited to `f_*` keys, every refresh fails with `500`. **Action:** Skip v4.12.0. Go to v4.13.0. | | v4.13.0 | Sign-out can also end the IdP session (`AUTH_OAUTH_IDP_LOGOUT_URL`), with an optional `id_token_hint`. Off by default (no action). To use it, see [End the IdP session too](sessions.md#end-the-idp-session-too). | +| v4.13.1 | Dependency updates (no action). | +| v4.14.0 | New `AUTH_GENERIC_REQUIRE_ACCESS_TOKEN_ISSUER` makes the generic backend check the issuer of the IdP access token. Off by default (no action). | +| v4.15.0 | The CSRF token cookie lasts the whole refresh session (no action). If the refresh store revoke fails two times, sign-out answers `503`. It deletes the access cookie and keeps the refresh, CSRF token and ID token cookies. A login with an empty `sub` is refused. **Action:** Handle a `503` from sign-out as a failed sign-out in every app, for a `fetch` and for a form post. See [CSRF Protection and Cookie Mode](csrf.md) and [Sign out](sessions.md#sign-out). | ### Magic link rate limits @@ -92,9 +95,9 @@ At `0`, LutherAuth ignores `X-Forwarded-For`, because a client can forge it. If ## Upgrade LutherAuth -Change the image tag to `luthersystems/auth:v4.13.0` and roll the pods. The image is multi-architecture (`linux/amd64` and `linux/arm64`). +Change the image tag to `luthersystems/auth:v4.15.0` and roll the pods. The image is multi-architecture (`linux/amd64` and `linux/arm64`). -To roll back, set the previous tag. Access tokens and service tokens keep working after a rollback, because they are signed with the same key. With refresh tokens on, a rollback to a version before v4.10.0 signs out the users who logged in after the upgrade. The token endpoint and `mint-token` are not in versions before v4.9.0. Do not roll back from v4.13.0 to v4.12.0: refresh fails again (see [Known Issues](../../known-issues.md)). A rollback to v4.12.0 or earlier also ignores the `revoke-user` records that v4.13.0 wrote, so revoked users can refresh again. Run `revoke-user` again after you upgrade back. +To roll back, set the previous tag. Access tokens and service tokens keep working after a rollback, because they are signed with the same key. With refresh tokens on, a rollback to a version before v4.10.0 signs out the users who logged in after the upgrade. The token endpoint and `mint-token` are not in versions before v4.9.0. With refresh tokens and `AUTH_OAUTH_PROTECT_CSRF` on, a rollback to a version before v4.15.0 brings back the short CSRF token cookie lifetime: the cookie expires after the access token lifetime, and every checked call fails after that. Turn off the CSRF check in your services before such a rollback. Do not roll back from v4.13.0 or later to v4.12.0: refresh fails again (see [Known Issues](../../known-issues.md)). A rollback to v4.12.0 or earlier also ignores the `revoke-user` records that v4.13.0 or later wrote, so revoked users can refresh again. Run `revoke-user` again after you upgrade back. --- @@ -146,7 +149,7 @@ Run these checks in a test environment, then in production. | Check | Expected result | |---|---| -| The image tag of the LutherAuth pods | `v4.13.0` | +| The image tag of the LutherAuth pods | `v4.15.0` | | With refresh tokens on: log in, wait past `AUTH_JWT_REFRESH_INTERVAL`, then an API call | Works. `POST /v1/auth/refresh` returns `200`, not `500` | | `https:///v1/auth/.well-known/jwks.json` | Returns the JWKS | | Browser login, then an API call | Works | diff --git a/operations/cloud-cost-optimization.md b/operations/cloud-cost-optimization.md new file mode 100644 index 0000000..fc52c64 --- /dev/null +++ b/operations/cloud-cost-optimization.md @@ -0,0 +1,103 @@ +--- +description: Decrease the AWS cost of a Luther network with ARM (Graviton) worker nodes, tiered peer storage and peers that join from a snapshot. +--- + +# Cloud Cost Optimization + +Three changes decrease the AWS cost of a Luther network the most. Read the +limits of each change on its details page before you make it. + +| Change | What it decreases | Typical saving | Details | +|---|---|---|---| +| Run the EKS worker nodes on ARM (Graviton) | Compute | About 20% per node-hour | [Run on ARM](#run-on-arm-graviton) | +| Keep old blocks on `sc1`, recent blocks on `gp3` | Peer storage | About 80% compared with all `gp3` | [Peer Ledger Storage](peer-storage.md#tiered-storage) | +| Join most peers from a snapshot | Peer storage | More than 95% compared with all `gp3` | [Fabric Snapshot Archiver](fabric-snapshot-archiver.md) | + +The savings are estimates from us-west-2 list prices. Your savings depend on +your region, your load and your discounts. + +## Run on ARM (Graviton) + +AWS Graviton instances use ARM (`arm64`) processors. A Graviton instance costs +about 20% less per hour than the x86 instance of the same size, for example +`m7g.large` in place of `m7i.large`. Luther's own platform runs its EKS worker +nodes on Graviton (`t4g`). + +### Images + +Kubernetes pulls the `arm64` image on an ARM node. You do not change the image +names or tags. These images publish for `amd64` and `arm64`: + +| Image | `arm64` | +|---|---| +| `luthersystems/substrate`, `luthersystems/shiroclient` | Yes | +| `luthersystems/auth` (lutherauth) | Yes | +| `luthersystems/connectorhub` | Yes | +| `luthersystems/fabric-snapshot-archiver` | Yes | +| `luthersystems/pdfserv`, `luthersystems/martin` | Yes | +| `luthersystems/awscli`, `luthersystems/dind`, `luthersystems/fabric-tools` | Yes | +| `hyperledger/fabric-peer`, `fabric-orderer` 2.5 and 3.1 | Yes | +| `hyperledger/fabric-peer`, `fabric-orderer` 2.2 and 2.4 | **No** | +| `hyperledger/fabric-ca` 1.5 | Yes | +| `hyperledger/fabric-ca` 1.4 | **No** | +| `luthersystems/fabric-ccenv` 2.5 and 3.1 | Yes | +| `luthersystems/fabric-ccenv`, `fabric-baseos` 2.2 and 2.4 | **No** | + +**Fabric 2.5 or later, and Fabric CA 1.5 or later, are necessary for ARM.** +The Fabric 2.2 and 2.4 images and the Fabric CA 1.4 image are `amd64` only. On those versions, keep the nodes that run Fabric on x86, or +upgrade Fabric first (see [Fabric 3 Upgrade](fabric-3-upgrade.md)). + +Check every other image that runs in your cluster, for example your own +services and monitoring agents. This command lists the architectures of one +image: + +```sh +docker buildx imagetools inspect luthersystems/connectorhub: | grep Platform +``` + +### Move a node group to Graviton + +1. Make sure that every image on the node group publishes `arm64`. +2. Set the worker instance type to a Graviton type, for example `t4g`, `m7g` + or `c7g`. The Luther `eks-vpc` Terraform module selects the ARM node AMI + from the instance type. All instance types of one node group must be the + same architecture. +3. Apply the change. EKS replaces the nodes, and Kubernetes starts each pod + again with the `arm64` image. +4. Check that each pod is running: + + ```sh + kubectl get pods -A -o wide + ``` + +Move a test environment first, then production. + +## Storage + +The peer ledger is often the largest storage cost. For the measurements and +the cost of each layout, see [Peer Ledger Storage](peer-storage.md). +In summary: + +- Put orderer volumes on `sc1`. The orderer's write-ahead log is as fast on + `sc1` as on `gp3`. +- Keep 2 or 3 archive peers with the full chain history, with tiered storage: + recent blocks on `gp3`, old blocks on `sc1`. +- Join all other peers from a snapshot, on small `gp3` volumes. A peer that + joined from a snapshot keeps only the blocks after the snapshot. See + [Fabric Snapshot Archiver](fabric-snapshot-archiver.md). + +For 40 peers with 4 TB each, this layout costs about $300 to $500 each month. +All `gp3` costs about $13,000 each month. + +## Test environments + +These changes are for test environments only: + +- Use Spot instances for the EKS worker nodes. Give the node group several + instance types, so that EKS can find Spot capacity. Each type must allow at + least as many pods as the base type. For example, `t4g.large` allows 35 + pods, but `m7g.large` allows 29, so a rolling deploy can fail with "Too many + pods". Use the `.xlarge` sizes for the other types. A Spot node can stop at + any time, so do not use Spot for production peers or orderers. +- Use smaller node and volume sizes than production. An `sc1` volume must be + 125 GiB or larger. diff --git a/operations/fabric-3-upgrade.md b/operations/fabric-3-upgrade.md new file mode 100644 index 0000000..ae397aa --- /dev/null +++ b/operations/fabric-3-upgrade.md @@ -0,0 +1,87 @@ +--- +description: Upgrade a running Luther network from Hyperledger Fabric 2.5 to 3.x with Raft ordering, with the mars upgrade scripts. +--- + +# Fabric 3 Upgrade + +This page tells you how to upgrade a running network from Hyperledger Fabric +2.5 to Fabric 3.1 or later. The network keeps Raft ordering. The scripts and +the full runbook are in [mars](https://github.com/luthersystems/mars), the +infrastructure tool that deploys Luther networks. + +## Why upgrade + +| Reason | Detail | +|---|---| +| Support | Fabric 2.5 is the last 2.x release line. Fixes go to 3.x. | +| Speed | A 3.x peer can send chaincode reads and writes in batches. Substrate uses this. | +| BFT | Fabric 3 adds BFT ordering. The upgrade keeps Raft. BFT is a separate change. | + +## Where the scripts are + +| Item | Location in mars | +|---|---| +| Runbook (full detail, errors and fixes) | [`docs/fabric-3-upgrade.md`](https://github.com/luthersystems/mars/blob/main/docs/fabric-3-upgrade.md) | +| Ansible role: pre-flight check and system channel removal | [`ansible-roles/k8s_fabric_upgrade_v3`](https://github.com/luthersystems/mars/tree/main/ansible-roles/k8s_fabric_upgrade_v3) | +| Shell scripts: `fabric3-preflight.sh`, `remove-system-channel.sh`, `fabric3-utils.sh`, `upgrade-pauser.sh` | [`ansible-roles/k8s_fabric_scripts/files/scripts`](https://github.com/luthersystems/mars/tree/main/ansible-roles/k8s_fabric_scripts/files/scripts) | +| End-to-end test on a local network | [`scripts/test-fabric3-upgrade.sh`](https://github.com/luthersystems/mars/blob/main/scripts/test-fabric3-upgrade.sh) | + +Your infrastructure repository calls the role and the scripts from +playbooks. You run each playbook with mars, for example +`mars ansible-playbook fabric3_preflight.yaml`. + +## Steps + +Do each step on a test network first. Do the next step only when the step +before it is done on every node. + +| Step | What | Playbook | Undo | +|---|---|---|---| +| 0 | Take EBS snapshots of the orderer and peer volumes. Run the pre-flight check. | `fabric3_preflight.yaml` | Not needed | +| 1 | Restart the 2.5 orderers with channel participation on | `fabric_remove_system_channel.yaml` (first part) | Set the flag false and run the orderer playbook again | +| 2 | Remove the system channel | `fabric_remove_system_channel.yaml` (second part) | **None.** Restore the orderer volumes from the snapshots. | +| 3 | Upgrade the orderers, then the peers, to 3.x, one node at a time | `fabric_upgrade.yaml` with the new version | Restore the volumes from the snapshots | +| 4 | Run the pre-flight check again, check the health endpoint, and do one write | `fabric3_preflight.yaml` | Not needed | + +**Caution:** step 2 has no undo. Take the snapshots in step 0 before you start +step 1. + +The pre-flight check reads the network and changes nothing, so you can run it +at any time. It must say `READY` before step 3 and after it. + +Step 1 sets this in the inventory of the environment. Keep it set from then +on: + +```yaml +k8s_fabric_orderer_channel_participation: true +``` + +## What Fabric 3 removes + +A 3.x node does not start when the network uses one of these items. The +pre-flight check finds each one before you change a node. + +| Removed in Fabric 3 | Luther networks | Action | +|---|---|---| +| Upgrade from a version before 2.5 | 2.5 | None. Upgrade an older network to 2.5 first. | +| System channel | Each network has one. Nothing uses it at run time. | Steps 1 and 2 remove it. | +| Kafka and Solo ordering | Raft | None | +| Legacy chaincode lifecycle (`lscc`) | `_lifecycle` | None. The pre-flight check confirms it. | + +You do not need the `V3_0` channel capability. It is only for BFT ordering. + +## Prevent an upgrade by accident + +A change of the Fabric version in the inventory is an upgrade. Do it only with +`fabric_upgrade.yaml`, which runs the pre-flight check and restarts one node +at a time. A plain orderer or peer playbook does neither. Make the plain +playbooks refuse to run when a running node has a different Fabric version +than the inventory. Then a version change cannot reach a network before its +upgrade runs. + +## Known limits + +| Limit | Detail | +|---|---| +| New networks | A new 3.x network has no system channel, and mars cannot create the first channel without one yet ([mars#343](https://github.com/luthersystems/mars/issues/343)). Create a new network on 2.5, then upgrade it. | +| Docker chaincode | A 3.x peer that builds chaincode with Docker uses the upstream `fabric-ccenv` image. Chaincode as a service (`ccaas`) is not affected. The pre-flight check shows a warning. | diff --git a/operations/peer-storage.md b/operations/peer-storage.md index f4bbf70..94bb790 100644 --- a/operations/peer-storage.md +++ b/operations/peer-storage.md @@ -51,7 +51,7 @@ The monthly cost of 40 peers with 4 TB each, at us-west-2 list prices: |---|---| | All `sc1` | about $2,400 | | All `gp3` | about $13,000 | -| Tiered on every peer: recent blocks on `gp3`, older blocks on `sc1` (upcoming, see below) | about $2,800 | +| Tiered on every peer: recent blocks on `gp3`, older blocks on `sc1` (see below) | about $2,800 | | 2 or 3 archive peers with the full history (tiered), and all other peers joined from a snapshot on small `gp3` volumes | about $300 to $500 | The last layout is the least expensive and the fastest. A peer that joined from @@ -104,17 +104,64 @@ variable `init_volume_type`), and the peer claims them by label. `k8s_fabric_peer_storage_class` must match the storage class of those volumes; it does not set their type. -## Tiered storage (upcoming) +## Tiered storage A peer keeps all of its old block files, but it reads them only to serve old -blocks. Tiered storage keeps the state database and the recent block files on -`gp3`, and moves the sealed (complete) block files to an `sc1` archive -volume, with a link in their old place so the peer still finds them. This gives `gp3` commit speed at near `sc1` cost for a peer that must keep the -full history. The archiver's `archive-blocks` mode, which moves the files, -is merged and goes out in fabric-snapshot-archiver v0.0.9 -([fabric-snapshot-archiver#38](https://github.com/luthersystems/fabric-snapshot-archiver/pull/38)). -Support in the mars fabric-peer chart is in progress -([luthersystems/mars#337](https://github.com/luthersystems/mars/pull/337)). +blocks. Tiered storage keeps the state database, the indexes and the current +block file on `gp3`. A `block-archiver` sidecar moves each sealed (complete) +block file to a second, `sc1` volume and leaves a link in its old place, so +the peer still finds it. This gives `gp3` commit speed at near `sc1` cost for +a peer that must keep the full history. + +### Requirements + +| Item | Version | +|---|---| +| mars | v0.145.0 or later (fabric-peer chart, [mars#337](https://github.com/luthersystems/mars/pull/337)) | +| fabric-snapshot-archiver | v0.0.9 or later (`archive-blocks`, [fabric-snapshot-archiver#38](https://github.com/luthersystems/fabric-snapshot-archiver/pull/38)). From the first mars release after v0.145.0 ([mars#353](https://github.com/luthersystems/mars/pull/353)), the peer role stops before any change if the version is older. With an older archiver the sidecar crash-loops and the peer pod drops out of its Service | + +### Turn it on + +Set these in the environment's Ansible inventory: + +```yaml +k8s_fabric_peer_volume_type_override: "gp3" +k8s_fabric_peer_block_archive: true +k8s_fabric_peer_block_archive_storage_size: 125Gi # the sc1 minimum +k8s_fabric_peer_snapshot_archiver_version: v0.0.9 # or later +``` + +| Setting | Default | Meaning | +|---|---|---| +| `k8s_fabric_peer_block_archive` | `false` | Adds the archive volume and the sidecar | +| `k8s_fabric_peer_block_archive_storage_size` | `125Gi` | Archive volume size. `sc1` needs 125 GiB or more | +| `k8s_fabric_peer_block_archive_volume_type_override` | `sc1` | EBS type of the archive volume | +| `k8s_fabric_peer_block_archive_interval` | `60s` | Time between passes (a Go duration) | +| `k8s_fabric_peer_block_archive_max_mb_per_second` | `20` | Copy limit in MiB/s, so the first pass of a large ledger does not take the disk from the peer. `0` is no limit | + +Then run `fabric_peer.yaml` for the environment. It adds the volume +`blockarchive-fabric-peer-`, so it restarts each peer one time. +Later passes do not restart the peer. + +### Check it + +| Check | How | +|---|---| +| Archive pass | The `block-archiver` log shows `archived block file` for each sealed file, then `block archive pass done` | +| Volume types | `fabric_snapshot_volumes.yaml` with `check_mode=true` lists each Fabric volume with its EBS type | +| Peer restart | `fabric_peer_restart.yaml` restarts each peer, waits until it is ready and prints its first log lines | + +### Results on platform-test + +| Check | Result | +|---|---| +| First pass | Moved 2 sealed block files in 32 s | +| Volume types | Archive volume `sc1` (125 GiB), peer storage `gp3` (126 GiB) | +| Peer restart | Ready in about 40 s, 0 container restarts, ledger opened at height 15424, no block file errors | +| Commit time (mean) | Block storage 6.2 ms, state database 3.1 ms. Before (Fabric 2.5, `gp3` only): 5.8 ms and 2.9 ms. Fabric 3.1.5 came in the same window, so this does not isolate the archive | +| Read of old blocks | The peer served blocks 0, 4780, 9560 and 14340 (height 19120) through `qscc` `GetBlockByNumber`, 5 KB to 12 KB each. Block 0 is in `blockfile_000000`, which is on the archive volume | + +To do the same check, run the `fabric_block_read.yaml` playbook. It reads block 0 and the blocks at 1/4, 1/2 and 3/4 of the height from the peer, not from the orderer, and changes nothing. ## Other options we examined diff --git a/release-notes.md b/release-notes.md index 5252289..fa80de9 100644 --- a/release-notes.md +++ b/release-notes.md @@ -22,6 +22,107 @@ Issues.

Operator settings applied to the chaincode container are catalogued in [Chaincode Environment Variables](common-operation-script/substrate/chaincode-environment-variables.md). +## 2026-11-13: substrate v2.259.0 + +- **substrate**: **Upgrade note:** some transactions take fewer ELPS steps. If you set a step limit (`SUBSTRATE_ELPS_MAX_STEPS`), upgrade every peer and pre-deploy checker of a channel together. See the [upgrade notes](https://github.com/luthersystems/substrate/blob/v2.259.0/docs/substrate-api.md) +- **substrate**: Substrate runs on Fabric 3.1 peers. On a 3.1 peer the chaincode sends a transaction's writes to the peer in one batch. Set `CHAINCODE_WRITE_BATCH=false` to turn this off. A Fabric 2.5 peer does not change. **Upgrade note:** set the variable the same on every peer of a channel. During a rolling upgrade from 2.5 to 3.1, set it to `false` if your phylum catches the error of a failed put +- **substrate**: The gateway does not retry a write batch that the peer refused. A transport failure is still retried +- **substrate**: New `statedb:get-many`, `statedb:get-many-bytes`, `sidedb:get-many` and `sidedb:get-many-bytes` read a list of keys in one call. They return what `(map 'list statedb:get keys)` returns. On a Fabric 3.1 peer the keys are read in one batched call: 100 keys take 16 ms, not 43 ms. A refused key fails the whole call before any key is read +- **substrate**: A data migration defined with `def-migration-ctx` takes fewer ELPS steps +- **substrate**: The in-process mock of the Go SDK takes the cc-fetchurl proxy and downgrade options. It refuses a request whose fetchurl settings are different from the mock's +- **substrate**: Bundles elps v1.98.0. A path read checks only the containers on its path, so its cost follows the path, not the document size +- **Tooling**: In the VS Code extension, hover and completion know the new `get-many` functions. The language server no longer reports a function that a phylum defines with its own macro as undefined. The extension is published with Marketplace Trusted Publishing; nothing changes for users + +Details: [substrate v2.259.0 release](https://github.com/luthersystems/substrate/releases/tag/v2.259.0). + +## 2026-11-13: elps v1.98.0 + +- **Language**: An `elpspath` read (`?`) with a path checks only the containers that path steps into. A fault elsewhere in the document does not change its answer, and its cost follows the path, not the document size. Writes, and a read with no path, still check the whole document +- **Language**: `elpspath` refuses a malformed array or map header with an error. Before, it could panic +- **Language**: `macroexpand-all` takes `:lower true`, and the new `macroexpand-all-lowered` is the same call with one argument. Each expands `get-default` as the macro it replaced expanded it, for tools that model that macro +- **Tooling**: The VS Code extension is published with Marketplace Trusted Publishing, and its `minimatch` dependency is 11.0.1 or later, except where `vscode-languageclient` needs version 10 + +Details: [elps v1.98.0 release](https://github.com/luthersystems/elps/releases/tag/v1.98.0). + +## 2026-11-12: elps v1.97.0 + +- **Language**: `get-default` is a special operator, not a macro. Calls give the same results, and the map and key are each evaluated once. A call costs fewer steps +- **Language**: `to-float` accepts only decimal text, such as `"-12.75"`, `".5"` or `"1e5"`. The strings `"nan"`, `"inf"` and `"infinity"`, hexadecimal floats, digits with `_` and text with spaces are now errors, as is a number too large for a float +- **Language**: `time:time-from` returns an error for two times more than about 292 years apart. Before, it returned about 292 years with no error +- **Language**: The reader refuses a sign joined to a `#x` or `#o` literal, such as `-#x10`. Before, it read the symbol `-` and then the number 16 +- **Language**: `?set!`, `?nil!` and `?del!` with a path that ends in `'*` write each element of a vector +- **Language**: `regexp:regexp-match?` charges steps by the size of the text and of the pattern +- **JSON**: The JSON functions give the same output and the same error text on both of Go 1.27's JSON engines, also for native Go values. A strict load refuses non-canonical integer text past the 64-bit range +- **Tooling**: The language server, `elps lint` and `elps mcp` expand macros defined in the workspace, so a function that a macro in another file defines is found, not reported as undefined. References, hover, definition and rename work on files the editor has not opened. A file with no problems now clears its old diagnostics +- **Tooling**: `elps fmt -d` prints a unified diff, and `elps fmt` keeps LF line endings for a file with CRLF endings +- **Performance**: Fewer allocations for function calls, integers, sorted-map copies and the JSON functions +- **Security**: built with Go 1.27.4, which fixes GO-2026-6859 + +Details: [elps v1.97.0 release](https://github.com/luthersystems/elps/releases/tag/v1.97.0). + +## 2026-11-12: substrate v2.258.0 + +- **substrate**: **Upgrade note:** this release is consensus-visible. It has no setting that turns the changes off. Upgrade every peer and pre-deploy checker of a channel in one maintenance window. The upgrade notes in [substrate-api.md](https://github.com/luthersystems/substrate/blob/v2.258.0/docs/substrate-api.md) give the read sets, write sets and step counts of each change +- **substrate**: Security: `cc:valid-json?` no longer loads a document that a schema `$ref` names. Before, each peer read an `http://`, `https://` or `file://` reference itself, with no timeout, so two peers could give different results. Now a `$ref` to another document is an error. A `$ref` to the schema's own text works as before +- **substrate**: Security: decimal calls have bounds. `decimal-string` with `:places` and `decimal-round` take places from -1000 to 1000, and `decimal-pow` refuses a result too large to compute. Before, one call could run until the transaction timed out. Now each such call raises an ordinary error before the work starts +- **substrate**: The batch package keeps one state key per queued item. Schedules of one queue no longer conflict in a block. A batch holds at most 100 items, and an item with no request record no longer blocks the queue. **Upgrade note:** queues of an earlier release are not migrated. If your phylum uses the batch package, empty its queues before you upgrade: pause the schedules and let the batch processor collect and answer every batch +- **substrate**: B+-tree fixes. A query or a delete of a tree that does not exist no longer creates the tree, and writes nothing. A delete that shrinks the root removes the old root from state. After `cc:bptree-delete-tree`, the handle works again. A node is searched with a binary search. Step counts change, see the upgrade notes +- **substrate**: `utils:storage-pcounter` and `utils:storage-pcounter-vec` without `:prand` pick the shard from the transaction ID, not from the clock. Before, all transactions in one second wrote the same shard, and all but one failed validation +- **substrate**: `validations:validate-address` checks the whole GB postcode. Before, a postcode with text after it, such as `"SW1A 1AA, London"`, passed +- **substrate**: A request for the `latest` phylum version finds a version that was enabled up to 10 years ago. Before, the limit was 5.33 years +- **substrate**: A failed flow migration records its outcome for any error text. Before, an error text with many quotes, newlines or `<` characters could not be recorded +- **substrate**: Less work for each request: the request is decoded once, a B+-tree flush writes only its changed keys, and the state cache iterator uses no goroutine. A large request is up to 47% faster +- **substrate**: Security: built with Go 1.27.4, which fixes GO-2026-6859 (a crash of the HTTP/1 server in `net/http`) +- **shiroclient**: Fixes in the gateway: a deadlock when `Close` runs during a write, and three panics on a failed proposal or a missing block cache. A request body that takes more than 60 seconds to read gets HTTP 408. The Prometheus `method` label holds only known method names +- **shiroclient**: With tracing on, the gateway starts as a user that has no entry in `/etc/passwd`. Before, it stopped with `user: Current requires cgo or $USER set in environment` (svc v0.21.1) +- **shiroclient**: In the plugin mock, `cc-fetchurl` sends an `https` URL as `https`. Before, it sent it as plain `http`. A test on the in-process mock that sends `https` through a plain-`http` proxy must add `shiroclient.WithCCFetchURLDowngrade(true)` +- **shirotester**: `shirotester package` needs Go 1.27 or later for a WebAssembly module. A deployed `.wasm` file still loads and runs + +Details: [substrate v2.258.0 release](https://github.com/luthersystems/substrate/releases/tag/v2.258.0). + +## 2026-11-11: svc v0.21.1 + +- **svc**: With tracing on, a service starts when its process owner is not known, for example a container that runs as a user with no entry in `/etc/passwd`. The trace resource then has no `process.owner` and the service logs a warning. Before, `opttrace.New` failed and the service stopped. Other errors in the resource attributes (`OTEL_RESOURCE_ATTRIBUTES`) still stop it +- **svc**: Security: built with Go 1.26.11, which fixes GO-2026-6859 + +Details: [svc v0.21.1 release](https://github.com/luthersystems/svc/releases/tag/v0.21.1). + +## 2026-11-11: fabric-network-builder v0.1.2 + +- **fabric-network-builder**: Each `CHAINCODE_*`, `SUBSTRATE_*` and `OTEL_*` variable that you give the builder goes to the chaincode containers. For example, `OTEL_BSP_SCHEDULE_DELAY` now reaches the chaincode's tracer. A value that the compose file sets is kept +- **fabric-network-builder**: Security: the network's ports are published on `127.0.0.1` only. Before, they were open on every interface of the host. To publish on another address, set `FNB_PUBLISH_ADDR`. The first `up` after the upgrade recreates the containers + +Details: [fabric-network-builder v0.1.2 release](https://github.com/luthersystems/fabric-network-builder/releases/tag/v0.1.2). + +## 2026-11-11: buildenv v0.1.59 + +- **General**: Security: the Go images (`build-api`, `build-go`, `build-go-alpine`, `build-godynamic`) and `build-java`, `build-js` and `build-e2e` ship Go 1.27.4, which fixes GO-2026-6859 +- **build-api, build-go**: The Debian openssh-client finding CVE-2026-123158 does not apply: the vulnerable code is in `sshd`, and the images have only the client. The images carry an OpenVEX `not_affected` statement for it +- **General**: All images are rebuilt on the latest base images and OS packages + +Details: [buildenv v0.1.59 release](https://github.com/luthersystems/buildenv/releases/tag/v0.1.59). + +## 2026-11-06: substrate v2.257.0 + +- **substrate**: **Upgrade note:** substrate is built with Go 1.27.3. Go 1.27 changes the bytes of zlib output and, for strings with invalid UTF-8, of JSON output. Both are consensus-visible. Upgrade every peer and pre-deploy checker of a channel in one maintenance window. While peers mix Go 1.26 and Go 1.27 builds, a transaction that compresses data fails endorsement: for example a `put-mxf` with the `zlib` compressor, or a response that holds `compress:zlib-compress` output. Reads and other transactions are not affected. Data that a Go 1.26 peer wrote still decodes +- **substrate**: Bump ELPS to v1.96.0: the Lisp JSON functions write the same bytes on Go 1.27's JSON v2 engine as before, invalid UTF-8 included +- **shiroclient**: The gateway health check can make one real write, to show that the network can endorse, order and commit a transaction, not only answer a query. It is off by default. Turn it on with `--health-check-write` or `SHIROCLIENT_GATEWAY_HEALTH_CHECK_WRITE=true`, then call `GET /health_check?write=true`. The write overwrites one key (`luther:healthcheck:heartbeat`), so the state does not grow. The report gets a `write` row with the transaction id and the commit time. A failed write is a `DOWN` row and HTTP 503. Checks that arrive while a write is in flight share it +- **substrate**: Security: only a creator of the channel's admin MSP can run the heartbeat write. The chaincode cannot tell one user from another, so the service in front of the gateway (the oracle) must decide which callers may ask for it. Turn the write on only on a gateway that such a service fronts. The `luther:healthcheck:` key space is reserved: a phylum read or write of it fails + +Details: [substrate v2.257.0 release](https://github.com/luthersystems/substrate/releases/tag/v2.257.0). + +## 2026-11-05: Luther Auth v4.15.0 + +- **General**: Fix: with refresh tokens and `AUTH_OAUTH_PROTECT_CSRF` on, the CSRF token cookie now lasts the whole refresh session, and each `POST /v1/auth/refresh` sets it again. Before, it expired after the access token lifetime (30 minutes by default), and every call that a service checked for CSRF failed after that. See [CSRF Protection and Cookie Mode](connectors/lutherauth/csrf.md) +- **General**: Security: a refresh that is in progress when `revoke-user` runs no longer returns a new access token. A rotated refresh token keeps the time of its login, so `revoke-user` refuses every token of a session that started before it +- **General**: Sign-out revokes the refresh token family also when the token lookup fails. **Behavior change:** if the revoke fails two times (for example, Redis is down), sign-out answers `503` and does not redirect. It keeps the refresh, CSRF token and ID token cookies, so the user can sign out again. See [Sign out](connectors/lutherauth/sessions.md#sign-out) +- **General**: Every Redis call of the refresh store has a 5-second timeout. The connection pool holds at most 64 connections. A call that waits for a free connection gives up after the timeout +- **General**: **Behavior change:** a login whose `sub` is empty is refused, because `revoke-user` cannot end it +- **General**: Security: the OAuth callback refuses a nonce cookie of the wrong length with `400`. Before, a short nonce cookie such as `0` gave every such login the same refresh token family and CSRF token +- **General**: Security: logs no longer contain refresh tokens or raw issuer values + +Details: [Luther Auth v4.15.0 release](https://github.com/luthersystems/lutherauth/releases/tag/v4.15.0). + ## 2026-11-05: fabric-snapshot-archiver v0.0.9 - **fabric-snapshot-archiver**: `serve --snapshot-schedule` takes a snapshot on a cron schedule (UTC), uploads it to S3, and keeps the newest `--keep-archives` archives of the channel (default 14). The IAM role needs `s3:DeleteObject` to delete old archives. Prometheus metrics report the last success, so you can alert when snapshots stop. See [Take snapshots on a schedule](operations/fabric-snapshot-archiver.md#take-snapshots-on-a-schedule) @@ -30,6 +131,18 @@ Operator settings applied to the chaincode container are catalogued in - **fabric-snapshot-archiver**: `serve` stops cleanly on SIGTERM, so a pod shutdown cancels a running snapshot - **fabric-snapshot-archiver**: Updates `golang.org/x/net` to v0.62.0 +## 2026-11-05: Luther Auth v4.14.0 + +- **General**: New `AUTH_GENERIC_REQUIRE_ACCESS_TOKEN_ISSUER`, off by default. On, the generic OAuth backend refuses an IdP access token whose issuer is not `AUTH_GENERIC_BACKEND_ISSUER`. Turn it on only if your IdP puts the same issuer in the access token and the ID token. The ID token issuer check does not change + +Details: [Luther Auth v4.14.0 release](https://github.com/luthersystems/lutherauth/releases/tag/v4.14.0). + +## 2026-11-05: Luther Auth v4.13.1 + +- **General**: Updates dependencies, including the AWS SDK for Go and `golang.org/x/crypto` v0.58.0. No change in behavior + +Details: [Luther Auth v4.13.1 release](https://github.com/luthersystems/lutherauth/releases/tag/v4.13.1). + ## 2026-11-04: substrate v2.256.0 - **substrate**: The in-memory mock ledger (`shirotester`, the Go mock client and offline tests) uses less memory and restores snapshots faster. Its memtable drops from 16 MB to 8 MB. An empty mock holds about 10.5 MB, down from about 26 MB. A snapshot restore plus its first call is about 30% faster. Peers do not change