From fbab3dc97d2a706fbde9edc75275815142988238 Mon Sep 17 00:00:00 2001 From: Ivan Diatchenko <2518263+inhuman@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:05:00 +0300 Subject: [PATCH 1/8] examples: two applications that embed the engine, and skills in their own folder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The examples showed the FORMAT and never the seam. What an embedder actually needs to see is the other half: what you hand the engine, what it hands back, and which of the two descriptions you are supposed to run. examples/skills/ the skill files, where they were examples/simple-llm-app/ the engine embedded in ~200 lines, net/http only examples/eino-llm-app/ the same, with the model reached through eino Both are SEPARATE modules, and that is the point rather than tidiness. The engine's promise is that embedding it adds no dependencies; an example pulling in a framework would break exactly that promise unless it is its own module. The engine's go.mod has never heard of eino, and `go list -deps ./...` says so. Two guards keep it that way, both in imports_test.go: the self-containment walk now skips nested modules, and a new test fails if a directory with Go code in it loses its go.mod. Proven by deleting one — eino is reported as a dependency of the engine on the next run, which at that moment is what it is. CI gained a step that vets and tests each example module, because `go test ./...` at the top never descends into them. An example that does not build teaches the format wrong, and it would rot silently. Each application is verified end to end against a stub — an httptest server for the simple one, eino's own interface for the other — so "it works" is something a reader can check rather than something a README claims. The eino test also checks that a step's `model:` and `sampling:` arrive as options: a field the skill sets and the executor drops is decoration, and nothing tells the author. Both go.mod files carry `replace => ../..` so CI checks the examples against the engine as it is now rather than the last release. The READMEs say to delete that line when copying the example out. --- .github/workflows/ci.yml | 14 ++ .gitignore | 3 + README.md | 2 +- README.ru.md | 2 +- examples/README.md | 72 ++++--- examples/eino-llm-app/README.md | 75 +++++++ examples/eino-llm-app/go.mod | 44 ++++ examples/eino-llm-app/go.sum | 151 +++++++++++++ examples/eino-llm-app/main.go | 121 +++++++++++ examples/eino-llm-app/main_test.go | 90 ++++++++ examples/eino-llm-app/runner.go | 73 +++++++ examples/eino-llm-app/testdata/declared.yaml | 16 ++ examples/simple-llm-app/README.md | 73 +++++++ examples/simple-llm-app/go.mod | 10 + examples/simple-llm-app/go.sum | 10 + examples/simple-llm-app/main.go | 210 +++++++++++++++++++ examples/simple-llm-app/main_test.go | 150 +++++++++++++ examples/simple-llm-app/openai.go | 123 +++++++++++ examples/skills/README.md | 27 +++ examples/{ => skills}/audit.yaml | 0 examples/{ => skills}/contract.yaml | 0 examples/{ => skills}/expenses.yaml | 0 examples/{ => skills}/glossary.yaml | 0 examples/{ => skills}/inbox.yaml | 0 examples/{ => skills}/menu.yaml | 0 examples/{ => skills}/pods.yaml | 0 examples/{ => skills}/proofread.yaml | 0 examples/{ => skills}/research.yaml | 0 examples/{ => skills}/triage.yaml | 0 examples/{ => skills}/vocabulary.yaml | 0 examples/{ => skills}/weather.yaml | 0 examples_test.go | 8 +- imports_test.go | 47 +++++ lint/fixtures_test.go | 2 +- skill.schema.ru.yaml | 16 +- skill.schema.yaml | 16 +- skill_test.go | 2 +- 37 files changed, 1308 insertions(+), 49 deletions(-) create mode 100644 examples/eino-llm-app/README.md create mode 100644 examples/eino-llm-app/go.mod create mode 100644 examples/eino-llm-app/go.sum create mode 100644 examples/eino-llm-app/main.go create mode 100644 examples/eino-llm-app/main_test.go create mode 100644 examples/eino-llm-app/runner.go create mode 100644 examples/eino-llm-app/testdata/declared.yaml create mode 100644 examples/simple-llm-app/README.md create mode 100644 examples/simple-llm-app/go.mod create mode 100644 examples/simple-llm-app/go.sum create mode 100644 examples/simple-llm-app/main.go create mode 100644 examples/simple-llm-app/main_test.go create mode 100644 examples/simple-llm-app/openai.go create mode 100644 examples/skills/README.md rename examples/{ => skills}/audit.yaml (100%) rename examples/{ => skills}/contract.yaml (100%) rename examples/{ => skills}/expenses.yaml (100%) rename examples/{ => skills}/glossary.yaml (100%) rename examples/{ => skills}/inbox.yaml (100%) rename examples/{ => skills}/menu.yaml (100%) rename examples/{ => skills}/pods.yaml (100%) rename examples/{ => skills}/proofread.yaml (100%) rename examples/{ => skills}/research.yaml (100%) rename examples/{ => skills}/triage.yaml (100%) rename examples/{ => skills}/vocabulary.yaml (100%) rename examples/{ => skills}/weather.yaml (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11530d0..bba01ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,3 +23,17 @@ jobs: run: go test ./... -count=1 - name: govulncheck run: go run golang.org/x/vuln/cmd/govulncheck@v1.3.0 ./... + + # The example applications are SEPARATE modules — that is what keeps their + # dependencies (eino, an SDK, whatever the next one uses) out of the + # engine's go.mod, and it also means `go test ./...` above never sees + # them. Without this step an example rots quietly, and an example that + # does not build teaches the format wrong. + - name: Examples + run: | + for mod in examples/*/go.mod; do + dir=$(dirname "$mod") + echo "── $dir" + go vet -C "$dir" ./... + go test -C "$dir" ./... -count=1 + done diff --git a/.gitignore b/.gitignore index c303947..a481fef 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ PLAN.md *.out *.test coverage.* +# Собранные примеры: `go build` в их каталоге кладёт бинарник рядом с исходником +examples/*/simple-llm-app +examples/*/eino-llm-app # Окружение .env diff --git a/README.md b/README.md index f7a28c3..4da5245 100644 --- a/README.md +++ b/README.md @@ -387,6 +387,6 @@ limitation worth knowing before relying on it are in ## Tests `example_flow_test.go` — runnable examples of the format, a good first entry -point. `examples_test.go` parses every file from `examples/` with the engine: an +point. `examples_test.go` parses every file from `examples/skills/` with the engine: an example that stopped parsing is worse than a missing one — it teaches the wrong thing. diff --git a/README.ru.md b/README.ru.md index ed75c0e..0a3c348 100644 --- a/README.ru.md +++ b/README.ru.md @@ -373,5 +373,5 @@ rep, err := lint.Lint(raw, facts, lint.Options{Unmarshal: yaml.Unmarshal}) ## Тесты `example_flow_test.go` — исполняемые примеры формата, годятся как первая точка -входа. `examples_test.go` разбирает движком все файлы из `examples/`: пример, +входа. `examples_test.go` разбирает движком все файлы из `examples/skills/`: пример, переставший разбираться, хуже отсутствующего — он учит неверному. diff --git a/examples/README.md b/examples/README.md index f6ca001..55c71e2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,27 +1,49 @@ # Examples -The format's schema (`skill.schema.yaml`) deliberately **does not close the -lists of values** for `role`, `kind`, `source`, `deliver`, `reasoning` or the -shape of `ref`: behind those words stands the design of a particular -application, and someone else's is of no use to yours. - -Here are samples of what those slots get filled with, plus working skills from -different areas. These are EXAMPLES, not requirements: invent your own values. - -Every file is checked by a test (`examples_test.go`): an example that stopped -parsing is worse than a missing one — it teaches the wrong thing. - -| file | about | what it shows | -|---|---|---| -| [`vocabulary.yaml`](vocabulary.yaml) | a vocabulary of values | what the schema's open slots get filled with | -| [`pods.yaml`](pods.yaml) | listing machines in a cluster | parse the request → call → answer; the server is computed, but only within the declared set | -| [`weather.yaml`](weather.yaml) | weather through a browser | a `call` step without generation; the browser goes only to the step that needs it | -| [`proofread.yaml`](proofread.yaml) | proofreading text | a reference asset is SUBSTITUTED into the instruction — otherwise the model never reads it | -| [`expenses.yaml`](expenses.yaml) | spending as a chart | a code asset and the data go BY REFERENCE past the context; `deliver` is declared in advance | -| [`inbox.yaml`](inbox.yaml) | triaging an email | `switch` + `delegate`: in the "spam" branch the step that replies to the customer simply is not there | -| [`research.yaml`](research.yaml) | an answer from several sources | `parallel` and `.skipped` — "we never went" differs from "it was empty" | -| [`glossary.yaml`](glossary.yaml) | translating terms | `for_each` and `collect`; `in` takes a variable NAME, not a template | -| [`contract.yaml`](contract.yaml) | checking a contract | `if` + `exit` (wrong document — an honest exit) and an external asset with a `fetch` policy | -| [`triage.yaml`](triage.yaml) | triaging an incident | a composite skill: branching and delegation | -| [`audit.yaml`](audit.yaml) | checking a document | `profiles` shared by the classifier steps, and all four `on_empty` outcomes | -| [`menu.yaml`](menu.yaml) | suggesting dishes by section | `contains` — a classifier step replaced by a condition, dictionary and all | +Two things live here: skills, and applications that run them. + +| | | +|---|---| +| [`skills/`](skills/) | the format itself — working skills from different areas, plus a vocabulary of the values the schema deliberately leaves open | +| [`simple-llm-app/`](simple-llm-app/) | the engine embedded in ~200 lines: an OpenAI-compatible endpoint over `net/http`, no dependencies beyond the engine and a YAML parser | +| [`eino-llm-app/`](eino-llm-app/) | the same application with the model reached through [eino](https://github.com/cloudwego/eino) — the whole framework-shaped part is one forty-line adapter | + +## Why each application is its own module + +Both apps have their own `go.mod`, and that is the point rather than tidiness. + +The engine's promise is that embedding it adds no dependencies: production code +is stdlib only, and a guard test fails the build the moment that stops being +true. An example that pulls in a framework would break exactly that promise — +unless it is a separate module, which is what `go.mod` next to it makes it. The +engine's `go.mod` never learns that eino exists. + +Two guards hold this in place, both in `imports_test.go` upstairs: one skips +nested modules while checking that the engine imports nothing, the other checks +that a directory with Go code in it still HAS a `go.mod`. Delete one of those +files and the first guard immediately reports eino as a dependency of the +engine — which is what it is at that moment. + +CI runs `go vet` and `go test` inside each example module separately, because +`go test ./...` at the top never descends into them. An example that does not +build teaches the format wrong. + +## Running one + +``` +cd simple-llm-app +export OPENAI_BASE_URL=http://localhost:8000/v1 # vLLM, Ollama, LM Studio, … +export OPENAI_API_KEY=… +export OPENAI_MODEL=… + +go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" +``` + +Both applications print the answer and then the trace — which step ran, which +was skipped and why, how many tool calls each made. That trace is the engine's +whole observability contract: it logs nothing, stores nothing and reaches +nowhere on its own. + +Neither needs a key to be **tested**: each has a test that runs the whole +application against a stub model, so "it works" is something you can check +rather than something this file claims. diff --git a/examples/eino-llm-app/README.md b/examples/eino-llm-app/README.md new file mode 100644 index 0000000..ca29180 --- /dev/null +++ b/examples/eino-llm-app/README.md @@ -0,0 +1,75 @@ +# eino-llm-app + +The same application as `../simple-llm-app`, with the model reached through +[eino](https://github.com/cloudwego/eino). + +``` +export OPENAI_BASE_URL=http://localhost:8000/v1 +export OPENAI_API_KEY=… +export OPENAI_MODEL=… + +go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" +``` + +## The whole point is `runner.go` + +Forty lines, and everything eino-shaped in this example lives in them. The +adapter takes `model.BaseChatModel` — the interface, not a vendor — so eino's +OpenAI, Ark, Ollama or Qwen components drop in without touching it, and so does +a stub in a test. + +The direction is what matters: **the engine does not know eino, and eino does +not know the engine.** The application owns the seam between them. That is what +makes the engine embeddable in an application built on some other framework +tomorrow — and it is why this directory has its own `go.mod`. + +``` +examples/eino-llm-app/go.mod → requires eino +skill-engine/go.mod → has never heard of it +``` + +A guard test upstairs (`imports_test.go`) enforces both halves: it skips nested +modules when checking that the engine imports nothing, and it fails if a +directory with Go code in it loses its `go.mod`. Delete this one and eino is +reported as a dependency of the engine on the next run — which, at that moment, +is exactly what it would be. + +## What the adapter carries across + +**Declarations.** A step's `model:` and `sampling:` become `model.WithModel`, +`WithTemperature`, `WithTopP`, `WithMaxTokens`. A field the skill sets and the +executor drops is decoration — the author writes `temperature: 0` on a +classifier for a reason, and nothing downstream would tell them it never +arrived. There is a test for exactly this. + +**The step's radius.** An empty tool set is not "no preference": it is the +skill's guard, and the adapter hands the model nothing. Binding tools "just in +case" is how a prohibition quietly stops being one. + +**What the executor knows and the engine cannot derive.** A generation stopped +at the token ceiling comes back as `Result.Truncated` with a reason, and the +engine marks the step degraded on the strength of it. A truncated answer and a +step that simply had nothing to say are indistinguishable from the text alone. + +## Testing it + +``` +go test ./... +``` + +The tests run the application end to end against a stub chat model — no network, +no key — because the adapter takes eino's interface rather than its client. They +check the same flow as the simple app, plus that the skill's declarations really +reach the model as options. + +## The `replace` in go.mod + +``` +replace github.com/inhuman/skill-engine => ../.. +``` + +It points at the engine in this repository, so CI checks the example against the +code as it is now rather than against the last release — an API break is caught +the day it happens, not a version later. **Delete that line when you copy this +example into your own project**; the `require` above it is what you actually +want. diff --git a/examples/eino-llm-app/go.mod b/examples/eino-llm-app/go.mod new file mode 100644 index 0000000..0ea8d09 --- /dev/null +++ b/examples/eino-llm-app/go.mod @@ -0,0 +1,44 @@ +module github.com/inhuman/skill-engine/examples/eino-llm-app + +go 1.26.1 + +require ( + github.com/cloudwego/eino v0.9.13 + github.com/cloudwego/eino-ext/components/model/openai v0.1.13 + github.com/inhuman/skill-engine v0.5.2 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/eino-contrib/jsonschema v1.0.3 // indirect + github.com/evanphx/json-patch v0.5.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/goph/emperror v0.17.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/meguminnnnnnnnn/go-openai v0.1.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nikolalohinski/gonja v1.5.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/yargevad/filepathx v1.0.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect + golang.org/x/sys v0.29.0 // indirect +) + +replace github.com/inhuman/skill-engine => ../.. diff --git a/examples/eino-llm-app/go.sum b/examples/eino-llm-app/go.sum new file mode 100644 index 0000000..d9d92e9 --- /dev/null +++ b/examples/eino-llm-app/go.sum @@ -0,0 +1,151 @@ +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/mockey v1.3.0 h1:ONLRdvhqmCfr9rTasUB8ZKCfvbdD2tohOg4u+4Q/ed0= +github.com/bytedance/mockey v1.3.0/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cloudwego/eino v0.9.13 h1:iD/ETS+lxnNp1VeNPqWVGPWdND6Dbf4LyINbLUlDRcM= +github.com/cloudwego/eino v0.9.13/go.mod h1:OBD1mrkfkt/pJa4rkg1P0VnaMeOVl7l8IAdEqY//3IQ= +github.com/cloudwego/eino-ext/components/model/openai v0.1.13 h1:5XHRTiTD5bt9KQrMHcfvuWNklEC3tpm3XHejdozt9vM= +github.com/cloudwego/eino-ext/components/model/openai v0.1.13/go.mod h1:mgIoqYYOc0eECCqvLbEYpOJrQNTNxkwXzSJzFU+v5sQ= +github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 h1:EeVcR1TslRA2IdNW1h/2LaGbPlffwGhQm99jM3zWZiI= +github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17/go.mod h1:Zkcx6DPTR2NfWmtSXbhItswGw6hqUezNPhNcke0pOG8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eino-contrib/jsonschema v1.0.3 h1:2Kfsm1xlMV0ssY2nuxshS4AwbLFuqmPmzIjLVJ1Fsp0= +github.com/eino-contrib/jsonschema v1.0.3/go.mod h1:cpnX4SyKjWjGC7iN2EbhxaTdLqGjCi0e9DxpLYxddD4= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/meguminnnnnnnnn/go-openai v0.1.2 h1:iXombGGjqjBrmE9WaSidUhhi3YQhf42QTHvHLMkgvCA= +github.com/meguminnnnnnnnn/go-openai v0.1.2/go.mod h1:qs96ysDmxhE4BZoU45I43zcyfnaYxU3X+aRzLko/htY= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/eino-llm-app/main.go b/examples/eino-llm-app/main.go new file mode 100644 index 0000000..6352d34 --- /dev/null +++ b/examples/eino-llm-app/main.go @@ -0,0 +1,121 @@ +// The same application, with the model reached through eino. +// +// The point of this example is one type — the adapter below — and one fact +// about it: eino appears in THIS module's go.mod and nowhere else. The engine +// stays dependency-free, and an application built on a framework plugs the +// framework in at the only seam that touches a model. +// +// go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" +// +// Set OPENAI_BASE_URL, OPENAI_API_KEY and OPENAI_MODEL for your endpoint; eino's +// OpenAI component speaks to anything OpenAI-compatible. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + + "github.com/cloudwego/eino-ext/components/model/openai" + "github.com/cloudwego/eino/components/model" + "gopkg.in/yaml.v3" + + se "github.com/inhuman/skill-engine" +) + +func main() { + skillPath := flag.String("skill", "../skills/menu.yaml", "path to a skill file") + input := flag.String("input", "подбери десерт и напиток", "the user's request") + flag.Parse() + + chat, err := newChatModel(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, "cannot build the model:", err) + os.Exit(1) + } + if err := run(chat, *skillPath, *input, os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +// newChatModel builds eino's OpenAI component. Any other implementation from +// eino-ext — Ark, Ollama, Qwen — drops in here without touching the adapter: +// the adapter knows the INTERFACE, not the vendor. +func newChatModel(ctx context.Context) (model.BaseChatModel, error) { + return openai.NewChatModel(ctx, &openai.ChatModelConfig{ + BaseURL: env("OPENAI_BASE_URL", "https://api.openai.com/v1"), + APIKey: os.Getenv("OPENAI_API_KEY"), + Model: env("OPENAI_MODEL", "gpt-4o-mini"), + }) +} + +func env(name, fallback string) string { + if v := os.Getenv(name); v != "" { + return v + } + return fallback +} + +func run(chat model.BaseChatModel, skillPath, input string, out io.Writer) error { + raw, err := os.ReadFile(skillPath) + if err != nil { + return err + } + skill, err := se.ParseSkill(raw, yaml.Unmarshal) + if err != nil { + return err + } + if err := skill.Validate(); err != nil { + return err + } + + vars, outcome, err := se.ExecuteWith(context.Background(), skill.Workflow, se.Deps{ + Runner: runner{chat: chat, log: out}, + Caller: tools{log: out}, + Assets: assets{}, + Vocabulary: se.Vocabulary{ + DecisionMarkers: []string{"Result:", "Ответ:"}, + }, + OnStepStart: func(name, kind string) { fmt.Fprintf(out, "→ %s (%s)\n", name, kind) }, + }, map[string]string{"input": input}) + if err != nil { + if errors.Is(err, se.ErrExit) { + fmt.Fprintln(out, "the skill stepped aside:", err) + return nil + } + return err + } + + fmt.Fprintln(out, "\n=== answer ===") + fmt.Fprintln(out, vars[se.AnswerVar]) + fmt.Fprintln(out, "\n=== steps ===") + for _, s := range outcome.Steps { + fmt.Fprintf(out, " %-16s %-11s %-8s %s\n", s.Name, s.Kind, s.Outcome, s.Reason) + } + return nil +} + +// tools and assets are the same as in ../simple-llm-app: eino changes how the +// MODEL is reached and nothing else about embedding the engine. +type tools struct{ log io.Writer } + +func (t tools) CallTool(_ context.Context, server, tool string, args map[string]any) (string, error) { + fmt.Fprintf(t.log, " call %s:%s %v\n", server, tool, args) + if server+":"+tool == "recipes:search" { + return "1. Тирамису — 30 минут\n2. Панна-котта — 15 минут", nil + } + return "", fmt.Errorf("no such tool: %s:%s", server, tool) +} + +type assets struct{} + +func (assets) Resolve(_ context.Context, name string, a se.Asset) (string, error) { + if a.Source == "inline" || a.Content != "" { + return a.Content, nil + } + return "", fmt.Errorf("asset %q: source %q is not implemented in this example", name, a.Source) +} diff --git a/examples/eino-llm-app/main_test.go b/examples/eino-llm-app/main_test.go new file mode 100644 index 0000000..b02f0a6 --- /dev/null +++ b/examples/eino-llm-app/main_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" +) + +// stubChat implements eino's chat model interface and records what it was +// asked. Because the adapter takes the INTERFACE rather than a vendor's client, +// the example can be verified end to end with no network and no key — and so +// can yours. +type stubChat struct { + answer string + seen []*schema.Message + opts int +} + +func (s *stubChat) Generate(_ context.Context, in []*schema.Message, opts ...model.Option) (*schema.Message, error) { + s.seen = append(s.seen, in...) + s.opts = len(opts) + return schema.AssistantMessage(s.answer, nil), nil +} + +func (s *stubChat) Stream(context.Context, []*schema.Message, ...model.Option) (*schema.StreamReader[*schema.Message], error) { + panic("this example only uses Generate") +} + +// The whole example against a shipped skill: conditions choose the sections the +// request names, `call` steps fetch them with no model involved, and the last +// step words the answer through eino. +func TestExampleRunsAShippedSkill(t *testing.T) { + chat := &stubChat{answer: "Тирамису и чай — 30 минут."} + + var out bytes.Buffer + if err := run(chat, "../skills/menu.yaml", "подбери десерт и напиток", &out); err != nil { + t.Fatalf("the example does not run: %v", err) + } + got := out.String() + + for _, want := range []string{"Тирамису и чай", "call recipes:search", "pick_dessert"} { + if !strings.Contains(got, want) { + t.Errorf("the output does not mention %q:\n%s", want, got) + } + } + if n := strings.Count(got, "call recipes:search"); n != 2 { + t.Errorf("expected two sections fetched, got %d:\n%s", n, got) + } + if len(chat.seen) != 1 { + t.Fatalf("expected one generation, got %d", len(chat.seen)) + } + if !strings.Contains(chat.seen[0].Content, "Тирамису") { + t.Errorf("what the tools returned never reached the model:\n%s", chat.seen[0].Content) + } +} + +// What a skill declares must arrive at the model. A step's `model:` and +// `sampling:` are translated into eino options; an adapter that drops them +// turns those fields into decoration, and nothing tells the author. +func TestDeclarationsReachTheModel(t *testing.T) { + chat := &stubChat{answer: "ok"} + var out bytes.Buffer + + if err := run(chat, "testdata/declared.yaml", "что угодно", &out); err != nil { + t.Fatalf("the example does not run: %v", err) + } + // model + temperature + max_tokens + if chat.opts != 3 { + t.Errorf("expected three options passed to eino, got %d", chat.opts) + } +} + +// A skill that leaves through `exit` did not fail: it decided the request was +// not its case. An embedder reporting that as an error reports a bug where the +// skill made a decision. +func TestExampleHandlesAnExit(t *testing.T) { + chat := &stubChat{answer: "unused"} + + var out bytes.Buffer + if err := run(chat, "../skills/menu.yaml", "расскажи что-нибудь", &out); err != nil { + t.Fatalf("an exit was reported as a failure: %v", err) + } + if !strings.Contains(out.String(), "stepped aside") { + t.Errorf("the exit was not recognised:\n%s", out.String()) + } +} diff --git a/examples/eino-llm-app/runner.go b/examples/eino-llm-app/runner.go new file mode 100644 index 0000000..5e34de7 --- /dev/null +++ b/examples/eino-llm-app/runner.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "fmt" + "io" + + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + + se "github.com/inhuman/skill-engine" +) + +// runner is the whole adapter: skill-engine's Runner on top of any eino chat +// model. Everything eino-shaped in this example lives in these forty lines. +// +// The direction matters. The engine does not know eino, and eino does not know +// the engine — the application owns the seam between them, which is exactly +// what makes the engine embeddable in an application built on some other +// framework tomorrow. +type runner struct { + chat model.BaseChatModel + log io.Writer +} + +func (r runner) Run(ctx context.Context, req se.StepRequest) (se.Result, error) { + // What the SKILL declared, translated into what eino calls it. A field the + // skill sets and the executor drops is decoration: the author writes + // `sampling: {temperature: 0}` on a classifier for a reason, and nothing + // downstream tells them it never arrived. + var opts []model.Option + if req.Model != "" { + opts = append(opts, model.WithModel(req.Model)) + } + if s := req.Sampling; s != nil { + if s.Temperature != nil { + opts = append(opts, model.WithTemperature(*s.Temperature)) + } + if s.TopP != nil { + opts = append(opts, model.WithTopP(*s.TopP)) + } + if s.MaxTokens != nil { + opts = append(opts, model.WithMaxTokens(*s.MaxTokens)) + } + } + + // The step's radius. An EMPTY list is not "no preference" — it is the + // skill's guard, and a step that was handed no tools must be given none + // here either. Binding tools "just in case" is how a prohibition quietly + // stops being one. + if len(req.Tools) > 0 { + fmt.Fprintf(r.log, " (step %q may use: %v)\n", req.Name, req.Tools) + // A real application resolves these names to eino tools and passes them + // with model.WithTools, or hands the step to an eino agent. This example + // keeps `call` steps for the deterministic work and leaves the model to + // think, which is the cheaper shape anyway. + } + + msg, err := r.chat.Generate(ctx, []*schema.Message{schema.UserMessage(req.Instruction)}, opts...) + if err != nil { + return se.Result{}, err + } + + // A truncated generation is not an empty one, and the engine cannot tell + // them apart from the text. Whatever the executor KNOWS goes back in the + // Result — that is what a step is marked degraded on. + res := se.Result{Text: msg.Content} + if msg.ResponseMeta != nil && msg.ResponseMeta.FinishReason == "length" { + res.Truncated = true + res.Note = "the model stopped at the token ceiling" + } + return res, nil +} diff --git a/examples/eino-llm-app/testdata/declared.yaml b/examples/eino-llm-app/testdata/declared.yaml new file mode 100644 index 0000000..f1c719b --- /dev/null +++ b/examples/eino-llm-app/testdata/declared.yaml @@ -0,0 +1,16 @@ +# A one-step skill whose only job is to declare things and check they arrive. +skill_engine_version: "2.2.2" +skill_version: "1.0.0" +name: declared +description: A fixture — a step that names its model and its sampling. +trigger_examples: ["anything"] + +workflow: + steps: + - name: answer + instruction: "Ответь на: {{input}}" + tools: [] + model: some-small-model + sampling: + temperature: 0 + max_tokens: 128 diff --git a/examples/simple-llm-app/README.md b/examples/simple-llm-app/README.md new file mode 100644 index 0000000..1ce5feb --- /dev/null +++ b/examples/simple-llm-app/README.md @@ -0,0 +1,73 @@ +# simple-llm-app + +The engine embedded in an application, with nothing between it and the model but +`net/http`. + +``` +export OPENAI_BASE_URL=http://localhost:8000/v1 # vLLM, Ollama, LM Studio, … +export OPENAI_API_KEY=… +export OPENAI_MODEL=… + +go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" +``` + +## What it shows + +**Loading a skill.** `ParseSkill` reads the whole file — header and description +alike — and `Validate` is a separate call on purpose: "this is not YAML" and +"this YAML says something wrong" are different answers, and an application +reporting on a file has to tell them apart. + +**Both halves of the format.** `ResolveMode` says which description runs. A +skill may carry a `playbook` instead of steps, and then the engine takes no part +at all — the application runs the prompt itself. Handling that case is what makes +an embedder complete rather than half-written. + +**The four seams.** `Deps` is the entire contract: + +| | | +|---|---| +| `Runner` | the ONE place the engine touches a model. The step arrives resolved — variables substituted, assets inlined, tools narrowed — and all that is left is to generate | +| `Caller` | a `call` step: a tool invocation with no model involved. Here it is a table so the example runs offline; in your application it is your MCP client | +| `Assets` | payloads a skill declares. Even `source: inline` comes through here — an asset is the application's to fetch, cache and police | +| `Memory` | a large result by the handle the host appended to it | + +**Vocabulary.** The engine ships no words of its own. What your model writes +before naming a choice, and how your host marks a shortened result, are declared +here — an agent about a kitchen and one about a car fleet share the format, not +a language. + +**What the skill declared has to arrive.** A step's `model:`, its `sampling:`, +its `response_schema:` are forwarded to the endpoint. An executor that drops +them turns every one of those fields into decoration: the skill declares, and +nothing happens, and nothing says so. + +**Failing properly.** A tool that does not exist returns an error rather than an +apology — the skill's `on_error` decides what happens next, and it can only +decide if it is told. A skill leaving through `exit` is not a failure: it decided +the request was not its case. + +## Testing it + +``` +go test ./... +``` + +The test runs the whole application against a stub OpenAI-compatible server, on +a skill shipped in `../skills`. No key, no network. It checks that the +conditions picked the sections the request named, that the `call` steps ran +without a model, that a section nobody asked for was skipped **and still +reported as skipped** — "we did not go there" and "it came back empty" are +different answers — and that every shipped skill still loads through this path. + +## The `replace` in go.mod + +``` +replace github.com/inhuman/skill-engine => ../.. +``` + +It points at the engine in this repository, so CI checks the example against the +code as it is now rather than against the last release — an API break is caught +the day it happens, not a version later. **Delete that line when you copy this +example into your own project**; the `require` above it is what you actually +want. diff --git a/examples/simple-llm-app/go.mod b/examples/simple-llm-app/go.mod new file mode 100644 index 0000000..93397e3 --- /dev/null +++ b/examples/simple-llm-app/go.mod @@ -0,0 +1,10 @@ +module github.com/inhuman/skill-engine/examples/simple-llm-app + +go 1.26.1 + +require ( + github.com/inhuman/skill-engine v0.5.2 + gopkg.in/yaml.v3 v3.0.1 +) + +replace github.com/inhuman/skill-engine => ../.. diff --git a/examples/simple-llm-app/go.sum b/examples/simple-llm-app/go.sum new file mode 100644 index 0000000..c4c1710 --- /dev/null +++ b/examples/simple-llm-app/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/simple-llm-app/main.go b/examples/simple-llm-app/main.go new file mode 100644 index 0000000..f0db2a7 --- /dev/null +++ b/examples/simple-llm-app/main.go @@ -0,0 +1,210 @@ +// A minimal application that embeds the skill engine. +// +// It shows the whole contract in one file: load a skill, hand the engine the +// four things it cannot do itself — talk to a model, call a tool, resolve an +// asset, read working memory — and print what came back. +// +// The model is reached over the OpenAI-compatible /chat/completions endpoint, +// which is what vLLM, Ollama, LM Studio and the hosted APIs all speak, using +// nothing but net/http. That keeps this example honest about the engine's own +// promise: it adds no dependencies of its own, and neither does using it. +// +// go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" +// +// Set OPENAI_BASE_URL and OPENAI_API_KEY for your endpoint. There is no key in +// the code and no default that quietly points somewhere. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "strings" + + "gopkg.in/yaml.v3" + + se "github.com/inhuman/skill-engine" +) + +func main() { + skillPath := flag.String("skill", "../skills/menu.yaml", "path to a skill file") + input := flag.String("input", "подбери десерт и напиток", "the user's request") + flag.Parse() + + if err := run(*skillPath, *input, os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +func run(skillPath, input string, out io.Writer) error { + raw, err := os.ReadFile(skillPath) + if err != nil { + return err + } + + // One call reads the whole file — header and description alike. Validate is + // separate on purpose: "this is not YAML" and "this YAML says something + // wrong" are different answers, and a tool reporting on a file needs to + // tell them apart. + skill, err := se.ParseSkill(raw, yaml.Unmarshal) + if err != nil { + return err + } + if err := skill.Validate(); err != nil { + return err + } + + // Which of the two descriptions to run. A skill may carry a `playbook` as + // well, and then the ENGINE takes no part: the application runs the prompt + // itself. Handling that case is what makes an embedder complete. + mode, err := skill.ResolveMode() + if err != nil { + return err + } + if mode == se.ModePlaybook { + answer, err := newModel().complete(context.Background(), skill.Playbook+"\n\n"+input, nil) + if err != nil { + return err + } + fmt.Fprintln(out, answer) + return nil + } + + vars, outcome, err := se.ExecuteWith(context.Background(), skill.Workflow, deps(out), map[string]string{ + "input": input, + }) + if err != nil { + // A skill leaving on purpose is not a failure: it decided the request + // was not its case, and the turn goes back to its ordinary path. + if errors.Is(err, se.ErrExit) { + fmt.Fprintln(out, "the skill stepped aside:", err) + return nil + } + return err + } + + fmt.Fprintln(out, "\n=== answer ===") + fmt.Fprintln(out, vars[se.AnswerVar]) + fmt.Fprintln(out, "\n=== steps ===") + for _, s := range outcome.Steps { + line := fmt.Sprintf(" %-16s %-11s %-8s calls=%d", s.Name, s.Kind, s.Outcome, s.Calls) + if s.Reason != "" { + line += " " + s.Reason + } + fmt.Fprintln(out, line) + } + if len(outcome.Skipped) > 0 { + fmt.Fprintln(out, " skipped:", strings.Join(outcome.Skipped, ", ")) + } + return nil +} + +// deps is the whole contract between an application and the engine. +// +// Everything the engine cannot know — how to reach a model, what a tool is, +// where an asset's content lives, what your host calls things — arrives here. +// Nothing else is injected, and the engine logs nothing, stores nothing and +// reaches nowhere on its own. +func deps(out io.Writer) se.Deps { + m := newModel() + return se.Deps{ + Runner: runner{model: m, log: out}, + Caller: tools{log: out}, + Assets: assets{}, + Delegate: nil, // no composite skills here: a `delegate` step would fail loudly + Memory: memory{}, + + // The words of THIS application. The engine ships none — an agent about + // a kitchen and one about a car fleet share the format, not a language. + Vocabulary: se.Vocabulary{ + DecisionMarkers: []string{"Result:", "Answer:", "Итог:", "Ответ:"}, + TruncationNotes: []string{"shortened:"}, + }, + + // A step that runs for ten seconds emits nothing until it finishes, and + // there is nothing to show a human all that time. + OnStepStart: func(name, kind string) { + fmt.Fprintf(out, "→ %s (%s)\n", name, kind) + }, + } +} + +// runner executes an instruction step: it is the ONE place the engine touches a +// model. +// +// The step arrives fully resolved — variables substituted, assets inlined, the +// tool set narrowed, the sampling decided. All that is left is to generate. +type runner struct { + model *openAI + log io.Writer +} + +func (r runner) Run(ctx context.Context, req se.StepRequest) (se.Result, error) { + // The tool set is the step's radius, and an EMPTY one is not "no + // preference": the step is meant to answer from what has already been + // gathered. Handing it tools anyway would undo the guard the skill relies + // on — so it is passed through as it came. + if len(req.Tools) > 0 { + fmt.Fprintf(r.log, " (step %q may use: %s)\n", req.Name, strings.Join(req.Tools, ", ")) + } + + text, err := r.model.complete(ctx, req.Instruction, &req) + if err != nil { + return se.Result{}, err + } + + // Result carries more than the text: what the executor KNOWS and the engine + // cannot derive. A truncated generation and a step that simply had nothing + // to say look identical from the outside, and the engine marks a step + // degraded on the strength of these fields. + return se.Result{Text: text}, nil +} + +// tools executes a `call` step — a tool invocation with no model involved. +// +// In a real application this is your MCP client, your HTTP client, your +// function registry. Here it is a table, so the example runs offline and every +// skill in ../skills can be tried out. +type tools struct{ log io.Writer } + +func (t tools) CallTool(_ context.Context, server, tool string, args map[string]any) (string, error) { + fmt.Fprintf(t.log, " call %s:%s %v\n", server, tool, args) + + switch server + ":" + tool { + case "recipes:search": + return "1. Тирамису — 30 минут\n2. Панна-котта — 15 минут", nil + } + // A tool that does not exist must FAIL, not return an apology: the skill's + // on_error policy decides what happens next, and it can only decide if it + // is told. + return "", fmt.Errorf("no such tool: %s:%s", server, tool) +} + +// assets resolve the payloads a skill declares. +// +// `source: inline` is the engine's own case and still comes through here — the +// engine deliberately does not read `content` itself, because an asset is the +// application's to fetch, cache and police. Every other source is yours: a +// repository, a URL, a file the user uploaded. +type assets struct{} + +func (assets) Resolve(_ context.Context, name string, a se.Asset) (string, error) { + if a.Source == "inline" || a.Content != "" { + return a.Content, nil + } + return "", fmt.Errorf("asset %q: source %q is not implemented in this example", name, a.Source) +} + +// memory returns a large result by the handle the host appended to it. +// +// This example never truncates anything, so the map stays empty and the engine +// simply never asks. It is here to show WHERE the seam is: without a reader, a +// step that cannot fetch the rest of a value is told so in its trace instead of +// quietly working on a fragment. +type memory map[string]string + +func (m memory) Get(id string) (string, bool) { v, ok := m[id]; return v, ok } diff --git a/examples/simple-llm-app/main_test.go b/examples/simple-llm-app/main_test.go new file mode 100644 index 0000000..cad42bc --- /dev/null +++ b/examples/simple-llm-app/main_test.go @@ -0,0 +1,150 @@ +package main + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// stubModel — an OpenAI-compatible endpoint that answers whatever it is told +// to. The example is verified end to end against it: an example that only +// compiles teaches nothing about whether it works. +func stubModel(t *testing.T, answer string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat/completions" { + t.Errorf("unexpected path %q — the endpoint is not the one an OpenAI-compatible server exposes", r.URL.Path) + } + var body struct { + Model string `json:"model"` + Messages []struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"messages"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("the request is not JSON: %v", err) + } + if len(body.Messages) == 0 { + t.Error("the instruction never reached the model") + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{ + "choices": []any{map[string]any{ + "message": map[string]string{"content": answer}, + "finish_reason": "stop", + }}, + }) + })) + t.Cleanup(srv.Close) + return srv +} + +func withStub(t *testing.T, answer string) { + t.Helper() + srv := stubModel(t, answer) + t.Setenv("OPENAI_BASE_URL", srv.URL) + t.Setenv("OPENAI_API_KEY", "not-a-real-key") + t.Setenv("OPENAI_MODEL", "stub") +} + +// The whole example, on a skill shipped in ../skills: conditions pick the +// sections the request names, `call` steps fetch them without a model, and the +// last step words the answer. +func TestExampleRunsAShippedSkill(t *testing.T) { + withStub(t, "Тирамису и чай — 30 минут.") + + var out bytes.Buffer + if err := run("../skills/menu.yaml", "подбери десерт и напиток", &out); err != nil { + t.Fatalf("the example does not run: %v", err) + } + got := out.String() + + for _, want := range []string{ + "Тирамису и чай", // the model's answer came back as the turn's answer + "call recipes:search", // a `call` step ran without a model + "pick_dessert", // the trace names the steps + "answer", // + } { + if !strings.Contains(got, want) { + t.Errorf("the output does not mention %q:\n%s", want, got) + } + } + // A section the request never named must not be FETCHED — but it must still + // be visible as skipped. "we did not go there" and "it came back empty" are + // different answers, and the trace is where the difference survives. + if n := strings.Count(got, "call recipes:search"); n != 2 { + t.Errorf("expected two sections fetched, got %d:\n%s", n, got) + } + if !strings.Contains(got, "skipped: nothing_named, pick_main") { + t.Errorf("the skipped steps are not reported:\n%s", got) + } +} + +// A request naming no section leaves the skill through `exit`, and that is not +// a failure: the turn goes back to its ordinary path. An embedder that treats +// it as an error reports a bug where the skill made a decision. +func TestExampleHandlesAnExit(t *testing.T) { + withStub(t, "unused") + + var out bytes.Buffer + if err := run("../skills/menu.yaml", "расскажи что-нибудь", &out); err != nil { + t.Fatalf("an exit was reported as a failure: %v", err) + } + if !strings.Contains(out.String(), "stepped aside") { + t.Errorf("the exit was not recognised:\n%s", out.String()) + } +} + +// Every skill in ../skills must at least load and validate through the same +// path the application uses. A shipped example that stopped parsing teaches the +// wrong thing, and this is the cheapest place to notice. +func TestEveryShippedSkillLoads(t *testing.T) { + withStub(t, "answer") + + for _, path := range shippedSkills(t) { + t.Run(path, func(t *testing.T) { + var out bytes.Buffer + err := run(path, "подбери десерт", &out) + // Running is allowed to fail — most of these skills need tools this + // example does not implement. Loading is not. + if err != nil && strings.Contains(err.Error(), "skill-engine:") { + t.Fatalf("the skill did not load: %v", err) + } + }) + } +} + +// shippedSkills lists the skill files next door, minus the vocabulary of values +// (it has no name and is not a skill). +func shippedSkills(t *testing.T) []string { + t.Helper() + all, err := filepath.Glob(filepath.Join("..", "skills", "*.yaml")) + if err != nil || len(all) == 0 { + t.Fatalf("the skills are gone: %v", err) + } + var out []string + for _, path := range all { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var doc struct { + Name string `yaml:"name"` + } + if err := yaml.Unmarshal(raw, &doc); err != nil { + t.Fatalf("%s: %v", path, err) + } + if doc.Name != "" { + out = append(out, path) + } + } + return out +} diff --git a/examples/simple-llm-app/openai.go b/examples/simple-llm-app/openai.go new file mode 100644 index 0000000..c11a48b --- /dev/null +++ b/examples/simple-llm-app/openai.go @@ -0,0 +1,123 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + "time" + + se "github.com/inhuman/skill-engine" +) + +// openAI — a chat completion over the OpenAI-compatible endpoint, in net/http. +// +// Sixty lines, no client library. That is not thrift for its own sake: the +// engine's promise is that embedding it adds no dependencies, and an example +// that pulls in an SDK to prove the point would disprove it. +type openAI struct { + baseURL string + apiKey string + model string + client *http.Client +} + +func newModel() *openAI { + return &openAI{ + baseURL: strings.TrimSuffix(env("OPENAI_BASE_URL", "https://api.openai.com/v1"), "/"), + apiKey: os.Getenv("OPENAI_API_KEY"), + model: env("OPENAI_MODEL", "gpt-4o-mini"), + client: &http.Client{Timeout: 2 * time.Minute}, + } +} + +func env(name, fallback string) string { + if v := os.Getenv(name); v != "" { + return v + } + return fallback +} + +// complete sends one instruction and returns the answer's text. +// +// req is the step as the engine resolved it, and it carries more than the +// prompt. Passing those fields on is what makes a step's declarations real: +// a `model:` the skill named, its `sampling:`, its `response_schema:`. An +// executor that ignores them turns every one of those fields into decoration — +// the skill declares, the engine forwards, and nothing happens. +func (o *openAI) complete(ctx context.Context, instruction string, req *se.StepRequest) (string, error) { + body := map[string]any{ + "model": o.model, + "messages": []map[string]string{{"role": "user", "content": instruction}}, + } + if req != nil { + if req.Model != "" { + body["model"] = req.Model + } + if s := req.Sampling; s != nil { + if s.Temperature != nil { + body["temperature"] = *s.Temperature + } + if s.TopP != nil { + body["top_p"] = *s.TopP + } + if s.MaxTokens != nil { + body["max_tokens"] = *s.MaxTokens + } + } + // A structured answer is only structured where the decoding grammar + // holds it. Dropping the schema here is the silent hole the format + // warns about: the model "usually" answers JSON, and the step that + // parses it fails on the day it does not. + if len(req.ResponseSchema) > 0 { + body["response_format"] = map[string]any{ + "type": "json_schema", + "json_schema": map[string]any{ + "name": "step", + "strict": true, + "schema": req.ResponseSchema, + }, + } + } + } + + raw, err := json.Marshal(body) + if err != nil { + return "", err + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, o.baseURL+"/chat/completions", bytes.NewReader(raw)) + if err != nil { + return "", err + } + httpReq.Header.Set("Content-Type", "application/json") + if o.apiKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+o.apiKey) + } + + resp, err := o.client.Do(httpReq) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var out struct { + Choices []struct { + Message struct{ Content string } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Error *struct{ Message string } `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", fmt.Errorf("decoding the answer: %w", err) + } + if out.Error != nil { + return "", fmt.Errorf("model: %s", out.Error.Message) + } + if len(out.Choices) == 0 { + return "", fmt.Errorf("model returned no choices") + } + return out.Choices[0].Message.Content, nil +} diff --git a/examples/skills/README.md b/examples/skills/README.md new file mode 100644 index 0000000..f6ca001 --- /dev/null +++ b/examples/skills/README.md @@ -0,0 +1,27 @@ +# Examples + +The format's schema (`skill.schema.yaml`) deliberately **does not close the +lists of values** for `role`, `kind`, `source`, `deliver`, `reasoning` or the +shape of `ref`: behind those words stands the design of a particular +application, and someone else's is of no use to yours. + +Here are samples of what those slots get filled with, plus working skills from +different areas. These are EXAMPLES, not requirements: invent your own values. + +Every file is checked by a test (`examples_test.go`): an example that stopped +parsing is worse than a missing one — it teaches the wrong thing. + +| file | about | what it shows | +|---|---|---| +| [`vocabulary.yaml`](vocabulary.yaml) | a vocabulary of values | what the schema's open slots get filled with | +| [`pods.yaml`](pods.yaml) | listing machines in a cluster | parse the request → call → answer; the server is computed, but only within the declared set | +| [`weather.yaml`](weather.yaml) | weather through a browser | a `call` step without generation; the browser goes only to the step that needs it | +| [`proofread.yaml`](proofread.yaml) | proofreading text | a reference asset is SUBSTITUTED into the instruction — otherwise the model never reads it | +| [`expenses.yaml`](expenses.yaml) | spending as a chart | a code asset and the data go BY REFERENCE past the context; `deliver` is declared in advance | +| [`inbox.yaml`](inbox.yaml) | triaging an email | `switch` + `delegate`: in the "spam" branch the step that replies to the customer simply is not there | +| [`research.yaml`](research.yaml) | an answer from several sources | `parallel` and `.skipped` — "we never went" differs from "it was empty" | +| [`glossary.yaml`](glossary.yaml) | translating terms | `for_each` and `collect`; `in` takes a variable NAME, not a template | +| [`contract.yaml`](contract.yaml) | checking a contract | `if` + `exit` (wrong document — an honest exit) and an external asset with a `fetch` policy | +| [`triage.yaml`](triage.yaml) | triaging an incident | a composite skill: branching and delegation | +| [`audit.yaml`](audit.yaml) | checking a document | `profiles` shared by the classifier steps, and all four `on_empty` outcomes | +| [`menu.yaml`](menu.yaml) | suggesting dishes by section | `contains` — a classifier step replaced by a condition, dictionary and all | diff --git a/examples/audit.yaml b/examples/skills/audit.yaml similarity index 100% rename from examples/audit.yaml rename to examples/skills/audit.yaml diff --git a/examples/contract.yaml b/examples/skills/contract.yaml similarity index 100% rename from examples/contract.yaml rename to examples/skills/contract.yaml diff --git a/examples/expenses.yaml b/examples/skills/expenses.yaml similarity index 100% rename from examples/expenses.yaml rename to examples/skills/expenses.yaml diff --git a/examples/glossary.yaml b/examples/skills/glossary.yaml similarity index 100% rename from examples/glossary.yaml rename to examples/skills/glossary.yaml diff --git a/examples/inbox.yaml b/examples/skills/inbox.yaml similarity index 100% rename from examples/inbox.yaml rename to examples/skills/inbox.yaml diff --git a/examples/menu.yaml b/examples/skills/menu.yaml similarity index 100% rename from examples/menu.yaml rename to examples/skills/menu.yaml diff --git a/examples/pods.yaml b/examples/skills/pods.yaml similarity index 100% rename from examples/pods.yaml rename to examples/skills/pods.yaml diff --git a/examples/proofread.yaml b/examples/skills/proofread.yaml similarity index 100% rename from examples/proofread.yaml rename to examples/skills/proofread.yaml diff --git a/examples/research.yaml b/examples/skills/research.yaml similarity index 100% rename from examples/research.yaml rename to examples/skills/research.yaml diff --git a/examples/triage.yaml b/examples/skills/triage.yaml similarity index 100% rename from examples/triage.yaml rename to examples/skills/triage.yaml diff --git a/examples/vocabulary.yaml b/examples/skills/vocabulary.yaml similarity index 100% rename from examples/vocabulary.yaml rename to examples/skills/vocabulary.yaml diff --git a/examples/weather.yaml b/examples/skills/weather.yaml similarity index 100% rename from examples/weather.yaml rename to examples/skills/weather.yaml diff --git a/examples_test.go b/examples_test.go index 78d8533..ffa1bc6 100644 --- a/examples_test.go +++ b/examples_test.go @@ -18,7 +18,7 @@ import ( // fields would draw the wrong conclusion from it. func exampleFiles(t *testing.T) []string { t.Helper() - all, err := filepath.Glob(filepath.Join("examples", "*.yaml")) + all, err := filepath.Glob(filepath.Join("examples", "skills", "*.yaml")) require.NoError(t, err) require.NotEmpty(t, all, "the examples are gone") @@ -69,7 +69,7 @@ func readWorkflow(t *testing.T, path string) (se.Flow, bool) { // example that stopped parsing is worse than a missing one — it teaches the // wrong thing. func TestExamplesParseAndValidate(t *testing.T) { - files, err := filepath.Glob(filepath.Join("examples", "*.yaml")) + files, err := filepath.Glob(filepath.Join("examples", "skills", "*.yaml")) require.NoError(t, err) require.NotEmpty(t, files, "the examples are gone") @@ -89,7 +89,7 @@ func TestExamplesParseAndValidate(t *testing.T) { // a skill of another major is refused, so an example carrying a stale version // teaches a file that will not run. func TestExamplesDeclareCurrentFormat(t *testing.T) { - files, err := filepath.Glob(filepath.Join("examples", "*.yaml")) + files, err := filepath.Glob(filepath.Join("examples", "skills", "*.yaml")) require.NoError(t, err) for _, path := range files { @@ -112,7 +112,7 @@ func TestExamplesDeclareCurrentFormat(t *testing.T) { // An example must WORK, not merely parse: steps reach the executors, the server // is computed, the answer arrives. func TestExamplePodsRuns(t *testing.T) { - f, ok := readWorkflow(t, filepath.Join("examples", "pods.yaml")) + f, ok := readWorkflow(t, filepath.Join("examples", "skills", "pods.yaml")) require.True(t, ok) answers := map[string]string{ diff --git a/imports_test.go b/imports_test.go index 5b19be4..a013a8c 100644 --- a/imports_test.go +++ b/imports_test.go @@ -3,6 +3,7 @@ package skillengine_test import ( "go/build" "io/fs" + "os" "path/filepath" "strings" "testing" @@ -75,6 +76,19 @@ func packageDirs(t *testing.T) []string { if name := d.Name(); path != "." && (strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_")) { return fs.SkipDir } + // A directory with its own go.mod is a DIFFERENT module and not part of + // this one. That is what lets the examples show the engine embedded in + // an application built on a framework — eino appears in that module's + // go.mod and never in the engine's. + // + // The guard is not weakened by this: delete such a go.mod and the files + // join this module, the walk finds them again, and their imports are + // reported like anyone else's. + if path != "." { + if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil { + return fs.SkipDir + } + } if files, _ := filepath.Glob(filepath.Join(path, "*.go")); len(files) > 0 { dirs = append(dirs, path) } @@ -111,3 +125,36 @@ func slicesContains(list []string, v string) bool { } return false } + +// The examples must stay SEPARATE modules, and this is the guard for that. +// +// It is the only thing standing between "an example shows the engine used with +// a framework" and "the engine depends on a framework": delete one of these +// go.mod files and every import inside becomes an import of this module, which +// TestEngineStaysSelfContained would then report — but only if somebody +// remembers the example was supposed to be its own module in the first place. +// This says so out loud. +func TestExampleAppsAreSeparateModules(t *testing.T) { + entries, err := os.ReadDir("examples") + if err != nil { + t.Fatalf("reading examples: %v", err) + } + var apps int + for _, e := range entries { + if !e.IsDir() { + continue + } + dir := filepath.Join("examples", e.Name()) + files, _ := filepath.Glob(filepath.Join(dir, "*.go")) + if len(files) == 0 { + continue // examples/skills — YAML, part of no module + } + apps++ + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil { + t.Errorf("%s holds Go code and no go.mod — its dependencies would become the engine's", dir) + } + } + if apps == 0 { + t.Fatal("no example application found — the walk is looking at the wrong place") + } +} diff --git a/lint/fixtures_test.go b/lint/fixtures_test.go index e05cba4..f03d155 100644 --- a/lint/fixtures_test.go +++ b/lint/fixtures_test.go @@ -125,7 +125,7 @@ func TestCatalogueIdsAreUnique(t *testing.T) { // clean — an example that trips a rule teaches the defect along with the format, // and this is also the false-positive guard for every rule at once. func TestShippedExamplesAreClean(t *testing.T) { - files, err := filepath.Glob(filepath.Join("..", "examples", "*.yaml")) + files, err := filepath.Glob(filepath.Join("..", "examples", "skills", "*.yaml")) require.NoError(t, err) require.NotEmpty(t, files, "the examples are gone") diff --git a/skill.schema.ru.yaml b/skill.schema.ru.yaml index e61b2d7..ee42ec6 100644 --- a/skill.schema.ru.yaml +++ b/skill.schema.ru.yaml @@ -145,14 +145,14 @@ properties: стоят бюджеты вызовов, wall-clock, температура, усилие рассуждения. Схема список не закрывает — профили у каждого свои. Примеры имён и того, - что за ними может стоять, — в examples/. + что за ними может стоять, — в examples/skills/. kind: type: string description: | Род скилла — метка для приложения: по ней оно решает, как скилл подавать и исполнять. Схема значение не ограничивает; пример разделения на - самостоятельные и составные скиллы — в examples/. + самостоятельные и составные скиллы — в examples/skills/. temperature: type: number @@ -238,7 +238,7 @@ properties: Произвольные данные для приложения: расписание запуска, владелец, теги, что угодно ещё. Движок сюда не смотрит и форму не проверяет — поле существует, чтобы такие вещи не приходилось изобретать заново - каждому встраивающему. Образцы наполнения — в examples/. + каждому встраивающему. Образцы наполнения — в examples/skills/. # Явный режим обязан иметь под собой описание. Эти два правила ловят в # редакторе и в CI ровно то, на чём иначе спотыкаются на прогоне: режим @@ -992,7 +992,7 @@ $defs: description: | ЧТО это по существу. Схема значение не ограничивает — у каждого приложения свой набор; примеры («код», «справочный текст», - «конфигурация», «данные») — в examples/. + «конфигурация», «данные») — в examples/skills/. Метка вспомогательная: способ передачи выбирается МЕСТОМ использования, а не ею (см. ниже про две формы ссылки). @@ -1001,7 +1001,7 @@ $defs: type: string description: | ОТКУДА берётся содержимое. Имя источника разбирает резолвер - приложения — схема список не закрывает (примеры в examples/). + приложения — схема список не закрывает (примеры в examples/skills/). Формату важно одно: содержимое либо ЗДЕСЬ (`content`), либо ПО АДРЕСУ (`ref`). Ровно одно из двух — иначе непонятно, что @@ -1015,7 +1015,7 @@ $defs: type: string description: | Адрес содержимого — когда оно лежит не в самом скилле. Форму адреса - определяет резолвер приложения (примеры — в examples/). + определяет резолвер приложения (примеры — в examples/skills/). args: type: object @@ -1036,7 +1036,7 @@ $defs: пустым. Ровно так здесь и жил `lang`, нужный одному роду из четырёх. Движок сюда не смотрит и формы не проверяет — отдаёт резолверу - приложения как есть. Наполнение по родам — в examples/vocabulary.yaml. + приложения как есть. Наполнение по родам — в examples/skills/vocabulary.yaml. А значит, ВАЛИДАЦИЯ КЛЮЧЕЙ — ОБЯЗАННОСТЬ РЕЗОЛВЕРА, и она не факультативна. За открытость платят: `params: {langauge: python}` @@ -1050,7 +1050,7 @@ $defs: description: | Маршрут ВЫВОДА инструмента, потребившего ассет: куда деть результат вместо возврата в переменную шага. Имена маршрутов задаёт - приложение — схема их не ограничивает (примеры в examples/). + приложение — схема их не ограничивает (примеры в examples/skills/). Зачем объявлять здесь, а не решать по ходу: маршрут, который должна выбрать модель, забывается ровно тогда, когда всё остальное уже diff --git a/skill.schema.yaml b/skill.schema.yaml index b35fad2..c86517b 100644 --- a/skill.schema.yaml +++ b/skill.schema.yaml @@ -147,7 +147,7 @@ properties: temperature, reasoning effort. The schema does not close the list — everyone has their own profiles. - Example names, and what may stand behind them, are in examples/. + Example names, and what may stand behind them, are in examples/skills/. kind: type: string @@ -155,7 +155,7 @@ properties: The skill's kind — a label for the application: it decides from this how to present and execute the skill. The schema does not restrict the value; an example of splitting into standalone and composite skills is in - examples/. + examples/skills/. temperature: type: number @@ -244,7 +244,7 @@ properties: Arbitrary data for the application: a run schedule, an owner, tags, anything else. The engine does not look in here and does not check the shape — the field exists so that every embedder does not have to reinvent - such things. Sample contents are in examples/. + such things. Sample contents are in examples/skills/. # An explicit mode must have a description under it. These two rules catch in # the editor and in CI exactly what people otherwise trip over at run time: the @@ -1018,7 +1018,7 @@ $defs: description: | WHAT it essentially is. The schema does not restrict the value — every application has its own set; examples ("code", "reference text", - "configuration", "data") are in examples/. + "configuration", "data") are in examples/skills/. The label is auxiliary: the way it is passed is chosen by the PLACE of use, not by this (see the two reference forms above). @@ -1028,7 +1028,7 @@ $defs: description: | WHERE the content comes from. The source's name is interpreted by the application's resolver — the schema does not close the list (examples - are in examples/). + are in examples/skills/). The format cares about one thing: the content is either HERE (`content`) or AT AN ADDRESS (`ref`). Exactly one of the two — @@ -1043,7 +1043,7 @@ $defs: description: | The address of the content — when it does not live in the skill itself. The form of the address is defined by the application's - resolver (examples are in examples/). + resolver (examples are in examples/skills/). args: type: object @@ -1066,7 +1066,7 @@ $defs: The engine does not look in here and does not check the shape — it hands this to the application's resolver as is. Contents by kind are - in examples/vocabulary.yaml. + in examples/skills/vocabulary.yaml. Which means VALIDATING THE KEYS IS THE RESOLVER'S JOB, and it is not optional. The openness is paid for: `params: {langauge: python}` @@ -1081,7 +1081,7 @@ $defs: The route of the OUTPUT of the tool that consumed the asset: where to put the result instead of returning it into the step's variable. Route names are defined by the application — the schema does not restrict - them (examples are in examples/). + them (examples are in examples/skills/). Why declare it here rather than decide as you go: a route the model is supposed to choose gets forgotten exactly when everything else has diff --git a/skill_test.go b/skill_test.go index 7e831e9..19318c2 100644 --- a/skill_test.go +++ b/skill_test.go @@ -177,7 +177,7 @@ func TestSkillCoversEverySchemaField(t *testing.T) { // how an embedder loads them, and a header that stopped validating would go // unnoticed by a test that only looks at the steps. func TestExamplesLoadAsSkills(t *testing.T) { - files, err := filepath.Glob(filepath.Join("examples", "*.yaml")) + files, err := filepath.Glob(filepath.Join("examples", "skills", "*.yaml")) require.NoError(t, err) require.NotEmpty(t, files) From 35045f668ce38fd927763561613c8ea985da4850 Mon Sep 17 00:00:00 2001 From: Ivan Diatchenko <2518263+inhuman@users.noreply.github.com> Date: Tue, 4 Aug 2026 23:47:26 +0300 Subject: [PATCH 2/8] docs: write the README for a reader who arrives cold MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README explained HOW the format works to somebody already inside. A reader coming from an article decides in a minute whether to run `go get`, and asks three things the file never answered. **What is this instead of.** A section naming the neighbours, because the differences are architectural and checkable in the code rather than a matter of taste: not a process orchestrator — the engine owns no state between turns, has no storage and does not survive a restart; not an agent framework — there the graph is the application developer's code, here the steps are the skill author's data, portable between hosts, which is why the format is versioned and why Migrate exists; zero dependencies as a consequence of being embedded, not a pose; and when NOT to use it, said again next to the comparison. **What can be relied on.** A Status section: the library is v0.5.x and the Go API may still move, while the FORMAT is what is stable at 2.2.2 — a number that lived in version.go and nowhere a reader would look. Where it runs: the engine executes the whole skill catalogue of a working assistant, which is the strongest argument for it and was missing entirely. And the caveat nobody wants to discover on their own: skills written for format 1.x do not load until Migrate is run, schedules included. **How to start.** go get, badges, and a link to examples/ on the first screen — two applications that embed the engine are the best answer to "how do I use this" and were invisible from the top. Coverage is stated because a reader choosing a dependency looks: 89.3% engine, 95.1% linter, both measured, not quoted. **The measurements are qualified.** The three before/after pairs now say what is NOT established about them — runs per scenario, spread, whether anything besides the form of the description changed — and the counterexample moved up beside them: a skill that cost 43k tokens as steps against 36k as prose, because knowledge is expensive in a step WITH tools. Three wins and no losses read as advertising; one named loss is what makes the other three credible. Re-running those measurements properly is still open — it needs a model and a catalogue, not an edit. A guard test keeps the two READMEs one document in two languages: same number of sections, same number of code samples, and every local link resolving. Proven by deleting a section from one of them. --- README.md | 97 +++++++++++++++++++++++++++++++++++++++++++++++--- README.ru.md | 88 +++++++++++++++++++++++++++++++++++++++++++-- readme_test.go | 67 ++++++++++++++++++++++++++++++++++ 3 files changed, 245 insertions(+), 7 deletions(-) create mode 100644 readme_test.go diff --git a/README.md b/README.md index 4da5245..de8d1d6 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,64 @@ **English** · [Русский](README.ru.md) +[![Version](https://img.shields.io/github/v/tag/inhuman/skill-engine?sort=semver&style=flat-square&label=version)](https://github.com/inhuman/skill-engine/tags) +[![Build](https://img.shields.io/github/actions/workflow/status/inhuman/skill-engine/ci.yml?style=flat-square&logo=github)](https://github.com/inhuman/skill-engine/actions/workflows/ci.yml) +[![Go Reference](https://pkg.go.dev/badge/github.com/inhuman/skill-engine.svg)](https://pkg.go.dev/github.com/inhuman/skill-engine) +[![Go Report Card](https://goreportcard.com/badge/github.com/inhuman/skill-engine?style=flat-square)](https://goreportcard.com/report/github.com/inhuman/skill-engine) +[![Go Version](https://img.shields.io/github/go-mod/go-version/inhuman/skill-engine?style=flat-square&logo=go)](https://go.dev/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow?style=flat-square)](LICENSE) + An engine for declarative programs for an LLM agent: a skill is described in **steps**, and control over the turn belongs to the code, not to the model. Steps are not the only form: a skill with only a `playbook` (a free-form instruction) is a full skill too (see "A prompt works as well"). +``` +go get github.com/inhuman/skill-engine +``` + +**Start here:** [`examples/`](examples/) — the format in +[`examples/skills/`](examples/skills/), and two working applications that embed +the engine: [`simple-llm-app`](examples/simple-llm-app/) on an +OpenAI-compatible endpoint with nothing but `net/http`, and +[`eino-llm-app`](examples/eino-llm-app/) with the model reached through a +framework. Both run offline in their tests. + **No dependencies** — production code runs on the standard library alone: the engine is embedded into someone else's application, and every dependency here would become a dependency of the embedder. YAML parsing is passed in as a parameter (the `Unmarshal` type), version comparison is implemented in place. The boundary is held by a guard test, `imports_test.go`, test imports included. +## Status + +**Where it runs.** The engine was taken out of a working assistant, where it +executes that assistant's whole skill catalogue — around thirty skills, in +production. It is not a design sketch: every field in the format is there +because something broke without it, and the comment beside the field says what. + +**Library version — `v0.5.x`.** Below `1.0` the **Go API may still move**: a +type can gain a field, a function a parameter. What is already stable is the +**format** — skill files are versioned separately and on their own rules. + +**Format version — `2.2.2`** (`EngineVersion` in `version.go`). A skill declares +the minimum it needs in `skill_engine_version`, and a foreign MAJOR is refused +in both directions: a description of a previous major would parse without a +single complaint, silently losing fields the structs no longer have. What +changed in each version, and what a migration does, is in +[CHANGELOG.md](CHANGELOG.md). + +**If you have skills written for format 1.x**, they do not load: that is the +refusal above, working as intended. `Migrate(raw)` rewrites them — it edits the +file as text, so comments, key order and block scalars survive — but until you +run it those skills do not execute at all, including on a schedule. Better said +here than discovered on a Monday morning. + +**Tests.** 89.3% statement coverage in the engine, 95.1% in the linter, plus +guard tests for the properties that prose cannot hold: no dependencies, no +direct reads of the variable map, the two schema translations staying +structurally identical, and the example applications staying separate modules. + ## Why A restriction written in words is a request: "do NOT call retract without @@ -21,10 +68,52 @@ the model read before it started acting. In steps the same thing is expressed structurally: in the unconfirmed branch the `retract` call **is not there**, a `call` step cannot be repeated, a branch that does not apply does not run. -Measurements on live skills (tool calls / seconds, before → after): three -skills of one catalogue went 18/95 → **2/6**, 7/33 → **2/5** and 9/29 → -**5/11**. What they did is beside the point — what changed is who held the -control flow. +Measured on a live catalogue — tool calls / seconds, before → after moving a +skill from prose into steps: 18/95 → **2/6**, 7/33 → **2/5**, 9/29 → **5/11**. +What those skills did is beside the point; what changed is who held the control +flow. + +And one that went the other way. A log-searching skill cost **43k tokens as +steps against 36k as prose** — because knowledge is expensive in a step WITH +tools: an asset rides along into every generation of the react loop. Three wins +and no losses read as advertising, and this is the shape of the case where the +format does not pay. + +What is NOT established about these numbers: how many runs per scenario, the +spread between them, and whether anything besides the form of the description +changed. Until that is measured, read them as an order of magnitude rather than +as a benchmark. + +## What this is not + +The neighbours are worth naming, because the differences are architectural +rather than a matter of taste. + +**Not a process orchestrator** (Temporal, n8n). The engine owns no state between +turns, has no storage of its own and does not survive a restart: a turn runs +inside somebody else's application and ends with it. Comparing durability is +comparing different jobs — if you need a workflow that resumes after a crash +three days later, this is the wrong tool and nothing here will make it right. + +**Not an agent framework** (LangGraph and its relatives). There the graph is +written by the application's developer, in the application's language, and it +ships with the application. Here the steps are written by the SKILL'S author, in +YAML, and the skill is portable between hosts — which is why the format has a +version of its own and why `Migrate` exists at all. A skill is data your users +can write; a graph is code you deploy. + +**Zero dependencies is a consequence, not a pose.** An engine embedded into +someone else's application makes every one of its dependencies theirs, with +their versions and their conflicts. So YAML arrives as a parameter (the +`Unmarshal` type), version comparison is thirty lines instead of a library, and +`imports_test.go` fails the build the moment production code imports anything at +all. Rare enough to be worth naming where you are comparing. + +**When you do NOT need this.** One or two steps and no branching — prose is +cheaper, and the format says so itself (see "A prompt works as well"). The +engine starts paying where a turn has branches, a loop, a tool set that must +narrow, or a guard that has to be impossible to violate rather than merely +asked for. ## A prompt works as well diff --git a/README.ru.md b/README.ru.md index 0a3c348..001af2d 100644 --- a/README.ru.md +++ b/README.ru.md @@ -2,16 +2,61 @@ **Русский** · [English](README.md) +[![Version](https://img.shields.io/github/v/tag/inhuman/skill-engine?sort=semver&style=flat-square&label=version)](https://github.com/inhuman/skill-engine/tags) +[![Build](https://img.shields.io/github/actions/workflow/status/inhuman/skill-engine/ci.yml?style=flat-square&logo=github)](https://github.com/inhuman/skill-engine/actions/workflows/ci.yml) +[![Go Reference](https://pkg.go.dev/badge/github.com/inhuman/skill-engine.svg)](https://pkg.go.dev/github.com/inhuman/skill-engine) +[![Go Report Card](https://goreportcard.com/badge/github.com/inhuman/skill-engine?style=flat-square)](https://goreportcard.com/report/github.com/inhuman/skill-engine) +[![Go Version](https://img.shields.io/github/go-mod/go-version/inhuman/skill-engine?style=flat-square&logo=go)](https://go.dev/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow?style=flat-square)](LICENSE) + Движок декларативных программ для LLM-агента: скилл описывается **шагами**, и управление ходом принадлежит коду, а не модели. Шаги — не единственная форма: скилл, у которого задан только `playbook` (свободная инструкция), тоже полноценный скилл (см. «Промптом тоже можно»). +``` +go get github.com/inhuman/skill-engine +``` + +**Начинать отсюда:** [`examples/`](examples/) — сам формат в +[`examples/skills/`](examples/skills/) и два рабочих приложения, встраивающих +движок: [`simple-llm-app`](examples/simple-llm-app/) на OpenAI-совместимом +эндпоинте и голом `net/http` и [`eino-llm-app`](examples/eino-llm-app/), где +модель достаётся через фреймворк. Оба гоняются в тестах без сети. + **Зависимостей нет** — боевой код на одной stdlib: движок встраивают в чужое приложение, и каждая зависимость здесь стала бы зависимостью встраивающего. Разбор YAML передаётся параметром (тип `Unmarshal`), сравнение версий сделано на месте. Границу держит тест-страж `imports_test.go`, включая тестовые импорты. +## Статус + +**Где работает.** Движок вынесен из работающего ассистента, где исполняет весь +его каталог системных скиллов — около тридцати штук, в проде. Это не эскиз: у +каждого поля формата есть отказ, без которого поля бы не было, и комментарий +рядом с полем говорит какой. + +**Версия библиотеки — `v0.5.x`.** До `1.0` **Go-API может двигаться**: у типа +появится поле, у функции параметр. Что уже стабильно — это **формат**: файлы +скиллов версионируются отдельно и по своим правилам. + +**Версия формата — `2.2.2`** (`EngineVersion` в `version.go`). Скилл объявляет +нужный ему минимум в `skill_engine_version`, и чужой МАЖОР отвергается в обе +стороны: описание прошлого мажора разобралось бы без единой жалобы, молча +потеряв поля, которых в структурах больше нет. Что изменилось в каждой версии и +что делает миграция — в [CHANGELOG.md](CHANGELOG.md). + +**Если у тебя есть скиллы под формат 1.x** — они не загрузятся: это тот самый +отказ, работающий как задумано. `Migrate(raw)` их переписывает, правя файл +текстом, так что комментарии, порядок ключей и блочные скаляры переживают +миграцию, — но до её запуска такие скиллы не исполняются вовсе, в том числе по +расписанию. Лучше сказать это здесь, чем обнаружить в понедельник утром. + +**Тесты.** 89.3% покрытия по операторам в движке, 95.1% в линтере, плюс +стражи на свойства, которые прозой не удержать: отсутствие зависимостей, +отсутствие прямых чтений карты переменных, структурная одинаковость двух +переводов схемы и отдельность модулей у примеров-приложений. + ## Зачем Ограничение, записанное словами, — просьба: «НЕ вызывай retract без @@ -20,9 +65,46 @@ Шагами то же самое выражается структурой: в неподтверждённой ветке вызова `retract` **нет**, шаг `call` нельзя повторить, лишняя ветка не исполняется. -Замеры на живых скиллах (вызовов инструментов / секунд, до → после): три скилла -одного каталога прошли 18/95 → **2/6**, 7/33 → **2/5** и 9/29 → **5/11**. Что -именно они делали — неважно; изменилось то, кто держит поток управления. +Замеры на живом каталоге — вызовов инструментов / секунд, до → после переноса +скилла из прозы в шаги: 18/95 → **2/6**, 7/33 → **2/5**, 9/29 → **5/11**. Что +именно эти скиллы делали — неважно; изменилось то, кто держит поток управления. + +И один, который пошёл в обратную сторону. Скилл поиска по логам стоил **43k +токенов шагами против 36k прозой** — потому что знание дорого в шаге С +ИНСТРУМЕНТАМИ: ассет едет в каждую генерацию react-цикла. Три победы без единого +поражения читаются как реклама, а это — форма случая, где формат не окупается. + +Чего про эти числа НЕ установлено: сколько прогонов на сценарий, какой между +ними разброс и менялось ли что-то ещё, кроме формы описания. Пока это не +измерено, читай их как порядок величины, а не как замер. + +## Чем это не является + +Соседей стоит назвать, потому что различия здесь архитектурные, а не вкусовые. + +**Это не оркестратор процессов** (Temporal, n8n). Движок не владеет состоянием +между ходами, не имеет своего хранилища и не переживает рестарт: ход исполняется +внутри чужого приложения и заканчивается вместе с ним. Сравнивать надёжность — +сравнивать разные задачи: если нужен процесс, продолжающийся после падения через +трое суток, это не тот инструмент, и ничего здесь этого не исправит. + +**Это не фреймворк агента** (LangGraph и родня). Там граф пишет разработчик +приложения, на языке приложения, и граф едет вместе с приложением. Здесь шаги +пишет АВТОР СКИЛЛА, на YAML, и скилл переносим между хостами — отсюда и +собственная версия формата, и существование `Migrate`. Скилл — это данные, +которые могут писать твои пользователи; граф — это код, который ты выкатываешь. + +**Ноль зависимостей — следствие, а не поза.** Движок, встроенный в чужое +приложение, делает каждую свою зависимость его зависимостью, с её версиями и её +конфликтами. Поэтому YAML приезжает параметром (тип `Unmarshal`), сравнение +версий — тридцать строк вместо библиотеки, а `imports_test.go` роняет сборку в +тот момент, когда боевой код импортирует хоть что-нибудь. Свойство редкое, и +назвать его стоит именно там, где читатель сравнивает. + +**Когда это НЕ нужно.** Один-два шага и никаких ветвлений — проза дешевле, и +формат сам об этом говорит (см. «Промптом тоже можно»). Движок начинает +окупаться там, где у хода есть ветки, цикл, набор инструментов, который надо +сузить, или запрет, который должен быть невыполнимым, а не попрошенным. ## Промптом тоже можно diff --git a/readme_test.go b/readme_test.go new file mode 100644 index 0000000..469c35b --- /dev/null +++ b/readme_test.go @@ -0,0 +1,67 @@ +package skillengine + +import ( + "os" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The two READMEs are one document in two languages, and they drift the moment +// somebody edits one of them. The titles cannot be compared — that is the whole +// point of having two — so what is compared is the SHAPE: how many sections, in +// what order by kind, and how many code samples. +// +// The same guard the schema translations already have, for the same reason: a +// reader of the Russian file must not be reading a version of the library that +// no longer exists. +func TestBothReadmesHaveTheSameShape(t *testing.T) { + en := mustRead(t, "README.md") + ru := mustRead(t, "README.ru.md") + + enSections, ruSections := sectionsOf(string(en)), sectionsOf(string(ru)) + require.Equal(t, len(enSections), len(ruSections), + "the two READMEs have a different number of sections:\nEN: %s\nRU: %s", + strings.Join(enSections, " | "), strings.Join(ruSections, " | ")) + + assert.Equal(t, strings.Count(string(en), "```"), strings.Count(string(ru), "```"), + "one README gained or lost a code sample") + + // The links a cold reader arrives by. A broken one in a file linked from an + // article is the cheapest possible way to lose them. + for _, doc := range []string{string(en), string(ru)} { + for _, path := range localLinks(doc) { + _, err := os.Stat(path) + assert.NoErrorf(t, err, "README links to %s, which does not exist", path) + } + } +} + +func sectionsOf(doc string) []string { + var out []string + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(line, "## ") { + out = append(out, strings.TrimPrefix(line, "## ")) + } + } + return out +} + +// localLinks — markdown links pointing inside the repository, badges and +// external URLs excluded. +var linkRe = regexp.MustCompile(`\]\(([^)]+)\)`) + +func localLinks(doc string) []string { + var out []string + for _, m := range linkRe.FindAllStringSubmatch(doc, -1) { + target := m[1] + if strings.HasPrefix(target, "http") || strings.HasPrefix(target, "#") { + continue + } + out = append(out, strings.TrimSuffix(target, "/")) + } + return out +} From 70b4bc146d58fb442de0960a7bd31edb199548ef Mon Sep 17 00:00:00 2001 From: Ivan Diatchenko <2518263+inhuman@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:00:10 +0300 Subject: [PATCH 3/8] docs: a quickstart that shows the difference instead of asserting it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reader arriving from an article wants to see the claim, not read it. So the quickstart is built around an A/B they run themselves: one skill carrying BOTH descriptions, the same request, one word changed on the command line. go run . -skill ../skills/menu.yaml -mode playbook -input "…" 227 tokens go run . -skill ../skills/menu.yaml -mode workflow -input "…" 102 tokens Three things had to exist for that to be real rather than a diagram. `examples/skills/menu.yaml` gained a `playbook` half and `mode: workflow`. No example showed `mode` before, which is odd for a field whose whole purpose is keeping both descriptions while a prompt is being moved into steps — the comparison this quickstart walks through is exactly what it is for. `examples/simple-llm-app` gained a `-mode` flag that overrides the skill's declared mode, and counts what a turn cost: generations, prompt and completion tokens, printed after the trace. "Steps are cheaper" is a claim until something puts a number next to it. The test stub now charges by prompt length the way a real endpoint does, so the A/B is checkable offline — and there is a test asserting the direction: the steps half must reach the model with fewer prompt tokens, because the dictionary of synonyms never gets there at all. The minimal skill the quickstart tells a reader to write has its own test too: it is the first thing they will run. The quickstart is honest about what its numbers are. They come from a stub, so they show the shape rather than a bill; with a real endpoint the prose half also spends extra generations deciding to call a tool and reading the result back, so the gap shown is understated; the accuracy figure is somebody else's measurement and the README says what is not established about it; and the counterexample is there — a skill that cost 43k tokens as steps against 36k as prose. Both READMEs now point at the quickstart first and `examples/` second, and the translation guard covers the new pair as well: same sections, same code samples, every local link resolving. --- QUICKSTART.md | 351 +++++++++++++++++++++ QUICKSTART.ru.md | 349 ++++++++++++++++++++ README.md | 9 +- README.ru.md | 9 +- examples/simple-llm-app/README.md | 8 + examples/simple-llm-app/main.go | 61 +++- examples/simple-llm-app/main_test.go | 71 ++++- examples/simple-llm-app/openai.go | 26 +- examples/simple-llm-app/quickstart_test.go | 44 +++ examples/skills/menu.yaml | 27 +- readme_test.go | 18 +- 11 files changed, 943 insertions(+), 30 deletions(-) create mode 100644 QUICKSTART.md create mode 100644 QUICKSTART.ru.md create mode 100644 examples/simple-llm-app/quickstart_test.go diff --git a/QUICKSTART.md b/QUICKSTART.md new file mode 100644 index 0000000..db966ae --- /dev/null +++ b/QUICKSTART.md @@ -0,0 +1,351 @@ +# Quickstart + +**English** · [Русский](QUICKSTART.ru.md) + +Fifteen minutes, at the end of which you will have run the **same skill both +ways** — as a prompt and as steps — on the same request, and seen the difference +in what it cost and in what it did. + +Written to be followed literally, by a person or by a model: every step has the +command, the file, and the output to expect. + +--- + +## What you are going to see + +One file describes a turn twice: as prose (`playbook`) and as steps +(`workflow`). Running it both ways on `подбери десерт и напиток`: + +``` +=== cost (playbook) === + generations: 1 + tokens: 214 prompt + 13 completion = 227 + +=== cost (workflow) === + generations: 1 + tokens: 89 prompt + 13 completion = 102 +``` + +And on a request that names no section at all: + +``` +-mode playbook → the model answers something. It was asked not to invent, and + whether it obeys is a probability. +-mode workflow → the skill stepped aside: exit: the request names no section + of the menu +``` + +Two different things happened there, and only one of them is about tokens. + +--- + +## Step 0 — what you need + +- **Go 1.26+** +- **an OpenAI-compatible endpoint.** Anything that speaks + `POST /v1/chat/completions`: vLLM, Ollama, LM Studio, LocalAI, or a hosted + API. The fastest local one: + + ``` + ollama serve + ollama pull qwen3:8b + ``` + + which gives you `http://localhost:11434/v1` and needs no key. + +You can do **Steps 1–5 without any endpoint at all** — the example's tests run +the whole thing against a stub. The numbers above come from exactly that stub, +which charges by prompt length the way a real endpoint does. + +--- + +## Step 1 — get the repository and run the tests + +``` +git clone https://github.com/inhuman/skill-engine +cd skill-engine/examples/simple-llm-app +go test ./... +``` + +Expected: + +``` +ok github.com/inhuman/skill-engine/examples/simple-llm-app +``` + +That run just executed a skill end to end, twice, and compared the two. If it +passes, everything below will work. + +--- + +## Step 2 — point the example at your model + +``` +export OPENAI_BASE_URL=http://localhost:11434/v1 +export OPENAI_API_KEY=ollama # any non-empty string for a local server +export OPENAI_MODEL=qwen3:8b +``` + +--- + +## Step 3 — run the skill as PROSE + +``` +go run . -skill ../skills/menu.yaml -mode playbook -input "подбери десерт и напиток" +``` + +The whole task goes to the model in one prompt: here are the sections, here are +the words people call them by, decide which were named, look them up, answer. + +Write down the two numbers it prints under `=== cost (playbook) ===`. + +--- + +## Step 4 — run the SAME FILE as steps + +``` +go run . -skill ../skills/menu.yaml -mode workflow -input "подбери десерт и напиток" +``` + +One word changed on the command line. Same file, same model, same request. + +``` +→ nothing_named (exit) +→ pick_dessert (call) + call recipes:search map[query:подбери десерт и напиток section:dessert] +→ pick_drink (call) + call recipes:search map[query:подбери десерт и напиток section:drink] +→ pick_main (call) +→ answer (instruction) + +=== steps === + nothing_named exit skipped calls=0 condition … is false + pick_dessert call ok calls=1 + pick_drink call ok calls=1 + pick_main call skipped calls=0 condition input contains горяч | второе | main course is false + answer instruction ok calls=0 + +=== cost (workflow) === + generations: 1 + tokens: 89 prompt + 13 completion = 102 +``` + +Compare with Step 3. + +--- + +## Step 5 — where the difference came from + +Open [`examples/skills/menu.yaml`](examples/skills/menu.yaml) next to the output +above. Four things did the work. + +**1. The dictionary never reached the model.** + +```yaml +- name: pick_dessert + when: "input contains десерт | сладк | dessert" +``` + +In the prose half those synonyms are in the prompt, and applying them is a +generation the model can get wrong. Here they are a condition: the words are +matched in code, before anything is sent anywhere. That is most of the token +difference — and all of the accuracy difference. + +**2. Two steps ran without a model at all.** + +```yaml + call: + tool: "recipes:search" + args: {section: dessert, query: "{{input}}"} +``` + +A `call` step is a tool invocation with the arguments already known. In prose +the same work is: one generation to decide to call, the call, another to read +the result back. Here it is the call. + +**3. The answering step was handed no tools.** + +```yaml +- name: answer + instruction: | + … + tools: [] +``` + +An empty list is not "no preference" — the step physically cannot call +anything. In prose, "do not go looking further" is a request; here there is +nothing to go with. + +**4. What did not run is visible.** + +``` +skipped: nothing_named, pick_main +``` + +"We did not look there" and "we looked and it was empty" are different answers. +The trace keeps them apart, so the answering step cannot report one as the +other. + +--- + +## Step 6 — make it decide something it cannot invent + +``` +go run . -skill ../skills/menu.yaml -mode workflow -input "что посоветуешь" +go run . -skill ../skills/menu.yaml -mode playbook -input "что посоветуешь" +``` + +Steps: + +``` +→ nothing_named (exit) +the skill stepped aside: skill-engine: exit: the request names no section of the menu +``` + +Prose: the model answers. It was told not to invent, and that instruction is +followed as far as it read before it started writing. + +This is the property the format is for, and it is not about cost: **in steps the +impossible is impossible, not discouraged.** The branch that would have searched +does not exist on that path. + +--- + +## Step 7 — write your own skill + +The smallest useful file. Save it as `my-skill.yaml`: + +```yaml +skill_engine_version: "2.2.0" +skill_version: "1.0.0" +name: my-skill +description: What this skill is FOR — and what it is NOT for. +trigger_examples: + - "a phrasing a user would actually type" + +workflow: + steps: + # A step with no tools cannot go anywhere. It thinks, and that is all. + - name: answer + instruction: | + The request: {{input}} + + Answer it in two sentences. + tools: [] +``` + +``` +go run . -skill ./my-skill.yaml -input "привет" +``` + +Then grow it in this order — each of these is one line in the file: + +| you want | you add | +|---|---| +| branch on the words of the request | `when: "input contains word \| synonym"` | +| call a tool with known arguments | a `call:` step | +| loop over what a step returned | `for_each: {in: var, as: item, collect: out}` | +| stop when it is not your case | `exit: {reason: "…"}` | +| a structured answer to branch on | `response_schema:` **plus** `model:` | +| hand the work to another skill | `delegate: {skill: other, task: "…"}` | + +The full field list with the reason each one exists is +[`skill.schema.yaml`](skill.schema.yaml); more shapes are in +[`examples/skills/`](examples/skills/). + +--- + +## Step 8 — check a skill before you run it + +Two levels, and they answer different questions. + +```go +// Will it run at all? +if err := skill.Validate(); err != nil { … } +``` + +```go +// Will it run WELL? 27 rules for the defects that stay quiet. +rep, err := lint.Lint(raw, facts, lint.Options{ + Unmarshal: yaml.Unmarshal, + EmptyWords: []string{"empty", "пусто"}, + HostVars: []string{"input"}, +}) +fmt.Println(rep.Text()) +``` + +The linter catches things a run would not: a loop collecting into a variable +nobody writes, a typo in a variable name that resolves to an empty string, a +required field the instruction beside it allows to be empty. See +[`lint/README.md`](lint/README.md). + +--- + +## Step 9 — embed it in your application + +The whole contract is one struct. From +[`examples/simple-llm-app/main.go`](examples/simple-llm-app/main.go): + +```go +skill, err := se.ParseSkill(raw, yaml.Unmarshal) // the whole file +if err := skill.Validate(); err != nil { … } + +mode, err := skill.ResolveMode() // steps or prose? +if mode == se.ModePlaybook { + // the engine takes no part: run skill.Playbook as your own prompt +} + +vars, outcome, err := se.ExecuteWith(ctx, skill.Workflow, se.Deps{ + Runner: yourModel, // executes an instruction step + Caller: yourTools, // executes a call step + Assets: yourResolver, // resolves an asset's content + Memory: yourMemory, // returns a large result by its handle + Vocabulary: se.Vocabulary{ + DecisionMarkers: []string{"Result:", "Ответ:"}, + }, + OnStepStart: func(name, kind string) { … }, +}, map[string]string{"input": userText}) +``` + +`vars[se.AnswerVar]` is the turn's answer. `outcome.Steps` is the trace you saw +above — the engine logs nothing and stores nothing, so that struct is the whole +of its observability. + +Two working applications to copy from: + +- [`examples/simple-llm-app`](examples/simple-llm-app/) — `net/http` and nothing + else; +- [`examples/eino-llm-app`](examples/eino-llm-app/) — the model reached through a + framework, with the whole framework-shaped part in one forty-line adapter. + +--- + +## Honest notes about the numbers above + +- They come from a **stub** that charges by prompt length, so they show the + SHAPE of the difference rather than your model's bill. Run Steps 3–4 against + a real endpoint for real numbers. +- With a real endpoint the prose half also spends **extra generations**: it has + to decide to call a tool, call it, and read the result back. The stub cannot + call tools, so it does that work in one generation and the gap in the table + above is understated, not overstated. +- The accuracy claim — a classifier step at temperature 0 getting 5 of 10 live + requests right where the same dictionary in a condition got 10 — is measured + on somebody else's catalogue, and the README says plainly what is not + established about it. +- One counterexample, on purpose: a log-searching skill cost **43k tokens as + steps against 36k as prose**, because knowledge is expensive in a step WITH + tools — an asset rides into every generation of the loop. Steps are not + free; they are cheaper where the turn has branches, loops and guards. + +--- + +## Where to go next + +| | | +|---|---| +| [README.md](README.md) | what the engine is, what it is NOT, and what you can rely on | +| [`examples/`](examples/) | the format and two applications that embed it | +| [`skill.schema.yaml`](skill.schema.yaml) | every field, and the failure that paid for it | +| [CHANGELOG.md](CHANGELOG.md) | what changed in the format, and what a migration does | +| [`lint/README.md`](lint/README.md) | the 27 rules and what each one catches | diff --git a/QUICKSTART.ru.md b/QUICKSTART.ru.md new file mode 100644 index 0000000..7ca4980 --- /dev/null +++ b/QUICKSTART.ru.md @@ -0,0 +1,349 @@ +# Быстрый старт + +**Русский** · [English](QUICKSTART.md) + +Пятнадцать минут, в конце которых ты прогонишь **один и тот же скилл двумя +способами** — промптом и шагами — на одном запросе и увидишь разницу в том, +сколько это стоило и что при этом произошло. + +Написано так, чтобы выполнять буквально — человеку или модели: у каждого шага +есть команда, файл и ожидаемый вывод. + +--- + +## Что ты увидишь + +Один файл описывает ход дважды: прозой (`playbook`) и шагами (`workflow`). +Прогон обоими способами на запросе `подбери десерт и напиток`: + +``` +=== cost (playbook) === + generations: 1 + tokens: 214 prompt + 13 completion = 227 + +=== cost (workflow) === + generations: 1 + tokens: 89 prompt + 13 completion = 102 +``` + +И на запросе, где не назван ни один раздел: + +``` +-mode playbook → модель что-нибудь отвечает. Её попросили не выдумывать, и + исполнение просьбы — это вероятность. +-mode workflow → the skill stepped aside: exit: the request names no section + of the menu +``` + +Здесь произошли две разные вещи, и только одна из них про токены. + +--- + +## Шаг 0 — что нужно + +- **Go 1.26+** +- **OpenAI-совместимый эндпоинт.** Любой, кто понимает + `POST /v1/chat/completions`: vLLM, Ollama, LM Studio, LocalAI или хостовый + API. Самый быстрый локальный: + + ``` + ollama serve + ollama pull qwen3:8b + ``` + + это даёт `http://localhost:11434/v1` и не требует ключа. + +**Шаги 1–5 можно пройти вообще без эндпоинта** — тесты примера гоняют всё +против заглушки. Числа выше сняты именно с неё, а считает она по длине промпта, +как это делает настоящий эндпоинт. + +--- + +## Шаг 1 — забрать репозиторий и прогнать тесты + +``` +git clone https://github.com/inhuman/skill-engine +cd skill-engine/examples/simple-llm-app +go test ./... +``` + +Ожидается: + +``` +ok github.com/inhuman/skill-engine/examples/simple-llm-app +``` + +Этот прогон только что исполнил скилл целиком, дважды, и сравнил результаты. +Если он зелёный, всё нижеследующее заработает. + +--- + +## Шаг 2 — направить пример на свою модель + +``` +export OPENAI_BASE_URL=http://localhost:11434/v1 +export OPENAI_API_KEY=ollama # для локального сервера любая непустая строка +export OPENAI_MODEL=qwen3:8b +``` + +--- + +## Шаг 3 — прогнать скилл ПРОЗОЙ + +``` +go run . -skill ../skills/menu.yaml -mode playbook -input "подбери десерт и напиток" +``` + +Вся задача уезжает в модель одним промптом: вот разделы, вот слова, которыми их +называют, реши, какие названы, найди их, ответь. + +Запиши два числа из блока `=== cost (playbook) ===`. + +--- + +## Шаг 4 — прогнать ТОТ ЖЕ ФАЙЛ шагами + +``` +go run . -skill ../skills/menu.yaml -mode workflow -input "подбери десерт и напиток" +``` + +На командной строке изменилось одно слово. Тот же файл, та же модель, тот же +запрос. + +``` +→ nothing_named (exit) +→ pick_dessert (call) + call recipes:search map[query:подбери десерт и напиток section:dessert] +→ pick_drink (call) + call recipes:search map[query:подбери десерт и напиток section:drink] +→ pick_main (call) +→ answer (instruction) + +=== steps === + nothing_named exit skipped calls=0 condition … is false + pick_dessert call ok calls=1 + pick_drink call ok calls=1 + pick_main call skipped calls=0 condition input contains горяч | второе | main course is false + answer instruction ok calls=0 + +=== cost (workflow) === + generations: 1 + tokens: 89 prompt + 13 completion = 102 +``` + +Сравни с шагом 3. + +--- + +## Шаг 5 — откуда взялась разница + +Открой [`examples/skills/menu.yaml`](examples/skills/menu.yaml) рядом с выводом +выше. Работу сделали четыре вещи. + +**1. Словарь вообще не доехал до модели.** + +```yaml +- name: pick_dessert + when: "input contains десерт | сладк | dessert" +``` + +В прозе эти синонимы лежат в промпте, и их применение — генерация, которую +модель может провалить. Здесь они условие: слова сопоставляются кодом, до того +как хоть что-то куда-то отправлено. Отсюда бо́льшая часть разницы в токенах — и +вся разница в точности. + +**2. Два шага отработали вообще без модели.** + +```yaml + call: + tool: "recipes:search" + args: {section: dessert, query: "{{input}}"} +``` + +Шаг `call` — это вызов инструмента, у которого аргументы уже известны. В прозе +та же работа — это генерация «решить позвать», сам вызов и ещё генерация +«пересказать результат». Здесь это вызов. + +**3. Отвечающему шагу не выдали инструментов.** + +```yaml +- name: answer + instruction: | + … + tools: [] +``` + +Пустой список — не «без предпочтений»: шаг физически не может никуда пойти. В +прозе «дальше не ходи» — просьба, здесь идти нечем. + +**4. Видно то, что НЕ исполнялось.** + +``` +skipped: nothing_named, pick_main +``` + +«Мы туда не ходили» и «сходили, там пусто» — разные ответы. След держит их +порознь, поэтому отвечающий шаг не может выдать одно за другое. + +--- + +## Шаг 6 — заставить его решить то, чего он не может выдумать + +``` +go run . -skill ../skills/menu.yaml -mode workflow -input "что посоветуешь" +go run . -skill ../skills/menu.yaml -mode playbook -input "что посоветуешь" +``` + +Шагами: + +``` +→ nothing_named (exit) +the skill stepped aside: skill-engine: exit: the request names no section of the menu +``` + +Прозой: модель отвечает. Ей сказали не выдумывать, и эта инструкция исполняется +ровно настолько, насколько она дочитала до неё, прежде чем начать писать. + +Это и есть свойство, ради которого формат существует, и оно не про стоимость: +**в шагах невозможное невозможно, а не нежелательно.** Ветки, которая пошла бы +искать, на этом пути просто нет. + +--- + +## Шаг 7 — написать свой скилл + +Минимальный полезный файл. Сохрани как `my-skill.yaml`: + +```yaml +skill_engine_version: "2.2.0" +skill_version: "1.0.0" +name: my-skill +description: Для ЧЕГО этот скилл — и для чего он НЕ нужен. +trigger_examples: + - "формулировка, которую пользователь реально напишет" + +workflow: + steps: + # Шаг без инструментов никуда пойти не может. Он думает, и всё. + - name: answer + instruction: | + Запрос: {{input}} + + Ответь в двух предложениях. + tools: [] +``` + +``` +go run . -skill ./my-skill.yaml -input "привет" +``` + +Дальше наращивай в таком порядке — каждое из этого одна строка в файле: + +| что нужно | что добавить | +|---|---| +| ветвиться по словам запроса | `when: "input contains слово \| синоним"` | +| позвать инструмент с известными аргументами | шаг `call:` | +| пройти циклом по тому, что вернул шаг | `for_each: {in: var, as: item, collect: out}` | +| выйти, если случай не твой | `exit: {reason: "…"}` | +| структурный ответ, по которому ветвиться | `response_schema:` **и рядом** `model:` | +| отдать работу другому скиллу | `delegate: {skill: other, task: "…"}` | + +Полный список полей с причиной существования каждого — +[`skill.schema.ru.yaml`](skill.schema.ru.yaml); другие формы — +в [`examples/skills/`](examples/skills/). + +--- + +## Шаг 8 — проверить скилл до запуска + +Два уровня, и они отвечают на разные вопросы. + +```go +// Запустится ли вообще? +if err := skill.Validate(); err != nil { … } +``` + +```go +// Хорошо ли запустится? 27 правил про дефекты, которые молчат. +rep, err := lint.Lint(raw, facts, lint.Options{ + Unmarshal: yaml.Unmarshal, + EmptyWords: []string{"пусто", "empty"}, + HostVars: []string{"input"}, +}) +fmt.Println(rep.Text()) +``` + +Линтер ловит то, чего не покажет прогон: цикл, собирающий в переменную, куда +никто не пишет; опечатку в имени переменной, которая разрешится в пустую строку; +обязательное поле, которому инструкция рядом разрешает быть пустым. См. +[`lint/README.md`](lint/README.md). + +--- + +## Шаг 9 — встроить в своё приложение + +Весь контракт — одна структура. Из +[`examples/simple-llm-app/main.go`](examples/simple-llm-app/main.go): + +```go +skill, err := se.ParseSkill(raw, yaml.Unmarshal) // файл целиком +if err := skill.Validate(); err != nil { … } + +mode, err := skill.ResolveMode() // шаги или проза? +if mode == se.ModePlaybook { + // движок не участвует: гони skill.Playbook как свой промпт +} + +vars, outcome, err := se.ExecuteWith(ctx, skill.Workflow, se.Deps{ + Runner: yourModel, // исполняет шаг-инструкцию + Caller: yourTools, // исполняет шаг-вызов + Assets: yourResolver, // достаёт содержимое ассета + Memory: yourMemory, // отдаёт крупный результат по рукояти + Vocabulary: se.Vocabulary{ + DecisionMarkers: []string{"Result:", "Ответ:"}, + }, + OnStepStart: func(name, kind string) { … }, +}, map[string]string{"input": userText}) +``` + +`vars[se.AnswerVar]` — ответ хода. `outcome.Steps` — тот самый след из вывода +выше: движок ничего не логирует и ничего не хранит, поэтому эта структура и есть +вся его наблюдаемость. + +Два рабочих приложения, с которых можно копировать: + +- [`examples/simple-llm-app`](examples/simple-llm-app/) — `net/http` и ничего + больше; +- [`examples/eino-llm-app`](examples/eino-llm-app/) — модель через фреймворк, + причём весь фреймворк-специфичный код в одном адаптере на сорок строк. + +--- + +## Честные оговорки про числа выше + +- Они сняты с **заглушки**, считающей по длине промпта, — то есть показывают + ФОРМУ разницы, а не твой счёт. Прогоняй шаги 3–4 против настоящего эндпоинта, + если нужны настоящие числа. +- С настоящим эндпоинтом прозаическая половина тратит ещё и **лишние + генерации**: решить позвать инструмент, позвать, прочитать результат. + Заглушка звать инструменты не умеет и делает всё одной генерацией, поэтому + разрыв в таблице выше занижен, а не завышен. +- Утверждение про точность — шаг-классификатор при temperature 0 берёт 5 живых + запросов из 10 там, где тот же словарь условием берёт 10 — измерено на чужом + каталоге, и README прямо говорит, чего про этот замер не установлено. +- Один контрпример, намеренно: скилл поиска по логам стоил **43k токенов шагами + против 36k прозой**, потому что знание дорого в шаге С ИНСТРУМЕНТАМИ — ассет + едет в каждую генерацию цикла. Шаги не бесплатны; они дешевле там, где у хода + есть ветки, циклы и запреты. + +--- + +## Куда дальше + +| | | +|---|---| +| [README.ru.md](README.ru.md) | что это за движок, чем он НЕ является и на что можно опираться | +| [`examples/`](examples/) | формат и два приложения, которые его встраивают | +| [`skill.schema.ru.yaml`](skill.schema.ru.yaml) | каждое поле и отказ, которым оно оплачено | +| [CHANGELOG.md](CHANGELOG.md) | что менялось в формате и что делает миграция | +| [`lint/README.md`](lint/README.md) | 27 правил и что ловит каждое | diff --git a/README.md b/README.md index de8d1d6..8894513 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,14 @@ instruction) is a full skill too (see "A prompt works as well"). go get github.com/inhuman/skill-engine ``` -**Start here:** [`examples/`](examples/) — the format in +**Start here → [QUICKSTART.md](QUICKSTART.md).** Fifteen minutes, at the end of +which you have run the same skill BOTH ways — as a prompt and as steps — on the +same request, and seen the difference in what it cost and in what it did. Steps +1–5 need no model at all. + +**Then → [`examples/`](examples/):** the format in [`examples/skills/`](examples/skills/), and two working applications that embed -the engine: [`simple-llm-app`](examples/simple-llm-app/) on an +the engine — [`simple-llm-app`](examples/simple-llm-app/) on an OpenAI-compatible endpoint with nothing but `net/http`, and [`eino-llm-app`](examples/eino-llm-app/) with the model reached through a framework. Both run offline in their tests. diff --git a/README.ru.md b/README.ru.md index 001af2d..4b475b9 100644 --- a/README.ru.md +++ b/README.ru.md @@ -18,9 +18,14 @@ go get github.com/inhuman/skill-engine ``` -**Начинать отсюда:** [`examples/`](examples/) — сам формат в +**Начинать отсюда → [QUICKSTART.ru.md](QUICKSTART.ru.md).** Пятнадцать минут, в +конце которых ты прогнал один и тот же скилл ДВУМЯ способами — промптом и +шагами — на одном запросе и увидел разницу в том, сколько это стоило и что при +этом произошло. Шаги 1–5 не требуют модели вовсе. + +**Дальше → [`examples/`](examples/):** сам формат в [`examples/skills/`](examples/skills/) и два рабочих приложения, встраивающих -движок: [`simple-llm-app`](examples/simple-llm-app/) на OpenAI-совместимом +движок, — [`simple-llm-app`](examples/simple-llm-app/) на OpenAI-совместимом эндпоинте и голом `net/http` и [`eino-llm-app`](examples/eino-llm-app/), где модель достаётся через фреймворк. Оба гоняются в тестах без сети. diff --git a/examples/simple-llm-app/README.md b/examples/simple-llm-app/README.md index 1ce5feb..a95f446 100644 --- a/examples/simple-llm-app/README.md +++ b/examples/simple-llm-app/README.md @@ -9,8 +9,16 @@ export OPENAI_API_KEY=… export OPENAI_MODEL=… go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" + +# the same file run the other way, when a skill carries both descriptions +go run . -skill ../skills/menu.yaml -mode playbook -input "подбери десерт и напиток" ``` +`-mode` overrides the skill's own `mode`, which is what makes the A/B in +[QUICKSTART.md](../../QUICKSTART.md) a one-word change: same file, same model, +same request. Each run reports what it cost — generations and tokens — so the +two halves can be compared rather than argued about. + ## What it shows **Loading a skill.** `ParseSkill` reads the whole file — header and description diff --git a/examples/simple-llm-app/main.go b/examples/simple-llm-app/main.go index f0db2a7..f5d4c44 100644 --- a/examples/simple-llm-app/main.go +++ b/examples/simple-llm-app/main.go @@ -32,15 +32,19 @@ import ( func main() { skillPath := flag.String("skill", "../skills/menu.yaml", "path to a skill file") input := flag.String("input", "подбери десерт и напиток", "the user's request") + // Overrides the skill's own `mode`. A skill carrying BOTH descriptions can + // then be run either way on the same request — which is the A/B in + // QUICKSTART.md, and the reason `mode` exists in the format at all. + mode := flag.String("mode", "", "run the skill as `workflow` or `playbook` (default: what the skill says)") flag.Parse() - if err := run(*skillPath, *input, os.Stdout); err != nil { + if err := run(*skillPath, *input, *mode, os.Stdout); err != nil { fmt.Fprintln(os.Stderr, "error:", err) os.Exit(1) } } -func run(skillPath, input string, out io.Writer) error { +func run(skillPath, input, forceMode string, out io.Writer) error { raw, err := os.ReadFile(skillPath) if err != nil { return err @@ -61,20 +65,36 @@ func run(skillPath, input string, out io.Writer) error { // Which of the two descriptions to run. A skill may carry a `playbook` as // well, and then the ENGINE takes no part: the application runs the prompt // itself. Handling that case is what makes an embedder complete. - mode, err := skill.ResolveMode() + // + // An explicit mode with an empty half is an ERROR rather than a fallback to + // the other one, and the engine enforces that: a silent fallback would give + // a clean run over the description that was NOT selected, and a conclusion + // drawn from a turn the chosen half never took part in. + declared := skill.Mode + if forceMode != "" { + declared = forceMode + } + mode, err := se.ResolveMode(declared, skill.HasWorkflow(), skill.HasPlaybook()) if err != nil { return err } + + st := &stats{} if mode == se.ModePlaybook { - answer, err := newModel().complete(context.Background(), skill.Playbook+"\n\n"+input, nil) + // The whole task in one prompt: the model decides what to look up, calls + // what it decides, and words the answer. The engine takes no part. + answer, u, err := newModel().complete(context.Background(), skill.Playbook+"\n\nЗапрос: "+input, nil) if err != nil { return err } + st.add(u) + fmt.Fprintln(out, "\n=== answer ===") fmt.Fprintln(out, answer) + st.report(out, "playbook") return nil } - vars, outcome, err := se.ExecuteWith(context.Background(), skill.Workflow, deps(out), map[string]string{ + vars, outcome, err := se.ExecuteWith(context.Background(), skill.Workflow, deps(out, st), map[string]string{ "input": input, }) if err != nil { @@ -100,19 +120,42 @@ func run(skillPath, input string, out io.Writer) error { if len(outcome.Skipped) > 0 { fmt.Fprintln(out, " skipped:", strings.Join(outcome.Skipped, ", ")) } + st.report(out, "workflow") return nil } +// stats — what the turn cost. Counted because "steps are cheaper" is a claim +// until somebody puts a number next to it, and the same skill run both ways on +// the same request is the cheapest way to get one. +type stats struct { + generations int + prompt int + completion int +} + +func (s *stats) add(u usage) { + s.generations++ + s.prompt += u.PromptTokens + s.completion += u.CompletionTokens +} + +func (s *stats) report(out io.Writer, mode string) { + fmt.Fprintf(out, "\n=== cost (%s) ===\n", mode) + fmt.Fprintf(out, " generations: %d\n", s.generations) + fmt.Fprintf(out, " tokens: %d prompt + %d completion = %d\n", + s.prompt, s.completion, s.prompt+s.completion) +} + // deps is the whole contract between an application and the engine. // // Everything the engine cannot know — how to reach a model, what a tool is, // where an asset's content lives, what your host calls things — arrives here. // Nothing else is injected, and the engine logs nothing, stores nothing and // reaches nowhere on its own. -func deps(out io.Writer) se.Deps { +func deps(out io.Writer, st *stats) se.Deps { m := newModel() return se.Deps{ - Runner: runner{model: m, log: out}, + Runner: runner{model: m, log: out, stats: st}, Caller: tools{log: out}, Assets: assets{}, Delegate: nil, // no composite skills here: a `delegate` step would fail loudly @@ -141,6 +184,7 @@ func deps(out io.Writer) se.Deps { type runner struct { model *openAI log io.Writer + stats *stats } func (r runner) Run(ctx context.Context, req se.StepRequest) (se.Result, error) { @@ -152,10 +196,11 @@ func (r runner) Run(ctx context.Context, req se.StepRequest) (se.Result, error) fmt.Fprintf(r.log, " (step %q may use: %s)\n", req.Name, strings.Join(req.Tools, ", ")) } - text, err := r.model.complete(ctx, req.Instruction, &req) + text, u, err := r.model.complete(ctx, req.Instruction, &req) if err != nil { return se.Result{}, err } + r.stats.add(u) // Result carries more than the text: what the executor KNOWS and the engine // cannot derive. A truncated generation and a step that simply had nothing diff --git a/examples/simple-llm-app/main_test.go b/examples/simple-llm-app/main_test.go index cad42bc..23b36e9 100644 --- a/examples/simple-llm-app/main_test.go +++ b/examples/simple-llm-app/main_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" @@ -35,12 +36,23 @@ func stubModel(t *testing.T, answer string) *httptest.Server { if len(body.Messages) == 0 { t.Error("the instruction never reached the model") } + // Charged by length, the way a real endpoint charges. That is what makes + // the A/B in QUICKSTART.md checkable without a model: the prose half + // carries its whole dictionary into the prompt, the steps half does not. + prompt := 0 + for _, m := range body.Messages { + prompt += len(m.Content) / 4 + } w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]any{ "choices": []any{map[string]any{ "message": map[string]string{"content": answer}, "finish_reason": "stop", }}, + "usage": map[string]int{ + "prompt_tokens": prompt, + "completion_tokens": len(answer) / 4, + }, }) })) t.Cleanup(srv.Close) @@ -62,7 +74,7 @@ func TestExampleRunsAShippedSkill(t *testing.T) { withStub(t, "Тирамису и чай — 30 минут.") var out bytes.Buffer - if err := run("../skills/menu.yaml", "подбери десерт и напиток", &out); err != nil { + if err := run("../skills/menu.yaml", "подбери десерт и напиток", "", &out); err != nil { t.Fatalf("the example does not run: %v", err) } got := out.String() @@ -95,7 +107,7 @@ func TestExampleHandlesAnExit(t *testing.T) { withStub(t, "unused") var out bytes.Buffer - if err := run("../skills/menu.yaml", "расскажи что-нибудь", &out); err != nil { + if err := run("../skills/menu.yaml", "расскажи что-нибудь", "", &out); err != nil { t.Fatalf("an exit was reported as a failure: %v", err) } if !strings.Contains(out.String(), "stepped aside") { @@ -112,7 +124,7 @@ func TestEveryShippedSkillLoads(t *testing.T) { for _, path := range shippedSkills(t) { t.Run(path, func(t *testing.T) { var out bytes.Buffer - err := run(path, "подбери десерт", &out) + err := run(path, "подбери десерт", "", &out) // Running is allowed to fail — most of these skills need tools this // example does not implement. Loading is not. if err != nil && strings.Contains(err.Error(), "skill-engine:") { @@ -148,3 +160,56 @@ func shippedSkills(t *testing.T) []string { } return out } + +// The A/B QUICKSTART.md walks a reader through: one skill carrying both +// descriptions, the same request, one flag changed. +// +// What is checked here is the mechanism and the direction, not a benchmark — +// the stub charges by prompt length, so the numbers are the shape of the real +// ones rather than the real ones. In prose the dictionary of sections rides +// into the prompt and the model has to apply it; in steps the conditions +// applied it before any generation happened, and the prompt carries only what +// was found. +func TestSameSkillBothWays(t *testing.T) { + withStub(t, "Тирамису и чай.") + + var prose, steps bytes.Buffer + if err := run("../skills/menu.yaml", "подбери десерт и напиток", "playbook", &prose); err != nil { + t.Fatalf("the prose half does not run: %v", err) + } + if err := run("../skills/menu.yaml", "подбери десерт и напиток", "workflow", &steps); err != nil { + t.Fatalf("the steps half does not run: %v", err) + } + + for _, out := range []*bytes.Buffer{&prose, &steps} { + if !strings.Contains(out.String(), "=== cost (") { + t.Fatalf("the run did not report what it cost:\n%s", out.String()) + } + } + if p, s := promptTokens(t, &prose), promptTokens(t, &steps); s >= p { + t.Errorf("steps cost %d prompt tokens against prose %d — the dictionary should not reach the model at all", s, p) + } + + // And the part a token count cannot show: in steps the sections are chosen + // before any generation, so a section nobody named cannot be fetched. In + // prose that is a request, and requests are followed probabilistically. + if !strings.Contains(steps.String(), "skipped: nothing_named, pick_main") { + t.Errorf("the steps half did not decide the sections deterministically:\n%s", steps.String()) + } +} + +func promptTokens(t *testing.T, out *bytes.Buffer) int { + t.Helper() + for _, line := range strings.Split(out.String(), "\n") { + if !strings.Contains(line, "prompt + ") { + continue + } + var p, c, total int + if _, err := fmt.Sscanf(strings.TrimSpace(line), "tokens: %d prompt + %d completion = %d", &p, &c, &total); err != nil { + t.Fatalf("cannot read the cost line %q: %v", line, err) + } + return p + } + t.Fatalf("no cost line in:\n%s", out.String()) + return 0 +} diff --git a/examples/simple-llm-app/openai.go b/examples/simple-llm-app/openai.go index c11a48b..02ebfe5 100644 --- a/examples/simple-llm-app/openai.go +++ b/examples/simple-llm-app/openai.go @@ -48,7 +48,7 @@ func env(name, fallback string) string { // a `model:` the skill named, its `sampling:`, its `response_schema:`. An // executor that ignores them turns every one of those fields into decoration — // the skill declares, the engine forwards, and nothing happens. -func (o *openAI) complete(ctx context.Context, instruction string, req *se.StepRequest) (string, error) { +func (o *openAI) complete(ctx context.Context, instruction string, req *se.StepRequest) (string, usage, error) { body := map[string]any{ "model": o.model, "messages": []map[string]string{{"role": "user", "content": instruction}}, @@ -86,11 +86,11 @@ func (o *openAI) complete(ctx context.Context, instruction string, req *se.StepR raw, err := json.Marshal(body) if err != nil { - return "", err + return "", usage{}, err } httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, o.baseURL+"/chat/completions", bytes.NewReader(raw)) if err != nil { - return "", err + return "", usage{}, err } httpReq.Header.Set("Content-Type", "application/json") if o.apiKey != "" { @@ -99,7 +99,7 @@ func (o *openAI) complete(ctx context.Context, instruction string, req *se.StepR resp, err := o.client.Do(httpReq) if err != nil { - return "", err + return "", usage{}, err } defer resp.Body.Close() @@ -109,15 +109,25 @@ func (o *openAI) complete(ctx context.Context, instruction string, req *se.StepR FinishReason string `json:"finish_reason"` } `json:"choices"` Error *struct{ Message string } `json:"error"` + // What the run COST. Without it "steps are cheaper" is a claim; with it + // the two halves of a skill can be compared on the same request, which + // is what QUICKSTART.md has the reader do. + Usage usage `json:"usage"` } if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { - return "", fmt.Errorf("decoding the answer: %w", err) + return "", usage{}, fmt.Errorf("decoding the answer: %w", err) } if out.Error != nil { - return "", fmt.Errorf("model: %s", out.Error.Message) + return "", usage{}, fmt.Errorf("model: %s", out.Error.Message) } if len(out.Choices) == 0 { - return "", fmt.Errorf("model returned no choices") + return "", usage{}, fmt.Errorf("model returned no choices") } - return out.Choices[0].Message.Content, nil + return out.Choices[0].Message.Content, out.Usage, nil +} + +// usage — what one generation cost, as the endpoint reports it. +type usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` } diff --git a/examples/simple-llm-app/quickstart_test.go b/examples/simple-llm-app/quickstart_test.go new file mode 100644 index 0000000..a550c50 --- /dev/null +++ b/examples/simple-llm-app/quickstart_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// The minimal skill QUICKSTART.md tells a reader to save and run. It is the +// first thing they will execute, so it is the first thing that must work. +func TestQuickstartMinimalSkill(t *testing.T) { + const skill = `skill_engine_version: "2.2.0" +skill_version: "1.0.0" +name: my-skill +description: What this skill is FOR — and what it is NOT for. +trigger_examples: + - "a phrasing a user would actually type" + +workflow: + steps: + # A step with no tools cannot go anywhere. It thinks, and that is all. + - name: answer + instruction: | + The request: {{input}} + + Answer it in two sentences. + tools: [] +` + path := filepath.Join(t.TempDir(), "my-skill.yaml") + if err := os.WriteFile(path, []byte(skill), 0o600); err != nil { + t.Fatal(err) + } + withStub(t, "Привет. Чем помочь?") + + var out bytes.Buffer + if err := run(path, "привет", "", &out); err != nil { + t.Fatalf("the skill QUICKSTART tells the reader to write does not run: %v", err) + } + if !strings.Contains(out.String(), "Чем помочь") { + t.Errorf("no answer:\n%s", out.String()) + } +} diff --git a/examples/skills/menu.yaml b/examples/skills/menu.yaml index 4b0a742..8fc9f06 100644 --- a/examples/skills/menu.yaml +++ b/examples/skills/menu.yaml @@ -14,7 +14,7 @@ # warehouse shares this mechanism and not one of these words. skill_engine_version: "2.2.0" -skill_version: "1.0.0" +skill_version: "1.1.0" name: menu description: Suggest dishes from the sections of the menu the request names — a dessert, a drink, a main course. trigger_examples: @@ -25,6 +25,31 @@ kind: leaf servers: [recipes] +# This skill carries BOTH descriptions, which is what `mode` is for: while a +# prompt is being moved into steps it is worth keeping the old half and +# switching between them on live requests. Deleting it to compare would mean +# deleting the work with nowhere to get it back from. +# +# `mode: workflow` is what actually runs. Point it at `playbook` — or override +# it from the outside, as ../simple-llm-app does with -mode — and the same +# request goes through the prose version instead. That is the A/B in +# QUICKSTART.md: same file, same model, one word changed. +mode: workflow + +playbook: | + Ты подбираешь блюда по разделам меню. + + Разделы и слова, которыми их называют: + десерт, сладкое, dessert → dessert + напиток, чай, кофе, коктейль → drink + горячее, второе, main course → main + + Посмотри, какие разделы названы в запросе, и возьми РОВНО названные. + По каждому названному разделу вызови recipes:search с этим разделом. + Если не назван ни один — так и скажи, ничего не выдумывай. + + Ответь по тому, что нашлось. Про раздел, который не спрашивали, не пиши. + workflow: tools: [recipes] diff --git a/readme_test.go b/readme_test.go index 469c35b..8992640 100644 --- a/readme_test.go +++ b/readme_test.go @@ -19,23 +19,29 @@ import ( // reader of the Russian file must not be reading a version of the library that // no longer exists. func TestBothReadmesHaveTheSameShape(t *testing.T) { - en := mustRead(t, "README.md") - ru := mustRead(t, "README.ru.md") + forEachTranslatedPair(t, "README.md", "README.ru.md") + forEachTranslatedPair(t, "QUICKSTART.md", "QUICKSTART.ru.md") +} + +func forEachTranslatedPair(t *testing.T, enPath, ruPath string) { + t.Helper() + en := mustRead(t, enPath) + ru := mustRead(t, ruPath) enSections, ruSections := sectionsOf(string(en)), sectionsOf(string(ru)) require.Equal(t, len(enSections), len(ruSections), - "the two READMEs have a different number of sections:\nEN: %s\nRU: %s", - strings.Join(enSections, " | "), strings.Join(ruSections, " | ")) + "%s and %s have a different number of sections:\nEN: %s\nRU: %s", + enPath, ruPath, strings.Join(enSections, " | "), strings.Join(ruSections, " | ")) assert.Equal(t, strings.Count(string(en), "```"), strings.Count(string(ru), "```"), - "one README gained or lost a code sample") + "%s and %s: one of them gained or lost a code sample", enPath, ruPath) // The links a cold reader arrives by. A broken one in a file linked from an // article is the cheapest possible way to lose them. for _, doc := range []string{string(en), string(ru)} { for _, path := range localLinks(doc) { _, err := os.Stat(path) - assert.NoErrorf(t, err, "README links to %s, which does not exist", path) + assert.NoErrorf(t, err, "%s links to %s, which does not exist", enPath, path) } } } From 7da4a9da88dd641d63183a24c0fd7efa00bae99e Mon Sep 17 00:00:00 2001 From: Ivan Diatchenko <2518263+inhuman@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:11:50 +0300 Subject: [PATCH 4/8] docs: replace the unbacked numbers with the live measurement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README carried three pairs of numbers with no methodology behind them and a paragraph admitting as much. There is now a real measurement, and it is both stronger and more honest than what it replaces. The event log of a working installation, five weeks, every turn where a skill matched, questions asked by people rather than by the author of the skill. The metric is LLM generations per turn, orchestrator and subagents together. 23 skills with at least 5 turns on each side, 5 280 turns 20 significantly cheaper, 1 significantly more expensive, 2 unchanged effect from -18 to -0.5 generations per turn; largest 38 -> 20, typical 7 -> 3 Recomputed from the raw rows rather than quoted: Mann-Whitney with a tie correction, a stated inclusion threshold, and the aggregate checked against the summary that came with the data. The threshold is why the counts here differ slightly from that summary — it is written down. Two things are stated because leaving them out would be the expensive kind of silence. The comparison is OBSERVATIONAL. The periods are split by a date rather than randomised, and other things changed in those same days: the engine was being edited alongside the skills. So it shows the catalogue got cheaper across that boundary, not that nothing else contributed. Named in the README in those words. And the loss. One skill went the other way — median 6 -> 10 generations, p<0.001. Twenty wins and no losses read as advertising; one measured loss is what makes the other twenty worth reading. What caused it the measurement does not say, and the text says that too: it counts generations, not reasons. The old counterexample (43k tokens against 36k) is gone with the rest of the unbacked numbers, replaced by this measured one. No skill names travelled with the data: the log belongs to a private installation, so what is published is the aggregate. Both quickstarts point at the measurement instead of restating figures nobody could check. --- QUICKSTART.md | 17 ++++++++-------- QUICKSTART.ru.md | 14 ++++++------- README.md | 52 ++++++++++++++++++++++++++++++++++-------------- README.ru.md | 46 +++++++++++++++++++++++++++++++----------- 4 files changed, 87 insertions(+), 42 deletions(-) diff --git a/QUICKSTART.md b/QUICKSTART.md index db966ae..110cac6 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -329,14 +329,15 @@ Two working applications to copy from: to decide to call a tool, call it, and read the result back. The stub cannot call tools, so it does that work in one generation and the gap in the table above is understated, not overstated. -- The accuracy claim — a classifier step at temperature 0 getting 5 of 10 live - requests right where the same dictionary in a condition got 10 — is measured - on somebody else's catalogue, and the README says plainly what is not - established about it. -- One counterexample, on purpose: a log-searching skill cost **43k tokens as - steps against 36k as prose**, because knowledge is expensive in a step WITH - tools — an asset rides into every generation of the loop. Steps are not - free; they are cheaper where the turn has branches, loops and guards. +- The real numbers are in [README.md](README.md): a live installation's event + log, 5 280 turns over five weeks, 23 skills — 20 significantly cheaper, 1 + significantly more expensive, 2 unchanged. That comparison is + **observational**: the periods are split by a date rather than randomised, and + other things changed in those same days. +- One counterexample, on purpose: a health-checking skill went the other way, + a median of **6 → 10 generations per turn**. Steps are not free; they are + cheaper where the turn has branches, loops and guards, and this is what the + other case looks like. --- diff --git a/QUICKSTART.ru.md b/QUICKSTART.ru.md index 7ca4980..43793a0 100644 --- a/QUICKSTART.ru.md +++ b/QUICKSTART.ru.md @@ -328,13 +328,13 @@ vars, outcome, err := se.ExecuteWith(ctx, skill.Workflow, se.Deps{ генерации**: решить позвать инструмент, позвать, прочитать результат. Заглушка звать инструменты не умеет и делает всё одной генерацией, поэтому разрыв в таблице выше занижен, а не завышен. -- Утверждение про точность — шаг-классификатор при temperature 0 берёт 5 живых - запросов из 10 там, где тот же словарь условием берёт 10 — измерено на чужом - каталоге, и README прямо говорит, чего про этот замер не установлено. -- Один контрпример, намеренно: скилл поиска по логам стоил **43k токенов шагами - против 36k прозой**, потому что знание дорого в шаге С ИНСТРУМЕНТАМИ — ассет - едет в каждую генерацию цикла. Шаги не бесплатны; они дешевле там, где у хода - есть ветки, циклы и запреты. +- Настоящие числа — в [README.ru.md](README.ru.md): журнал событий работающей + установки, 5 280 ходов за пять недель, 23 скилла — 20 значимо дешевле, 1 + значимо дороже, 2 без разницы. То сравнение **наблюдательное**: периоды + разделены датой, а не рандомизацией, и в те же дни менялось другое. +- Один контрпример, намеренно: скилл проверки здоровья пошёл в обратную сторону, + медиана **6 → 10 генераций на ход**. Шаги не бесплатны; они дешевле там, где у + хода есть ветки, циклы и запреты, а вот так выглядит противоположный случай. --- diff --git a/README.md b/README.md index 8894513..c9cf8bd 100644 --- a/README.md +++ b/README.md @@ -73,21 +73,43 @@ the model read before it started acting. In steps the same thing is expressed structurally: in the unconfirmed branch the `retract` call **is not there**, a `call` step cannot be repeated, a branch that does not apply does not run. -Measured on a live catalogue — tool calls / seconds, before → after moving a -skill from prose into steps: 18/95 → **2/6**, 7/33 → **2/5**, 9/29 → **5/11**. -What those skills did is beside the point; what changed is who held the control -flow. - -And one that went the other way. A log-searching skill cost **43k tokens as -steps against 36k as prose** — because knowledge is expensive in a step WITH -tools: an asset rides along into every generation of the react loop. Three wins -and no losses read as advertising, and this is the shape of the case where the -format does not pay. - -What is NOT established about these numbers: how many runs per scenario, the -spread between them, and whether anything besides the form of the description -changed. Until that is measured, read them as an order of magnitude rather than -as a benchmark. +### What it changed, measured on live traffic + +Not a benchmark of one question run twice — the event log of a working +installation: every turn where a skill matched, over five weeks, questions asked +by people rather than by the author of the skill. The metric is **LLM +generations per turn**, orchestrator and subagents together. + +The catalogue moved from prose to steps on one day, and the periods are split by +that date. + +| | | +|---|---:| +| skills with at least 5 turns on each side | **23** | +| turns compared | **5 280** (3 469 prose, 1 811 steps) | +| significantly cheaper (Mann–Whitney, p<0.05) | **20** | +| significantly more expensive | **1** | +| no significant difference | **2** | + +The effect is a median of −18 to −0.5 generations per turn. The largest: a +triage skill went from a median of **38 generations per turn to 20**. Typical: +**7 → 3**. + +**And the one that got worse.** A health-checking skill went the other way — +median **6 → 10** generations, p<0.001. It is in the table on purpose: twenty +wins and no losses read as advertising, and one measured loss is what makes the +other twenty worth reading. Which mechanism did it the measurement does not +say — it counts generations, not reasons — and the invariants below list the +ways a step gets MORE expensive, starting with knowledge inside a step that has +tools. + +**What this is and is not.** The periods are separated by a DATE, not by +randomisation, and other things changed in those same days — the engine was +being edited alongside the skills. So this is an **observational before/after +comparison, not an experiment**: it shows that the catalogue got cheaper across +that boundary, not that nothing else contributed. The underlying event log +belongs to a private installation, so what is published here is the aggregate +rather than the raw data. ## What this is not diff --git a/README.ru.md b/README.ru.md index 4b475b9..c38a118 100644 --- a/README.ru.md +++ b/README.ru.md @@ -70,18 +70,40 @@ go get github.com/inhuman/skill-engine Шагами то же самое выражается структурой: в неподтверждённой ветке вызова `retract` **нет**, шаг `call` нельзя повторить, лишняя ветка не исполняется. -Замеры на живом каталоге — вызовов инструментов / секунд, до → после переноса -скилла из прозы в шаги: 18/95 → **2/6**, 7/33 → **2/5**, 9/29 → **5/11**. Что -именно эти скиллы делали — неважно; изменилось то, кто держит поток управления. - -И один, который пошёл в обратную сторону. Скилл поиска по логам стоил **43k -токенов шагами против 36k прозой** — потому что знание дорого в шаге С -ИНСТРУМЕНТАМИ: ассет едет в каждую генерацию react-цикла. Три победы без единого -поражения читаются как реклама, а это — форма случая, где формат не окупается. - -Чего про эти числа НЕ установлено: сколько прогонов на сценарий, какой между -ними разброс и менялось ли что-то ещё, кроме формы описания. Пока это не -измерено, читай их как порядок величины, а не как замер. +### Что изменилось, замер на живом трафике + +Это не прогон одного вопроса дважды, а журнал событий работающей установки: все +ходы, где сматчился скилл, за пять недель, и вопросы задавали люди, а не автор +скилла. Метрика — **число генераций LLM на один ход**, оркестратор и субагенты +вместе. + +Каталог перевели с прозы на шаги в один день, и периоды разделены этой датой. + +| | | +|---|---:| +| скиллов, где не меньше 5 ходов с каждой стороны | **23** | +| сравниваемых ходов | **5 280** (3 469 прозой, 1 811 шагами) | +| значимо дешевле (Манн–Уитни, p<0.05) | **20** | +| значимо дороже | **1** | +| без значимой разницы | **2** | + +Эффект — от −18 до −0.5 генерации на ход по медиане. Наибольший: скилл разбора +инцидентов прошёл с медианы **38 генераций на ход до 20**. Типичный: **7 → 3**. + +**И тот, которому стало хуже.** Скилл проверки здоровья сервиса пошёл в обратную +сторону: медиана **6 → 10** генераций, p<0.001. Он в таблице намеренно — +двадцать побед без единого поражения читаются как реклама, а одно измеренное +поражение и делает остальные двадцать читаемыми. Какой именно механизм это +сделал, замер не говорит — он считает генерации, а не причины, — а инварианты +ниже перечисляют способы подорожать, начиная со знания внутри шага, у которого +есть инструменты. + +**Чем это является и чем нет.** Периоды разделены ДАТОЙ, а не рандомизацией, и +в те же дни менялось другое — движок правился одновременно со скиллами. То есть +это **наблюдательное сравнение до/после, а не эксперимент**: оно показывает, что +каталог подешевел через эту границу, но не то, что больше ничто не повлияло. +Журнал событий принадлежит приватной установке, поэтому опубликована сводка, а +не сырьё. ## Чем это не является From 5a118a97de1c524185b0a84dbd90234bfab34cee Mon Sep 17 00:00:00 2001 From: Ivan Diatchenko <2518263+inhuman@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:20:15 +0300 Subject: [PATCH 5/8] docs: say plainly that one skill got worse, and where the limits are MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The paragraph about the skill that got more expensive was written as a credibility move — "one measured loss is what makes the other twenty worth reading". That is advertising with extra steps, and it buries the useful part. It now says what happened and what follows from it: one skill went from a median of 6 generations per turn to 10, the measurement does not say why, steps are not automatically cheaper, and the format does not replace checking. Then the part that was missing — the known ways a rewrite costs MORE: - an asset inside a step that has tools, riding into every generation of the loop rather than just the first; - splitting a turn that had nothing to split, so the second prompt adds a generation without removing work from the first; - a decision that is genuinely open — wording, judging, reading intent — which a condition cannot replace, and pretending otherwise only moves the model call somewhere less visible; - one or two steps and no branching, where prose was already cheaper. Ending on the thing a reader actually needs: measure each skill on its own after rewriting it. The engine gives the trace to measure with and a linter for the quiet defects; it does not promise a rewrite pays. The same tone crept into both quickstarts ("one counterexample, on purpose") and is gone from there too. --- QUICKSTART.md | 8 ++++---- QUICKSTART.ru.md | 7 ++++--- README.md | 29 ++++++++++++++++++++++------- README.ru.md | 29 ++++++++++++++++++++++------- 4 files changed, 52 insertions(+), 21 deletions(-) diff --git a/QUICKSTART.md b/QUICKSTART.md index 110cac6..946b782 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -334,10 +334,10 @@ Two working applications to copy from: significantly more expensive, 2 unchanged. That comparison is **observational**: the periods are split by a date rather than randomised, and other things changed in those same days. -- One counterexample, on purpose: a health-checking skill went the other way, - a median of **6 → 10 generations per turn**. Steps are not free; they are - cheaper where the turn has branches, loops and guards, and this is what the - other case looks like. +- In that same measurement one skill got **more** expensive: a median of + **6 → 10 generations per turn**. Steps are not automatically cheaper — the + README lists the known ways a rewrite costs more, and every skill is worth + measuring on its own afterwards. --- diff --git a/QUICKSTART.ru.md b/QUICKSTART.ru.md index 43793a0..b2c51c8 100644 --- a/QUICKSTART.ru.md +++ b/QUICKSTART.ru.md @@ -332,9 +332,10 @@ vars, outcome, err := se.ExecuteWith(ctx, skill.Workflow, se.Deps{ установки, 5 280 ходов за пять недель, 23 скилла — 20 значимо дешевле, 1 значимо дороже, 2 без разницы. То сравнение **наблюдательное**: периоды разделены датой, а не рандомизацией, и в те же дни менялось другое. -- Один контрпример, намеренно: скилл проверки здоровья пошёл в обратную сторону, - медиана **6 → 10 генераций на ход**. Шаги не бесплатны; они дешевле там, где у - хода есть ветки, циклы и запреты, а вот так выглядит противоположный случай. +- В том же замере одному скиллу стало **дороже**: медиана **6 → 10 генераций на + ход**. Шаги не дешевле автоматически — README перечисляет известные способы + подорожать при переписывании, и каждый скилл после него стоит измерить + отдельно. --- diff --git a/README.md b/README.md index c9cf8bd..56fd2e5 100644 --- a/README.md +++ b/README.md @@ -95,13 +95,28 @@ The effect is a median of −18 to −0.5 generations per turn. The largest: a triage skill went from a median of **38 generations per turn to 20**. Typical: **7 → 3**. -**And the one that got worse.** A health-checking skill went the other way — -median **6 → 10** generations, p<0.001. It is in the table on purpose: twenty -wins and no losses read as advertising, and one measured loss is what makes the -other twenty worth reading. Which mechanism did it the measurement does not -say — it counts generations, not reasons — and the invariants below list the -ways a step gets MORE expensive, starting with knowledge inside a step that has -tools. +**One skill got worse.** A health-checking skill went the other way — median +**6 → 10** generations, p<0.001. The measurement does not say why: it counts +generations, not reasons. + +So steps are not automatically cheaper, and the format is not a substitute for +checking. Known ways a skill gets MORE expensive when it moves into steps: + +- **an asset inside a step that has tools.** The knowledge rides along into + every generation of the react loop, not just the first one. Splitting into + "decide" (knowledge, no tools) and "do" (tools, no knowledge) is the fix; +- **splitting a turn that had nothing to split.** Two steps means two prompts, + each carrying its own context. If the second step does not remove work from + the first, it only adds a generation; +- **a decision that is genuinely open.** Wording an answer, judging quality, + reading somebody's intent — a condition cannot replace that, and pretending + otherwise just moves the model call somewhere less visible; +- **one or two steps and no branching at all** — prose is cheaper, and the + format says so itself (see "A prompt works as well"). + +Every skill is worth measuring on its own after it is rewritten. This engine +gives you the trace to measure with (`Outcome.Steps`) and a linter for the +defects that stay quiet; it does not promise that a rewrite pays. **What this is and is not.** The periods are separated by a DATE, not by randomisation, and other things changed in those same days — the engine was diff --git a/README.ru.md b/README.ru.md index c38a118..b0f7fea 100644 --- a/README.ru.md +++ b/README.ru.md @@ -90,13 +90,28 @@ go get github.com/inhuman/skill-engine Эффект — от −18 до −0.5 генерации на ход по медиане. Наибольший: скилл разбора инцидентов прошёл с медианы **38 генераций на ход до 20**. Типичный: **7 → 3**. -**И тот, которому стало хуже.** Скилл проверки здоровья сервиса пошёл в обратную -сторону: медиана **6 → 10** генераций, p<0.001. Он в таблице намеренно — -двадцать побед без единого поражения читаются как реклама, а одно измеренное -поражение и делает остальные двадцать читаемыми. Какой именно механизм это -сделал, замер не говорит — он считает генерации, а не причины, — а инварианты -ниже перечисляют способы подорожать, начиная со знания внутри шага, у которого -есть инструменты. +**Одному скиллу стало хуже.** Скилл проверки здоровья сервиса пошёл в обратную +сторону: медиана **6 → 10** генераций, p<0.001. Почему — замер не говорит: он +считает генерации, а не причины. + +То есть шаги не дешевле автоматически, и формат не заменяет проверку. Известные +способы подорожать при переносе в шаги: + +- **ассет внутри шага, у которого есть инструменты.** Знание едет в каждую + генерацию react-цикла, а не только в первую. Лечится разбиением на «решить» + (знание, без тулов) и «выполнить» (тулы, без знания); +- **разбиение хода, который нечего было разбивать.** Два шага — это два + промпта, каждый со своим контекстом. Если второй шаг не снимает работу с + первого, он просто добавляет генерацию; +- **решение, которое действительно открыто.** Сформулировать ответ, оценить + качество, понять намерение — условием это не заменить, а попытка заменить + просто уводит вызов модели туда, где его хуже видно; +- **один-два шага и никаких ветвлений** — проза дешевле, и формат сам об этом + говорит (см. «Промптом тоже можно»). + +Каждый скилл после переписывания стоит измерить отдельно. Движок даёт, чем +мерить (`Outcome.Steps`), и линтер на дефекты, которые молчат; он не обещает, +что переписывание окупится. **Чем это является и чем нет.** Периоды разделены ДАТОЙ, а не рандомизацией, и в те же дни менялось другое — движок правился одновременно со скиллами. То есть From 118ba545cd8d4e3106031ec164a1de432153d6f0 Mon Sep 17 00:00:00 2001 From: Ivan Diatchenko <2518263+inhuman@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:33:48 +0300 Subject: [PATCH 6/8] docs: state the multiple-comparison correction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 23 Mann-Whitney tests at p<0.05 is 23 chances for a fluke, and a reader who knows that will say so. Computed from the same raw rows: with Holm-Bonferroni 16 of the 23 stay significant — 15 cheaper and the one that got more expensive. The skills that drop out are mostly those with the fewest turns, and the conclusion does not move: 20 cheaper without correction, 15 with it, against 1 more expensive either way. Worth its own sentence: the loss survives the correction too, so it is not an artifact of testing many skills at once. A counterexample that vanished under correction would have been worth less than no counterexample at all. --- README.md | 7 +++++++ README.ru.md | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/README.md b/README.md index 56fd2e5..88a745f 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,13 @@ Every skill is worth measuring on its own after it is rewritten. This engine gives you the trace to measure with (`Outcome.Steps`) and a linter for the defects that stay quiet; it does not promise that a rewrite pays. +**Multiple comparisons.** Those 23 tests are 23 chances for a fluke, so: with +the Holm–Bonferroni correction **16 of the 23 stay significant — 15 cheaper and +the one that got more expensive**. The skills that drop out are mostly the ones +with the fewest turns, and the conclusion does not move. Worth noting that the +loss survives the correction too: it is not an artifact of testing many skills +at once. + **What this is and is not.** The periods are separated by a DATE, not by randomisation, and other things changed in those same days — the engine was being edited alongside the skills. So this is an **observational before/after diff --git a/README.ru.md b/README.ru.md index b0f7fea..f3fd9ba 100644 --- a/README.ru.md +++ b/README.ru.md @@ -113,6 +113,12 @@ go get github.com/inhuman/skill-engine мерить (`Outcome.Steps`), и линтер на дефекты, которые молчат; он не обещает, что переписывание окупится. +**Множественные сравнения.** 23 теста — это 23 шанса на случайность, поэтому: с +поправкой Холма–Бонферрони **значимыми остаются 16 из 23 — 15 подешевевших и тот +один, что подорожал**. Выпадают в основном скиллы с наименьшим числом ходов, и +вывод от этого не двигается. Отдельно стоит сказать, что поражение поправку тоже +переживает: это не артефакт того, что скиллов проверяли много сразу. + **Чем это является и чем нет.** Периоды разделены ДАТОЙ, а не рандомизацией, и в те же дни менялось другое — движок правился одновременно со скиллами. То есть это **наблюдательное сравнение до/после, а не эксперимент**: оно показывает, что From 89ba6c1f091e543ebf0b8c9f5535dd7e7d54b176 Mon Sep 17 00:00:00 2001 From: Ivan Diatchenko <2518263+inhuman@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:39:21 +0300 Subject: [PATCH 7/8] fork a parallel branch's state instead of rebuilding it from a list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A branch of a `parallel` step did not inherit six of the flow's fields: the assets, their resolver, their cache and context, working memory, and the application's vocabulary. The sub-state was assembled by naming fields, and those six were not named. Nothing failed loudly, which is why it survived. An unknown asset expands to an empty string by contract — that contract exists so a marker never reaches a model — so `{{asset:x}}` inside a branch became "" and the tool call that needed it lost a required argument. The error named the argument, not the substitution. Working memory and the vocabulary went the same way: a step in a branch read a preview instead of the whole value, and `one_of` lost its tie-breaker. It stayed hidden because in a live catalogue of 29 skills not one `call` step with an asset had ever sat inside a `parallel` branch. The path never ran. The branch state is now FORKED from the flow's, with the few branch-local fields reset right after: its own copy of the variables (branches must not see each other's work, or the result would depend on who finished first), and an empty trace, skip list and answer until the join. A list of fields has to be extended by whoever adds a field to the engine, and that person is not thinking about `parallel`; forking inverts the default, and what does NOT reach a branch is visible in one place. The asset cache is shared with the branches rather than cloned into them, so an asset three branches need is fetched once — which makes it concurrent state, so it has a lock, held across the resolve rather than around the map alone. The lock is a pointer: a forked state copies the struct, and a mutex copied by value guards nothing. Removing the lock and running the tests under -race shows the race it prevents, and CI now runs the race detector. Engine fix, not a format change: EngineVersion 2.2.2 -> 2.2.3. --- .github/workflows/ci.yml | 7 ++ CHANGELOG.md | 28 +++++++ exec.go | 9 ++- expand.go | 7 ++ parallel_state_test.go | 163 +++++++++++++++++++++++++++++++++++++++ steps.go | 35 ++++++--- version.go | 2 +- 7 files changed, 239 insertions(+), 12 deletions(-) create mode 100644 parallel_state_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bba01ce..bdff463 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,13 @@ jobs: # outside the declared list. - name: Test run: go test ./... -count=1 + # Branches of a `parallel` step run in goroutines that share what the flow + # was given. That sharing is deliberate — an asset three branches need is + # fetched once — and it is exactly the kind of thing only the race + # detector notices: a bug here is a wrong answer on a busy day, not a + # crash on a quiet one. + - name: Race + run: go test ./... -count=1 -race - name: govulncheck run: go run golang.org/x/vuln/cmd/govulncheck@v1.3.0 ./... diff --git a/CHANGELOG.md b/CHANGELOG.md index 178fdbb..cf26bf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,34 @@ wrap skills in something of your own — front matter, a markdown body, several documents in one file — unwrap before calling and wrap the result back; anything else is refused rather than guessed at. +## 2.2.3 + +An engine fix; the format itself did not change. + +- **Fixed**: a branch of a `parallel` step did not inherit the assets, their + resolver, cache and context, working memory, or the application's vocabulary. + The sub-state was assembled by listing fields, and those six were not on the + list. + + Nothing failed loudly. An unknown asset expands to an empty string by + contract, so `{{asset:x}}` inside a branch quietly became "" and the tool call + that needed it lost a required argument — with the error pointing at the + argument rather than at the substitution. Working memory and the vocabulary + went the same way: a step in a branch read a preview instead of the whole + value, and `one_of` lost its tie-breaker. + + It stayed hidden because in a live catalogue of 29 skills no `call` step with + an asset had ever sat inside a `parallel` branch. + + The branch state is now FORKED from the flow's and the few branch-local + fields are reset explicitly, so a field added to the engine reaches branches + by default. The list had the opposite default, and whoever adds a field is + not thinking about `parallel`. + + The asset cache is shared with the branches rather than copied into them, so + an asset three branches need is still fetched once — under a lock held across + the resolve. CI now runs the race detector. + ## 2.2.2 An engine fix; the format itself did not change. diff --git a/exec.go b/exec.go index 555018b..861f758 100644 --- a/exec.go +++ b/exec.go @@ -6,6 +6,7 @@ package skillengine import ( "context" + "sync" "time" ) @@ -114,7 +115,7 @@ func newState(f *Flow, deps Deps, vars map[string]string) (*state, error) { onStep: deps.OnStep, onStepStart: deps.OnStepStart, assets: f.Assets, assetsRes: deps.Assets, memory: deps.Memory, vocab: deps.Vocabulary, - assetCache: map[string]string{}, seeded: map[string]bool{}} + assetCache: map[string]string{}, assetMu: &sync.Mutex{}, seeded: map[string]bool{}} for k, v := range f.Vars { st.vars[k] = v st.seeded[k] = true @@ -151,7 +152,13 @@ type state struct { vocab Vocabulary // assetCache — content already fetched in THIS turn: one asset consumed by // three steps is fetched once. + // + // Shared with the branches of a `parallel` step rather than copied into + // them, so an asset three branches need is still fetched once — which is + // why it needs a lock, and why the lock is a POINTER: a forked state copies + // the struct, and a mutex copied by value is a mutex that guards nothing. assetCache map[string]string + assetMu *sync.Mutex // assetCtx — the turn's context for resolving payloads. expand() is called // from places with no ctx at hand, and threading it through every // signature for the sake of one branch costs more than storing it here: diff --git a/expand.go b/expand.go index 7ddc807..1c932eb 100644 --- a/expand.go +++ b/expand.go @@ -146,6 +146,13 @@ func (s *state) payload(name string) string { // instruction. The failure is not silent, though — the resolver reports it to // the caller. func (s *state) asset(name string) string { + // The lock is held across the resolve, not just around the map: branches of + // a `parallel` step share this cache, and releasing it to fetch would let + // three branches fetch the same asset three times — the thing the cache + // exists to prevent. Resolving is not the hot path; a race here would be. + s.assetMu.Lock() + defer s.assetMu.Unlock() + if v, ok := s.assetCache[name]; ok { return v } diff --git a/parallel_state_test.go b/parallel_state_test.go new file mode 100644 index 0000000..c4096af --- /dev/null +++ b/parallel_state_test.go @@ -0,0 +1,163 @@ +package skillengine + +import ( + "context" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A branch of `parallel` runs in a state forked from the flow's, and everything +// the flow was given has to survive the fork. +// +// It did not. The sub-state was assembled by listing fields, and six of them +// were missing — assets, their resolver, their cache and context, working +// memory, and the application's vocabulary. Nothing failed loudly: an unknown +// asset expands to an empty string by contract, so `{{asset:x}}` inside a +// branch quietly became "", and the tool call that needed it lost a required +// argument. The error pointed at the argument, not at the substitution. +// +// It stayed hidden because no skill in a live catalogue of 29 had an asset +// inside a parallel branch — the path simply never ran. +func TestParallelBranchKeepsEverythingTheFlowHas(t *testing.T) { + f := parseFlow(t, ` +tools: ["srv"] +assets: + payload: + kind: code + source: inline + content: "print('the asset content')" +steps: + - name: fork + parallel: + branches: + - - name: uses_asset + call: + tool: "srv:run" + args: {code: "{{asset:payload}}"} + save_as: from_asset + - - name: uses_memory + instruction: "the whole of it: {{big}}" + tools: [] + save_as: from_memory +`) + + var got map[string]any + caller := ToolCallerFunc(func(_ context.Context, _, _ string, args map[string]any) (string, error) { + got = args + return "ran", nil + }) + r := &fakeRunner{answer: map[string]string{"uses_memory": "read it"}} + + _, _, err := ExecuteWith(context.Background(), f, Deps{ + Runner: r, + Caller: caller, + Assets: assetFunc(func(_ context.Context, _ string, a Asset) (string, error) { + return a.Content, nil + }), + Memory: fakeMemory{"res-1": "THE WHOLE VALUE"}, + }, map[string]string{"big": "preview…\n[mem:res-1]"}) + require.NoError(t, err) + + assert.Equal(t, "print('the asset content')", got["code"], + "the asset resolved to an empty string inside the branch, and the call lost a required argument") + assert.Contains(t, r.seen[0].Instruction, "THE WHOLE VALUE", + "working memory did not reach the branch, so the step saw a fragment") +} + +// The vocabulary is the application's words, and a branch that does not have +// them normalises an answer differently from the same step outside a branch. +func TestParallelBranchKeepsTheVocabulary(t *testing.T) { + f := parseFlow(t, ` +steps: + - name: fork + parallel: + branches: + - - name: classify + instruction: decide + one_of: [t1, foreign] + save_as: verdict + - - name: other + set: {var: x, value: "y"} +`) + r := &fakeRunner{answer: map[string]string{"classify": "Result: t1, not foreign"}} + + vars, _, err := ExecuteWith(context.Background(), f, Deps{ + Runner: r, + Vocabulary: Vocabulary{DecisionMarkers: []string{"result:"}}, + }, nil) + require.NoError(t, err) + assert.Equal(t, "t1", vars["verdict"], + "the decision markers did not reach the branch, so a tie stayed unresolved") +} + +// An asset needed by several branches is fetched ONCE: the cache is shared +// across the fork rather than cloned into it. Without a lock that sharing is a +// data race, which is why the cache has one. +func TestParallelBranchesShareTheAssetCache(t *testing.T) { + f := parseFlow(t, ` +tools: ["srv"] +assets: + payload: {kind: code, source: inline, content: "shared"} +steps: + - name: fork + parallel: + branches: + - - name: a + call: {tool: "srv:run", args: {code: "{{asset:payload}}"}, save_as: ra} + - - name: b + call: {tool: "srv:run", args: {code: "{{asset:payload}}"}, save_as: rb} + - - name: c + call: {tool: "srv:run", args: {code: "{{asset:payload}}"}, save_as: rc} +`) + var mu sync.Mutex + fetches := 0 + _, _, err := ExecuteWith(context.Background(), f, Deps{ + Caller: ToolCallerFunc(func(context.Context, string, string, map[string]any) (string, error) { + return "ok", nil + }), + Assets: assetFunc(func(_ context.Context, _ string, a Asset) (string, error) { + mu.Lock() + defer mu.Unlock() + fetches++ + return a.Content, nil + }), + }, nil) + require.NoError(t, err) + assert.Equal(t, 1, fetches, "the same asset was fetched once per branch") +} + +// What a branch must NOT inherit: the flow's collected traces and skips. A +// branch reporting the steps that ran before the fork would double-count them. +func TestParallelBranchStartsWithACleanTrace(t *testing.T) { + f := parseFlow(t, ` +steps: + - name: before + when: "missing == yes" + set: {var: a, value: "1"} + - name: fork + parallel: + branches: + - - name: x + set: {var: b, value: "2"} + - - name: y + when: "missing == yes" + set: {var: c, value: "3"} +`) + _, outcome, err := ExecuteWith(context.Background(), f, Deps{}, nil) + require.NoError(t, err) + + assert.Equal(t, []string{"before", "y"}, outcome.Skipped, + "a branch inherited the flow's skip list and reported it again") + assert.Equal(t, 1, strings.Count(strings.Join(outcome.Skipped, " "), "before")) +} + +// assetFunc — a resolver as a function, so a test does not need a type. +type assetFunc func(ctx context.Context, name string, a Asset) (string, error) + +func (f assetFunc) Resolve(ctx context.Context, name string, a Asset) (string, error) { + return f(ctx, name, a) +} diff --git a/steps.go b/steps.go index 02bf01c..c999010 100644 --- a/steps.go +++ b/steps.go @@ -559,16 +559,31 @@ func (s *state) parallelStep(ctx context.Context, step Step) (bool, error) { wg.Add(1) go func(i int, branch []Step) { defer wg.Done() - sub := &state{ - vars: maps.Clone(s.vars), - seeded: maps.Clone(s.seeded), - tools: s.tools, - runner: s.runner, - caller: s.caller, - delegate: s.delegate, - onStep: s.onStep, - onStepStart: s.onStepStart, - } + // FORKED from the flow's state, not assembled from a list of + // fields. The list was the bug: six of them were missing — the + // assets, their resolver, cache and context, working memory, and + // the application's vocabulary — so an `{{asset:x}}` inside a + // branch expanded to an empty string by contract, and the call + // that needed it lost a required argument. Nothing failed; the + // error pointed at the argument. + // + // A list has to be extended by whoever adds a field to `state`, + // and the person adding a field is not thinking about `parallel`. + // Forking inverts the default: everything reaches a branch unless + // it is explicitly reset below, and what is reset is visible in + // one place. + forked := *s + sub := &forked + // What a branch must NOT inherit: the variables are its own copy + // (the branches do not see each other's work — otherwise the + // result would depend on who finished first), and the trace, the + // skips and the answer belong to the branch alone until they are + // merged back after the join. + sub.vars = maps.Clone(s.vars) + sub.seeded = maps.Clone(s.seeded) + sub.skipped = nil + sub.traces = nil + sub.answeredBy = "" for k := range sub.vars { sub.seeded[k] = true // everything from before the fork is the branch's input } diff --git a/version.go b/version.go index ada1796..ab8c80b 100644 --- a/version.go +++ b/version.go @@ -28,7 +28,7 @@ import ( // minor — an optional field was added (a skill using it requires an engine // no older than that minor); // patch — engine fixes, the format did not change. -const EngineVersion = "2.2.2" +const EngineVersion = "2.2.3" // LegacyEngineVersion — what counts as the declared version when the field is // absent (skills written before it was introduced). From 90fa5d8a5a9e7ab19ba1956af86bfc10d371071e Mon Sep 17 00:00:00 2001 From: Ivan Diatchenko <2518263+inhuman@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:54:42 +0300 Subject: [PATCH 8/8] document two properties of the contract that only a reader could not guess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both came out of the parallel fix — one from the race detector it added, one from reading the merge at the join. Neither changes behaviour. **The step callbacks are concurrent.** OnStep and OnStepStart fire from the goroutine that ran the step, so inside a `parallel` they fire from several at once, and an embedder appending to a slice without a lock has a data race in production. The engine does not serialise them on purpose: a lock there would hold up a branch for as long as somebody else's telemetry write takes, and that is not the engine's call to make. Now said on Deps and in both READMEs, and the test that found it is written the way an embedder has to write it — with the lock in plain sight. Worth noting how it surfaced: the race detector went into CI with the previous commit, and the first thing it caught was my own test. Without it this would have reached embedders as an occasional wrong answer under load. **Outcome.Steps stops at a parallel, Outcome.Skipped does not.** A branch runs in a forked state, and only its variables and its skips are merged back. Nothing is lost — branch steps reach OnStep as they happen, which is where per-step telemetry comes from — but the two fields disagree, and the asymmetry is invisible until somebody checks one and assumes the other. Left as it is rather than "fixed": merging branch traces into Outcome.Steps would add entries for every embedder reading that field, which is a behaviour change and not a bug fix. A test now pins the property and says in its comment what to do if it ever starts failing. --- CHANGELOG.md | 13 +++++++++++ README.md | 16 +++++++++++++- README.ru.md | 16 +++++++++++++- exec.go | 24 ++++++++++++++++++++- parallel_state_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 115 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf26bf0..578f727 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,19 @@ An engine fix; the format itself did not change. an asset three branches need is still fetched once — under a lock held across the resolve. CI now runs the race detector. +- **Documented, not changed**: `Deps.OnStep` and `Deps.OnStepStart` fire from + the goroutine that ran the step, so inside a `parallel` they fire from several + at once and a callback that appends to a slice needs its own lock. The engine + does not serialise them on purpose — a lock there would hold up a branch for + the duration of somebody else's telemetry write. Found by the race detector + added above, in a test written the way an embedder would write it. + +- **Documented, not changed**: `Outcome.Steps` stops at a `parallel` — the steps + inside its branches are not in it, while `Outcome.Skipped` does include them. + Branch steps reach `OnStep` as they happen, so nothing is lost; but the two + fields disagree, and a reader who checks one and assumes the other loses an + afternoon. Now stated on the type and pinned by a test. + ## 2.2.2 An engine fix; the format itself did not change. diff --git a/README.md b/README.md index 88a745f..16f9b61 100644 --- a/README.md +++ b/README.md @@ -376,10 +376,24 @@ out, outcome, err := skillengine.ExecuteWith(ctx, flow, skillengine.Deps{ messages in chat instead of an answer. - `Outcome.Steps` — the trace of every step (name, kind, outcome, reason, duration, number of calls and failures); -- `Outcome.Skipped` — steps not executed because of `when`; +- `Outcome.Skipped` — steps not executed because of `when`, **including those + inside `parallel` branches**; +- the one asymmetry worth knowing: `Outcome.Steps` stops at a `parallel` — the + steps INSIDE its branches are not there, while `Skipped` above does include + them. A branch runs in a forked state, and only its variables and its skips + are merged back at the join. Nothing is lost by it: branch steps reach + `OnStep` as they happen, which is where per-step telemetry comes from. + `Steps` is the flow's shape, `OnStep` is the event stream, and only the first + one stops at the fork; - `Outcome.AnsweredBy` — `instruction` or `call`: what wrote the answer. Needed so that post-processing does not rewrite a script's deterministic output. +`OnStepStart` and `OnStep` **must be safe for concurrent use**: they fire from +the goroutine that ran the step, and the branches of a `parallel` run in several +at once. A callback appending to a slice needs its own lock. The engine does not +serialise them on purpose — a lock there would hold up a branch for the duration +of somebody else's telemetry write. + The engine logs nothing, persists nothing and goes nowhere: the input and the steps' output are the caller's data. Everything visible from outside is handed over as a structure (`Outcome`) and through callbacks (`OnStepStart` — before a diff --git a/README.ru.md b/README.ru.md index f3fd9ba..dd6e633 100644 --- a/README.ru.md +++ b/README.ru.md @@ -360,10 +360,24 @@ out, outcome, err := skillengine.ExecuteWith(ctx, flow, skillengine.Deps{ же реплик вместо ответа. - `Outcome.Steps` — след каждого шага (имя, вид, исход, причина, длительность, число вызовов и отказов); -- `Outcome.Skipped` — шаги, не исполнённые по `when`; +- `Outcome.Skipped` — шаги, не исполнённые по `when`, **включая те, что внутри + веток `parallel`**; +- единственная асимметрия, о которой стоит знать: `Outcome.Steps` на `parallel` + останавливается — шагов ВНУТРИ его веток там нет, хотя `Skipped` выше их + включает. Ветка исполняется в форкнутом состоянии, и на стыке сливаются только + её переменные и её пропуски. Ничего при этом не теряется: шаги веток + приезжают в `OnStep` по мере исполнения, и именно оттуда их берёт пошаговая + телеметрия. `Steps` — это форма потока, `OnStep` — поток событий, и на + развилке останавливается только первое; - `Outcome.AnsweredBy` — `instruction` или `call`: чем записан ответ. Нужно, чтобы не переписывать постобработкой детерминированный вывод скрипта. +`OnStepStart` и `OnStep` **обязаны быть безопасны для конкурентного вызова**: +они срабатывают из той горутины, что исполнила шаг, а ветки `parallel` идут +несколькими сразу. Колбэку, дописывающему в срез, нужен свой замок. Движок их +намеренно не сериализует — замок там держал бы ветку на время чужой записи +телеметрии. + Движок ничего не логирует, не персистит и никуда не ходит: вход и вывод шагов — данные вызывающего. Всё, что видно снаружи, он отдаёт структурой (`Outcome`) и колбэками (`OnStepStart` — перед шагом, для показа работы человеку; `OnStep` — diff --git a/exec.go b/exec.go index 861f758..5ef7fa2 100644 --- a/exec.go +++ b/exec.go @@ -38,12 +38,21 @@ type Deps struct { // OnStepStart is called BEFORE a step. Needed by anyone showing work to a // human: a step that runs for 14 seconds emits no event until it // finishes, and there is nothing to show all that time. + // + // CONCURRENT — see OnStep. OnStepStart func(name, kind string) // OnStep is called RIGHT AFTER each step, not at the end of the flow. // // Otherwise the caller learns about all the steps at once, when the turn // is already over: the progress post shows a finished list instead of work // as it happens, and the user stares at "brewing…" for nine seconds. + // + // BOTH CALLBACKS MUST BE SAFE FOR CONCURRENT USE. They fire from the + // goroutine that ran the step, and the branches of a `parallel` step run in + // several at once — so a callback that appends to a slice or writes to a + // map needs its own lock. The engine deliberately does not serialise them: + // a lock here would hold up a branch for the duration of somebody else's + // telemetry write, and the engine has no business deciding that. OnStep func(StepTrace) } @@ -91,11 +100,24 @@ type StepTrace struct { // Outcome — what happened to the flow beyond the variables. type Outcome struct { - // Steps — the trace of every executed (and skipped) step. + // Steps — the trace of every executed (and skipped) step of the FLOW. + // + // Steps inside the branches of a `parallel` are NOT here, and this is the + // one asymmetry in this struct worth knowing before you read it: Skipped + // below DOES include them. A branch runs in a forked state and only its + // variables and its skips are merged back at the join. + // + // Nothing about branch steps is lost, though — they reach Deps.OnStep as + // they happen, which is where an embedder recording per-step telemetry + // takes them from. This field is the flow's shape, OnStep is the event + // stream, and only the first one stops at the fork. Steps []StepTrace // Skipped — steps not executed because of a false `when`. Empty for a flow // without conditions; non-empty means the task matched only PARTIALLY, and // this is the only way to notice that. + // + // Includes steps skipped inside `parallel` branches — see Steps above for + // why the two differ. Skipped []string // AnsweredBy — the kind of step that wrote the turn's ANSWER: // "instruction" (the model wrote the text) or "call" (a tool printed it). diff --git a/parallel_state_test.go b/parallel_state_test.go index c4096af..cf47fd9 100644 --- a/parallel_state_test.go +++ b/parallel_state_test.go @@ -161,3 +161,52 @@ type assetFunc func(ctx context.Context, name string, a Asset) (string, error) func (f assetFunc) Resolve(ctx context.Context, name string, a Asset) (string, error) { return f(ctx, name, a) } + +// The one asymmetry in Outcome, pinned so it stays a documented property rather +// than an accident: Skipped includes what a `parallel` branch skipped, Steps +// does not include what a branch ran. +// +// Nothing is lost by it — branch steps reach Deps.OnStep as they happen, which +// is where per-step telemetry comes from. But the two fields disagree, and a +// reader who checks one and assumes the other is the next person to lose an +// afternoon. If this test starts failing because branch traces were merged in, +// that is a behaviour change for every embedder reading Outcome.Steps: say so +// in the CHANGELOG and update the doc comment on Outcome. +func TestOutcomeReportsBranchesOnlyThroughSkippedAndOnStep(t *testing.T) { + f := parseFlow(t, ` +steps: + - name: fork + parallel: + branches: + - - name: ran_in_branch + set: {var: a, value: "1"} + - - name: skipped_in_branch + when: "missing == yes" + set: {var: b, value: "2"} +`) + // The lock is not test hygiene — it is the contract. OnStep fires from the + // goroutine that ran the step, so inside a `parallel` it fires from several + // at once, and a callback appending to a slice without one is a data race + // in the embedder. Written here the way an embedder has to write it. + var mu sync.Mutex + var live []string + _, outcome, err := ExecuteWith(context.Background(), f, Deps{ + OnStep: func(tr StepTrace) { + mu.Lock() + defer mu.Unlock() + live = append(live, tr.Name) + }, + }, nil) + require.NoError(t, err) + + var inOutcome []string + for _, s := range outcome.Steps { + inOutcome = append(inOutcome, s.Name) + } + assert.Equal(t, []string{"fork"}, inOutcome, + "Outcome.Steps gained the branch steps — a change for everyone reading it") + assert.Contains(t, outcome.Skipped, "skipped_in_branch", + "Outcome.Skipped must include what a branch skipped") + assert.Subset(t, live, []string{"ran_in_branch", "skipped_in_branch", "fork"}, + "OnStep is where branch steps are visible, and they were not") +}