diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11530d0..bdff463 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,5 +21,26 @@ jobs: # outside the declared list. - name: Test run: go test ./... -count=1 + # Branches of a `parallel` step run in goroutines that share what the flow + # was given. That sharing is deliberate — an asset three branches need is + # fetched once — and it is exactly the kind of thing only the race + # detector notices: a bug here is a wrong answer on a busy day, not a + # crash on a quiet one. + - name: Race + run: go test ./... -count=1 -race - name: govulncheck run: go run golang.org/x/vuln/cmd/govulncheck@v1.3.0 ./... + + # The example applications are SEPARATE modules — that is what keeps their + # dependencies (eino, an SDK, whatever the next one uses) out of the + # engine's go.mod, and it also means `go test ./...` above never sees + # them. Without this step an example rots quietly, and an example that + # does not build teaches the format wrong. + - name: Examples + run: | + for mod in examples/*/go.mod; do + dir=$(dirname "$mod") + echo "── $dir" + go vet -C "$dir" ./... + go test -C "$dir" ./... -count=1 + done diff --git a/.gitignore b/.gitignore index c303947..a481fef 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,9 @@ PLAN.md *.out *.test coverage.* +# Собранные примеры: `go build` в их каталоге кладёт бинарник рядом с исходником +examples/*/simple-llm-app +examples/*/eino-llm-app # Окружение .env diff --git a/CHANGELOG.md b/CHANGELOG.md index 178fdbb..578f727 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,47 @@ wrap skills in something of your own — front matter, a markdown body, several documents in one file — unwrap before calling and wrap the result back; anything else is refused rather than guessed at. +## 2.2.3 + +An engine fix; the format itself did not change. + +- **Fixed**: a branch of a `parallel` step did not inherit the assets, their + resolver, cache and context, working memory, or the application's vocabulary. + The sub-state was assembled by listing fields, and those six were not on the + list. + + Nothing failed loudly. An unknown asset expands to an empty string by + contract, so `{{asset:x}}` inside a branch quietly became "" and the tool call + that needed it lost a required argument — with the error pointing at the + argument rather than at the substitution. Working memory and the vocabulary + went the same way: a step in a branch read a preview instead of the whole + value, and `one_of` lost its tie-breaker. + + It stayed hidden because in a live catalogue of 29 skills no `call` step with + an asset had ever sat inside a `parallel` branch. + + The branch state is now FORKED from the flow's and the few branch-local + fields are reset explicitly, so a field added to the engine reaches branches + by default. The list had the opposite default, and whoever adds a field is + not thinking about `parallel`. + + The asset cache is shared with the branches rather than copied into them, so + an asset three branches need is still fetched once — under a lock held across + the resolve. CI now runs the race detector. + +- **Documented, not changed**: `Deps.OnStep` and `Deps.OnStepStart` fire from + the goroutine that ran the step, so inside a `parallel` they fire from several + at once and a callback that appends to a slice needs its own lock. The engine + does not serialise them on purpose — a lock there would hold up a branch for + the duration of somebody else's telemetry write. Found by the race detector + added above, in a test written the way an embedder would write it. + +- **Documented, not changed**: `Outcome.Steps` stops at a `parallel` — the steps + inside its branches are not in it, while `Outcome.Skipped` does include them. + Branch steps reach `OnStep` as they happen, so nothing is lost; but the two + fields disagree, and a reader who checks one and assumes the other loses an + afternoon. Now stated on the type and pinned by a test. + ## 2.2.2 An engine fix; the format itself did not change. diff --git a/QUICKSTART.md b/QUICKSTART.md new file mode 100644 index 0000000..946b782 --- /dev/null +++ b/QUICKSTART.md @@ -0,0 +1,352 @@ +# Quickstart + +**English** · [Русский](QUICKSTART.ru.md) + +Fifteen minutes, at the end of which you will have run the **same skill both +ways** — as a prompt and as steps — on the same request, and seen the difference +in what it cost and in what it did. + +Written to be followed literally, by a person or by a model: every step has the +command, the file, and the output to expect. + +--- + +## What you are going to see + +One file describes a turn twice: as prose (`playbook`) and as steps +(`workflow`). Running it both ways on `подбери десерт и напиток`: + +``` +=== cost (playbook) === + generations: 1 + tokens: 214 prompt + 13 completion = 227 + +=== cost (workflow) === + generations: 1 + tokens: 89 prompt + 13 completion = 102 +``` + +And on a request that names no section at all: + +``` +-mode playbook → the model answers something. It was asked not to invent, and + whether it obeys is a probability. +-mode workflow → the skill stepped aside: exit: the request names no section + of the menu +``` + +Two different things happened there, and only one of them is about tokens. + +--- + +## Step 0 — what you need + +- **Go 1.26+** +- **an OpenAI-compatible endpoint.** Anything that speaks + `POST /v1/chat/completions`: vLLM, Ollama, LM Studio, LocalAI, or a hosted + API. The fastest local one: + + ``` + ollama serve + ollama pull qwen3:8b + ``` + + which gives you `http://localhost:11434/v1` and needs no key. + +You can do **Steps 1–5 without any endpoint at all** — the example's tests run +the whole thing against a stub. The numbers above come from exactly that stub, +which charges by prompt length the way a real endpoint does. + +--- + +## Step 1 — get the repository and run the tests + +``` +git clone https://github.com/inhuman/skill-engine +cd skill-engine/examples/simple-llm-app +go test ./... +``` + +Expected: + +``` +ok github.com/inhuman/skill-engine/examples/simple-llm-app +``` + +That run just executed a skill end to end, twice, and compared the two. If it +passes, everything below will work. + +--- + +## Step 2 — point the example at your model + +``` +export OPENAI_BASE_URL=http://localhost:11434/v1 +export OPENAI_API_KEY=ollama # any non-empty string for a local server +export OPENAI_MODEL=qwen3:8b +``` + +--- + +## Step 3 — run the skill as PROSE + +``` +go run . -skill ../skills/menu.yaml -mode playbook -input "подбери десерт и напиток" +``` + +The whole task goes to the model in one prompt: here are the sections, here are +the words people call them by, decide which were named, look them up, answer. + +Write down the two numbers it prints under `=== cost (playbook) ===`. + +--- + +## Step 4 — run the SAME FILE as steps + +``` +go run . -skill ../skills/menu.yaml -mode workflow -input "подбери десерт и напиток" +``` + +One word changed on the command line. Same file, same model, same request. + +``` +→ nothing_named (exit) +→ pick_dessert (call) + call recipes:search map[query:подбери десерт и напиток section:dessert] +→ pick_drink (call) + call recipes:search map[query:подбери десерт и напиток section:drink] +→ pick_main (call) +→ answer (instruction) + +=== steps === + nothing_named exit skipped calls=0 condition … is false + pick_dessert call ok calls=1 + pick_drink call ok calls=1 + pick_main call skipped calls=0 condition input contains горяч | второе | main course is false + answer instruction ok calls=0 + +=== cost (workflow) === + generations: 1 + tokens: 89 prompt + 13 completion = 102 +``` + +Compare with Step 3. + +--- + +## Step 5 — where the difference came from + +Open [`examples/skills/menu.yaml`](examples/skills/menu.yaml) next to the output +above. Four things did the work. + +**1. The dictionary never reached the model.** + +```yaml +- name: pick_dessert + when: "input contains десерт | сладк | dessert" +``` + +In the prose half those synonyms are in the prompt, and applying them is a +generation the model can get wrong. Here they are a condition: the words are +matched in code, before anything is sent anywhere. That is most of the token +difference — and all of the accuracy difference. + +**2. Two steps ran without a model at all.** + +```yaml + call: + tool: "recipes:search" + args: {section: dessert, query: "{{input}}"} +``` + +A `call` step is a tool invocation with the arguments already known. In prose +the same work is: one generation to decide to call, the call, another to read +the result back. Here it is the call. + +**3. The answering step was handed no tools.** + +```yaml +- name: answer + instruction: | + … + tools: [] +``` + +An empty list is not "no preference" — the step physically cannot call +anything. In prose, "do not go looking further" is a request; here there is +nothing to go with. + +**4. What did not run is visible.** + +``` +skipped: nothing_named, pick_main +``` + +"We did not look there" and "we looked and it was empty" are different answers. +The trace keeps them apart, so the answering step cannot report one as the +other. + +--- + +## Step 6 — make it decide something it cannot invent + +``` +go run . -skill ../skills/menu.yaml -mode workflow -input "что посоветуешь" +go run . -skill ../skills/menu.yaml -mode playbook -input "что посоветуешь" +``` + +Steps: + +``` +→ nothing_named (exit) +the skill stepped aside: skill-engine: exit: the request names no section of the menu +``` + +Prose: the model answers. It was told not to invent, and that instruction is +followed as far as it read before it started writing. + +This is the property the format is for, and it is not about cost: **in steps the +impossible is impossible, not discouraged.** The branch that would have searched +does not exist on that path. + +--- + +## Step 7 — write your own skill + +The smallest useful file. Save it as `my-skill.yaml`: + +```yaml +skill_engine_version: "2.2.0" +skill_version: "1.0.0" +name: my-skill +description: What this skill is FOR — and what it is NOT for. +trigger_examples: + - "a phrasing a user would actually type" + +workflow: + steps: + # A step with no tools cannot go anywhere. It thinks, and that is all. + - name: answer + instruction: | + The request: {{input}} + + Answer it in two sentences. + tools: [] +``` + +``` +go run . -skill ./my-skill.yaml -input "привет" +``` + +Then grow it in this order — each of these is one line in the file: + +| you want | you add | +|---|---| +| branch on the words of the request | `when: "input contains word \| synonym"` | +| call a tool with known arguments | a `call:` step | +| loop over what a step returned | `for_each: {in: var, as: item, collect: out}` | +| stop when it is not your case | `exit: {reason: "…"}` | +| a structured answer to branch on | `response_schema:` **plus** `model:` | +| hand the work to another skill | `delegate: {skill: other, task: "…"}` | + +The full field list with the reason each one exists is +[`skill.schema.yaml`](skill.schema.yaml); more shapes are in +[`examples/skills/`](examples/skills/). + +--- + +## Step 8 — check a skill before you run it + +Two levels, and they answer different questions. + +```go +// Will it run at all? +if err := skill.Validate(); err != nil { … } +``` + +```go +// Will it run WELL? 27 rules for the defects that stay quiet. +rep, err := lint.Lint(raw, facts, lint.Options{ + Unmarshal: yaml.Unmarshal, + EmptyWords: []string{"empty", "пусто"}, + HostVars: []string{"input"}, +}) +fmt.Println(rep.Text()) +``` + +The linter catches things a run would not: a loop collecting into a variable +nobody writes, a typo in a variable name that resolves to an empty string, a +required field the instruction beside it allows to be empty. See +[`lint/README.md`](lint/README.md). + +--- + +## Step 9 — embed it in your application + +The whole contract is one struct. From +[`examples/simple-llm-app/main.go`](examples/simple-llm-app/main.go): + +```go +skill, err := se.ParseSkill(raw, yaml.Unmarshal) // the whole file +if err := skill.Validate(); err != nil { … } + +mode, err := skill.ResolveMode() // steps or prose? +if mode == se.ModePlaybook { + // the engine takes no part: run skill.Playbook as your own prompt +} + +vars, outcome, err := se.ExecuteWith(ctx, skill.Workflow, se.Deps{ + Runner: yourModel, // executes an instruction step + Caller: yourTools, // executes a call step + Assets: yourResolver, // resolves an asset's content + Memory: yourMemory, // returns a large result by its handle + Vocabulary: se.Vocabulary{ + DecisionMarkers: []string{"Result:", "Ответ:"}, + }, + OnStepStart: func(name, kind string) { … }, +}, map[string]string{"input": userText}) +``` + +`vars[se.AnswerVar]` is the turn's answer. `outcome.Steps` is the trace you saw +above — the engine logs nothing and stores nothing, so that struct is the whole +of its observability. + +Two working applications to copy from: + +- [`examples/simple-llm-app`](examples/simple-llm-app/) — `net/http` and nothing + else; +- [`examples/eino-llm-app`](examples/eino-llm-app/) — the model reached through a + framework, with the whole framework-shaped part in one forty-line adapter. + +--- + +## Honest notes about the numbers above + +- They come from a **stub** that charges by prompt length, so they show the + SHAPE of the difference rather than your model's bill. Run Steps 3–4 against + a real endpoint for real numbers. +- With a real endpoint the prose half also spends **extra generations**: it has + to decide to call a tool, call it, and read the result back. The stub cannot + call tools, so it does that work in one generation and the gap in the table + above is understated, not overstated. +- The real numbers are in [README.md](README.md): a live installation's event + log, 5 280 turns over five weeks, 23 skills — 20 significantly cheaper, 1 + significantly more expensive, 2 unchanged. That comparison is + **observational**: the periods are split by a date rather than randomised, and + other things changed in those same days. +- In that same measurement one skill got **more** expensive: a median of + **6 → 10 generations per turn**. Steps are not automatically cheaper — the + README lists the known ways a rewrite costs more, and every skill is worth + measuring on its own afterwards. + +--- + +## Where to go next + +| | | +|---|---| +| [README.md](README.md) | what the engine is, what it is NOT, and what you can rely on | +| [`examples/`](examples/) | the format and two applications that embed it | +| [`skill.schema.yaml`](skill.schema.yaml) | every field, and the failure that paid for it | +| [CHANGELOG.md](CHANGELOG.md) | what changed in the format, and what a migration does | +| [`lint/README.md`](lint/README.md) | the 27 rules and what each one catches | diff --git a/QUICKSTART.ru.md b/QUICKSTART.ru.md new file mode 100644 index 0000000..b2c51c8 --- /dev/null +++ b/QUICKSTART.ru.md @@ -0,0 +1,350 @@ +# Быстрый старт + +**Русский** · [English](QUICKSTART.md) + +Пятнадцать минут, в конце которых ты прогонишь **один и тот же скилл двумя +способами** — промптом и шагами — на одном запросе и увидишь разницу в том, +сколько это стоило и что при этом произошло. + +Написано так, чтобы выполнять буквально — человеку или модели: у каждого шага +есть команда, файл и ожидаемый вывод. + +--- + +## Что ты увидишь + +Один файл описывает ход дважды: прозой (`playbook`) и шагами (`workflow`). +Прогон обоими способами на запросе `подбери десерт и напиток`: + +``` +=== cost (playbook) === + generations: 1 + tokens: 214 prompt + 13 completion = 227 + +=== cost (workflow) === + generations: 1 + tokens: 89 prompt + 13 completion = 102 +``` + +И на запросе, где не назван ни один раздел: + +``` +-mode playbook → модель что-нибудь отвечает. Её попросили не выдумывать, и + исполнение просьбы — это вероятность. +-mode workflow → the skill stepped aside: exit: the request names no section + of the menu +``` + +Здесь произошли две разные вещи, и только одна из них про токены. + +--- + +## Шаг 0 — что нужно + +- **Go 1.26+** +- **OpenAI-совместимый эндпоинт.** Любой, кто понимает + `POST /v1/chat/completions`: vLLM, Ollama, LM Studio, LocalAI или хостовый + API. Самый быстрый локальный: + + ``` + ollama serve + ollama pull qwen3:8b + ``` + + это даёт `http://localhost:11434/v1` и не требует ключа. + +**Шаги 1–5 можно пройти вообще без эндпоинта** — тесты примера гоняют всё +против заглушки. Числа выше сняты именно с неё, а считает она по длине промпта, +как это делает настоящий эндпоинт. + +--- + +## Шаг 1 — забрать репозиторий и прогнать тесты + +``` +git clone https://github.com/inhuman/skill-engine +cd skill-engine/examples/simple-llm-app +go test ./... +``` + +Ожидается: + +``` +ok github.com/inhuman/skill-engine/examples/simple-llm-app +``` + +Этот прогон только что исполнил скилл целиком, дважды, и сравнил результаты. +Если он зелёный, всё нижеследующее заработает. + +--- + +## Шаг 2 — направить пример на свою модель + +``` +export OPENAI_BASE_URL=http://localhost:11434/v1 +export OPENAI_API_KEY=ollama # для локального сервера любая непустая строка +export OPENAI_MODEL=qwen3:8b +``` + +--- + +## Шаг 3 — прогнать скилл ПРОЗОЙ + +``` +go run . -skill ../skills/menu.yaml -mode playbook -input "подбери десерт и напиток" +``` + +Вся задача уезжает в модель одним промптом: вот разделы, вот слова, которыми их +называют, реши, какие названы, найди их, ответь. + +Запиши два числа из блока `=== cost (playbook) ===`. + +--- + +## Шаг 4 — прогнать ТОТ ЖЕ ФАЙЛ шагами + +``` +go run . -skill ../skills/menu.yaml -mode workflow -input "подбери десерт и напиток" +``` + +На командной строке изменилось одно слово. Тот же файл, та же модель, тот же +запрос. + +``` +→ nothing_named (exit) +→ pick_dessert (call) + call recipes:search map[query:подбери десерт и напиток section:dessert] +→ pick_drink (call) + call recipes:search map[query:подбери десерт и напиток section:drink] +→ pick_main (call) +→ answer (instruction) + +=== steps === + nothing_named exit skipped calls=0 condition … is false + pick_dessert call ok calls=1 + pick_drink call ok calls=1 + pick_main call skipped calls=0 condition input contains горяч | второе | main course is false + answer instruction ok calls=0 + +=== cost (workflow) === + generations: 1 + tokens: 89 prompt + 13 completion = 102 +``` + +Сравни с шагом 3. + +--- + +## Шаг 5 — откуда взялась разница + +Открой [`examples/skills/menu.yaml`](examples/skills/menu.yaml) рядом с выводом +выше. Работу сделали четыре вещи. + +**1. Словарь вообще не доехал до модели.** + +```yaml +- name: pick_dessert + when: "input contains десерт | сладк | dessert" +``` + +В прозе эти синонимы лежат в промпте, и их применение — генерация, которую +модель может провалить. Здесь они условие: слова сопоставляются кодом, до того +как хоть что-то куда-то отправлено. Отсюда бо́льшая часть разницы в токенах — и +вся разница в точности. + +**2. Два шага отработали вообще без модели.** + +```yaml + call: + tool: "recipes:search" + args: {section: dessert, query: "{{input}}"} +``` + +Шаг `call` — это вызов инструмента, у которого аргументы уже известны. В прозе +та же работа — это генерация «решить позвать», сам вызов и ещё генерация +«пересказать результат». Здесь это вызов. + +**3. Отвечающему шагу не выдали инструментов.** + +```yaml +- name: answer + instruction: | + … + tools: [] +``` + +Пустой список — не «без предпочтений»: шаг физически не может никуда пойти. В +прозе «дальше не ходи» — просьба, здесь идти нечем. + +**4. Видно то, что НЕ исполнялось.** + +``` +skipped: nothing_named, pick_main +``` + +«Мы туда не ходили» и «сходили, там пусто» — разные ответы. След держит их +порознь, поэтому отвечающий шаг не может выдать одно за другое. + +--- + +## Шаг 6 — заставить его решить то, чего он не может выдумать + +``` +go run . -skill ../skills/menu.yaml -mode workflow -input "что посоветуешь" +go run . -skill ../skills/menu.yaml -mode playbook -input "что посоветуешь" +``` + +Шагами: + +``` +→ nothing_named (exit) +the skill stepped aside: skill-engine: exit: the request names no section of the menu +``` + +Прозой: модель отвечает. Ей сказали не выдумывать, и эта инструкция исполняется +ровно настолько, насколько она дочитала до неё, прежде чем начать писать. + +Это и есть свойство, ради которого формат существует, и оно не про стоимость: +**в шагах невозможное невозможно, а не нежелательно.** Ветки, которая пошла бы +искать, на этом пути просто нет. + +--- + +## Шаг 7 — написать свой скилл + +Минимальный полезный файл. Сохрани как `my-skill.yaml`: + +```yaml +skill_engine_version: "2.2.0" +skill_version: "1.0.0" +name: my-skill +description: Для ЧЕГО этот скилл — и для чего он НЕ нужен. +trigger_examples: + - "формулировка, которую пользователь реально напишет" + +workflow: + steps: + # Шаг без инструментов никуда пойти не может. Он думает, и всё. + - name: answer + instruction: | + Запрос: {{input}} + + Ответь в двух предложениях. + tools: [] +``` + +``` +go run . -skill ./my-skill.yaml -input "привет" +``` + +Дальше наращивай в таком порядке — каждое из этого одна строка в файле: + +| что нужно | что добавить | +|---|---| +| ветвиться по словам запроса | `when: "input contains слово \| синоним"` | +| позвать инструмент с известными аргументами | шаг `call:` | +| пройти циклом по тому, что вернул шаг | `for_each: {in: var, as: item, collect: out}` | +| выйти, если случай не твой | `exit: {reason: "…"}` | +| структурный ответ, по которому ветвиться | `response_schema:` **и рядом** `model:` | +| отдать работу другому скиллу | `delegate: {skill: other, task: "…"}` | + +Полный список полей с причиной существования каждого — +[`skill.schema.ru.yaml`](skill.schema.ru.yaml); другие формы — +в [`examples/skills/`](examples/skills/). + +--- + +## Шаг 8 — проверить скилл до запуска + +Два уровня, и они отвечают на разные вопросы. + +```go +// Запустится ли вообще? +if err := skill.Validate(); err != nil { … } +``` + +```go +// Хорошо ли запустится? 27 правил про дефекты, которые молчат. +rep, err := lint.Lint(raw, facts, lint.Options{ + Unmarshal: yaml.Unmarshal, + EmptyWords: []string{"пусто", "empty"}, + HostVars: []string{"input"}, +}) +fmt.Println(rep.Text()) +``` + +Линтер ловит то, чего не покажет прогон: цикл, собирающий в переменную, куда +никто не пишет; опечатку в имени переменной, которая разрешится в пустую строку; +обязательное поле, которому инструкция рядом разрешает быть пустым. См. +[`lint/README.md`](lint/README.md). + +--- + +## Шаг 9 — встроить в своё приложение + +Весь контракт — одна структура. Из +[`examples/simple-llm-app/main.go`](examples/simple-llm-app/main.go): + +```go +skill, err := se.ParseSkill(raw, yaml.Unmarshal) // файл целиком +if err := skill.Validate(); err != nil { … } + +mode, err := skill.ResolveMode() // шаги или проза? +if mode == se.ModePlaybook { + // движок не участвует: гони skill.Playbook как свой промпт +} + +vars, outcome, err := se.ExecuteWith(ctx, skill.Workflow, se.Deps{ + Runner: yourModel, // исполняет шаг-инструкцию + Caller: yourTools, // исполняет шаг-вызов + Assets: yourResolver, // достаёт содержимое ассета + Memory: yourMemory, // отдаёт крупный результат по рукояти + Vocabulary: se.Vocabulary{ + DecisionMarkers: []string{"Result:", "Ответ:"}, + }, + OnStepStart: func(name, kind string) { … }, +}, map[string]string{"input": userText}) +``` + +`vars[se.AnswerVar]` — ответ хода. `outcome.Steps` — тот самый след из вывода +выше: движок ничего не логирует и ничего не хранит, поэтому эта структура и есть +вся его наблюдаемость. + +Два рабочих приложения, с которых можно копировать: + +- [`examples/simple-llm-app`](examples/simple-llm-app/) — `net/http` и ничего + больше; +- [`examples/eino-llm-app`](examples/eino-llm-app/) — модель через фреймворк, + причём весь фреймворк-специфичный код в одном адаптере на сорок строк. + +--- + +## Честные оговорки про числа выше + +- Они сняты с **заглушки**, считающей по длине промпта, — то есть показывают + ФОРМУ разницы, а не твой счёт. Прогоняй шаги 3–4 против настоящего эндпоинта, + если нужны настоящие числа. +- С настоящим эндпоинтом прозаическая половина тратит ещё и **лишние + генерации**: решить позвать инструмент, позвать, прочитать результат. + Заглушка звать инструменты не умеет и делает всё одной генерацией, поэтому + разрыв в таблице выше занижен, а не завышен. +- Настоящие числа — в [README.ru.md](README.ru.md): журнал событий работающей + установки, 5 280 ходов за пять недель, 23 скилла — 20 значимо дешевле, 1 + значимо дороже, 2 без разницы. То сравнение **наблюдательное**: периоды + разделены датой, а не рандомизацией, и в те же дни менялось другое. +- В том же замере одному скиллу стало **дороже**: медиана **6 → 10 генераций на + ход**. Шаги не дешевле автоматически — README перечисляет известные способы + подорожать при переписывании, и каждый скилл после него стоит измерить + отдельно. + +--- + +## Куда дальше + +| | | +|---|---| +| [README.ru.md](README.ru.md) | что это за движок, чем он НЕ является и на что можно опираться | +| [`examples/`](examples/) | формат и два приложения, которые его встраивают | +| [`skill.schema.ru.yaml`](skill.schema.ru.yaml) | каждое поле и отказ, которым оно оплачено | +| [CHANGELOG.md](CHANGELOG.md) | что менялось в формате и что делает миграция | +| [`lint/README.md`](lint/README.md) | 27 правил и что ловит каждое | diff --git a/README.md b/README.md index f7a28c3..16f9b61 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,69 @@ **English** · [Русский](README.ru.md) +[![Version](https://img.shields.io/github/v/tag/inhuman/skill-engine?sort=semver&style=flat-square&label=version)](https://github.com/inhuman/skill-engine/tags) +[![Build](https://img.shields.io/github/actions/workflow/status/inhuman/skill-engine/ci.yml?style=flat-square&logo=github)](https://github.com/inhuman/skill-engine/actions/workflows/ci.yml) +[![Go Reference](https://pkg.go.dev/badge/github.com/inhuman/skill-engine.svg)](https://pkg.go.dev/github.com/inhuman/skill-engine) +[![Go Report Card](https://goreportcard.com/badge/github.com/inhuman/skill-engine?style=flat-square)](https://goreportcard.com/report/github.com/inhuman/skill-engine) +[![Go Version](https://img.shields.io/github/go-mod/go-version/inhuman/skill-engine?style=flat-square&logo=go)](https://go.dev/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow?style=flat-square)](LICENSE) + An engine for declarative programs for an LLM agent: a skill is described in **steps**, and control over the turn belongs to the code, not to the model. Steps are not the only form: a skill with only a `playbook` (a free-form instruction) is a full skill too (see "A prompt works as well"). +``` +go get github.com/inhuman/skill-engine +``` + +**Start here → [QUICKSTART.md](QUICKSTART.md).** Fifteen minutes, at the end of +which you have run the same skill BOTH ways — as a prompt and as steps — on the +same request, and seen the difference in what it cost and in what it did. Steps +1–5 need no model at all. + +**Then → [`examples/`](examples/):** the format in +[`examples/skills/`](examples/skills/), and two working applications that embed +the engine — [`simple-llm-app`](examples/simple-llm-app/) on an +OpenAI-compatible endpoint with nothing but `net/http`, and +[`eino-llm-app`](examples/eino-llm-app/) with the model reached through a +framework. Both run offline in their tests. + **No dependencies** — production code runs on the standard library alone: the engine is embedded into someone else's application, and every dependency here would become a dependency of the embedder. YAML parsing is passed in as a parameter (the `Unmarshal` type), version comparison is implemented in place. The boundary is held by a guard test, `imports_test.go`, test imports included. +## Status + +**Where it runs.** The engine was taken out of a working assistant, where it +executes that assistant's whole skill catalogue — around thirty skills, in +production. It is not a design sketch: every field in the format is there +because something broke without it, and the comment beside the field says what. + +**Library version — `v0.5.x`.** Below `1.0` the **Go API may still move**: a +type can gain a field, a function a parameter. What is already stable is the +**format** — skill files are versioned separately and on their own rules. + +**Format version — `2.2.2`** (`EngineVersion` in `version.go`). A skill declares +the minimum it needs in `skill_engine_version`, and a foreign MAJOR is refused +in both directions: a description of a previous major would parse without a +single complaint, silently losing fields the structs no longer have. What +changed in each version, and what a migration does, is in +[CHANGELOG.md](CHANGELOG.md). + +**If you have skills written for format 1.x**, they do not load: that is the +refusal above, working as intended. `Migrate(raw)` rewrites them — it edits the +file as text, so comments, key order and block scalars survive — but until you +run it those skills do not execute at all, including on a schedule. Better said +here than discovered on a Monday morning. + +**Tests.** 89.3% statement coverage in the engine, 95.1% in the linter, plus +guard tests for the properties that prose cannot hold: no dependencies, no +direct reads of the variable map, the two schema translations staying +structurally identical, and the example applications staying separate modules. + ## Why A restriction written in words is a request: "do NOT call retract without @@ -21,10 +73,96 @@ the model read before it started acting. In steps the same thing is expressed structurally: in the unconfirmed branch the `retract` call **is not there**, a `call` step cannot be repeated, a branch that does not apply does not run. -Measurements on live skills (tool calls / seconds, before → after): three -skills of one catalogue went 18/95 → **2/6**, 7/33 → **2/5** and 9/29 → -**5/11**. What they did is beside the point — what changed is who held the -control flow. +### What it changed, measured on live traffic + +Not a benchmark of one question run twice — the event log of a working +installation: every turn where a skill matched, over five weeks, questions asked +by people rather than by the author of the skill. The metric is **LLM +generations per turn**, orchestrator and subagents together. + +The catalogue moved from prose to steps on one day, and the periods are split by +that date. + +| | | +|---|---:| +| skills with at least 5 turns on each side | **23** | +| turns compared | **5 280** (3 469 prose, 1 811 steps) | +| significantly cheaper (Mann–Whitney, p<0.05) | **20** | +| significantly more expensive | **1** | +| no significant difference | **2** | + +The effect is a median of −18 to −0.5 generations per turn. The largest: a +triage skill went from a median of **38 generations per turn to 20**. Typical: +**7 → 3**. + +**One skill got worse.** A health-checking skill went the other way — median +**6 → 10** generations, p<0.001. The measurement does not say why: it counts +generations, not reasons. + +So steps are not automatically cheaper, and the format is not a substitute for +checking. Known ways a skill gets MORE expensive when it moves into steps: + +- **an asset inside a step that has tools.** The knowledge rides along into + every generation of the react loop, not just the first one. Splitting into + "decide" (knowledge, no tools) and "do" (tools, no knowledge) is the fix; +- **splitting a turn that had nothing to split.** Two steps means two prompts, + each carrying its own context. If the second step does not remove work from + the first, it only adds a generation; +- **a decision that is genuinely open.** Wording an answer, judging quality, + reading somebody's intent — a condition cannot replace that, and pretending + otherwise just moves the model call somewhere less visible; +- **one or two steps and no branching at all** — prose is cheaper, and the + format says so itself (see "A prompt works as well"). + +Every skill is worth measuring on its own after it is rewritten. This engine +gives you the trace to measure with (`Outcome.Steps`) and a linter for the +defects that stay quiet; it does not promise that a rewrite pays. + +**Multiple comparisons.** Those 23 tests are 23 chances for a fluke, so: with +the Holm–Bonferroni correction **16 of the 23 stay significant — 15 cheaper and +the one that got more expensive**. The skills that drop out are mostly the ones +with the fewest turns, and the conclusion does not move. Worth noting that the +loss survives the correction too: it is not an artifact of testing many skills +at once. + +**What this is and is not.** The periods are separated by a DATE, not by +randomisation, and other things changed in those same days — the engine was +being edited alongside the skills. So this is an **observational before/after +comparison, not an experiment**: it shows that the catalogue got cheaper across +that boundary, not that nothing else contributed. The underlying event log +belongs to a private installation, so what is published here is the aggregate +rather than the raw data. + +## What this is not + +The neighbours are worth naming, because the differences are architectural +rather than a matter of taste. + +**Not a process orchestrator** (Temporal, n8n). The engine owns no state between +turns, has no storage of its own and does not survive a restart: a turn runs +inside somebody else's application and ends with it. Comparing durability is +comparing different jobs — if you need a workflow that resumes after a crash +three days later, this is the wrong tool and nothing here will make it right. + +**Not an agent framework** (LangGraph and its relatives). There the graph is +written by the application's developer, in the application's language, and it +ships with the application. Here the steps are written by the SKILL'S author, in +YAML, and the skill is portable between hosts — which is why the format has a +version of its own and why `Migrate` exists at all. A skill is data your users +can write; a graph is code you deploy. + +**Zero dependencies is a consequence, not a pose.** An engine embedded into +someone else's application makes every one of its dependencies theirs, with +their versions and their conflicts. So YAML arrives as a parameter (the +`Unmarshal` type), version comparison is thirty lines instead of a library, and +`imports_test.go` fails the build the moment production code imports anything at +all. Rare enough to be worth naming where you are comparing. + +**When you do NOT need this.** One or two steps and no branching — prose is +cheaper, and the format says so itself (see "A prompt works as well"). The +engine starts paying where a turn has branches, a loop, a tool set that must +narrow, or a guard that has to be impossible to violate rather than merely +asked for. ## A prompt works as well @@ -238,10 +376,24 @@ out, outcome, err := skillengine.ExecuteWith(ctx, flow, skillengine.Deps{ messages in chat instead of an answer. - `Outcome.Steps` — the trace of every step (name, kind, outcome, reason, duration, number of calls and failures); -- `Outcome.Skipped` — steps not executed because of `when`; +- `Outcome.Skipped` — steps not executed because of `when`, **including those + inside `parallel` branches**; +- the one asymmetry worth knowing: `Outcome.Steps` stops at a `parallel` — the + steps INSIDE its branches are not there, while `Skipped` above does include + them. A branch runs in a forked state, and only its variables and its skips + are merged back at the join. Nothing is lost by it: branch steps reach + `OnStep` as they happen, which is where per-step telemetry comes from. + `Steps` is the flow's shape, `OnStep` is the event stream, and only the first + one stops at the fork; - `Outcome.AnsweredBy` — `instruction` or `call`: what wrote the answer. Needed so that post-processing does not rewrite a script's deterministic output. +`OnStepStart` and `OnStep` **must be safe for concurrent use**: they fire from +the goroutine that ran the step, and the branches of a `parallel` run in several +at once. A callback appending to a slice needs its own lock. The engine does not +serialise them on purpose — a lock there would hold up a branch for the duration +of somebody else's telemetry write. + The engine logs nothing, persists nothing and goes nowhere: the input and the steps' output are the caller's data. Everything visible from outside is handed over as a structure (`Outcome`) and through callbacks (`OnStepStart` — before a @@ -387,6 +539,6 @@ limitation worth knowing before relying on it are in ## Tests `example_flow_test.go` — runnable examples of the format, a good first entry -point. `examples_test.go` parses every file from `examples/` with the engine: an +point. `examples_test.go` parses every file from `examples/skills/` with the engine: an example that stopped parsing is worse than a missing one — it teaches the wrong thing. diff --git a/README.ru.md b/README.ru.md index ed75c0e..dd6e633 100644 --- a/README.ru.md +++ b/README.ru.md @@ -2,16 +2,66 @@ **Русский** · [English](README.md) +[![Version](https://img.shields.io/github/v/tag/inhuman/skill-engine?sort=semver&style=flat-square&label=version)](https://github.com/inhuman/skill-engine/tags) +[![Build](https://img.shields.io/github/actions/workflow/status/inhuman/skill-engine/ci.yml?style=flat-square&logo=github)](https://github.com/inhuman/skill-engine/actions/workflows/ci.yml) +[![Go Reference](https://pkg.go.dev/badge/github.com/inhuman/skill-engine.svg)](https://pkg.go.dev/github.com/inhuman/skill-engine) +[![Go Report Card](https://goreportcard.com/badge/github.com/inhuman/skill-engine?style=flat-square)](https://goreportcard.com/report/github.com/inhuman/skill-engine) +[![Go Version](https://img.shields.io/github/go-mod/go-version/inhuman/skill-engine?style=flat-square&logo=go)](https://go.dev/) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow?style=flat-square)](LICENSE) + Движок декларативных программ для LLM-агента: скилл описывается **шагами**, и управление ходом принадлежит коду, а не модели. Шаги — не единственная форма: скилл, у которого задан только `playbook` (свободная инструкция), тоже полноценный скилл (см. «Промптом тоже можно»). +``` +go get github.com/inhuman/skill-engine +``` + +**Начинать отсюда → [QUICKSTART.ru.md](QUICKSTART.ru.md).** Пятнадцать минут, в +конце которых ты прогнал один и тот же скилл ДВУМЯ способами — промптом и +шагами — на одном запросе и увидел разницу в том, сколько это стоило и что при +этом произошло. Шаги 1–5 не требуют модели вовсе. + +**Дальше → [`examples/`](examples/):** сам формат в +[`examples/skills/`](examples/skills/) и два рабочих приложения, встраивающих +движок, — [`simple-llm-app`](examples/simple-llm-app/) на OpenAI-совместимом +эндпоинте и голом `net/http` и [`eino-llm-app`](examples/eino-llm-app/), где +модель достаётся через фреймворк. Оба гоняются в тестах без сети. + **Зависимостей нет** — боевой код на одной stdlib: движок встраивают в чужое приложение, и каждая зависимость здесь стала бы зависимостью встраивающего. Разбор YAML передаётся параметром (тип `Unmarshal`), сравнение версий сделано на месте. Границу держит тест-страж `imports_test.go`, включая тестовые импорты. +## Статус + +**Где работает.** Движок вынесен из работающего ассистента, где исполняет весь +его каталог системных скиллов — около тридцати штук, в проде. Это не эскиз: у +каждого поля формата есть отказ, без которого поля бы не было, и комментарий +рядом с полем говорит какой. + +**Версия библиотеки — `v0.5.x`.** До `1.0` **Go-API может двигаться**: у типа +появится поле, у функции параметр. Что уже стабильно — это **формат**: файлы +скиллов версионируются отдельно и по своим правилам. + +**Версия формата — `2.2.2`** (`EngineVersion` в `version.go`). Скилл объявляет +нужный ему минимум в `skill_engine_version`, и чужой МАЖОР отвергается в обе +стороны: описание прошлого мажора разобралось бы без единой жалобы, молча +потеряв поля, которых в структурах больше нет. Что изменилось в каждой версии и +что делает миграция — в [CHANGELOG.md](CHANGELOG.md). + +**Если у тебя есть скиллы под формат 1.x** — они не загрузятся: это тот самый +отказ, работающий как задумано. `Migrate(raw)` их переписывает, правя файл +текстом, так что комментарии, порядок ключей и блочные скаляры переживают +миграцию, — но до её запуска такие скиллы не исполняются вовсе, в том числе по +расписанию. Лучше сказать это здесь, чем обнаружить в понедельник утром. + +**Тесты.** 89.3% покрытия по операторам в движке, 95.1% в линтере, плюс +стражи на свойства, которые прозой не удержать: отсутствие зависимостей, +отсутствие прямых чтений карты переменных, структурная одинаковость двух +переводов схемы и отдельность модулей у примеров-приложений. + ## Зачем Ограничение, записанное словами, — просьба: «НЕ вызывай retract без @@ -20,9 +70,89 @@ Шагами то же самое выражается структурой: в неподтверждённой ветке вызова `retract` **нет**, шаг `call` нельзя повторить, лишняя ветка не исполняется. -Замеры на живых скиллах (вызовов инструментов / секунд, до → после): три скилла -одного каталога прошли 18/95 → **2/6**, 7/33 → **2/5** и 9/29 → **5/11**. Что -именно они делали — неважно; изменилось то, кто держит поток управления. +### Что изменилось, замер на живом трафике + +Это не прогон одного вопроса дважды, а журнал событий работающей установки: все +ходы, где сматчился скилл, за пять недель, и вопросы задавали люди, а не автор +скилла. Метрика — **число генераций LLM на один ход**, оркестратор и субагенты +вместе. + +Каталог перевели с прозы на шаги в один день, и периоды разделены этой датой. + +| | | +|---|---:| +| скиллов, где не меньше 5 ходов с каждой стороны | **23** | +| сравниваемых ходов | **5 280** (3 469 прозой, 1 811 шагами) | +| значимо дешевле (Манн–Уитни, p<0.05) | **20** | +| значимо дороже | **1** | +| без значимой разницы | **2** | + +Эффект — от −18 до −0.5 генерации на ход по медиане. Наибольший: скилл разбора +инцидентов прошёл с медианы **38 генераций на ход до 20**. Типичный: **7 → 3**. + +**Одному скиллу стало хуже.** Скилл проверки здоровья сервиса пошёл в обратную +сторону: медиана **6 → 10** генераций, p<0.001. Почему — замер не говорит: он +считает генерации, а не причины. + +То есть шаги не дешевле автоматически, и формат не заменяет проверку. Известные +способы подорожать при переносе в шаги: + +- **ассет внутри шага, у которого есть инструменты.** Знание едет в каждую + генерацию react-цикла, а не только в первую. Лечится разбиением на «решить» + (знание, без тулов) и «выполнить» (тулы, без знания); +- **разбиение хода, который нечего было разбивать.** Два шага — это два + промпта, каждый со своим контекстом. Если второй шаг не снимает работу с + первого, он просто добавляет генерацию; +- **решение, которое действительно открыто.** Сформулировать ответ, оценить + качество, понять намерение — условием это не заменить, а попытка заменить + просто уводит вызов модели туда, где его хуже видно; +- **один-два шага и никаких ветвлений** — проза дешевле, и формат сам об этом + говорит (см. «Промптом тоже можно»). + +Каждый скилл после переписывания стоит измерить отдельно. Движок даёт, чем +мерить (`Outcome.Steps`), и линтер на дефекты, которые молчат; он не обещает, +что переписывание окупится. + +**Множественные сравнения.** 23 теста — это 23 шанса на случайность, поэтому: с +поправкой Холма–Бонферрони **значимыми остаются 16 из 23 — 15 подешевевших и тот +один, что подорожал**. Выпадают в основном скиллы с наименьшим числом ходов, и +вывод от этого не двигается. Отдельно стоит сказать, что поражение поправку тоже +переживает: это не артефакт того, что скиллов проверяли много сразу. + +**Чем это является и чем нет.** Периоды разделены ДАТОЙ, а не рандомизацией, и +в те же дни менялось другое — движок правился одновременно со скиллами. То есть +это **наблюдательное сравнение до/после, а не эксперимент**: оно показывает, что +каталог подешевел через эту границу, но не то, что больше ничто не повлияло. +Журнал событий принадлежит приватной установке, поэтому опубликована сводка, а +не сырьё. + +## Чем это не является + +Соседей стоит назвать, потому что различия здесь архитектурные, а не вкусовые. + +**Это не оркестратор процессов** (Temporal, n8n). Движок не владеет состоянием +между ходами, не имеет своего хранилища и не переживает рестарт: ход исполняется +внутри чужого приложения и заканчивается вместе с ним. Сравнивать надёжность — +сравнивать разные задачи: если нужен процесс, продолжающийся после падения через +трое суток, это не тот инструмент, и ничего здесь этого не исправит. + +**Это не фреймворк агента** (LangGraph и родня). Там граф пишет разработчик +приложения, на языке приложения, и граф едет вместе с приложением. Здесь шаги +пишет АВТОР СКИЛЛА, на YAML, и скилл переносим между хостами — отсюда и +собственная версия формата, и существование `Migrate`. Скилл — это данные, +которые могут писать твои пользователи; граф — это код, который ты выкатываешь. + +**Ноль зависимостей — следствие, а не поза.** Движок, встроенный в чужое +приложение, делает каждую свою зависимость его зависимостью, с её версиями и её +конфликтами. Поэтому YAML приезжает параметром (тип `Unmarshal`), сравнение +версий — тридцать строк вместо библиотеки, а `imports_test.go` роняет сборку в +тот момент, когда боевой код импортирует хоть что-нибудь. Свойство редкое, и +назвать его стоит именно там, где читатель сравнивает. + +**Когда это НЕ нужно.** Один-два шага и никаких ветвлений — проза дешевле, и +формат сам об этом говорит (см. «Промптом тоже можно»). Движок начинает +окупаться там, где у хода есть ветки, цикл, набор инструментов, который надо +сузить, или запрет, который должен быть невыполнимым, а не попрошенным. ## Промптом тоже можно @@ -230,10 +360,24 @@ out, outcome, err := skillengine.ExecuteWith(ctx, flow, skillengine.Deps{ же реплик вместо ответа. - `Outcome.Steps` — след каждого шага (имя, вид, исход, причина, длительность, число вызовов и отказов); -- `Outcome.Skipped` — шаги, не исполнённые по `when`; +- `Outcome.Skipped` — шаги, не исполнённые по `when`, **включая те, что внутри + веток `parallel`**; +- единственная асимметрия, о которой стоит знать: `Outcome.Steps` на `parallel` + останавливается — шагов ВНУТРИ его веток там нет, хотя `Skipped` выше их + включает. Ветка исполняется в форкнутом состоянии, и на стыке сливаются только + её переменные и её пропуски. Ничего при этом не теряется: шаги веток + приезжают в `OnStep` по мере исполнения, и именно оттуда их берёт пошаговая + телеметрия. `Steps` — это форма потока, `OnStep` — поток событий, и на + развилке останавливается только первое; - `Outcome.AnsweredBy` — `instruction` или `call`: чем записан ответ. Нужно, чтобы не переписывать постобработкой детерминированный вывод скрипта. +`OnStepStart` и `OnStep` **обязаны быть безопасны для конкурентного вызова**: +они срабатывают из той горутины, что исполнила шаг, а ветки `parallel` идут +несколькими сразу. Колбэку, дописывающему в срез, нужен свой замок. Движок их +намеренно не сериализует — замок там держал бы ветку на время чужой записи +телеметрии. + Движок ничего не логирует, не персистит и никуда не ходит: вход и вывод шагов — данные вызывающего. Всё, что видно снаружи, он отдаёт структурой (`Outcome`) и колбэками (`OnStepStart` — перед шагом, для показа работы человеку; `OnStep` — @@ -373,5 +517,5 @@ rep, err := lint.Lint(raw, facts, lint.Options{Unmarshal: yaml.Unmarshal}) ## Тесты `example_flow_test.go` — исполняемые примеры формата, годятся как первая точка -входа. `examples_test.go` разбирает движком все файлы из `examples/`: пример, +входа. `examples_test.go` разбирает движком все файлы из `examples/skills/`: пример, переставший разбираться, хуже отсутствующего — он учит неверному. diff --git a/examples/README.md b/examples/README.md index f6ca001..55c71e2 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,27 +1,49 @@ # Examples -The format's schema (`skill.schema.yaml`) deliberately **does not close the -lists of values** for `role`, `kind`, `source`, `deliver`, `reasoning` or the -shape of `ref`: behind those words stands the design of a particular -application, and someone else's is of no use to yours. - -Here are samples of what those slots get filled with, plus working skills from -different areas. These are EXAMPLES, not requirements: invent your own values. - -Every file is checked by a test (`examples_test.go`): an example that stopped -parsing is worse than a missing one — it teaches the wrong thing. - -| file | about | what it shows | -|---|---|---| -| [`vocabulary.yaml`](vocabulary.yaml) | a vocabulary of values | what the schema's open slots get filled with | -| [`pods.yaml`](pods.yaml) | listing machines in a cluster | parse the request → call → answer; the server is computed, but only within the declared set | -| [`weather.yaml`](weather.yaml) | weather through a browser | a `call` step without generation; the browser goes only to the step that needs it | -| [`proofread.yaml`](proofread.yaml) | proofreading text | a reference asset is SUBSTITUTED into the instruction — otherwise the model never reads it | -| [`expenses.yaml`](expenses.yaml) | spending as a chart | a code asset and the data go BY REFERENCE past the context; `deliver` is declared in advance | -| [`inbox.yaml`](inbox.yaml) | triaging an email | `switch` + `delegate`: in the "spam" branch the step that replies to the customer simply is not there | -| [`research.yaml`](research.yaml) | an answer from several sources | `parallel` and `.skipped` — "we never went" differs from "it was empty" | -| [`glossary.yaml`](glossary.yaml) | translating terms | `for_each` and `collect`; `in` takes a variable NAME, not a template | -| [`contract.yaml`](contract.yaml) | checking a contract | `if` + `exit` (wrong document — an honest exit) and an external asset with a `fetch` policy | -| [`triage.yaml`](triage.yaml) | triaging an incident | a composite skill: branching and delegation | -| [`audit.yaml`](audit.yaml) | checking a document | `profiles` shared by the classifier steps, and all four `on_empty` outcomes | -| [`menu.yaml`](menu.yaml) | suggesting dishes by section | `contains` — a classifier step replaced by a condition, dictionary and all | +Two things live here: skills, and applications that run them. + +| | | +|---|---| +| [`skills/`](skills/) | the format itself — working skills from different areas, plus a vocabulary of the values the schema deliberately leaves open | +| [`simple-llm-app/`](simple-llm-app/) | the engine embedded in ~200 lines: an OpenAI-compatible endpoint over `net/http`, no dependencies beyond the engine and a YAML parser | +| [`eino-llm-app/`](eino-llm-app/) | the same application with the model reached through [eino](https://github.com/cloudwego/eino) — the whole framework-shaped part is one forty-line adapter | + +## Why each application is its own module + +Both apps have their own `go.mod`, and that is the point rather than tidiness. + +The engine's promise is that embedding it adds no dependencies: production code +is stdlib only, and a guard test fails the build the moment that stops being +true. An example that pulls in a framework would break exactly that promise — +unless it is a separate module, which is what `go.mod` next to it makes it. The +engine's `go.mod` never learns that eino exists. + +Two guards hold this in place, both in `imports_test.go` upstairs: one skips +nested modules while checking that the engine imports nothing, the other checks +that a directory with Go code in it still HAS a `go.mod`. Delete one of those +files and the first guard immediately reports eino as a dependency of the +engine — which is what it is at that moment. + +CI runs `go vet` and `go test` inside each example module separately, because +`go test ./...` at the top never descends into them. An example that does not +build teaches the format wrong. + +## Running one + +``` +cd simple-llm-app +export OPENAI_BASE_URL=http://localhost:8000/v1 # vLLM, Ollama, LM Studio, … +export OPENAI_API_KEY=… +export OPENAI_MODEL=… + +go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" +``` + +Both applications print the answer and then the trace — which step ran, which +was skipped and why, how many tool calls each made. That trace is the engine's +whole observability contract: it logs nothing, stores nothing and reaches +nowhere on its own. + +Neither needs a key to be **tested**: each has a test that runs the whole +application against a stub model, so "it works" is something you can check +rather than something this file claims. diff --git a/examples/eino-llm-app/README.md b/examples/eino-llm-app/README.md new file mode 100644 index 0000000..ca29180 --- /dev/null +++ b/examples/eino-llm-app/README.md @@ -0,0 +1,75 @@ +# eino-llm-app + +The same application as `../simple-llm-app`, with the model reached through +[eino](https://github.com/cloudwego/eino). + +``` +export OPENAI_BASE_URL=http://localhost:8000/v1 +export OPENAI_API_KEY=… +export OPENAI_MODEL=… + +go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" +``` + +## The whole point is `runner.go` + +Forty lines, and everything eino-shaped in this example lives in them. The +adapter takes `model.BaseChatModel` — the interface, not a vendor — so eino's +OpenAI, Ark, Ollama or Qwen components drop in without touching it, and so does +a stub in a test. + +The direction is what matters: **the engine does not know eino, and eino does +not know the engine.** The application owns the seam between them. That is what +makes the engine embeddable in an application built on some other framework +tomorrow — and it is why this directory has its own `go.mod`. + +``` +examples/eino-llm-app/go.mod → requires eino +skill-engine/go.mod → has never heard of it +``` + +A guard test upstairs (`imports_test.go`) enforces both halves: it skips nested +modules when checking that the engine imports nothing, and it fails if a +directory with Go code in it loses its `go.mod`. Delete this one and eino is +reported as a dependency of the engine on the next run — which, at that moment, +is exactly what it would be. + +## What the adapter carries across + +**Declarations.** A step's `model:` and `sampling:` become `model.WithModel`, +`WithTemperature`, `WithTopP`, `WithMaxTokens`. A field the skill sets and the +executor drops is decoration — the author writes `temperature: 0` on a +classifier for a reason, and nothing downstream would tell them it never +arrived. There is a test for exactly this. + +**The step's radius.** An empty tool set is not "no preference": it is the +skill's guard, and the adapter hands the model nothing. Binding tools "just in +case" is how a prohibition quietly stops being one. + +**What the executor knows and the engine cannot derive.** A generation stopped +at the token ceiling comes back as `Result.Truncated` with a reason, and the +engine marks the step degraded on the strength of it. A truncated answer and a +step that simply had nothing to say are indistinguishable from the text alone. + +## Testing it + +``` +go test ./... +``` + +The tests run the application end to end against a stub chat model — no network, +no key — because the adapter takes eino's interface rather than its client. They +check the same flow as the simple app, plus that the skill's declarations really +reach the model as options. + +## The `replace` in go.mod + +``` +replace github.com/inhuman/skill-engine => ../.. +``` + +It points at the engine in this repository, so CI checks the example against the +code as it is now rather than against the last release — an API break is caught +the day it happens, not a version later. **Delete that line when you copy this +example into your own project**; the `require` above it is what you actually +want. diff --git a/examples/eino-llm-app/go.mod b/examples/eino-llm-app/go.mod new file mode 100644 index 0000000..0ea8d09 --- /dev/null +++ b/examples/eino-llm-app/go.mod @@ -0,0 +1,44 @@ +module github.com/inhuman/skill-engine/examples/eino-llm-app + +go 1.26.1 + +require ( + github.com/cloudwego/eino v0.9.13 + github.com/cloudwego/eino-ext/components/model/openai v0.1.13 + github.com/inhuman/skill-engine v0.5.2 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/cloudwego/base64x v0.1.6 // indirect + github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/eino-contrib/jsonschema v1.0.3 // indirect + github.com/evanphx/json-patch v0.5.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/goph/emperror v0.17.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/meguminnnnnnnnn/go-openai v0.1.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/nikolalohinski/gonja v1.5.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + github.com/yargevad/filepathx v1.0.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect + golang.org/x/sys v0.29.0 // indirect +) + +replace github.com/inhuman/skill-engine => ../.. diff --git a/examples/eino-llm-app/go.sum b/examples/eino-llm-app/go.sum new file mode 100644 index 0000000..d9d92e9 --- /dev/null +++ b/examples/eino-llm-app/go.sum @@ -0,0 +1,151 @@ +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/mockey v1.3.0 h1:ONLRdvhqmCfr9rTasUB8ZKCfvbdD2tohOg4u+4Q/ed0= +github.com/bytedance/mockey v1.3.0/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= +github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= +github.com/cloudwego/eino v0.9.13 h1:iD/ETS+lxnNp1VeNPqWVGPWdND6Dbf4LyINbLUlDRcM= +github.com/cloudwego/eino v0.9.13/go.mod h1:OBD1mrkfkt/pJa4rkg1P0VnaMeOVl7l8IAdEqY//3IQ= +github.com/cloudwego/eino-ext/components/model/openai v0.1.13 h1:5XHRTiTD5bt9KQrMHcfvuWNklEC3tpm3XHejdozt9vM= +github.com/cloudwego/eino-ext/components/model/openai v0.1.13/go.mod h1:mgIoqYYOc0eECCqvLbEYpOJrQNTNxkwXzSJzFU+v5sQ= +github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17 h1:EeVcR1TslRA2IdNW1h/2LaGbPlffwGhQm99jM3zWZiI= +github.com/cloudwego/eino-ext/libs/acl/openai v0.1.17/go.mod h1:Zkcx6DPTR2NfWmtSXbhItswGw6hqUezNPhNcke0pOG8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eino-contrib/jsonschema v1.0.3 h1:2Kfsm1xlMV0ssY2nuxshS4AwbLFuqmPmzIjLVJ1Fsp0= +github.com/eino-contrib/jsonschema v1.0.3/go.mod h1:cpnX4SyKjWjGC7iN2EbhxaTdLqGjCi0e9DxpLYxddD4= +github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8P3k= +github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= +github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/meguminnnnnnnnn/go-openai v0.1.2 h1:iXombGGjqjBrmE9WaSidUhhi3YQhf42QTHvHLMkgvCA= +github.com/meguminnnnnnnnn/go-openai v0.1.2/go.mod h1:qs96ysDmxhE4BZoU45I43zcyfnaYxU3X+aRzLko/htY= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/eino-llm-app/main.go b/examples/eino-llm-app/main.go new file mode 100644 index 0000000..6352d34 --- /dev/null +++ b/examples/eino-llm-app/main.go @@ -0,0 +1,121 @@ +// The same application, with the model reached through eino. +// +// The point of this example is one type — the adapter below — and one fact +// about it: eino appears in THIS module's go.mod and nowhere else. The engine +// stays dependency-free, and an application built on a framework plugs the +// framework in at the only seam that touches a model. +// +// go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" +// +// Set OPENAI_BASE_URL, OPENAI_API_KEY and OPENAI_MODEL for your endpoint; eino's +// OpenAI component speaks to anything OpenAI-compatible. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + + "github.com/cloudwego/eino-ext/components/model/openai" + "github.com/cloudwego/eino/components/model" + "gopkg.in/yaml.v3" + + se "github.com/inhuman/skill-engine" +) + +func main() { + skillPath := flag.String("skill", "../skills/menu.yaml", "path to a skill file") + input := flag.String("input", "подбери десерт и напиток", "the user's request") + flag.Parse() + + chat, err := newChatModel(context.Background()) + if err != nil { + fmt.Fprintln(os.Stderr, "cannot build the model:", err) + os.Exit(1) + } + if err := run(chat, *skillPath, *input, os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +// newChatModel builds eino's OpenAI component. Any other implementation from +// eino-ext — Ark, Ollama, Qwen — drops in here without touching the adapter: +// the adapter knows the INTERFACE, not the vendor. +func newChatModel(ctx context.Context) (model.BaseChatModel, error) { + return openai.NewChatModel(ctx, &openai.ChatModelConfig{ + BaseURL: env("OPENAI_BASE_URL", "https://api.openai.com/v1"), + APIKey: os.Getenv("OPENAI_API_KEY"), + Model: env("OPENAI_MODEL", "gpt-4o-mini"), + }) +} + +func env(name, fallback string) string { + if v := os.Getenv(name); v != "" { + return v + } + return fallback +} + +func run(chat model.BaseChatModel, skillPath, input string, out io.Writer) error { + raw, err := os.ReadFile(skillPath) + if err != nil { + return err + } + skill, err := se.ParseSkill(raw, yaml.Unmarshal) + if err != nil { + return err + } + if err := skill.Validate(); err != nil { + return err + } + + vars, outcome, err := se.ExecuteWith(context.Background(), skill.Workflow, se.Deps{ + Runner: runner{chat: chat, log: out}, + Caller: tools{log: out}, + Assets: assets{}, + Vocabulary: se.Vocabulary{ + DecisionMarkers: []string{"Result:", "Ответ:"}, + }, + OnStepStart: func(name, kind string) { fmt.Fprintf(out, "→ %s (%s)\n", name, kind) }, + }, map[string]string{"input": input}) + if err != nil { + if errors.Is(err, se.ErrExit) { + fmt.Fprintln(out, "the skill stepped aside:", err) + return nil + } + return err + } + + fmt.Fprintln(out, "\n=== answer ===") + fmt.Fprintln(out, vars[se.AnswerVar]) + fmt.Fprintln(out, "\n=== steps ===") + for _, s := range outcome.Steps { + fmt.Fprintf(out, " %-16s %-11s %-8s %s\n", s.Name, s.Kind, s.Outcome, s.Reason) + } + return nil +} + +// tools and assets are the same as in ../simple-llm-app: eino changes how the +// MODEL is reached and nothing else about embedding the engine. +type tools struct{ log io.Writer } + +func (t tools) CallTool(_ context.Context, server, tool string, args map[string]any) (string, error) { + fmt.Fprintf(t.log, " call %s:%s %v\n", server, tool, args) + if server+":"+tool == "recipes:search" { + return "1. Тирамису — 30 минут\n2. Панна-котта — 15 минут", nil + } + return "", fmt.Errorf("no such tool: %s:%s", server, tool) +} + +type assets struct{} + +func (assets) Resolve(_ context.Context, name string, a se.Asset) (string, error) { + if a.Source == "inline" || a.Content != "" { + return a.Content, nil + } + return "", fmt.Errorf("asset %q: source %q is not implemented in this example", name, a.Source) +} diff --git a/examples/eino-llm-app/main_test.go b/examples/eino-llm-app/main_test.go new file mode 100644 index 0000000..b02f0a6 --- /dev/null +++ b/examples/eino-llm-app/main_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "bytes" + "context" + "strings" + "testing" + + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" +) + +// stubChat implements eino's chat model interface and records what it was +// asked. Because the adapter takes the INTERFACE rather than a vendor's client, +// the example can be verified end to end with no network and no key — and so +// can yours. +type stubChat struct { + answer string + seen []*schema.Message + opts int +} + +func (s *stubChat) Generate(_ context.Context, in []*schema.Message, opts ...model.Option) (*schema.Message, error) { + s.seen = append(s.seen, in...) + s.opts = len(opts) + return schema.AssistantMessage(s.answer, nil), nil +} + +func (s *stubChat) Stream(context.Context, []*schema.Message, ...model.Option) (*schema.StreamReader[*schema.Message], error) { + panic("this example only uses Generate") +} + +// The whole example against a shipped skill: conditions choose the sections the +// request names, `call` steps fetch them with no model involved, and the last +// step words the answer through eino. +func TestExampleRunsAShippedSkill(t *testing.T) { + chat := &stubChat{answer: "Тирамису и чай — 30 минут."} + + var out bytes.Buffer + if err := run(chat, "../skills/menu.yaml", "подбери десерт и напиток", &out); err != nil { + t.Fatalf("the example does not run: %v", err) + } + got := out.String() + + for _, want := range []string{"Тирамису и чай", "call recipes:search", "pick_dessert"} { + if !strings.Contains(got, want) { + t.Errorf("the output does not mention %q:\n%s", want, got) + } + } + if n := strings.Count(got, "call recipes:search"); n != 2 { + t.Errorf("expected two sections fetched, got %d:\n%s", n, got) + } + if len(chat.seen) != 1 { + t.Fatalf("expected one generation, got %d", len(chat.seen)) + } + if !strings.Contains(chat.seen[0].Content, "Тирамису") { + t.Errorf("what the tools returned never reached the model:\n%s", chat.seen[0].Content) + } +} + +// What a skill declares must arrive at the model. A step's `model:` and +// `sampling:` are translated into eino options; an adapter that drops them +// turns those fields into decoration, and nothing tells the author. +func TestDeclarationsReachTheModel(t *testing.T) { + chat := &stubChat{answer: "ok"} + var out bytes.Buffer + + if err := run(chat, "testdata/declared.yaml", "что угодно", &out); err != nil { + t.Fatalf("the example does not run: %v", err) + } + // model + temperature + max_tokens + if chat.opts != 3 { + t.Errorf("expected three options passed to eino, got %d", chat.opts) + } +} + +// A skill that leaves through `exit` did not fail: it decided the request was +// not its case. An embedder reporting that as an error reports a bug where the +// skill made a decision. +func TestExampleHandlesAnExit(t *testing.T) { + chat := &stubChat{answer: "unused"} + + var out bytes.Buffer + if err := run(chat, "../skills/menu.yaml", "расскажи что-нибудь", &out); err != nil { + t.Fatalf("an exit was reported as a failure: %v", err) + } + if !strings.Contains(out.String(), "stepped aside") { + t.Errorf("the exit was not recognised:\n%s", out.String()) + } +} diff --git a/examples/eino-llm-app/runner.go b/examples/eino-llm-app/runner.go new file mode 100644 index 0000000..5e34de7 --- /dev/null +++ b/examples/eino-llm-app/runner.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "fmt" + "io" + + "github.com/cloudwego/eino/components/model" + "github.com/cloudwego/eino/schema" + + se "github.com/inhuman/skill-engine" +) + +// runner is the whole adapter: skill-engine's Runner on top of any eino chat +// model. Everything eino-shaped in this example lives in these forty lines. +// +// The direction matters. The engine does not know eino, and eino does not know +// the engine — the application owns the seam between them, which is exactly +// what makes the engine embeddable in an application built on some other +// framework tomorrow. +type runner struct { + chat model.BaseChatModel + log io.Writer +} + +func (r runner) Run(ctx context.Context, req se.StepRequest) (se.Result, error) { + // What the SKILL declared, translated into what eino calls it. A field the + // skill sets and the executor drops is decoration: the author writes + // `sampling: {temperature: 0}` on a classifier for a reason, and nothing + // downstream tells them it never arrived. + var opts []model.Option + if req.Model != "" { + opts = append(opts, model.WithModel(req.Model)) + } + if s := req.Sampling; s != nil { + if s.Temperature != nil { + opts = append(opts, model.WithTemperature(*s.Temperature)) + } + if s.TopP != nil { + opts = append(opts, model.WithTopP(*s.TopP)) + } + if s.MaxTokens != nil { + opts = append(opts, model.WithMaxTokens(*s.MaxTokens)) + } + } + + // The step's radius. An EMPTY list is not "no preference" — it is the + // skill's guard, and a step that was handed no tools must be given none + // here either. Binding tools "just in case" is how a prohibition quietly + // stops being one. + if len(req.Tools) > 0 { + fmt.Fprintf(r.log, " (step %q may use: %v)\n", req.Name, req.Tools) + // A real application resolves these names to eino tools and passes them + // with model.WithTools, or hands the step to an eino agent. This example + // keeps `call` steps for the deterministic work and leaves the model to + // think, which is the cheaper shape anyway. + } + + msg, err := r.chat.Generate(ctx, []*schema.Message{schema.UserMessage(req.Instruction)}, opts...) + if err != nil { + return se.Result{}, err + } + + // A truncated generation is not an empty one, and the engine cannot tell + // them apart from the text. Whatever the executor KNOWS goes back in the + // Result — that is what a step is marked degraded on. + res := se.Result{Text: msg.Content} + if msg.ResponseMeta != nil && msg.ResponseMeta.FinishReason == "length" { + res.Truncated = true + res.Note = "the model stopped at the token ceiling" + } + return res, nil +} diff --git a/examples/eino-llm-app/testdata/declared.yaml b/examples/eino-llm-app/testdata/declared.yaml new file mode 100644 index 0000000..f1c719b --- /dev/null +++ b/examples/eino-llm-app/testdata/declared.yaml @@ -0,0 +1,16 @@ +# A one-step skill whose only job is to declare things and check they arrive. +skill_engine_version: "2.2.2" +skill_version: "1.0.0" +name: declared +description: A fixture — a step that names its model and its sampling. +trigger_examples: ["anything"] + +workflow: + steps: + - name: answer + instruction: "Ответь на: {{input}}" + tools: [] + model: some-small-model + sampling: + temperature: 0 + max_tokens: 128 diff --git a/examples/simple-llm-app/README.md b/examples/simple-llm-app/README.md new file mode 100644 index 0000000..a95f446 --- /dev/null +++ b/examples/simple-llm-app/README.md @@ -0,0 +1,81 @@ +# simple-llm-app + +The engine embedded in an application, with nothing between it and the model but +`net/http`. + +``` +export OPENAI_BASE_URL=http://localhost:8000/v1 # vLLM, Ollama, LM Studio, … +export OPENAI_API_KEY=… +export OPENAI_MODEL=… + +go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" + +# the same file run the other way, when a skill carries both descriptions +go run . -skill ../skills/menu.yaml -mode playbook -input "подбери десерт и напиток" +``` + +`-mode` overrides the skill's own `mode`, which is what makes the A/B in +[QUICKSTART.md](../../QUICKSTART.md) a one-word change: same file, same model, +same request. Each run reports what it cost — generations and tokens — so the +two halves can be compared rather than argued about. + +## What it shows + +**Loading a skill.** `ParseSkill` reads the whole file — header and description +alike — and `Validate` is a separate call on purpose: "this is not YAML" and +"this YAML says something wrong" are different answers, and an application +reporting on a file has to tell them apart. + +**Both halves of the format.** `ResolveMode` says which description runs. A +skill may carry a `playbook` instead of steps, and then the engine takes no part +at all — the application runs the prompt itself. Handling that case is what makes +an embedder complete rather than half-written. + +**The four seams.** `Deps` is the entire contract: + +| | | +|---|---| +| `Runner` | the ONE place the engine touches a model. The step arrives resolved — variables substituted, assets inlined, tools narrowed — and all that is left is to generate | +| `Caller` | a `call` step: a tool invocation with no model involved. Here it is a table so the example runs offline; in your application it is your MCP client | +| `Assets` | payloads a skill declares. Even `source: inline` comes through here — an asset is the application's to fetch, cache and police | +| `Memory` | a large result by the handle the host appended to it | + +**Vocabulary.** The engine ships no words of its own. What your model writes +before naming a choice, and how your host marks a shortened result, are declared +here — an agent about a kitchen and one about a car fleet share the format, not +a language. + +**What the skill declared has to arrive.** A step's `model:`, its `sampling:`, +its `response_schema:` are forwarded to the endpoint. An executor that drops +them turns every one of those fields into decoration: the skill declares, and +nothing happens, and nothing says so. + +**Failing properly.** A tool that does not exist returns an error rather than an +apology — the skill's `on_error` decides what happens next, and it can only +decide if it is told. A skill leaving through `exit` is not a failure: it decided +the request was not its case. + +## Testing it + +``` +go test ./... +``` + +The test runs the whole application against a stub OpenAI-compatible server, on +a skill shipped in `../skills`. No key, no network. It checks that the +conditions picked the sections the request named, that the `call` steps ran +without a model, that a section nobody asked for was skipped **and still +reported as skipped** — "we did not go there" and "it came back empty" are +different answers — and that every shipped skill still loads through this path. + +## The `replace` in go.mod + +``` +replace github.com/inhuman/skill-engine => ../.. +``` + +It points at the engine in this repository, so CI checks the example against the +code as it is now rather than against the last release — an API break is caught +the day it happens, not a version later. **Delete that line when you copy this +example into your own project**; the `require` above it is what you actually +want. diff --git a/examples/simple-llm-app/go.mod b/examples/simple-llm-app/go.mod new file mode 100644 index 0000000..93397e3 --- /dev/null +++ b/examples/simple-llm-app/go.mod @@ -0,0 +1,10 @@ +module github.com/inhuman/skill-engine/examples/simple-llm-app + +go 1.26.1 + +require ( + github.com/inhuman/skill-engine v0.5.2 + gopkg.in/yaml.v3 v3.0.1 +) + +replace github.com/inhuman/skill-engine => ../.. diff --git a/examples/simple-llm-app/go.sum b/examples/simple-llm-app/go.sum new file mode 100644 index 0000000..c4c1710 --- /dev/null +++ b/examples/simple-llm-app/go.sum @@ -0,0 +1,10 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/examples/simple-llm-app/main.go b/examples/simple-llm-app/main.go new file mode 100644 index 0000000..f5d4c44 --- /dev/null +++ b/examples/simple-llm-app/main.go @@ -0,0 +1,255 @@ +// A minimal application that embeds the skill engine. +// +// It shows the whole contract in one file: load a skill, hand the engine the +// four things it cannot do itself — talk to a model, call a tool, resolve an +// asset, read working memory — and print what came back. +// +// The model is reached over the OpenAI-compatible /chat/completions endpoint, +// which is what vLLM, Ollama, LM Studio and the hosted APIs all speak, using +// nothing but net/http. That keeps this example honest about the engine's own +// promise: it adds no dependencies of its own, and neither does using it. +// +// go run . -skill ../skills/menu.yaml -input "подбери десерт и напиток" +// +// Set OPENAI_BASE_URL and OPENAI_API_KEY for your endpoint. There is no key in +// the code and no default that quietly points somewhere. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + "strings" + + "gopkg.in/yaml.v3" + + se "github.com/inhuman/skill-engine" +) + +func main() { + skillPath := flag.String("skill", "../skills/menu.yaml", "path to a skill file") + input := flag.String("input", "подбери десерт и напиток", "the user's request") + // Overrides the skill's own `mode`. A skill carrying BOTH descriptions can + // then be run either way on the same request — which is the A/B in + // QUICKSTART.md, and the reason `mode` exists in the format at all. + mode := flag.String("mode", "", "run the skill as `workflow` or `playbook` (default: what the skill says)") + flag.Parse() + + if err := run(*skillPath, *input, *mode, os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +func run(skillPath, input, forceMode string, out io.Writer) error { + raw, err := os.ReadFile(skillPath) + if err != nil { + return err + } + + // One call reads the whole file — header and description alike. Validate is + // separate on purpose: "this is not YAML" and "this YAML says something + // wrong" are different answers, and a tool reporting on a file needs to + // tell them apart. + skill, err := se.ParseSkill(raw, yaml.Unmarshal) + if err != nil { + return err + } + if err := skill.Validate(); err != nil { + return err + } + + // Which of the two descriptions to run. A skill may carry a `playbook` as + // well, and then the ENGINE takes no part: the application runs the prompt + // itself. Handling that case is what makes an embedder complete. + // + // An explicit mode with an empty half is an ERROR rather than a fallback to + // the other one, and the engine enforces that: a silent fallback would give + // a clean run over the description that was NOT selected, and a conclusion + // drawn from a turn the chosen half never took part in. + declared := skill.Mode + if forceMode != "" { + declared = forceMode + } + mode, err := se.ResolveMode(declared, skill.HasWorkflow(), skill.HasPlaybook()) + if err != nil { + return err + } + + st := &stats{} + if mode == se.ModePlaybook { + // The whole task in one prompt: the model decides what to look up, calls + // what it decides, and words the answer. The engine takes no part. + answer, u, err := newModel().complete(context.Background(), skill.Playbook+"\n\nЗапрос: "+input, nil) + if err != nil { + return err + } + st.add(u) + fmt.Fprintln(out, "\n=== answer ===") + fmt.Fprintln(out, answer) + st.report(out, "playbook") + return nil + } + + vars, outcome, err := se.ExecuteWith(context.Background(), skill.Workflow, deps(out, st), map[string]string{ + "input": input, + }) + if err != nil { + // A skill leaving on purpose is not a failure: it decided the request + // was not its case, and the turn goes back to its ordinary path. + if errors.Is(err, se.ErrExit) { + fmt.Fprintln(out, "the skill stepped aside:", err) + return nil + } + return err + } + + fmt.Fprintln(out, "\n=== answer ===") + fmt.Fprintln(out, vars[se.AnswerVar]) + fmt.Fprintln(out, "\n=== steps ===") + for _, s := range outcome.Steps { + line := fmt.Sprintf(" %-16s %-11s %-8s calls=%d", s.Name, s.Kind, s.Outcome, s.Calls) + if s.Reason != "" { + line += " " + s.Reason + } + fmt.Fprintln(out, line) + } + if len(outcome.Skipped) > 0 { + fmt.Fprintln(out, " skipped:", strings.Join(outcome.Skipped, ", ")) + } + st.report(out, "workflow") + return nil +} + +// stats — what the turn cost. Counted because "steps are cheaper" is a claim +// until somebody puts a number next to it, and the same skill run both ways on +// the same request is the cheapest way to get one. +type stats struct { + generations int + prompt int + completion int +} + +func (s *stats) add(u usage) { + s.generations++ + s.prompt += u.PromptTokens + s.completion += u.CompletionTokens +} + +func (s *stats) report(out io.Writer, mode string) { + fmt.Fprintf(out, "\n=== cost (%s) ===\n", mode) + fmt.Fprintf(out, " generations: %d\n", s.generations) + fmt.Fprintf(out, " tokens: %d prompt + %d completion = %d\n", + s.prompt, s.completion, s.prompt+s.completion) +} + +// deps is the whole contract between an application and the engine. +// +// Everything the engine cannot know — how to reach a model, what a tool is, +// where an asset's content lives, what your host calls things — arrives here. +// Nothing else is injected, and the engine logs nothing, stores nothing and +// reaches nowhere on its own. +func deps(out io.Writer, st *stats) se.Deps { + m := newModel() + return se.Deps{ + Runner: runner{model: m, log: out, stats: st}, + Caller: tools{log: out}, + Assets: assets{}, + Delegate: nil, // no composite skills here: a `delegate` step would fail loudly + Memory: memory{}, + + // The words of THIS application. The engine ships none — an agent about + // a kitchen and one about a car fleet share the format, not a language. + Vocabulary: se.Vocabulary{ + DecisionMarkers: []string{"Result:", "Answer:", "Итог:", "Ответ:"}, + TruncationNotes: []string{"shortened:"}, + }, + + // A step that runs for ten seconds emits nothing until it finishes, and + // there is nothing to show a human all that time. + OnStepStart: func(name, kind string) { + fmt.Fprintf(out, "→ %s (%s)\n", name, kind) + }, + } +} + +// runner executes an instruction step: it is the ONE place the engine touches a +// model. +// +// The step arrives fully resolved — variables substituted, assets inlined, the +// tool set narrowed, the sampling decided. All that is left is to generate. +type runner struct { + model *openAI + log io.Writer + stats *stats +} + +func (r runner) Run(ctx context.Context, req se.StepRequest) (se.Result, error) { + // The tool set is the step's radius, and an EMPTY one is not "no + // preference": the step is meant to answer from what has already been + // gathered. Handing it tools anyway would undo the guard the skill relies + // on — so it is passed through as it came. + if len(req.Tools) > 0 { + fmt.Fprintf(r.log, " (step %q may use: %s)\n", req.Name, strings.Join(req.Tools, ", ")) + } + + text, u, err := r.model.complete(ctx, req.Instruction, &req) + if err != nil { + return se.Result{}, err + } + r.stats.add(u) + + // Result carries more than the text: what the executor KNOWS and the engine + // cannot derive. A truncated generation and a step that simply had nothing + // to say look identical from the outside, and the engine marks a step + // degraded on the strength of these fields. + return se.Result{Text: text}, nil +} + +// tools executes a `call` step — a tool invocation with no model involved. +// +// In a real application this is your MCP client, your HTTP client, your +// function registry. Here it is a table, so the example runs offline and every +// skill in ../skills can be tried out. +type tools struct{ log io.Writer } + +func (t tools) CallTool(_ context.Context, server, tool string, args map[string]any) (string, error) { + fmt.Fprintf(t.log, " call %s:%s %v\n", server, tool, args) + + switch server + ":" + tool { + case "recipes:search": + return "1. Тирамису — 30 минут\n2. Панна-котта — 15 минут", nil + } + // A tool that does not exist must FAIL, not return an apology: the skill's + // on_error policy decides what happens next, and it can only decide if it + // is told. + return "", fmt.Errorf("no such tool: %s:%s", server, tool) +} + +// assets resolve the payloads a skill declares. +// +// `source: inline` is the engine's own case and still comes through here — the +// engine deliberately does not read `content` itself, because an asset is the +// application's to fetch, cache and police. Every other source is yours: a +// repository, a URL, a file the user uploaded. +type assets struct{} + +func (assets) Resolve(_ context.Context, name string, a se.Asset) (string, error) { + if a.Source == "inline" || a.Content != "" { + return a.Content, nil + } + return "", fmt.Errorf("asset %q: source %q is not implemented in this example", name, a.Source) +} + +// memory returns a large result by the handle the host appended to it. +// +// This example never truncates anything, so the map stays empty and the engine +// simply never asks. It is here to show WHERE the seam is: without a reader, a +// step that cannot fetch the rest of a value is told so in its trace instead of +// quietly working on a fragment. +type memory map[string]string + +func (m memory) Get(id string) (string, bool) { v, ok := m[id]; return v, ok } diff --git a/examples/simple-llm-app/main_test.go b/examples/simple-llm-app/main_test.go new file mode 100644 index 0000000..23b36e9 --- /dev/null +++ b/examples/simple-llm-app/main_test.go @@ -0,0 +1,215 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// stubModel — an OpenAI-compatible endpoint that answers whatever it is told +// to. The example is verified end to end against it: an example that only +// compiles teaches nothing about whether it works. +func stubModel(t *testing.T, answer string) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/chat/completions" { + t.Errorf("unexpected path %q — the endpoint is not the one an OpenAI-compatible server exposes", r.URL.Path) + } + var body struct { + Model string `json:"model"` + Messages []struct { + Role string `json:"role"` + Content string `json:"content"` + } `json:"messages"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Errorf("the request is not JSON: %v", err) + } + if len(body.Messages) == 0 { + t.Error("the instruction never reached the model") + } + // Charged by length, the way a real endpoint charges. That is what makes + // the A/B in QUICKSTART.md checkable without a model: the prose half + // carries its whole dictionary into the prompt, the steps half does not. + prompt := 0 + for _, m := range body.Messages { + prompt += len(m.Content) / 4 + } + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]any{ + "choices": []any{map[string]any{ + "message": map[string]string{"content": answer}, + "finish_reason": "stop", + }}, + "usage": map[string]int{ + "prompt_tokens": prompt, + "completion_tokens": len(answer) / 4, + }, + }) + })) + t.Cleanup(srv.Close) + return srv +} + +func withStub(t *testing.T, answer string) { + t.Helper() + srv := stubModel(t, answer) + t.Setenv("OPENAI_BASE_URL", srv.URL) + t.Setenv("OPENAI_API_KEY", "not-a-real-key") + t.Setenv("OPENAI_MODEL", "stub") +} + +// The whole example, on a skill shipped in ../skills: conditions pick the +// sections the request names, `call` steps fetch them without a model, and the +// last step words the answer. +func TestExampleRunsAShippedSkill(t *testing.T) { + withStub(t, "Тирамису и чай — 30 минут.") + + var out bytes.Buffer + if err := run("../skills/menu.yaml", "подбери десерт и напиток", "", &out); err != nil { + t.Fatalf("the example does not run: %v", err) + } + got := out.String() + + for _, want := range []string{ + "Тирамису и чай", // the model's answer came back as the turn's answer + "call recipes:search", // a `call` step ran without a model + "pick_dessert", // the trace names the steps + "answer", // + } { + if !strings.Contains(got, want) { + t.Errorf("the output does not mention %q:\n%s", want, got) + } + } + // A section the request never named must not be FETCHED — but it must still + // be visible as skipped. "we did not go there" and "it came back empty" are + // different answers, and the trace is where the difference survives. + if n := strings.Count(got, "call recipes:search"); n != 2 { + t.Errorf("expected two sections fetched, got %d:\n%s", n, got) + } + if !strings.Contains(got, "skipped: nothing_named, pick_main") { + t.Errorf("the skipped steps are not reported:\n%s", got) + } +} + +// A request naming no section leaves the skill through `exit`, and that is not +// a failure: the turn goes back to its ordinary path. An embedder that treats +// it as an error reports a bug where the skill made a decision. +func TestExampleHandlesAnExit(t *testing.T) { + withStub(t, "unused") + + var out bytes.Buffer + if err := run("../skills/menu.yaml", "расскажи что-нибудь", "", &out); err != nil { + t.Fatalf("an exit was reported as a failure: %v", err) + } + if !strings.Contains(out.String(), "stepped aside") { + t.Errorf("the exit was not recognised:\n%s", out.String()) + } +} + +// Every skill in ../skills must at least load and validate through the same +// path the application uses. A shipped example that stopped parsing teaches the +// wrong thing, and this is the cheapest place to notice. +func TestEveryShippedSkillLoads(t *testing.T) { + withStub(t, "answer") + + for _, path := range shippedSkills(t) { + t.Run(path, func(t *testing.T) { + var out bytes.Buffer + err := run(path, "подбери десерт", "", &out) + // Running is allowed to fail — most of these skills need tools this + // example does not implement. Loading is not. + if err != nil && strings.Contains(err.Error(), "skill-engine:") { + t.Fatalf("the skill did not load: %v", err) + } + }) + } +} + +// shippedSkills lists the skill files next door, minus the vocabulary of values +// (it has no name and is not a skill). +func shippedSkills(t *testing.T) []string { + t.Helper() + all, err := filepath.Glob(filepath.Join("..", "skills", "*.yaml")) + if err != nil || len(all) == 0 { + t.Fatalf("the skills are gone: %v", err) + } + var out []string + for _, path := range all { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var doc struct { + Name string `yaml:"name"` + } + if err := yaml.Unmarshal(raw, &doc); err != nil { + t.Fatalf("%s: %v", path, err) + } + if doc.Name != "" { + out = append(out, path) + } + } + return out +} + +// The A/B QUICKSTART.md walks a reader through: one skill carrying both +// descriptions, the same request, one flag changed. +// +// What is checked here is the mechanism and the direction, not a benchmark — +// the stub charges by prompt length, so the numbers are the shape of the real +// ones rather than the real ones. In prose the dictionary of sections rides +// into the prompt and the model has to apply it; in steps the conditions +// applied it before any generation happened, and the prompt carries only what +// was found. +func TestSameSkillBothWays(t *testing.T) { + withStub(t, "Тирамису и чай.") + + var prose, steps bytes.Buffer + if err := run("../skills/menu.yaml", "подбери десерт и напиток", "playbook", &prose); err != nil { + t.Fatalf("the prose half does not run: %v", err) + } + if err := run("../skills/menu.yaml", "подбери десерт и напиток", "workflow", &steps); err != nil { + t.Fatalf("the steps half does not run: %v", err) + } + + for _, out := range []*bytes.Buffer{&prose, &steps} { + if !strings.Contains(out.String(), "=== cost (") { + t.Fatalf("the run did not report what it cost:\n%s", out.String()) + } + } + if p, s := promptTokens(t, &prose), promptTokens(t, &steps); s >= p { + t.Errorf("steps cost %d prompt tokens against prose %d — the dictionary should not reach the model at all", s, p) + } + + // And the part a token count cannot show: in steps the sections are chosen + // before any generation, so a section nobody named cannot be fetched. In + // prose that is a request, and requests are followed probabilistically. + if !strings.Contains(steps.String(), "skipped: nothing_named, pick_main") { + t.Errorf("the steps half did not decide the sections deterministically:\n%s", steps.String()) + } +} + +func promptTokens(t *testing.T, out *bytes.Buffer) int { + t.Helper() + for _, line := range strings.Split(out.String(), "\n") { + if !strings.Contains(line, "prompt + ") { + continue + } + var p, c, total int + if _, err := fmt.Sscanf(strings.TrimSpace(line), "tokens: %d prompt + %d completion = %d", &p, &c, &total); err != nil { + t.Fatalf("cannot read the cost line %q: %v", line, err) + } + return p + } + t.Fatalf("no cost line in:\n%s", out.String()) + return 0 +} diff --git a/examples/simple-llm-app/openai.go b/examples/simple-llm-app/openai.go new file mode 100644 index 0000000..02ebfe5 --- /dev/null +++ b/examples/simple-llm-app/openai.go @@ -0,0 +1,133 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + "time" + + se "github.com/inhuman/skill-engine" +) + +// openAI — a chat completion over the OpenAI-compatible endpoint, in net/http. +// +// Sixty lines, no client library. That is not thrift for its own sake: the +// engine's promise is that embedding it adds no dependencies, and an example +// that pulls in an SDK to prove the point would disprove it. +type openAI struct { + baseURL string + apiKey string + model string + client *http.Client +} + +func newModel() *openAI { + return &openAI{ + baseURL: strings.TrimSuffix(env("OPENAI_BASE_URL", "https://api.openai.com/v1"), "/"), + apiKey: os.Getenv("OPENAI_API_KEY"), + model: env("OPENAI_MODEL", "gpt-4o-mini"), + client: &http.Client{Timeout: 2 * time.Minute}, + } +} + +func env(name, fallback string) string { + if v := os.Getenv(name); v != "" { + return v + } + return fallback +} + +// complete sends one instruction and returns the answer's text. +// +// req is the step as the engine resolved it, and it carries more than the +// prompt. Passing those fields on is what makes a step's declarations real: +// a `model:` the skill named, its `sampling:`, its `response_schema:`. An +// executor that ignores them turns every one of those fields into decoration — +// the skill declares, the engine forwards, and nothing happens. +func (o *openAI) complete(ctx context.Context, instruction string, req *se.StepRequest) (string, usage, error) { + body := map[string]any{ + "model": o.model, + "messages": []map[string]string{{"role": "user", "content": instruction}}, + } + if req != nil { + if req.Model != "" { + body["model"] = req.Model + } + if s := req.Sampling; s != nil { + if s.Temperature != nil { + body["temperature"] = *s.Temperature + } + if s.TopP != nil { + body["top_p"] = *s.TopP + } + if s.MaxTokens != nil { + body["max_tokens"] = *s.MaxTokens + } + } + // A structured answer is only structured where the decoding grammar + // holds it. Dropping the schema here is the silent hole the format + // warns about: the model "usually" answers JSON, and the step that + // parses it fails on the day it does not. + if len(req.ResponseSchema) > 0 { + body["response_format"] = map[string]any{ + "type": "json_schema", + "json_schema": map[string]any{ + "name": "step", + "strict": true, + "schema": req.ResponseSchema, + }, + } + } + } + + raw, err := json.Marshal(body) + if err != nil { + return "", usage{}, err + } + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, o.baseURL+"/chat/completions", bytes.NewReader(raw)) + if err != nil { + return "", usage{}, err + } + httpReq.Header.Set("Content-Type", "application/json") + if o.apiKey != "" { + httpReq.Header.Set("Authorization", "Bearer "+o.apiKey) + } + + resp, err := o.client.Do(httpReq) + if err != nil { + return "", usage{}, err + } + defer resp.Body.Close() + + var out struct { + Choices []struct { + Message struct{ Content string } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Error *struct{ Message string } `json:"error"` + // What the run COST. Without it "steps are cheaper" is a claim; with it + // the two halves of a skill can be compared on the same request, which + // is what QUICKSTART.md has the reader do. + Usage usage `json:"usage"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return "", usage{}, fmt.Errorf("decoding the answer: %w", err) + } + if out.Error != nil { + return "", usage{}, fmt.Errorf("model: %s", out.Error.Message) + } + if len(out.Choices) == 0 { + return "", usage{}, fmt.Errorf("model returned no choices") + } + return out.Choices[0].Message.Content, out.Usage, nil +} + +// usage — what one generation cost, as the endpoint reports it. +type usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` +} diff --git a/examples/simple-llm-app/quickstart_test.go b/examples/simple-llm-app/quickstart_test.go new file mode 100644 index 0000000..a550c50 --- /dev/null +++ b/examples/simple-llm-app/quickstart_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// The minimal skill QUICKSTART.md tells a reader to save and run. It is the +// first thing they will execute, so it is the first thing that must work. +func TestQuickstartMinimalSkill(t *testing.T) { + const skill = `skill_engine_version: "2.2.0" +skill_version: "1.0.0" +name: my-skill +description: What this skill is FOR — and what it is NOT for. +trigger_examples: + - "a phrasing a user would actually type" + +workflow: + steps: + # A step with no tools cannot go anywhere. It thinks, and that is all. + - name: answer + instruction: | + The request: {{input}} + + Answer it in two sentences. + tools: [] +` + path := filepath.Join(t.TempDir(), "my-skill.yaml") + if err := os.WriteFile(path, []byte(skill), 0o600); err != nil { + t.Fatal(err) + } + withStub(t, "Привет. Чем помочь?") + + var out bytes.Buffer + if err := run(path, "привет", "", &out); err != nil { + t.Fatalf("the skill QUICKSTART tells the reader to write does not run: %v", err) + } + if !strings.Contains(out.String(), "Чем помочь") { + t.Errorf("no answer:\n%s", out.String()) + } +} diff --git a/examples/skills/README.md b/examples/skills/README.md new file mode 100644 index 0000000..f6ca001 --- /dev/null +++ b/examples/skills/README.md @@ -0,0 +1,27 @@ +# Examples + +The format's schema (`skill.schema.yaml`) deliberately **does not close the +lists of values** for `role`, `kind`, `source`, `deliver`, `reasoning` or the +shape of `ref`: behind those words stands the design of a particular +application, and someone else's is of no use to yours. + +Here are samples of what those slots get filled with, plus working skills from +different areas. These are EXAMPLES, not requirements: invent your own values. + +Every file is checked by a test (`examples_test.go`): an example that stopped +parsing is worse than a missing one — it teaches the wrong thing. + +| file | about | what it shows | +|---|---|---| +| [`vocabulary.yaml`](vocabulary.yaml) | a vocabulary of values | what the schema's open slots get filled with | +| [`pods.yaml`](pods.yaml) | listing machines in a cluster | parse the request → call → answer; the server is computed, but only within the declared set | +| [`weather.yaml`](weather.yaml) | weather through a browser | a `call` step without generation; the browser goes only to the step that needs it | +| [`proofread.yaml`](proofread.yaml) | proofreading text | a reference asset is SUBSTITUTED into the instruction — otherwise the model never reads it | +| [`expenses.yaml`](expenses.yaml) | spending as a chart | a code asset and the data go BY REFERENCE past the context; `deliver` is declared in advance | +| [`inbox.yaml`](inbox.yaml) | triaging an email | `switch` + `delegate`: in the "spam" branch the step that replies to the customer simply is not there | +| [`research.yaml`](research.yaml) | an answer from several sources | `parallel` and `.skipped` — "we never went" differs from "it was empty" | +| [`glossary.yaml`](glossary.yaml) | translating terms | `for_each` and `collect`; `in` takes a variable NAME, not a template | +| [`contract.yaml`](contract.yaml) | checking a contract | `if` + `exit` (wrong document — an honest exit) and an external asset with a `fetch` policy | +| [`triage.yaml`](triage.yaml) | triaging an incident | a composite skill: branching and delegation | +| [`audit.yaml`](audit.yaml) | checking a document | `profiles` shared by the classifier steps, and all four `on_empty` outcomes | +| [`menu.yaml`](menu.yaml) | suggesting dishes by section | `contains` — a classifier step replaced by a condition, dictionary and all | diff --git a/examples/audit.yaml b/examples/skills/audit.yaml similarity index 100% rename from examples/audit.yaml rename to examples/skills/audit.yaml diff --git a/examples/contract.yaml b/examples/skills/contract.yaml similarity index 100% rename from examples/contract.yaml rename to examples/skills/contract.yaml diff --git a/examples/expenses.yaml b/examples/skills/expenses.yaml similarity index 100% rename from examples/expenses.yaml rename to examples/skills/expenses.yaml diff --git a/examples/glossary.yaml b/examples/skills/glossary.yaml similarity index 100% rename from examples/glossary.yaml rename to examples/skills/glossary.yaml diff --git a/examples/inbox.yaml b/examples/skills/inbox.yaml similarity index 100% rename from examples/inbox.yaml rename to examples/skills/inbox.yaml diff --git a/examples/menu.yaml b/examples/skills/menu.yaml similarity index 71% rename from examples/menu.yaml rename to examples/skills/menu.yaml index 4b0a742..8fc9f06 100644 --- a/examples/menu.yaml +++ b/examples/skills/menu.yaml @@ -14,7 +14,7 @@ # warehouse shares this mechanism and not one of these words. skill_engine_version: "2.2.0" -skill_version: "1.0.0" +skill_version: "1.1.0" name: menu description: Suggest dishes from the sections of the menu the request names — a dessert, a drink, a main course. trigger_examples: @@ -25,6 +25,31 @@ kind: leaf servers: [recipes] +# This skill carries BOTH descriptions, which is what `mode` is for: while a +# prompt is being moved into steps it is worth keeping the old half and +# switching between them on live requests. Deleting it to compare would mean +# deleting the work with nowhere to get it back from. +# +# `mode: workflow` is what actually runs. Point it at `playbook` — or override +# it from the outside, as ../simple-llm-app does with -mode — and the same +# request goes through the prose version instead. That is the A/B in +# QUICKSTART.md: same file, same model, one word changed. +mode: workflow + +playbook: | + Ты подбираешь блюда по разделам меню. + + Разделы и слова, которыми их называют: + десерт, сладкое, dessert → dessert + напиток, чай, кофе, коктейль → drink + горячее, второе, main course → main + + Посмотри, какие разделы названы в запросе, и возьми РОВНО названные. + По каждому названному разделу вызови recipes:search с этим разделом. + Если не назван ни один — так и скажи, ничего не выдумывай. + + Ответь по тому, что нашлось. Про раздел, который не спрашивали, не пиши. + workflow: tools: [recipes] diff --git a/examples/pods.yaml b/examples/skills/pods.yaml similarity index 100% rename from examples/pods.yaml rename to examples/skills/pods.yaml diff --git a/examples/proofread.yaml b/examples/skills/proofread.yaml similarity index 100% rename from examples/proofread.yaml rename to examples/skills/proofread.yaml diff --git a/examples/research.yaml b/examples/skills/research.yaml similarity index 100% rename from examples/research.yaml rename to examples/skills/research.yaml diff --git a/examples/triage.yaml b/examples/skills/triage.yaml similarity index 100% rename from examples/triage.yaml rename to examples/skills/triage.yaml diff --git a/examples/vocabulary.yaml b/examples/skills/vocabulary.yaml similarity index 100% rename from examples/vocabulary.yaml rename to examples/skills/vocabulary.yaml diff --git a/examples/weather.yaml b/examples/skills/weather.yaml similarity index 100% rename from examples/weather.yaml rename to examples/skills/weather.yaml diff --git a/examples_test.go b/examples_test.go index 78d8533..ffa1bc6 100644 --- a/examples_test.go +++ b/examples_test.go @@ -18,7 +18,7 @@ import ( // fields would draw the wrong conclusion from it. func exampleFiles(t *testing.T) []string { t.Helper() - all, err := filepath.Glob(filepath.Join("examples", "*.yaml")) + all, err := filepath.Glob(filepath.Join("examples", "skills", "*.yaml")) require.NoError(t, err) require.NotEmpty(t, all, "the examples are gone") @@ -69,7 +69,7 @@ func readWorkflow(t *testing.T, path string) (se.Flow, bool) { // example that stopped parsing is worse than a missing one — it teaches the // wrong thing. func TestExamplesParseAndValidate(t *testing.T) { - files, err := filepath.Glob(filepath.Join("examples", "*.yaml")) + files, err := filepath.Glob(filepath.Join("examples", "skills", "*.yaml")) require.NoError(t, err) require.NotEmpty(t, files, "the examples are gone") @@ -89,7 +89,7 @@ func TestExamplesParseAndValidate(t *testing.T) { // a skill of another major is refused, so an example carrying a stale version // teaches a file that will not run. func TestExamplesDeclareCurrentFormat(t *testing.T) { - files, err := filepath.Glob(filepath.Join("examples", "*.yaml")) + files, err := filepath.Glob(filepath.Join("examples", "skills", "*.yaml")) require.NoError(t, err) for _, path := range files { @@ -112,7 +112,7 @@ func TestExamplesDeclareCurrentFormat(t *testing.T) { // An example must WORK, not merely parse: steps reach the executors, the server // is computed, the answer arrives. func TestExamplePodsRuns(t *testing.T) { - f, ok := readWorkflow(t, filepath.Join("examples", "pods.yaml")) + f, ok := readWorkflow(t, filepath.Join("examples", "skills", "pods.yaml")) require.True(t, ok) answers := map[string]string{ diff --git a/exec.go b/exec.go index 555018b..5ef7fa2 100644 --- a/exec.go +++ b/exec.go @@ -6,6 +6,7 @@ package skillengine import ( "context" + "sync" "time" ) @@ -37,12 +38,21 @@ type Deps struct { // OnStepStart is called BEFORE a step. Needed by anyone showing work to a // human: a step that runs for 14 seconds emits no event until it // finishes, and there is nothing to show all that time. + // + // CONCURRENT — see OnStep. OnStepStart func(name, kind string) // OnStep is called RIGHT AFTER each step, not at the end of the flow. // // Otherwise the caller learns about all the steps at once, when the turn // is already over: the progress post shows a finished list instead of work // as it happens, and the user stares at "brewing…" for nine seconds. + // + // BOTH CALLBACKS MUST BE SAFE FOR CONCURRENT USE. They fire from the + // goroutine that ran the step, and the branches of a `parallel` step run in + // several at once — so a callback that appends to a slice or writes to a + // map needs its own lock. The engine deliberately does not serialise them: + // a lock here would hold up a branch for the duration of somebody else's + // telemetry write, and the engine has no business deciding that. OnStep func(StepTrace) } @@ -90,11 +100,24 @@ type StepTrace struct { // Outcome — what happened to the flow beyond the variables. type Outcome struct { - // Steps — the trace of every executed (and skipped) step. + // Steps — the trace of every executed (and skipped) step of the FLOW. + // + // Steps inside the branches of a `parallel` are NOT here, and this is the + // one asymmetry in this struct worth knowing before you read it: Skipped + // below DOES include them. A branch runs in a forked state and only its + // variables and its skips are merged back at the join. + // + // Nothing about branch steps is lost, though — they reach Deps.OnStep as + // they happen, which is where an embedder recording per-step telemetry + // takes them from. This field is the flow's shape, OnStep is the event + // stream, and only the first one stops at the fork. Steps []StepTrace // Skipped — steps not executed because of a false `when`. Empty for a flow // without conditions; non-empty means the task matched only PARTIALLY, and // this is the only way to notice that. + // + // Includes steps skipped inside `parallel` branches — see Steps above for + // why the two differ. Skipped []string // AnsweredBy — the kind of step that wrote the turn's ANSWER: // "instruction" (the model wrote the text) or "call" (a tool printed it). @@ -114,7 +137,7 @@ func newState(f *Flow, deps Deps, vars map[string]string) (*state, error) { onStep: deps.OnStep, onStepStart: deps.OnStepStart, assets: f.Assets, assetsRes: deps.Assets, memory: deps.Memory, vocab: deps.Vocabulary, - assetCache: map[string]string{}, seeded: map[string]bool{}} + assetCache: map[string]string{}, assetMu: &sync.Mutex{}, seeded: map[string]bool{}} for k, v := range f.Vars { st.vars[k] = v st.seeded[k] = true @@ -151,7 +174,13 @@ type state struct { vocab Vocabulary // assetCache — content already fetched in THIS turn: one asset consumed by // three steps is fetched once. + // + // Shared with the branches of a `parallel` step rather than copied into + // them, so an asset three branches need is still fetched once — which is + // why it needs a lock, and why the lock is a POINTER: a forked state copies + // the struct, and a mutex copied by value is a mutex that guards nothing. assetCache map[string]string + assetMu *sync.Mutex // assetCtx — the turn's context for resolving payloads. expand() is called // from places with no ctx at hand, and threading it through every // signature for the sake of one branch costs more than storing it here: diff --git a/expand.go b/expand.go index 7ddc807..1c932eb 100644 --- a/expand.go +++ b/expand.go @@ -146,6 +146,13 @@ func (s *state) payload(name string) string { // instruction. The failure is not silent, though — the resolver reports it to // the caller. func (s *state) asset(name string) string { + // The lock is held across the resolve, not just around the map: branches of + // a `parallel` step share this cache, and releasing it to fetch would let + // three branches fetch the same asset three times — the thing the cache + // exists to prevent. Resolving is not the hot path; a race here would be. + s.assetMu.Lock() + defer s.assetMu.Unlock() + if v, ok := s.assetCache[name]; ok { return v } diff --git a/imports_test.go b/imports_test.go index 5b19be4..a013a8c 100644 --- a/imports_test.go +++ b/imports_test.go @@ -3,6 +3,7 @@ package skillengine_test import ( "go/build" "io/fs" + "os" "path/filepath" "strings" "testing" @@ -75,6 +76,19 @@ func packageDirs(t *testing.T) []string { if name := d.Name(); path != "." && (strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_")) { return fs.SkipDir } + // A directory with its own go.mod is a DIFFERENT module and not part of + // this one. That is what lets the examples show the engine embedded in + // an application built on a framework — eino appears in that module's + // go.mod and never in the engine's. + // + // The guard is not weakened by this: delete such a go.mod and the files + // join this module, the walk finds them again, and their imports are + // reported like anyone else's. + if path != "." { + if _, err := os.Stat(filepath.Join(path, "go.mod")); err == nil { + return fs.SkipDir + } + } if files, _ := filepath.Glob(filepath.Join(path, "*.go")); len(files) > 0 { dirs = append(dirs, path) } @@ -111,3 +125,36 @@ func slicesContains(list []string, v string) bool { } return false } + +// The examples must stay SEPARATE modules, and this is the guard for that. +// +// It is the only thing standing between "an example shows the engine used with +// a framework" and "the engine depends on a framework": delete one of these +// go.mod files and every import inside becomes an import of this module, which +// TestEngineStaysSelfContained would then report — but only if somebody +// remembers the example was supposed to be its own module in the first place. +// This says so out loud. +func TestExampleAppsAreSeparateModules(t *testing.T) { + entries, err := os.ReadDir("examples") + if err != nil { + t.Fatalf("reading examples: %v", err) + } + var apps int + for _, e := range entries { + if !e.IsDir() { + continue + } + dir := filepath.Join("examples", e.Name()) + files, _ := filepath.Glob(filepath.Join(dir, "*.go")) + if len(files) == 0 { + continue // examples/skills — YAML, part of no module + } + apps++ + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err != nil { + t.Errorf("%s holds Go code and no go.mod — its dependencies would become the engine's", dir) + } + } + if apps == 0 { + t.Fatal("no example application found — the walk is looking at the wrong place") + } +} diff --git a/lint/fixtures_test.go b/lint/fixtures_test.go index e05cba4..f03d155 100644 --- a/lint/fixtures_test.go +++ b/lint/fixtures_test.go @@ -125,7 +125,7 @@ func TestCatalogueIdsAreUnique(t *testing.T) { // clean — an example that trips a rule teaches the defect along with the format, // and this is also the false-positive guard for every rule at once. func TestShippedExamplesAreClean(t *testing.T) { - files, err := filepath.Glob(filepath.Join("..", "examples", "*.yaml")) + files, err := filepath.Glob(filepath.Join("..", "examples", "skills", "*.yaml")) require.NoError(t, err) require.NotEmpty(t, files, "the examples are gone") diff --git a/parallel_state_test.go b/parallel_state_test.go new file mode 100644 index 0000000..cf47fd9 --- /dev/null +++ b/parallel_state_test.go @@ -0,0 +1,212 @@ +package skillengine + +import ( + "context" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A branch of `parallel` runs in a state forked from the flow's, and everything +// the flow was given has to survive the fork. +// +// It did not. The sub-state was assembled by listing fields, and six of them +// were missing — assets, their resolver, their cache and context, working +// memory, and the application's vocabulary. Nothing failed loudly: an unknown +// asset expands to an empty string by contract, so `{{asset:x}}` inside a +// branch quietly became "", and the tool call that needed it lost a required +// argument. The error pointed at the argument, not at the substitution. +// +// It stayed hidden because no skill in a live catalogue of 29 had an asset +// inside a parallel branch — the path simply never ran. +func TestParallelBranchKeepsEverythingTheFlowHas(t *testing.T) { + f := parseFlow(t, ` +tools: ["srv"] +assets: + payload: + kind: code + source: inline + content: "print('the asset content')" +steps: + - name: fork + parallel: + branches: + - - name: uses_asset + call: + tool: "srv:run" + args: {code: "{{asset:payload}}"} + save_as: from_asset + - - name: uses_memory + instruction: "the whole of it: {{big}}" + tools: [] + save_as: from_memory +`) + + var got map[string]any + caller := ToolCallerFunc(func(_ context.Context, _, _ string, args map[string]any) (string, error) { + got = args + return "ran", nil + }) + r := &fakeRunner{answer: map[string]string{"uses_memory": "read it"}} + + _, _, err := ExecuteWith(context.Background(), f, Deps{ + Runner: r, + Caller: caller, + Assets: assetFunc(func(_ context.Context, _ string, a Asset) (string, error) { + return a.Content, nil + }), + Memory: fakeMemory{"res-1": "THE WHOLE VALUE"}, + }, map[string]string{"big": "preview…\n[mem:res-1]"}) + require.NoError(t, err) + + assert.Equal(t, "print('the asset content')", got["code"], + "the asset resolved to an empty string inside the branch, and the call lost a required argument") + assert.Contains(t, r.seen[0].Instruction, "THE WHOLE VALUE", + "working memory did not reach the branch, so the step saw a fragment") +} + +// The vocabulary is the application's words, and a branch that does not have +// them normalises an answer differently from the same step outside a branch. +func TestParallelBranchKeepsTheVocabulary(t *testing.T) { + f := parseFlow(t, ` +steps: + - name: fork + parallel: + branches: + - - name: classify + instruction: decide + one_of: [t1, foreign] + save_as: verdict + - - name: other + set: {var: x, value: "y"} +`) + r := &fakeRunner{answer: map[string]string{"classify": "Result: t1, not foreign"}} + + vars, _, err := ExecuteWith(context.Background(), f, Deps{ + Runner: r, + Vocabulary: Vocabulary{DecisionMarkers: []string{"result:"}}, + }, nil) + require.NoError(t, err) + assert.Equal(t, "t1", vars["verdict"], + "the decision markers did not reach the branch, so a tie stayed unresolved") +} + +// An asset needed by several branches is fetched ONCE: the cache is shared +// across the fork rather than cloned into it. Without a lock that sharing is a +// data race, which is why the cache has one. +func TestParallelBranchesShareTheAssetCache(t *testing.T) { + f := parseFlow(t, ` +tools: ["srv"] +assets: + payload: {kind: code, source: inline, content: "shared"} +steps: + - name: fork + parallel: + branches: + - - name: a + call: {tool: "srv:run", args: {code: "{{asset:payload}}"}, save_as: ra} + - - name: b + call: {tool: "srv:run", args: {code: "{{asset:payload}}"}, save_as: rb} + - - name: c + call: {tool: "srv:run", args: {code: "{{asset:payload}}"}, save_as: rc} +`) + var mu sync.Mutex + fetches := 0 + _, _, err := ExecuteWith(context.Background(), f, Deps{ + Caller: ToolCallerFunc(func(context.Context, string, string, map[string]any) (string, error) { + return "ok", nil + }), + Assets: assetFunc(func(_ context.Context, _ string, a Asset) (string, error) { + mu.Lock() + defer mu.Unlock() + fetches++ + return a.Content, nil + }), + }, nil) + require.NoError(t, err) + assert.Equal(t, 1, fetches, "the same asset was fetched once per branch") +} + +// What a branch must NOT inherit: the flow's collected traces and skips. A +// branch reporting the steps that ran before the fork would double-count them. +func TestParallelBranchStartsWithACleanTrace(t *testing.T) { + f := parseFlow(t, ` +steps: + - name: before + when: "missing == yes" + set: {var: a, value: "1"} + - name: fork + parallel: + branches: + - - name: x + set: {var: b, value: "2"} + - - name: y + when: "missing == yes" + set: {var: c, value: "3"} +`) + _, outcome, err := ExecuteWith(context.Background(), f, Deps{}, nil) + require.NoError(t, err) + + assert.Equal(t, []string{"before", "y"}, outcome.Skipped, + "a branch inherited the flow's skip list and reported it again") + assert.Equal(t, 1, strings.Count(strings.Join(outcome.Skipped, " "), "before")) +} + +// assetFunc — a resolver as a function, so a test does not need a type. +type assetFunc func(ctx context.Context, name string, a Asset) (string, error) + +func (f assetFunc) Resolve(ctx context.Context, name string, a Asset) (string, error) { + return f(ctx, name, a) +} + +// The one asymmetry in Outcome, pinned so it stays a documented property rather +// than an accident: Skipped includes what a `parallel` branch skipped, Steps +// does not include what a branch ran. +// +// Nothing is lost by it — branch steps reach Deps.OnStep as they happen, which +// is where per-step telemetry comes from. But the two fields disagree, and a +// reader who checks one and assumes the other is the next person to lose an +// afternoon. If this test starts failing because branch traces were merged in, +// that is a behaviour change for every embedder reading Outcome.Steps: say so +// in the CHANGELOG and update the doc comment on Outcome. +func TestOutcomeReportsBranchesOnlyThroughSkippedAndOnStep(t *testing.T) { + f := parseFlow(t, ` +steps: + - name: fork + parallel: + branches: + - - name: ran_in_branch + set: {var: a, value: "1"} + - - name: skipped_in_branch + when: "missing == yes" + set: {var: b, value: "2"} +`) + // The lock is not test hygiene — it is the contract. OnStep fires from the + // goroutine that ran the step, so inside a `parallel` it fires from several + // at once, and a callback appending to a slice without one is a data race + // in the embedder. Written here the way an embedder has to write it. + var mu sync.Mutex + var live []string + _, outcome, err := ExecuteWith(context.Background(), f, Deps{ + OnStep: func(tr StepTrace) { + mu.Lock() + defer mu.Unlock() + live = append(live, tr.Name) + }, + }, nil) + require.NoError(t, err) + + var inOutcome []string + for _, s := range outcome.Steps { + inOutcome = append(inOutcome, s.Name) + } + assert.Equal(t, []string{"fork"}, inOutcome, + "Outcome.Steps gained the branch steps — a change for everyone reading it") + assert.Contains(t, outcome.Skipped, "skipped_in_branch", + "Outcome.Skipped must include what a branch skipped") + assert.Subset(t, live, []string{"ran_in_branch", "skipped_in_branch", "fork"}, + "OnStep is where branch steps are visible, and they were not") +} diff --git a/readme_test.go b/readme_test.go new file mode 100644 index 0000000..8992640 --- /dev/null +++ b/readme_test.go @@ -0,0 +1,73 @@ +package skillengine + +import ( + "os" + "regexp" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The two READMEs are one document in two languages, and they drift the moment +// somebody edits one of them. The titles cannot be compared — that is the whole +// point of having two — so what is compared is the SHAPE: how many sections, in +// what order by kind, and how many code samples. +// +// The same guard the schema translations already have, for the same reason: a +// reader of the Russian file must not be reading a version of the library that +// no longer exists. +func TestBothReadmesHaveTheSameShape(t *testing.T) { + forEachTranslatedPair(t, "README.md", "README.ru.md") + forEachTranslatedPair(t, "QUICKSTART.md", "QUICKSTART.ru.md") +} + +func forEachTranslatedPair(t *testing.T, enPath, ruPath string) { + t.Helper() + en := mustRead(t, enPath) + ru := mustRead(t, ruPath) + + enSections, ruSections := sectionsOf(string(en)), sectionsOf(string(ru)) + require.Equal(t, len(enSections), len(ruSections), + "%s and %s have a different number of sections:\nEN: %s\nRU: %s", + enPath, ruPath, strings.Join(enSections, " | "), strings.Join(ruSections, " | ")) + + assert.Equal(t, strings.Count(string(en), "```"), strings.Count(string(ru), "```"), + "%s and %s: one of them gained or lost a code sample", enPath, ruPath) + + // The links a cold reader arrives by. A broken one in a file linked from an + // article is the cheapest possible way to lose them. + for _, doc := range []string{string(en), string(ru)} { + for _, path := range localLinks(doc) { + _, err := os.Stat(path) + assert.NoErrorf(t, err, "%s links to %s, which does not exist", enPath, path) + } + } +} + +func sectionsOf(doc string) []string { + var out []string + for _, line := range strings.Split(doc, "\n") { + if strings.HasPrefix(line, "## ") { + out = append(out, strings.TrimPrefix(line, "## ")) + } + } + return out +} + +// localLinks — markdown links pointing inside the repository, badges and +// external URLs excluded. +var linkRe = regexp.MustCompile(`\]\(([^)]+)\)`) + +func localLinks(doc string) []string { + var out []string + for _, m := range linkRe.FindAllStringSubmatch(doc, -1) { + target := m[1] + if strings.HasPrefix(target, "http") || strings.HasPrefix(target, "#") { + continue + } + out = append(out, strings.TrimSuffix(target, "/")) + } + return out +} diff --git a/skill.schema.ru.yaml b/skill.schema.ru.yaml index e61b2d7..ee42ec6 100644 --- a/skill.schema.ru.yaml +++ b/skill.schema.ru.yaml @@ -145,14 +145,14 @@ properties: стоят бюджеты вызовов, wall-clock, температура, усилие рассуждения. Схема список не закрывает — профили у каждого свои. Примеры имён и того, - что за ними может стоять, — в examples/. + что за ними может стоять, — в examples/skills/. kind: type: string description: | Род скилла — метка для приложения: по ней оно решает, как скилл подавать и исполнять. Схема значение не ограничивает; пример разделения на - самостоятельные и составные скиллы — в examples/. + самостоятельные и составные скиллы — в examples/skills/. temperature: type: number @@ -238,7 +238,7 @@ properties: Произвольные данные для приложения: расписание запуска, владелец, теги, что угодно ещё. Движок сюда не смотрит и форму не проверяет — поле существует, чтобы такие вещи не приходилось изобретать заново - каждому встраивающему. Образцы наполнения — в examples/. + каждому встраивающему. Образцы наполнения — в examples/skills/. # Явный режим обязан иметь под собой описание. Эти два правила ловят в # редакторе и в CI ровно то, на чём иначе спотыкаются на прогоне: режим @@ -992,7 +992,7 @@ $defs: description: | ЧТО это по существу. Схема значение не ограничивает — у каждого приложения свой набор; примеры («код», «справочный текст», - «конфигурация», «данные») — в examples/. + «конфигурация», «данные») — в examples/skills/. Метка вспомогательная: способ передачи выбирается МЕСТОМ использования, а не ею (см. ниже про две формы ссылки). @@ -1001,7 +1001,7 @@ $defs: type: string description: | ОТКУДА берётся содержимое. Имя источника разбирает резолвер - приложения — схема список не закрывает (примеры в examples/). + приложения — схема список не закрывает (примеры в examples/skills/). Формату важно одно: содержимое либо ЗДЕСЬ (`content`), либо ПО АДРЕСУ (`ref`). Ровно одно из двух — иначе непонятно, что @@ -1015,7 +1015,7 @@ $defs: type: string description: | Адрес содержимого — когда оно лежит не в самом скилле. Форму адреса - определяет резолвер приложения (примеры — в examples/). + определяет резолвер приложения (примеры — в examples/skills/). args: type: object @@ -1036,7 +1036,7 @@ $defs: пустым. Ровно так здесь и жил `lang`, нужный одному роду из четырёх. Движок сюда не смотрит и формы не проверяет — отдаёт резолверу - приложения как есть. Наполнение по родам — в examples/vocabulary.yaml. + приложения как есть. Наполнение по родам — в examples/skills/vocabulary.yaml. А значит, ВАЛИДАЦИЯ КЛЮЧЕЙ — ОБЯЗАННОСТЬ РЕЗОЛВЕРА, и она не факультативна. За открытость платят: `params: {langauge: python}` @@ -1050,7 +1050,7 @@ $defs: description: | Маршрут ВЫВОДА инструмента, потребившего ассет: куда деть результат вместо возврата в переменную шага. Имена маршрутов задаёт - приложение — схема их не ограничивает (примеры в examples/). + приложение — схема их не ограничивает (примеры в examples/skills/). Зачем объявлять здесь, а не решать по ходу: маршрут, который должна выбрать модель, забывается ровно тогда, когда всё остальное уже diff --git a/skill.schema.yaml b/skill.schema.yaml index b35fad2..c86517b 100644 --- a/skill.schema.yaml +++ b/skill.schema.yaml @@ -147,7 +147,7 @@ properties: temperature, reasoning effort. The schema does not close the list — everyone has their own profiles. - Example names, and what may stand behind them, are in examples/. + Example names, and what may stand behind them, are in examples/skills/. kind: type: string @@ -155,7 +155,7 @@ properties: The skill's kind — a label for the application: it decides from this how to present and execute the skill. The schema does not restrict the value; an example of splitting into standalone and composite skills is in - examples/. + examples/skills/. temperature: type: number @@ -244,7 +244,7 @@ properties: Arbitrary data for the application: a run schedule, an owner, tags, anything else. The engine does not look in here and does not check the shape — the field exists so that every embedder does not have to reinvent - such things. Sample contents are in examples/. + such things. Sample contents are in examples/skills/. # An explicit mode must have a description under it. These two rules catch in # the editor and in CI exactly what people otherwise trip over at run time: the @@ -1018,7 +1018,7 @@ $defs: description: | WHAT it essentially is. The schema does not restrict the value — every application has its own set; examples ("code", "reference text", - "configuration", "data") are in examples/. + "configuration", "data") are in examples/skills/. The label is auxiliary: the way it is passed is chosen by the PLACE of use, not by this (see the two reference forms above). @@ -1028,7 +1028,7 @@ $defs: description: | WHERE the content comes from. The source's name is interpreted by the application's resolver — the schema does not close the list (examples - are in examples/). + are in examples/skills/). The format cares about one thing: the content is either HERE (`content`) or AT AN ADDRESS (`ref`). Exactly one of the two — @@ -1043,7 +1043,7 @@ $defs: description: | The address of the content — when it does not live in the skill itself. The form of the address is defined by the application's - resolver (examples are in examples/). + resolver (examples are in examples/skills/). args: type: object @@ -1066,7 +1066,7 @@ $defs: The engine does not look in here and does not check the shape — it hands this to the application's resolver as is. Contents by kind are - in examples/vocabulary.yaml. + in examples/skills/vocabulary.yaml. Which means VALIDATING THE KEYS IS THE RESOLVER'S JOB, and it is not optional. The openness is paid for: `params: {langauge: python}` @@ -1081,7 +1081,7 @@ $defs: The route of the OUTPUT of the tool that consumed the asset: where to put the result instead of returning it into the step's variable. Route names are defined by the application — the schema does not restrict - them (examples are in examples/). + them (examples are in examples/skills/). Why declare it here rather than decide as you go: a route the model is supposed to choose gets forgotten exactly when everything else has diff --git a/skill_test.go b/skill_test.go index 7e831e9..19318c2 100644 --- a/skill_test.go +++ b/skill_test.go @@ -177,7 +177,7 @@ func TestSkillCoversEverySchemaField(t *testing.T) { // how an embedder loads them, and a header that stopped validating would go // unnoticed by a test that only looks at the steps. func TestExamplesLoadAsSkills(t *testing.T) { - files, err := filepath.Glob(filepath.Join("examples", "*.yaml")) + files, err := filepath.Glob(filepath.Join("examples", "skills", "*.yaml")) require.NoError(t, err) require.NotEmpty(t, files) diff --git a/steps.go b/steps.go index 02bf01c..c999010 100644 --- a/steps.go +++ b/steps.go @@ -559,16 +559,31 @@ func (s *state) parallelStep(ctx context.Context, step Step) (bool, error) { wg.Add(1) go func(i int, branch []Step) { defer wg.Done() - sub := &state{ - vars: maps.Clone(s.vars), - seeded: maps.Clone(s.seeded), - tools: s.tools, - runner: s.runner, - caller: s.caller, - delegate: s.delegate, - onStep: s.onStep, - onStepStart: s.onStepStart, - } + // FORKED from the flow's state, not assembled from a list of + // fields. The list was the bug: six of them were missing — the + // assets, their resolver, cache and context, working memory, and + // the application's vocabulary — so an `{{asset:x}}` inside a + // branch expanded to an empty string by contract, and the call + // that needed it lost a required argument. Nothing failed; the + // error pointed at the argument. + // + // A list has to be extended by whoever adds a field to `state`, + // and the person adding a field is not thinking about `parallel`. + // Forking inverts the default: everything reaches a branch unless + // it is explicitly reset below, and what is reset is visible in + // one place. + forked := *s + sub := &forked + // What a branch must NOT inherit: the variables are its own copy + // (the branches do not see each other's work — otherwise the + // result would depend on who finished first), and the trace, the + // skips and the answer belong to the branch alone until they are + // merged back after the join. + sub.vars = maps.Clone(s.vars) + sub.seeded = maps.Clone(s.seeded) + sub.skipped = nil + sub.traces = nil + sub.answeredBy = "" for k := range sub.vars { sub.seeded[k] = true // everything from before the fork is the branch's input } diff --git a/version.go b/version.go index ada1796..ab8c80b 100644 --- a/version.go +++ b/version.go @@ -28,7 +28,7 @@ import ( // minor — an optional field was added (a skill using it requires an engine // no older than that minor); // patch — engine fixes, the format did not change. -const EngineVersion = "2.2.2" +const EngineVersion = "2.2.3" // LegacyEngineVersion — what counts as the declared version when the field is // absent (skills written before it was introduced).