From 8fe526193094e4c22f83c246ab8c9a5716317ed2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 21 Aug 2026 15:54:31 +0200 Subject: [PATCH 1/9] SED-4885 Creating plan samples for RPA --- README.md | 26 +- automation-packages/README.md | 5 + plans/README.md | 132 ++++++ .../Demo_Data-driven.json | 0 .../Demo_Google-search.json | 0 plans/legacy-exports/README.md | 20 + plans/reference/README.md | 52 +++ plans/reference/basic-plan-syntax.yml | 67 +++ plans/reference/dynamic-values.yml | 82 ++++ .../performance-assert.yml} | 0 plans/rpa/01-linear-bot/README.md | 135 ++++++ .../rpa/01-linear-bot/automation-package.yaml | 250 ++++++++++ .../keywords/closeBackOffice.groovy | 3 + .../keywords/openBackOffice.groovy | 8 + .../01-linear-bot/keywords/readRecord.groovy | 13 + .../keywords/submitRecord.groovy | 17 + plans/rpa/02-parameterized-bot/README.md | 88 ++++ .../automation-package.yaml | 169 +++++++ .../keywords/login.groovy | 14 + .../keywords/logout.groovy | 3 + .../keywords/submitRecord.groovy | 12 + plans/rpa/03-data-driven-loops/README.md | 99 ++++ .../automation-package.yaml | 279 +++++++++++ .../rpa/03-data-driven-loops/data/records.csv | 4 + .../03-data-driven-loops/data/records.json | 4 + .../keywords/processFile.groovy | 4 + .../keywords/submitRecord.groovy | 4 + .../rpa/04-branching-and-variables/README.md | 117 +++++ .../automation-package.yaml | 269 +++++++++++ .../keywords/autoApprove.groovy | 3 + .../keywords/escalate.groovy | 3 + .../keywords/readRecord.groovy | 9 + plans/rpa/05-resilience-and-waiting/README.md | 178 ++++++++ .../automation-package.yaml | 432 ++++++++++++++++++ .../keywords/alwaysFails.groovy | 2 + .../keywords/checkQueueSize.groovy | 7 + .../keywords/flakyStep.groovy | 16 + .../keywords/login.groovy | 1 + .../keywords/logout.groovy | 3 + plans/rpa/06-session-and-scheduling/README.md | 124 +++++ .../automation-package.yaml | 187 ++++++++ .../keywords/closeBrowser.groovy | 3 + .../keywords/openBrowser.groovy | 5 + .../keywords/useBrowser.groovy | 10 + plans/rpa/07-composition-and-reuse/README.md | 92 ++++ .../automation-package.yaml | 260 +++++++++++ .../keywords/readRecord.groovy | 5 + .../keywords/reserveLicence.groovy | 2 + .../keywords/submitRecord.groovy | 4 + plans/rpa/README.md | 106 +++++ plans/yaml/README.md | 7 - plans/yaml/basic-sample-plan.yml | 26 -- plans/yaml/dynamic-values-sample-plan.yml | 28 -- 53 files changed, 3327 insertions(+), 62 deletions(-) create mode 100644 plans/README.md rename plans/{ => legacy-exports}/Demo_Data-driven.json (100%) rename plans/{ => legacy-exports}/Demo_Google-search.json (100%) create mode 100644 plans/legacy-exports/README.md create mode 100644 plans/reference/README.md create mode 100644 plans/reference/basic-plan-syntax.yml create mode 100644 plans/reference/dynamic-values.yml rename plans/{yaml/benchmark-sample-plan.yml => reference/performance-assert.yml} (100%) create mode 100644 plans/rpa/01-linear-bot/README.md create mode 100644 plans/rpa/01-linear-bot/automation-package.yaml create mode 100644 plans/rpa/01-linear-bot/keywords/closeBackOffice.groovy create mode 100644 plans/rpa/01-linear-bot/keywords/openBackOffice.groovy create mode 100644 plans/rpa/01-linear-bot/keywords/readRecord.groovy create mode 100644 plans/rpa/01-linear-bot/keywords/submitRecord.groovy create mode 100644 plans/rpa/02-parameterized-bot/README.md create mode 100644 plans/rpa/02-parameterized-bot/automation-package.yaml create mode 100644 plans/rpa/02-parameterized-bot/keywords/login.groovy create mode 100644 plans/rpa/02-parameterized-bot/keywords/logout.groovy create mode 100644 plans/rpa/02-parameterized-bot/keywords/submitRecord.groovy create mode 100644 plans/rpa/03-data-driven-loops/README.md create mode 100644 plans/rpa/03-data-driven-loops/automation-package.yaml create mode 100644 plans/rpa/03-data-driven-loops/data/records.csv create mode 100644 plans/rpa/03-data-driven-loops/data/records.json create mode 100644 plans/rpa/03-data-driven-loops/keywords/processFile.groovy create mode 100644 plans/rpa/03-data-driven-loops/keywords/submitRecord.groovy create mode 100644 plans/rpa/04-branching-and-variables/README.md create mode 100644 plans/rpa/04-branching-and-variables/automation-package.yaml create mode 100644 plans/rpa/04-branching-and-variables/keywords/autoApprove.groovy create mode 100644 plans/rpa/04-branching-and-variables/keywords/escalate.groovy create mode 100644 plans/rpa/04-branching-and-variables/keywords/readRecord.groovy create mode 100644 plans/rpa/05-resilience-and-waiting/README.md create mode 100644 plans/rpa/05-resilience-and-waiting/automation-package.yaml create mode 100644 plans/rpa/05-resilience-and-waiting/keywords/alwaysFails.groovy create mode 100644 plans/rpa/05-resilience-and-waiting/keywords/checkQueueSize.groovy create mode 100644 plans/rpa/05-resilience-and-waiting/keywords/flakyStep.groovy create mode 100644 plans/rpa/05-resilience-and-waiting/keywords/login.groovy create mode 100644 plans/rpa/05-resilience-and-waiting/keywords/logout.groovy create mode 100644 plans/rpa/06-session-and-scheduling/README.md create mode 100644 plans/rpa/06-session-and-scheduling/automation-package.yaml create mode 100644 plans/rpa/06-session-and-scheduling/keywords/closeBrowser.groovy create mode 100644 plans/rpa/06-session-and-scheduling/keywords/openBrowser.groovy create mode 100644 plans/rpa/06-session-and-scheduling/keywords/useBrowser.groovy create mode 100644 plans/rpa/07-composition-and-reuse/README.md create mode 100644 plans/rpa/07-composition-and-reuse/automation-package.yaml create mode 100644 plans/rpa/07-composition-and-reuse/keywords/readRecord.groovy create mode 100644 plans/rpa/07-composition-and-reuse/keywords/reserveLicence.groovy create mode 100644 plans/rpa/07-composition-and-reuse/keywords/submitRecord.groovy create mode 100644 plans/rpa/README.md delete mode 100644 plans/yaml/README.md delete mode 100644 plans/yaml/basic-sample-plan.yml delete mode 100644 plans/yaml/dynamic-values-sample-plan.yml diff --git a/README.md b/README.md index 15d27f2..9fcfe32 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,31 @@ Browse the **[automation-packages/README.md](automation-packages/README.md)** fo --- +## Plan samples + +The **[plans](plans/)** directory zooms in on one part of an Automation Package: the +**plan** — the control-flow tree of loops, branches, retries and keyword calls that turns +keywords into an automation that does something useful. + +The two directories answer different questions, and most people need both: + +- **[automation-packages](automation-packages/)** — *"what does a real project look like?"* + Complete, real-world blueprints for a given use case and stack: load testing with + Playwright/TypeScript, synthetic monitoring with Cypress, RPA with Selenium. Take one as + the starting point for your own project. +- **[plans](plans/)** — *"how do I express this logic in a plan?"* One plan concept per + sample, with keyword code deliberately reduced to 3-line stubs so the plan itself is the + subject. Look things up here when you are writing the plan inside your package. + +| Section | Contents | +|---------|----------| +| [plans/rpa](plans/rpa/) | Seven RPA plan samples: loops and data sources, branching, resilience, sessions, scheduling, reuse | +| [plans/reference](plans/reference/) | Standalone YAML plans illustrating the syntax — the shape of a plan, and static values vs expressions | + +Samples for functional testing, load testing and monitoring will follow the same structure. + +--- + ## Other directories These directories contain older, lower-level samples that predate Automation Packages. They remain useful as reference material for individual keywords or Step client usage. @@ -31,7 +56,6 @@ These directories contain older, lower-level samples that predate Automation Pac | Directory | Contents | |-----------|----------| | [keywords](keywords/) | Standalone keyword examples by technology (Java, .NET, Cypress, TypeScript/Playwright, JMeter, k6, Oryon, gRPC, SoapUI, …) | -| [plans](plans/) | Example Step plan files (JSON and YAML) | | [step-client](step-client/) | Sample projects using the Step Controller API (Java and REST) | | [plugins](plugins/) | Example Step plugin | | [maven-plugins](maven-plugins/) | Sample for the Step Maven upload plugin | diff --git a/automation-packages/README.md b/automation-packages/README.md index 49c710c..6f92eb8 100644 --- a/automation-packages/README.md +++ b/automation-packages/README.md @@ -82,6 +82,11 @@ level: intermediate |--------|-----------|----------|----------|-------| | [rpa-selenium](rpa-selenium/) | selenium | java | keyword-driven | intermediate | +> The sample above is a complete RPA project. For focused examples of the **plan** itself — +> one concept at a time, with the keyword code stubbed out — see [plans/rpa](../plans/rpa/): +> seven samples covering loops and data sources, branching, resilience, sessions, scheduling +> and reuse. + ### Reference | Sample | Description | Level | diff --git a/plans/README.md b/plans/README.md new file mode 100644 index 0000000..425ff59 --- /dev/null +++ b/plans/README.md @@ -0,0 +1,132 @@ +# Step plan samples + +This directory is about one thing: **how to write a Step plan**. + +A plan is the control-flow tree inside an Automation Package — the loops, branches, retries +and keyword calls that turn a handful of keywords into an automation that does something +useful. + +Every sample here is a self-contained, runnable Automation Package, so each plan can be +validated and executed rather than just read. Keyword code is deliberately reduced to +3-line stubs so the plan itself is the subject. + +### How this differs from `automation-packages/` + +[automation-packages/](../automation-packages/) holds **complete real-world blueprints** — +a full project for a given use case and stack (load testing with Playwright/TypeScript, +synthetic monitoring with Cypress, RPA with Selenium), including its build, its keywords and +its plans. That is where you go to start a project. + +This directory is the **plan-authoring reference**: one plan concept per sample, stripped of +everything else. That is where you go while writing the plan inside your package. + +## Samples by use case + +| Use case | Samples | Status | +|----------|---------|--------| +| [**RPA**](rpa/) | 7 samples — loops, branching, resilience, sessions, scheduling, reuse | available | +| Functional testing | — | planned | +| Load testing | — | planned | +| Monitoring | — | planned | + +## Reference + +For what each control does and how to configure it, see the official +[controls documentation](https://step.dev/knowledgebase/userdocs/plans/controls/). + +[reference/](reference/) holds small standalone YAML plans illustrating the syntax: + +| File | Shows | +|------|-------| +| [reference/basic-plan-syntax.yml](reference/basic-plan-syntax.yml) | The shape of a plan: root artefact, `callKeyword` with inputs, capturing an output, `if`, `assert`, `check` | +| [reference/dynamic-values.yml](reference/dynamic-values.yml) | Static values vs `expression:`, where plan variables come from, dynamic keyword names and `routing` | +| [reference/performance-assert.yml](reference/performance-assert.yml) | A `threadGroup` with a `performanceAssert` — the load-testing shape | + +## Plan formats + +Step has three plan formats: + +| Format | Written as | Used by these samples | +|--------|-----------|-----------------------| +| **YAML** | The tree of controls documented at [step.dev](https://step.dev/knowledgebase/userdocs/plans/controls/) | Yes — the whole `rpa/` set | +| **Plain text** | A compact line-based syntax, one keyword call per line | No | +| **UI** | Built in the Step plan editor, stored serialized as JSON | No — see [legacy-exports/](legacy-exports/) for what an export looks like | + +**Automation packages support YAML and plain text.** Editing an automation package's plans in +the UI is planned but not currently supported. + +### Where a YAML plan lives + +Inside an automation package, a YAML plan can be declared either way: + +```yaml +plans: # directly in the main descriptor + - name: "My plan" + root: + testCase: + children: [] + +fragments: # or pulled in from a fragment file + - "plans/my-plan.yml" +``` + +A **standalone YAML plan** — a file with a top-level `root:`, like the three in +[reference/](reference/) — is not a separate format. It is the same tree, and it can be +either incorporated into an automation package like any other plan, or created centrally in +the Step UI with **Add plan → Create from YAML**. + +Plain-text plans are declared with `plansPlainText`, each entry naming a `file`, a `name` and +a `rootType`: + +```yaml +plansPlainText: + - name: "Open the site" + file: "plans/open-site.plan" + rootType: TestCase +``` + +## Schema + +All YAML here targets Automation Package schema **1.2.0**. Any Step instance serves its own +schema at: + +``` +/rest/automation-packages/schema +``` + +Point your IDE at it to get completion and validation while editing +`automation-package.yaml`. + +## Frontmatter + +Each sample README carries the same descriptor block used across this repository (see +[automation-packages/README.md](../automation-packages/README.md)), plus `focus: plans` to +mark it as plan-authoring material rather than a technology sample: + +```yaml +--- +use-case: rpa +focus: plans +framework: none +language: groovy +target-platform: web +approach: keyword-driven +level: beginner +--- +``` + +## Running a sample + +```bash +step ap execute -p -u --token --projectName +``` + +`--includePlans` runs a subset. It is comma-separated, so plan names containing a comma +cannot be selected individually — worth avoiding when naming plans. + +`execute` runs the plans and nothing else. To register a package in a project — its plans, +keywords, **schedules** and parameters — deploy it: + +```bash +step ap deploy -p -u --token --projectName +``` diff --git a/plans/Demo_Data-driven.json b/plans/legacy-exports/Demo_Data-driven.json similarity index 100% rename from plans/Demo_Data-driven.json rename to plans/legacy-exports/Demo_Data-driven.json diff --git a/plans/Demo_Google-search.json b/plans/legacy-exports/Demo_Google-search.json similarity index 100% rename from plans/Demo_Google-search.json rename to plans/legacy-exports/Demo_Google-search.json diff --git a/plans/legacy-exports/README.md b/plans/legacy-exports/README.md new file mode 100644 index 0000000..80ec7ea --- /dev/null +++ b/plans/legacy-exports/README.md @@ -0,0 +1,20 @@ +# Legacy plan exports + +Two plans exported from the Step plan editor as JSON. + +| File | Plan | +|------|------| +| `Demo_Google-search.json` | A sequence calling an Echo keyword and asserting on its output | +| `Demo_Data-driven.json` | A data-driven plan iterating over a data source | + +Plans built in the Step plan editor are stored serialized as JSON, and these two files are +what that serialization looks like: `step.core.plans.Plan` with its `_class` discriminators, +`DynamicValue` wrappers and null fields. They are kept here as a sample of the shape, since +it is also what the plan REST API returns. + +**This format is not meant to be hand-written.** For authoring a plan, use YAML: + +- [../reference/](../reference/) — the YAML syntax reference +- [../rpa/](../rpa/) — worked, runnable samples + +See the [plans README](../README.md) for how the three plan formats relate. diff --git a/plans/reference/README.md b/plans/reference/README.md new file mode 100644 index 0000000..12993a3 --- /dev/null +++ b/plans/reference/README.md @@ -0,0 +1,52 @@ +# Plan syntax reference + +Small, self-contained YAML plans illustrating the syntax, independent of any use case. + +For the controls themselves — what each one does and how to configure it — see the official +documentation: +**[step.dev/knowledgebase/userdocs/plans/controls](https://step.dev/knowledgebase/userdocs/plans/controls/)**. + +| File | Shows | +|------|-------| +| [basic-plan-syntax.yml](basic-plan-syntax.yml) | The shape of a plan: a root artefact, `callKeyword` with inputs, capturing an output with a nested `set`, `if`, `assert` and `check` | +| [dynamic-values.yml](dynamic-values.yml) | Static values vs `expression:`, where plan variables come from, and dynamic keyword names and `routing` | +| [performance-assert.yml](performance-assert.yml) | A `threadGroup` with a `performanceAssert` — the load-testing shape | + +These files are **syntax illustrations, not runnable plans**: the keywords they call do not +exist. For plans that execute and assert their own outcome, see [../rpa/](../rpa/). + +## Standalone YAML plan files + +Each file here is a **standalone plan**: a top-level `root:` with no package around it. This +is not a separate plan format — it is the same YAML tree, just not nested inside a +descriptor. + +Such a file can be used two ways: + +- **Straight into the Step UI** — **Add plan → Create from YAML**, which creates the plan + centrally from the YAML. +- **Inside an automation package** — incorporated like any other plan. + +```yaml +version: 1.0.0 +name: "Basic plan syntax" +root: + testCase: + children: [] +``` + +Inside an `automation-package.yaml` the same tree sits one level deeper, under a named entry +in `plans:`: + +```yaml +version: "1.2.0" +name: "my-package" +plans: + - name: "Basic plan syntax" + root: + testCase: + children: [] +``` + +The node syntax is identical either way. See the [plans README](../README.md) for how the +three plan formats relate. diff --git a/plans/reference/basic-plan-syntax.yml b/plans/reference/basic-plan-syntax.yml new file mode 100644 index 0000000..47c3316 --- /dev/null +++ b/plans/reference/basic-plan-syntax.yml @@ -0,0 +1,67 @@ +# A standalone YAML plan: the same tree you would write inside an automation +# package, with a top-level `root:` and no package around it. +# +# A file like this can be dropped straight into the Step UI with +# "Add plan" > "Create from YAML", or referenced from an automation package +# like any other plan. +# +# This file is a syntax illustration, not a runnable plan - the keywords it +# calls do not exist. For plans you can execute, see ../rpa/. +--- +version: 1.0.0 +name: "Basic plan syntax" + +root: + # The root is one of: sequence, testCase, testSet, testScenario, + # threadGroup, assertionPlan. + testCase: + nodeName: "Process one record" + children: + + # A keyword call with inputs. A plain value is a static literal; + # `expression:` evaluates Groovy against the plan's variables. + - callKeyword: + keyword: "Read Record" + nodeName: "Read the record" + inputs: + - recordId: "REC-001" + children: + + # A `set` nested in the call reads the keyword's output and + # promotes the variable to the enclosing block, so later siblings + # can still see it. + - set: + key: amount + value: + expression: "output.amount as Integer" + + # `assert` checks a field of the keyword's output report, so it is + # only valid HERE - as a child of the call that produced it. + # `actual` names an output field; it is not an expression. + - assert: + actual: "status" + operator: EQUALS + expected: "PENDING" + customErrorMessage: "The record was not ready to process." + + # `if` takes a boolean expression. + - if: + nodeName: "Only large amounts need approval" + condition: + expression: "amount > 1000" + children: + - callKeyword: + keyword: "Request Approval" + inputs: + - amount: + expression: "amount" + + - echo: + text: + expression: "'Processed for ' + amount" + + # `check` evaluates an expression over PLAN VARIABLES - use it where + # `assert` would not apply, i.e. outside a keyword call. + - check: + nodeName: "An amount was read" + expression: "amount > 0" diff --git a/plans/reference/dynamic-values.yml b/plans/reference/dynamic-values.yml new file mode 100644 index 0000000..ad1eed3 --- /dev/null +++ b/plans/reference/dynamic-values.yml @@ -0,0 +1,82 @@ +# Static values vs dynamic expressions - the distinction that governs every +# field in a Step YAML plan. +# +# A file like this can be dropped straight into the Step UI with +# "Add plan" > "Create from YAML", or referenced from an automation package +# like any other plan. +# +# This file is a syntax illustration, not a runnable plan - the keywords it +# calls do not exist. For plans you can execute, see ../rpa/. +--- +version: 1.0.0 +name: "Dynamic values and inputs" + +root: + sequence: + nodeName: "Static and dynamic values" + children: + + # ------------------------------------------------------------------ + # Every value is one of two things. + # + # a plain value a STATIC literal, passed through verbatim + # expression: "..." Groovy, evaluated against the plan variables + # + # There is NO string interpolation: "${amount}" is the literal seven + # characters, not the value of `amount`. To build a string from + # variables, concatenate inside the expression. + # ------------------------------------------------------------------ + - set: + key: recordId + value: "REC-001" # the string "REC-001" + - set: + key: attempts + value: + expression: "0" # the NUMBER 0, not "0" + - set: + key: attempts + value: + expression: "attempts + 1" # arithmetic, not string work + + - echo: + text: + expression: "'Record ' + recordId + ' attempt ' + attempts" + + # ------------------------------------------------------------------ + # The same choice applies to keyword inputs - and to the keyword name + # itself, which lets one node call different keywords per iteration. + # ------------------------------------------------------------------ + - callKeyword: + keyword: "Submit Record" + inputs: + - literalInput: "abc" # static + - numericInput: 777 # static number + - fromVariable: + expression: "recordId" # a plan variable + - computed: + expression: "attempts * 10" # any Groovy + + - callKeyword: + keyword: + expression: "'Submit ' + recordId" # the keyword NAME is dynamic + # `routing` picks the agent token by attribute, rather than taking + # whichever token is free. + routing: + - role: "rpa-workstation" + + # ------------------------------------------------------------------ + # Where the values come from: + # + # output. the calling node's own output - in scope only + # inside that callKeyword's children + # previous. the preceding sibling's output; replaced by the + # next keyword call, so capture it if you need it + # later + # a `set` variable the block it was declared in, and everything + # nested below + # row / `item` the current row inside forEach / for / dataSet + # parameters execution parameters and Step `parameters`, both + # available as ordinary plan variables + # ------------------------------------------------------------------ + - check: + expression: "previous.status == 'SUBMITTED'" diff --git a/plans/yaml/benchmark-sample-plan.yml b/plans/reference/performance-assert.yml similarity index 100% rename from plans/yaml/benchmark-sample-plan.yml rename to plans/reference/performance-assert.yml diff --git a/plans/rpa/01-linear-bot/README.md b/plans/rpa/01-linear-bot/README.md new file mode 100644 index 0000000..c80313e --- /dev/null +++ b/plans/rpa/01-linear-bot/README.md @@ -0,0 +1,135 @@ +--- +use-case: rpa +focus: plans +framework: none +language: groovy +target-platform: web +approach: keyword-driven +level: beginner +--- + +# 01 — Linear bot + +The baseline shape of an RPA plan: one unattended bot run that opens an application, reads a +record, submits it, verifies the result and closes down. Every other sample in this set builds +on this structure. + +The subject of this sample is the **plan**. The keywords are 3-line Groovy stubs that simulate a +back-office application, so the package runs on any Java agent — no browser, no build, no +external system. + +## What this sample shows + +- The anatomy of an RPA plan: a `testCase` root wrapping a linear sequence of steps +- **Chaining one keyword's output into the next keyword's input** — the most-used idiom in RPA + plans, and the main lesson here +- A `session` block, so every keyword runs on the same agent and shares that context +- `echo` for a cheap audit trail in the execution report of an unattended run +- `check` for verifying plan variables, and `assert` for verifying keyword outputs +- `sleep` for asynchronous processing in the system under test — and why it is the wrong + tool for waiting on a UI + +## Chaining keyword outputs: the three bindings + +This is where most hand-written Step plans go wrong. Three bindings can carry a keyword's output +forward, and they do **not** have the same reach. + +### The recommended idiom — a `set` nested inside the `callKeyword` + +```yaml +- callKeyword: + keyword: "Read Record" + children: + - set: # child of the keyword call + key: amount + value: + expression: "output.amount" +- callKeyword: + keyword: "Submit Record" + inputs: + - amount: + expression: "amount" # still available here, and later +``` + +A `set` placed inside a keyword call is a special case: **its variable is promoted to the parent +scope**, so it stays readable by every following sibling — not just the next one. + +### The full picture + +| Binding | Where it is in scope | Use it for | +|---------|----------------------|------------| +| `output.` | **Only inside the calling node's own `children`** | Feeding a nested `set` or a nested `assert` | +| `previous.` | The **immediately preceding sibling** only — the next keyword call replaces it | A quick check right after a call | +| `expression: "myVar"` | The block the `set` belongs to, and everything nested below it | Anything that must survive further steps | + +`previous` is shown once in the plan (the `check` after *Submit the record*) and is deliberately +flagged there as the fragile option. + +### There is no string interpolation + +`"${myVar}"` is **not** expanded anywhere in this YAML. A plain string is a static value and +is passed through verbatim, so the keyword receives the literal characters `${myVar}`. This +holds in keyword inputs, `echo` text and `return` outputs alike. + +It fails silently: if the keyword ignores the input, or only checks that it is non-empty, +the plan goes green while the bot was fed nonsense. Always use +`expression:`, and concatenate to build strings: + +```yaml +text: + expression: "'Record ' + recordId + ' submitted'" +``` + +## What the plan does *not* carry: the application context + +The bot opens an application, then several keywords work in it. The browser, the driver, the +logged-in connection — none of that appears in this plan. + +It lives in the **keyword session object** instead. `Open Back Office` does: + +```groovy +session.put("appContext", driver) +``` + +and every keyword after it on the same agent token reads it back with `session.get(...)`. +The plan's only job is to wrap them in a `session` block so they share that token. + +Passing a `sessionId` from keyword to keyword as an input is a common mistake in +hand-written plans. It clutters every call, and for a real driver object it cannot work at +all — a Playwright or Selenium handle does not serialise into a plan variable. + +The rule of thumb: **the plan carries business data, the session carries technical context.** + +`Read Record` and `Submit Record` here fail with a business error if the context is +missing, so the sample proves the mechanism rather than just describing it. Sessions get +fuller treatment in [06-session-and-scheduling](../06-session-and-scheduling/). + +## `assert` vs `check` + +One distinction worth knowing before you write your first plan: + +- **`assert`** reads the **output report of a keyword**. It is only valid as a **child of a + `callKeyword`**. Its `actual` field is the *name of an output field*, not an expression. + Used as a standalone sibling it fails the execution with + `Keyword report unreachable. Asserts should be wrapped in Keyword nodes in the test plan.` +- **`check`** evaluates a Groovy expression against **plan variables**. This is what you want for + verifying a value you captured with `set`. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | The plan — heavily commented, this is what to read | +| `keywords/openBackOffice.groovy` | Parks the application context in the agent `session` | +| `keywords/readRecord.groovy` | Returns `customer`, `recordType`, `amount`, `status` | +| `keywords/submitRecord.groovy` | Returns a `confirmationId` | +| `keywords/closeBackOffice.groovy` | Cleanup | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` + +The execution report should show four passing keyword calls, four passing `Set` nodes and two +passing `Check` nodes. diff --git a/plans/rpa/01-linear-bot/automation-package.yaml b/plans/rpa/01-linear-bot/automation-package.yaml new file mode 100644 index 0000000..20b6fd1 --- /dev/null +++ b/plans/rpa/01-linear-bot/automation-package.yaml @@ -0,0 +1,250 @@ +--- +# --------------------------------------------------------------------------- +# RPA sample 01 — Linear bot +# +# The baseline shape of an RPA plan: a single unattended bot run that opens an +# application, reads a record, submits it, verifies the result and closes down. +# +# The focus of this sample is the PLAN. The keywords are 3-line Groovy stubs +# that simulate a back-office application, so the package runs on any Java +# agent with no browser, no build and no external system. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "rpa-01-linear-bot" + +plans: + - name: "Linear RPA bot" + categories: + - "RPA" + root: + # A `testCase` is the usual root for an RPA bot: one run = one business + # transaction, reported as a single test case in the execution tree. + testCase: + nodeName: "Process one record" + children: + + # ============================================================= + # The whole bot run happens inside ONE session. + # + # A `session` pins every keyword inside it to the same agent + # token. That matters because the keywords share the application + # itself: "Open Back Office" parks the application context (in a + # real bot, the Playwright or Selenium driver) in the agent's + # `session` object, and the keywords after it read it back from + # there. + # + # NOTE WHAT IS *NOT* IN THIS PLAN: no session id, no driver + # handle, no window reference is passed from keyword to keyword. + # Technical context travels through the keyword session object, + # invisibly to the plan. The plan only carries BUSINESS data - + # the record, its amount, the confirmation. + # + # Passing a session id around as a keyword input is a common + # mistake in hand-written plans: it clutters every call and it + # does not actually work for a real driver object, which cannot + # be serialised into a plan variable. + # + # Sessions get a sample of their own - see 06-session-and- + # scheduling for routing, and for where to put open/close so + # cleanup is guaranteed. + # ============================================================= + - session: + nodeName: "One agent for the whole bot run" + children: + + # ----------------------------------------------------- + # 1. Open the application. + # Its context goes into the keyword session, so there is + # nothing here for the plan to capture. + # ----------------------------------------------------- + - callKeyword: + keyword: "Open Back Office" + nodeName: "Open the back-office application" + description: "Parks the application context in the agent session." + + # ----------------------------------------------------- + # 2. Read the record. + # + # THE KEY IDIOM OF THIS SAMPLE: the nested `set`. + # + # `output.` is only in scope inside the calling + # node's own `children`. A `set` placed there reads the + # keyword output and promotes the variable to the PARENT + # scope, so it stays readable by every following sibling - + # not just the next one. + # + # This is for BUSINESS data flowing through the plan. The + # application context is not here; see the note above. + # ----------------------------------------------------- + - callKeyword: + keyword: "Read Record" + nodeName: "Read the record to process" + description: "Reads one record; its amount drives the submission below." + inputs: + - recordId: "REC-001" + children: + # Capture the fields this bot needs later on. + # + # `as Integer` is not decoration. A value read from a + # keyword output is a JsonNumber wrapper - neither a + # Groovy Integer nor a String - and passing it on + # unconverted hands the next keyword a STRING. Coerce it + # here and `amount` is a real number from then on. + - set: + key: amount + value: + expression: "output.amount as Integer" + - set: + key: customer + value: + expression: "output.customer" + # An `assert` nested under the call is the one place + # where `output.` is used directly - no `set`. + - assert: + actual: "status" + operator: EQUALS + expected: "PENDING" + customErrorMessage: "The record was not in PENDING state - nothing to submit." + + - echo: + text: + expression: "'Submitting ' + amount + ' for ' + customer" + description: "Echo writes to the execution report - cheap audit trail for an unattended bot." + + # ----------------------------------------------------- + # 3. Submit the record. + # + # This call consumes values produced two steps earlier, + # which is exactly what `previous` could NOT do (step 4). + # ----------------------------------------------------- + - callKeyword: + keyword: "Submit Record" + nodeName: "Submit the record" + inputs: + - recordId: "REC-001" + - amount: + expression: "amount" + children: + - set: + key: confirmationId + value: + expression: "output.confirmationId" + + # The keyword echoes back what it received, so this + # assert proves the VALUE arrived - not the variable + # name. Swap the input above for "${amount}" and + # this is what catches it. + - assert: + actual: "receivedAmount" + operator: EQUALS + expected: "1250" + customErrorMessage: "The amount did not arrive - check that the input uses expression:, not a static string." + + # ----------------------------------------------------- + # ASIDE - there is no string interpolation in this YAML. + # + # Every reference to a variable above uses `expression:`. + # A plain string is a STATIC value, passed through as-is: + # + # - amount: "${amount}" # WRONG - the keyword + # # receives the literal + # # 9 characters "${amount}" + # - amount: # RIGHT + # expression: "amount" + # + # This is easiest to miss when the keyword ignores the input + # or only checks that it is non-empty: the plan goes green + # while the bot has been fed nonsense. This holds in keyword + # inputs, echo text and return outputs alike. + # + # To build a message out of variables, concatenate in the + # expression: expression: "'Record ' + recordId + ' done'" + # ----------------------------------------------------- + + # ----------------------------------------------------- + # 4. `previous` - the fragile alternative. + # + # `previous.` holds the output of the PRECEDING + # SIBLING, so it is only valid immediately after that + # keyword call. The next keyword call replaces it. Fine + # for an immediate check like this one; prefer the nested + # `set` for anything that has to survive further steps. + # ----------------------------------------------------- + - check: + expression: "previous.status == 'SUBMITTED'" + nodeName: "Check the submission succeeded" + description: "Reads the output of the immediately preceding keyword call." + + # ----------------------------------------------------- + # A deliberate wait - and what it is NOT for. + # + # The back office books the submission ASYNCHRONOUSLY: + # the call above returns as soon as the form is accepted, + # and the record settles a moment later. That is a + # property of the system under test, so waiting for it + # belongs in the plan. + # + # Do NOT use sleep to wait for a UI to become ready - for + # an element to appear, a spinner to stop, a page to load. + # That belongs inside the keyword, where the automation + # library has waits that return as soon as the condition + # is met. A sleep in the plan for that is both slower (it + # always costs the full duration) and flakier (one day it + # is not long enough). + # + # Even for an async backend a fixed sleep is the crude + # option: if you can ASK the system whether it is done, + # poll instead - see sample 05, plan D2. Reach for sleep + # when there is nothing to observe, or when you are + # deliberately pacing against a rate-limited system. + # + # `releaseTokens: true` would hand the agent token back + # while waiting. Here it stays false: the application is + # open on this agent and must not be given away. + # ----------------------------------------------------- + - sleep: + duration: 500 + unit: "ms" + releaseTokens: false + nodeName: "Wait for the back office to book the record" + description: "The submission is processed asynchronously by the application, not by the UI." + + - callKeyword: + keyword: "Close Back Office" + nodeName: "Close the back-office application" + + # ------------------------------------------------------------- + # 5. Final proof that the value chained across several hops. + # `confirmationId` was set in step 3 and is still readable here. + # + # NOTE - `assert` vs `check`, a common authoring mistake: + # `assert` reads the OUTPUT REPORT of a keyword, so it is only + # valid as a CHILD of a callKeyword (see step 2). Used as a + # standalone sibling it fails the execution with + # "Keyword report unreachable. Asserts should be wrapped in + # Keyword nodes in the test plan." + # `check` evaluates a Groovy expression against plan variables + # and is the right control here. + # ------------------------------------------------------------- + - check: + expression: "confirmationId == 'CONF-REC-001'" + nodeName: "Verify the confirmation id survived the whole plan" + description: "Uses check, not assert: this reads a plan variable, not a keyword output report." + +keywords: + - GeneralScript: + name: "Open Back Office" + scriptLanguage: groovy + scriptFile: keywords/openBackOffice.groovy + - GeneralScript: + name: "Read Record" + scriptLanguage: groovy + scriptFile: keywords/readRecord.groovy + - GeneralScript: + name: "Submit Record" + scriptLanguage: groovy + scriptFile: keywords/submitRecord.groovy + - GeneralScript: + name: "Close Back Office" + scriptLanguage: groovy + scriptFile: keywords/closeBackOffice.groovy diff --git a/plans/rpa/01-linear-bot/keywords/closeBackOffice.groovy b/plans/rpa/01-linear-bot/keywords/closeBackOffice.groovy new file mode 100644 index 0000000..77d6916 --- /dev/null +++ b/plans/rpa/01-linear-bot/keywords/closeBackOffice.groovy @@ -0,0 +1,3 @@ +// Simulates closing the back-office application and clearing the context. +session.put("appContext", null) +output.add("status", "CLOSED") diff --git a/plans/rpa/01-linear-bot/keywords/openBackOffice.groovy b/plans/rpa/01-linear-bot/keywords/openBackOffice.groovy new file mode 100644 index 0000000..f19b101 --- /dev/null +++ b/plans/rpa/01-linear-bot/keywords/openBackOffice.groovy @@ -0,0 +1,8 @@ +// Simulates opening the back-office application. +// +// The application context (here a fake handle; in a real bot the Playwright +// or Selenium driver) goes into the agent SESSION - not into the output. +// Every keyword that runs on the same agent token can read it back, so the +// PLAN never has to carry a session id around. +session.put("appContext", "BACKOFFICE-" + System.currentTimeMillis()) +output.add("status", "OPEN") diff --git a/plans/rpa/01-linear-bot/keywords/readRecord.groovy b/plans/rpa/01-linear-bot/keywords/readRecord.groovy new file mode 100644 index 0000000..19ff35e --- /dev/null +++ b/plans/rpa/01-linear-bot/keywords/readRecord.groovy @@ -0,0 +1,13 @@ +// Simulates reading one record from the back-office application. +// The application context comes from the session, not from a plan variable. +def app = session.get("appContext") +if (app == null) { + output.setBusinessError("No application context - is this keyword inside the session block?") + return +} +def recordId = input.getString("recordId", "REC-001") +output.add("recordId", recordId) +output.add("customer", "ACME Corp") +output.add("recordType", "INVOICE") +output.add("amount", 1250) +output.add("status", "PENDING") diff --git a/plans/rpa/01-linear-bot/keywords/submitRecord.groovy b/plans/rpa/01-linear-bot/keywords/submitRecord.groovy new file mode 100644 index 0000000..69aba65 --- /dev/null +++ b/plans/rpa/01-linear-bot/keywords/submitRecord.groovy @@ -0,0 +1,17 @@ +// Simulates submitting the record through the back-office UI. +// +// It echoes back the amount it actually received, so the plan can assert the +// value really arrived - a static "${amount}" would show up here as the +// literal string instead of the number. +// +// `amount` arrives as a NUMBER, so it is read with getInt. getString on a +// numeric input throws ClassCastException. +def app = session.get("appContext") +if (app == null) { + output.setBusinessError("No application context - is this keyword inside the session block?") + return +} +def recordId = input.getString("recordId", "REC-001") +output.add("confirmationId", "CONF-" + recordId) +output.add("receivedAmount", input.getInt("amount", -1)) +output.add("status", "SUBMITTED") diff --git a/plans/rpa/02-parameterized-bot/README.md b/plans/rpa/02-parameterized-bot/README.md new file mode 100644 index 0000000..92a3738 --- /dev/null +++ b/plans/rpa/02-parameterized-bot/README.md @@ -0,0 +1,88 @@ +--- +use-case: rpa +focus: plans +framework: none +language: groovy +target-platform: web +approach: keyword-driven +level: beginner +--- + +# 02 — Parameterized bot (self-service RPA) + +A bot a business user launches **on demand** to perform one action, supplying the inputs at +execution time. This is the self-service RPA pattern: one plan, many callers, different +inputs each run. + +## What this sample shows + +- **Execution parameters** — values the caller chooses when starting the run +- The **defaulting idiom**, so the same plan still runs unattended with no parameters +- **Step parameters** for centrally managed values, including `protectedValue` credentials +- How a schedule pre-fills the same values for an unattended run (see + [06-session-and-scheduling](../06-session-and-scheduling/)) + +## The three ways a value reaches a plan + +| Source | Declared where | Use it for | +|--------|----------------|-----------| +| Execution parameter | Chosen at execution start (UI dialog, CLI `-ep`, Maven plugin) | What the caller decides: which record, which customer | +| Step parameter | `parameters:` in `automation-package.yaml` | Credentials, endpoints, anything centrally managed | +| Schedule parameter | `schedules[].executionParameters` | The values an unattended run starts with | + +All three end up as ordinary plan variables, read with `expression: "name"`. Note there is +**no** `${...}` interpolation in this YAML — see [01-linear-bot](../01-linear-bot/). + +## Defaulting an execution parameter + +Step declares execution parameters as plan variables **only if the caller supplied them**. +Referencing a missing one fails, so a plan that must also run unattended should default it: + +```yaml +- set: + key: recordId + value: + expression: "binding.variables.containsKey('recordId') ? recordId : 'REC-001'" +``` + +## The three calls share a session + +`Login`, `Submit Record` and `Logout` sit inside a `session` block, so they run on the same +agent token. `Login` parks the application context in the keyword `session` object and the +others read it from there — which is why **no session id is passed between them in the +plan**. See [01-linear-bot](../01-linear-bot/) for the full explanation. + +## Protected credentials + +`botPassword` is declared with `protectedValue: true`, which masks it in the UI and in +execution reports. The plan passes it into the `Login` keyword with `expression:` — the value +never appears in the plan tree. + +> The value in this sample is a dummy string so the package runs anywhere. In a real +> package, set protected parameters on the Step instance rather than committing them. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | The plan, plus the `parameters:` section | +| `keywords/login.groovy` | Fails with a business error if no password arrives; parks the app context in the agent `session` | +| `keywords/submitRecord.groovy` | Reads the context from the `session`; returns a `confirmationId` | +| `keywords/logout.groovy` | Cleanup | + +## Running it + +With defaults: + +```bash +step ap execute -p . -u --token --projectName +``` + +With execution parameters — the self-service path: + +```bash +step ap execute -p . -u --token --projectName -ep recordId=REC-042 -ep amount=7500 +``` + +In the Step UI, the same values go in the **Execution parameters** section of the execute +dialog. diff --git a/plans/rpa/02-parameterized-bot/automation-package.yaml b/plans/rpa/02-parameterized-bot/automation-package.yaml new file mode 100644 index 0000000..08e4e04 --- /dev/null +++ b/plans/rpa/02-parameterized-bot/automation-package.yaml @@ -0,0 +1,169 @@ +--- +# --------------------------------------------------------------------------- +# RPA sample 02 - Parameterized bot (self-service RPA) +# +# A bot a business user launches ON DEMAND to perform one action, supplying the +# inputs at execution time. This is the self-service RPA pattern: one plan, +# many callers, different inputs each run. +# +# It covers the three distinct ways a value reaches a plan from outside: +# 1. Execution parameters - chosen by the caller at execution start +# 2. Step parameters - centrally managed, incl. protected credentials +# 3. Schedule parameters - the same mechanism, pre-filled for an unattended +# run (see sample 06) +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "rpa-02-parameterized-bot" + +plans: + - name: "Self-service record submission" + categories: + - "RPA" + root: + testCase: + nodeName: "Submit a record on demand" + children: + + # ------------------------------------------------------------- + # 1. Defaulting execution parameters. + # + # Step declares every execution parameter as a plan variable at the + # start of the execution, so `recordId` is read like any other + # binding. But if the caller did NOT supply it, the variable does + # not exist and any expression referencing it fails. + # + # This is the defaulting idiom: ask the Groovy binding whether the + # variable exists before reading it. It makes the plan runnable with + # no parameters at all, which is what you want for a bot that also + # runs on a schedule. + # ------------------------------------------------------------- + - set: + key: recordId + value: + expression: "binding.variables.containsKey('recordId') ? recordId : 'REC-001'" + nodeName: "Default the recordId execution parameter" + description: "Falls back to REC-001 when the caller supplies nothing." + + - set: + key: amount + value: + expression: "binding.variables.containsKey('amount') ? amount : '1000'" + nodeName: "Default the amount execution parameter" + + - echo: + text: + expression: "'Bot starting for record ' + recordId + ', amount ' + amount" + + # ------------------------------------------------------------- + # 2. Step parameters, inside a session. + # + # `botUser` and `botPassword` are declared in the `parameters` + # section at the bottom of this file. They are available to the plan + # as ordinary variables, so a credential never appears in the plan + # tree. `botPassword` is declared with `protectedValue: true`, which + # masks it in the UI and in execution reports. + # + # The three calls share a `session` so they run on one agent token. + # `Login` parks the application context in the agent session and the + # later keywords read it from there - which is why NO session id is + # passed between them here. See 01-linear-bot for the full note. + # ------------------------------------------------------------- + - session: + nodeName: "One agent for the whole bot run" + children: + + - callKeyword: + keyword: "Login" + nodeName: "Log into the back office" + description: "Credentials come from protected Step parameters, not from the plan." + inputs: + - user: + expression: "botUser" + - password: + expression: "botPassword" + children: + - assert: + actual: "status" + operator: EQUALS + expected: "LOGGED_IN" + # Proves the protected parameter arrived as a value. + - assert: + actual: "loggedInAs" + operator: EQUALS + expected: "rpa-bot" + customErrorMessage: "botUser did not arrive - check the input uses expression:." + + # --------------------------------------------------- + # 3. Do the work the caller asked for. + # --------------------------------------------------- + - callKeyword: + keyword: "Submit Record" + nodeName: "Submit the requested record" + inputs: + - recordId: + expression: "recordId" + - amount: + expression: "amount" + children: + - set: + key: confirmationId + value: + expression: "output.confirmationId" + - assert: + actual: "status" + operator: EQUALS + expected: "SUBMITTED" + # Proves the execution parameter reached the keyword. + - assert: + actual: "receivedRecordId" + operator: EQUALS + expected: "REC-001" + customErrorMessage: "recordId did not arrive - with -ep recordId=... this expected value changes." + + - callKeyword: + keyword: "Logout" + nodeName: "Log out" + + # Stays INSIDE the session: `confirmationId` was promoted to + # the enclosing block by the nested `set`, and that block is + # the session body - not the test case around it. + - echo: + text: + expression: "'Record ' + recordId + ' submitted - confirmation ' + confirmationId" + description: "The value the caller cares about, surfaced in the execution report." + +# --------------------------------------------------------------------------- +# Step parameters. +# +# key the variable name the plan reads +# value the value; may be a dynamic expression +# protectedValue masks the value in the UI and in execution reports - always +# use it for credentials +# scope GLOBAL (everywhere) | APPLICATION (one app) | FUNCTION (one +# keyword). GLOBAL is the default and is what a plan variable +# needs. +# --------------------------------------------------------------------------- +parameters: + - key: "botUser" + value: "rpa-bot" + description: "The technical account the bot logs in with." + scope: GLOBAL + - key: "botPassword" + value: "s3cr3t-demo-value" + description: "Masked in the UI and in execution reports." + protectedValue: true + scope: GLOBAL + +keywords: + - GeneralScript: + name: "Login" + scriptLanguage: groovy + scriptFile: keywords/login.groovy + - GeneralScript: + name: "Submit Record" + scriptLanguage: groovy + scriptFile: keywords/submitRecord.groovy + - GeneralScript: + name: "Logout" + scriptLanguage: groovy + scriptFile: keywords/logout.groovy diff --git a/plans/rpa/02-parameterized-bot/keywords/login.groovy b/plans/rpa/02-parameterized-bot/keywords/login.groovy new file mode 100644 index 0000000..6d50f84 --- /dev/null +++ b/plans/rpa/02-parameterized-bot/keywords/login.groovy @@ -0,0 +1,14 @@ +// Simulates logging into the back-office application. +// The password arrives from a protected Step parameter - never hard-coded. +// +// The resulting application context goes into the agent SESSION, so the +// keywords that follow just find it there. The plan carries no session id. +def user = input.getString("user", "") +def password = input.getString("password", "") +if (password == null || password.isEmpty()) { + output.setBusinessError("No password supplied - check the 'botPassword' parameter.") + return +} +session.put("appContext", "BACKOFFICE-" + user) +output.add("loggedInAs", user) +output.add("status", "LOGGED_IN") diff --git a/plans/rpa/02-parameterized-bot/keywords/logout.groovy b/plans/rpa/02-parameterized-bot/keywords/logout.groovy new file mode 100644 index 0000000..f3c3c6e --- /dev/null +++ b/plans/rpa/02-parameterized-bot/keywords/logout.groovy @@ -0,0 +1,3 @@ +// Simulates closing the back-office session and clearing the context. +session.put("appContext", null) +output.add("status", "LOGGED_OUT") diff --git a/plans/rpa/02-parameterized-bot/keywords/submitRecord.groovy b/plans/rpa/02-parameterized-bot/keywords/submitRecord.groovy new file mode 100644 index 0000000..fe82272 --- /dev/null +++ b/plans/rpa/02-parameterized-bot/keywords/submitRecord.groovy @@ -0,0 +1,12 @@ +// Simulates submitting the requested record through the back-office UI. +// The application context comes from the session, not from a plan variable. +def app = session.get("appContext") +if (app == null) { + output.setBusinessError("No application context - is this keyword inside the session block?") + return +} +def recordId = input.getString("recordId", "REC-001") +output.add("confirmationId", "CONF-" + recordId) +output.add("receivedRecordId", recordId) +output.add("receivedAmount", input.getString("amount", "")) +output.add("status", "SUBMITTED") diff --git a/plans/rpa/03-data-driven-loops/README.md b/plans/rpa/03-data-driven-loops/README.md new file mode 100644 index 0000000..7575510 --- /dev/null +++ b/plans/rpa/03-data-driven-loops/README.md @@ -0,0 +1,99 @@ +--- +use-case: rpa +focus: plans +framework: none +language: groovy +target-platform: web +approach: keyword-driven +level: intermediate +--- + +# 03 — Data-driven loops + +The bread and butter of RPA: *do this for every row*. Five plans in one package, each +showing a different loop or data source, all doing the same "process each record" job. + +## What this sample shows + +- `forEach` over a data source — the main RPA loop +- Reading cells with `row.`, and renaming `row` with `item` +- Parallel bot workers with `threads`, and failure tolerance with `maxFailedLoops` +- Five of the nine data sources: `csv`, `json-array`, `sequence`, plus documented `folder` + and `sql` +- `for` — a plain counter loop +- **Writing results back into the source row** with `script` + `row.put(...)` + +## The plans + +| Plan | Shows | +|------|-------| +| A | `forEach` + `csv`, `row.` expressions | +| B | `item` to rename the row variable, `threads: 2`, `maxFailedLoops` | +| C | `json-array` and `sequence` sources; `folder` documented | +| D | `for` with `start` / `end` / `inc` | +| E | Write-back with `script` + `row.put(...)` | + +## Two things worth knowing before you copy this + +### `folder` is not a package resource + +`csv`, `excel` and `file` take a `file:` which is a **resource reference** — a +package-relative path, so the data travels with the package. `folder` takes a plain +**string path resolved on the agent's own filesystem**. It cannot point inside the package; +doing so fails with a `NullPointerException` in `step.datapool.file.FileDataPoolImpl`. + +That is right for a real drop-folder bot — the folder is normally a network share — but it +means the `folder` example in plan C is documented rather than executed. + +### Where write-back actually lands + +`row.put("Result", "OK")` inside a `script` control mutates the current row, and the new +value is readable immediately afterwards in the same iteration. Plan E executes and asserts +exactly that. + +But a data file **bundled in the automation package is not a persistent store**. With both a +packaged CSV and a packaged XLSX, a second `forEach` over the same file in the same +execution still reads the original values. + +To persist, point the data source at a **Step-managed resource** by id: + +```yaml +dataSource: + csv: + file: + id: "" # a resource uploaded to Step +``` + +…or the same with `excel:` plus `headers: true`. + +The write reaches the resource and survives the run, so a later execution reads the updated +values. Upload your file as a resource in Step and substitute its id — which is also why +this sample cannot ship a runnable version of it, since the id only exists on the instance +holding the resource. + +**Both `csv` and `excel` resources accept write-back.** + +Note that Step's documentation says *"writing to a CSV file from a dataset is not +supported"*. That sentence is about the **`dataSet`** control, not `forEach` — a CSV +resource written through `forEach` + `script` persists normally. + +For SQL, `writePKey` names the primary-key column Step uses to build the `UPDATE`. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | Five plans, one per loop / data-source pattern | +| `data/records.csv` | Input rows, with an empty `Result` column for the write-back | +| `data/records.json` | The same data as JSON | +| `keywords/submitRecord.groovy` | Returns a `confirmationId` | +| `keywords/processFile.groovy` | Used by the documented drop-folder example | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` + +To run one plan only, use `--includePlans`. Note it is **comma-separated**, so a plan whose +name contains a comma cannot be selected — worth avoiding when you name plans. diff --git a/plans/rpa/03-data-driven-loops/automation-package.yaml b/plans/rpa/03-data-driven-loops/automation-package.yaml new file mode 100644 index 0000000..8c44f8b --- /dev/null +++ b/plans/rpa/03-data-driven-loops/automation-package.yaml @@ -0,0 +1,279 @@ +--- +# --------------------------------------------------------------------------- +# RPA sample 03 - Data-driven loops +# +# The bread and butter of RPA: "do this for every row". This sample shows the +# loop controls and the data sources that feed them, plus how to WRITE RESULTS +# BACK into the source so a re-run can skip what is already done. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "rpa-03-data-driven-loops" + +plans: + + # ========================================================================= + # A. forEach over a CSV file - the most common RPA loop. + # + # Each row is exposed as the variable `row`; `row.` reads a cell. + # ========================================================================= + - name: "A - Process records from a CSV file" + categories: ["RPA"] + root: + testCase: + nodeName: "Process every row of records.csv" + children: + - forEach: + nodeName: "For each record" + description: "Iterates the CSV; one bot transaction per row." + dataSource: + csv: + file: "data/records.csv" + delimiter: "," + children: + - echo: + text: + expression: "'Processing ' + row.RecordId + ' for ' + row.Customer" + - callKeyword: + keyword: "Submit Record" + inputs: + - recordId: + expression: "row.RecordId" + - amount: + expression: "row.Amount" + + # ========================================================================= + # B. Renaming the row variable, parallel workers and failure tolerance. + # + # item renames `row` to something readable - useful when loops + # are nested and two `row` variables would collide + # threads how many rows are processed in parallel; each thread takes + # its own agent token, so this is how you scale a bot + # maxFailedLoops keep going after N failures instead of aborting the run - + # essential for an unattended batch where one bad record + # must not stop the other 999 + # ========================================================================= + - name: "B - Process records in parallel" + categories: ["RPA"] + root: + testCase: + nodeName: "Process records with 2 parallel workers" + children: + - forEach: + nodeName: "For each record (2 workers)" + item: "invoice" + threads: 2 + maxFailedLoops: 2 + dataSource: + csv: + file: "data/records.csv" + children: + - echo: + text: + expression: "'Worker processing ' + invoice.RecordId" + - callKeyword: + keyword: "Submit Record" + inputs: + - recordId: + expression: "invoice.RecordId" + + # ========================================================================= + # C. Other data sources. + # + # The loop body never changes - only the `dataSource` block does. Available + # sources: csv, excel, file, folder, gsheet, json, json-array, sequence, sql. + # ========================================================================= + - name: "C - Other data sources" + categories: ["RPA"] + root: + testCase: + nodeName: "json-array, folder and sequence sources" + children: + + # A JSON array - handy when the work list comes from an API. + - forEach: + nodeName: "For each record in a JSON array" + dataSource: + json-array: + json: '[{"RecordId":"REC-101"},{"RecordId":"REC-102"}]' + children: + - callKeyword: + keyword: "Submit Record" + inputs: + - recordId: + expression: "row.RecordId" + + # --------------------------------------------------------------- + # A drop folder - the classic "process whatever lands here" bot. + # + # NOT EXECUTED HERE, and the reason matters: + # + # `csv`, `excel` and `file` take a `file:` which is a RESOURCE + # REFERENCE - a path relative to the automation package (or a Step + # resource id), so the data travels with the package. + # + # `folder` takes a plain STRING PATH that is resolved on the AGENT's + # own filesystem. It cannot point inside the package. Pointing it at + # a package-relative path fails with a NullPointerException in + # step.datapool.file.FileDataPoolImpl, because the directory does + # not exist on the agent. + # + # That is the right design for a real drop-folder bot - the folder is + # normally a network share the agent can see - but it means the + # example below needs a path that exists on your agent: + # + # - forEach: + # nodeName: "For each file in the drop folder" + # dataSource: + # folder: + # folder: "/mnt/rpa/dropfolder" # absolute, on the agent + # children: + # - callKeyword: + # keyword: "Process File" + # inputs: + # - file: + # expression: "row.file" + # --------------------------------------------------------------- + + # An integer sequence - no external data needed. + - forEach: + nodeName: "For each number in a sequence" + item: "n" + dataSource: + sequence: + start: 1 + end: 3 + inc: 1 + children: + - echo: + text: + expression: "'Iteration ' + n" + + # ========================================================================= + # D. `for` - a plain counter loop. + # + # Use it when there is no data source at all: retry a fixed number of times, + # process N pages of a paginated back-office list, etc. + # ========================================================================= + - name: "D - Counter loop" + categories: ["RPA"] + root: + testCase: + nodeName: "Walk 3 pages of a back-office list" + children: + - for: + nodeName: "For page 1 to 3" + start: 1 + end: 3 + inc: 1 + item: "page" + children: + - echo: + text: + expression: "'Opening page ' + page" + + + # ========================================================================= + # E. Writing results BACK to the data source. + # + # The RPA closing-the-loop pattern: stamp each source row with the outcome, + # so a re-run can skip what is already done and the business gets the + # confirmation ids next to their records. + # + # THE MECHANISM - a `script` control inside the loop mutating the row map: + # + # - script: + # script: 'row.put("Result", "OK")' + # + # `row` is a plain Map, so `row.put(column, value)` sets a cell on the + # current row. That part is what this plan executes, and it passes. + # + # WHERE THE VALUES ACTUALLY LAND - read this before relying on it: + # + # * A data file BUNDLED IN THE PACKAGE (like the `data/records.csv` used + # below) is NOT a persistent store. The package is unpacked per + # execution, so `row.put` succeeds and the value is readable in the + # same iteration, but it is gone afterwards. With both a packaged CSV + # and a packaged XLSX, a second forEach over the same file in the same + # execution still reads the ORIGINAL values. + # + # * To actually persist, point the data source at a STEP-MANAGED RESOURCE + # by id instead of a package-relative path: + # + # dataSource: + # csv: + # file: + # id: "" # a Step resource + # + # ...or the same thing with `excel:` plus `headers: true`. + # + # The write reaches the resource and survives the run, so a later + # execution reads the updated values. + # + # Upload the file as a resource in Step first, then substitute its id. + # That is why this package cannot ship a runnable version of it - the + # id only exists on the instance that holds the resource. + # + # * BOTH csv and excel resources accept write-back. + # + # Step's documentation says "writing to a CSV file from a dataset is + # not supported" - note that sentence is about the `dataSet` control, + # not `forEach`. A csv resource written through `forEach` + `script` + # persists normally. + # + # A SQL source writes back through `writePKey`, which names the primary-key + # column Step uses to build the UPDATE statement: + # + # dataSource: + # sql: + # connectionString: "jdbc:postgresql://db:5432/backoffice" + # driverClass: "org.postgresql.Driver" + # user: "rpa" + # password: + # expression: "dbPassword" + # query: "SELECT id, record_id, status FROM records WHERE status = 'NEW'" + # writePKey: "id" + # ========================================================================= + - name: "E - Write results back to the data source" + categories: ["RPA"] + root: + testCase: + nodeName: "Stamp each row with its result" + children: + - forEach: + nodeName: "Process and stamp each row" + dataSource: + csv: + file: "data/records.csv" + children: + - callKeyword: + keyword: "Submit Record" + inputs: + - recordId: + expression: "row.RecordId" + children: + - set: + key: confirmationId + value: + expression: "output.confirmationId" + + # The write-back itself. + - script: + nodeName: "Write the result back into the row" + script: 'row.put("Result", "OK:" + confirmationId)' + + # Reading the value straight back proves the row was mutated. + # (Persisting it beyond the run needs a Step resource - see + # the note above.) + - check: + nodeName: "The row now carries the confirmation" + expression: "row.Result.startsWith('OK:')" + +keywords: + - GeneralScript: + name: "Submit Record" + scriptLanguage: groovy + scriptFile: keywords/submitRecord.groovy + - GeneralScript: + name: "Process File" + scriptLanguage: groovy + scriptFile: keywords/processFile.groovy diff --git a/plans/rpa/03-data-driven-loops/data/records.csv b/plans/rpa/03-data-driven-loops/data/records.csv new file mode 100644 index 0000000..5e41812 --- /dev/null +++ b/plans/rpa/03-data-driven-loops/data/records.csv @@ -0,0 +1,4 @@ +RecordId,Customer,Amount,Result +REC-001,ACME Corp,1250, +REC-002,Globex,890, +REC-003,Initech,2400, diff --git a/plans/rpa/03-data-driven-loops/data/records.json b/plans/rpa/03-data-driven-loops/data/records.json new file mode 100644 index 0000000..85766da --- /dev/null +++ b/plans/rpa/03-data-driven-loops/data/records.json @@ -0,0 +1,4 @@ +[ + { "RecordId": "REC-101", "Customer": "Umbrella", "Amount": 500 }, + { "RecordId": "REC-102", "Customer": "Soylent", "Amount": 1750 } +] diff --git a/plans/rpa/03-data-driven-loops/keywords/processFile.groovy b/plans/rpa/03-data-driven-loops/keywords/processFile.groovy new file mode 100644 index 0000000..c4d6ecf --- /dev/null +++ b/plans/rpa/03-data-driven-loops/keywords/processFile.groovy @@ -0,0 +1,4 @@ +// Simulates picking up one file from a drop folder and processing it. +def file = input.getString("file", "") +output.add("processedFile", file) +output.add("status", "PROCESSED") diff --git a/plans/rpa/03-data-driven-loops/keywords/submitRecord.groovy b/plans/rpa/03-data-driven-loops/keywords/submitRecord.groovy new file mode 100644 index 0000000..f1ba36c --- /dev/null +++ b/plans/rpa/03-data-driven-loops/keywords/submitRecord.groovy @@ -0,0 +1,4 @@ +// Simulates submitting one record through the back-office UI. +def recordId = input.getString("recordId", "UNKNOWN") +output.add("confirmationId", "CONF-" + recordId) +output.add("status", "SUBMITTED") diff --git a/plans/rpa/04-branching-and-variables/README.md b/plans/rpa/04-branching-and-variables/README.md new file mode 100644 index 0000000..0b43596 --- /dev/null +++ b/plans/rpa/04-branching-and-variables/README.md @@ -0,0 +1,117 @@ +--- +use-case: rpa +focus: plans +framework: none +language: groovy +target-platform: web +approach: keyword-driven +level: intermediate +--- + +# 04 — Branching and variables + +An approval bot that decides what to do with each record: route it by type, auto-approve +small amounts, escalate large ones. This is where an RPA plan stops being a script and +starts encoding business rules. + +## What this sample shows + +- `switch` / `case` to route by document type, and the fallback pattern for values no case + handles +- `if` for a threshold decision +- `assert` with `doNegate` and `customErrorMessage`, and `check` for plan variables +- **How `set` scoping works**, and where to declare a variable that must outlive a branch +- `skipNode` to switch a step off without deleting it + +## The plans + +| Plan | Shows | +|------|-------| +| A | `switch` / `case` per record type, plus the fallback pattern for unmatched values | +| B | `if` threshold routing, `set` scoping, `assert` with `doNegate` | +| C | `skipNode` on a temporarily disabled step | + +## Two ways `switch` silently does nothing + +Both leave the plan reporting **PASSED** while no branch ran at all. + +### 1. The expression must be dynamic + +```yaml +- switch: + expression: "recordType" # WRONG - the static string "recordType" +- switch: + expression: + expression: "recordType" # RIGHT - reads the variable +``` + +The static form matches no case, so the switch executes nothing. Same rule as everywhere +else in this YAML: a plain string is a literal, never a variable reference. + +### 2. There is no `default` case + +`value: "default"` is just a case matching the literal string `"default"`. When the +expression matches nothing, the switch runs **nothing** and passes. + +The fallback pattern — what plan A does — is to set a sentinel before the switch, have each +case overwrite it, and test it afterwards: + +```yaml +- set: {key: routed, value: "NONE"} +- switch: + expression: {expression: "recordType"} + children: + - case: {value: "INVOICE", children: [ ... , {set: {key: routed, value: "INVOICE"}}]} + # ... +- if: + condition: {expression: "routed == NONE"} + children: + - set: {key: routed, value: "MANUAL"} +``` + +Note `routed` is declared **before** the switch — a `set` living only inside a case is +scoped to that case (see `set` scoping below). + +Because both failure modes are silent, plan A ends each iteration with a `check` that the +record took the branch its type demands. Break the switch and that check goes red. + +## How `set` scoping works + +A variable declared by `set` belongs to **the block the `set` is in**. A `set` inside an +`if` is therefore *not* visible to that `if`'s siblings afterwards: + +```yaml +- if: + condition: {expression: "amount > 1000"} + children: + - set: {key: decision, value: "ESCALATED"} # scoped to the if +- check: + expression: "decision != 'UNDECIDED'" # would not see it +``` + +Declare the variable in the outer block first, then re-set it inside the branches — that is +what plan B does. + +> There is one deliberate exception: a `set` nested inside a `callKeyword` is **promoted to +> the parent scope**. That is the idiom for capturing keyword outputs — see +> [01-linear-bot](../01-linear-bot/). + +## `if` conditions must be boolean + +`if.condition` is a Groovy expression that has to evaluate to a boolean. Write a real +comparison (`amount > 1000`), not a bare string. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | Three plans covering the branching controls | +| `keywords/readRecord.groovy` | Returns `recordType`, `amount`, `status` | +| `keywords/autoApprove.groovy` | The straight-through path | +| `keywords/escalate.groovy` | The human-approval path | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` diff --git a/plans/rpa/04-branching-and-variables/automation-package.yaml b/plans/rpa/04-branching-and-variables/automation-package.yaml new file mode 100644 index 0000000..218cfc8 --- /dev/null +++ b/plans/rpa/04-branching-and-variables/automation-package.yaml @@ -0,0 +1,269 @@ +--- +# --------------------------------------------------------------------------- +# RPA sample 04 - Branching and variables +# +# An approval bot that decides what to do with each record: route it by type, +# auto-approve small amounts, escalate large ones. This is where an RPA plan +# stops being a script and starts encoding business rules. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "rpa-04-branching-and-variables" + +plans: + + # ========================================================================= + # A. switch / case - routing by record type. + # + # TWO THINGS TO GET RIGHT: + # + # 1. `switch.expression` MUST be the dynamic form. + # + # expression: "recordType" # WRONG - the static + # # string "recordType", + # # which matches no case + # expression: # RIGHT - reads the + # expression: "recordType" # variable + # + # With the static form nothing matches and the switch runs NOTHING, + # while still reporting PASSED. A silent no-op, like every other + # static-vs-dynamic mistake in this YAML. + # + # 2. THERE IS NO `default` CASE. + # + # A case with value "default" is just a case whose value is the string + # "default". When the expression matches no case, the switch executes + # nothing at all and passes. If you need a fallback, test for it AFTER + # the switch - as this plan does with the `if` below. + # + # The fourth record has a type no case handles, so it exercises exactly + # that path. + # ========================================================================= + - name: "A - Route records by type" + categories: ["RPA"] + root: + testCase: + nodeName: "Route each record to the right sub-process" + children: + - forEach: + nodeName: "For each incoming record" + dataSource: + json-array: + json: '[{"RecordId":"REC-001","Type":"INVOICE","Amount":"500"},{"RecordId":"REC-002","Type":"ORDER","Amount":"2500"},{"RecordId":"REC-003","Type":"CREDIT_NOTE","Amount":"100"},{"RecordId":"REC-004","Type":"UNKNOWN_TYPE","Amount":"75"}]' + children: + + - callKeyword: + keyword: "Read Record" + nodeName: "Read the record" + inputs: + - recordId: + expression: "row.RecordId" + - recordType: + expression: "row.Type" + - amount: + expression: "row.Amount" + children: + # Promote the fields the routing below needs. + - set: + key: recordType + value: + expression: "output.recordType" + - set: + key: amount + value: + expression: "output.amount as Integer" + + # Declared out here, before the switch, so the branches can + # write to it and the check below can still read it. A `set` + # that only exists inside a case is scoped to that case. + - set: + key: routed + value: "NONE" + nodeName: "Declare the routing outcome" + + - switch: + nodeName: "Route by record type" + expression: + expression: "recordType" + children: + - case: + value: "INVOICE" + children: + - echo: + text: + expression: "'Invoice path for ' + recordType" + - set: + key: routed + value: "INVOICE" + - case: + value: "ORDER" + children: + - echo: + text: + expression: "'Order path for ' + recordType" + - set: + key: routed + value: "ORDER" + - case: + value: "CREDIT_NOTE" + children: + - echo: + text: + expression: "'Credit note path for ' + recordType" + - set: + key: routed + value: "CREDIT_NOTE" + + # The replacement for the `default` case Step does not have: + # if nothing matched, `routed` is still NONE. + - if: + nodeName: "Nothing matched?" + condition: + expression: "routed == 'NONE'" + children: + - echo: + text: + expression: "'Unhandled type ' + recordType + ' - sending to manual review'" + - set: + key: routed + value: "MANUAL" + + # Proves the routing actually happened. If the switch silently + # matched nothing - the static-expression mistake above - every + # record would come out MANUAL and this fails. + - check: + nodeName: "The record took the branch its type demands" + expression: "recordType == 'UNKNOWN_TYPE' ? routed == 'MANUAL' : routed == recordType" + + + # ========================================================================= + # B. if - conditional execution, and how `set` scoping works. + # + # `if.condition` is a Groovy expression that must evaluate to a boolean. + # A common mistake is leaving it as a plain string: a non-empty string is + # not automatically truthy here, so write a real comparison. + # ========================================================================= + - name: "B - Approve or escalate" + categories: ["RPA"] + root: + testCase: + nodeName: "Auto-approve small amounts, escalate large ones" + children: + + - callKeyword: + keyword: "Read Record" + nodeName: "Read the record" + inputs: + - recordId: "REC-100" + - amount: "2500" + children: + - set: + key: amount + value: + expression: "output.amount as Integer" + - set: + key: recordId + value: + expression: "output.recordId" + + # --------------------------------------------------------------- + # The decision. Note the variable `decision` is declared BEFORE the + # branches, at this level. + # + # NOTE: a `set` inside an `if` belongs to the if's block and is + # NOT visible to the if's siblings afterwards. Declaring `decision` + # here first, then re-setting it inside the branches, keeps it + # readable after the branches - which is what the final check needs. + # --------------------------------------------------------------- + - set: + key: decision + value: "UNDECIDED" + nodeName: "Declare the decision variable in the outer scope" + + - if: + nodeName: "Amount over 1000?" + condition: + expression: "amount > 1000" + description: "Large amounts need a human approver." + children: + - callKeyword: + keyword: "Escalate" + inputs: + - recordId: + expression: "recordId" + children: + - assert: + actual: "decision" + operator: EQUALS + expected: "ESCALATED" + - set: + key: decision + value: "ESCALATED" + + - if: + nodeName: "Amount 1000 or less?" + condition: + expression: "amount <= 1000" + children: + - callKeyword: + keyword: "Auto Approve" + inputs: + - recordId: + expression: "recordId" + children: + # `doNegate` inverts the operator: assert that the + # decision is NOT the escalation outcome. + - assert: + actual: "decision" + operator: EQUALS + expected: "ESCALATED" + doNegate: true + customErrorMessage: "A small amount must not be escalated." + - set: + key: decision + value: "AUTO_APPROVED" + + # `check` verifies a PLAN VARIABLE (unlike `assert`, which reads a + # keyword output report and must sit inside a callKeyword). + - check: + nodeName: "A decision was reached" + expression: "decision != 'UNDECIDED'" + + - echo: + text: + expression: "'Record ' + recordId + ' (' + amount + ') -> ' + decision" + + # ========================================================================= + # C. skipNode - switching a branch off without deleting it. + # + # Handy for a step that is temporarily disabled (a downstream system under + # maintenance) or for a manual-only step during an unattended run. + # ========================================================================= + - name: "C - Temporarily disabled steps" + categories: ["RPA"] + root: + testCase: + nodeName: "Run with one step switched off" + children: + - echo: + text: "This step runs" + - callKeyword: + keyword: "Escalate" + nodeName: "Notify the approvals team (disabled)" + description: "skipNode leaves the step in the plan but does not execute it." + skipNode: true + - echo: + text: "This step runs too" + +keywords: + - GeneralScript: + name: "Read Record" + scriptLanguage: groovy + scriptFile: keywords/readRecord.groovy + - GeneralScript: + name: "Auto Approve" + scriptLanguage: groovy + scriptFile: keywords/autoApprove.groovy + - GeneralScript: + name: "Escalate" + scriptLanguage: groovy + scriptFile: keywords/escalate.groovy diff --git a/plans/rpa/04-branching-and-variables/keywords/autoApprove.groovy b/plans/rpa/04-branching-and-variables/keywords/autoApprove.groovy new file mode 100644 index 0000000..b5e3b1a --- /dev/null +++ b/plans/rpa/04-branching-and-variables/keywords/autoApprove.groovy @@ -0,0 +1,3 @@ +// Simulates the straight-through path: approve without human involvement. +output.add("decision", "AUTO_APPROVED") +output.add("recordId", input.getString("recordId", "")) diff --git a/plans/rpa/04-branching-and-variables/keywords/escalate.groovy b/plans/rpa/04-branching-and-variables/keywords/escalate.groovy new file mode 100644 index 0000000..2aea5c9 --- /dev/null +++ b/plans/rpa/04-branching-and-variables/keywords/escalate.groovy @@ -0,0 +1,3 @@ +// Simulates routing the record to a human approver. +output.add("decision", "ESCALATED") +output.add("assignedTo", "approvals-team") diff --git a/plans/rpa/04-branching-and-variables/keywords/readRecord.groovy b/plans/rpa/04-branching-and-variables/keywords/readRecord.groovy new file mode 100644 index 0000000..9ca21d8 --- /dev/null +++ b/plans/rpa/04-branching-and-variables/keywords/readRecord.groovy @@ -0,0 +1,9 @@ +// Simulates reading one record. The record type and amount drive the routing +// decisions in the plan. +def recordId = input.getString("recordId", "REC-001") +def type = input.getString("recordType", "INVOICE") +def amount = Integer.parseInt(input.getString("amount", "500")) +output.add("recordId", recordId) +output.add("recordType", type) +output.add("amount", amount) +output.add("status", "PENDING") diff --git a/plans/rpa/05-resilience-and-waiting/README.md b/plans/rpa/05-resilience-and-waiting/README.md new file mode 100644 index 0000000..55eb9f2 --- /dev/null +++ b/plans/rpa/05-resilience-and-waiting/README.md @@ -0,0 +1,178 @@ +--- +use-case: rpa +focus: plans +framework: none +language: groovy +target-platform: web +approach: keyword-driven +level: advanced +--- + +# 05 — Resilience and waiting + +The heart of unattended RPA. A bot that runs at 03:00 with nobody watching has to cope with +a slow UI, a flaky click, and an upstream job that has not finished yet — without either +giving up too early or hanging forever. + +> **Some plans in this package fail on purpose.** That is the lesson: look at which steps +> still ran after the failure. The plan names say which ones. + +## What this sample shows + +- `retryIfFails` absorbing a transient failure +- `sequence.before` / `after` for cleanup that runs even when the bot crashes +- `continueOnError` vs `continueParentNodeExecutionOnError`, side by side +- `while` vs `retryIfFails` for waiting — what actually separates them +- `failure` for aborting with your own message + +## The plans + +| Plan | Expected outcome | Shows | +|------|------------------|-------| +| A — Retry a flaky step | **PASSED** | `retryIfFails` absorbs two failures, passes on attempt 3 | +| B — Guaranteed cleanup | **FAILED** (on purpose) | `after` cleanup runs; the step after the failure does not | +| C — Error propagation flags | **FAILED** (on purpose) | Both flags, and how they combine | +| D1 — Wait with while | **PASSED** | `condition`, `postCondition`, `pacing`, `maxIterations`, `timeout` | +| D2 — Wait with retryIfFails | **PASSED** | `retryIfFails` + a nested `assert` as the not-ready signal | +| E — Explicit business failure | **TECHNICAL_ERROR** (on purpose) | `failure` with a custom message | + +## Notes on the controls + +### Calling a keyword inside `while` on auto-provisioned agents + +Step forecasts how many agents an execution needs before it starts, and that forecast does +**not** account for keyword calls nested in a `while`. On an auto-provisioned setup the loop +therefore fails with + +``` +Not able to find any agent token matching selection criteria $agenttype=default +and accepting attributes {$tokenPartition=} +``` + +**Only the forecast is affected — the construct itself is fine.** With permanent agents, or +with provisioning declared on the plan, a `while` calling a keyword works normally. So if +you want the `while` form on an auto-provisioned instance, declare the agents: + +```yaml +- name: "Poll until the queue is empty" + agents: + - replicas: 1 + pool: "" +``` + +This is a provisioning limitation, not a reason to avoid `while`. Plan D2 uses +`retryIfFails` simply because it needs no pool name and therefore runs on any instance: + +```yaml +- retryIfFails: + maxRetries: 10 + gracePeriod: 500 + timeout: 60000 + children: + - callKeyword: + keyword: "Check Queue Size" + children: + - assert: {actual: "queueSize", operator: EQUALS, expected: "0"} +``` + +### `while` or `retryIfFails` for waiting? + +Both call keywords perfectly well, and both wait. What separates them is **how you express +"not ready yet"**: + +| | `while` | `retryIfFails` | +|---|---|---| +| Exit criterion | a **condition** stops holding | the block stops **failing** | +| "Not ready" is… | a normal state | a failure you have to fabricate (usually a failing `assert`) | +| Execution report | clean — nothing failed | one failed attempt per wait (soften with `reportLastTryOnly`) | + +**Prefer `while`** when the system gives you an answer you can test — *how many items are +left?*, *is the status DONE?*. The condition reads as a condition, and a long wait does not +fill the report with failures. + +**Prefer `retryIfFails`** when the thing you are waiting on genuinely **fails** rather than +reporting a state — a call that errors until the service is up. That is retry, not polling, +and it is also what you want for absorbing flakiness (plan A). + +### Retry on a fresh session, not the same one + +Neither plan A nor plan D2 wraps its retry in a `session`, and that is deliberate. + +A retry is usually better on a **fresh** session — a new agent token, a new browser, a clean +slate. Half the reason a step is flaky is state left behind by the attempt that just failed; +retrying inside the same dirty session retries the problem along with the step. + +Wrap a retry in a session only when the retried block genuinely depends on something +established **earlier and outside it** — typically a login you do not want to redo on every +attempt. That is a deliberate trade-off, not the default. + +This has a consequence for the keywords: **keep them stateless.** Both samples count +attempts in the *plan* and pass the number in as an input: + +```yaml +- set: {key: attempt, value: {expression: "0"}} # a NUMBER, not "0" +- retryIfFails: + children: + - set: + key: attempt + value: + expression: "attempt + 1" + - callKeyword: + keyword: "Flaky Step" + inputs: + - attempt: + expression: "attempt" +``` + +Declaring the counter with `expression:` makes it a real number, so the arithmetic reads as +arithmetic. A plain `value: "0"` would be the *string* `"0"`, and every use would need +`.toInteger()` / `.toString()` around it. + +Re-setting a variable declared outside the block updates *that* variable, so the count +survives across attempts. A keyword that remembered the count itself would force every +caller to pin to one agent — which is how you end up needing a session you did not want. + +### `continueOnError` on an `after` block + +It governs whether one failing cleanup step stops the **remaining** cleanup steps — the same +"keep going" semantics as on a sequence, applied to the cleanup list. Without it, a failing +logout means the licence never gets released. + +It does **not** suppress or hide anything: both the body error and the cleanup error are +reported either way. And a failing cleanup step fails the run even when the body passed, so +keep cleanup steps defensive. + +### The two error flags are complementary, not alternatives + +- `continueOnError` on a **container** keeps the **inside** of the block going. +- `continueParentNodeExecutionOnError` on a **child** keeps the **outside** going. + +Plan C needs both: `continueOnError` lets the first sequence finish its own children, but +that sequence still ends up failed — and a failed child stops its parent. Without +`continueParentNodeExecutionOnError` on it too, the test case would stop there and the +second demo would never run. + +Neither flag hides the error. The run still reports as failed, which is usually what an RPA +batch wants: finish the remaining work, but report the failure. + +### `failure` reports TECHNICAL_ERROR + +A `failure` node aborts the run with your message attached, but the status is +`TECHNICAL_ERROR`, not `FAILED`. For a run that comes out as `FAILED`, express the rule as a +`check`, or raise `output.setBusinessError(...)` from inside a keyword. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | Six plans covering the resilience controls | +| `keywords/flakyStep.groovy` | Stateless: fails when the `attempt` input is below 3 | +| `keywords/checkQueueSize.groovy` | Stateless: returns `3 - polls`, so the queue drains as polling goes on | +| `keywords/alwaysFails.groovy` | Always raises a business error | +| `keywords/login.groovy`, `logout.groovy` | Used by the `before` / `after` cleanup demo | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` diff --git a/plans/rpa/05-resilience-and-waiting/automation-package.yaml b/plans/rpa/05-resilience-and-waiting/automation-package.yaml new file mode 100644 index 0000000..b0c1261 --- /dev/null +++ b/plans/rpa/05-resilience-and-waiting/automation-package.yaml @@ -0,0 +1,432 @@ +--- +# --------------------------------------------------------------------------- +# RPA sample 05 - Resilience and waiting +# +# The heart of unattended RPA. A bot that runs at 03:00 with nobody watching +# has to cope with a slow UI, a flaky click, and an upstream job that has not +# finished yet - without either giving up too early or hanging forever. +# +# Some plans in this package FAIL ON PURPOSE. That is the lesson: see which +# steps still run after a failure. The plan names say which ones. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "rpa-05-resilience-and-waiting" + +plans: + + # ========================================================================= + # A. retryIfFails - absorbing a transient failure. + # + # maxRetries how many attempts in total + # gracePeriod milliseconds to wait between attempts - give the UI time + # to recover instead of hammering it + # timeout give up after this many ms no matter how many attempts + # are left; the safety net against an endless retry + # reportLastTryOnly keep the report clean: only the final attempt is shown, + # so two absorbed failures do not look like real errors + # releaseTokens hand the agent token back while waiting (only meaningful + # inside a session) + # + # The keyword fails twice and succeeds on the third attempt, so this plan + # PASSES - the failures are absorbed. + # + # NOTE THERE IS NO `session` HERE, ON PURPOSE. + # + # Retries are BETTER on a fresh session: a new agent token, a new browser, a + # clean slate. Half the reason a step is flaky is state left behind by the + # attempt that just failed, and retrying in the same dirty session retries + # the problem with it. + # + # Wrap a retry in a session only when the retried block genuinely depends on + # something established EARLIER and outside it - typically a login you do + # not want to redo on every attempt. That is a deliberate trade-off, not the + # default. + # + # ========================================================================= + - name: "A - Retry a flaky step" + categories: ["RPA"] + root: + testCase: + nodeName: "Absorb a transient UI failure" + children: + + # `expression:` makes this a NUMBER. A plain "0" would be the + # string "0", and every later use would need .toInteger(). + - set: + key: attempt + value: + expression: "0" + nodeName: "Declare the attempt counter in the outer scope" + + - retryIfFails: + nodeName: "Retry the flaky step" + maxRetries: 5 + gracePeriod: 1000 + timeout: 30000 + reportLastTryOnly: true + children: + # Re-setting a variable declared outside the block updates + # THAT variable, so the count survives across attempts. + - set: + key: attempt + value: + expression: "attempt + 1" + - callKeyword: + keyword: "Flaky Step" + nodeName: "A step that fails the first two times" + inputs: + - attempt: + expression: "attempt" + + - check: + nodeName: "It took three attempts" + expression: "attempt == 3" + + # ========================================================================= + # B. Guaranteed cleanup with before / after. (INTENTIONALLY FAILS) + # + # An RPA bot that dies mid-run must still log out, close the browser and + # release the licence - otherwise the next run finds a locked session. + # + # `before` and `after` belong to the sequence, not to its children, so the + # `after` steps run even when a child failed. This is the correct place for + # bot cleanup - a plain last child would be skipped on failure. + # + # `continueOnError: true` on the after block means one failing cleanup step + # does not stop the REMAINING cleanup steps - same meaning as on a sequence. + # Close the browser even if releasing the licence blew up. + # + # With a two-step after block whose first step fails: + # with continueOnError: true -> the second cleanup step runs + # without it (the default) -> the second cleanup step is skipped + # + # It does NOT hide anything: both the body error and the cleanup error are + # reported either way, and a failing cleanup step fails the run even when + # the body passed. + # ========================================================================= + - name: "B - Guaranteed cleanup (intentionally fails)" + categories: ["RPA"] + root: + testCase: + nodeName: "The bot crashes but still logs out" + children: + - sequence: + nodeName: "Bot run with guaranteed cleanup" + before: + steps: + - callKeyword: + keyword: "Login" + nodeName: "Log in (before)" + after: + # Two cleanup steps, so continueOnError is doing something: + # if the logout failed, the second step would still run. + continueOnError: true + steps: + - callKeyword: + keyword: "Logout" + nodeName: "Log out (after) - runs even after a failure" + - echo: + text: "Session released" + nodeName: "Release the licence (after)" + children: + - echo: + text: "Doing the work..." + - callKeyword: + keyword: "Always Fails" + nodeName: "The step that breaks the bot" + - echo: + text: "This is NOT reached - the sequence stops at the first error" + + # ========================================================================= + # C. continueOnError vs continueParentNodeExecutionOnError. + # (INTENTIONALLY FAILS) + # + # These two are constantly confused. They act at different levels: + # + # continueOnError set on a CONTAINER (sequence). + # "If one of MY children fails, keep + # running my remaining children." + # + # continueParentNodeExecutionOnError set on a CHILD node. + # "If I fail, let MY PARENT carry on + # with its next sibling." + # + # Both blocks below still report as failed - neither flag hides the error, + # they only control what keeps running. That is usually what an RPA batch + # wants: finish the remaining work, but report the failure. + # + # Watch what this plan demonstrates about the difference: + # + # The first sequence carries `continueOnError`, so its own echo IS reached. + # But the sequence still ends up FAILED, and a failed child stops its + # parent - so without also setting `continueParentNodeExecutionOnError` on + # it, the testCase would stop there and the second demo below would never + # run. + # + # In other words the two flags are complementary, not alternatives: + # continueOnError keeps the INSIDE of a block going, + # continueParentNodeExecutionOnError keeps the OUTSIDE going. + # ========================================================================= + - name: "C - Error propagation flags (intentionally fails)" + categories: ["RPA"] + root: + testCase: + nodeName: "Two ways to keep going after an error" + children: + + - sequence: + nodeName: "continueOnError on the container" + description: "The failing child does not stop its siblings inside this block." + continueOnError: true + # ...and this lets the testCase move on to the second block, + # even though this sequence ends up failed. + continueParentNodeExecutionOnError: true + children: + - callKeyword: + keyword: "Always Fails" + nodeName: "Fails here" + - echo: + text: "REACHED - continueOnError let the sequence carry on" + + - sequence: + nodeName: "continueParentNodeExecutionOnError on the child" + description: "The failing child tells this sequence to carry on." + continueParentNodeExecutionOnError: true + children: + - callKeyword: + keyword: "Always Fails" + nodeName: "Fails, but lets the parent continue" + continueParentNodeExecutionOnError: true + - echo: + text: "REACHED - the child released the parent" + + - echo: + text: "End of plan" + + + # ========================================================================= + # D. Waiting for something to be ready. + # + # The RPA "wait for the upstream batch to finish" pattern. Two controls can + # express it, and they differ in HOW YOU SAY "not ready yet": + # + # `while` you evaluate a CONDITION and loop while it holds. + # "Keep going while the queue is not empty." + # Not-ready is a normal state, so the report stays clean. + # + # `retryIfFails` you re-run a block until it stops FAILING. + # "Not ready" has to be expressed as a failure - usually + # an assert that fails until the system is ready - so + # every wait shows up as failed attempts in the report + # (use reportLastTryOnly to keep it readable). + # + # Prefer `while` when the system gives you an answer you can test ("how + # many items are left?"). Prefer `retryIfFails` when the thing you are + # waiting on genuinely FAILS rather than reporting a state - a call that + # errors until the service is up - since that is retry, not polling. + # + # Both call keywords perfectly well. The one caveat, described next, + # affects `while` on auto-provisioned agents. + # + # This sample ships D1 (`while`, no keyword) and D2 (`retryIfFails`) + # because both run on any instance. The `while`-plus-keyword form needs an + # agent pool name, so it is documented below rather than executed. + # + # CALLING A KEYWORD INSIDE A `while` ON AUTO-PROVISIONED AGENTS. + # + # Step forecasts how many agents an execution needs before it starts, and + # that forecast does not account for keyword calls nested in a `while`. On + # an auto-provisioned setup the loop therefore fails with + # "Not able to find any agent token matching selection criteria + # $agenttype=default and accepting attributes {$tokenPartition=}" + # + # It is ONLY the provisioning forecast that is wrong. The construct itself + # is fine: with PERMANENT agents, or with provisioning declared explicitly + # on the plan, a `while` calling a keyword works normally. + # + # So if you want the `while` form and you are on auto-provisioning, declare + # the agents on the plan (substituting your own pool name): + # + # - name: "Poll until the queue is empty" + # agents: + # - replicas: 1 + # pool: "" + # root: + # testCase: + # children: + # - while: + # condition: + # expression: "queueSize > 0" + # maxIterations: 10 + # timeout: 60000 + # children: + # - callKeyword: + # keyword: "Check Queue Size" + # children: + # - set: + # key: queueSize + # value: + # expression: "output.queueSize as Integer" + # + # Again: this is a provisioning limitation, not a reason to avoid `while`. + # ========================================================================= + - name: "D1 - Wait with while" + categories: ["RPA"] + root: + testCase: + nodeName: "Count a backlog down to zero" + children: + + # The loop counts down a plan variable so this plan runs on any + # instance. In a real bot the body would call a keyword that asks + # the system, and `set` the answer - see the block above the plan + # for that form and its provisioning caveat. + + # A number, not the string "3" - see plan A. + - set: + key: remaining + value: + expression: "3" + nodeName: "Seed the loop variable" + + # condition evaluated BEFORE each iteration (a classic while) + # postCondition evaluated AFTER each iteration - use it for do-while, + # when the first pass is what produces the value tested + # pacing ms between iterations - do not hammer the target + # maxIterations hard cap on the number of loops + # timeout hard cap on wall-clock time; the guard that stops an + # unattended bot from hanging until someone notices + # + # Always set maxIterations AND timeout. A polling loop without both + # is how an unattended bot hangs until someone notices. + - while: + nodeName: "While there is backlog left" + condition: + expression: "remaining > 0" + pacing: 200 + maxIterations: 10 + timeout: 60000 + children: + - set: + key: remaining + value: + expression: "remaining - 1" + - echo: + text: + expression: "'Backlog is now ' + remaining" + + - check: + nodeName: "The backlog drained before the guards fired" + expression: "remaining == 0" + + - name: "D2 - Wait with retryIfFails" + categories: ["RPA"] + root: + testCase: + nodeName: "Wait for the back-office queue to empty" + children: + + # The polling idiom: retryIfFails around a keyword call whose nested + # `assert` fails while the system is not ready yet. Each failed + # assert triggers another attempt after `gracePeriod` ms, until the + # assert passes or the guards fire. + # + # No `session` here either - see plan A. Polling a system is a + # read: each poll can happen on whatever agent is free. Only wrap it + # in a session if the poll goes through a UI you already have open. + # + # Note what this costs: the two "not ready yet" polls are recorded + # as FAILED attempts, because retryIfFails only knows "failed" and + # "succeeded". `reportLastTryOnly` keeps the report readable. A + # `while` loop testing the same value reports nothing but passes - + # which is the main reason to prefer it when you have a value to + # test. + # + # The poll counter lives in the plan, so "Check Queue Size" stays a + # stateless query - which is what a real one would be. + - set: + key: polls + value: + expression: "0" + nodeName: "Declare the poll counter in the outer scope" + + - retryIfFails: + nodeName: "Poll until the queue is empty" + maxRetries: 10 + gracePeriod: 500 + timeout: 60000 + reportLastTryOnly: true + children: + - set: + key: polls + value: + expression: "polls + 1" + - callKeyword: + keyword: "Check Queue Size" + nodeName: "Ask how many items are left" + inputs: + - polls: + expression: "polls" + children: + - assert: + actual: "queueSize" + operator: EQUALS + expected: "0" + customErrorMessage: "Queue not drained yet - retrying." + + - check: + nodeName: "The queue drained within the guards" + expression: "polls == 3" + + # ========================================================================= + # E. failure - stopping on purpose. (INTENTIONALLY FAILS) + # + # Sometimes the bot works perfectly and the ANSWER is unacceptable: the + # record is in the wrong state, the total does not reconcile. `failure` + # aborts the run and puts your own message in the report, so whoever reads + # it knows the bot did its job and the DATA was wrong. + # + # Note a `failure` node reports the run as TECHNICAL_ERROR (with your + # message attached), not FAILED. If you want the + # run to come out as FAILED instead, express the rule as a `check` - a + # false `check` expression fails the node - or raise a business error from + # inside a keyword with `output.setBusinessError(...)`. + # ========================================================================= + - name: "E - Explicit business failure (intentionally fails)" + categories: ["RPA"] + root: + testCase: + nodeName: "Stop the run with a business message" + children: + - set: + key: reconciled + value: "false" + - if: + nodeName: "Did the totals reconcile?" + condition: + expression: "reconciled == 'false'" + children: + - failure: + message: "Totals did not reconcile - the bot stopped instead of booking a wrong entry." + +keywords: + - GeneralScript: + name: "Flaky Step" + scriptLanguage: groovy + scriptFile: keywords/flakyStep.groovy + - GeneralScript: + name: "Check Queue Size" + scriptLanguage: groovy + scriptFile: keywords/checkQueueSize.groovy + - GeneralScript: + name: "Always Fails" + scriptLanguage: groovy + scriptFile: keywords/alwaysFails.groovy + - GeneralScript: + name: "Login" + scriptLanguage: groovy + scriptFile: keywords/login.groovy + - GeneralScript: + name: "Logout" + scriptLanguage: groovy + scriptFile: keywords/logout.groovy diff --git a/plans/rpa/05-resilience-and-waiting/keywords/alwaysFails.groovy b/plans/rpa/05-resilience-and-waiting/keywords/alwaysFails.groovy new file mode 100644 index 0000000..ede9585 --- /dev/null +++ b/plans/rpa/05-resilience-and-waiting/keywords/alwaysFails.groovy @@ -0,0 +1,2 @@ +// Simulates a step that always fails - used to show error handling. +output.setBusinessError("This step always fails on purpose.") diff --git a/plans/rpa/05-resilience-and-waiting/keywords/checkQueueSize.groovy b/plans/rpa/05-resilience-and-waiting/keywords/checkQueueSize.groovy new file mode 100644 index 0000000..dd5f205 --- /dev/null +++ b/plans/rpa/05-resilience-and-waiting/keywords/checkQueueSize.groovy @@ -0,0 +1,7 @@ +// Simulates asking the back office how many items are still queued: the queue +// drains as polling goes on (2 -> 1 -> 0). +// +// The poll number comes IN FROM THE PLAN as a number, so it is read with +// getInt. This keyword is stateless, so the polling loop needs no session. +def polls = input.getInt("polls", 1) +output.add("queueSize", Math.max(0, 3 - polls)) diff --git a/plans/rpa/05-resilience-and-waiting/keywords/flakyStep.groovy b/plans/rpa/05-resilience-and-waiting/keywords/flakyStep.groovy new file mode 100644 index 0000000..a04c128 --- /dev/null +++ b/plans/rpa/05-resilience-and-waiting/keywords/flakyStep.groovy @@ -0,0 +1,16 @@ +// Simulates a UI step that fails on the first two attempts and succeeds on the +// third. +// +// The attempt number comes IN FROM THE PLAN as a number, so it is read with +// getInt - getString on a numeric input throws ClassCastException. +// +// This keyword holds no state of its own, which is what lets each retry run on +// a fresh agent - no session required. A keyword that remembers things between +// calls forces every caller to pin itself to one agent. +def attempt = input.getInt("attempt", 1) +output.add("attempt", attempt) +if (attempt < 3) { + output.setBusinessError("Simulated transient UI failure on attempt " + attempt) +} else { + output.add("status", "OK") +} diff --git a/plans/rpa/05-resilience-and-waiting/keywords/login.groovy b/plans/rpa/05-resilience-and-waiting/keywords/login.groovy new file mode 100644 index 0000000..d3f8a2a --- /dev/null +++ b/plans/rpa/05-resilience-and-waiting/keywords/login.groovy @@ -0,0 +1 @@ +output.add("status", "LOGGED_IN") diff --git a/plans/rpa/05-resilience-and-waiting/keywords/logout.groovy b/plans/rpa/05-resilience-and-waiting/keywords/logout.groovy new file mode 100644 index 0000000..3698a8c --- /dev/null +++ b/plans/rpa/05-resilience-and-waiting/keywords/logout.groovy @@ -0,0 +1,3 @@ +// Cleanup keyword - the point of this sample is that it runs even when the +// body of the sequence failed. +output.add("status", "LOGGED_OUT") diff --git a/plans/rpa/06-session-and-scheduling/README.md b/plans/rpa/06-session-and-scheduling/README.md new file mode 100644 index 0000000..064f511 --- /dev/null +++ b/plans/rpa/06-session-and-scheduling/README.md @@ -0,0 +1,124 @@ +--- +use-case: rpa +focus: plans +framework: none +language: groovy +target-platform: web +approach: keyword-driven +level: advanced +--- + +# 06 — Sessions and scheduling + +Two things every unattended bot needs: a **session**, so all the steps of one run land on +the same agent and can share a browser; and a **schedule**, so it runs at 03:00 without +anybody launching it. + +## What this sample shows + +- `session` pinning a whole bot run to one agent token +- Where to put open/close so cleanup is guaranteed — and where **not** to put it +- `agents` and `routing` for sending a bot to the right machine +- `schedules` with a Quartz cron, `cronExclusions` and `executionParameters` + +## Why a session + +By default each keyword call takes whatever agent token is free, so two consecutive calls +may run on **different agents**. That is fine for stateless keywords and fatal for UI +automation: the second keyword would not find the browser the first one opened. + +`session` wraps a block so every keyword inside uses the same token. Anything a keyword puts +into its `session` map is then visible to the next keyword. + +## Do not use the session's own `before` block for stateful setup + +A keyword in a **session's** `before` block runs on a **different +agent token** than the session body, so the body cannot see what `before` put into the +session. + +Put a `sequence` inside the session and use **its** `before` / `after` instead. Those steps +run within the session's token, and `after` still runs when the body fails — so cleanup is +still guaranteed: + +```yaml +- session: + children: + - sequence: + before: + steps: + - callKeyword: {keyword: "Open Browser"} + after: + continueOnError: true + steps: + - callKeyword: {keyword: "Close Browser"} + children: + - callKeyword: {keyword: "Use Browser"} +``` + +## Routing a bot to the right agent + +| Level | Attribute | Selects | +|-------|-----------|---------| +| Plan | `agents: auto_detect` or `[{image: "..."}]` | Which agent **pool** the plan runs on | +| Session / keyword | `routing:` | A token by agent **attributes** | + +`routing` is commented out in plan B because it needs agents carrying those attributes on +your instance — an unmatched criterion means no token is ever granted and the plan waits. + +## Scheduling + +```yaml +schedules: + - name: "Nightly stateful bot" + cron: "0 0 3 * * ?" + planName: "A - Stateful bot in one session" + active: false + cronExclusions: ["0 0 0-23 ? * SUN"] + executionParameters: + env: "production" +``` + +Step uses **Quartz** cron: six fields, **seconds first**. `"0 0 3 * * ?"` is 03:00 daily, +not the five-field Unix form — this is the usual trip-up. + +`executionParameters` pre-fills the same values a user would type by hand for an on-demand +run — see [02-parameterized-bot](../02-parameterized-bot/). + +### A schedule needs the package deployed, not executed + +`step ap execute` runs the plans a package contains and nothing else — it never creates the +schedule. Schedules exist only once the package is **deployed** into a Step project: + +```bash +step ap deploy -p . -u --token --projectName +``` + +So the `schedules:` block in this sample has no effect when you run the package the way the +other samples are run. To see it, deploy the package and look under **Scheduler** in the +Step UI. + +The schedule ships with `active: false`, so deploying it registers the entry without arming +a recurring job on your instance. Set it to `true` when you actually want it to fire. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | Two plans plus the `schedules:` section | +| `keywords/openBrowser.groovy` | Parks a browser handle in the agent session | +| `keywords/useBrowser.groovy` | Reads the handle back — fails if the token changed | +| `keywords/closeBrowser.groovy` | Cleanup | + +## Running it + +To run the two plans: + +```bash +step ap execute -p . -u --token --projectName +``` + +To register the package — plans, keywords **and** the schedule — in a project: + +```bash +step ap deploy -p . -u --token --projectName +``` diff --git a/plans/rpa/06-session-and-scheduling/automation-package.yaml b/plans/rpa/06-session-and-scheduling/automation-package.yaml new file mode 100644 index 0000000..8d9c275 --- /dev/null +++ b/plans/rpa/06-session-and-scheduling/automation-package.yaml @@ -0,0 +1,187 @@ +--- +# --------------------------------------------------------------------------- +# RPA sample 06 - Sessions and scheduling +# +# Two things every unattended bot needs: +# +# 1. A SESSION, so all the steps of one bot run land on the same agent and +# can share a browser, a login, or an application handle. +# 2. A SCHEDULE, so the bot runs at 03:00 without anybody launching it. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "rpa-06-session-and-scheduling" + +plans: + + # ========================================================================= + # A. session - pinning a bot run to one agent token. + # + # By default each keyword call takes whatever agent token is free, so two + # consecutive calls may run on DIFFERENT agents. That is fine for stateless + # keywords and fatal for UI automation: the second keyword would not find + # the browser the first one opened. + # + # `session` wraps a block so every keyword inside it uses the same token. + # Anything a keyword puts into its `session` map is then visible to the + # next keyword. + # + # WHERE TO PUT open/close - this bit is easy to get wrong. + # + # Do NOT use the session's own `before` / `after` blocks to open and close a + # stateful resource. A keyword in the session's + # `before` block runs on a DIFFERENT agent token than the session body, so + # the body cannot see what `before` put into the session and fails with + # "No browser in this session". + # + # Put a `sequence` INSIDE the session instead and use the sequence's + # before / after. Those steps run within the session's token, and `after` + # still runs when the body fails - so cleanup is still guaranteed. + # ========================================================================= + - name: "A - Stateful bot in one session" + categories: ["RPA"] + root: + testCase: + nodeName: "Drive one browser across several keywords" + children: + - session: + nodeName: "One agent, one browser, whole run" + description: "All keywords below share an agent token and therefore a browser." + children: + - sequence: + nodeName: "Bot run with guaranteed browser cleanup" + before: + steps: + - callKeyword: + keyword: "Open Browser" + nodeName: "Open the browser (before)" + after: + continueOnError: true + steps: + - callKeyword: + keyword: "Close Browser" + nodeName: "Close the browser (after) - always runs" + children: + + # Each of these finds the browser the `before` step opened. + - callKeyword: + keyword: "Use Browser" + nodeName: "Open the orders page" + inputs: + - page: "orders" + + - callKeyword: + keyword: "Use Browser" + nodeName: "Open the invoices page" + inputs: + - page: "invoices" + children: + - set: + key: browserHandle + value: + expression: "output.browser" + + - check: + nodeName: "The same browser served both pages" + expression: "browserHandle != null && browserHandle.startsWith('BROWSER-')" + + # ========================================================================= + # B. routing a bot to the right agent. + # + # An RPA bot usually needs a specific machine: the one with the fat client + # installed, or the one inside the customer's network. + # + # Two levels of control: + # + # plan-level `agents:` where the plan runs. `auto_detect` lets Step + # forecast what it needs; a list of entries with + # `pool` / `replicas` / `image` declares + # provisioning explicitly, e.g. + # agents: + # - replicas: 1 + # pool: "" + # + # `routing:` on a session or a keyword call + # selects a token by agent ATTRIBUTES, e.g. only + # agents tagged as RPA workstations + # + # `routing` is commented out below because it needs agents carrying those + # attributes on your instance - an unmatched criterion means no token is + # ever granted and the plan waits. + # ========================================================================= + - name: "B - Route the bot to the right agent" + categories: ["RPA"] + agents: auto_detect + root: + testCase: + nodeName: "Run on an agent that can do RPA" + children: + - session: + nodeName: "Session on a selected agent" + # routing: + # - role: "rpa-workstation" + # - os: "windows" + children: + - sequence: + before: + steps: + - callKeyword: + keyword: "Open Browser" + after: + continueOnError: true + steps: + - callKeyword: + keyword: "Close Browser" + children: + - callKeyword: + keyword: "Use Browser" + inputs: + - page: "home" + +# --------------------------------------------------------------------------- +# Schedules - running the bot unattended. +# +# A schedule is created when the package is DEPLOYED, not when it is executed: +# +# step ap deploy -> registers plans, keywords and schedules in a project +# step ap execute -> runs the plans, and nothing else +# +# So this block has no effect if you only ever run the package with +# `step ap execute`, which is how the other samples in this set are run. +# +# name what the schedule is called in the Step UI +# cron a QUARTZ expression - SIX fields, seconds first. This is +# the usual trip-up: "0 0 3 * * ?" is 03:00 every day, NOT +# the five-field Unix form. +# planName the plan to run; must match a `name` above exactly +# active false ships the schedule switched off, which is what you +# want for a sample - flip it to true to arm it +# cronExclusions windows where the bot must NOT run (maintenance, holidays) +# executionParameters +# the values the run starts with - the same mechanism a user +# fills in by hand for an on-demand run (see sample 02) +# --------------------------------------------------------------------------- +schedules: + - name: "Nightly stateful bot" + cron: "0 0 3 * * ?" + planName: "A - Stateful bot in one session" + active: false + cronExclusions: + # No runs on Sunday - the back office is down for maintenance. + - "0 0 0-23 ? * SUN" + executionParameters: + env: "production" + recordId: "REC-NIGHTLY" + +keywords: + - GeneralScript: + name: "Open Browser" + scriptLanguage: groovy + scriptFile: keywords/openBrowser.groovy + - GeneralScript: + name: "Use Browser" + scriptLanguage: groovy + scriptFile: keywords/useBrowser.groovy + - GeneralScript: + name: "Close Browser" + scriptLanguage: groovy + scriptFile: keywords/closeBrowser.groovy diff --git a/plans/rpa/06-session-and-scheduling/keywords/closeBrowser.groovy b/plans/rpa/06-session-and-scheduling/keywords/closeBrowser.groovy new file mode 100644 index 0000000..034a7b5 --- /dev/null +++ b/plans/rpa/06-session-and-scheduling/keywords/closeBrowser.groovy @@ -0,0 +1,3 @@ +// Simulates closing the browser and clearing the session handle. +session.put("browser", null) +output.add("status", "CLOSED") diff --git a/plans/rpa/06-session-and-scheduling/keywords/openBrowser.groovy b/plans/rpa/06-session-and-scheduling/keywords/openBrowser.groovy new file mode 100644 index 0000000..1028b9e --- /dev/null +++ b/plans/rpa/06-session-and-scheduling/keywords/openBrowser.groovy @@ -0,0 +1,5 @@ +// Simulates opening a browser and parking the handle in the AGENT SESSION. +// Anything put in `session` lives on the agent token, not in the plan. +def handle = "BROWSER-" + System.currentTimeMillis() +session.put("browser", handle) +output.add("browser", handle) diff --git a/plans/rpa/06-session-and-scheduling/keywords/useBrowser.groovy b/plans/rpa/06-session-and-scheduling/keywords/useBrowser.groovy new file mode 100644 index 0000000..8e3f893 --- /dev/null +++ b/plans/rpa/06-session-and-scheduling/keywords/useBrowser.groovy @@ -0,0 +1,10 @@ +// Reads the browser handle back out of the agent session. +// This only works if this keyword ran on the SAME agent token as the one that +// opened the browser - which is exactly what the session control guarantees. +def handle = session.get("browser") +if (handle == null) { + output.setBusinessError("No browser in this session - the keyword landed on a different agent token.") + return +} +output.add("browser", handle) +output.add("page", input.getString("page", "home")) diff --git a/plans/rpa/07-composition-and-reuse/README.md b/plans/rpa/07-composition-and-reuse/README.md new file mode 100644 index 0000000..c006968 --- /dev/null +++ b/plans/rpa/07-composition-and-reuse/README.md @@ -0,0 +1,92 @@ +--- +use-case: rpa +focus: plans +framework: none +language: groovy +target-platform: web +approach: keyword-driven +level: advanced +--- + +# 07 — Composition and reuse + +Once you have more than one bot, the same sub-process shows up everywhere: *log in*, *look +up a customer*, *book an entry*. This sample shows the three ways Step lets you package a +piece of plan once and call it from many bots. + +## What this sample shows + +- A **Composite keyword** — a plan that behaves like a keyword, with inputs and outputs +- `callPlan` to delegate to another whole plan +- `testSet` to run several bots as one orchestrated batch +- `synchronized` to serialise access to a single shared licence + +## The three reuse mechanisms + +| Mechanism | Returns outputs? | Use it when | +|-----------|------------------|-------------| +| **Composite keyword** | Yes, via `return` | The thing you are reusing is a *step* inside someone else's plan. **Reach for this first** | +| `callPlan` | No — it produces its own report branch | The thing you are reusing is a whole bot in its own right, with its own schedule | +| `testSet` | n/a | You want to run several independent bots as one batch | + +## The Composite keyword + +A Composite's implementation is a **plan**, declared under `keywords:`: + +```yaml +keywords: + - Composite: + name: "Process One Record" + plan: + root: + sequence: # a sequence, not a testCase + children: + - callKeyword: + keyword: "Read Record" + inputs: + - recordId: + expression: "input.recordId" # the composite's own inputs + # ... + - return: + output: + - confirmationId: + expression: "confirmationId" +``` + +The caller invokes it exactly like any other keyword, and reads its `return` values with +`output.*`. + +Two details that are easy to get wrong: + +- The root is a **`sequence`**, not a `testCase` — a composite is a step inside someone + else's plan, not a test case of its own. +- `return.output` values need `expression:`, like every other value in this YAML. There is + no string interpolation — `"${confirmationId}"` would hand the caller that literal string. + See [01-linear-bot](../01-linear-bot/). + +## `forEach threads` vs `testSet threads` + +- `threads` on a **loop** parallelises **rows** of one bot. +- `threads` on a **`testSet`** parallelises **whole bots**. + +## A plan that is called is still a plan + +`Shared - Archive processed records` exists to be called by plan B, but nothing marks it as +"a sub-plan" — so it is also executed on its own when the whole package runs. That is why it +defaults its input: a plan meant to be called by others should still stand up when launched +directly. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | Four plans plus the Composite keyword definition | +| `keywords/readRecord.groovy` | Returns `amount`, `status` | +| `keywords/submitRecord.groovy` | Returns a `confirmationId` | +| `keywords/reserveLicence.groovy` | Stands in for the single-licence legacy app | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` diff --git a/plans/rpa/07-composition-and-reuse/automation-package.yaml b/plans/rpa/07-composition-and-reuse/automation-package.yaml new file mode 100644 index 0000000..b7cd6ba --- /dev/null +++ b/plans/rpa/07-composition-and-reuse/automation-package.yaml @@ -0,0 +1,260 @@ +--- +# --------------------------------------------------------------------------- +# RPA sample 07 - Composition and reuse +# +# Once you have more than one bot, the same sub-process shows up everywhere: +# "log in", "look up a customer", "book an entry". This sample shows the three +# ways Step lets you package a piece of plan once and call it from many bots. +# +# Composite keyword a plan that behaves like a keyword - inputs, outputs, +# and it shows up in the keyword list. The cleanest +# reuse mechanism. +# callPlan call another plan by name +# testSet run several bot plans as one orchestrated batch +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "rpa-07-composition-and-reuse" + +plans: + + # ========================================================================= + # A. Calling a composite keyword. + # + # "Process One Record" is declared at the bottom of this file as a Composite + # keyword: its implementation is a PLAN, not code. From here it is called + # exactly like any other keyword - inputs go in, `return` sends outputs back. + # + # This is the reuse mechanism to reach for first: the caller does not need to + # know the sub-process has five steps in it. + # ========================================================================= + - name: "A - Use a composite keyword as a sub-bot" + categories: ["RPA"] + root: + testCase: + nodeName: "Process records through a reusable sub-bot" + children: + - forEach: + nodeName: "For each record" + dataSource: + json-array: + json: '[{"RecordId":"REC-001"},{"RecordId":"REC-002"}]' + children: + - callKeyword: + keyword: "Process One Record" + nodeName: "Run the reusable sub-bot" + inputs: + - recordId: + expression: "row.RecordId" + children: + # The composite's `return` outputs are read exactly like + # any other keyword's outputs. + - set: + key: confirmationId + value: + expression: "output.confirmationId" + - assert: + actual: "confirmationId" + operator: BEGINS_WITH + expected: "CONF-" + + - echo: + text: + expression: "'Sub-bot returned ' + confirmationId" + + # ========================================================================= + # B. callPlan - invoking another plan by name. + # + # Use this when the thing you want to reuse is a whole bot in its own right + # (it has its own schedule, its own report), rather than a step inside one. + # + # `selectionAttributes` picks the target plan by its attributes - `name` is + # the usual one. `input` passes values in. + # + # Unlike a composite keyword, a called plan does NOT return outputs to the + # caller; it produces its own branch in the execution report. + # ========================================================================= + - name: "B - Call another plan" + categories: ["RPA"] + root: + testCase: + nodeName: "Delegate to a standalone bot plan" + children: + - callPlan: + nodeName: "Run the shared cleanup bot" + selectionAttributes: + - name: "Shared - Archive processed records" + input: + - archiveFolder: "/archive/2026-08" + + # ------------------------------------------------------------------------- + # The plan that B calls. An ordinary plan - nothing marks it as "a sub-plan", + # which means it is also executed on its own when the whole package runs. + # + # That is why it defaults its input: a plan meant to be called by others + # should still stand up when launched directly. + # ------------------------------------------------------------------------- + - name: "Shared - Archive processed records" + categories: ["RPA"] + root: + testCase: + nodeName: "Archive what the bots processed" + children: + - set: + key: archiveFolder + value: + expression: "binding.variables.containsKey('archiveFolder') ? archiveFolder : '/archive/default'" + - echo: + text: + expression: "'Archiving into ' + archiveFolder" + description: "Reads the value the caller passed through callPlan input." + + # ========================================================================= + # C. testSet - orchestrating several bots as one batch. + # + # A `testSet` runs its children (usually testCases or callPlans) as separate + # test cases, `threads` of them at a time. This is the "nightly RPA batch" + # shape: five independent bots, two agents, one report. + # + # Note the difference from `forEach threads`: forEach parallelises ROWS of + # one bot, testSet parallelises WHOLE BOTS. + # ========================================================================= + - name: "C - Nightly batch of bots" + categories: ["RPA"] + root: + testSet: + nodeName: "Run the nightly bots" + threads: 2 + children: + - testCase: + nodeName: "Bot 1 - invoices" + children: + - callKeyword: + keyword: "Process One Record" + inputs: + - recordId: "INV-001" + - testCase: + nodeName: "Bot 2 - orders" + children: + - callKeyword: + keyword: "Process One Record" + inputs: + - recordId: "ORD-001" + - testCase: + nodeName: "Bot 3 - credit notes" + children: + - callKeyword: + keyword: "Process One Record" + inputs: + - recordId: "CRN-001" + + # ========================================================================= + # D. synchronized - serialising access to a shared resource. + # + # The classic RPA constraint: the legacy application has ONE floating + # licence, or the back office refuses two concurrent sessions for the same + # user. Parallel bots must queue up for it. + # + # `synchronized` lets only one thread at a time into the block. + # lockName the name of the lock; blocks sharing a name share the queue + # globalLock true serialises across the WHOLE controller (every running + # execution), not just within this one + # + # Here three parallel workers contend for a single licence. + # ========================================================================= + - name: "D - Queue for a single licence" + categories: ["RPA"] + root: + testCase: + nodeName: "Three bots, one licence" + children: + - forEach: + nodeName: "For each record (3 workers)" + threads: 3 + dataSource: + json-array: + json: '[{"RecordId":"REC-001"},{"RecordId":"REC-002"},{"RecordId":"REC-003"}]' + children: + - synchronized: + nodeName: "Only one bot in the legacy app at a time" + lockName: "legacy-app-licence" + globalLock: false + children: + - callKeyword: + keyword: "Reserve Licence" + - callKeyword: + keyword: "Submit Record" + inputs: + - recordId: + expression: "row.RecordId" + +keywords: + + # ------------------------------------------------------------------------- + # A COMPOSITE KEYWORD - a plan wearing a keyword's clothes. + # + # `plan.root` is an ordinary plan tree. Inputs arrive as plan variables + # (`input.recordId`), and `return.output` hands values back to the caller, + # where they appear as normal keyword outputs. + # + # Note the root here is a `sequence`, not a testCase: a composite is a step + # inside someone else's plan, not a test case of its own. + # ------------------------------------------------------------------------- + - Composite: + name: "Process One Record" + description: "Reads a record and submits it. Reused by every bot in this package." + plan: + root: + sequence: + nodeName: "Process one record" + children: + + - callKeyword: + keyword: "Read Record" + inputs: + # The composite's own inputs arrive under `input`. + - recordId: + expression: "input.recordId" + children: + - set: + key: amount + value: + expression: "output.amount as Integer" + + - callKeyword: + keyword: "Submit Record" + inputs: + - recordId: + expression: "input.recordId" + - amount: + expression: "amount" + children: + - set: + key: confirmationId + value: + expression: "output.confirmationId" + + # `return` is what makes this a keyword rather than a plan: + # these become the outputs the caller reads with `output.*`. + # + # NOTE: these MUST use `expression:`. A plain string is a + # static value - "${confirmationId}" would hand the caller the + # literal seven-character string, not the id. + - return: + output: + - confirmationId: + expression: "confirmationId" + - amount: + expression: "amount" + + - GeneralScript: + name: "Read Record" + scriptLanguage: groovy + scriptFile: keywords/readRecord.groovy + - GeneralScript: + name: "Submit Record" + scriptLanguage: groovy + scriptFile: keywords/submitRecord.groovy + - GeneralScript: + name: "Reserve Licence" + scriptLanguage: groovy + scriptFile: keywords/reserveLicence.groovy diff --git a/plans/rpa/07-composition-and-reuse/keywords/readRecord.groovy b/plans/rpa/07-composition-and-reuse/keywords/readRecord.groovy new file mode 100644 index 0000000..fddcf7f --- /dev/null +++ b/plans/rpa/07-composition-and-reuse/keywords/readRecord.groovy @@ -0,0 +1,5 @@ +// Simulates reading one record from the back-office application. +def recordId = input.getString("recordId", "REC-001") +output.add("recordId", recordId) +output.add("amount", 1250) +output.add("status", "PENDING") diff --git a/plans/rpa/07-composition-and-reuse/keywords/reserveLicence.groovy b/plans/rpa/07-composition-and-reuse/keywords/reserveLicence.groovy new file mode 100644 index 0000000..69cc2d5 --- /dev/null +++ b/plans/rpa/07-composition-and-reuse/keywords/reserveLicence.groovy @@ -0,0 +1,2 @@ +// Simulates taking the single floating licence of a legacy application. +output.add("licence", "LIC-1") diff --git a/plans/rpa/07-composition-and-reuse/keywords/submitRecord.groovy b/plans/rpa/07-composition-and-reuse/keywords/submitRecord.groovy new file mode 100644 index 0000000..93af72c --- /dev/null +++ b/plans/rpa/07-composition-and-reuse/keywords/submitRecord.groovy @@ -0,0 +1,4 @@ +// Simulates submitting one record. +def recordId = input.getString("recordId", "REC-001") +output.add("confirmationId", "CONF-" + recordId) +output.add("status", "SUBMITTED") diff --git a/plans/rpa/README.md b/plans/rpa/README.md new file mode 100644 index 0000000..20c531d --- /dev/null +++ b/plans/rpa/README.md @@ -0,0 +1,106 @@ +# RPA plan samples + +Seven small Automation Packages, each teaching one aspect of writing a **Step plan** for +RPA. Read them in order — each builds on the one before. + +The subject is the **plan**, not the keywords. Every keyword here is a 3-line Groovy +`GeneralScript` stub simulating a back-office application, so every package runs on any Java +agent with **no build, no browser and no external system**. Every plan in the set is +executable, and each one asserts its own outcome rather than merely running. + +## The samples + +| # | Sample | Level | Controls covered | +|---|--------|-------|------------------| +| 01 | [Linear bot](01-linear-bot/) | beginner | `testCase`, `session`, `callKeyword`, `set`, `echo`, `assert`, `check`, `sleep` | +| 02 | [Parameterized bot](02-parameterized-bot/) | beginner | execution parameters, `parameters` (`protectedValue`), defaulting | +| 03 | [Data-driven loops](03-data-driven-loops/) | intermediate | `forEach`, `for`, `item`, `threads`, `maxFailedLoops`, data sources, `script` write-back | +| 04 | [Branching and variables](04-branching-and-variables/) | intermediate | `if`, `switch`/`case`, `assert` operators, `set` scope, `skipNode` | +| 05 | [Resilience and waiting](05-resilience-and-waiting/) | advanced | `retryIfFails`, `before`/`after`, `continueOnError`, `while`, `failure` | +| 06 | [Sessions and scheduling](06-session-and-scheduling/) | advanced | `session`, `routing`, `agents`, `schedules` | +| 07 | [Composition and reuse](07-composition-and-reuse/) | advanced | `Composite` + `return`, `callPlan`, `testSet`, `synchronized` | + +For what each control does and how to configure it, see the official +[controls documentation](https://step.dev/knowledgebase/userdocs/plans/controls/). For the YAML shape of a standalone plan, see +[../reference/](../reference/). + +## Control coverage matrix + +| Control | Sample | +|---------|--------| +| `testCase` / `sequence` | 01 | +| `callKeyword` + input/output chaining | 01 | +| `set`, `echo`, `check`, `assert`, `sleep` | 01, 04 | +| execution parameters, `parameters` | 02 | +| `forEach` + `csv` / `json-array` / `sequence` / `folder` / `sql` | 03 | +| `for`, `item`, `threads`, `maxFailedLoops` | 03 | +| `script` + `row.put(...)` write-back | 03 | +| `if`, `switch` / `case`, `skipNode` | 04 | +| `retryIfFails` | 05 | +| `before` / `after` | 05, 06 | +| `continueOnError`, `continueParentNodeExecutionOnError` | 05 | +| `while`, `failure` | 05 | +| `session` | 01, 02, 06 | +| `routing`, `agents`, `schedules` | 06 | +| `Composite` + `return`, `callPlan`, `testSet`, `synchronized` | 07 | + +## Running any of them + +```bash +step ap execute -p . -u --token --projectName +``` + +Or point the Step MCP server at the directory and use `step_validate_plan` / +`step_execute_automation_package`. + +`execute` runs the plans and nothing else. To register a package — plans, keywords, +schedules and parameters — in a project, **deploy** it instead. That matters for +[06](06-session-and-scheduling/), whose `schedules:` block only takes effect on deploy: + +```bash +step ap deploy -p . -u --token --projectName +``` + +Three plans in [05](05-resilience-and-waiting/) are **expected to fail** — that is the lesson +in them. Their names say so, and that sample's README lists the expected outcome per plan. +Everything else passes. + +## What these samples teach + +The controls are the vocabulary; these are the ideas that decide whether a plan is any good. + +1. **The plan orchestrates, the keyword acts.** The plan carries *business* data — the + record, the amount, the confirmation. Technical context — a browser, a driver, a + logged-in connection — lives in the keyword's `session` object and never appears in the + plan. Getting that boundary right is most of what makes a plan readable. + +2. **One run is one business transaction.** A `testCase` root models a single unit of work + the business would recognise, which is what turns an execution report into an audit + trail rather than a log. + +3. **RPA is data-driven by default.** The usual shape is `forEach` over a work list, one + transaction per row — `threads` to scale it, `maxFailedLoops` so one bad record does not + strand the other 999. + +4. **Unattended means resilience is designed, not hoped for.** Nobody is watching at 03:00: + absorb transient failures with `retryIfFails`, put cleanup in an `after` block so it + survives a crash, and give every wait both `maxIterations` and `timeout`. + +5. **Business rules belong in the plan.** Routing and thresholds written as `if` and + `switch` are visible in the report and changeable without touching keyword code — which + is the reason to use a plan at all instead of one large script. + +6. **Values come from outside the plan.** The same bot serves a person on demand + (execution parameters) and a nightly `schedules` entry, with credentials held in + protected `parameters` rather than written into the tree. + +7. **Close the loop.** Writing the outcome back to the source — so a re-run skips what is + done, and the business sees the confirmation next to its record — is much of what + separates a bot from a script. + +8. **Reuse sub-processes as Composite keywords.** Once there is more than one bot, "log in" + and "look up a customer" want to be callable units with inputs and outputs, not copied + blocks. + +Each sample README calls out the pitfalls for its own controls — the ones that silently do +nothing, and the pairs that are easy to confuse. diff --git a/plans/yaml/README.md b/plans/yaml/README.md deleted file mode 100644 index de68340..0000000 --- a/plans/yaml/README.md +++ /dev/null @@ -1,7 +0,0 @@ -## Overview - -Basic samples of Step Yaml Plans - - **basic-sample-plan.yml**: basic sample illustrating the Yaml plan structure and usages of some operators like `Sequence, If, Assert, CallKeyword` - **benchmark-sample-plan.yml**: the Yaml format for the plan used in **maven-plugins/run-automation-package-sample** - **dynamic-values-sample-plan.yml**: various examples of dynamic values and inputs for `CallKeyword` control diff --git a/plans/yaml/basic-sample-plan.yml b/plans/yaml/basic-sample-plan.yml deleted file mode 100644 index 5fd4cfd..0000000 --- a/plans/yaml/basic-sample-plan.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 1.0.0 -name: "Basic test plan" -root: - sequence: - continueOnError: false - children: - - if: - condition: - expression: "controllerSettings.getSettingByKey('housekeeping_enabled').getValue()=='true'" - description: "my description" - children: - - assert: - actual: - expression: "'status'" - operator: "EQUALS" - doNegate: false - expected: - expression: "'ok'" - customErrorMessage: "my custom error" - - callKeyword: - keyword: "callExisting3" - remote: true - inputs: - - stringInput: - expression: "'abc'" - - intInput: 777 \ No newline at end of file diff --git a/plans/yaml/dynamic-values-sample-plan.yml b/plans/yaml/dynamic-values-sample-plan.yml deleted file mode 100644 index 744e946..0000000 --- a/plans/yaml/dynamic-values-sample-plan.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: 1.0.0 -name: "Dynamic values and inputs sample" -root: - threadGroup: - users: 1 - pacing: 0 - maxDuration: 0 - iterations: - expression: "someVariable" - children: - - callKeyword: - keyword: "myKeyword" - - callKeyword: - keyword: - expression: "'myKeyword2'" - - callKeyword: - keyword: "myKeyword3" - routing: - - criteria1: "criteriaValue1" - - callKeyword: - keyword: - expression: "'myKeyword4'" - routing: - - criteria1: "criteriaValue1" - inputs: - - stringInput: - expression: "'abc'" - - intInput: 777 \ No newline at end of file From 0eb04301d191bc36bee42f3b733219aa2c41b24e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 21 Aug 2026 22:51:34 +0200 Subject: [PATCH 2/9] SED-4885 Creating plan samples for load testing (draft) --- README.md | 3 +- plans/README.md | 6 +- .../load-testing/01-first-load-test/README.md | 163 +++++++ .../automation-package.yaml | 209 +++++++++ .../keywords/addToCart.groovy | 3 + .../keywords/checkout.groovy | 4 + .../keywords/searchProducts.groovy | 4 + .../02-thread-group-configuration/README.md | 166 +++++++ .../automation-package.yaml | 368 ++++++++++++++++ .../keywords/checkout.groovy | 3 + .../keywords/login.groovy | 3 + .../keywords/logout.groovy | 3 + .../keywords/recordIteration.groovy | 6 + .../keywords/warmUpCache.groovy | 3 + .../03-scenarios-and-mixed-load/README.md | 126 ++++++ .../automation-package.yaml | 283 ++++++++++++ .../keywords/checkout.groovy | 3 + .../keywords/runBatchReport.groovy | 3 + .../keywords/searchProducts.groovy | 3 + .../keywords/warmUp.groovy | 3 + .../04-test-data-and-datasets/README.md | 172 ++++++++ .../automation-package.yaml | 417 ++++++++++++++++++ .../04-test-data-and-datasets/data/users.csv | 5 + .../keywords/login.groovy | 5 + .../keywords/placeOrder.groovy | 4 + plans/load-testing/05-measurements/README.md | 126 ++++++ .../05-measurements/automation-package.yaml | 264 +++++++++++ .../keywords/browseCatalog.groovy | 9 + .../05-measurements/keywords/checkout.groovy | 4 + .../keywords/completePurchase.groovy | 16 + .../06-thresholds-and-slas/README.md | 167 +++++++ .../automation-package.yaml | 385 ++++++++++++++++ .../keywords/checkout.groovy | 3 + .../keywords/flakyPayment.groovy | 20 + .../keywords/searchProducts.groovy | 3 + .../keywords/slowSearch.groovy | 3 + plans/load-testing/README.md | 116 +++++ plans/reference/README.md | 5 +- plans/reference/performance-assert.yml | 66 ++- 39 files changed, 3137 insertions(+), 18 deletions(-) create mode 100644 plans/load-testing/01-first-load-test/README.md create mode 100644 plans/load-testing/01-first-load-test/automation-package.yaml create mode 100644 plans/load-testing/01-first-load-test/keywords/addToCart.groovy create mode 100644 plans/load-testing/01-first-load-test/keywords/checkout.groovy create mode 100644 plans/load-testing/01-first-load-test/keywords/searchProducts.groovy create mode 100644 plans/load-testing/02-thread-group-configuration/README.md create mode 100644 plans/load-testing/02-thread-group-configuration/automation-package.yaml create mode 100644 plans/load-testing/02-thread-group-configuration/keywords/checkout.groovy create mode 100644 plans/load-testing/02-thread-group-configuration/keywords/login.groovy create mode 100644 plans/load-testing/02-thread-group-configuration/keywords/logout.groovy create mode 100644 plans/load-testing/02-thread-group-configuration/keywords/recordIteration.groovy create mode 100644 plans/load-testing/02-thread-group-configuration/keywords/warmUpCache.groovy create mode 100644 plans/load-testing/03-scenarios-and-mixed-load/README.md create mode 100644 plans/load-testing/03-scenarios-and-mixed-load/automation-package.yaml create mode 100644 plans/load-testing/03-scenarios-and-mixed-load/keywords/checkout.groovy create mode 100644 plans/load-testing/03-scenarios-and-mixed-load/keywords/runBatchReport.groovy create mode 100644 plans/load-testing/03-scenarios-and-mixed-load/keywords/searchProducts.groovy create mode 100644 plans/load-testing/03-scenarios-and-mixed-load/keywords/warmUp.groovy create mode 100644 plans/load-testing/04-test-data-and-datasets/README.md create mode 100644 plans/load-testing/04-test-data-and-datasets/automation-package.yaml create mode 100644 plans/load-testing/04-test-data-and-datasets/data/users.csv create mode 100644 plans/load-testing/04-test-data-and-datasets/keywords/login.groovy create mode 100644 plans/load-testing/04-test-data-and-datasets/keywords/placeOrder.groovy create mode 100644 plans/load-testing/05-measurements/README.md create mode 100644 plans/load-testing/05-measurements/automation-package.yaml create mode 100644 plans/load-testing/05-measurements/keywords/browseCatalog.groovy create mode 100644 plans/load-testing/05-measurements/keywords/checkout.groovy create mode 100644 plans/load-testing/05-measurements/keywords/completePurchase.groovy create mode 100644 plans/load-testing/06-thresholds-and-slas/README.md create mode 100644 plans/load-testing/06-thresholds-and-slas/automation-package.yaml create mode 100644 plans/load-testing/06-thresholds-and-slas/keywords/checkout.groovy create mode 100644 plans/load-testing/06-thresholds-and-slas/keywords/flakyPayment.groovy create mode 100644 plans/load-testing/06-thresholds-and-slas/keywords/searchProducts.groovy create mode 100644 plans/load-testing/06-thresholds-and-slas/keywords/slowSearch.groovy create mode 100644 plans/load-testing/README.md diff --git a/README.md b/README.md index 9fcfe32..4622ed2 100644 --- a/README.md +++ b/README.md @@ -43,9 +43,10 @@ The two directories answer different questions, and most people need both: | Section | Contents | |---------|----------| | [plans/rpa](plans/rpa/) | Seven RPA plan samples: loops and data sources, branching, resilience, sessions, scheduling, reuse | +| [plans/load-testing](plans/load-testing/) | Six load-testing plan samples: thread groups, scenarios and mixed load, test data, measurements, SLA gates | | [plans/reference](plans/reference/) | Standalone YAML plans illustrating the syntax — the shape of a plan, and static values vs expressions | -Samples for functional testing, load testing and monitoring will follow the same structure. +Samples for functional testing and monitoring will follow the same structure. --- diff --git a/plans/README.md b/plans/README.md index 425ff59..4e3d8d5 100644 --- a/plans/README.md +++ b/plans/README.md @@ -25,8 +25,8 @@ everything else. That is where you go while writing the plan inside your package | Use case | Samples | Status | |----------|---------|--------| | [**RPA**](rpa/) | 7 samples — loops, branching, resilience, sessions, scheduling, reuse | available | +| [**Load testing**](load-testing/) | 6 samples — thread groups, scenarios, data sets, measurements, SLA gates | available | | Functional testing | — | planned | -| Load testing | — | planned | | Monitoring | — | planned | ## Reference @@ -40,7 +40,7 @@ For what each control does and how to configure it, see the official |------|-------| | [reference/basic-plan-syntax.yml](reference/basic-plan-syntax.yml) | The shape of a plan: root artefact, `callKeyword` with inputs, capturing an output, `if`, `assert`, `check` | | [reference/dynamic-values.yml](reference/dynamic-values.yml) | Static values vs `expression:`, where plan variables come from, dynamic keyword names and `routing` | -| [reference/performance-assert.yml](reference/performance-assert.yml) | A `threadGroup` with a `performanceAssert` — the load-testing shape | +| [reference/performance-assert.yml](reference/performance-assert.yml) | A `threadGroup` with a `performanceAssert` — the load-testing shape, and the `after`-block rule | ## Plan formats @@ -48,7 +48,7 @@ Step has three plan formats: | Format | Written as | Used by these samples | |--------|-----------|-----------------------| -| **YAML** | The tree of controls documented at [step.dev](https://step.dev/knowledgebase/userdocs/plans/controls/) | Yes — the whole `rpa/` set | +| **YAML** | The tree of controls documented at [step.dev](https://step.dev/knowledgebase/userdocs/plans/controls/) | Yes — the whole `rpa/` and `load-testing/` sets | | **Plain text** | A compact line-based syntax, one keyword call per line | No | | **UI** | Built in the Step plan editor, stored serialized as JSON | No — see [legacy-exports/](legacy-exports/) for what an export looks like | diff --git a/plans/load-testing/01-first-load-test/README.md b/plans/load-testing/01-first-load-test/README.md new file mode 100644 index 0000000..e8d4924 --- /dev/null +++ b/plans/load-testing/01-first-load-test/README.md @@ -0,0 +1,163 @@ +--- +use-case: load-testing +focus: plans +framework: none +language: groovy +target-platform: api +approach: keyword-driven +level: beginner +--- + +# 01 — First load test + +The baseline shape of a load-testing plan: a thread group runs one business transaction +repeatedly, from several virtual users at once, and the plan states the SLA that transaction +has to meet. Every other sample in this set builds on this structure. + +The subject of this sample is the **plan**. The keywords are 3-line Groovy stubs that simulate +a shop API, so the package runs on any Java agent — no build, no browser, no system under test. + +## What this sample shows + +- `threadGroup` as the root of a load plan, with `users` and `iterations` +- **One iteration = one business transaction** — the decision that makes or breaks a load plan +- `instrumentNode` for an end-to-end transaction measurement +- `performanceAssert` as the SLA gate, and **the two rules about where it may go** +- Why a load test still needs a functional `assert` + +## The anatomy of a load plan + +```yaml +root: + threadGroup: + users: 2 # virtual users, running in parallel + iterations: 3 # repetitions per user -> 6 transactions + children: + - sequence: + nodeName: "Search and buy" + instrumentNode: true # measure the transaction end to end + children: + - callKeyword: {...} + after: + continueOnError: true + steps: + - performanceAssert: {...} # the SLA +``` + +A `threadGroup` is to a load test what a `testCase` is to a functional test or a bot: the root +that says what "one run" means. `users` sizes the load — each virtual user holds its own agent +token for the whole thread group, so it also sizes the agent capacity you need. + +## One iteration is one business transaction + +The children of a thread group should be the smallest thing a *user of the system* would call a +complete action — search, add to cart, buy — not a single HTTP call. + +That choice decides what the report can tell you. Time the whole transaction and you can answer +"can a customer buy something in under two seconds?". Time only the individual calls and you +have a pile of numbers that never adds up to an answer, because the sum of the parts is not the +experience. + +## Where measurements come from + +| Measurement | Named after | Created by | +|-------------|-------------|-----------| +| Keyword call | the **keyword** | Step, automatically, for every call | +| Instrumented node | the node's `nodeName` | `instrumentNode: true` on the node | +| Custom | whatever the keyword chooses | the keyword itself — see [05](../05-measurements/) | + +`instrumentNode: true` on the `sequence` is what gives you an end-to-end "Search and buy" series +in the performance dashboards. Without it the report has three keyword timings and no number for +the transaction as a whole. + +## `performanceAssert`: two rules about placement + +`performanceAssert` compares an aggregate of a measurement against a threshold after the load is +over. It is what turns a load test from "here are some numbers, go and look" into a pass/fail +gate a pipeline can use. + +```yaml +- performanceAssert: + measurementName: "Checkout" + aggregator: AVG # AVG | MAX | MIN | COUNT | SUM + comparator: LOWER_THAN # LOWER_THAN | HIGHER_THAN | EQUALS + expectedValue: 5000 +``` + +**1. It must live in an `after` or `afterThread` block.** As an ordinary child — of the thread +group, of a `testCase`, anywhere — the execution ends in `TECHNICAL_ERROR` with: + +``` +PerformanceAssert can only be defined in an 'after' or 'after thread' block +``` + +| Block | Runs | +|-------|------| +| `after` | once, when the whole thread group has finished — the right place for a run-wide SLA | +| `afterThread` | once per virtual user, as that user finishes | + +**2. `measurementName` must name a keyword measurement** — or a custom measurement a keyword +created itself. The measurement produced by `instrumentNode` is *not* matched. Asserting on +`"Search and buy"` fails with: + +``` +No measurement is matching the defined filters. +``` + +which is the same error a typo in a keyword name produces, so the mistake is easy to miss. To +put an SLA on a multi-step transaction, emit a custom measurement from the keyword — see +[05-measurements](../05-measurements/). + +Set `continueOnError: true` on the `after` block. Without it the block stops at the first +breached threshold and you learn about one SLA violation per run instead of all of them. + +### Bound both sides + +The plan asserts a floor as well as a ceiling: + +```yaml +- performanceAssert: {measurementName: "Checkout", aggregator: MIN, + comparator: HIGHER_THAN, expectedValue: 50} +``` + +An upper bound alone passes happily when the measurement is missing, empty, or the keyword has +been stubbed down to nothing. A floor fails the moment the transaction stops doing real work. + +### `COUNT` is the cheapest check there is + +`COUNT` on the transaction keyword catches a thread group that quietly did less work than you +asked for — a `maxDuration` that cut the run short, users that never started, a data source that +ran dry. Response times over a run that only did a third of the work look wonderful. + +## A load test still has to check its answers + +A system under load starts returning fast, cheap, **wrong** responses — an error page renders +quicker than a checkout. The nested `assert` on the checkout status runs once per iteration, so a +failure at high load appears as a failure count next to the timings: + +```yaml +- assert: + actual: "status" + operator: EQUALS + expected: "CONFIRMED" +``` + +Without it, the response times would look excellent and mean nothing. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | The plan — heavily commented, this is what to read | +| `keywords/searchProducts.groovy` | Returns a product id | +| `keywords/addToCart.groovy` | Returns a cart id | +| `keywords/checkout.groovy` | Returns an order id and `CONFIRMED` | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` + +The execution report should show 6 passing transactions, 18 passing keyword calls and three +passing performance asserts. diff --git a/plans/load-testing/01-first-load-test/automation-package.yaml b/plans/load-testing/01-first-load-test/automation-package.yaml new file mode 100644 index 0000000..4b2b6df --- /dev/null +++ b/plans/load-testing/01-first-load-test/automation-package.yaml @@ -0,0 +1,209 @@ +--- +# --------------------------------------------------------------------------- +# Load-testing sample 01 - First load test +# +# The baseline shape of a load-testing plan: a thread group runs one business +# transaction over and over, from several virtual users at once, and the plan +# states the SLA the transaction has to meet. +# +# The focus of this sample is the PLAN. The keywords are 3-line Groovy stubs +# that simulate a shop API, so the package runs on any Java agent with no +# build, no browser and no system under test. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "load-01-first-load-test" + +plans: + - name: "First load test" + categories: + - "Load testing" + root: + + # ===================================================================== + # A `threadGroup` is the usual root for a load test, the way `testCase` + # is for a functional test or a bot. + # + # It says: start `users` virtual users, and have each of them run the + # children `iterations` times. + # + # users how many virtual users run IN PARALLEL. Each one takes + # its own agent token for the whole thread group, so this + # is what actually sizes the load - and what sizes the + # agent capacity you need. + # iterations how many times EACH user repeats the children. Total + # transactions = users x iterations = 6 here. + # + # Both are deliberately tiny in this sample so it runs in seconds. The + # other knobs - pacing, rampup, startOffset, maxDuration - are the + # subject of sample 02. + # ===================================================================== + threadGroup: + nodeName: "2 virtual users, 3 iterations each" + users: 2 + iterations: 3 + children: + + # ------------------------------------------------------------- + # ONE ITERATION IS ONE BUSINESS TRANSACTION. + # + # This is the load-testing counterpart of "one testCase run is one + # business transaction": the children of a thread group should be + # the smallest thing a user of the system would call a complete + # action - search, add to cart, buy - not a single HTTP call. + # + # `instrumentNode: true` wraps the sequence in a MEASUREMENT named + # after its `nodeName`. Without it the report would show three + # separate keyword timings and no end-to-end number for the + # transaction as a whole; with it, "Search and buy" appears as its + # own series in the performance dashboards. + # + # Note what instrumenting does NOT give you: a measurement created + # this way is not visible to `performanceAssert` (see the `after` + # block below for what is). It is for the dashboards. + # ------------------------------------------------------------- + - sequence: + nodeName: "Search and buy" + instrumentNode: true + children: + + # Every keyword call is measured automatically, under the + # KEYWORD's name - "Search Products" here. That happens with + # no configuration at all, and it is the raw material of + # every load-test report. + - callKeyword: + keyword: "Search Products" + nodeName: "Search the catalogue" + inputs: + - term: "laptop" + children: + # Chaining outputs works exactly as in any other plan: + # a nested `set` promotes the value to the parent scope + # so later siblings can read it. + - set: + key: productId + value: + expression: "output.firstProductId" + + - callKeyword: + keyword: "Add To Cart" + nodeName: "Add the product to the cart" + inputs: + - productId: + expression: "productId" + children: + - set: + key: cartId + value: + expression: "output.cartId" + + - callKeyword: + keyword: "Checkout" + nodeName: "Check out" + inputs: + - cartId: + expression: "cartId" + children: + + # --------------------------------------------- + # A LOAD TEST STILL HAS TO CHECK ITS ANSWERS. + # + # A system under load frequently starts returning + # fast, cheap, WRONG responses - an error page is + # quicker to render than a checkout. Without this + # assert the response times would look excellent + # and mean nothing. + # + # This runs once per iteration, so a failure at + # high load shows up as a failure count in the + # report, next to the timings. + # --------------------------------------------- + - assert: + actual: "status" + operator: EQUALS + expected: "CONFIRMED" + customErrorMessage: "Checkout did not confirm - the response was fast but wrong." + + # ===================================================================== + # THE SLA, EXPRESSED IN THE PLAN. + # + # `performanceAssert` compares an aggregate of a measurement against a + # threshold once the load is over. It turns a load test from "here are + # some numbers, go and look" into a pass/fail gate you can put in a + # pipeline. + # + # measurementName which measurement to aggregate + # aggregator AVG | MAX | MIN | COUNT | SUM + # comparator LOWER_THAN | HIGHER_THAN | EQUALS + # expectedValue the threshold + # + # TWO RULES ABOUT WHERE IT GOES, both easy to get wrong: + # + # 1. A `performanceAssert` MUST live in an `after` or `afterThread` + # block. As an ordinary child - of the thread group, of a testCase, + # anywhere - the execution ends in TECHNICAL_ERROR with + # "PerformanceAssert can only be defined in an 'after' or + # 'after thread' block". + # + # `after` runs once, when the whole thread group is done - + # the right place for an SLA over the whole run. + # `afterThread` runs once per virtual user, as that user finishes. + # + # 2. `measurementName` must name a KEYWORD measurement (or a custom + # one a keyword created itself - see sample 05). The measurement + # produced by `instrumentNode` on the sequence above is NOT + # matched: asserting on "Search and buy" fails with + # "No measurement is matching the defined filters." - the same + # error a typo in the keyword name produces, which is why the + # mistake is easy to miss. + # + # `continueOnError: true` matters here. Without it the block stops at + # the first breached threshold, and you learn about one SLA violation + # per run instead of all of them. + # ===================================================================== + after: + continueOnError: true + steps: + + # Did the load actually happen? COUNT is the cheapest sanity + # check there is, and it catches a thread group that quietly did + # less work than you asked for - a `maxDuration` that cut the run + # short, users that failed to start, a data source that ran dry. + - performanceAssert: + nodeName: "All 6 checkouts ran" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 6 + + # The SLA proper. + - performanceAssert: + nodeName: "Checkout stays under 5s on average" + measurementName: "Checkout" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 5000 + + # A lower bound as well, which looks odd until you have been + # fooled once: a measurement that is missing, empty or mocked out + # sails through an upper-bound threshold. Bounding both sides + # means this assert fails if the keyword ever stops doing work. + - performanceAssert: + nodeName: "Checkout is really doing something" + measurementName: "Checkout" + aggregator: MIN + comparator: HIGHER_THAN + expectedValue: 50 + +keywords: + - GeneralScript: + name: "Search Products" + scriptLanguage: groovy + scriptFile: keywords/searchProducts.groovy + - GeneralScript: + name: "Add To Cart" + scriptLanguage: groovy + scriptFile: keywords/addToCart.groovy + - GeneralScript: + name: "Checkout" + scriptLanguage: groovy + scriptFile: keywords/checkout.groovy diff --git a/plans/load-testing/01-first-load-test/keywords/addToCart.groovy b/plans/load-testing/01-first-load-test/keywords/addToCart.groovy new file mode 100644 index 0000000..fbe56ef --- /dev/null +++ b/plans/load-testing/01-first-load-test/keywords/addToCart.groovy @@ -0,0 +1,3 @@ +// Simulates adding one product to the cart. +Thread.sleep(60) +output.add("cartId", "CART-" + input.getString("productId", "P-UNKNOWN")) diff --git a/plans/load-testing/01-first-load-test/keywords/checkout.groovy b/plans/load-testing/01-first-load-test/keywords/checkout.groovy new file mode 100644 index 0000000..f5c22b6 --- /dev/null +++ b/plans/load-testing/01-first-load-test/keywords/checkout.groovy @@ -0,0 +1,4 @@ +// Simulates the checkout call - the transaction the SLA is written against. +Thread.sleep(120) +output.add("orderId", "ORD-" + input.getString("cartId", "CART-UNKNOWN")) +output.add("status", "CONFIRMED") diff --git a/plans/load-testing/01-first-load-test/keywords/searchProducts.groovy b/plans/load-testing/01-first-load-test/keywords/searchProducts.groovy new file mode 100644 index 0000000..b3fe1bc --- /dev/null +++ b/plans/load-testing/01-first-load-test/keywords/searchProducts.groovy @@ -0,0 +1,4 @@ +// Simulates a product search against the shop API. +Thread.sleep(80) +output.add("firstProductId", "P-" + input.getString("term", "laptop").toUpperCase()) +output.add("resultCount", 12) diff --git a/plans/load-testing/02-thread-group-configuration/README.md b/plans/load-testing/02-thread-group-configuration/README.md new file mode 100644 index 0000000..56e9b17 --- /dev/null +++ b/plans/load-testing/02-thread-group-configuration/README.md @@ -0,0 +1,166 @@ +--- +use-case: load-testing +focus: plans +framework: none +language: groovy +target-platform: api +approach: keyword-driven +level: beginner +--- + +# 02 — Configuring a thread group + +`users` and `iterations` describe a load, but not a realistic one. A real population does not +appear all at once, does not hammer the system as fast as it can answer, and does not stop after +exactly N clicks. + +This sample covers every knob a thread group has, and the four blocks that decide what runs once, +what runs once per virtual user, and what runs on every iteration. + +## The plans + +| Plan | Shows | +|------|-------| +| A — The three counters | `gcounter`, `userId`, `literationId`, and `item` / `userItem` / `localItem` | +| B — Pacing sets the throughput | `pacing` — controlling throughput instead of concurrency | +| C — Ramping the load up | `rampup`, `pack`, `startOffset` | +| D — Run for a fixed time | `maxDuration`, and the trap that comes with it | +| E — Setup per test versus per user | `before`, `beforeThread`, `afterThread`, `after` | + +All five pass. + +## The knobs + +| Field | Meaning | +|-------|---------| +| `users` | virtual users running in parallel — each holds an agent token for the whole thread group | +| `iterations` | repetitions per user | +| `pacing` | fixed period between the **starts** of consecutive iterations | +| `rampup` | time taken to start all the users | +| `pack` | how many users are released together at each ramp-up step | +| `startOffset` | delay before the ramp-up begins | +| `maxDuration` | wall-clock cap on the whole thread group | + +### `pacing` — the field that makes runs comparable + +Without pacing, a virtual user starts its next iteration the instant the previous one returns. +That is a stress test: the faster the system answers, the harder you hit it, so throughput is +whatever the system allows and two runs cannot be compared. + +`pacing` makes throughput a number you choose: + +``` +iterations per second = users / (pacing in seconds) +``` + +One user with a 3000 ms pacing is 20 transactions an hour whether the system answers in 100 ms or +in two seconds. That is also what lets response time degrade *visibly* instead of being masked by +the load quietly backing off. + +`sequence` has a `pacing` field too, for pacing an inner block. + +### `rampup` and `pack` + +`rampup` is the time the group takes to start all its users. Four users over an 8000 ms ramp-up +start at 0, 2, 4 and 6 seconds. Starting everyone at once is a spike test — a different question, +measuring the cold start rather than the steady state. + +`pack` is how many users are released **together** at each step. The same four users over 8000 ms +with `pack: 2` start at 0 s and 4 s, two at a time. Use it for a stepped ramp, or simply to stop +a long ramp-up from trickling in one user at a time. + +`startOffset` delays the ramp-up. On its own in a single thread group it does nothing useful; its +purpose is staggering several thread groups inside a scenario — see [03](../03-scenarios-and-mixed-load/). + +### `maxDuration` and the trap in it + +Most load tests are specified as "an hour at this rate", not "500 iterations". The idiom is to +set `iterations` to a number nobody will reach and let `maxDuration` be the real stop condition, +so the run takes the same time whether the system is fast or slow that day. + +The trap: **the iteration count is now an outcome, not an input.** A slow system produces both a +longer response time *and* fewer iterations. An SLA that reads only response times will pass a run +that managed a tenth of the work. Always assert throughput as well: + +```yaml +- performanceAssert: {measurementName: "Checkout", aggregator: COUNT, + comparator: HIGHER_THAN, expectedValue: 2} +``` + +## The counters every thread group publishes + +Three variables are always in scope inside a thread group: + +| Variable | Value | +|----------|-------| +| `userId` | which virtual user this is — 1..`users` | +| `literationId` | the iteration within **this** user — 1..`iterations` | +| `gcounter` | the iteration across the **whole** thread group — 1..(`users` × `iterations`), unique | + +They are how you build unique test data — order numbers, e-mail addresses, search terms — with no +external data source at all. `gcounter` when the value must be unique across the run, `userId` +when it must be stable for one virtual user. + +`item`, `userItem` and `localItem` rename them: + +| Field | Renames | +|-------|---------| +| `item` | `gcounter` | +| `userItem` | `userId` | +| `localItem` | `literationId` | + +**Renaming replaces.** Once `userItem: "shopperNo"` is set, `userId` no longer exists — worth +knowing before renaming one counter in a plan that reads another by its default name. + +### Coerce the counter before passing it on + +```yaml +- iteration: + expression: "gcounter as Integer" +``` + +Without `as Integer` the counter reaches the keyword as a string, `input.getInt` silently returns +its default, and every iteration takes the same branch. Nothing errors; the test just stops +testing what you meant. Sample [06](../06-thresholds-and-slas/) has a keyword that reports this +case explicitly rather than absorbing it. + +## The four blocks + +| Block | Runs | +|-------|------| +| `before` | once, before the thread group starts | +| `beforeThread` | once per virtual user, before its first iteration | +| `children` | every iteration | +| `afterThread` | once per virtual user, after its last iteration | +| `after` | once, when the whole thread group has finished | + +Getting this wrong is the most common way to make a load test lie. Logging in inside `children` +means every iteration measures a login the real user performs once a day, and the reported average +becomes a blend of two unrelated things. + +**The rule of thumb: `children` holds exactly the transaction the SLA is written about.** Setup +goes in `beforeThread`; setup the whole test needs once — warming a cache, seeding a data set — +goes in `before`. `afterThread` is the reliable place for per-user cleanup: it runs even when an +iteration failed, which a last child would not. + +Plan E makes all of this visible with counts rather than prose: 1 warm-up, 2 logins, 6 checkouts, +2 logouts. + +A variable `set` in `before` is visible to the iterations and to the `after` block — which is how +plan B measures its own duration to prove pacing worked. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | Five plans, one knob group each | +| `keywords/recordIteration.groovy` | Echoes the counters back so the plan can assert they arrived | +| `keywords/checkout.groovy` | The measured transaction | +| `keywords/login.groovy`, `logout.groovy` | Per-virtual-user setup and cleanup | +| `keywords/warmUpCache.groovy` | The once-per-test setup | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` diff --git a/plans/load-testing/02-thread-group-configuration/automation-package.yaml b/plans/load-testing/02-thread-group-configuration/automation-package.yaml new file mode 100644 index 0000000..7babdc5 --- /dev/null +++ b/plans/load-testing/02-thread-group-configuration/automation-package.yaml @@ -0,0 +1,368 @@ +--- +# --------------------------------------------------------------------------- +# Load-testing sample 02 - Configuring a thread group +# +# `users` and `iterations` from sample 01 describe a load, but not a REALISTIC +# one. A real population does not appear all at once, does not hammer the +# system as fast as it can answer, and does not stop after exactly N clicks. +# +# This sample covers every knob a thread group has, and the four blocks that +# decide what runs once, what runs once per virtual user, and what runs on +# every iteration. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "load-02-thread-group-configuration" + +plans: + + # ========================================================================= + # A. The counters every thread group publishes. + # + # Inside a thread group three variables are always in scope: + # + # userId which virtual user this is 1 .. users + # literationId the iteration within THIS user 1 .. iterations + # gcounter the iteration across the WHOLE thread group, unique + # 1 .. users x iterations + # + # They are what you build unique test data out of - order numbers, e-mail + # addresses, search terms - without any external data source. `gcounter` is + # the one to use when the value has to be unique across the run; + # `userId` when it has to be stable for one virtual user. + # + # `item`, `userItem` and `localItem` RENAME them: + # + # item renames gcounter + # userItem renames userId + # localItem renames literationId + # + # Renaming REPLACES: once `userItem: "shopperNo"` is set, `userId` no longer + # exists. Worth knowing before renaming one counter in a plan that reads + # another by its default name. + # ========================================================================= + - name: "A - The three counters" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "2 users x 3 iterations with renamed counters" + users: 2 + iterations: 3 + item: "orderNo" + userItem: "shopperNo" + localItem: "shopperOrderNo" + children: + - callKeyword: + keyword: "Record Iteration" + nodeName: "Place an order with a unique reference" + inputs: + # gcounter, renamed: unique across the whole thread group, so + # every order reference in the run is different. + - orderRef: + expression: "'ORD-' + orderNo" + # userId, renamed: stable for the life of this virtual user. + - shopper: + expression: "shopperNo" + # literationId, renamed: 1, 2, 3 for each user separately. + - orderOfThisShopper: + expression: "shopperOrderNo" + children: + # The keyword echoes back what it received, so the plan can + # prove the counters carry real values rather than the + # literal text of the expression. + - assert: + actual: "orderRef" + operator: MATCHES + expected: "ORD-[1-6]" + customErrorMessage: "The global counter did not arrive as a number between 1 and 6." + - assert: + actual: "shopper" + operator: MATCHES + expected: "[12]" + - assert: + actual: "orderOfThisShopper" + operator: MATCHES + expected: "[123]" + + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Six distinct orders were placed" + measurementName: "Record Iteration" + aggregator: COUNT + comparator: EQUALS + expectedValue: 6 + + # ========================================================================= + # B. `pacing` - controlling THROUGHPUT rather than concurrency. + # + # Without pacing, a virtual user starts its next iteration the instant the + # previous one returns. That is a stress test: the faster the system + # answers, the harder you hit it, so throughput is whatever the system + # happens to allow and you cannot compare two runs. + # + # `pacing` is the fixed period a thread group tries to keep between the + # STARTS of consecutive iterations. If an iteration takes less than that, + # the user waits; if it takes longer, the next one starts immediately. + # + # That makes throughput a number you CHOOSE: + # + # iterations per second = users / (pacing in seconds) + # + # 1 user with a 3000 ms pacing is 20 transactions an hour, whether the + # system answers in 100 ms or in 2 seconds. That is the property that makes + # a load test repeatable - and the one that lets response time degrade + # visibly instead of being masked by the load backing off. + # + # `sequence` has a `pacing` field too, for pacing an inner block. + # ========================================================================= + - name: "B - Pacing sets the throughput" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "One user checking out every 3 seconds" + users: 1 + iterations: 3 + pacing: 3000 + + # A variable set in `before` is visible to the iterations and to the + # `after` block, which is how this plan measures its own duration. + before: + steps: + - set: + key: startedAt + value: + expression: "System.currentTimeMillis()" + + children: + - callKeyword: + keyword: "Checkout" + + after: + continueOnError: true + steps: + # Three iterations paced at 3 s start at 0 s, 3 s and 6 s, so the + # run cannot finish before 6 s. Unpaced, the same three + # iterations take about a second. This check fails if pacing + # stops working. + - check: + nodeName: "Pacing stretched the run out" + expression: "System.currentTimeMillis() - startedAt > 5500" + + - performanceAssert: + nodeName: "Pacing slowed the run without dropping iterations" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 3 + + # ========================================================================= + # C. Starting the load gradually - `rampup`, `pack` and `startOffset`. + # + # rampup the time the thread group takes to start ALL its users. + # 4 users over an 8000 ms ramp-up start at 0, 2, 4 and 6 s. + # Starting every user at once is a spike test, which is a + # different question from a load test: it measures the cold + # start rather than the steady state. + # + # pack how many users are released TOGETHER at each ramp-up step. + # The same 4 users over 8000 ms with `pack: 2` start at 0 s + # and 4 s - two at a time. Use it for a stepped ramp, or + # simply to keep a long ramp-up from trickling. + # + # startOffset a delay before the ramp-up begins. It does nothing on its + # own in a single thread group; its purpose is staggering + # several thread groups inside a scenario - see sample 03. + # + # A real ramp is much longer than this. Sample values stay in seconds so + # the package runs quickly. + # ========================================================================= + - name: "C - Ramping the load up" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "4 users released two at a time over 8 seconds" + users: 4 + iterations: 2 + rampup: 8000 + pack: 2 + startOffset: 500 + children: + - callKeyword: + keyword: "Checkout" + + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Every user completed its iterations" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 8 + + # ========================================================================= + # D. `maxDuration` - running for a TIME instead of for a count. + # + # Most load tests are specified as "an hour at this rate", not "500 + # iterations". `maxDuration` is the wall-clock cap on the whole thread + # group: when it expires, the group stops wherever it is. + # + # The idiom is to set `iterations` to a number nobody will reach and let + # `maxDuration` be the real stop condition, so the run takes the same time + # whether the system is fast or slow that day. Comparable runs are the + # whole point. + # + # THE TRAP THAT COMES WITH IT: the iteration count is now an OUTCOME, not + # an input - and a slow system produces both a longer response time and + # fewer iterations. So never write an SLA that only reads response times + # here; assert the throughput too, or a run that managed a tenth of the + # work will still report a healthy average. + # + # This plan asks for 100 iterations and cuts them off after 5 seconds. + # ========================================================================= + - name: "D - Run for a fixed time" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "One user for 5 seconds" + users: 1 + iterations: 100 + maxDuration: 5000 + children: + - callKeyword: + keyword: "Checkout" + + after: + continueOnError: true + steps: + # It stopped early: nowhere near the 100 iterations requested. + - performanceAssert: + nodeName: "maxDuration cut the run short" + measurementName: "Checkout" + aggregator: COUNT + comparator: LOWER_THAN + expectedValue: 50 + + # ...but it did do work. This is the throughput floor the note + # above is about - the assertion that catches a run which + # achieved almost nothing. + - performanceAssert: + nodeName: "And still did a useful amount of work" + measurementName: "Checkout" + aggregator: COUNT + comparator: HIGHER_THAN + expectedValue: 2 + + # ========================================================================= + # E. The four blocks: what runs once, per user, per iteration. + # + # before once, before the thread group starts + # beforeThread once per VIRTUAL USER, before its first iteration + # children every ITERATION + # afterThread once per VIRTUAL USER, after its last iteration + # after once, when the whole thread group has finished + # + # Getting this wrong is the most common way to make a load test lie. + # Logging in inside `children` means every iteration measures a login the + # real user does once a day, and the average response time is then a + # blend of two completely different things. + # + # The rule of thumb: `children` holds exactly the transaction the SLA is + # written about. Everything that sets it up goes in `beforeThread`. + # + # `before` is for the setup the whole test needs once - warming a cache, + # seeding a data set, resetting a counter on the target. + # + # `afterThread` is the reliable place for per-user cleanup. It runs even + # when an iteration failed, which a last child would not. + # + # The counts in the `after` block below are what make all of this visible: + # 1 warm-up, 2 logins, 6 checkouts, 2 logouts. + # ========================================================================= + - name: "E - Setup per test versus per user" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "2 users x 3 checkouts, logging in once each" + users: 2 + iterations: 3 + + before: + steps: + - callKeyword: + keyword: "Warm Up Cache" + nodeName: "Warm the cache once for the whole test" + + beforeThread: + steps: + - callKeyword: + keyword: "Login" + nodeName: "Log in once per virtual user" + + children: + - callKeyword: + keyword: "Checkout" + nodeName: "The transaction the SLA is about" + + afterThread: + steps: + - callKeyword: + keyword: "Logout" + nodeName: "Log out once per virtual user" + + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "The cache was warmed once for the whole run" + measurementName: "Warm Up Cache" + aggregator: COUNT + comparator: EQUALS + expectedValue: 1 + - performanceAssert: + nodeName: "Each virtual user logged in once" + measurementName: "Login" + aggregator: COUNT + comparator: EQUALS + expectedValue: 2 + - performanceAssert: + nodeName: "Each virtual user logged out once" + measurementName: "Logout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 2 + - performanceAssert: + nodeName: "And the transaction ran on every iteration" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 6 + - performanceAssert: + nodeName: "The SLA covers checkouts only, not the logins" + measurementName: "Checkout" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 5000 + +keywords: + - GeneralScript: + name: "Record Iteration" + scriptLanguage: groovy + scriptFile: keywords/recordIteration.groovy + - GeneralScript: + name: "Checkout" + scriptLanguage: groovy + scriptFile: keywords/checkout.groovy + - GeneralScript: + name: "Login" + scriptLanguage: groovy + scriptFile: keywords/login.groovy + - GeneralScript: + name: "Logout" + scriptLanguage: groovy + scriptFile: keywords/logout.groovy + - GeneralScript: + name: "Warm Up Cache" + scriptLanguage: groovy + scriptFile: keywords/warmUpCache.groovy diff --git a/plans/load-testing/02-thread-group-configuration/keywords/checkout.groovy b/plans/load-testing/02-thread-group-configuration/keywords/checkout.groovy new file mode 100644 index 0000000..31e8ea6 --- /dev/null +++ b/plans/load-testing/02-thread-group-configuration/keywords/checkout.groovy @@ -0,0 +1,3 @@ +// Simulates the checkout call. +Thread.sleep(120) +output.add("status", "CONFIRMED") diff --git a/plans/load-testing/02-thread-group-configuration/keywords/login.groovy b/plans/load-testing/02-thread-group-configuration/keywords/login.groovy new file mode 100644 index 0000000..fa7b6b5 --- /dev/null +++ b/plans/load-testing/02-thread-group-configuration/keywords/login.groovy @@ -0,0 +1,3 @@ +// Simulates the login one virtual user does once, before its first iteration. +Thread.sleep(80) +output.add("status", "LOGGED_IN") diff --git a/plans/load-testing/02-thread-group-configuration/keywords/logout.groovy b/plans/load-testing/02-thread-group-configuration/keywords/logout.groovy new file mode 100644 index 0000000..e372f63 --- /dev/null +++ b/plans/load-testing/02-thread-group-configuration/keywords/logout.groovy @@ -0,0 +1,3 @@ +// Simulates the logout one virtual user does after its last iteration. +Thread.sleep(40) +output.add("status", "LOGGED_OUT") diff --git a/plans/load-testing/02-thread-group-configuration/keywords/recordIteration.groovy b/plans/load-testing/02-thread-group-configuration/keywords/recordIteration.groovy new file mode 100644 index 0000000..9a98bdd --- /dev/null +++ b/plans/load-testing/02-thread-group-configuration/keywords/recordIteration.groovy @@ -0,0 +1,6 @@ +// Echoes the thread-group counters back, so the plan can prove they arrived +// as values rather than as the literal text of the expression. +Thread.sleep(80) +output.add("orderRef", input.getString("orderRef", "NONE")) +output.add("shopper", String.valueOf(input.getInt("shopper", -1))) +output.add("orderOfThisShopper", String.valueOf(input.getInt("orderOfThisShopper", -1))) diff --git a/plans/load-testing/02-thread-group-configuration/keywords/warmUpCache.groovy b/plans/load-testing/02-thread-group-configuration/keywords/warmUpCache.groovy new file mode 100644 index 0000000..1ac7a47 --- /dev/null +++ b/plans/load-testing/02-thread-group-configuration/keywords/warmUpCache.groovy @@ -0,0 +1,3 @@ +// Simulates the one-off warm-up the whole test needs before load starts. +Thread.sleep(100) +output.add("status", "WARM") diff --git a/plans/load-testing/03-scenarios-and-mixed-load/README.md b/plans/load-testing/03-scenarios-and-mixed-load/README.md new file mode 100644 index 0000000..f7a818d --- /dev/null +++ b/plans/load-testing/03-scenarios-and-mixed-load/README.md @@ -0,0 +1,126 @@ +--- +use-case: load-testing +focus: plans +framework: none +language: groovy +target-platform: api +approach: keyword-driven +level: intermediate +--- + +# 03 — Scenarios and mixed load + +Real systems are never hit by one kind of user. While a few hundred people browse, a handful buy, +and at 02:00 a batch job runs straight through the middle of it. A load test that models only the +happy path measures a system nobody is using. + +`testScenario` is the control that composes those populations: it runs its children **in +parallel**, each with its own load profile. + +## The plans + +| Plan | Shows | +|------|-------| +| A — Browsers and buyers at the same time | `testScenario` with two thread groups, `startOffset`, scenario-wide `after` | +| B — Step ramp in three stages | A staged ramp built from several thread groups | +| C — Measure the site while the batch job runs | A thread group next to a plain `sequence` | +| D — Warm-up phase then measured phase | `sequence` as the root — phases instead of parallelism | + +All four pass. + +## The three composing roots + +| Root | Children run | Use for | +|------|--------------|---------| +| `testScenario` | **in parallel** | mixed populations, staged ramps, load next to a batch job | +| `sequence` | **one after the other** | phases — warm-up, measured, cool-down | +| `testSet` | as separate test cases, `threads` at a time | a batch of independent tests, not a load profile | + +Confusing `testScenario` with `testSet` is the usual slip. `testSet` parallelises *test cases*; +`testScenario` runs *load profiles* side by side. + +## The mixed-workload shape + +Each thread group keeps its own configuration — that is the whole point. Browsers are many and +cheap; buyers are few and expensive. Modelling them as one thread group doing "browse then buy" +produces a traffic mix no real shop ever sees. + +```yaml +root: + testScenario: + children: + - threadGroup: {nodeName: "Browsers", users: 2, iterations: 3, pacing: 1000, ...} + - threadGroup: {nodeName: "Buyers", users: 1, iterations: 2, startOffset: 2000, ...} +``` + +`startOffset` on the buyers means the checkout numbers are measured against a system that is +already busy, rather than an idle one. + +## Where the SLA goes + +| Placement | Evaluated | +|-----------|-----------| +| `after` on the **testScenario** | once, when every thread group has finished — the scenario-wide gate | +| `after` on a **thread group** | when that population finishes — the place for a threshold that concerns only it | + +Both work. Prefer the thread group's own block when the threshold belongs to one population: it +reads next to the profile it describes. [06](../06-thresholds-and-slas/) covers this in more +detail. + +Assert exact `COUNT`s in a scenario. It is entirely possible for one population to be starved of +agent tokens and quietly contribute nothing while the report still looks green. + +## A staged ramp + +`rampup` inside one thread group gives a smooth ramp to a single target. What it cannot express is +a **step profile**: hold 1 user, then 2, then 3, each for a while, and compare the response times +between the steps. That is the shape that answers *where does it start to hurt?* + +Several thread groups in a scenario, staggered with `startOffset`, is how you write it. Each stage +is its own node in the report, so each has its own numbers. In a real test each stage also carries +`maxDuration`, to hold the load for a fixed time rather than a fixed number of iterations. + +A `performanceAssert` on a measurement name spans **every** node that produced it, so a COUNT over +the whole ramp is the total across all stages. To compare the stages against each other — the +reason to build a ramp this way — read them in the performance dashboards, where each stage is a +separate node; a per-stage threshold goes in that stage's own `after` block. + +## Load next to a batch job + +A scenario's children do not all have to be thread groups. "What do response times look like while +the nightly batch runs?" is a thread group next to a plain `sequence`. The sequence runs once, in +parallel with the load, and its duration is measured like anything else. + +## Phases, and why a warm-up needs its own + +Nesting a warm-up in the same thread group as the measured load pollutes the measurement with +cold-start timings. A separate warm-up **phase** keeps them in different nodes, so the threshold +applies only to the measured phase: + +```yaml +root: + sequence: + children: + - threadGroup: {nodeName: "Warm-up phase", ...} + - threadGroup: + nodeName: "Measured phase" + after: + steps: + - performanceAssert: {...} # on the measured group, not the sequence +``` + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | Four plans — the composing controls | +| `keywords/searchProducts.groovy` | The browse traffic | +| `keywords/checkout.groovy` | The buy traffic | +| `keywords/runBatchReport.groovy` | The batch job that runs alongside | +| `keywords/warmUp.groovy` | The warm-up phase | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` diff --git a/plans/load-testing/03-scenarios-and-mixed-load/automation-package.yaml b/plans/load-testing/03-scenarios-and-mixed-load/automation-package.yaml new file mode 100644 index 0000000..df464a9 --- /dev/null +++ b/plans/load-testing/03-scenarios-and-mixed-load/automation-package.yaml @@ -0,0 +1,283 @@ +--- +# --------------------------------------------------------------------------- +# Load-testing sample 03 - Scenarios and mixed load +# +# Real systems are never hit by one kind of user. While a few hundred people +# browse, a handful buy, and at 02:00 a batch job runs straight through the +# middle of it. A load test that models only the happy path measures a system +# nobody is using. +# +# `testScenario` is the control that composes those populations: it runs its +# children IN PARALLEL, each with its own load profile. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "load-03-scenarios-and-mixed-load" + +plans: + + # ========================================================================= + # A. The mixed-workload shape. + # + # Two populations with different sizes, different transactions and + # different rhythms, running at the same time against the same system. + # + # Each thread group keeps its own configuration - that is the whole point. + # Browsers are many and cheap; buyers are few and expensive. Modelling them + # as one thread group doing "browse then buy" would produce a traffic mix + # no real shop ever sees. + # ========================================================================= + - name: "A - Browsers and buyers at the same time" + categories: ["Load testing"] + root: + testScenario: + nodeName: "Shop under mixed traffic" + children: + + - threadGroup: + nodeName: "Browsers" + users: 2 + iterations: 3 + pacing: 1000 + children: + - callKeyword: + keyword: "Search Products" + inputs: + - term: "laptop" + + - threadGroup: + nodeName: "Buyers" + users: 1 + iterations: 2 + # The buyers join once the browse traffic is already flowing, + # so the checkout numbers are measured against a busy system + # rather than an idle one. + startOffset: 2000 + children: + - callKeyword: + keyword: "Checkout" + + # ------------------------------------------------------------------- + # WHERE THE SLA GOES IN A SCENARIO. + # + # `after` on the testScenario runs once, when every thread group in it + # has finished - so this is the scenario-wide gate. A thread group's + # own `after` block still works and is the place for a threshold that + # only concerns that one population. + # + # Both counts are exact, and that is deliberate: in a scenario it is + # entirely possible for one population to be starved of agent tokens + # and quietly contribute nothing while the report still looks green. + # ------------------------------------------------------------------- + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "The browse traffic ran in full" + measurementName: "Search Products" + aggregator: COUNT + comparator: EQUALS + expectedValue: 6 + - performanceAssert: + nodeName: "The buy traffic ran in full" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 2 + - performanceAssert: + nodeName: "Checkout stays under 5s while browsing is going on" + measurementName: "Checkout" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 5000 + + # ========================================================================= + # B. A staged ramp built out of several thread groups. + # + # `rampup` inside one thread group gives a smooth ramp to a single target + # (sample 02). What it cannot express is a STEP profile: hold 1 user, then + # 2, then 3, each for a while, and compare the response times between the + # steps. That is the shape that answers "where does it start to hurt?". + # + # Several thread groups in a scenario, staggered with `startOffset`, is how + # you write it. Each stage is its own node in the report, so each one has + # its own numbers. + # + # In a real test each stage would also carry `maxDuration` to hold the load + # for a fixed time rather than a fixed number of iterations. + # ========================================================================= + - name: "B - Step ramp in three stages" + categories: ["Load testing"] + root: + testScenario: + nodeName: "Find the level where response times turn" + children: + + - threadGroup: + nodeName: "Stage 1 - 1 user" + users: 1 + iterations: 2 + startOffset: 0 + children: + - callKeyword: + keyword: "Checkout" + + - threadGroup: + nodeName: "Stage 2 - 2 users" + users: 2 + iterations: 2 + startOffset: 3000 + children: + - callKeyword: + keyword: "Checkout" + + - threadGroup: + nodeName: "Stage 3 - 2 users" + users: 2 + iterations: 2 + startOffset: 6000 + children: + - callKeyword: + keyword: "Checkout" + + after: + continueOnError: true + steps: + # A measurement name is shared by every node that produces it, so + # this COUNT spans all three stages: 2 + 4 + 4. + # + # To compare the stages against each OTHER - which is the reason + # to build a ramp this way - read them in the performance + # dashboards, where each stage is a separate node. A per-stage + # threshold goes in that stage's own `after` block. + - performanceAssert: + nodeName: "Every stage contributed its transactions" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 10 + + # ========================================================================= + # C. Load and a batch job at the same time. + # + # A scenario's children do not all have to be thread groups. The classic + # question - "what do response times look like while the nightly batch is + # running?" - is a thread group next to a plain `sequence`. + # + # The sequence runs once, in parallel with the load, and its own duration + # is measured like anything else. + # ========================================================================= + - name: "C - Measure the site while the batch job runs" + categories: ["Load testing"] + root: + testScenario: + nodeName: "Shoppers versus the nightly batch" + children: + + - threadGroup: + nodeName: "Shoppers" + users: 2 + iterations: 3 + children: + - callKeyword: + keyword: "Search Products" + inputs: + - term: "laptop" + + - sequence: + nodeName: "Nightly batch" + children: + - callKeyword: + keyword: "Run Batch Report" + + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "The batch ran exactly once" + measurementName: "Run Batch Report" + aggregator: COUNT + comparator: EQUALS + expectedValue: 1 + - performanceAssert: + nodeName: "Search survives the batch window" + measurementName: "Search Products" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 5000 + + # ========================================================================= + # D. Phases, one after the other - and the control that is NOT testScenario. + # + # `testScenario` runs its children in parallel. When you want thread groups + # to run in SEQUENCE - a warm-up phase, then the measured phase, then a + # cool-down - the root is a plain `sequence`. + # + # This matters more than it looks. A warm-up nested in the same thread group + # as the measured load pollutes the measurement with the cold-start + # timings; a separate warm-up PHASE keeps them in different nodes, so the + # threshold below applies only to the measured phase. + # + # The three composing roots, side by side: + # + # testScenario children run in PARALLEL -> mixed populations + # sequence children run ONE AFTER THE OTHER -> phases + # testSet children run as separate test cases, `threads` at a time + # -> a batch of independent tests, not a load profile + # ========================================================================= + - name: "D - Warm-up phase then measured phase" + categories: ["Load testing"] + root: + sequence: + nodeName: "Two phases in order" + children: + + - threadGroup: + nodeName: "Warm-up phase" + users: 1 + iterations: 2 + children: + - callKeyword: + keyword: "Warm Up" + + - threadGroup: + nodeName: "Measured phase" + users: 2 + iterations: 3 + children: + - callKeyword: + keyword: "Checkout" + # The threshold sits on the measured thread group, not on the + # sequence, so the warm-up cannot drag the average around. + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Checkout meets its SLA once the system is warm" + measurementName: "Checkout" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 5000 + - performanceAssert: + nodeName: "The measured phase ran in full" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 6 + +keywords: + - GeneralScript: + name: "Search Products" + scriptLanguage: groovy + scriptFile: keywords/searchProducts.groovy + - GeneralScript: + name: "Checkout" + scriptLanguage: groovy + scriptFile: keywords/checkout.groovy + - GeneralScript: + name: "Run Batch Report" + scriptLanguage: groovy + scriptFile: keywords/runBatchReport.groovy + - GeneralScript: + name: "Warm Up" + scriptLanguage: groovy + scriptFile: keywords/warmUp.groovy diff --git a/plans/load-testing/03-scenarios-and-mixed-load/keywords/checkout.groovy b/plans/load-testing/03-scenarios-and-mixed-load/keywords/checkout.groovy new file mode 100644 index 0000000..85c21ce --- /dev/null +++ b/plans/load-testing/03-scenarios-and-mixed-load/keywords/checkout.groovy @@ -0,0 +1,3 @@ +// Simulates the buy traffic: a checkout. +Thread.sleep(120) +output.add("status", "CONFIRMED") diff --git a/plans/load-testing/03-scenarios-and-mixed-load/keywords/runBatchReport.groovy b/plans/load-testing/03-scenarios-and-mixed-load/keywords/runBatchReport.groovy new file mode 100644 index 0000000..be20c5f --- /dev/null +++ b/plans/load-testing/03-scenarios-and-mixed-load/keywords/runBatchReport.groovy @@ -0,0 +1,3 @@ +// Simulates the nightly report the shop runs while users are on the site. +Thread.sleep(200) +output.add("status", "REPORT_DONE") diff --git a/plans/load-testing/03-scenarios-and-mixed-load/keywords/searchProducts.groovy b/plans/load-testing/03-scenarios-and-mixed-load/keywords/searchProducts.groovy new file mode 100644 index 0000000..555b8fb --- /dev/null +++ b/plans/load-testing/03-scenarios-and-mixed-load/keywords/searchProducts.groovy @@ -0,0 +1,3 @@ +// Simulates the browse traffic: a product search. +Thread.sleep(80) +output.add("resultCount", 12) diff --git a/plans/load-testing/03-scenarios-and-mixed-load/keywords/warmUp.groovy b/plans/load-testing/03-scenarios-and-mixed-load/keywords/warmUp.groovy new file mode 100644 index 0000000..e8101fa --- /dev/null +++ b/plans/load-testing/03-scenarios-and-mixed-load/keywords/warmUp.groovy @@ -0,0 +1,3 @@ +// Simulates the warm-up phase: a first pass that pays the cold-start cost. +Thread.sleep(150) +output.add("status", "WARM") diff --git a/plans/load-testing/04-test-data-and-datasets/README.md b/plans/load-testing/04-test-data-and-datasets/README.md new file mode 100644 index 0000000..e6cde09 --- /dev/null +++ b/plans/load-testing/04-test-data-and-datasets/README.md @@ -0,0 +1,172 @@ +--- +use-case: load-testing +focus: plans +framework: none +language: groovy +target-platform: api +approach: keyword-driven +level: intermediate +--- + +# 04 — Test data and data sets + +A load test that sends the same account and the same product on every iteration measures the +target's caches, not the target. Realistic load needs a pool of test data, handed out so that no +two virtual users collide. + +`dataSet` is the control for that, and it does **not** work like `forEach`. + +> **One plan in this package fails on purpose.** That is the lesson in it: the plan name says +> which one. + +## The plans + +| Plan | Expected outcome | Shows | +|------|------------------|-------| +| A — One pool feeding one thread group | **PASSED** | The two-sibling shape, `item`, `.next()` | +| B — One pool shared by two thread groups | **PASSED** | A shared cursor across populations | +| C — One account per virtual user | **PASSED** | Pulling in `beforeThread` instead of the loop body | +| D — The pool runs dry | **FAILED** (on purpose) | `resetAtEnd: false` returns `null`, silently | +| E — A pool that needs no file | **PASSED** | `json-array`, and the other sources | + +## `dataSet` is a declaration, not a loop + +This is the one thing to understand, and it is the opposite of what the name suggests: + +| Control | What it is | +|---------|-----------| +| `forEach` | a **loop**. It runs its children once per row. | +| `dataSet` | a **declaration**. It opens the data source, binds a **cursor** over it to the variable named by `item`, and runs nothing. | + +A `dataSet` node with children is a common and completely silent mistake: **the children are never +executed, and the node still reports PASSED.** + +So the pattern is two **siblings** — the `dataSet` declares the pool, and the `threadGroup` after +it pulls one row per iteration: + +```yaml +root: + testCase: + children: + - dataSet: + item: "shopperPool" # names the CURSOR, not the row + resetAtEnd: true + dataSource: + csv: {file: "data/users.csv"} + + - threadGroup: + users: 1 + iterations: 3 + children: + - set: + key: shopper + value: + expression: "shopperPool.next()" # one pull per iteration + - callKeyword: + keyword: "Login" + inputs: + - user: + expression: "shopper.Username" +``` + +`item` names the cursor. `shopperPool` is an object you call `.next()` on — it is **not** a map of +columns. `.next()` advances the cursor and returns the row as a map. + +## One pool, several thread groups + +The cursor is shared. Every `.next()` in the execution, from whichever thread group, takes the +*following* row — so two populations drawing from one pool never collide on the same account, +which is exactly what you want when the system under test locks a session per user. + +The consequence: **which** rows a given thread group gets is not deterministic. It depends on the +order the threads happen to reach their `.next()`. Never write a plan that assumes "the buyers get +shopper1". + +## Where you pull decides how much data you need + +| Pull site | Rows consumed | +|-----------|---------------| +| the loop body | one per **iteration** | +| `beforeThread` | one per **virtual user**, held for the thread's whole life | + +Per-virtual-user is the shape most load tests actually need: a virtual user logs in once and then +does twenty things as that account — it does not become a different person between two clicks. + +```yaml +beforeThread: + steps: + - set: + key: myShopper + value: + expression: "shopperPool.next()" + - callKeyword: {keyword: "Login", ...} +``` + +It is also much cheaper. Plan C runs 2 users × 3 iterations against a pool of 4 rows with +`resetAtEnd: false`. Pulling per iteration would exhaust the pool; pulling per thread uses 2 rows. +The plan proves it with counts — two logins for six orders. + +## When the pool runs dry + +With `resetAtEnd: false`, `.next()` past the last row does **not** stop the thread group and does +**not** raise an error. It returns `null`, and the run carries on feeding `null` into the keywords. + +Depending on the keyword that is either a confusing `NullPointerException` deep in the report or — +far worse — a keyword that shrugs, sends an empty value, and keeps the run green while half the +load was meaningless. + +Guard the pull: + +```yaml +- check: + nodeName: "The pool still had a row to give" + expression: "shopper != null" +``` + +Decide `resetAtEnd` deliberately: + +| Value | Meaning | +|-------|---------| +| `true` | recycle the rows — fine when the target does not care that the same account comes back | +| `false` | each row used at most once — then size the pool for the whole run, and guard the pull | + +## Data sources + +Only the `dataSource` block changes; the pull is always `.next()`. Available: `csv`, `excel`, +`file`, `folder`, `gsheet`, `json`, `json-array`, `sequence`, `sql`. + +The two that come up most in load tests are `csv` — a generated pool checked in next to the plan — +and `sql`, which reads the pool straight out of the system under test: + +```yaml +dataSource: + sql: + connectionString: "jdbc:postgresql://db:5432/shop" + driverClass: "org.postgresql.Driver" + user: "loadtest" + password: + expression: "dbPassword" + query: "SELECT username, product_id FROM test_accounts" +``` + +`sequence` is the one to reach for when the data does not need to exist beforehand — a pool of +unique order numbers, say. And for data that needs no pool at all, the thread-group counters +(`gcounter`, `userId`) from [02](../02-thread-group-configuration/) are often enough. + +A source holding credentials can be declared with `protect: true`, which obfuscates its values in +the report. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | Five plans covering the data-set mechanics | +| `data/users.csv` | The pool — four accounts, each with a product | +| `keywords/login.groovy` | Echoes the account back so the plan can prove the row arrived | +| `keywords/placeOrder.groovy` | Echoes the product back | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` diff --git a/plans/load-testing/04-test-data-and-datasets/automation-package.yaml b/plans/load-testing/04-test-data-and-datasets/automation-package.yaml new file mode 100644 index 0000000..cd46eb3 --- /dev/null +++ b/plans/load-testing/04-test-data-and-datasets/automation-package.yaml @@ -0,0 +1,417 @@ +--- +# --------------------------------------------------------------------------- +# Load-testing sample 04 - Test data and data sets +# +# A load test that sends the same account and the same product on every +# iteration measures the target's caches, not the target. Realistic load needs +# a pool of test data, handed out so that no two virtual users collide. +# +# `dataSet` is the control for that, and it does NOT work like `forEach`. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "load-04-test-data-and-datasets" + +plans: + + # ========================================================================= + # A. The shape: a pool declared once, pulled from inside the load. + # + # THE ONE THING TO UNDERSTAND ABOUT `dataSet`: + # + # `forEach` is a LOOP. It runs its children once per row. + # `dataSet` is a DECLARATION. It opens the data source, binds a CURSOR + # over it to the variable named by `item`, and runs nothing. + # + # A `dataSet` node with children is a common and completely silent mistake: + # the children are never executed, and the node still reports PASSED. + # + # So the pattern is two SIBLINGS: the `dataSet` declares the pool, and the + # `threadGroup` after it pulls one row per iteration with `.next()`. + # ========================================================================= + - name: "A - One pool feeding one thread group" + categories: ["Load testing"] + root: + testCase: + nodeName: "Each iteration gets its own account" + children: + + # `item` names the CURSOR, not the row. `shopperPool` is an object + # you call `.next()` on - it is not a map of columns. + # + # `resetAtEnd: true` wraps around when the pool is used up, so a + # long run keeps going with the same rows recycled. See plan D for + # what `false` does, which is not what most people expect. + # + # A data source holding credentials can be declared with + # `protect: true`, which obfuscates its values in the report. + - dataSet: + nodeName: "Shopper accounts" + item: "shopperPool" + resetAtEnd: true + dataSource: + csv: + file: "data/users.csv" + delimiter: "," + + - threadGroup: + nodeName: "1 user placing 3 orders" + users: 1 + iterations: 3 + children: + + # One pull per iteration. `.next()` advances the shared + # cursor and returns the row as a map of columns. + - set: + key: shopper + value: + expression: "shopperPool.next()" + nodeName: "Take the next account from the pool" + + - callKeyword: + keyword: "Login" + inputs: + - user: + expression: "shopper.Username" + - password: + expression: "shopper.Password" + children: + # The keyword echoes the account back, so this proves + # the row really arrived - a static input, or a pool + # that quietly handed out nothing, fails here. + - assert: + actual: "account" + operator: BEGINS_WITH + expected: "shopper" + customErrorMessage: "The virtual user was not given an account from the pool." + + - callKeyword: + keyword: "Place Order" + inputs: + - productId: + expression: "shopper.ProductId" + children: + - assert: + actual: "product" + operator: BEGINS_WITH + expected: "P-" + + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Three orders were placed" + measurementName: "Place Order" + aggregator: COUNT + comparator: EQUALS + expectedValue: 3 + + # ========================================================================= + # B. One pool shared by several thread groups. + # + # The cursor is shared: every `.next()` in the execution, from whichever + # thread group, takes the FOLLOWING row. Two populations drawing from one + # pool therefore never collide on the same account - which is exactly what + # you want when the system under test locks a session per user. + # + # The consequence: WHICH rows a given thread group gets is not + # deterministic. It depends on the order the threads happen to reach their + # `.next()`. Never write a plan that assumes "the buyers get shopper1". + # ========================================================================= + - name: "B - One pool shared by two thread groups" + categories: ["Load testing"] + root: + testScenario: + nodeName: "Browsers and buyers drawing from one pool" + children: + + - dataSet: + nodeName: "Shopper accounts" + item: "shopperPool" + resetAtEnd: true + dataSource: + csv: + file: "data/users.csv" + + - threadGroup: + nodeName: "Browsers" + users: 1 + iterations: 2 + children: + - set: + key: shopper + value: + expression: "shopperPool.next()" + - callKeyword: + keyword: "Login" + inputs: + - user: + expression: "shopper.Username" + children: + - assert: + actual: "account" + operator: BEGINS_WITH + expected: "shopper" + + - threadGroup: + nodeName: "Buyers" + users: 1 + iterations: 2 + children: + - set: + key: shopper + value: + expression: "shopperPool.next()" + - callKeyword: + keyword: "Place Order" + inputs: + - productId: + expression: "shopper.ProductId" + children: + - assert: + actual: "product" + operator: BEGINS_WITH + expected: "P-" + + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Both populations drew from the pool" + measurementName: "Login" + aggregator: COUNT + comparator: EQUALS + expectedValue: 2 + - performanceAssert: + nodeName: "And both did their work" + measurementName: "Place Order" + aggregator: COUNT + comparator: EQUALS + expectedValue: 2 + + # ========================================================================= + # C. One account per VIRTUAL USER, not one per iteration. + # + # This is the shape most load tests actually need. A virtual user logs in + # once and then does twenty things as that account - it does not become a + # different person between two clicks. + # + # Pulling in `beforeThread` instead of in the loop body is the whole trick: + # + # pull in the body one row per ITERATION - burns the pool fast + # pull in `beforeThread` one row per VIRTUAL USER - held for the + # thread's whole life + # + # And it is not just tidier, it is cheaper. This plan has 2 users x 3 + # iterations = 6 iterations against a pool of 4 rows with + # `resetAtEnd: false`. Pulling per iteration would exhaust the pool and the + # last two iterations would get null. Pulling per thread uses 2 rows. + # ========================================================================= + - name: "C - One account per virtual user" + categories: ["Load testing"] + root: + testCase: + nodeName: "Each virtual user keeps its own account" + children: + + - dataSet: + nodeName: "Shopper accounts" + item: "shopperPool" + resetAtEnd: false + dataSource: + csv: + file: "data/users.csv" + + - threadGroup: + nodeName: "2 users doing 3 orders each" + users: 2 + iterations: 3 + + # Runs once per virtual user, before its first iteration. + # `myShopper` then belongs to that thread and every iteration + # of it reads the same account. + beforeThread: + steps: + - set: + key: myShopper + value: + expression: "shopperPool.next()" + nodeName: "Claim one account for this virtual user" + - callKeyword: + keyword: "Login" + nodeName: "Log in once per virtual user" + inputs: + - user: + expression: "myShopper.Username" + - password: + expression: "myShopper.Password" + children: + - assert: + actual: "account" + operator: BEGINS_WITH + expected: "shopper" + + children: + - callKeyword: + keyword: "Place Order" + inputs: + - productId: + expression: "myShopper.ProductId" + children: + - assert: + actual: "product" + operator: BEGINS_WITH + expected: "P-" + customErrorMessage: "The account was lost between iterations - was the pull moved into the loop body?" + + after: + continueOnError: true + steps: + # Two logins for six orders: the proof that the pull ran + # once per virtual user rather than once per iteration. + - performanceAssert: + nodeName: "One login per virtual user" + measurementName: "Login" + aggregator: COUNT + comparator: EQUALS + expectedValue: 2 + - performanceAssert: + nodeName: "Six orders in total" + measurementName: "Place Order" + aggregator: COUNT + comparator: EQUALS + expectedValue: 6 + + # ========================================================================= + # D. What happens when the pool runs dry. (INTENTIONALLY FAILS) + # + # With `resetAtEnd: false`, `.next()` past the last row does NOT stop the + # thread group and does NOT raise an error. It returns null, and the run + # carries on feeding null into the keywords. + # + # Depending on the keyword that is either a confusing NullPointerException + # deep in the report, or - far worse - a keyword that shrugs, sends an + # empty value and keeps the run green while half the load was meaningless. + # + # The pool below has 4 rows and the thread group asks for 6. This plan + # fails on purpose, at the `check` guarding the pull - which is the cheap + # way to make the problem loud instead of silent. + # + # In a real test, decide deliberately: + # resetAtEnd: true recycle the rows - fine when the target does not + # care that the same account comes back + # resetAtEnd: false each row used at most once - then size the pool for + # the whole run, and guard the pull like this + # ========================================================================= + - name: "D - The pool runs dry (intentionally fails)" + categories: ["Load testing"] + root: + testCase: + nodeName: "Six iterations against a pool of four" + children: + + - dataSet: + nodeName: "Shopper accounts used once each" + item: "shopperPool" + resetAtEnd: false + dataSource: + csv: + file: "data/users.csv" + + - threadGroup: + nodeName: "1 user asking for 6 accounts" + users: 1 + iterations: 6 + children: + - set: + key: shopper + value: + expression: "shopperPool.next()" + + - check: + nodeName: "The pool still had a row to give" + expression: "shopper != null" + + - callKeyword: + keyword: "Place Order" + inputs: + - productId: + expression: "shopper.ProductId" + + # ========================================================================= + # E. Other data sources. + # + # Only the `dataSource` block changes; the pull is always `.next()`. + # Available sources: csv, excel, file, folder, gsheet, json, json-array, + # sequence, sql. + # + # `json-array` is used here because it needs no file. In a real load test + # the two that come up most are `csv` - a generated pool checked in next to + # the plan - and `sql`, which reads the pool straight out of the system + # under test: + # + # dataSource: + # sql: + # connectionString: "jdbc:postgresql://db:5432/shop" + # driverClass: "org.postgresql.Driver" + # user: "loadtest" + # password: + # expression: "dbPassword" + # query: "SELECT username, product_id FROM test_accounts" + # + # `sequence` is the one to reach for when the data does not need to exist + # beforehand - a pool of unique order numbers, say. + # ========================================================================= + - name: "E - A pool that needs no file" + categories: ["Load testing"] + root: + testCase: + nodeName: "An inline pool of products" + children: + + - dataSet: + nodeName: "Products" + item: "productPool" + resetAtEnd: true + dataSource: + json-array: + json: '[{"ProductId":"P-LAPTOP"},{"ProductId":"P-PHONE"}]' + + - threadGroup: + nodeName: "1 user ordering 2 products" + users: 1 + iterations: 2 + children: + - set: + key: product + value: + expression: "productPool.next()" + - callKeyword: + keyword: "Place Order" + inputs: + - productId: + expression: "product.ProductId" + children: + - assert: + actual: "product" + operator: BEGINS_WITH + expected: "P-" + + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Two orders were placed" + measurementName: "Place Order" + aggregator: COUNT + comparator: EQUALS + expectedValue: 2 + +keywords: + - GeneralScript: + name: "Login" + scriptLanguage: groovy + scriptFile: keywords/login.groovy + - GeneralScript: + name: "Place Order" + scriptLanguage: groovy + scriptFile: keywords/placeOrder.groovy diff --git a/plans/load-testing/04-test-data-and-datasets/data/users.csv b/plans/load-testing/04-test-data-and-datasets/data/users.csv new file mode 100644 index 0000000..f1b0da8 --- /dev/null +++ b/plans/load-testing/04-test-data-and-datasets/data/users.csv @@ -0,0 +1,5 @@ +Username,Password,ProductId +shopper1,pw-1,P-LAPTOP +shopper2,pw-2,P-PHONE +shopper3,pw-3,P-TABLET +shopper4,pw-4,P-MONITOR diff --git a/plans/load-testing/04-test-data-and-datasets/keywords/login.groovy b/plans/load-testing/04-test-data-and-datasets/keywords/login.groovy new file mode 100644 index 0000000..c4e7e89 --- /dev/null +++ b/plans/load-testing/04-test-data-and-datasets/keywords/login.groovy @@ -0,0 +1,5 @@ +// Simulates a login and echoes the account back, so the plan can prove which +// row of the data set this virtual user was handed. +Thread.sleep(60) +output.add("account", input.getString("user", "NONE")) +output.add("status", "LOGGED_IN") diff --git a/plans/load-testing/04-test-data-and-datasets/keywords/placeOrder.groovy b/plans/load-testing/04-test-data-and-datasets/keywords/placeOrder.groovy new file mode 100644 index 0000000..63ffecc --- /dev/null +++ b/plans/load-testing/04-test-data-and-datasets/keywords/placeOrder.groovy @@ -0,0 +1,4 @@ +// Simulates placing an order for one product, echoing the product back. +Thread.sleep(100) +output.add("product", input.getString("productId", "NONE")) +output.add("status", "CONFIRMED") diff --git a/plans/load-testing/05-measurements/README.md b/plans/load-testing/05-measurements/README.md new file mode 100644 index 0000000..8fda418 --- /dev/null +++ b/plans/load-testing/05-measurements/README.md @@ -0,0 +1,126 @@ +--- +use-case: load-testing +focus: plans +framework: none +language: groovy +target-platform: api +approach: keyword-driven +level: advanced +--- + +# 05 — Measurements + +Everything a load test reports comes out of measurements. Which ones exist, what they are called +and how they nest is decided partly by Step and partly by the plan — and getting it wrong is how +you end up with a run full of numbers that cannot answer the question you ran it to answer. + +## The plans + +| Plan | Shows | +|------|-------| +| A — Where measurements come from | The three sources side by side, and which ones carry an SLA | +| B — A custom measurement around a whole transaction | `startMeasure` / `stopMeasure`, nested | +| C — Count the failures too | Why a response-time average alone is the most misleading number in load testing | + +All three pass. + +## The three sources + +| Source | Named after | Created by | Can carry a `performanceAssert` | +|--------|-------------|-----------|-------------------------------| +| Keyword call | the **keyword** | Step, automatically, for every call | **yes** | +| Instrumented node | the node's `nodeName` | `instrumentNode: true` | **no** | +| Custom | whatever the keyword chooses | `output.startMeasure(...)` in the keyword | **yes** | + +A keyword measurement is named after the **keyword**, never the node. Two calls to `Checkout` from +different nodes land in the same series — usually what you want, occasionally not. + +`instrumentNode: true` on any node times that node and reports it under its `nodeName`. This is +how a multi-keyword transaction gets an end-to-end number in the dashboards. + +Custom measurements are the only way to time something **smaller** than a keyword call — the two +page loads inside one browser keyword, for instance. + +## The difference that catches people out + +`performanceAssert` sees keyword measurements and custom measurements. It does **not** see +instrumented-node measurements. Asserting on an instrumented sequence fails with: + +``` +No measurement is matching the defined filters. +``` + +which is the same message a misspelled keyword name produces — so the mistake reads as a typo and +gets "fixed" by changing a name that was already right. + +**Instrumented nodes are for the dashboards. SLAs go on keyword or custom measurements.** To put a +threshold on a whole multi-step transaction, emit a custom measurement around it from inside the +keyword — which is what plan B does. ([06](../06-thresholds-and-slas/) shows the other route: +`assertMetric`, which *can* read instrumented nodes but has different rules.) + +## Custom measurements + +```groovy +output.startMeasure("Catalog page") +// ... do the work +output.stopMeasure() + +output.startMeasure("Product page") +// ... do the work +output.stopMeasure(["page": "product-detail"]) // with attributes +``` + +Measurements **nest**: `stopMeasure()` closes the most recently opened one. Plan B's keyword wraps +three inner steps in one `Purchase journey`, so the report shows both the end-to-end journey and +where inside it the time went — and the journey, being a custom measurement, can carry the SLA +that the equivalent instrumented sequence could not. + +The plan proves the nesting rather than asserting it in prose: the journey's `MIN` must exceed the +sum of its parts' sleeps, and the two page measurements in plan A are bounded on opposite sides of +the same threshold, so they cannot be the same number reported twice. + +## Naming measurements + +A measurement name is the axis every dashboard, threshold and comparison is grouped by. The names +**are** the report. Two rules pay for themselves: + +**Name the business step, not the technical one.** `Checkout` tells you what broke. +`POST /api/v3/order` tells you where, which you can find out afterwards anyway. + +**Keep the name stable and bounded.** A name built out of a variable — `Checkout for shopper 417` +— creates one series per virtual user. Nothing aggregates, the dashboards fill with noise, and no +threshold can be written against it. Put the varying part in an **attribute** instead: + +```groovy +output.stopMeasure(["region": "eu"]) +``` + +## Measure the failures, not just the successes + +A response-time average computed over successful calls only is the most comfortable and most +misleading number in load testing. When a system under stress starts failing fast, **the average +improves.** + +Two habits prevent it: + +- **Assert the count as well as the time.** A run that did half the work has a great average. + Every plan in this set carries a `COUNT` assertion for that reason. +- **Check the answers**, so failures are counted as failures. A nested `assert` on the keyword + output turns a wrong response into a failure count in the report, next to the timings. + +[06](../06-thresholds-and-slas/) puts thresholds on that failure count. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | Three plans on where measurements come from and what they are worth | +| `keywords/browseCatalog.groovy` | Two custom measurements inside one keyword call | +| `keywords/completePurchase.groovy` | A custom measurement wrapping three nested ones | +| `keywords/checkout.groovy` | A plain keyword — its measurement is created for it | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` diff --git a/plans/load-testing/05-measurements/automation-package.yaml b/plans/load-testing/05-measurements/automation-package.yaml new file mode 100644 index 0000000..ae9dad5 --- /dev/null +++ b/plans/load-testing/05-measurements/automation-package.yaml @@ -0,0 +1,264 @@ +--- +# --------------------------------------------------------------------------- +# Load-testing sample 05 - Measurements +# +# Everything a load test reports comes out of measurements. Which ones exist, +# what they are called, and how they nest is decided partly by Step and partly +# by the plan - and getting it wrong is how you end up with a run full of +# numbers that cannot answer the question you ran it to answer. +# +# There are three sources of measurements, and only two of them can carry an +# SLA. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "load-05-measurements" + +plans: + + # ========================================================================= + # A. The three sources, side by side. + # + # 1. AUTOMATIC, per keyword call. + # Every `callKeyword` is timed and reported under the KEYWORD's name - + # never the node name. Two calls to "Checkout" from different nodes + # land in the same series. + # + # 2. INSTRUMENTED NODES, per plan node. + # `instrumentNode: true` on any node times that node and reports it + # under its `nodeName`. This is how a multi-keyword transaction gets an + # end-to-end number. + # + # 3. CUSTOM, from inside the keyword. + # `output.startMeasure(name)` / `output.stopMeasure()` create + # measurements the keyword decides on. This is the only way to time + # something SMALLER than a keyword call - the two page loads inside one + # browser keyword, for instance. + # + # THE DIFFERENCE THAT MATTERS: `performanceAssert` sees keyword + # measurements and custom measurements. It does NOT see instrumented-node + # measurements. Asserting on "Search and buy" below would fail with + # "No measurement is matching the defined filters." - the same message a + # misspelled keyword name produces, which is why this catches people out. + # + # So instrumented nodes are for the dashboards; SLAs go on keyword or + # custom measurements. + # ========================================================================= + - name: "A - Where measurements come from" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "2 users x 2 iterations" + users: 2 + iterations: 2 + children: + + # (2) Instrumented node: one measurement named "Search and buy", + # covering both keyword calls below it. + - sequence: + nodeName: "Search and buy" + instrumentNode: true + children: + + # (3) The keyword creates its own measurements - see + # keywords/browseCatalog.groovy. The call still produces its + # own automatic "Browse Catalog" measurement as well, so this + # one call yields three series. + - callKeyword: + keyword: "Browse Catalog" + nodeName: "Browse two pages" + + # (1) Automatic: one measurement named "Checkout". + - callKeyword: + keyword: "Checkout" + nodeName: "Check out" + + after: + continueOnError: true + steps: + + # A keyword measurement. + - performanceAssert: + nodeName: "Checkout ran on every iteration" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 4 + + # A custom measurement, created by the keyword. Note the name is + # the one the SCRIPT chose, not the keyword name. + - performanceAssert: + nodeName: "The catalog page was measured separately" + measurementName: "Catalog page" + aggregator: COUNT + comparator: EQUALS + expectedValue: 4 + + - performanceAssert: + nodeName: "And so was the product page" + measurementName: "Product page" + aggregator: COUNT + comparator: EQUALS + expectedValue: 4 + + # The product page sleeps longer than the catalog page, so this + # pair proves the two custom measurements are really independent + # timings rather than the same number reported twice. + - performanceAssert: + nodeName: "The product page is the slower of the two" + measurementName: "Product page" + aggregator: MIN + comparator: HIGHER_THAN + expectedValue: 120 + - performanceAssert: + nodeName: "The catalog page is the faster of the two" + measurementName: "Catalog page" + aggregator: MIN + comparator: LOWER_THAN + expectedValue: 120 + + # ------------------------------------------------------------- + # The one that does NOT work, kept here as a comment because it + # fails the execution: + # + # - performanceAssert: + # measurementName: "Search and buy" # instrumented node + # aggregator: AVG + # comparator: LOWER_THAN + # expectedValue: 5000 + # + # -> "No measurement is matching the defined filters." + # + # To put a threshold on a whole transaction, emit a custom + # measurement around it from inside the keyword instead - which + # is what plan B is about. + # ------------------------------------------------------------- + + # ========================================================================= + # B. Naming measurements so the report can be read. + # + # A measurement name is the axis every dashboard, threshold and comparison + # is grouped by, so the names ARE the report. Two rules pay for themselves: + # + # Name the business step, not the technical one. + # "Checkout" tells you what broke. "POST /api/v3/order" tells you where, + # which you can find out afterwards anyway. + # + # Keep the name STABLE and BOUNDED. + # A name built out of a variable - "Checkout for shopper 417" - creates + # one series per virtual user. Nothing aggregates, the dashboards fill + # with noise, and no threshold can be written against it. Put the + # varying part in a measurement ATTRIBUTE instead: + # + # output.stopMeasure(["region": "eu"]) + # + # A keyword can also be called from several places in a plan. All those + # calls share one measurement name, which is usually what you want - and + # occasionally is not. When "Checkout" is called both as a warm-up and as + # the measured transaction, the warm-up timings land in the SLA. Sample 03 + # plan D shows the fix: keep them in separate thread groups and put the + # threshold on the measured one. + # ========================================================================= + - name: "B - A custom measurement around a whole transaction" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "2 users x 2 iterations" + users: 2 + iterations: 2 + children: + - callKeyword: + keyword: "Complete Purchase" + nodeName: "Search, add to cart and buy in one keyword" + + after: + continueOnError: true + steps: + # "Purchase journey" is created by the keyword with + # startMeasure/stopMeasure, so - unlike an instrumented sequence - + # it CAN carry a threshold. + - performanceAssert: + nodeName: "The whole journey stays under 5s" + measurementName: "Purchase journey" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 5000 + - performanceAssert: + nodeName: "Every iteration produced a journey" + measurementName: "Purchase journey" + aggregator: COUNT + comparator: EQUALS + expectedValue: 4 + # The journey wraps three inner steps, so it must be slower than + # any of them. This is what proves the nesting is real. + - performanceAssert: + nodeName: "The journey covers more than its slowest step" + measurementName: "Purchase journey" + aggregator: MIN + comparator: HIGHER_THAN + expectedValue: 250 + + # ========================================================================= + # C. Measuring the failures, not just the successes. + # + # A response time average computed over successful calls only is the most + # comfortable and most misleading number in load testing. When a system + # under stress starts failing fast, the average IMPROVES. + # + # Two habits prevent it: + # + # Assert the count as well as the time. A run that did half the work has + # a great average - see the COUNT assertions in every plan here. + # + # Check the answers, so failures are counted as failures. The keyword + # below fails every third iteration; the nested `assert` turns that into + # a failure count in the report, next to the timings. + # + # Sample 06 puts thresholds on that failure count. + # ========================================================================= + - name: "C - Count the failures too" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "2 users x 2 iterations against a wobbly service" + users: 2 + iterations: 2 + children: + - callKeyword: + keyword: "Checkout" + nodeName: "Check out" + children: + # Without this the run is green whatever comes back. + - assert: + actual: "status" + operator: EQUALS + expected: "CONFIRMED" + + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "The work was actually done" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 4 + - performanceAssert: + nodeName: "The slowest checkout was still acceptable" + measurementName: "Checkout" + aggregator: MAX + comparator: LOWER_THAN + expectedValue: 10000 + +keywords: + - GeneralScript: + name: "Browse Catalog" + scriptLanguage: groovy + scriptFile: keywords/browseCatalog.groovy + - GeneralScript: + name: "Checkout" + scriptLanguage: groovy + scriptFile: keywords/checkout.groovy + - GeneralScript: + name: "Complete Purchase" + scriptLanguage: groovy + scriptFile: keywords/completePurchase.groovy diff --git a/plans/load-testing/05-measurements/keywords/browseCatalog.groovy b/plans/load-testing/05-measurements/keywords/browseCatalog.groovy new file mode 100644 index 0000000..8772b68 --- /dev/null +++ b/plans/load-testing/05-measurements/keywords/browseCatalog.groovy @@ -0,0 +1,9 @@ +// Two custom measurements inside a single keyword call, so the report shows +// the two pages separately instead of one opaque "Browse Catalog" total. +output.startMeasure("Catalog page") +Thread.sleep(90) +output.stopMeasure() +output.startMeasure("Product page") +Thread.sleep(140) +output.stopMeasure(["page": "product-detail"]) +output.add("status", "OK") diff --git a/plans/load-testing/05-measurements/keywords/checkout.groovy b/plans/load-testing/05-measurements/keywords/checkout.groovy new file mode 100644 index 0000000..3a0a41e --- /dev/null +++ b/plans/load-testing/05-measurements/keywords/checkout.groovy @@ -0,0 +1,4 @@ +// One keyword call - one measurement, named after the keyword, created by +// Step itself. Nothing in this script asks for it. +Thread.sleep(120) +output.add("status", "CONFIRMED") diff --git a/plans/load-testing/05-measurements/keywords/completePurchase.groovy b/plans/load-testing/05-measurements/keywords/completePurchase.groovy new file mode 100644 index 0000000..fa8e1c2 --- /dev/null +++ b/plans/load-testing/05-measurements/keywords/completePurchase.groovy @@ -0,0 +1,16 @@ +// One custom measurement wrapping three inner ones, so the report shows both +// the end-to-end journey and where inside it the time went. +// +// Measurements nest: stopMeasure() closes the most recently opened one. +output.startMeasure("Purchase journey") +output.startMeasure("Search") +Thread.sleep(80) +output.stopMeasure() +output.startMeasure("Add to cart") +Thread.sleep(60) +output.stopMeasure() +output.startMeasure("Pay") +Thread.sleep(140) +output.stopMeasure(["paymentMethod": "card"]) +output.stopMeasure() +output.add("status", "CONFIRMED") diff --git a/plans/load-testing/06-thresholds-and-slas/README.md b/plans/load-testing/06-thresholds-and-slas/README.md new file mode 100644 index 0000000..78b9628 --- /dev/null +++ b/plans/load-testing/06-thresholds-and-slas/README.md @@ -0,0 +1,167 @@ +--- +use-case: load-testing +focus: plans +framework: none +language: groovy +target-platform: api +approach: keyword-driven +level: advanced +--- + +# 06 — Thresholds and SLA gates + +A load test that produces a report somebody has to interpret is a load test nobody runs twice. A +load test that comes back PASSED or FAILED is a gate you can put in a pipeline. + +This sample covers the two threshold controls, what each can and cannot see, and the thresholds a +load test needs beyond "average response time". + +> **Two plans in this package fail on purpose.** That is the lesson in them: the plan names say +> which ones. + +## The plans + +| Plan | Expected outcome | Shows | +|------|------------------|-------| +| A — A complete SLA gate | **PASSED** | All five aggregators, and which thresholds are worth writing | +| B — One SLA per population | **PASSED** | Per-thread-group `after` blocks in a scenario | +| C — Threshold on the failure rate | **FAILED** (on purpose) | Counting successes, and why a run with real errors is red anyway | +| D — A threshold over the metric time series | **PASSED** | `assertMetric`, and how it differs from `performanceAssert` | +| E — A breached SLA | **FAILED** (on purpose) | What a violation looks like in the report | + +## The aggregators + +| Aggregator | What it is good for | +|------------|---------------------| +| `AVG` | the headline number — and the one that hides the tail. An average of 800 ms is compatible with one user in twenty waiting eight seconds. | +| `MAX` | the worst single transaction. Brutal, and useful precisely because it is: a MAX threshold catches the timeout nobody saw. | +| `MIN` | a floor. Mostly used to prove the measurement is real rather than empty. | +| `COUNT` | how many transactions happened — the throughput check, and the one most often forgotten. | +| `SUM` | total time spent. Rarely a threshold; useful for capacity sums. | + +**A good gate uses several together.** Response time on its own can always be met by doing less +work — which is exactly what a struggling system does. + +Set `continueOnError: true` on the `after` block. Without it the block stops at the first breach, +and a run that violates three thresholds tells you about one of them. Plan E shows the payoff: two +breaches reported in one run, plus a third threshold that passed. + +## Where the threshold lives + +| Placement | Evaluated | +|-----------|-----------| +| `after` on a **thread group** | when that population finishes | +| `after` on a **testScenario** | once, when every thread group has finished | +| `afterThread` | once per virtual user | + +Different populations have different SLAs — search may have to answer in under a second while a +checkout is allowed three. Putting both in the scenario's `after` block works, but the threshold +then sits far from the load it describes. Each thread group's own `after` block is usually the +better home. + +## Failure rate as a threshold + +"Under 1% errors" is part of every real SLA, and it is not a response time. There is no +failure-rate aggregator, so the pattern is to make the failures countable and then count them: + +| Number | Where it comes from | +|--------|---------------------| +| attempts | `COUNT` of the keyword measurement — every call, successful or not | +| successes | `COUNT` of a measurement the keyword emits **only** on success | + +```groovy +// in the keyword, on the success path only +output.startMeasure("Payment accepted") +output.stopMeasure() +``` + +Six attempts, two of which fail, leaves four successes — so a threshold on the success count is a +threshold on the failure rate. + +`continueParentNodeExecutionOnError: true` on the keyword call is what keeps the thread group +running past a failed iteration. Without it the first failure ends that virtual user, and the run +measures a fraction of the load you asked for — the opposite of what you want when the question is +*how often does it fail?* + +### Why plan C is red even though both thresholds pass + +Two payments genuinely failed, and a failed node fails the run. Step has no error-budget flag that +makes a run with real errors come back green. The thresholds tell you the failure *rate* was +acceptable; the run status tells you failures happened. Both are true and both are worth +reporting. + +If a rejection is **expected traffic** rather than an error — a payment the business means to +decline — do not raise a business error for it. Return it as an ordinary output and branch on it in +the plan, and the run status then reflects your error budget rather than the provider's mood. + +## `performanceAssert` vs `assertMetric` + +They look similar and answer different questions. + +| | `performanceAssert` | `assertMetric` | +|---|---|---| +| Reads | the measurements **of this execution** | the **metric time series** | +| Scope | this run only | **not scoped to this run** — the series spans every execution | +| Sees instrumented-node measurements | no | **yes** | +| Placement | must be in `after` / `afterThread` | anywhere | +| Answers | *did this run meet its SLA?* | *is the system drifting?* | + +```yaml +- assertMetric: + metric: "response-time" # required + aggregation: AVG # required + comparator: LOWER_THAN + expectedValue: 60000 + filters: + - field: "name" # `name` selects a measurement name + filterType: EQUALS + filter: "Search transaction" +``` + +Two things to know before using `assertMetric` as a gate: + +1. **It is not scoped to the current execution.** A `COUNT` over a measurement name this run + produced twice comes back with every point the series holds, from every run that ever used the + same name. Count assertions are meaningless here — use `performanceAssert` for those. +2. **It reaches measurements `performanceAssert` cannot**, including instrumented-node + measurements, and it can read gauges and counters a keyword pushed. Together with + `slidingWindow`, that is its real use: thresholds over *time* — is the response time drifting up + across the last ten minutes, across yesterday's run and today's — which is what monitoring and + alerting rules are built out of. + +Note `field: "name"` and not `attributes.name`; the latter matches nothing and reports +`No metric found for the defined filters`. + +## What a breach looks like + +``` +Average of Slow Search expected to be lower than 100 but was 1672 +Max of Slow Search expected to be lower than 200 but was 2894 +``` + +The control reports the **actual value** alongside the threshold, so a breached SLA tells you how +far off it was without opening the dashboards. + +An empty series reads differently: + +``` +No measurement is matching the defined filters. +``` + +That is what you get from a misspelled measurement name, from an instrumented-node name, and from +a success measurement that was never emitted — three quite different problems with one message. + +## Key files + +| File | Purpose | +|------|---------| +| `automation-package.yaml` | Five plans covering both threshold controls | +| `keywords/checkout.groovy`, `searchProducts.groovy` | Transactions that meet their SLA | +| `keywords/slowSearch.groovy` | Always breaches its threshold | +| `keywords/flakyPayment.groovy` | Fails every third iteration, and emits a success measurement otherwise | + +## Running it + +```bash +step ap execute -p . -u --token --projectName +``` diff --git a/plans/load-testing/06-thresholds-and-slas/automation-package.yaml b/plans/load-testing/06-thresholds-and-slas/automation-package.yaml new file mode 100644 index 0000000..ccaeefb --- /dev/null +++ b/plans/load-testing/06-thresholds-and-slas/automation-package.yaml @@ -0,0 +1,385 @@ +--- +# --------------------------------------------------------------------------- +# Load-testing sample 06 - Thresholds and SLA gates +# +# A load test that produces a report somebody has to interpret is a load test +# nobody runs twice. A load test that comes back PASSED or FAILED is a gate +# you can put in a pipeline. +# +# This sample covers the two threshold controls, what they can and cannot see, +# and the thresholds a load test needs beyond "average response time". +# +# TWO PLANS HERE FAIL ON PURPOSE. That is the lesson in them: the plan names +# say which ones. +# --------------------------------------------------------------------------- +version: "1.2.0" +name: "load-06-thresholds-and-slas" + +plans: + + # ========================================================================= + # A. The aggregators, and the thresholds worth writing. + # + # AVG the headline number - and the one that hides the tail. An + # average of 800 ms is compatible with one user in twenty waiting + # eight seconds. + # MAX the worst single transaction. Brutal, and useful precisely + # because it is: a MAX threshold catches the timeout nobody saw. + # MIN a floor. Mostly used, as below, to prove the measurement is real + # rather than empty. + # COUNT how many transactions happened - the throughput check, and the + # one most often forgotten. + # SUM total time spent. Rarely a threshold; useful for capacity sums. + # + # A good gate uses several of these together. Response time on its own can + # always be met by doing less work. + # ========================================================================= + - name: "A - A complete SLA gate" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "2 users x 3 checkouts" + users: 2 + iterations: 3 + children: + - callKeyword: + keyword: "Checkout" + children: + # Correctness is part of the SLA. A fast error is not a pass. + - assert: + actual: "status" + operator: EQUALS + expected: "CONFIRMED" + + after: + # Without this the block stops at the first breach, and a run that + # violates three thresholds only tells you about one of them. + continueOnError: true + steps: + + - performanceAssert: + nodeName: "Throughput - all the work was done" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 6 + + - performanceAssert: + nodeName: "Average response time under 5s" + measurementName: "Checkout" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 5000 + + - performanceAssert: + nodeName: "No single checkout over 15s" + measurementName: "Checkout" + aggregator: MAX + comparator: LOWER_THAN + expectedValue: 15000 + + - performanceAssert: + nodeName: "The measurement is real, not empty" + measurementName: "Checkout" + aggregator: MIN + comparator: HIGHER_THAN + expectedValue: 50 + + # ========================================================================= + # B. Per-population thresholds in a scenario. + # + # Different populations have different SLAs. Search may have to answer in + # under a second while a checkout is allowed three. Putting both thresholds + # in the scenario's `after` block works, but the threshold then sits far + # from the load it describes. + # + # Each thread group's own `after` block is usually the better home: the + # threshold reads next to the profile it belongs to, and it is evaluated as + # soon as that population finishes rather than at the end of the scenario. + # ========================================================================= + - name: "B - One SLA per population" + categories: ["Load testing"] + root: + testScenario: + nodeName: "Two populations with two SLAs" + children: + + - threadGroup: + nodeName: "Searchers" + users: 2 + iterations: 2 + children: + - callKeyword: + keyword: "Search Products" + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Search answers within its own budget" + measurementName: "Search Products" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 5000 + - performanceAssert: + nodeName: "All searches ran" + measurementName: "Search Products" + aggregator: COUNT + comparator: EQUALS + expectedValue: 4 + + - threadGroup: + nodeName: "Buyers" + users: 1 + iterations: 2 + children: + - callKeyword: + keyword: "Checkout" + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Checkout has a looser budget than search" + measurementName: "Checkout" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 8000 + - performanceAssert: + nodeName: "All checkouts ran" + measurementName: "Checkout" + aggregator: COUNT + comparator: EQUALS + expectedValue: 2 + + # ========================================================================= + # C. Failure rate as a threshold. (INTENTIONALLY FAILS) + # + # "Under 1% errors" is part of every real SLA, and it is not a response + # time. There is no failure-rate aggregator, so the pattern is to make the + # failures countable and then count them. + # + # WHY THIS PLAN STILL REPORTS FAILED, EVEN THOUGH BOTH THRESHOLDS PASS. + # Two payments genuinely failed, and a failed node fails the run. Step has + # no error-budget flag that makes a run with real errors come back green: + # the thresholds tell you the failure RATE was acceptable, the run status + # tells you failures happened. Both are true and both are worth reporting. + # + # If a rejection is EXPECTED traffic rather than an error - a payment the + # business means to decline - do not raise a business error for it. Return + # it as an ordinary output and branch on it in the plan, and the run status + # then reflects your error budget rather than the provider's mood. + # + # Here "Flaky Payment" fails on every third iteration, and the plan checks + # the successes separately from the attempts: + # + # attempts the COUNT of the keyword measurement - every call, whether + # it succeeded or not + # successes the COUNT of a measurement the keyword only emits when the + # payment went through + # + # 6 attempts, 2 of which fail, leaves 4 successes - so a threshold on the + # success count is a threshold on the failure rate. + # + # `continueParentNodeExecutionOnError` on the keyword call is what keeps + # the thread group running past a failed iteration. Without it the first + # failure ends that virtual user, and the run measures a fraction of the + # load you asked for - which is the opposite of what you want when the + # question is "how often does it fail?". + # ========================================================================= + - name: "C - Threshold on the failure rate (intentionally fails)" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "2 users x 3 payments against a wobbly provider" + users: 2 + iterations: 3 + children: + - callKeyword: + keyword: "Flaky Payment" + nodeName: "Pay" + continueParentNodeExecutionOnError: true + inputs: + # The iteration number comes from the plan, so the keyword + # stays stateless and the failures land on fixed iterations. + # + # `as Integer` is not decoration. Without it the counter + # reaches the keyword as a STRING, `input.getInt` silently + # falls back to its default, and every iteration takes the + # same branch - a load test that looks uniformly broken for + # a reason that has nothing to do with the system under test. + - iteration: + expression: "gcounter as Integer" + + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Every payment was attempted" + measurementName: "Flaky Payment" + aggregator: COUNT + comparator: EQUALS + expectedValue: 6 + + # 6 attempts, 2 failures, 4 successes: a 33% failure rate. + # Tighten this number and the assert catches a provider that got + # worse. + # + # If the success measurement is never emitted at all, this fails + # with "No measurement is matching the defined filters." rather + # than with a count of zero - an empty series and a misspelled + # name look identical here. + - performanceAssert: + nodeName: "4 of the 6 payments went through" + measurementName: "Payment accepted" + aggregator: COUNT + comparator: EQUALS + expectedValue: 4 + + # ========================================================================= + # D. `assertMetric` - the other threshold control, and what it is for. + # + # `performanceAssert` aggregates the measurements OF THIS EXECUTION. + # `assertMetric` queries the METRIC TIME SERIES, which is a different thing + # with different rules: + # + # metric the series to query, e.g. "response-time" (required) + # aggregation AVG | MAX | MIN | COUNT | SUM (required) + # comparator LOWER_THAN | HIGHER_THAN | EQUALS + # expectedValue the threshold + # filters narrow the series - `field: "name"` selects a + # measurement name + # slidingWindow query a rolling window rather than the whole range + # + # TWO THINGS TO KNOW BEFORE USING IT AS A GATE: + # + # 1. It is NOT scoped to the current execution. A COUNT over a measurement + # name this run produced twice comes back with every point the series + # holds, from every run that used the same name. Count assertions are + # therefore not meaningful here - use `performanceAssert` for those. + # + # 2. It reaches measurements `performanceAssert` cannot, including + # instrumented-node measurements, and it can read gauges and counters + # a keyword pushed. That is its real use: thresholds over + # TIME - is the response time drifting up across the last ten minutes, + # across yesterday's run and today's - which is what monitoring and + # alerting rules are built out of. + # + # The rule of thumb: `performanceAssert` for "did THIS run meet its SLA", + # `assertMetric` for "is the system drifting". + # + # Unlike `performanceAssert`, it does not have to sit in an `after` block - + # but keeping it there is still the right place for a gate. + # ========================================================================= + - name: "D - A threshold over the metric time series" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "2 users x 2 searches" + users: 2 + iterations: 2 + children: + - sequence: + nodeName: "Search transaction" + instrumentNode: true + children: + - callKeyword: + keyword: "Search Products" + + after: + continueOnError: true + steps: + # An instrumented-node measurement - which performanceAssert + # would refuse - carrying a threshold via assertMetric. + - assertMetric: + nodeName: "The search transaction is not drifting" + metric: "response-time" + aggregation: AVG + comparator: LOWER_THAN + expectedValue: 60000 + filters: + - field: "name" + filterType: EQUALS + filter: "Search transaction" + + # The same run gated the reliable way, for comparison. + - performanceAssert: + nodeName: "And this run met its own SLA" + measurementName: "Search Products" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 5000 + - performanceAssert: + nodeName: "All four searches ran" + measurementName: "Search Products" + aggregator: COUNT + comparator: EQUALS + expectedValue: 4 + + # ========================================================================= + # E. A breached threshold. (INTENTIONALLY FAILS) + # + # What a violation looks like, so you recognise it in a report. "Slow + # Search" takes about 600 ms and the threshold below allows 100, so this + # plan fails with: + # + # Average of Slow Search expected to be lower than 100 but was + # + # Note the shape of that message: the control reports the ACTUAL value + # alongside the threshold, so a breached SLA tells you how far off it was + # without opening the dashboards. + # + # `continueOnError: true` on the `after` block matters most exactly here. + # The second threshold below is also breached, and without the flag you + # would fix the first one, re-run, and only then discover the second. + # ========================================================================= + - name: "E - A breached SLA (intentionally fails)" + categories: ["Load testing"] + root: + threadGroup: + nodeName: "2 users x 2 slow searches" + users: 2 + iterations: 2 + children: + - callKeyword: + keyword: "Slow Search" + + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Average must stay under 100ms - it does not" + measurementName: "Slow Search" + aggregator: AVG + comparator: LOWER_THAN + expectedValue: 100 + - performanceAssert: + nodeName: "Worst case must stay under 200ms - it does not" + measurementName: "Slow Search" + aggregator: MAX + comparator: LOWER_THAN + expectedValue: 200 + # This one passes, which is the point: the run reports exactly + # which thresholds failed, not just that something did. + - performanceAssert: + nodeName: "All four searches ran - this one passes" + measurementName: "Slow Search" + aggregator: COUNT + comparator: EQUALS + expectedValue: 4 + +keywords: + - GeneralScript: + name: "Checkout" + scriptLanguage: groovy + scriptFile: keywords/checkout.groovy + - GeneralScript: + name: "Search Products" + scriptLanguage: groovy + scriptFile: keywords/searchProducts.groovy + - GeneralScript: + name: "Slow Search" + scriptLanguage: groovy + scriptFile: keywords/slowSearch.groovy + - GeneralScript: + name: "Flaky Payment" + scriptLanguage: groovy + scriptFile: keywords/flakyPayment.groovy diff --git a/plans/load-testing/06-thresholds-and-slas/keywords/checkout.groovy b/plans/load-testing/06-thresholds-and-slas/keywords/checkout.groovy new file mode 100644 index 0000000..81be19c --- /dev/null +++ b/plans/load-testing/06-thresholds-and-slas/keywords/checkout.groovy @@ -0,0 +1,3 @@ +// A checkout that comfortably meets its SLA. +Thread.sleep(120) +output.add("status", "CONFIRMED") diff --git a/plans/load-testing/06-thresholds-and-slas/keywords/flakyPayment.groovy b/plans/load-testing/06-thresholds-and-slas/keywords/flakyPayment.groovy new file mode 100644 index 0000000..17d28cb --- /dev/null +++ b/plans/load-testing/06-thresholds-and-slas/keywords/flakyPayment.groovy @@ -0,0 +1,20 @@ +// Fails on every third iteration, so the failure-rate threshold has something +// to measure. The iteration number comes from the plan, so the keyword stays +// stateless. +// +// "Payment accepted" is emitted only on success, which is what makes the +// success count - and therefore the failure rate - assertable from the plan. +Thread.sleep(80) +def iteration = input.getInt("iteration", -1) +if (iteration < 0) { + // Distinguishing a mis-wired input from a simulated rejection matters: + // getInt falls back to its default when the value did not arrive as a + // NUMBER, and without this the plan would just look uniformly broken. + output.setBusinessError("No usable iteration number arrived - check that the input coerces the counter.") +} else if (iteration % 3 == 0) { + output.setBusinessError("Payment provider rejected the transaction.") +} else { + output.startMeasure("Payment accepted") + output.stopMeasure() + output.add("status", "PAID") +} diff --git a/plans/load-testing/06-thresholds-and-slas/keywords/searchProducts.groovy b/plans/load-testing/06-thresholds-and-slas/keywords/searchProducts.groovy new file mode 100644 index 0000000..87d9561 --- /dev/null +++ b/plans/load-testing/06-thresholds-and-slas/keywords/searchProducts.groovy @@ -0,0 +1,3 @@ +// Simulates a product search against the shop API. +Thread.sleep(80) +output.add("resultCount", 12) diff --git a/plans/load-testing/06-thresholds-and-slas/keywords/slowSearch.groovy b/plans/load-testing/06-thresholds-and-slas/keywords/slowSearch.groovy new file mode 100644 index 0000000..0b9d789 --- /dev/null +++ b/plans/load-testing/06-thresholds-and-slas/keywords/slowSearch.groovy @@ -0,0 +1,3 @@ +// A search that always breaches its SLA - the point of the failing plan. +Thread.sleep(600) +output.add("resultCount", 12) diff --git a/plans/load-testing/README.md b/plans/load-testing/README.md new file mode 100644 index 0000000..5847745 --- /dev/null +++ b/plans/load-testing/README.md @@ -0,0 +1,116 @@ +# Load-testing plan samples + +Six small Automation Packages, each teaching one aspect of writing a **Step plan** for load and +performance testing. Read them in order — each builds on the one before. + +The subject is the **plan**, not the keywords. Every keyword here is a 3-line Groovy +`GeneralScript` stub simulating a shop API, so every package runs on any Java agent with **no +build, no browser and no system under test**. Every plan in the set is executable, and each one +asserts its own outcome rather than merely running. + +## The samples + +| # | Sample | Level | Controls covered | +|---|--------|-------|------------------| +| 01 | [First load test](01-first-load-test/) | beginner | `threadGroup`, `users`, `iterations`, `instrumentNode`, `performanceAssert` | +| 02 | [Configuring a thread group](02-thread-group-configuration/) | beginner | `pacing`, `rampup`, `pack`, `startOffset`, `maxDuration`, `before` / `beforeThread` / `afterThread` / `after`, the counters | +| 03 | [Scenarios and mixed load](03-scenarios-and-mixed-load/) | intermediate | `testScenario`, staged ramps, `sequence` as phases, `testSet` | +| 04 | [Test data and data sets](04-test-data-and-datasets/) | intermediate | `dataSet`, `item`, `.next()`, `resetAtEnd`, data sources | +| 05 | [Measurements](05-measurements/) | advanced | keyword / instrumented / custom measurements, `startMeasure`, naming | +| 06 | [Thresholds and SLA gates](06-thresholds-and-slas/) | advanced | aggregators, failure rate, `assertMetric` vs `performanceAssert` | + +For what each control does and how to configure it, see the official +[controls documentation](https://step.dev/knowledgebase/userdocs/plans/controls/). For the YAML +shape of a standalone plan, see [../reference/](../reference/). + +## Control coverage matrix + +| Control | Sample | +|---------|--------| +| `threadGroup` as a plan root | 01, 02, 05, 06 | +| `users`, `iterations` | 01 | +| `pacing`, `rampup`, `pack`, `startOffset`, `maxDuration` | 02 | +| `item`, `userItem`, `localItem` and the `gcounter` / `userId` / `literationId` counters | 02 | +| `before`, `beforeThread`, `afterThread`, `after` | 02, 04 | +| `testScenario`, `sequence` and `testSet` as composing roots | 03 | +| `dataSet` + `.next()`, `resetAtEnd`, `csv` / `json-array` / `sql` | 04 | +| `instrumentNode` | 01, 05, 06 | +| `output.startMeasure` / `stopMeasure` | 05 | +| `performanceAssert` — `AVG`, `MAX`, `MIN`, `COUNT`, `SUM` | 01, 02, 03, 04, 05, 06 | +| `assertMetric` | 06 | +| `assert` inside a load test | 01, 05, 06 | +| `continueOnError`, `continueParentNodeExecutionOnError` | 01, 06 | + +## Running any of them + +```bash +step ap execute -p . -u --token --projectName +``` + +Or point the Step MCP server at the directory and use `step_validate_plan` / +`step_execute_automation_package`. + +Three plans are **expected to fail** — that is the lesson in them. Their names say so, and each +sample's README lists the expected outcome per plan: + +| Sample | Plan | +|--------|------| +| [04](04-test-data-and-datasets/) | D — The pool runs dry | +| [06](06-thresholds-and-slas/) | C — Threshold on the failure rate | +| [06](06-thresholds-and-slas/) | E — A breached SLA | + +Everything else passes. + +## Two rules that are easy to get wrong + +Both cost an afternoon the first time, and neither is visible from the YAML: + +**`performanceAssert` must live in an `after` or `afterThread` block.** Anywhere else — as a child +of the thread group, of a `testCase`, of anything — the execution ends in `TECHNICAL_ERROR` with +`PerformanceAssert can only be defined in an 'after' or 'after thread' block`. + +**`performanceAssert` cannot see `instrumentNode` measurements.** It matches keyword measurements +and custom ones a keyword created itself. Asserting on an instrumented sequence gives +`No measurement is matching the defined filters.` — the same message a misspelled keyword name +produces, so the mistake reads as a typo. [05](05-measurements/) covers the ways round it. + +## What these samples teach + +The controls are the vocabulary; these are the ideas that decide whether a load plan is any good. + +1. **One iteration is one business transaction.** The children of a thread group should be the + smallest thing a *user* would call a complete action — search, add to cart, buy — not a single + HTTP call. That choice decides what the report can answer, because the sum of the parts is not + the experience. + +2. **`children` holds exactly what the SLA is about.** Logging in inside the loop means every + iteration measures a login the real user performs once a day, and the reported average becomes a + blend of two unrelated things. Setup belongs in `beforeThread`, once-per-test setup in `before`. + +3. **Pace the load, or you are not testing — you are being tested.** Without `pacing`, throughput + is whatever the system happens to allow, so two runs cannot be compared and a degrading system + quietly reduces its own load. + +4. **Always assert throughput, not just response time.** A struggling system meets any latency + target by doing less work. `COUNT` is the cheapest check there is and the one most often + missing — and it is essential under `maxDuration`, where the iteration count becomes an outcome + rather than an input. + +5. **A load test still has to check its answers.** A system under stress starts returning fast, + cheap, *wrong* responses; an error page renders quicker than a checkout. Without a functional + `assert` in the loop, the response times look excellent and mean nothing. + +6. **Real traffic is mixed.** Browsers, buyers and the nightly batch hit the system at once. + `testScenario` composes those populations, each with its own profile — modelling them as one + thread group produces a traffic mix nobody ever sees. + +7. **Data is part of the load.** The same account on every iteration measures the target's caches. + A `dataSet` hands out a shared pool; where you pull from it — per iteration or per virtual user + — decides both realism and how much data you need. + +8. **Names are the report.** Every dashboard, threshold and comparison groups by measurement name. + Name the business step rather than the endpoint, and keep the name bounded — a name built from + a variable creates one series per virtual user and nothing aggregates. + +Each sample README calls out the pitfalls for its own controls — the ones that silently do +nothing, and the pairs that are easy to confuse. diff --git a/plans/reference/README.md b/plans/reference/README.md index 12993a3..0699bb3 100644 --- a/plans/reference/README.md +++ b/plans/reference/README.md @@ -10,10 +10,11 @@ documentation: |------|-------| | [basic-plan-syntax.yml](basic-plan-syntax.yml) | The shape of a plan: a root artefact, `callKeyword` with inputs, capturing an output with a nested `set`, `if`, `assert` and `check` | | [dynamic-values.yml](dynamic-values.yml) | Static values vs `expression:`, where plan variables come from, and dynamic keyword names and `routing` | -| [performance-assert.yml](performance-assert.yml) | A `threadGroup` with a `performanceAssert` — the load-testing shape | +| [performance-assert.yml](performance-assert.yml) | A `threadGroup` with a `performanceAssert` — the load-testing shape, and the `after`-block rule | These files are **syntax illustrations, not runnable plans**: the keywords they call do not -exist. For plans that execute and assert their own outcome, see [../rpa/](../rpa/). +exist. For plans that execute and assert their own outcome, see [../rpa/](../rpa/) and +[../load-testing/](../load-testing/). ## Standalone YAML plan files diff --git a/plans/reference/performance-assert.yml b/plans/reference/performance-assert.yml index a4b60b8..2e4c5d3 100644 --- a/plans/reference/performance-assert.yml +++ b/plans/reference/performance-assert.yml @@ -1,16 +1,58 @@ +# A thread group with a performance assert - the load-testing shape. +# +# A file like this can be dropped straight into the Step UI with +# "Add plan" > "Create from YAML", or referenced from an automation package +# like any other plan. +# +# This file is a syntax illustration, not a runnable plan - the keyword it +# calls does not exist. For plans you can execute, see ../load-testing/. +--- version: 1.0.0 name: "Performance assert example" + root: - testCase: + # A `threadGroup` is the usual root of a load plan: `users` virtual users in + # parallel, each repeating the children `iterations` times. + threadGroup: + users: 1 + iterations: 10 + pacing: 0 + maxDuration: 0 children: - - threadGroup: - users: 1 - pacing: 0 - maxDuration: 0 - iterations: 10 - children: - - callKeyword: - keyword: "Buy MacBook in OpenCart" - - performanceAssert: - measurementName: "Buy MacBook in OpenCart" - expectedValue: 10000 \ No newline at end of file + - callKeyword: + keyword: "Buy MacBook in OpenCart" + + # ------------------------------------------------------------------ + # `performanceAssert` MUST live in an `after` or `afterThread` block. + # + # As an ordinary child - of the thread group, of a testCase, of anything - + # the execution ends in TECHNICAL_ERROR with + # "PerformanceAssert can only be defined in an 'after' or + # 'after thread' block" + # + # after once, when the whole thread group has finished + # afterThread once per virtual user, as that user finishes + # + # `measurementName` names a KEYWORD measurement, or a custom measurement a + # keyword created with output.startMeasure(...). A measurement produced by + # `instrumentNode` on a plan node is NOT matched. + # + # `continueOnError: true` keeps the block going past the first breach, so + # one run reports every violated threshold rather than just one. + # ------------------------------------------------------------------ + after: + continueOnError: true + steps: + - performanceAssert: + measurementName: "Buy MacBook in OpenCart" + aggregator: AVG # AVG | MAX | MIN | COUNT | SUM + comparator: LOWER_THAN # LOWER_THAN | HIGHER_THAN | EQUALS + expectedValue: 10000 + + # Assert the throughput too. A system that meets a latency target by + # doing less work passes the threshold above and fails this one. + - performanceAssert: + measurementName: "Buy MacBook in OpenCart" + aggregator: COUNT + comparator: EQUALS + expectedValue: 10 From 1a9dbc1d19aaea42ce7156890ffa4a712d9b5758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 21 Aug 2026 23:03:03 +0200 Subject: [PATCH 3/9] SED-4885 Creating plan samples for load testing - Precising iterations --- .../load-testing/01-first-load-test/README.md | 35 +++++++++++++------ .../automation-package.yaml | 23 +++++++++--- .../02-thread-group-configuration/README.md | 26 +++++++++----- .../automation-package.yaml | 30 +++++++++++----- plans/load-testing/README.md | 18 +++++----- 5 files changed, 92 insertions(+), 40 deletions(-) diff --git a/plans/load-testing/01-first-load-test/README.md b/plans/load-testing/01-first-load-test/README.md index e8d4924..bc8033e 100644 --- a/plans/load-testing/01-first-load-test/README.md +++ b/plans/load-testing/01-first-load-test/README.md @@ -10,9 +10,9 @@ level: beginner # 01 — First load test -The baseline shape of a load-testing plan: a thread group runs one business transaction -repeatedly, from several virtual users at once, and the plan states the SLA that transaction -has to meet. Every other sample in this set builds on this structure. +The baseline shape of a load-testing plan: a thread group repeats one transaction, from several +virtual users at once, and the plan states the SLA that transaction has to meet. Every other +sample in this set builds on this structure. The subject of this sample is the **plan**. The keywords are 3-line Groovy stubs that simulate a shop API, so the package runs on any Java agent — no build, no browser, no system under test. @@ -20,7 +20,7 @@ a shop API, so the package runs on any Java agent — no build, no browser, no s ## What this sample shows - `threadGroup` as the root of a load plan, with `users` and `iterations` -- **One iteration = one business transaction** — the decision that makes or breaks a load plan +- **The iteration is the unit your load numbers are denominated in** — what to put in it, and why - `instrumentNode` for an end-to-end transaction measurement - `performanceAssert` as the SLA gate, and **the two rules about where it may go** - Why a load test still needs a functional `assert` @@ -48,15 +48,28 @@ A `threadGroup` is to a load test what a `testCase` is to a functional test or a that says what "one run" means. `users` sizes the load — each virtual user holds its own agent token for the whole thread group, so it also sizes the agent capacity you need. -## One iteration is one business transaction +## What goes in one iteration -The children of a thread group should be the smallest thing a *user of the system* would call a -complete action — search, add to cart, buy — not a single HTTP call. +Everything a thread group reports is *per iteration*, so what you put in `children` decides what +`users`, `pacing` and throughput actually mean. Take `users: 10` with `pacing: 30000`: -That choice decides what the report can tell you. Time the whole transaction and you can answer -"can a customer buy something in under two seconds?". Time only the individual calls and you -have a pile of numbers that never adds up to an answer, because the sum of the parts is not the -experience. +| If `children` is… | …the run means | +|-------------------|----------------| +| one HTTP call | 20 HTTP calls a minute, and a 30-second pause between **every call** | +| search → add to cart → check out | 20 **purchases** a minute, and a 30-second pause between purchases | + +Same numbers, different tests. The second is a load somebody can state a requirement about — *the +shop must sustain 1200 orders an hour* — and its pacing models a real user, who pauses between +purchases rather than between two clicks of the same purchase. + +**Choose the iteration to be the thing your requirement is stated in.** Usually that is a complete +user action: search, add to cart, buy. Sometimes it genuinely is one call — an API whose +requirement reads *500 GET /products per second* is correctly modelled with one call per +iteration. What you want to avoid is an iteration nobody has a target for, because then the +throughput figure has to be divided by something before anyone can act on it. + +Sizing a transaction this way also gives the report an end-to-end number to gate on, which is what +`instrumentNode` below is for. ## Where measurements come from diff --git a/plans/load-testing/01-first-load-test/automation-package.yaml b/plans/load-testing/01-first-load-test/automation-package.yaml index 4b2b6df..8d30616 100644 --- a/plans/load-testing/01-first-load-test/automation-package.yaml +++ b/plans/load-testing/01-first-load-test/automation-package.yaml @@ -44,12 +44,25 @@ plans: children: # ------------------------------------------------------------- - # ONE ITERATION IS ONE BUSINESS TRANSACTION. + # THE ITERATION IS THE UNIT THE LOAD NUMBERS ARE COUNTED IN. # - # This is the load-testing counterpart of "one testCase run is one - # business transaction": the children of a thread group should be - # the smallest thing a user of the system would call a complete - # action - search, add to cart, buy - not a single HTTP call. + # Everything a thread group reports is PER ITERATION, so what goes + # in `children` decides what `users`, `pacing` and throughput + # actually mean. `users: 10` with `pacing: 30000` is 20 iterations + # a minute - but 20 of WHAT? + # + # one HTTP call per iteration -> 20 calls a minute, and a + # 30s pause between every call + # a whole purchase per iteration -> 20 PURCHASES a minute, and a + # 30s pause between purchases + # + # Choose the iteration to be the thing your requirement is stated + # in. Usually that is a complete user action, as below. Sometimes + # it really is one call - an API whose target reads "500 GET + # /products per second" is correctly modelled that way. What to + # avoid is an iteration nobody has a target for, because then the + # throughput figure has to be divided by something before anyone + # can act on it. # # `instrumentNode: true` wraps the sequence in a MEASUREMENT named # after its `nodeName`. Without it the report would show three diff --git a/plans/load-testing/02-thread-group-configuration/README.md b/plans/load-testing/02-thread-group-configuration/README.md index 56e9b17..f1b4297 100644 --- a/plans/load-testing/02-thread-group-configuration/README.md +++ b/plans/load-testing/02-thread-group-configuration/README.md @@ -134,14 +134,24 @@ case explicitly rather than absorbing it. | `afterThread` | once per virtual user, after its last iteration | | `after` | once, when the whole thread group has finished | -Getting this wrong is the most common way to make a load test lie. Logging in inside `children` -means every iteration measures a login the real user performs once a day, and the reported average -becomes a blend of two unrelated things. - -**The rule of thumb: `children` holds exactly the transaction the SLA is written about.** Setup -goes in `beforeThread`; setup the whole test needs once — warming a cache, seeding a data set — -goes in `before`. `afterThread` is the reliable place for per-user cleanup: it runs even when an -iteration failed, which a last child would not. +**Put each step in the block that matches how often a real user does it.** That is the whole rule, +and getting it wrong distorts the traffic mix rather than producing any visible error. + +A real user logs in once per session and then does twenty things. Move `Login` from `beforeThread` +into `children` in plan E and the run sends **6 logins instead of 2** — triple the load on the +authentication service, and a traffic mix the production system never sees. Nothing fails; the +test just stops describing reality. + +So: once-per-session steps in `beforeThread`, once-per-transaction steps in `children`, and setup +the whole test needs once — warming a cache, seeding a data set — in `before`. `afterThread` is +the reliable place for per-user cleanup: it runs even when an iteration failed, which a last child +would not. + +There is a second, narrower consequence for measurements. Per-keyword measurements are **not** +affected — `performanceAssert` aggregates by keyword name, so `Login` and `Checkout` stay separate +series wherever they sit. What does get polluted is any **transaction-level** measurement that +wraps the iteration: an `instrumentNode` sequence around both calls, or a custom measurement +spanning them, would fold the login time into the transaction number you gate on. Plan E makes all of this visible with counts rather than prose: 1 warm-up, 2 logins, 6 checkouts, 2 logouts. diff --git a/plans/load-testing/02-thread-group-configuration/automation-package.yaml b/plans/load-testing/02-thread-group-configuration/automation-package.yaml index 7babdc5..fe4fe07 100644 --- a/plans/load-testing/02-thread-group-configuration/automation-package.yaml +++ b/plans/load-testing/02-thread-group-configuration/automation-package.yaml @@ -263,13 +263,24 @@ plans: # afterThread once per VIRTUAL USER, after its last iteration # after once, when the whole thread group has finished # - # Getting this wrong is the most common way to make a load test lie. - # Logging in inside `children` means every iteration measures a login the - # real user does once a day, and the average response time is then a - # blend of two completely different things. + # THE RULE: put each step in the block that matches HOW OFTEN A REAL USER + # DOES IT. Getting it wrong distorts the traffic mix without producing any + # visible error. # - # The rule of thumb: `children` holds exactly the transaction the SLA is - # written about. Everything that sets it up goes in `beforeThread`. + # A real user logs in once per session and then does twenty things. Move + # the Login below from `beforeThread` into `children` and this plan sends + # 6 LOGINS INSTEAD OF 2 - triple the load on the authentication service, + # and a traffic mix production never sees. Nothing fails; the test just + # stops describing reality. (The counts in the `after` block are what make + # that visible, so try the move and watch them.) + # + # A narrower consequence for measurements, since it is easy to overstate: + # per-keyword measurements are NOT affected - performanceAssert aggregates + # by keyword name, so Login and Checkout stay separate series wherever + # they sit. What DOES get polluted is a TRANSACTION-LEVEL measurement + # wrapping the iteration - an instrumented sequence around both calls, or + # a custom measurement spanning them - which would fold the login time + # into the number you gate on. # # `before` is for the setup the whole test needs once - warming a cache, # seeding a data set, resetting a counter on the target. @@ -303,7 +314,7 @@ plans: children: - callKeyword: keyword: "Checkout" - nodeName: "The transaction the SLA is about" + nodeName: "What the user does on every visit" afterThread: steps: @@ -338,8 +349,11 @@ plans: aggregator: COUNT comparator: EQUALS expectedValue: 6 + # Named per keyword, so this threshold covers checkouts and + # nothing else - it would do so even if the login had been put + # in the wrong block. The counts above are what catch that. - performanceAssert: - nodeName: "The SLA covers checkouts only, not the logins" + nodeName: "And the checkouts met their threshold" measurementName: "Checkout" aggregator: AVG comparator: LOWER_THAN diff --git a/plans/load-testing/README.md b/plans/load-testing/README.md index 5847745..170ff8a 100644 --- a/plans/load-testing/README.md +++ b/plans/load-testing/README.md @@ -78,14 +78,16 @@ produces, so the mistake reads as a typo. [05](05-measurements/) covers the ways The controls are the vocabulary; these are the ideas that decide whether a load plan is any good. -1. **One iteration is one business transaction.** The children of a thread group should be the - smallest thing a *user* would call a complete action — search, add to cart, buy — not a single - HTTP call. That choice decides what the report can answer, because the sum of the parts is not - the experience. - -2. **`children` holds exactly what the SLA is about.** Logging in inside the loop means every - iteration measures a login the real user performs once a day, and the reported average becomes a - blend of two unrelated things. Setup belongs in `beforeThread`, once-per-test setup in `before`. +1. **The iteration is the unit your load numbers are counted in.** Everything a thread group + reports is per iteration, so `users: 10` with `pacing: 30000` is 20 iterations a minute — but + 20 of *what*? Choose the iteration to be the thing your requirement is stated in, usually a + complete user action. An iteration nobody has a target for gives you a throughput figure that + has to be divided by something before anyone can act on it. + +2. **Put each step in the block that matches how often a real user does it.** A real user logs in + once per session, so a login belongs in `beforeThread`; move it into `children` and a 2 × 3 run + sends six logins instead of two — triple the load on the authentication service, and no error + anywhere. Once-per-test setup goes in `before`. 3. **Pace the load, or you are not testing — you are being tested.** Without `pacing`, throughput is whatever the system happens to allow, so two runs cannot be compared and a degrading system From 29687c7fdc0f19922e8462bf1850fb6a6151088e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 21 Aug 2026 23:10:02 +0200 Subject: [PATCH 4/9] SED-4885 Creating plan samples for load testing - Moving dataSets to before --- .../04-test-data-and-datasets/README.md | 99 ++-- .../automation-package.yaml | 457 +++++++++--------- 2 files changed, 306 insertions(+), 250 deletions(-) diff --git a/plans/load-testing/04-test-data-and-datasets/README.md b/plans/load-testing/04-test-data-and-datasets/README.md index e6cde09..b8c7fd6 100644 --- a/plans/load-testing/04-test-data-and-datasets/README.md +++ b/plans/load-testing/04-test-data-and-datasets/README.md @@ -23,8 +23,8 @@ two virtual users collide. | Plan | Expected outcome | Shows | |------|------------------|-------| -| A — One pool feeding one thread group | **PASSED** | The two-sibling shape, `item`, `.next()` | -| B — One pool shared by two thread groups | **PASSED** | A shared cursor across populations | +| A — One pool feeding one thread group | **PASSED** | Declaring in `before`, `item`, `.next()` | +| B — One pool shared by two thread groups | **PASSED** | A shared cursor across parallel populations | | C — One account per virtual user | **PASSED** | Pulling in `beforeThread` instead of the loop body | | D — The pool runs dry | **FAILED** (on purpose) | `resetAtEnd: false` returns `null`, silently | | E — A pool that needs no file | **PASSED** | `json-array`, and the other sources | @@ -41,42 +41,70 @@ This is the one thing to understand, and it is the opposite of what the name sug A `dataSet` node with children is a common and completely silent mistake: **the children are never executed, and the node still reports PASSED.** -So the pattern is two **siblings** — the `dataSet` declares the pool, and the `threadGroup` after -it pulls one row per iteration: +## Declare the pool in a `before` block + +The declaration belongs in the `before` block of whatever node contains the load — and the thread +group stays the root of the plan: ```yaml root: - testCase: + threadGroup: + users: 1 + iterations: 3 + + before: # runs once, before the load starts + steps: + - dataSet: + item: "shopperPool" # names the CURSOR, not the row + resetAtEnd: true + dataSource: + csv: {file: "data/users.csv"} + children: - - dataSet: - item: "shopperPool" # names the CURSOR, not the row - resetAtEnd: true - dataSource: - csv: {file: "data/users.csv"} - - - threadGroup: - users: 1 - iterations: 3 - children: - - set: - key: shopper - value: - expression: "shopperPool.next()" # one pull per iteration - - callKeyword: - keyword: "Login" - inputs: - - user: - expression: "shopper.Username" + - set: + key: shopper + value: + expression: "shopperPool.next()" # one pull per iteration + - callKeyword: + keyword: "Login" + inputs: + - user: + expression: "shopper.Username" ``` `item` names the cursor. `shopperPool` is an object you call `.next()` on — it is **not** a map of columns. `.next()` advances the cursor and returns the row as a map. +Two reasons for `before` rather than a sibling node: + +- **It removes a race.** See the next section — in a `testScenario` a sibling declaration is not + guaranteed to be bound before the load starts pulling from it. +- **It keeps the thread group at the root.** A load plan's root should be the load profile. + Wrapping the whole thing in a `testCase` just to have somewhere to put the declaration buries + the profile one level down for no benefit. + ## One pool, several thread groups -The cursor is shared. Every `.next()` in the execution, from whichever thread group, takes the -*following* row — so two populations drawing from one pool never collide on the same account, -which is exactly what you want when the system under test locks a session per user. +`testScenario` runs its children **in parallel**. A `dataSet` written as a plain sibling of the +thread groups is therefore racing them: nothing guarantees the cursor is bound before the first +`.next()`, and the failure mode is a plan that works on a quiet instance and breaks on a busy one. + +Put it in the scenario's `before` block, which runs to completion before any child starts: + +```yaml +root: + testScenario: + before: + steps: + - dataSet: {item: "shopperPool", resetAtEnd: true, dataSource: {...}} + children: + - threadGroup: {nodeName: "Browsers", ...} + - threadGroup: {nodeName: "Buyers", ...} +``` + +Once bound, the cursor is **shared**. Every `.next()` in the execution, from whichever thread +group, takes the *following* row — so two populations drawing from one pool never collide on the +same account, which is exactly what you want when the system under test locks a session per user. The consequence: **which** rows a given thread group gets is not deterministic. It depends on the order the threads happen to reach their `.next()`. Never write a plan that assumes "the buyers get @@ -84,15 +112,24 @@ shopper1". ## Where you pull decides how much data you need -| Pull site | Rows consumed | -|-----------|---------------| -| the loop body | one per **iteration** | -| `beforeThread` | one per **virtual user**, held for the thread's whole life | +Three blocks, three frequencies: + +| Block | Runs | Holds | +|-------|------|-------| +| `before` | once for the test | the **declaration** | +| `beforeThread` | once per virtual user | a pull, if the account belongs to the user | +| `children` | every iteration | a pull, if the data belongs to the transaction | + +`before` is guaranteed to run before `beforeThread`, so the cursor is always bound by the time a +thread claims its row. Per-virtual-user is the shape most load tests actually need: a virtual user logs in once and then does twenty things as that account — it does not become a different person between two clicks. ```yaml +before: + steps: + - dataSet: {item: "shopperPool", resetAtEnd: false, dataSource: {...}} beforeThread: steps: - set: diff --git a/plans/load-testing/04-test-data-and-datasets/automation-package.yaml b/plans/load-testing/04-test-data-and-datasets/automation-package.yaml index cd46eb3..844f9b1 100644 --- a/plans/load-testing/04-test-data-and-datasets/automation-package.yaml +++ b/plans/load-testing/04-test-data-and-datasets/automation-package.yaml @@ -14,7 +14,7 @@ name: "load-04-test-data-and-datasets" plans: # ========================================================================= - # A. The shape: a pool declared once, pulled from inside the load. + # A. The shape: a pool declared in `before`, pulled from inside the load. # # THE ONE THING TO UNDERSTAND ABOUT `dataSet`: # @@ -25,92 +25,107 @@ plans: # A `dataSet` node with children is a common and completely silent mistake: # the children are never executed, and the node still reports PASSED. # - # So the pattern is two SIBLINGS: the `dataSet` declares the pool, and the - # `threadGroup` after it pulls one row per iteration with `.next()`. + # SO WHERE DOES IT GO? In the `before` block of the node that contains the + # load. That guarantees the cursor is bound before anything tries to pull + # from it, and it leaves the THREAD GROUP as the root of the plan - which + # is what a load plan's root should be. Wrapping the whole thing in a + # `testCase` just to have somewhere to put the declaration buries the load + # profile one level down for no benefit. # ========================================================================= - name: "A - One pool feeding one thread group" categories: ["Load testing"] root: - testCase: - nodeName: "Each iteration gets its own account" - children: + threadGroup: + nodeName: "1 user placing 3 orders" + users: 1 + iterations: 3 - # `item` names the CURSOR, not the row. `shopperPool` is an object - # you call `.next()` on - it is not a map of columns. - # - # `resetAtEnd: true` wraps around when the pool is used up, so a - # long run keeps going with the same rows recycled. See plan D for - # what `false` does, which is not what most people expect. - # - # A data source holding credentials can be declared with - # `protect: true`, which obfuscates its values in the report. - - dataSet: - nodeName: "Shopper accounts" - item: "shopperPool" - resetAtEnd: true - dataSource: - csv: - file: "data/users.csv" - delimiter: "," + # Runs once, before the thread group starts. + # + # `item` names the CURSOR, not the row. `shopperPool` is an object + # you call `.next()` on - it is not a map of columns. + # + # `resetAtEnd: true` wraps around when the pool is used up, so a long + # run keeps going with the same rows recycled. See plan D for what + # `false` does, which is not what most people expect. + # + # A data source holding credentials can be declared with + # `protect: true`, which obfuscates its values in the report. + before: + steps: + - dataSet: + nodeName: "Shopper accounts" + item: "shopperPool" + resetAtEnd: true + dataSource: + csv: + file: "data/users.csv" + delimiter: "," - - threadGroup: - nodeName: "1 user placing 3 orders" - users: 1 - iterations: 3 - children: + children: - # One pull per iteration. `.next()` advances the shared - # cursor and returns the row as a map of columns. - - set: - key: shopper - value: - expression: "shopperPool.next()" - nodeName: "Take the next account from the pool" + # One pull per iteration. `.next()` advances the shared cursor and + # returns the row as a map of columns. + - set: + key: shopper + value: + expression: "shopperPool.next()" + nodeName: "Take the next account from the pool" - - callKeyword: - keyword: "Login" - inputs: - - user: - expression: "shopper.Username" - - password: - expression: "shopper.Password" - children: - # The keyword echoes the account back, so this proves - # the row really arrived - a static input, or a pool - # that quietly handed out nothing, fails here. - - assert: - actual: "account" - operator: BEGINS_WITH - expected: "shopper" - customErrorMessage: "The virtual user was not given an account from the pool." + - callKeyword: + keyword: "Login" + inputs: + - user: + expression: "shopper.Username" + - password: + expression: "shopper.Password" + children: + # The keyword echoes the account back, so this proves the row + # really arrived - a static input, or a pool that quietly + # handed out nothing, fails here. + - assert: + actual: "account" + operator: BEGINS_WITH + expected: "shopper" + customErrorMessage: "The virtual user was not given an account from the pool." - - callKeyword: - keyword: "Place Order" - inputs: - - productId: - expression: "shopper.ProductId" - children: - - assert: - actual: "product" - operator: BEGINS_WITH - expected: "P-" + - callKeyword: + keyword: "Place Order" + inputs: + - productId: + expression: "shopper.ProductId" + children: + - assert: + actual: "product" + operator: BEGINS_WITH + expected: "P-" - after: - continueOnError: true - steps: - - performanceAssert: - nodeName: "Three orders were placed" - measurementName: "Place Order" - aggregator: COUNT - comparator: EQUALS - expectedValue: 3 + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Three orders were placed" + measurementName: "Place Order" + aggregator: COUNT + comparator: EQUALS + expectedValue: 3 # ========================================================================= # B. One pool shared by several thread groups. # - # The cursor is shared: every `.next()` in the execution, from whichever - # thread group, takes the FOLLOWING row. Two populations drawing from one - # pool therefore never collide on the same account - which is exactly what + # THE DECLARATION MUST GO IN THE SCENARIO'S `before` BLOCK, NOT NEXT TO THE + # THREAD GROUPS. + # + # `testScenario` runs its children IN PARALLEL. A `dataSet` written as a + # plain sibling of the thread groups is therefore racing them: nothing + # guarantees the cursor is bound before the first `.next()` runs, and the + # failure mode is a plan that works on a quiet instance and breaks on a busy + # one. `before` runs to completion before any child starts, which removes + # the race entirely. + # + # Once bound, the cursor is SHARED: every `.next()` in the execution, from + # whichever thread group, takes the FOLLOWING row. Two populations drawing + # from one pool therefore never collide on the same account - which is what # you want when the system under test locks a session per user. # # The consequence: WHICH rows a given thread group gets is not @@ -122,15 +137,18 @@ plans: root: testScenario: nodeName: "Browsers and buyers drawing from one pool" - children: - - dataSet: - nodeName: "Shopper accounts" - item: "shopperPool" - resetAtEnd: true - dataSource: - csv: - file: "data/users.csv" + before: + steps: + - dataSet: + nodeName: "Shopper accounts" + item: "shopperPool" + resetAtEnd: true + dataSource: + csv: + file: "data/users.csv" + + children: - threadGroup: nodeName: "Browsers" @@ -195,91 +213,94 @@ plans: # once and then does twenty things as that account - it does not become a # different person between two clicks. # - # Pulling in `beforeThread` instead of in the loop body is the whole trick: + # Three blocks, three frequencies, and the whole trick is which one holds + # the pull: # - # pull in the body one row per ITERATION - burns the pool fast - # pull in `beforeThread` one row per VIRTUAL USER - held for the - # thread's whole life + # before once for the test - DECLARE the pool + # beforeThread once per VIRTUAL USER - PULL one row, and log in with it + # children every iteration - use the row already held # - # And it is not just tidier, it is cheaper. This plan has 2 users x 3 - # iterations = 6 iterations against a pool of 4 rows with - # `resetAtEnd: false`. Pulling per iteration would exhaust the pool and the - # last two iterations would get null. Pulling per thread uses 2 rows. + # `before` is guaranteed to run before `beforeThread`, so the cursor is + # always bound by the time a thread claims its row. + # + # Pulling per thread rather than per iteration is not just tidier, it is + # cheaper. This plan runs 2 users x 3 iterations = 6 iterations against a + # pool of 4 rows with `resetAtEnd: false`. Pulling in the loop body would + # exhaust the pool and the last two iterations would get null. Pulling per + # thread uses 2 rows. # ========================================================================= - name: "C - One account per virtual user" categories: ["Load testing"] root: - testCase: - nodeName: "Each virtual user keeps its own account" - children: + threadGroup: + nodeName: "2 users doing 3 orders each" + users: 2 + iterations: 3 - - dataSet: - nodeName: "Shopper accounts" - item: "shopperPool" - resetAtEnd: false - dataSource: - csv: - file: "data/users.csv" - - - threadGroup: - nodeName: "2 users doing 3 orders each" - users: 2 - iterations: 3 + before: + steps: + - dataSet: + nodeName: "Shopper accounts" + item: "shopperPool" + resetAtEnd: false + dataSource: + csv: + file: "data/users.csv" - # Runs once per virtual user, before its first iteration. - # `myShopper` then belongs to that thread and every iteration - # of it reads the same account. - beforeThread: - steps: - - set: - key: myShopper - value: - expression: "shopperPool.next()" - nodeName: "Claim one account for this virtual user" - - callKeyword: - keyword: "Login" - nodeName: "Log in once per virtual user" - inputs: - - user: - expression: "myShopper.Username" - - password: - expression: "myShopper.Password" - children: - - assert: - actual: "account" - operator: BEGINS_WITH - expected: "shopper" + # Runs once per virtual user, before its first iteration. + # `myShopper` then belongs to that thread, and every iteration of it + # reads the same account. + beforeThread: + steps: + - set: + key: myShopper + value: + expression: "shopperPool.next()" + nodeName: "Claim one account for this virtual user" + - callKeyword: + keyword: "Login" + nodeName: "Log in once per virtual user" + inputs: + - user: + expression: "myShopper.Username" + - password: + expression: "myShopper.Password" + children: + - assert: + actual: "account" + operator: BEGINS_WITH + expected: "shopper" + children: + - callKeyword: + keyword: "Place Order" + inputs: + - productId: + expression: "myShopper.ProductId" children: - - callKeyword: - keyword: "Place Order" - inputs: - - productId: - expression: "myShopper.ProductId" - children: - - assert: - actual: "product" - operator: BEGINS_WITH - expected: "P-" - customErrorMessage: "The account was lost between iterations - was the pull moved into the loop body?" + - assert: + actual: "product" + operator: BEGINS_WITH + expected: "P-" + customErrorMessage: "The account was lost between iterations - was the pull moved into the loop body?" - after: - continueOnError: true - steps: - # Two logins for six orders: the proof that the pull ran - # once per virtual user rather than once per iteration. - - performanceAssert: - nodeName: "One login per virtual user" - measurementName: "Login" - aggregator: COUNT - comparator: EQUALS - expectedValue: 2 - - performanceAssert: - nodeName: "Six orders in total" - measurementName: "Place Order" - aggregator: COUNT - comparator: EQUALS - expectedValue: 6 + after: + continueOnError: true + steps: + # Two logins for six orders: the proof that the pull ran once per + # virtual user rather than once per iteration. + - performanceAssert: + nodeName: "One login per virtual user" + measurementName: "Login" + aggregator: COUNT + comparator: EQUALS + expectedValue: 2 + - performanceAssert: + nodeName: "Six orders in total" + measurementName: "Place Order" + aggregator: COUNT + comparator: EQUALS + expectedValue: 6 # ========================================================================= # D. What happens when the pool runs dry. (INTENTIONALLY FAILS) @@ -305,37 +326,36 @@ plans: - name: "D - The pool runs dry (intentionally fails)" categories: ["Load testing"] root: - testCase: - nodeName: "Six iterations against a pool of four" - children: + threadGroup: + nodeName: "1 user asking for 6 accounts" + users: 1 + iterations: 6 - - dataSet: - nodeName: "Shopper accounts used once each" - item: "shopperPool" - resetAtEnd: false - dataSource: - csv: - file: "data/users.csv" + before: + steps: + - dataSet: + nodeName: "Shopper accounts used once each" + item: "shopperPool" + resetAtEnd: false + dataSource: + csv: + file: "data/users.csv" - - threadGroup: - nodeName: "1 user asking for 6 accounts" - users: 1 - iterations: 6 - children: - - set: - key: shopper - value: - expression: "shopperPool.next()" + children: + - set: + key: shopper + value: + expression: "shopperPool.next()" - - check: - nodeName: "The pool still had a row to give" - expression: "shopper != null" + - check: + nodeName: "The pool still had a row to give" + expression: "shopper != null" - - callKeyword: - keyword: "Place Order" - inputs: - - productId: - expression: "shopper.ProductId" + - callKeyword: + keyword: "Place Order" + inputs: + - productId: + expression: "shopper.ProductId" # ========================================================================= # E. Other data sources. @@ -364,47 +384,46 @@ plans: - name: "E - A pool that needs no file" categories: ["Load testing"] root: - testCase: - nodeName: "An inline pool of products" - children: + threadGroup: + nodeName: "1 user ordering 2 products" + users: 1 + iterations: 2 - - dataSet: - nodeName: "Products" - item: "productPool" - resetAtEnd: true - dataSource: - json-array: - json: '[{"ProductId":"P-LAPTOP"},{"ProductId":"P-PHONE"}]' + before: + steps: + - dataSet: + nodeName: "Products" + item: "productPool" + resetAtEnd: true + dataSource: + json-array: + json: '[{"ProductId":"P-LAPTOP"},{"ProductId":"P-PHONE"}]' - - threadGroup: - nodeName: "1 user ordering 2 products" - users: 1 - iterations: 2 + children: + - set: + key: product + value: + expression: "productPool.next()" + - callKeyword: + keyword: "Place Order" + inputs: + - productId: + expression: "product.ProductId" children: - - set: - key: product - value: - expression: "productPool.next()" - - callKeyword: - keyword: "Place Order" - inputs: - - productId: - expression: "product.ProductId" - children: - - assert: - actual: "product" - operator: BEGINS_WITH - expected: "P-" + - assert: + actual: "product" + operator: BEGINS_WITH + expected: "P-" - after: - continueOnError: true - steps: - - performanceAssert: - nodeName: "Two orders were placed" - measurementName: "Place Order" - aggregator: COUNT - comparator: EQUALS - expectedValue: 2 + after: + continueOnError: true + steps: + - performanceAssert: + nodeName: "Two orders were placed" + measurementName: "Place Order" + aggregator: COUNT + comparator: EQUALS + expectedValue: 2 keywords: - GeneralScript: From 829c5a1428369d28ea8d8a6bf194bb2ac8ffd4ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Fri, 21 Aug 2026 23:13:37 +0200 Subject: [PATCH 5/9] SED-4885 Creating plan samples for load testing - Fixing register slips --- plans/load-testing/02-thread-group-configuration/README.md | 2 +- plans/load-testing/03-scenarios-and-mixed-load/README.md | 2 +- plans/load-testing/05-measurements/README.md | 4 ++-- plans/load-testing/05-measurements/automation-package.yaml | 2 +- plans/load-testing/README.md | 7 ++++--- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/plans/load-testing/02-thread-group-configuration/README.md b/plans/load-testing/02-thread-group-configuration/README.md index f1b4297..ff44e27 100644 --- a/plans/load-testing/02-thread-group-configuration/README.md +++ b/plans/load-testing/02-thread-group-configuration/README.md @@ -27,7 +27,7 @@ what runs once per virtual user, and what runs on every iteration. | D — Run for a fixed time | `maxDuration`, and the trap that comes with it | | E — Setup per test versus per user | `before`, `beforeThread`, `afterThread`, `after` | -All five pass. +Every plan in this package is expected to pass; none of them fails by design. ## The knobs diff --git a/plans/load-testing/03-scenarios-and-mixed-load/README.md b/plans/load-testing/03-scenarios-and-mixed-load/README.md index f7a818d..2d4d17e 100644 --- a/plans/load-testing/03-scenarios-and-mixed-load/README.md +++ b/plans/load-testing/03-scenarios-and-mixed-load/README.md @@ -26,7 +26,7 @@ parallel**, each with its own load profile. | C — Measure the site while the batch job runs | A thread group next to a plain `sequence` | | D — Warm-up phase then measured phase | `sequence` as the root — phases instead of parallelism | -All four pass. +Every plan in this package is expected to pass; none of them fails by design. ## The three composing roots diff --git a/plans/load-testing/05-measurements/README.md b/plans/load-testing/05-measurements/README.md index 8fda418..a80ba7e 100644 --- a/plans/load-testing/05-measurements/README.md +++ b/plans/load-testing/05-measurements/README.md @@ -22,7 +22,7 @@ you end up with a run full of numbers that cannot answer the question you ran it | B — A custom measurement around a whole transaction | `startMeasure` / `stopMeasure`, nested | | C — Count the failures too | Why a response-time average alone is the most misleading number in load testing | -All three pass. +Every plan in this package is expected to pass; none of them fails by design. ## The three sources @@ -41,7 +41,7 @@ how a multi-keyword transaction gets an end-to-end number in the dashboards. Custom measurements are the only way to time something **smaller** than a keyword call — the two page loads inside one browser keyword, for instance. -## The difference that catches people out +## Which measurements can carry an SLA `performanceAssert` sees keyword measurements and custom measurements. It does **not** see instrumented-node measurements. Asserting on an instrumented sequence fails with: diff --git a/plans/load-testing/05-measurements/automation-package.yaml b/plans/load-testing/05-measurements/automation-package.yaml index ae9dad5..791129a 100644 --- a/plans/load-testing/05-measurements/automation-package.yaml +++ b/plans/load-testing/05-measurements/automation-package.yaml @@ -38,7 +38,7 @@ plans: # measurements and custom measurements. It does NOT see instrumented-node # measurements. Asserting on "Search and buy" below would fail with # "No measurement is matching the defined filters." - the same message a - # misspelled keyword name produces, which is why this catches people out. + # misspelled keyword name produces, so the mistake reads as a typo. # # So instrumented nodes are for the dashboards; SLAs go on keyword or # custom measurements. diff --git a/plans/load-testing/README.md b/plans/load-testing/README.md index 170ff8a..d5653d6 100644 --- a/plans/load-testing/README.md +++ b/plans/load-testing/README.md @@ -59,11 +59,12 @@ sample's README lists the expected outcome per plan: | [06](06-thresholds-and-slas/) | C — Threshold on the failure rate | | [06](06-thresholds-and-slas/) | E — A breached SLA | -Everything else passes. +Every other plan is expected to pass. -## Two rules that are easy to get wrong +## Two placement rules for `performanceAssert` -Both cost an afternoon the first time, and neither is visible from the YAML: +Neither follows from the structure of the YAML, and both report an error that names something +other than the cause: **`performanceAssert` must live in an `after` or `afterThread` block.** Anywhere else — as a child of the thread group, of a `testCase`, of anything — the execution ends in `TECHNICAL_ERROR` with From cd3b63c5c0a1428cfa45ff03d886329c01966d11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Mon, 24 Aug 2026 08:50:28 +0200 Subject: [PATCH 6/9] SED-4885 Creating plan samples for load testing - Improving clarity about throughput measurements --- .../load-testing/01-first-load-test/README.md | 24 ++++- .../automation-package.yaml | 32 ++++++- .../02-thread-group-configuration/README.md | 44 +++++++--- .../automation-package.yaml | 87 ++++++++++++------- .../03-scenarios-and-mixed-load/README.md | 6 +- .../automation-package.yaml | 11 ++- .../06-thresholds-and-slas/README.md | 49 ++++++++++- .../automation-package.yaml | 42 ++++++++- plans/load-testing/README.md | 9 +- 9 files changed, 237 insertions(+), 67 deletions(-) diff --git a/plans/load-testing/01-first-load-test/README.md b/plans/load-testing/01-first-load-test/README.md index bc8033e..576bdf1 100644 --- a/plans/load-testing/01-first-load-test/README.md +++ b/plans/load-testing/01-first-load-test/README.md @@ -136,11 +136,27 @@ The plan asserts a floor as well as a ceiling: An upper bound alone passes happily when the measurement is missing, empty, or the keyword has been stubbed down to nothing. A floor fails the moment the transaction stops doing real work. -### `COUNT` is the cheapest check there is +### `COUNT` checks completeness -`COUNT` on the transaction keyword catches a thread group that quietly did less work than you -asked for — a `maxDuration` that cut the run short, users that never started, a data source that -ran dry. Response times over a run that only did a third of the work look wonderful. +A keyword's `COUNT` is not simply `users` × `iterations`. It is how many times **that step was +reached**, and the two come apart as soon as an iteration is more than a single call: + +| Cause | Effect on the count | +|-------|---------------------| +| an iteration **fails part way** | keywords after the failure never run, so their counts fall short of the ones before | +| a keyword sits inside an `if` / `switch` | it runs only when the branch is taken — the count tells you how often that was | +| a data source **ran dry** | iterations carry on with nothing to work on | + +So the `COUNT` worth asserting is the one on the **last** step of the transaction: that is the +number saying the whole thing completed rather than merely started. Comparing counts across the +three keywords in this iteration is how you find where iterations were breaking off. + +Response times over a run that only completed a third of its transactions look wonderful. + +What `COUNT` does **not** measure here is throughput. This plan fixes `iterations`, so the count is +6 whether the system was quick or crawling — a slowdown stretches the *run* instead. To gate the +rate, pin the duration with `iterations: 0` + `maxDuration` and put the `COUNT` threshold on that; +[02](../02-thread-group-configuration/) plan D shows it. ## A load test still has to check its answers diff --git a/plans/load-testing/01-first-load-test/automation-package.yaml b/plans/load-testing/01-first-load-test/automation-package.yaml index 8d30616..aa49f76 100644 --- a/plans/load-testing/01-first-load-test/automation-package.yaml +++ b/plans/load-testing/01-first-load-test/automation-package.yaml @@ -177,10 +177,34 @@ plans: continueOnError: true steps: - # Did the load actually happen? COUNT is the cheapest sanity - # check there is, and it catches a thread group that quietly did - # less work than you asked for - a `maxDuration` that cut the run - # short, users that failed to start, a data source that ran dry. + # Did this STEP actually run as often as the load implies? + # + # A keyword's COUNT is not simply users x iterations. It is how + # many times that particular step was reached, and the two come + # apart as soon as an iteration is more than a single call: + # + # an iteration that FAILS PART WAY the keywords after the + # failure never run, so + # their counts fall short of + # the ones before it + # a keyword inside an `if`/`switch` it runs only when the + # branch is taken, so its + # count tells you how often + # that was + # a data source that RAN DRY iterations continue with + # nothing to work on + # + # Which is why the COUNT to assert is the one on the LAST step of + # the transaction: it is the number that says the whole thing + # completed, not just started. Comparing counts across the three + # keywords of this iteration is how you find where iterations + # were breaking off. + # + # It is a COMPLETENESS check, not a throughput one: `iterations` + # is fixed here, so this is 6 however slow the system was. A + # slowdown stretches the RUN instead. To gate the RATE, pin the + # duration with `iterations: 0` + `maxDuration` and put the COUNT + # threshold on that - see sample 02 plan D. - performanceAssert: nodeName: "All 6 checkouts ran" measurementName: "Checkout" diff --git a/plans/load-testing/02-thread-group-configuration/README.md b/plans/load-testing/02-thread-group-configuration/README.md index ff44e27..21687ff 100644 --- a/plans/load-testing/02-thread-group-configuration/README.md +++ b/plans/load-testing/02-thread-group-configuration/README.md @@ -72,20 +72,35 @@ a long ramp-up from trickling in one user at a time. `startOffset` delays the ramp-up. On its own in a single thread group it does nothing useful; its purpose is staggering several thread groups inside a scenario — see [03](../03-scenarios-and-mixed-load/). -### `maxDuration` and the trap in it +### Running for a time — `iterations` and `maxDuration` together -Most load tests are specified as "an hour at this rate", not "500 iterations". The idiom is to -set `iterations` to a number nobody will reach and let `maxDuration` be the real stop condition, -so the run takes the same time whether the system is fast or slow that day. +Most load tests are specified as "an hour at this rate", not "500 iterations". `maxDuration` is +the wall-clock cap on the whole thread group. How it combines with `iterations` is the part worth +getting right, because the obvious value is not the one you want: -The trap: **the iteration count is now an outcome, not an input.** A slow system produces both a -longer response time *and* fewer iterations. An SLA that reads only response times will pass a run -that managed a tenth of the work. Always assert throughput as well: +| `iterations` | Each user | `maxDuration` acts as | +|--------------|-----------|-----------------------| +| `N` | runs N times | a ceiling — cuts the run short if it fires first | +| `0` | **loops until `maxDuration`** | the real stop condition | +| omitted | runs **once** | a ceiling on that single iteration — it does **not** loop | -```yaml -- performanceAssert: {measurementName: "Checkout", aggregator: COUNT, - comparator: HIGHER_THAN, expectedValue: 2} -``` +So a duration-bounded run is **`iterations: 0` with `maxDuration`** — not a large iteration count +picked to be "unreachable", and not `maxDuration` on its own (which runs once). The run then takes +the same wall-clock time whether the system is fast or slow that day, which is the whole point: +comparable runs. + +That configuration is also what makes a `COUNT` threshold mean throughput. Throughput is iterations +÷ elapsed time, and the two configurations pin different halves: + +| Configuration | Pinned | A slow system produces | `COUNT` is therefore | +|---------------|--------|------------------------|----------------------| +| `iterations: 0` + `maxDuration` | the duration | **fewer iterations** | a **throughput** measure | +| fixed `iterations` | the count | a **longer run** | a **completeness** check — same number quick or slow | + +Either way throughput fell, so never gate on response time alone: a struggling system meets any +latency target by doing less work per unit time. A throughput aggregator that works whichever way +the group is configured is planned for `performanceAssert` in a future release; until then, gate a +rate by pinning the duration and counting. ## The counters every thread group publishes @@ -156,8 +171,11 @@ spanning them, would fold the login time into the transaction number you gate on Plan E makes all of this visible with counts rather than prose: 1 warm-up, 2 logins, 6 checkouts, 2 logouts. -A variable `set` in `before` is visible to the iterations and to the `after` block — which is how -plan B measures its own duration to prove pacing worked. +A variable `set` in `before` is visible to the iterations and to the `after` block. Plan B uses +that to time itself and prove pacing did something — a `COUNT` assertion alone would pass whether +pacing worked or not. **That timestamp is a self-check for the sample, not a pattern to copy:** +timing a run from inside the plan is not how thresholds are written in Step. See +[06](../06-thresholds-and-slas/) for the controls that are. ## Key files diff --git a/plans/load-testing/02-thread-group-configuration/automation-package.yaml b/plans/load-testing/02-thread-group-configuration/automation-package.yaml index fe4fe07..326e4b1 100644 --- a/plans/load-testing/02-thread-group-configuration/automation-package.yaml +++ b/plans/load-testing/02-thread-group-configuration/automation-package.yaml @@ -125,8 +125,15 @@ plans: iterations: 3 pacing: 3000 - # A variable set in `before` is visible to the iterations and to the - # `after` block, which is how this plan measures its own duration. + # NOT A PATTERN TO COPY. The timestamp below exists so that THIS + # SAMPLE can prove pacing did something - a COUNT assertion alone + # would pass whether pacing worked or not, which would make the + # sample worthless as evidence. + # + # Do not reach for this to express a real requirement. Timing a run + # from inside the plan is not how thresholds are written in Step: + # see sample 06 for the controls that are, and for the note on the + # throughput aggregator still to come. before: steps: - set: @@ -144,9 +151,9 @@ plans: # Three iterations paced at 3 s start at 0 s, 3 s and 6 s, so the # run cannot finish before 6 s. Unpaced, the same three # iterations take about a second. This check fails if pacing - # stops working. + # stops working - which is the only reason it is here. - check: - nodeName: "Pacing stretched the run out" + nodeName: "Pacing stretched the run out - sample self-check" expression: "System.currentTimeMillis() - startedAt > 5500" - performanceAssert: @@ -202,33 +209,57 @@ plans: expectedValue: 8 # ========================================================================= - # D. `maxDuration` - running for a TIME instead of for a count. + # D. Running for a TIME instead of for a count. # # Most load tests are specified as "an hour at this rate", not "500 # iterations". `maxDuration` is the wall-clock cap on the whole thread # group: when it expires, the group stops wherever it is. # - # The idiom is to set `iterations` to a number nobody will reach and let - # `maxDuration` be the real stop condition, so the run takes the same time - # whether the system is fast or slow that day. Comparable runs are the - # whole point. + # HOW `iterations` AND `maxDuration` COMBINE - the part that is easy to get + # wrong, because the sensible-looking value is not the one you want: # - # THE TRAP THAT COMES WITH IT: the iteration count is now an OUTCOME, not - # an input - and a slow system produces both a longer response time and - # fewer iterations. So never write an SLA that only reads response times - # here; assert the throughput too, or a run that managed a tenth of the - # work will still report a healthy average. + # iterations: N each user runs N times, UNLESS maxDuration fires first + # and cuts it short. `maxDuration` is a ceiling here. + # iterations: 0 UNLIMITED - each user loops until maxDuration expires. + # This is what "run for a duration" actually means. + # iterations omitted -> defaults to 1 per user. `maxDuration` then + # omitted only caps that single iteration; it does NOT make the + # group loop. A frequent surprise. # - # This plan asks for 100 iterations and cuts them off after 5 seconds. + # So the form for a duration-bounded run is `iterations: 0` WITH + # `maxDuration`, as below - not a large iteration count picked to be + # "unreachable". The run then takes the same wall-clock time whether the + # system is fast or slow that day, which is the whole point: comparable + # runs. (`maxDuration` without `iterations` runs once - see the table.) + # + # AND THIS IS THE FORM THAT MAKES `COUNT` MEAN THROUGHPUT. + # + # Throughput is iterations divided by elapsed time. Here the DURATION is + # pinned, so a COUNT threshold reads directly as a rate: a slow system does + # fewer iterations in the same window. With fixed `iterations` it is the + # other way round - the count is pinned and a slowdown STRETCHES the run, + # so there COUNT is a COMPLETENESS check (how many times the step was + # REACHED - short of users x iterations when an iteration fails part way, + # when the step sits in a branch, or when a data pool ran dry), not a rate. + # + # A throughput aggregator that works whichever way the group is configured + # is planned for `performanceAssert` in a future release; until then, gate + # a rate by pinning the duration and counting, rather than computing a rate + # inside the plan. Either way, never gate on response time alone: a + # struggling system meets any latency target by doing less work per unit + # time. + # + # This plan loops one user for 4 seconds and asserts the rate it sustained. # ========================================================================= - name: "D - Run for a fixed time" categories: ["Load testing"] root: threadGroup: - nodeName: "One user for 5 seconds" + nodeName: "One user for 4 seconds" users: 1 - iterations: 100 - maxDuration: 5000 + iterations: 0 # unlimited: loop until maxDuration + maxDuration: 4000 + pacing: 1000 # aim for one checkout per second children: - callKeyword: keyword: "Checkout" @@ -236,23 +267,17 @@ plans: after: continueOnError: true steps: - # It stopped early: nowhere near the 100 iterations requested. + # The duration is pinned at 4 s, so this COUNT is a throughput + # floor: it fails a run that crawled, which no response-time + # threshold would have caught. One checkout per second over 4 s + # is 4 or 5 starts; asserting >1 keeps the sample robust on a + # shared agent while still proving the loop ran for the window. - performanceAssert: - nodeName: "maxDuration cut the run short" - measurementName: "Checkout" - aggregator: COUNT - comparator: LOWER_THAN - expectedValue: 50 - - # ...but it did do work. This is the throughput floor the note - # above is about - the assertion that catches a run which - # achieved almost nothing. - - performanceAssert: - nodeName: "And still did a useful amount of work" + nodeName: "Sustained more than one checkout per second window" measurementName: "Checkout" aggregator: COUNT comparator: HIGHER_THAN - expectedValue: 2 + expectedValue: 1 # ========================================================================= # E. The four blocks: what runs once, per user, per iteration. diff --git a/plans/load-testing/03-scenarios-and-mixed-load/README.md b/plans/load-testing/03-scenarios-and-mixed-load/README.md index 2d4d17e..a055b85 100644 --- a/plans/load-testing/03-scenarios-and-mixed-load/README.md +++ b/plans/load-testing/03-scenarios-and-mixed-load/README.md @@ -67,8 +67,10 @@ Both work. Prefer the thread group's own block when the threshold belongs to one reads next to the profile it describes. [06](../06-thresholds-and-slas/) covers this in more detail. -Assert exact `COUNT`s in a scenario. It is entirely possible for one population to be starved of -agent tokens and quietly contribute nothing while the report still looks green. +Assert a `COUNT` **per population**, and make each one exact. A scenario mixes populations running +different transactions, so no single aggregate describes it — and if the buyers broke off half way +through their transaction while the browsers ran in full, the totals still look plausible. Only the +per-population counts show it. ## A staged ramp diff --git a/plans/load-testing/03-scenarios-and-mixed-load/automation-package.yaml b/plans/load-testing/03-scenarios-and-mixed-load/automation-package.yaml index df464a9..cc3676d 100644 --- a/plans/load-testing/03-scenarios-and-mixed-load/automation-package.yaml +++ b/plans/load-testing/03-scenarios-and-mixed-load/automation-package.yaml @@ -64,9 +64,14 @@ plans: # own `after` block still works and is the place for a threshold that # only concerns that one population. # - # Both counts are exact, and that is deliberate: in a scenario it is - # entirely possible for one population to be starved of agent tokens - # and quietly contribute nothing while the report still looks green. + # A COUNT PER POPULATION, and both exact. + # + # A scenario mixes populations that run different transactions, so one + # aggregate number cannot describe it. Asserting each population's own + # step separately is what keeps the traffic MIX honest: if the buyers + # broke off half way through their transaction while the browsers ran + # in full, the totals still look plausible and only the per-population + # counts show it. # ------------------------------------------------------------------- after: continueOnError: true diff --git a/plans/load-testing/06-thresholds-and-slas/README.md b/plans/load-testing/06-thresholds-and-slas/README.md index 78b9628..95d55c7 100644 --- a/plans/load-testing/06-thresholds-and-slas/README.md +++ b/plans/load-testing/06-thresholds-and-slas/README.md @@ -36,11 +36,56 @@ load test needs beyond "average response time". | `AVG` | the headline number — and the one that hides the tail. An average of 800 ms is compatible with one user in twenty waiting eight seconds. | | `MAX` | the worst single transaction. Brutal, and useful precisely because it is: a MAX threshold catches the timeout nobody saw. | | `MIN` | a floor. Mostly used to prove the measurement is real rather than empty. | -| `COUNT` | how many transactions happened — the throughput check, and the one most often forgotten. | +| `COUNT` | how many transactions happened. With `iterations` fixed this is a **completeness** check, not a throughput one — see below. | | `SUM` | total time spent. Rarely a threshold; useful for capacity sums. | **A good gate uses several together.** Response time on its own can always be met by doing less -work — which is exactly what a struggling system does. +work per unit time — which is exactly what a struggling system does. + +## What `COUNT` tells you, and how to gate throughput + +Throughput is transactions ÷ elapsed time, and a thread group fixes exactly one of the two. That +decides what a `COUNT` threshold on it actually means: + +| Configuration | Pinned | A `COUNT` threshold is | +|---------------|--------|------------------------| +| fixed `iterations` | the count | a **completeness** check — the same number however slow the run was | +| `iterations: 0` + `maxDuration` | the duration | a genuine **throughput** threshold | + +Plan A fixes `iterations`, so its `COUNT` of 6 confirms the work was done but says nothing about +the rate. + +It is still worth asserting, because a keyword's count is **how often that step was reached**, not +`users` × `iterations`. It falls short when an iteration fails part way — the steps after the +failure never run — when the step sits inside an `if` or `switch` that was not always taken, or +when a data pool ran dry. On a single-keyword iteration like plan A's the two numbers coincide; on +the multi-step transactions in [01](../01-first-load-test/) and [05](../05-measurements/) they do +not, and comparing counts across the steps of one transaction is how you find where iterations were +breaking off. + +**To gate throughput, bound the duration and count**, which is [02](../02-thread-group-configuration/) +plan D's shape: + +```yaml +threadGroup: + users: 1 + iterations: 0 # unlimited: loop until maxDuration + maxDuration: 5000 # the real stop condition + pacing: 1000 + after: + steps: + - performanceAssert: {measurementName: "Checkout", aggregator: COUNT, + comparator: HIGHER_THAN, expectedValue: 2} +``` + +`iterations: 0` is the loop-for-a-duration form; a fixed count would pin the count instead of the +duration, and omitting `iterations` runs a single time. This is also how load requirements are +usually written — *an hour at this rate* — so it is rarely a +compromise. + +> A throughput aggregator that works whichever of the two is pinned is **not available yet**; it is +> planned for `performanceAssert` in a future release. Until then, express a throughput requirement +> as a duration-bounded run with a `COUNT` threshold, rather than computing a rate inside the plan. Set `continueOnError: true` on the `after` block. Without it the block stops at the first breach, and a run that violates three thresholds tells you about one of them. Plan E shows the payoff: two diff --git a/plans/load-testing/06-thresholds-and-slas/automation-package.yaml b/plans/load-testing/06-thresholds-and-slas/automation-package.yaml index ccaeefb..42fd5ad 100644 --- a/plans/load-testing/06-thresholds-and-slas/automation-package.yaml +++ b/plans/load-testing/06-thresholds-and-slas/automation-package.yaml @@ -27,12 +27,33 @@ plans: # because it is: a MAX threshold catches the timeout nobody saw. # MIN a floor. Mostly used, as below, to prove the measurement is real # rather than empty. - # COUNT how many transactions happened - the throughput check, and the - # one most often forgotten. + # COUNT how many transactions happened. What that tells you depends on + # the thread group - see below. # SUM total time spent. Rarely a threshold; useful for capacity sums. # # A good gate uses several of these together. Response time on its own can - # always be met by doing less work. + # always be met by doing less work per unit time. + # + # WHAT `COUNT` MEANS DEPENDS ON WHAT THE THREAD GROUP PINS. + # + # Throughput is transactions divided by elapsed time, and a thread group + # fixes exactly one of the two: + # + # fixed `iterations` the COUNT is pinned. It is 6 below whether the + # system was quick or crawling, so this assertion is + # a COMPLETENESS check: how many times that step was + # actually REACHED. + # + # `maxDuration` the DURATION is pinned, so COUNT becomes a genuine + # THROUGHPUT threshold. That is the supported way to + # gate throughput today, and sample 02 plan D shows + # it. It also matches how load requirements are + # usually written - "an hour at this rate". + # + # A throughput aggregator that works regardless of which one is pinned is + # not available yet; it is planned for `performanceAssert` in a future + # release. Until then, express a throughput requirement as a duration-bound + # run with a COUNT threshold rather than computing a rate in the plan. # ========================================================================= - name: "A - A complete SLA gate" categories: ["Load testing"] @@ -57,8 +78,21 @@ plans: continueOnError: true steps: + # COMPLETENESS, not throughput: `iterations` is fixed here, so + # this is 6 however slow the run was. + # + # It is worth asserting anyway, because a keyword's count is how + # often that STEP was reached - not users x iterations. It falls + # short when iterations fail part way (the steps after the + # failure never run), when the step sits inside a branch that was + # not always taken, or when a data pool ran dry. On a one-keyword + # iteration like this one the two coincide; on the multi-step + # transactions of samples 01 and 05 they do not. + # + # To gate the RATE instead, bound the duration - see the note + # above and sample 02 plan D. - performanceAssert: - nodeName: "Throughput - all the work was done" + nodeName: "All the work was done" measurementName: "Checkout" aggregator: COUNT comparator: EQUALS diff --git a/plans/load-testing/README.md b/plans/load-testing/README.md index d5653d6..a1ecda6 100644 --- a/plans/load-testing/README.md +++ b/plans/load-testing/README.md @@ -94,10 +94,11 @@ The controls are the vocabulary; these are the ideas that decide whether a load is whatever the system happens to allow, so two runs cannot be compared and a degrading system quietly reduces its own load. -4. **Always assert throughput, not just response time.** A struggling system meets any latency - target by doing less work. `COUNT` is the cheapest check there is and the one most often - missing — and it is essential under `maxDuration`, where the iteration count becomes an outcome - rather than an input. +4. **Response time alone is never the whole SLA.** A struggling system meets any latency target by + doing less work per unit time. Which threshold catches that depends on the thread group: run it + `iterations: 0` + `maxDuration` and the duration is pinned, so a `COUNT` threshold *is* a + throughput gate; with `iterations` fixed the count is pinned, so the same threshold only checks + completeness. State rate requirements as `iterations: 0` duration-bounded runs. 5. **A load test still has to check its answers.** A system under stress starts returning fast, cheap, *wrong* responses; an error page renders quicker than a checkout. Without a functional From 74a22504af4223299806dacd9d9def68a31584bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Mon, 24 Aug 2026 09:04:19 +0200 Subject: [PATCH 7/9] SED-4885 Creating plan samples for load testing - Remove assertMetric sample --- plans/load-testing/05-measurements/README.md | 3 +- .../06-thresholds-and-slas/README.md | 69 ++++++------- .../automation-package.yaml | 99 +++++-------------- plans/load-testing/README.md | 6 +- 4 files changed, 57 insertions(+), 120 deletions(-) diff --git a/plans/load-testing/05-measurements/README.md b/plans/load-testing/05-measurements/README.md index a80ba7e..793b3a6 100644 --- a/plans/load-testing/05-measurements/README.md +++ b/plans/load-testing/05-measurements/README.md @@ -55,8 +55,7 @@ gets "fixed" by changing a name that was already right. **Instrumented nodes are for the dashboards. SLAs go on keyword or custom measurements.** To put a threshold on a whole multi-step transaction, emit a custom measurement around it from inside the -keyword — which is what plan B does. ([06](../06-thresholds-and-slas/) shows the other route: -`assertMetric`, which *can* read instrumented nodes but has different rules.) +keyword — which is what plan B does. ## Custom measurements diff --git a/plans/load-testing/06-thresholds-and-slas/README.md b/plans/load-testing/06-thresholds-and-slas/README.md index 95d55c7..221ae75 100644 --- a/plans/load-testing/06-thresholds-and-slas/README.md +++ b/plans/load-testing/06-thresholds-and-slas/README.md @@ -26,8 +26,11 @@ load test needs beyond "average response time". | A — A complete SLA gate | **PASSED** | All five aggregators, and which thresholds are worth writing | | B — One SLA per population | **PASSED** | Per-thread-group `after` blocks in a scenario | | C — Threshold on the failure rate | **FAILED** (on purpose) | Counting successes, and why a run with real errors is red anyway | -| D — A threshold over the metric time series | **PASSED** | `assertMetric`, and how it differs from `performanceAssert` | -| E — A breached SLA | **FAILED** (on purpose) | What a violation looks like in the report | +| D — A breached SLA | **FAILED** (on purpose) | What a violation looks like in the report | + +`performanceAssert` is the control for every one of these. There is a second threshold control, +`assertMetric`, which this sample deliberately does **not** use for gating — see the note at the +end of this page for what it is actually for. ## The aggregators @@ -139,44 +142,6 @@ If a rejection is **expected traffic** rather than an error — a payment the bu decline — do not raise a business error for it. Return it as an ordinary output and branch on it in the plan, and the run status then reflects your error budget rather than the provider's mood. -## `performanceAssert` vs `assertMetric` - -They look similar and answer different questions. - -| | `performanceAssert` | `assertMetric` | -|---|---|---| -| Reads | the measurements **of this execution** | the **metric time series** | -| Scope | this run only | **not scoped to this run** — the series spans every execution | -| Sees instrumented-node measurements | no | **yes** | -| Placement | must be in `after` / `afterThread` | anywhere | -| Answers | *did this run meet its SLA?* | *is the system drifting?* | - -```yaml -- assertMetric: - metric: "response-time" # required - aggregation: AVG # required - comparator: LOWER_THAN - expectedValue: 60000 - filters: - - field: "name" # `name` selects a measurement name - filterType: EQUALS - filter: "Search transaction" -``` - -Two things to know before using `assertMetric` as a gate: - -1. **It is not scoped to the current execution.** A `COUNT` over a measurement name this run - produced twice comes back with every point the series holds, from every run that ever used the - same name. Count assertions are meaningless here — use `performanceAssert` for those. -2. **It reaches measurements `performanceAssert` cannot**, including instrumented-node - measurements, and it can read gauges and counters a keyword pushed. Together with - `slidingWindow`, that is its real use: thresholds over *time* — is the response time drifting up - across the last ten minutes, across yesterday's run and today's — which is what monitoring and - alerting rules are built out of. - -Note `field: "name"` and not `attributes.name`; the latter matches nothing and reports -`No metric found for the defined filters`. - ## What a breach looks like ``` @@ -196,11 +161,33 @@ No measurement is matching the defined filters. That is what you get from a misspelled measurement name, from an instrumented-node name, and from a success measurement that was never emitted — three quite different problems with one message. +## A note on `assertMetric` + +Step has a second threshold control, `assertMetric`, that looks like an alternative to +`performanceAssert`. It is not one for load testing, and this sample deliberately leaves it out. + +| | `performanceAssert` | `assertMetric` | +|---|---|---| +| Reads | the measurements **of this execution** | the stored **metric time series**, across every execution | +| Answers | *did this run meet its SLA?* | *is the system slower than it used to be?* | +| Use it for | gating a load test | cross-run trend and regression detection | + +Because `assertMetric` is **not scoped to the current execution**, its aggregates fold in every +past run that used the same measurement name — so on a single load test the numbers are +meaningless, and it cannot serve as the gate. Its real home is a **cross-execution** assertion: +comparing today's run against last week's, watching an error rate creep up over the last ten runs, +typically inside an assertion plan attached to a **scheduled** execution. That is monitoring +territory, covered by the monitoring samples rather than this set. + +So: **gate a load test with `performanceAssert`; reach for `assertMetric` only for cross-run +assertions.** (A future Step release may let `assertMetric` default to the current execution's +scope, which would make it usable here too — until then the split above holds.) + ## Key files | File | Purpose | |------|---------| -| `automation-package.yaml` | Five plans covering both threshold controls | +| `automation-package.yaml` | Four plans, all gated with `performanceAssert` | | `keywords/checkout.groovy`, `searchProducts.groovy` | Transactions that meet their SLA | | `keywords/slowSearch.groovy` | Always breaches its threshold | | `keywords/flakyPayment.groovy` | Fails every third iteration, and emits a success measurement otherwise | diff --git a/plans/load-testing/06-thresholds-and-slas/automation-package.yaml b/plans/load-testing/06-thresholds-and-slas/automation-package.yaml index 42fd5ad..009fb1d 100644 --- a/plans/load-testing/06-thresholds-and-slas/automation-package.yaml +++ b/plans/load-testing/06-thresholds-and-slas/automation-package.yaml @@ -269,87 +269,38 @@ plans: expectedValue: 4 # ========================================================================= - # D. `assertMetric` - the other threshold control, and what it is for. + # A NOTE ON `assertMetric` - and why this load-test sample does not use it. # - # `performanceAssert` aggregates the measurements OF THIS EXECUTION. - # `assertMetric` queries the METRIC TIME SERIES, which is a different thing - # with different rules: + # `assertMetric` looks like a second threshold control, and it is a real + # one, but it answers a DIFFERENT question and is the WRONG tool for gating + # a load test: # - # metric the series to query, e.g. "response-time" (required) - # aggregation AVG | MAX | MIN | COUNT | SUM (required) - # comparator LOWER_THAN | HIGHER_THAN | EQUALS - # expectedValue the threshold - # filters narrow the series - `field: "name"` selects a - # measurement name - # slidingWindow query a rolling window rather than the whole range + # `performanceAssert` aggregates the measurements OF THIS EXECUTION. + # This is what a load-test SLA gate needs, and it is + # what every plan in this sample uses. # - # TWO THINGS TO KNOW BEFORE USING IT AS A GATE: + # `assertMetric` queries the stored METRIC TIME SERIES, which is + # NOT scoped to the current execution. A COUNT over a + # measurement name comes back with every point the + # series holds, from every run that ever used that + # name - so on a single load test its numbers are + # meaningless. # - # 1. It is NOT scoped to the current execution. A COUNT over a measurement - # name this run produced twice comes back with every point the series - # holds, from every run that used the same name. Count assertions are - # therefore not meaningful here - use `performanceAssert` for those. + # Its actual purpose is CROSS-EXECUTION assertion - "is today's run slower + # than last week's", "has the error rate crept up across the last ten + # runs" - typically inside an assertion plan attached to a SCHEDULED + # execution. That is trend and regression detection, not the pass/fail gate + # on one load test, and it is the subject of the monitoring samples rather + # than this set. # - # 2. It reaches measurements `performanceAssert` cannot, including - # instrumented-node measurements, and it can read gauges and counters - # a keyword pushed. That is its real use: thresholds over - # TIME - is the response time drifting up across the last ten minutes, - # across yesterday's run and today's - which is what monitoring and - # alerting rules are built out of. - # - # The rule of thumb: `performanceAssert` for "did THIS run meet its SLA", - # `assertMetric` for "is the system drifting". - # - # Unlike `performanceAssert`, it does not have to sit in an `after` block - - # but keeping it there is still the right place for a gate. + # (A future Step release may let `assertMetric` default to the current + # execution's scope, which would make it usable here too; until then, gate + # a load test with `performanceAssert` and leave `assertMetric` for + # cross-run assertions.) # ========================================================================= - - name: "D - A threshold over the metric time series" - categories: ["Load testing"] - root: - threadGroup: - nodeName: "2 users x 2 searches" - users: 2 - iterations: 2 - children: - - sequence: - nodeName: "Search transaction" - instrumentNode: true - children: - - callKeyword: - keyword: "Search Products" - - after: - continueOnError: true - steps: - # An instrumented-node measurement - which performanceAssert - # would refuse - carrying a threshold via assertMetric. - - assertMetric: - nodeName: "The search transaction is not drifting" - metric: "response-time" - aggregation: AVG - comparator: LOWER_THAN - expectedValue: 60000 - filters: - - field: "name" - filterType: EQUALS - filter: "Search transaction" - - # The same run gated the reliable way, for comparison. - - performanceAssert: - nodeName: "And this run met its own SLA" - measurementName: "Search Products" - aggregator: AVG - comparator: LOWER_THAN - expectedValue: 5000 - - performanceAssert: - nodeName: "All four searches ran" - measurementName: "Search Products" - aggregator: COUNT - comparator: EQUALS - expectedValue: 4 # ========================================================================= - # E. A breached threshold. (INTENTIONALLY FAILS) + # D. A breached threshold. (INTENTIONALLY FAILS) # # What a violation looks like, so you recognise it in a report. "Slow # Search" takes about 600 ms and the threshold below allows 100, so this @@ -365,7 +316,7 @@ plans: # The second threshold below is also breached, and without the flag you # would fix the first one, re-run, and only then discover the second. # ========================================================================= - - name: "E - A breached SLA (intentionally fails)" + - name: "D - A breached SLA (intentionally fails)" categories: ["Load testing"] root: threadGroup: diff --git a/plans/load-testing/README.md b/plans/load-testing/README.md index a1ecda6..c6fcfce 100644 --- a/plans/load-testing/README.md +++ b/plans/load-testing/README.md @@ -17,7 +17,7 @@ asserts its own outcome rather than merely running. | 03 | [Scenarios and mixed load](03-scenarios-and-mixed-load/) | intermediate | `testScenario`, staged ramps, `sequence` as phases, `testSet` | | 04 | [Test data and data sets](04-test-data-and-datasets/) | intermediate | `dataSet`, `item`, `.next()`, `resetAtEnd`, data sources | | 05 | [Measurements](05-measurements/) | advanced | keyword / instrumented / custom measurements, `startMeasure`, naming | -| 06 | [Thresholds and SLA gates](06-thresholds-and-slas/) | advanced | aggregators, failure rate, `assertMetric` vs `performanceAssert` | +| 06 | [Thresholds and SLA gates](06-thresholds-and-slas/) | advanced | `performanceAssert` aggregators, failure rate, and why `assertMetric` is not a load-test gate | For what each control does and how to configure it, see the official [controls documentation](https://step.dev/knowledgebase/userdocs/plans/controls/). For the YAML @@ -37,7 +37,7 @@ shape of a standalone plan, see [../reference/](../reference/). | `instrumentNode` | 01, 05, 06 | | `output.startMeasure` / `stopMeasure` | 05 | | `performanceAssert` — `AVG`, `MAX`, `MIN`, `COUNT`, `SUM` | 01, 02, 03, 04, 05, 06 | -| `assertMetric` | 06 | +| `assertMetric` (cross-execution — why *not* to use it to gate a load test) | 06 | | `assert` inside a load test | 01, 05, 06 | | `continueOnError`, `continueParentNodeExecutionOnError` | 01, 06 | @@ -57,7 +57,7 @@ sample's README lists the expected outcome per plan: |--------|------| | [04](04-test-data-and-datasets/) | D — The pool runs dry | | [06](06-thresholds-and-slas/) | C — Threshold on the failure rate | -| [06](06-thresholds-and-slas/) | E — A breached SLA | +| [06](06-thresholds-and-slas/) | D — A breached SLA | Every other plan is expected to pass. From 3c5083510289fd9951751cf9542f493b010d61f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Mon, 24 Aug 2026 11:19:25 +0200 Subject: [PATCH 8/9] SED-4885 Creating plan samples for load testing - Removing duplicated content --- .../load-testing/01-first-load-test/README.md | 176 +++------------- .../automation-package.yaml | 37 +--- .../02-thread-group-configuration/README.md | 168 +++++----------- .../automation-package.yaml | 38 +++- .../03-scenarios-and-mixed-load/README.md | 102 +++------- .../04-test-data-and-datasets/README.md | 189 ++++-------------- plans/load-testing/05-measurements/README.md | 102 +++------- .../06-thresholds-and-slas/README.md | 175 ++++++---------- .../automation-package.yaml | 13 +- plans/load-testing/README.md | 5 +- 10 files changed, 274 insertions(+), 731 deletions(-) diff --git a/plans/load-testing/01-first-load-test/README.md b/plans/load-testing/01-first-load-test/README.md index 576bdf1..f73314e 100644 --- a/plans/load-testing/01-first-load-test/README.md +++ b/plans/load-testing/01-first-load-test/README.md @@ -14,164 +14,40 @@ The baseline shape of a load-testing plan: a thread group repeats one transactio virtual users at once, and the plan states the SLA that transaction has to meet. Every other sample in this set builds on this structure. -The subject of this sample is the **plan**. The keywords are 3-line Groovy stubs that simulate -a shop API, so the package runs on any Java agent — no build, no browser, no system under test. +**The lesson is the commented [`automation-package.yaml`](automation-package.yaml)** — read that +for the reasoning at each node. This page orients you and collects the reference tables. The +keywords are 3-line Groovy stubs simulating a shop API, so the package runs on any Java agent — +no build, no browser, no system under test. -## What this sample shows +## What it covers - `threadGroup` as the root of a load plan, with `users` and `iterations` -- **The iteration is the unit your load numbers are denominated in** — what to put in it, and why +- choosing what one iteration contains — the unit your load numbers are denominated in - `instrumentNode` for an end-to-end transaction measurement -- `performanceAssert` as the SLA gate, and **the two rules about where it may go** -- Why a load test still needs a functional `assert` - -## The anatomy of a load plan - -```yaml -root: - threadGroup: - users: 2 # virtual users, running in parallel - iterations: 3 # repetitions per user -> 6 transactions - children: - - sequence: - nodeName: "Search and buy" - instrumentNode: true # measure the transaction end to end - children: - - callKeyword: {...} - after: - continueOnError: true - steps: - - performanceAssert: {...} # the SLA -``` - -A `threadGroup` is to a load test what a `testCase` is to a functional test or a bot: the root -that says what "one run" means. `users` sizes the load — each virtual user holds its own agent -token for the whole thread group, so it also sizes the agent capacity you need. - -## What goes in one iteration - -Everything a thread group reports is *per iteration*, so what you put in `children` decides what -`users`, `pacing` and throughput actually mean. Take `users: 10` with `pacing: 30000`: - -| If `children` is… | …the run means | -|-------------------|----------------| -| one HTTP call | 20 HTTP calls a minute, and a 30-second pause between **every call** | -| search → add to cart → check out | 20 **purchases** a minute, and a 30-second pause between purchases | - -Same numbers, different tests. The second is a load somebody can state a requirement about — *the -shop must sustain 1200 orders an hour* — and its pacing models a real user, who pauses between -purchases rather than between two clicks of the same purchase. - -**Choose the iteration to be the thing your requirement is stated in.** Usually that is a complete -user action: search, add to cart, buy. Sometimes it genuinely is one call — an API whose -requirement reads *500 GET /products per second* is correctly modelled with one call per -iteration. What you want to avoid is an iteration nobody has a target for, because then the -throughput figure has to be divided by something before anyone can act on it. - -Sizing a transaction this way also gives the report an end-to-end number to gate on, which is what -`instrumentNode` below is for. +- `performanceAssert` as the SLA gate, and the two rules about where it may go +- why a load test still needs a functional `assert` ## Where measurements come from -| Measurement | Named after | Created by | -|-------------|-------------|-----------| -| Keyword call | the **keyword** | Step, automatically, for every call | -| Instrumented node | the node's `nodeName` | `instrumentNode: true` on the node | -| Custom | whatever the keyword chooses | the keyword itself — see [05](../05-measurements/) | - -`instrumentNode: true` on the `sequence` is what gives you an end-to-end "Search and buy" series -in the performance dashboards. Without it the report has three keyword timings and no number for -the transaction as a whole. - -## `performanceAssert`: two rules about placement - -`performanceAssert` compares an aggregate of a measurement against a threshold after the load is -over. It is what turns a load test from "here are some numbers, go and look" into a pass/fail -gate a pipeline can use. - -```yaml -- performanceAssert: - measurementName: "Checkout" - aggregator: AVG # AVG | MAX | MIN | COUNT | SUM - comparator: LOWER_THAN # LOWER_THAN | HIGHER_THAN | EQUALS - expectedValue: 5000 -``` - -**1. It must live in an `after` or `afterThread` block.** As an ordinary child — of the thread -group, of a `testCase`, anywhere — the execution ends in `TECHNICAL_ERROR` with: - -``` -PerformanceAssert can only be defined in an 'after' or 'after thread' block -``` - -| Block | Runs | -|-------|------| -| `after` | once, when the whole thread group has finished — the right place for a run-wide SLA | -| `afterThread` | once per virtual user, as that user finishes | +| Measurement | Named after | Created by | Can carry a `performanceAssert` | +|-------------|-------------|-----------|-------------------------------| +| Keyword call | the **keyword** | Step, automatically, for every call | yes | +| Instrumented node | the node's `nodeName` | `instrumentNode: true` | **no** — dashboards only | +| Custom | whatever the keyword chooses | the keyword itself — see [05](../05-measurements/) | yes | -**2. `measurementName` must name a keyword measurement** — or a custom measurement a keyword -created itself. The measurement produced by `instrumentNode` is *not* matched. Asserting on -`"Search and buy"` fails with: +## Two rules for `performanceAssert` -``` -No measurement is matching the defined filters. -``` - -which is the same error a typo in a keyword name produces, so the mistake is easy to miss. To -put an SLA on a multi-step transaction, emit a custom measurement from the keyword — see -[05-measurements](../05-measurements/). - -Set `continueOnError: true` on the `after` block. Without it the block stops at the first -breached threshold and you learn about one SLA violation per run instead of all of them. - -### Bound both sides - -The plan asserts a floor as well as a ceiling: - -```yaml -- performanceAssert: {measurementName: "Checkout", aggregator: MIN, - comparator: HIGHER_THAN, expectedValue: 50} -``` - -An upper bound alone passes happily when the measurement is missing, empty, or the keyword has -been stubbed down to nothing. A floor fails the moment the transaction stops doing real work. - -### `COUNT` checks completeness - -A keyword's `COUNT` is not simply `users` × `iterations`. It is how many times **that step was -reached**, and the two come apart as soon as an iteration is more than a single call: - -| Cause | Effect on the count | -|-------|---------------------| -| an iteration **fails part way** | keywords after the failure never run, so their counts fall short of the ones before | -| a keyword sits inside an `if` / `switch` | it runs only when the branch is taken — the count tells you how often that was | -| a data source **ran dry** | iterations carry on with nothing to work on | - -So the `COUNT` worth asserting is the one on the **last** step of the transaction: that is the -number saying the whole thing completed rather than merely started. Comparing counts across the -three keywords in this iteration is how you find where iterations were breaking off. - -Response times over a run that only completed a third of its transactions look wonderful. - -What `COUNT` does **not** measure here is throughput. This plan fixes `iterations`, so the count is -6 whether the system was quick or crawling — a slowdown stretches the *run* instead. To gate the -rate, pin the duration with `iterations: 0` + `maxDuration` and put the `COUNT` threshold on that; -[02](../02-thread-group-configuration/) plan D shows it. - -## A load test still has to check its answers - -A system under load starts returning fast, cheap, **wrong** responses — an error page renders -quicker than a checkout. The nested `assert` on the checkout status runs once per iteration, so a -failure at high load appears as a failure count next to the timings: - -```yaml -- assert: - actual: "status" - operator: EQUALS - expected: "CONFIRMED" -``` +1. **It must live in an `after` or `afterThread` block.** Anywhere else the run ends in + `TECHNICAL_ERROR`: `PerformanceAssert can only be defined in an 'after' or 'after thread' block`. + `after` runs once when the thread group finishes (run-wide SLA); `afterThread` runs once per + virtual user. +2. **`measurementName` must name a keyword or custom measurement**, never an `instrumentNode` one — + that fails with `No measurement is matching the defined filters.`, the same message a misspelled + name gives. For an SLA on a multi-step transaction, emit a custom measurement — see + [05](../05-measurements/). -Without it, the response times would look excellent and mean nothing. +Set `continueOnError: true` on the `after` block, or it stops at the first breach and hides the +rest. Bound thresholds on both sides — an upper bound alone passes when the measurement is empty. ## Key files @@ -188,5 +64,5 @@ Without it, the response times would look excellent and mean nothing. step ap execute -p . -u --token --projectName ``` -The execution report should show 6 passing transactions, 18 passing keyword calls and three -passing performance asserts. +The report should show 6 passing transactions, 18 passing keyword calls and three passing +performance asserts. diff --git a/plans/load-testing/01-first-load-test/automation-package.yaml b/plans/load-testing/01-first-load-test/automation-package.yaml index aa49f76..a169b85 100644 --- a/plans/load-testing/01-first-load-test/automation-package.yaml +++ b/plans/load-testing/01-first-load-test/automation-package.yaml @@ -177,34 +177,17 @@ plans: continueOnError: true steps: - # Did this STEP actually run as often as the load implies? + # A sanity check that every transaction completed. COUNT here is + # how many times "Checkout" ran; if an earlier step failed - as + # it might under load - that iteration aborts before checkout and + # the count falls short of 6. Paired with the AVG/MAX below, it + # stops a run that quietly did less work from hiding behind a + # healthy average. # - # A keyword's COUNT is not simply users x iterations. It is how - # many times that particular step was reached, and the two come - # apart as soon as an iteration is more than a single call: - # - # an iteration that FAILS PART WAY the keywords after the - # failure never run, so - # their counts fall short of - # the ones before it - # a keyword inside an `if`/`switch` it runs only when the - # branch is taken, so its - # count tells you how often - # that was - # a data source that RAN DRY iterations continue with - # nothing to work on - # - # Which is why the COUNT to assert is the one on the LAST step of - # the transaction: it is the number that says the whole thing - # completed, not just started. Comparing counts across the three - # keywords of this iteration is how you find where iterations - # were breaking off. - # - # It is a COMPLETENESS check, not a throughput one: `iterations` - # is fixed here, so this is 6 however slow the system was. A - # slowdown stretches the RUN instead. To gate the RATE, pin the - # duration with `iterations: 0` + `maxDuration` and put the COUNT - # threshold on that - see sample 02 plan D. + # Note it is a COMPLETENESS check, not throughput: `iterations` is + # fixed, so this stays 6 however slow the run was - a slowdown + # stretches the RUN instead. To gate the RATE, pin the duration + # with `iterations: 0` + `maxDuration` - see sample 02 plan D. - performanceAssert: nodeName: "All 6 checkouts ran" measurementName: "Checkout" diff --git a/plans/load-testing/02-thread-group-configuration/README.md b/plans/load-testing/02-thread-group-configuration/README.md index 21687ff..6d5207e 100644 --- a/plans/load-testing/02-thread-group-configuration/README.md +++ b/plans/load-testing/02-thread-group-configuration/README.md @@ -12,10 +12,10 @@ level: beginner `users` and `iterations` describe a load, but not a realistic one. A real population does not appear all at once, does not hammer the system as fast as it can answer, and does not stop after -exactly N clicks. +exactly N clicks. This sample covers every thread-group knob and the four before/after blocks. -This sample covers every knob a thread group has, and the four blocks that decide what runs once, -what runs once per virtual user, and what runs on every iteration. +**The lesson is the commented [`automation-package.yaml`](automation-package.yaml).** This page maps +the knobs and calls out the handful that most often trip people up. ## The plans @@ -24,122 +24,69 @@ what runs once per virtual user, and what runs on every iteration. | A — The three counters | `gcounter`, `userId`, `literationId`, and `item` / `userItem` / `localItem` | | B — Pacing sets the throughput | `pacing` — controlling throughput instead of concurrency | | C — Ramping the load up | `rampup`, `pack`, `startOffset` | -| D — Run for a fixed time | `maxDuration`, and the trap that comes with it | +| D — Run for a fixed time | `iterations: 0` + `maxDuration` | | E — Setup per test versus per user | `before`, `beforeThread`, `afterThread`, `after` | -Every plan in this package is expected to pass; none of them fails by design. +All five are expected to pass. ## The knobs | Field | Meaning | |-------|---------| | `users` | virtual users running in parallel — each holds an agent token for the whole thread group | -| `iterations` | repetitions per user | -| `pacing` | fixed period between the **starts** of consecutive iterations | -| `rampup` | time taken to start all the users | +| `iterations` | repetitions per user (but see the table below) | +| `pacing` | fixed period between the **starts** of consecutive iterations — makes throughput a number you choose, so two runs compare | +| `rampup` | time taken to start all the users; starting all at once is a spike test, not a load test | | `pack` | how many users are released together at each ramp-up step | -| `startOffset` | delay before the ramp-up begins | +| `startOffset` | delay before the ramp-up begins — only useful for staggering thread groups in a [scenario](../03-scenarios-and-mixed-load/) | | `maxDuration` | wall-clock cap on the whole thread group | -### `pacing` — the field that makes runs comparable +## Notes worth knowing -Without pacing, a virtual user starts its next iteration the instant the previous one returns. -That is a stress test: the faster the system answers, the harder you hit it, so throughput is -whatever the system allows and two runs cannot be compared. +Not a full reference — the [controls documentation](https://step.dev/knowledgebase/userdocs/plans/controls/) +and the commented descriptor cover every field. These are the ones this sample dwells on because +they are the easiest to get wrong. -`pacing` makes throughput a number you choose: +### Controlling throughput — `pacing` with `users` + +Define the load by `users` alone and the throughput is **uncontrolled** — it rises and falls with +response time, so no two runs compare. This is the most common mistake. `pacing` (the fixed gap +between iteration *starts*) turns throughput into a number you set: ``` -iterations per second = users / (pacing in seconds) +iterations/sec = users ÷ pacing(s) pacing(s) = users ÷ target-rate ``` -One user with a 3000 ms pacing is 20 transactions an hour whether the system answers in 100 ms or -in two seconds. That is also what lets response time degrade *visibly* instead of being masked by -the load quietly backing off. - -`sequence` has a `pacing` field too, for pacing an inner block. - -### `rampup` and `pack` - -`rampup` is the time the group takes to start all its users. Four users over an 8000 ms ramp-up -start at 0, 2, 4 and 6 seconds. Starting everyone at once is a spike test — a different question, -measuring the cold start rather than the steady state. - -`pack` is how many users are released **together** at each step. The same four users over 8000 ms -with `pack: 2` start at 0 s and 4 s, two at a time. Use it for a stepped ramp, or simply to stop -a long ramp-up from trickling in one user at a time. - -`startOffset` delays the ramp-up. On its own in a single thread group it does nothing useful; its -purpose is staggering several thread groups inside a scenario — see [03](../03-scenarios-and-mixed-load/). - -### Running for a time — `iterations` and `maxDuration` together - -Most load tests are specified as "an hour at this rate", not "500 iterations". `maxDuration` is -the wall-clock cap on the whole thread group. How it combines with `iterations` is the part worth -getting right, because the obvious value is not the one you want: - -| `iterations` | Each user | `maxDuration` acts as | -|--------------|-----------|-----------------------| -| `N` | runs N times | a ceiling — cuts the run short if it fires first | -| `0` | **loops until `maxDuration`** | the real stop condition | -| omitted | runs **once** | a ceiling on that single iteration — it does **not** loop | - -So a duration-bounded run is **`iterations: 0` with `maxDuration`** — not a large iteration count -picked to be "unreachable", and not `maxDuration` on its own (which runs once). The run then takes -the same wall-clock time whether the system is fast or slow that day, which is the whole point: -comparable runs. +So 1 user at 3 s pacing is 20 transactions a minute, fast system or slow. One caveat: it holds only +while each iteration finishes **within** its pacing window — if an iteration outlasts `pacing`, that +user falls behind and the rate drops back to response-time-bound. Give `pacing` headroom over the +slowest iteration, or add users. -That configuration is also what makes a `COUNT` threshold mean throughput. Throughput is iterations -÷ elapsed time, and the two configurations pin different halves: +### Running for a duration -| Configuration | Pinned | A slow system produces | `COUNT` is therefore | -|---------------|--------|------------------------|----------------------| -| `iterations: 0` + `maxDuration` | the duration | **fewer iterations** | a **throughput** measure | -| fixed `iterations` | the count | a **longer run** | a **completeness** check — same number quick or slow | +A fixed count needs nothing special — `iterations: N`. Running for a *time* is the non-obvious one: +it needs **`iterations: 0`** (unlimited) together with `maxDuration`. `maxDuration` on its own does +**not** give a duration-bounded run, because the `iterations` default is 1: -Either way throughput fell, so never gate on response time alone: a struggling system meets any -latency target by doing less work per unit time. A throughput aggregator that works whichever way -the group is configured is planned for `performanceAssert` in a future release; until then, gate a -rate by pinning the duration and counting. +| `iterations` | Result | +|--------------|--------| +| `0` | loops until `maxDuration` — the duration-bounded run | +| omitted | runs **once** (defaults to 1); `maxDuration` only caps that single iteration | -## The counters every thread group publishes +### The counters every thread group publishes -Three variables are always in scope inside a thread group: - -| Variable | Value | -|----------|-------| -| `userId` | which virtual user this is — 1..`users` | -| `literationId` | the iteration within **this** user — 1..`iterations` | -| `gcounter` | the iteration across the **whole** thread group — 1..(`users` × `iterations`), unique | - -They are how you build unique test data — order numbers, e-mail addresses, search terms — with no -external data source at all. `gcounter` when the value must be unique across the run, `userId` -when it must be stable for one virtual user. - -`item`, `userItem` and `localItem` rename them: - -| Field | Renames | -|-------|---------| -| `item` | `gcounter` | -| `userItem` | `userId` | -| `localItem` | `literationId` | - -**Renaming replaces.** Once `userItem: "shopperNo"` is set, `userId` no longer exists — worth -knowing before renaming one counter in a plan that reads another by its default name. - -### Coerce the counter before passing it on - -```yaml -- iteration: - expression: "gcounter as Integer" -``` +| Variable | Value | Renamed by | +|----------|-------|-----------| +| `userId` | which virtual user this is — 1..`users` | `userItem` | +| `literationId` | the iteration within **this** user — 1..`iterations` | `localItem` | +| `gcounter` | the iteration across the **whole** group — 1..(`users`×`iterations`), unique | `item` | -Without `as Integer` the counter reaches the keyword as a string, `input.getInt` silently returns -its default, and every iteration takes the same branch. Nothing errors; the test just stops -testing what you meant. Sample [06](../06-thresholds-and-slas/) has a keyword that reports this -case explicitly rather than absorbing it. +Build unique test data from these — `gcounter` when it must be unique across the run, `userId` when +it must be stable for one user. **Renaming replaces:** once `userItem: "shopperNo"` is set, `userId` +no longer exists. Coerce with `as Integer` before passing a counter to a keyword, or it arrives as a +string and `input.getInt` silently returns its default (see [06](../06-thresholds-and-slas/)). -## The four blocks +### The four blocks | Block | Runs | |-------|------| @@ -149,33 +96,10 @@ case explicitly rather than absorbing it. | `afterThread` | once per virtual user, after its last iteration | | `after` | once, when the whole thread group has finished | -**Put each step in the block that matches how often a real user does it.** That is the whole rule, -and getting it wrong distorts the traffic mix rather than producing any visible error. - -A real user logs in once per session and then does twenty things. Move `Login` from `beforeThread` -into `children` in plan E and the run sends **6 logins instead of 2** — triple the load on the -authentication service, and a traffic mix the production system never sees. Nothing fails; the -test just stops describing reality. - -So: once-per-session steps in `beforeThread`, once-per-transaction steps in `children`, and setup -the whole test needs once — warming a cache, seeding a data set — in `before`. `afterThread` is -the reliable place for per-user cleanup: it runs even when an iteration failed, which a last child -would not. - -There is a second, narrower consequence for measurements. Per-keyword measurements are **not** -affected — `performanceAssert` aggregates by keyword name, so `Login` and `Checkout` stay separate -series wherever they sit. What does get polluted is any **transaction-level** measurement that -wraps the iteration: an `instrumentNode` sequence around both calls, or a custom measurement -spanning them, would fold the login time into the transaction number you gate on. - -Plan E makes all of this visible with counts rather than prose: 1 warm-up, 2 logins, 6 checkouts, -2 logouts. - -A variable `set` in `before` is visible to the iterations and to the `after` block. Plan B uses -that to time itself and prove pacing did something — a `COUNT` assertion alone would pass whether -pacing worked or not. **That timestamp is a self-check for the sample, not a pattern to copy:** -timing a run from inside the plan is not how thresholds are written in Step. See -[06](../06-thresholds-and-slas/) for the controls that are. +**Put each step in the block that matches how often a real user does it.** Getting it wrong +distorts the traffic mix with no visible error: a login in `children` instead of `beforeThread` +sends one login per iteration instead of one per session. Plan E proves the placement with counts +(1 warm-up, 2 logins, 6 checkouts, 2 logouts). ## Key files diff --git a/plans/load-testing/02-thread-group-configuration/automation-package.yaml b/plans/load-testing/02-thread-group-configuration/automation-package.yaml index 326e4b1..931ba30 100644 --- a/plans/load-testing/02-thread-group-configuration/automation-package.yaml +++ b/plans/load-testing/02-thread-group-configuration/automation-package.yaml @@ -105,14 +105,31 @@ plans: # STARTS of consecutive iterations. If an iteration takes less than that, # the user waits; if it takes longer, the next one starts immediately. # - # That makes throughput a number you CHOOSE: + # That makes throughput a number you CHOOSE, out of `users` and `pacing`: # # iterations per second = users / (pacing in seconds) # - # 1 user with a 3000 ms pacing is 20 transactions an hour, whether the - # system answers in 100 ms or in 2 seconds. That is the property that makes - # a load test repeatable - and the one that lets response time degrade - # visibly instead of being masked by the load backing off. + # and, turned around, to hit a target rate: + # + # pacing (seconds) = users / (target iterations per second) + # + # 1 user at 3000 ms pacing is one checkout every 3 s - 20 a minute, 1200 an + # hour - whether the system answers in 100 ms or in 2 s. That is the + # property that makes a load test repeatable, and the one that lets response + # time degrade visibly instead of being masked by the load backing off. + # + # TWO THINGS PEOPLE GET WRONG: + # + # - Setting only `users` and leaving `pacing` out gives UNCONTROLLED + # throughput - the stress-test case above, where the rate is whatever + # the system allows and no two runs compare. Defining the load by users + # alone is the most common mistake. + # + # - The formula holds only while each iteration FITS INSIDE its pacing + # window. If an iteration outlasts `pacing`, that user starts the next + # one immediately and falls behind, so the real rate drops below + # users/pacing - back to response-time-bound. Give `pacing` headroom + # over the slowest expected iteration, or add users. # # `sequence` has a `pacing` field too, for pacing an inner block. # ========================================================================= @@ -215,16 +232,17 @@ plans: # iterations". `maxDuration` is the wall-clock cap on the whole thread # group: when it expires, the group stops wherever it is. # - # HOW `iterations` AND `maxDuration` COMBINE - the part that is easy to get - # wrong, because the sensible-looking value is not the one you want: + # HOW `iterations` AND `maxDuration` COMBINE. A specified count is a target + # and 0 means unlimited; either way `maxDuration` caps the run. The one + # thing that surprises people is the DEFAULT when `iterations` is left out: # - # iterations: N each user runs N times, UNLESS maxDuration fires first - # and cuts it short. `maxDuration` is a ceiling here. + # iterations: N each user runs UP TO N times - `maxDuration` is a + # ceiling that cuts it short if it fires first. # iterations: 0 UNLIMITED - each user loops until maxDuration expires. # This is what "run for a duration" actually means. # iterations omitted -> defaults to 1 per user. `maxDuration` then # omitted only caps that single iteration; it does NOT make the - # group loop. A frequent surprise. + # group loop. The one real gotcha here. # # So the form for a duration-bounded run is `iterations: 0` WITH # `maxDuration`, as below - not a large iteration count picked to be diff --git a/plans/load-testing/03-scenarios-and-mixed-load/README.md b/plans/load-testing/03-scenarios-and-mixed-load/README.md index a055b85..518fc41 100644 --- a/plans/load-testing/03-scenarios-and-mixed-load/README.md +++ b/plans/load-testing/03-scenarios-and-mixed-load/README.md @@ -10,12 +10,13 @@ level: intermediate # 03 — Scenarios and mixed load -Real systems are never hit by one kind of user. While a few hundred people browse, a handful buy, -and at 02:00 a batch job runs straight through the middle of it. A load test that models only the -happy path measures a system nobody is using. +Real systems are never hit by one kind of user. While a few hundred browse, a handful buy, and at +02:00 a batch job runs through the middle of it. A load test that models only the happy path +measures a system nobody is using. `testScenario` composes those populations — it runs its children +**in parallel**, each with its own profile. -`testScenario` is the control that composes those populations: it runs its children **in -parallel**, each with its own load profile. +**The lesson is the commented [`automation-package.yaml`](automation-package.yaml).** This page +holds the plan index and the reference tables. ## The plans @@ -26,7 +27,7 @@ parallel**, each with its own load profile. | C — Measure the site while the batch job runs | A thread group next to a plain `sequence` | | D — Warm-up phase then measured phase | `sequence` as the root — phases instead of parallelism | -Every plan in this package is expected to pass; none of them fails by design. +All four are expected to pass. ## The three composing roots @@ -36,80 +37,37 @@ Every plan in this package is expected to pass; none of them fails by design. | `sequence` | **one after the other** | phases — warm-up, measured, cool-down | | `testSet` | as separate test cases, `threads` at a time | a batch of independent tests, not a load profile | -Confusing `testScenario` with `testSet` is the usual slip. `testSet` parallelises *test cases*; +Confusing `testScenario` with `testSet` is the usual slip: `testSet` parallelises *test cases*, `testScenario` runs *load profiles* side by side. -## The mixed-workload shape +## Notes worth knowing -Each thread group keeps its own configuration — that is the whole point. Browsers are many and -cheap; buyers are few and expensive. Modelling them as one thread group doing "browse then buy" -produces a traffic mix no real shop ever sees. +Selective notes, not a full reference — the commented descriptor covers every case. These are the +points most worth getting right. -```yaml -root: - testScenario: - children: - - threadGroup: {nodeName: "Browsers", users: 2, iterations: 3, pacing: 1000, ...} - - threadGroup: {nodeName: "Buyers", users: 1, iterations: 2, startOffset: 2000, ...} -``` - -`startOffset` on the buyers means the checkout numbers are measured against a system that is -already busy, rather than an idle one. - -## Where the SLA goes +### Where the SLA goes | Placement | Evaluated | |-----------|-----------| | `after` on the **testScenario** | once, when every thread group has finished — the scenario-wide gate | -| `after` on a **thread group** | when that population finishes — the place for a threshold that concerns only it | - -Both work. Prefer the thread group's own block when the threshold belongs to one population: it -reads next to the profile it describes. [06](../06-thresholds-and-slas/) covers this in more -detail. - -Assert a `COUNT` **per population**, and make each one exact. A scenario mixes populations running -different transactions, so no single aggregate describes it — and if the buyers broke off half way -through their transaction while the browsers ran in full, the totals still look plausible. Only the -per-population counts show it. - -## A staged ramp - -`rampup` inside one thread group gives a smooth ramp to a single target. What it cannot express is -a **step profile**: hold 1 user, then 2, then 3, each for a while, and compare the response times -between the steps. That is the shape that answers *where does it start to hurt?* - -Several thread groups in a scenario, staggered with `startOffset`, is how you write it. Each stage -is its own node in the report, so each has its own numbers. In a real test each stage also carries -`maxDuration`, to hold the load for a fixed time rather than a fixed number of iterations. - -A `performanceAssert` on a measurement name spans **every** node that produced it, so a COUNT over -the whole ramp is the total across all stages. To compare the stages against each other — the -reason to build a ramp this way — read them in the performance dashboards, where each stage is a -separate node; a per-stage threshold goes in that stage's own `after` block. - -## Load next to a batch job - -A scenario's children do not all have to be thread groups. "What do response times look like while -the nightly batch runs?" is a thread group next to a plain `sequence`. The sequence runs once, in -parallel with the load, and its duration is measured like anything else. - -## Phases, and why a warm-up needs its own - -Nesting a warm-up in the same thread group as the measured load pollutes the measurement with -cold-start timings. A separate warm-up **phase** keeps them in different nodes, so the threshold -applies only to the measured phase: - -```yaml -root: - sequence: - children: - - threadGroup: {nodeName: "Warm-up phase", ...} - - threadGroup: - nodeName: "Measured phase" - after: - steps: - - performanceAssert: {...} # on the measured group, not the sequence -``` +| `after` on a **thread group** | when that population finishes — for a threshold concerning only it | + +Prefer the thread group's own block when the threshold belongs to one population. Assert a `COUNT` +**per population** and make each exact: a scenario mixes different transactions, so no single +aggregate describes it, and one population breaking off mid-transaction still leaves the totals +looking plausible. + +### Patterns in the plans + +- **Staged ramp** (B) — `rampup` inside one thread group ramps to a single target; a *step* profile + (hold 1 user, then 2, then 3, comparing the steps) is several thread groups staggered with + `startOffset`, each its own node in the report. Compare the stages in the dashboards, not with one + cross-stage `COUNT`. +- **Load next to a batch** (C) — a scenario child need not be a thread group; a plain `sequence` + runs once, in parallel with the load, and is measured like anything else. +- **Phases** (D) — a warm-up nested in the measured thread group pollutes its measurement with + cold-start timings; a separate warm-up *phase* under a `sequence` root keeps them in different + nodes, so the threshold applies only to the measured phase. ## Key files diff --git a/plans/load-testing/04-test-data-and-datasets/README.md b/plans/load-testing/04-test-data-and-datasets/README.md index b8c7fd6..83c2d63 100644 --- a/plans/load-testing/04-test-data-and-datasets/README.md +++ b/plans/load-testing/04-test-data-and-datasets/README.md @@ -10,14 +10,14 @@ level: intermediate # 04 — Test data and data sets -A load test that sends the same account and the same product on every iteration measures the -target's caches, not the target. Realistic load needs a pool of test data, handed out so that no -two virtual users collide. +A load test that sends the same account and product on every iteration measures the target's +caches, not the target. Realistic load needs a pool of test data, handed out so that no two virtual +users collide. `dataSet` is the control for that — and it does **not** work like `forEach`. -`dataSet` is the control for that, and it does **not** work like `forEach`. +**The lesson is the commented [`automation-package.yaml`](automation-package.yaml)**, which walks +each shape at its node. This page holds the plan index and the reference tables. -> **One plan in this package fails on purpose.** That is the lesson in it: the plan name says -> which one. +> **One plan fails on purpose** — the plan name says which. ## The plans @@ -31,167 +31,56 @@ two virtual users collide. ## `dataSet` is a declaration, not a loop -This is the one thing to understand, and it is the opposite of what the name suggests: - | Control | What it is | |---------|-----------| -| `forEach` | a **loop**. It runs its children once per row. | -| `dataSet` | a **declaration**. It opens the data source, binds a **cursor** over it to the variable named by `item`, and runs nothing. | - -A `dataSet` node with children is a common and completely silent mistake: **the children are never -executed, and the node still reports PASSED.** - -## Declare the pool in a `before` block - -The declaration belongs in the `before` block of whatever node contains the load — and the thread -group stays the root of the plan: - -```yaml -root: - threadGroup: - users: 1 - iterations: 3 - - before: # runs once, before the load starts - steps: - - dataSet: - item: "shopperPool" # names the CURSOR, not the row - resetAtEnd: true - dataSource: - csv: {file: "data/users.csv"} - - children: - - set: - key: shopper - value: - expression: "shopperPool.next()" # one pull per iteration - - callKeyword: - keyword: "Login" - inputs: - - user: - expression: "shopper.Username" -``` - -`item` names the cursor. `shopperPool` is an object you call `.next()` on — it is **not** a map of -columns. `.next()` advances the cursor and returns the row as a map. - -Two reasons for `before` rather than a sibling node: - -- **It removes a race.** See the next section — in a `testScenario` a sibling declaration is not - guaranteed to be bound before the load starts pulling from it. -- **It keeps the thread group at the root.** A load plan's root should be the load profile. - Wrapping the whole thing in a `testCase` just to have somewhere to put the declaration buries - the profile one level down for no benefit. - -## One pool, several thread groups - -`testScenario` runs its children **in parallel**. A `dataSet` written as a plain sibling of the -thread groups is therefore racing them: nothing guarantees the cursor is bound before the first -`.next()`, and the failure mode is a plan that works on a quiet instance and breaks on a busy one. +| `forEach` | a **loop** — runs its children once per row | +| `dataSet` | a **declaration** — opens the source, binds a **cursor** to the variable named by `item`, and runs nothing | -Put it in the scenario's `before` block, which runs to completion before any child starts: +A `dataSet` node with children is a silent mistake: the children never execute, and the node still +reports PASSED. Instead, declare it in the **`before` block** of the node that holds the load (the +thread group stays the plan root), and pull one row with `item.next()` inside the load. In a +`testScenario`, `before` is also what removes the race — a sibling `dataSet` is not guaranteed to be +bound before parallel thread groups start pulling. -```yaml -root: - testScenario: - before: - steps: - - dataSet: {item: "shopperPool", resetAtEnd: true, dataSource: {...}} - children: - - threadGroup: {nodeName: "Browsers", ...} - - threadGroup: {nodeName: "Buyers", ...} -``` - -Once bound, the cursor is **shared**. Every `.next()` in the execution, from whichever thread -group, takes the *following* row — so two populations drawing from one pool never collide on the -same account, which is exactly what you want when the system under test locks a session per user. - -The consequence: **which** rows a given thread group gets is not deterministic. It depends on the -order the threads happen to reach their `.next()`. Never write a plan that assumes "the buyers get -shopper1". - -## Where you pull decides how much data you need - -Three blocks, three frequencies: - -| Block | Runs | Holds | -|-------|------|-------| -| `before` | once for the test | the **declaration** | -| `beforeThread` | once per virtual user | a pull, if the account belongs to the user | -| `children` | every iteration | a pull, if the data belongs to the transaction | - -`before` is guaranteed to run before `beforeThread`, so the cursor is always bound by the time a -thread claims its row. - -Per-virtual-user is the shape most load tests actually need: a virtual user logs in once and then -does twenty things as that account — it does not become a different person between two clicks. - -```yaml -before: - steps: - - dataSet: {item: "shopperPool", resetAtEnd: false, dataSource: {...}} -beforeThread: - steps: - - set: - key: myShopper - value: - expression: "shopperPool.next()" - - callKeyword: {keyword: "Login", ...} -``` +The cursor is **shared**: every `.next()` in the execution takes the following row, so populations +never collide on the same account — but *which* rows a given group gets is not deterministic. Never +assume "the buyers get shopper1". -It is also much cheaper. Plan C runs 2 users × 3 iterations against a pool of 4 rows with -`resetAtEnd: false`. Pulling per iteration would exhaust the pool; pulling per thread uses 2 rows. -The plan proves it with counts — two logins for six orders. +## Notes worth knowing -## When the pool runs dry +Selective notes, not a full reference — the commented descriptor covers every case. These are the +points most worth getting right. -With `resetAtEnd: false`, `.next()` past the last row does **not** stop the thread group and does -**not** raise an error. It returns `null`, and the run carries on feeding `null` into the keywords. +### Where you pull decides how much data you need -Depending on the keyword that is either a confusing `NullPointerException` deep in the report or — -far worse — a keyword that shrugs, sends an empty value, and keeps the run green while half the -load was meaningless. +| Block | Runs | Pull here when… | +|-------|------|-----------------| +| `before` | once for the test | never — this is the **declaration** | +| `beforeThread` | once per virtual user | the account belongs to the user (log in once, act many times) | +| `children` | every iteration | the data belongs to the transaction | -Guard the pull: +`before` always runs before `beforeThread`, so the cursor is bound by the time a thread claims its +row. Per-virtual-user is the common case *and* the cheaper one: plan C serves 2×3 iterations from a +4-row pool by pulling twice, not six times. -```yaml -- check: - nodeName: "The pool still had a row to give" - expression: "shopper != null" -``` +### `resetAtEnd`, and running dry -Decide `resetAtEnd` deliberately: +With `resetAtEnd: false`, `.next()` past the last row returns **`null`** — no stop, no error — and +the run feeds `null` into the keywords. Guard the pull with a `check` on `!= null`. | Value | Meaning | |-------|---------| -| `true` | recycle the rows — fine when the target does not care that the same account comes back | -| `false` | each row used at most once — then size the pool for the whole run, and guard the pull | +| `true` | recycle the rows — fine when the target does not mind the same account returning | +| `false` | each row used at most once — size the pool for the whole run, and guard the pull | -## Data sources +### Data sources Only the `dataSource` block changes; the pull is always `.next()`. Available: `csv`, `excel`, -`file`, `folder`, `gsheet`, `json`, `json-array`, `sequence`, `sql`. - -The two that come up most in load tests are `csv` — a generated pool checked in next to the plan — -and `sql`, which reads the pool straight out of the system under test: - -```yaml -dataSource: - sql: - connectionString: "jdbc:postgresql://db:5432/shop" - driverClass: "org.postgresql.Driver" - user: "loadtest" - password: - expression: "dbPassword" - query: "SELECT username, product_id FROM test_accounts" -``` - -`sequence` is the one to reach for when the data does not need to exist beforehand — a pool of -unique order numbers, say. And for data that needs no pool at all, the thread-group counters -(`gcounter`, `userId`) from [02](../02-thread-group-configuration/) are often enough. - -A source holding credentials can be declared with `protect: true`, which obfuscates its values in -the report. +`file`, `folder`, `gsheet`, `json`, `json-array`, `sequence`, `sql`. The common ones for load are +`csv` (a pool checked in beside the plan) and `sql` (read straight from the system under test); +`sequence` needs no data to exist beforehand, and for data needing no pool at all the thread-group +counters from [02](../02-thread-group-configuration/) are often enough. A credential source takes +`protect: true` to obfuscate its values in the report. ## Key files diff --git a/plans/load-testing/05-measurements/README.md b/plans/load-testing/05-measurements/README.md index 793b3a6..a8c5d65 100644 --- a/plans/load-testing/05-measurements/README.md +++ b/plans/load-testing/05-measurements/README.md @@ -11,8 +11,11 @@ level: advanced # 05 — Measurements Everything a load test reports comes out of measurements. Which ones exist, what they are called -and how they nest is decided partly by Step and partly by the plan — and getting it wrong is how -you end up with a run full of numbers that cannot answer the question you ran it to answer. +and how they nest is decided partly by Step and partly by the plan — and getting it wrong leaves a +run full of numbers that cannot answer the question you ran it to answer. + +**The lesson is the commented [`automation-package.yaml`](automation-package.yaml)** and the two +measurement keywords. This page holds the plan index and the reference tables. ## The plans @@ -22,92 +25,43 @@ you end up with a run full of numbers that cannot answer the question you ran it | B — A custom measurement around a whole transaction | `startMeasure` / `stopMeasure`, nested | | C — Count the failures too | Why a response-time average alone is the most misleading number in load testing | -Every plan in this package is expected to pass; none of them fails by design. +All three are expected to pass. ## The three sources | Source | Named after | Created by | Can carry a `performanceAssert` | |--------|-------------|-----------|-------------------------------| -| Keyword call | the **keyword** | Step, automatically, for every call | **yes** | -| Instrumented node | the node's `nodeName` | `instrumentNode: true` | **no** | +| Keyword call | the **keyword** (never the node) | Step, automatically, for every call | **yes** | +| Instrumented node | the node's `nodeName` | `instrumentNode: true` | **no** — dashboards only | | Custom | whatever the keyword chooses | `output.startMeasure(...)` in the keyword | **yes** | -A keyword measurement is named after the **keyword**, never the node. Two calls to `Checkout` from -different nodes land in the same series — usually what you want, occasionally not. - -`instrumentNode: true` on any node times that node and reports it under its `nodeName`. This is -how a multi-keyword transaction gets an end-to-end number in the dashboards. - -Custom measurements are the only way to time something **smaller** than a keyword call — the two -page loads inside one browser keyword, for instance. - -## Which measurements can carry an SLA - -`performanceAssert` sees keyword measurements and custom measurements. It does **not** see -instrumented-node measurements. Asserting on an instrumented sequence fails with: - -``` -No measurement is matching the defined filters. -``` - -which is the same message a misspelled keyword name produces — so the mistake reads as a typo and -gets "fixed" by changing a name that was already right. - -**Instrumented nodes are for the dashboards. SLAs go on keyword or custom measurements.** To put a -threshold on a whole multi-step transaction, emit a custom measurement around it from inside the -keyword — which is what plan B does. +Custom measurements are the only way to time something **smaller** than a keyword call (two page +loads inside one browser keyword), and they **nest** — `stopMeasure()` closes the most recently +opened one, so one keyword can report an end-to-end journey *and* where inside it the time went. +Asserting on an instrumented-node measurement fails with `No measurement is matching the defined +filters.` — the same message a misspelled name gives, so the mistake reads as a typo. For an SLA on +a multi-step transaction, wrap it in a custom measurement (plan B), not an instrumented sequence. -## Custom measurements +## Notes worth knowing -```groovy -output.startMeasure("Catalog page") -// ... do the work -output.stopMeasure() - -output.startMeasure("Product page") -// ... do the work -output.stopMeasure(["page": "product-detail"]) // with attributes -``` - -Measurements **nest**: `stopMeasure()` closes the most recently opened one. Plan B's keyword wraps -three inner steps in one `Purchase journey`, so the report shows both the end-to-end journey and -where inside it the time went — and the journey, being a custom measurement, can carry the SLA -that the equivalent instrumented sequence could not. - -The plan proves the nesting rather than asserting it in prose: the journey's `MIN` must exceed the -sum of its parts' sleeps, and the two page measurements in plan A are bounded on opposite sides of -the same threshold, so they cannot be the same number reported twice. - -## Naming measurements - -A measurement name is the axis every dashboard, threshold and comparison is grouped by. The names -**are** the report. Two rules pay for themselves: - -**Name the business step, not the technical one.** `Checkout` tells you what broke. -`POST /api/v3/order` tells you where, which you can find out afterwards anyway. - -**Keep the name stable and bounded.** A name built out of a variable — `Checkout for shopper 417` -— creates one series per virtual user. Nothing aggregates, the dashboards fill with noise, and no -threshold can be written against it. Put the varying part in an **attribute** instead: - -```groovy -output.stopMeasure(["region": "eu"]) -``` +Selective notes, not a full reference — the commented descriptor covers every case. These are the +points most worth getting right. -## Measure the failures, not just the successes +### Naming measurements -A response-time average computed over successful calls only is the most comfortable and most -misleading number in load testing. When a system under stress starts failing fast, **the average -improves.** +The name is the axis every dashboard and threshold groups by — the names **are** the report. -Two habits prevent it: +- **Name the business step, not the endpoint.** `Checkout`, not `POST /api/v3/order`. +- **Keep the name stable and bounded.** A name built from a variable (`Checkout for shopper 417`) + makes one series per user and nothing aggregates. Put the varying part in an **attribute**: + `output.stopMeasure(["region": "eu"])`. -- **Assert the count as well as the time.** A run that did half the work has a great average. - Every plan in this set carries a `COUNT` assertion for that reason. -- **Check the answers**, so failures are counted as failures. A nested `assert` on the keyword - output turns a wrong response into a failure count in the report, next to the timings. +### Measure the failures, not just the successes -[06](../06-thresholds-and-slas/) puts thresholds on that failure count. +An average over successful calls only is the most misleading number in load testing — when a system +starts failing fast, the average *improves*. Two habits prevent it: **assert the count** as well as +the time (every plan here carries a `COUNT`), and **check the answers** so a wrong response becomes +a failure count. [06](../06-thresholds-and-slas/) puts thresholds on that failure count. ## Key files diff --git a/plans/load-testing/06-thresholds-and-slas/README.md b/plans/load-testing/06-thresholds-and-slas/README.md index 221ae75..34e8cc8 100644 --- a/plans/load-testing/06-thresholds-and-slas/README.md +++ b/plans/load-testing/06-thresholds-and-slas/README.md @@ -10,14 +10,13 @@ level: advanced # 06 — Thresholds and SLA gates -A load test that produces a report somebody has to interpret is a load test nobody runs twice. A -load test that comes back PASSED or FAILED is a gate you can put in a pipeline. +A load test that produces a report somebody has to interpret is a load test nobody runs twice. One +that comes back PASSED or FAILED is a gate you can put in a pipeline. -This sample covers the two threshold controls, what each can and cannot see, and the thresholds a -load test needs beyond "average response time". +**The lesson is the commented [`automation-package.yaml`](automation-package.yaml).** This page +holds the plan index and the reference tables. -> **Two plans in this package fail on purpose.** That is the lesson in them: the plan names say -> which ones. +> **Two plans fail on purpose** — the plan names say which. ## The plans @@ -28,143 +27,86 @@ load test needs beyond "average response time". | C — Threshold on the failure rate | **FAILED** (on purpose) | Counting successes, and why a run with real errors is red anyway | | D — A breached SLA | **FAILED** (on purpose) | What a violation looks like in the report | -`performanceAssert` is the control for every one of these. There is a second threshold control, -`assertMetric`, which this sample deliberately does **not** use for gating — see the note at the -end of this page for what it is actually for. +`performanceAssert` is the control for every one of these. A second control, `assertMetric`, is +**not** a load-test gate — see the note at the end for what it is for. ## The aggregators | Aggregator | What it is good for | |------------|---------------------| -| `AVG` | the headline number — and the one that hides the tail. An average of 800 ms is compatible with one user in twenty waiting eight seconds. | -| `MAX` | the worst single transaction. Brutal, and useful precisely because it is: a MAX threshold catches the timeout nobody saw. | -| `MIN` | a floor. Mostly used to prove the measurement is real rather than empty. | -| `COUNT` | how many transactions happened. With `iterations` fixed this is a **completeness** check, not a throughput one — see below. | -| `SUM` | total time spent. Rarely a threshold; useful for capacity sums. | +| `AVG` | the headline number — and the one that hides the tail | +| `MAX` | the worst single transaction; catches the timeout nobody saw | +| `MIN` | a floor — mostly used to prove the measurement is real, not empty | +| `COUNT` | how many transactions happened (completeness or throughput — see below) | +| `SUM` | total time spent; rarely a threshold | -**A good gate uses several together.** Response time on its own can always be met by doing less -work per unit time — which is exactly what a struggling system does. +A good gate uses several together; response time alone is always met by doing less work per unit +time. Set `continueOnError: true` on the `after` block or it stops at the first breach. -## What `COUNT` tells you, and how to gate throughput +## Notes worth knowing -Throughput is transactions ÷ elapsed time, and a thread group fixes exactly one of the two. That -decides what a `COUNT` threshold on it actually means: +Selective notes, not a full reference — the commented descriptor covers every case. These are the +points most worth getting right. + +### What `COUNT` means, and gating throughput + +A thread group pins exactly one of the two halves of throughput (transactions ÷ elapsed time), and +that decides what a `COUNT` threshold means: | Configuration | Pinned | A `COUNT` threshold is | |---------------|--------|------------------------| -| fixed `iterations` | the count | a **completeness** check — the same number however slow the run was | +| fixed `iterations` | the count | a **completeness** check — same number however slow the run was | | `iterations: 0` + `maxDuration` | the duration | a genuine **throughput** threshold | -Plan A fixes `iterations`, so its `COUNT` of 6 confirms the work was done but says nothing about -the rate. - -It is still worth asserting, because a keyword's count is **how often that step was reached**, not -`users` × `iterations`. It falls short when an iteration fails part way — the steps after the -failure never run — when the step sits inside an `if` or `switch` that was not always taken, or -when a data pool ran dry. On a single-keyword iteration like plan A's the two numbers coincide; on -the multi-step transactions in [01](../01-first-load-test/) and [05](../05-measurements/) they do -not, and comparing counts across the steps of one transaction is how you find where iterations were -breaking off. - -**To gate throughput, bound the duration and count**, which is [02](../02-thread-group-configuration/) -plan D's shape: - -```yaml -threadGroup: - users: 1 - iterations: 0 # unlimited: loop until maxDuration - maxDuration: 5000 # the real stop condition - pacing: 1000 - after: - steps: - - performanceAssert: {measurementName: "Checkout", aggregator: COUNT, - comparator: HIGHER_THAN, expectedValue: 2} -``` - -`iterations: 0` is the loop-for-a-duration form; a fixed count would pin the count instead of the -duration, and omitting `iterations` runs a single time. This is also how load requirements are -usually written — *an hour at this rate* — so it is rarely a -compromise. - -> A throughput aggregator that works whichever of the two is pinned is **not available yet**; it is -> planned for `performanceAssert` in a future release. Until then, express a throughput requirement -> as a duration-bounded run with a `COUNT` threshold, rather than computing a rate inside the plan. +A completeness check is still worth asserting: a keyword's count is *how often that step was +reached*, which falls short of `users` × `iterations` when an iteration fails part way, sits in an +untaken branch, or hits a dry data pool. To gate the **rate**, use the `iterations: 0` + `maxDuration` +shape from [02](../02-thread-group-configuration/) plan D. -Set `continueOnError: true` on the `after` block. Without it the block stops at the first breach, -and a run that violates three thresholds tells you about one of them. Plan E shows the payoff: two -breaches reported in one run, plus a third threshold that passed. +> A throughput aggregator that works whichever half is pinned is **not available yet**; it is planned +> for `performanceAssert`. Until then, pin the duration and count — don't compute a rate in the plan. -## Where the threshold lives +### Where the threshold lives | Placement | Evaluated | |-----------|-----------| -| `after` on a **thread group** | when that population finishes | +| `after` on a **thread group** | when that population finishes — usually the best home | | `after` on a **testScenario** | once, when every thread group has finished | | `afterThread` | once per virtual user | -Different populations have different SLAs — search may have to answer in under a second while a -checkout is allowed three. Putting both in the scenario's `after` block works, but the threshold -then sits far from the load it describes. Each thread group's own `after` block is usually the -better home. +Different populations have different SLAs, so a per-population threshold reads best next to the load +it describes. -## Failure rate as a threshold +### Failure rate as a threshold -"Under 1% errors" is part of every real SLA, and it is not a response time. There is no -failure-rate aggregator, so the pattern is to make the failures countable and then count them: +There is no failure-rate aggregator, so make the failures countable and count them: | Number | Where it comes from | |--------|---------------------| -| attempts | `COUNT` of the keyword measurement — every call, successful or not | +| attempts | `COUNT` of the keyword measurement — every call | | successes | `COUNT` of a measurement the keyword emits **only** on success | -```groovy -// in the keyword, on the success path only -output.startMeasure("Payment accepted") -output.stopMeasure() -``` - -Six attempts, two of which fail, leaves four successes — so a threshold on the success count is a -threshold on the failure rate. - -`continueParentNodeExecutionOnError: true` on the keyword call is what keeps the thread group -running past a failed iteration. Without it the first failure ends that virtual user, and the run -measures a fraction of the load you asked for — the opposite of what you want when the question is -*how often does it fail?* - -### Why plan C is red even though both thresholds pass +A threshold on the success count is a threshold on the failure rate. +`continueParentNodeExecutionOnError: true` on the call keeps the thread group running past a failed +iteration — without it the first failure ends that virtual user and you measure a fraction of the +load. -Two payments genuinely failed, and a failed node fails the run. Step has no error-budget flag that -makes a run with real errors come back green. The thresholds tell you the failure *rate* was -acceptable; the run status tells you failures happened. Both are true and both are worth -reporting. +**Why plan C is red though both thresholds pass:** real payments failed, and a failed node fails the +run. Step has no error-budget flag that greens a run with real errors — the thresholds say the +*rate* was acceptable, the status says failures happened. If a rejection is *expected* traffic, don't +raise a business error for it: return it as an output and branch on it, so the status reflects your +error budget. -If a rejection is **expected traffic** rather than an error — a payment the business means to -decline — do not raise a business error for it. Return it as an ordinary output and branch on it in -the plan, and the run status then reflects your error budget rather than the provider's mood. - -## What a breach looks like - -``` -Average of Slow Search expected to be lower than 100 but was 1672 -Max of Slow Search expected to be lower than 200 but was 2894 -``` - -The control reports the **actual value** alongside the threshold, so a breached SLA tells you how -far off it was without opening the dashboards. - -An empty series reads differently: - -``` -No measurement is matching the defined filters. -``` +### Reading the report -That is what you get from a misspelled measurement name, from an instrumented-node name, and from -a success measurement that was never emitted — three quite different problems with one message. +- A breach names the actual value: `Average of Slow Search expected to be lower than 100 but was 1672`. +- An empty series reads `No measurement is matching the defined filters.` — which covers a misspelled + name, an instrumented-node name, and a success measurement never emitted alike. ## A note on `assertMetric` -Step has a second threshold control, `assertMetric`, that looks like an alternative to -`performanceAssert`. It is not one for load testing, and this sample deliberately leaves it out. +`assertMetric` looks like an alternative to `performanceAssert`. It is not one for load testing, and +this sample deliberately leaves it out. | | `performanceAssert` | `assertMetric` | |---|---|---| @@ -172,16 +114,15 @@ Step has a second threshold control, `assertMetric`, that looks like an alternat | Answers | *did this run meet its SLA?* | *is the system slower than it used to be?* | | Use it for | gating a load test | cross-run trend and regression detection | -Because `assertMetric` is **not scoped to the current execution**, its aggregates fold in every -past run that used the same measurement name — so on a single load test the numbers are -meaningless, and it cannot serve as the gate. Its real home is a **cross-execution** assertion: -comparing today's run against last week's, watching an error rate creep up over the last ten runs, -typically inside an assertion plan attached to a **scheduled** execution. That is monitoring -territory, covered by the monitoring samples rather than this set. +Because `assertMetric` is **not scoped to the current execution**, its aggregates fold in every past +run with the same measurement name — so on a single load test the numbers are meaningless and it +cannot serve as the gate. Its real home is **cross-execution** assertion — today's run against last +week's, an error rate creeping up over ten runs — typically in an assertion plan on a **scheduled** +execution. That is monitoring territory, covered by the monitoring samples. So: **gate a load test with `performanceAssert`; reach for `assertMetric` only for cross-run -assertions.** (A future Step release may let `assertMetric` default to the current execution's -scope, which would make it usable here too — until then the split above holds.) +assertions.** (A future Step release may let `assertMetric` default to the current execution's scope, +which would make it usable here too — until then the split holds.) ## Key files diff --git a/plans/load-testing/06-thresholds-and-slas/automation-package.yaml b/plans/load-testing/06-thresholds-and-slas/automation-package.yaml index 009fb1d..a124c83 100644 --- a/plans/load-testing/06-thresholds-and-slas/automation-package.yaml +++ b/plans/load-testing/06-thresholds-and-slas/automation-package.yaml @@ -82,12 +82,13 @@ plans: # this is 6 however slow the run was. # # It is worth asserting anyway, because a keyword's count is how - # often that STEP was reached - not users x iterations. It falls - # short when iterations fail part way (the steps after the - # failure never run), when the step sits inside a branch that was - # not always taken, or when a data pool ran dry. On a one-keyword - # iteration like this one the two coincide; on the multi-step - # transactions of samples 01 and 05 they do not. + # often that STEP was reached - which equals users x iterations + # only when every iteration runs it. Under real load it can fall + # short: an iteration that fails part way skips the steps after + # it, a step inside a branch runs only when the branch is taken, + # a dry data pool leaves iterations with nothing to do. None of + # those occur in these stub runs, but the assertion is what + # catches them when they do. # # To gate the RATE instead, bound the duration - see the note # above and sample 02 plan D. diff --git a/plans/load-testing/README.md b/plans/load-testing/README.md index c6fcfce..08fc51f 100644 --- a/plans/load-testing/README.md +++ b/plans/load-testing/README.md @@ -61,10 +61,9 @@ sample's README lists the expected outcome per plan: Every other plan is expected to pass. -## Two placement rules for `performanceAssert` +## Two rules for `performanceAssert` -Neither follows from the structure of the YAML, and both report an error that names something -other than the cause: +Two rules to keep in mind: **`performanceAssert` must live in an `after` or `afterThread` block.** Anywhere else — as a child of the thread group, of a `testCase`, of anything — the execution ends in `TECHNICAL_ERROR` with From b89c6990354c403d24e6be90adf4dfe81bed3251 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20Comte?= Date: Mon, 24 Aug 2026 21:12:34 +0200 Subject: [PATCH 9/9] SED-4885 PR Feedbacks RPA Samples --- README.md | 9 +- plans/README.md | 17 +-- plans/legacy-exports/README.md | 13 ++- plans/reference/basic-plan-syntax.yml | 17 ++- plans/reference/dynamic-values.yml | 24 ++-- plans/rpa/01-linear-bot/README.md | 10 +- plans/rpa/02-parameterized-bot/README.md | 13 ++- .../automation-package.yaml | 7 +- .../rpa/04-branching-and-variables/README.md | 22 ++-- plans/rpa/05-resilience-and-waiting/README.md | 79 +++++-------- .../automation-package.yaml | 55 ++------- plans/rpa/06-session-and-scheduling/README.md | 14 +-- .../automation-package.yaml | 17 ++- plans/rpa/07-composition-and-reuse/README.md | 108 +++++++++++++++--- .../automation-package.yaml | 66 +++++++---- plans/rpa/README.md | 11 +- 16 files changed, 271 insertions(+), 211 deletions(-) diff --git a/README.md b/README.md index 4622ed2..f9c7e4f 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,10 @@ Browse the **[automation-packages/README.md](automation-packages/README.md)** fo ## Plan samples -The **[plans](plans/)** directory zooms in on one part of an Automation Package: the -**plan** — the control-flow tree of loops, branches, retries and keyword calls that turns -keywords into an automation that does something useful. +The **[plans](plans/)** directory is about **plans** — the implementation of an automation +scenario, whether that is a functional test case, a load test, an RPA routine or a synthetic +monitoring probe. A plan combines keywords with the controls that build up its execution +logic: loops, branches, retries, waits. The two directories answer different questions, and most people need both: @@ -38,7 +39,7 @@ The two directories answer different questions, and most people need both: the starting point for your own project. - **[plans](plans/)** — *"how do I express this logic in a plan?"* One plan concept per sample, with keyword code deliberately reduced to 3-line stubs so the plan itself is the - subject. Look things up here when you are writing the plan inside your package. + subject. Look things up here when you are writing a plan. | Section | Contents | |---------|----------| diff --git a/plans/README.md b/plans/README.md index 4e3d8d5..09b8d7c 100644 --- a/plans/README.md +++ b/plans/README.md @@ -2,13 +2,14 @@ This directory is about one thing: **how to write a Step plan**. -A plan is the control-flow tree inside an Automation Package — the loops, branches, retries -and keyword calls that turn a handful of keywords into an automation that does something -useful. +A **plan** is the implementation of an automation scenario — a functional test case, a load +test, an RPA routine, a synthetic monitoring probe. It combines **keywords**, the building +blocks that do the work, with **controls** that build up the execution logic: loops, +branches, retries, waits. -Every sample here is a self-contained, runnable Automation Package, so each plan can be -validated and executed rather than just read. Keyword code is deliberately reduced to -3-line stubs so the plan itself is the subject. +Every sample here ships as a runnable automation package, so each plan can be validated and +executed rather than just read. Keyword code is deliberately reduced to 3-line stubs so the +plan itself is the subject. ### How this differs from `automation-packages/` @@ -18,7 +19,7 @@ synthetic monitoring with Cypress, RPA with Selenium), including its build, its its plans. That is where you go to start a project. This directory is the **plan-authoring reference**: one plan concept per sample, stripped of -everything else. That is where you go while writing the plan inside your package. +everything else. That is where you go while writing a plan. ## Samples by use case @@ -50,7 +51,7 @@ Step has three plan formats: |--------|-----------|-----------------------| | **YAML** | The tree of controls documented at [step.dev](https://step.dev/knowledgebase/userdocs/plans/controls/) | Yes — the whole `rpa/` and `load-testing/` sets | | **Plain text** | A compact line-based syntax, one keyword call per line | No | -| **UI** | Built in the Step plan editor, stored serialized as JSON | No — see [legacy-exports/](legacy-exports/) for what an export looks like | +| **UI** | Built in the Step plan editor; [imported and exported](https://step.dev/knowledgebase/userdocs/import-export-entities/) as JSON | No — see [legacy-exports/](legacy-exports/) for what an export looks like | **Automation packages support YAML and plain text.** Editing an automation package's plans in the UI is planned but not currently supported. diff --git a/plans/legacy-exports/README.md b/plans/legacy-exports/README.md index 80ec7ea..e5ca282 100644 --- a/plans/legacy-exports/README.md +++ b/plans/legacy-exports/README.md @@ -7,12 +7,15 @@ Two plans exported from the Step plan editor as JSON. | `Demo_Google-search.json` | A sequence calling an Echo keyword and asserting on its output | | `Demo_Data-driven.json` | A data-driven plan iterating over a data source | -Plans built in the Step plan editor are stored serialized as JSON, and these two files are -what that serialization looks like: `step.core.plans.Plan` with its `_class` discriminators, -`DynamicValue` wrappers and null fields. They are kept here as a sample of the shape, since -it is also what the plan REST API returns. +JSON is the format Step uses to **import and export** plans. It is not meant for authoring — +these two files are kept only as a sample of the shape. -**This format is not meant to be hand-written.** For authoring a plan, use YAML: +For how to produce and consume such files — exporting single or bulk entities, exporting a +plan recursively with the entities it references, and the import options — see +[Import/Export entities](https://step.dev/knowledgebase/userdocs/import-export-entities/) in +the Step documentation. + +To write a plan, use YAML: - [../reference/](../reference/) — the YAML syntax reference - [../rpa/](../rpa/) — worked, runnable samples diff --git a/plans/reference/basic-plan-syntax.yml b/plans/reference/basic-plan-syntax.yml index 47c3316..709227d 100644 --- a/plans/reference/basic-plan-syntax.yml +++ b/plans/reference/basic-plan-syntax.yml @@ -1,12 +1,17 @@ -# A standalone YAML plan: the same tree you would write inside an automation -# package, with a top-level `root:` and no package around it. +# ============================================================================== +# STANDALONE PLAN # -# A file like this can be dropped straight into the Step UI with -# "Add plan" > "Create from YAML", or referenced from an automation package -# like any other plan. +# This file represents a single, self-contained Step plan. +# +# Usage: +# - Upload directly: import into the Step UI via "Add plan" > "Create from YAML". +# - Convert/add to a package: move the contents under a `plans:` list entry +# inside an automation package manifest. # # This file is a syntax illustration, not a runnable plan - the keywords it -# calls do not exist. For plans you can execute, see ../rpa/. +# calls do not exist. For plans you can execute, see ../rpa/ and +# ../load-testing/ +# ============================================================================== --- version: 1.0.0 name: "Basic plan syntax" diff --git a/plans/reference/dynamic-values.yml b/plans/reference/dynamic-values.yml index ad1eed3..a705361 100644 --- a/plans/reference/dynamic-values.yml +++ b/plans/reference/dynamic-values.yml @@ -1,12 +1,17 @@ -# Static values vs dynamic expressions - the distinction that governs every -# field in a Step YAML plan. +# ============================================================================== +# STANDALONE PLAN - STATIC VALUES vs DYNAMIC EXPRESSIONS # -# A file like this can be dropped straight into the Step UI with -# "Add plan" > "Create from YAML", or referenced from an automation package -# like any other plan. +# The distinction that governs every field in a Step YAML plan. +# +# Usage: +# - Upload directly: import into the Step UI via "Add plan" > "Create from YAML". +# - Convert/add to a package: move the contents under a `plans:` list entry +# inside an automation package manifest. # # This file is a syntax illustration, not a runnable plan - the keywords it -# calls do not exist. For plans you can execute, see ../rpa/. +# calls do not exist. For plans you can execute, see ../rpa/ and +# ../load-testing/ +# ============================================================================== --- version: 1.0.0 name: "Dynamic values and inputs" @@ -49,8 +54,11 @@ root: - callKeyword: keyword: "Submit Record" inputs: - - literalInput: "abc" # static - - numericInput: 777 # static number + # Quoting decides the type of a static value: YAML parses an + # unquoted number as a number, and the keyword receives it as one. + - literalString: "abc" # static string + - quotedNumber: "777" # static STRING "777" + - unquotedNumber: 777 # static NUMBER 777 - fromVariable: expression: "recordId" # a plan variable - computed: diff --git a/plans/rpa/01-linear-bot/README.md b/plans/rpa/01-linear-bot/README.md index c80313e..11f0aea 100644 --- a/plans/rpa/01-linear-bot/README.md +++ b/plans/rpa/01-linear-bot/README.md @@ -56,11 +56,11 @@ scope**, so it stays readable by every following sibling — not just the next o ### The full picture -| Binding | Where it is in scope | Use it for | -|---------|----------------------|------------| -| `output.` | **Only inside the calling node's own `children`** | Feeding a nested `set` or a nested `assert` | -| `previous.` | The **immediately preceding sibling** only — the next keyword call replaces it | A quick check right after a call | -| `expression: "myVar"` | The block the `set` belongs to, and everything nested below it | Anything that must survive further steps | +| Binding | Holds | Readable from | +|---------|-------|---------------| +| `output.` | The output of the keyword call it sits under | **Only** the `children` of that calling keyword node | +| `previous.` | The output of the keyword call just before it | The node **immediately following** that call — the next keyword call overwrites it | +| a variable created by `set` | Whatever you put in it | The block the `set` belongs to, and everything nested below. Read it with `expression: "myVar"` | `previous` is shown once in the plan (the `check` after *Submit the record*) and is deliberately flagged there as the fragile option. diff --git a/plans/rpa/02-parameterized-bot/README.md b/plans/rpa/02-parameterized-bot/README.md index 92a3738..0ca95aa 100644 --- a/plans/rpa/02-parameterized-bot/README.md +++ b/plans/rpa/02-parameterized-bot/README.md @@ -35,8 +35,17 @@ All three end up as ordinary plan variables, read with `expression: "name"`. Not ## Defaulting an execution parameter -Step declares execution parameters as plan variables **only if the caller supplied them**. -Referencing a missing one fails, so a plan that must also run unattended should default it: +Step declares execution parameters as plan variables **only if the caller supplied them**, so +referencing one the caller omitted fails the plan. + +That failure is often the behaviour you want: a bot that submits the wrong record because an +input silently fell back to a default is worse than one that refuses to start. + +Default a parameter only where the value is genuinely **optional** — a dry-run flag that is +off unless asked for, a batch size, a target environment that is nearly always the same. +Anything that decides *what the bot acts on* is better left to fail. + +This sample defaults its parameters so it runs with no arguments; the idiom is: ```yaml - set: diff --git a/plans/rpa/02-parameterized-bot/automation-package.yaml b/plans/rpa/02-parameterized-bot/automation-package.yaml index 08e4e04..f5d8b2e 100644 --- a/plans/rpa/02-parameterized-bot/automation-package.yaml +++ b/plans/rpa/02-parameterized-bot/automation-package.yaml @@ -34,8 +34,11 @@ plans: # # This is the defaulting idiom: ask the Groovy binding whether the # variable exists before reading it. It makes the plan runnable with - # no parameters at all, which is what you want for a bot that also - # runs on a schedule. + # no parameters at all, which is what this sample wants. + # + # Use it sparingly. Defaulting suits a value that is genuinely + # optional - a dry-run flag, a batch size. For a value that decides + # WHAT the bot acts on, failing is usually better than guessing. # ------------------------------------------------------------- - set: key: recordId diff --git a/plans/rpa/04-branching-and-variables/README.md b/plans/rpa/04-branching-and-variables/README.md index 0b43596..1415432 100644 --- a/plans/rpa/04-branching-and-variables/README.md +++ b/plans/rpa/04-branching-and-variables/README.md @@ -31,11 +31,11 @@ starts encoding business rules. | B | `if` threshold routing, `set` scoping, `assert` with `doNegate` | | C | `skipNode` on a temporarily disabled step | -## Two ways `switch` silently does nothing +## Notes on `switch` -Both leave the plan reporting **PASSED** while no branch ran at all. +Two behaviours worth knowing before writing one: -### 1. The expression must be dynamic +### The expression must be dynamic ```yaml - switch: @@ -45,13 +45,13 @@ Both leave the plan reporting **PASSED** while no branch ran at all. expression: "recordType" # RIGHT - reads the variable ``` -The static form matches no case, so the switch executes nothing. Same rule as everywhere -else in this YAML: a plain string is a literal, never a variable reference. +The static form matches no case, so the switch executes nothing — and the plan still reports +PASSED. Same rule as everywhere else in this YAML: a plain string is a literal, never a +variable reference. -### 2. There is no `default` case +### There is no `default` case -`value: "default"` is just a case matching the literal string `"default"`. When the -expression matches nothing, the switch runs **nothing** and passes. +When the expression matches nothing, the switch runs **nothing** and passes. The fallback pattern — what plan A does — is to set a sentinel before the switch, have each case overwrite it, and test it afterwards: @@ -64,7 +64,7 @@ case overwrite it, and test it afterwards: - case: {value: "INVOICE", children: [ ... , {set: {key: routed, value: "INVOICE"}}]} # ... - if: - condition: {expression: "routed == NONE"} + condition: {expression: "routed == 'NONE'"} children: - set: {key: routed, value: "MANUAL"} ``` @@ -72,8 +72,8 @@ case overwrite it, and test it afterwards: Note `routed` is declared **before** the switch — a `set` living only inside a case is scoped to that case (see `set` scoping below). -Because both failure modes are silent, plan A ends each iteration with a `check` that the -record took the branch its type demands. Break the switch and that check goes red. +Because both behaviours are silent, plan A ends each iteration with a `check` that the record +took the branch its type demands. Break the switch and that check goes red. ## How `set` scoping works diff --git a/plans/rpa/05-resilience-and-waiting/README.md b/plans/rpa/05-resilience-and-waiting/README.md index 55eb9f2..1f8f796 100644 --- a/plans/rpa/05-resilience-and-waiting/README.md +++ b/plans/rpa/05-resilience-and-waiting/README.md @@ -30,7 +30,7 @@ giving up too early or hanging forever. | Plan | Expected outcome | Shows | |------|------------------|-------| | A — Retry a flaky step | **PASSED** | `retryIfFails` absorbs two failures, passes on attempt 3 | -| B — Guaranteed cleanup | **FAILED** (on purpose) | `after` cleanup runs; the step after the failure does not | +| B — Guaranteed cleanup | **FAILED** (on purpose) | `after` cleanup runs; subsequent steps in the sequence do not | | C — Error propagation flags | **FAILED** (on purpose) | Both flags, and how they combine | | D1 — Wait with while | **PASSED** | `condition`, `postCondition`, `pacing`, `maxIterations`, `timeout` | | D2 — Wait with retryIfFails | **PASSED** | `retryIfFails` + a nested `assert` as the not-ready signal | @@ -38,43 +38,6 @@ giving up too early or hanging forever. ## Notes on the controls -### Calling a keyword inside `while` on auto-provisioned agents - -Step forecasts how many agents an execution needs before it starts, and that forecast does -**not** account for keyword calls nested in a `while`. On an auto-provisioned setup the loop -therefore fails with - -``` -Not able to find any agent token matching selection criteria $agenttype=default -and accepting attributes {$tokenPartition=} -``` - -**Only the forecast is affected — the construct itself is fine.** With permanent agents, or -with provisioning declared on the plan, a `while` calling a keyword works normally. So if -you want the `while` form on an auto-provisioned instance, declare the agents: - -```yaml -- name: "Poll until the queue is empty" - agents: - - replicas: 1 - pool: "" -``` - -This is a provisioning limitation, not a reason to avoid `while`. Plan D2 uses -`retryIfFails` simply because it needs no pool name and therefore runs on any instance: - -```yaml -- retryIfFails: - maxRetries: 10 - gracePeriod: 500 - timeout: 60000 - children: - - callKeyword: - keyword: "Check Queue Size" - children: - - assert: {actual: "queueSize", operator: EQUALS, expected: "0"} -``` - ### `while` or `retryIfFails` for waiting? Both call keywords perfectly well, and both wait. What separates them is **how you express @@ -83,31 +46,40 @@ Both call keywords perfectly well, and both wait. What separates them is **how y | | `while` | `retryIfFails` | |---|---|---| | Exit criterion | a **condition** stops holding | the block stops **failing** | -| "Not ready" is… | a normal state | a failure you have to fabricate (usually a failing `assert`) | +| "Not ready" is… | a normal state | an error — either a real one, or a failing `assert` you add | | Execution report | clean — nothing failed | one failed attempt per wait (soften with `reportLastTryOnly`) | **Prefer `while`** when the system gives you an answer you can test — *how many items are left?*, *is the status DONE?*. The condition reads as a condition, and a long wait does not fill the report with failures. -**Prefer `retryIfFails`** when the thing you are waiting on genuinely **fails** rather than -reporting a state — a call that errors until the service is up. That is retry, not polling, -and it is also what you want for absorbing flakiness (plan A). +**Prefer `retryIfFails`** when the call genuinely **errors** until the system is ready — a +request refused while a service starts up. Then the failure is real, the retry is doing what +it was designed for, and this is also the control for absorbing flakiness (plan A). + +Using `retryIfFails` purely to wait, with an `assert` added only to force a retry, works but +is a workaround: it records every wait as a failed attempt. Plan D2 shows the shape, so that +both forms appear side by side. + +> On older Step versions, keyword calls nested in a `while` are not counted when forecasting +> how many agents to provision, so such a loop can fail to obtain an agent token on an +> auto-provisioned instance. Declaring `agents` on the plan avoids it. ### Retry on a fresh session, not the same one Neither plan A nor plan D2 wraps its retry in a `session`, and that is deliberate. -A retry is usually better on a **fresh** session — a new agent token, a new browser, a clean -slate. Half the reason a step is flaky is state left behind by the attempt that just failed; -retrying inside the same dirty session retries the problem along with the step. +A failed attempt often leaves state behind — a half-filled form, a stale selection. Retrying +inside that same session retries the mess along with the step, so a retry on a **fresh** +session is usually more likely to succeed. -Wrap a retry in a session only when the retried block genuinely depends on something -established **earlier and outside it** — typically a login you do not want to redo on every -attempt. That is a deliberate trade-off, not the default. +The exception is when the retried block depends on something set up **before** it, such as a +login you do not want to repeat on every attempt. Then wrap the retry in a session +deliberately. -This has a consequence for the keywords: **keep them stateless.** Both samples count -attempts in the *plan* and pass the number in as an input: +For this to be possible, **keywords must be stateless** — a keyword that remembers something +between calls forces every caller to pin to one agent. So both samples count attempts in the +*plan* and pass the number in as an input: ```yaml - set: {key: attempt, value: {expression: "0"}} # a NUMBER, not "0" @@ -134,9 +106,10 @@ caller to pin to one agent — which is how you end up needing a session you did ### `continueOnError` on an `after` block -It governs whether one failing cleanup step stops the **remaining** cleanup steps — the same -"keep going" semantics as on a sequence, applied to the cleanup list. Without it, a failing -logout means the licence never gets released. +On a `sequence`, `continueOnError` means "if one of my children fails, keep running the +others" — that is what plan C uses it for. An `after` block takes the same attribute with the +same meaning, applied to the cleanup steps: it governs whether one failing cleanup step stops +the **remaining** ones. Without it, a failing logout means the licence never gets released. It does **not** suppress or hide anything: both the body error and the cleanup error are reported either way. And a failing cleanup step fails the run even when the body passed, so diff --git a/plans/rpa/05-resilience-and-waiting/automation-package.yaml b/plans/rpa/05-resilience-and-waiting/automation-package.yaml index b0c1261..0d80b93 100644 --- a/plans/rpa/05-resilience-and-waiting/automation-package.yaml +++ b/plans/rpa/05-resilience-and-waiting/automation-package.yaml @@ -225,50 +225,12 @@ plans: # waiting on genuinely FAILS rather than reporting a state - a call that # errors until the service is up - since that is retry, not polling. # - # Both call keywords perfectly well. The one caveat, described next, - # affects `while` on auto-provisioned agents. + # Both call keywords perfectly well. # - # This sample ships D1 (`while`, no keyword) and D2 (`retryIfFails`) - # because both run on any instance. The `while`-plus-keyword form needs an - # agent pool name, so it is documented below rather than executed. - # - # CALLING A KEYWORD INSIDE A `while` ON AUTO-PROVISIONED AGENTS. - # - # Step forecasts how many agents an execution needs before it starts, and - # that forecast does not account for keyword calls nested in a `while`. On - # an auto-provisioned setup the loop therefore fails with - # "Not able to find any agent token matching selection criteria - # $agenttype=default and accepting attributes {$tokenPartition=}" - # - # It is ONLY the provisioning forecast that is wrong. The construct itself - # is fine: with PERMANENT agents, or with provisioning declared explicitly - # on the plan, a `while` calling a keyword works normally. - # - # So if you want the `while` form and you are on auto-provisioning, declare - # the agents on the plan (substituting your own pool name): - # - # - name: "Poll until the queue is empty" - # agents: - # - replicas: 1 - # pool: "" - # root: - # testCase: - # children: - # - while: - # condition: - # expression: "queueSize > 0" - # maxIterations: 10 - # timeout: 60000 - # children: - # - callKeyword: - # keyword: "Check Queue Size" - # children: - # - set: - # key: queueSize - # value: - # expression: "output.queueSize as Integer" - # - # Again: this is a provisioning limitation, not a reason to avoid `while`. + # NOTE: on older Step versions, keyword calls nested in a `while` are not + # counted when forecasting how many agents to provision, so such a loop can + # fail to obtain an agent token on an auto-provisioned instance. Declaring + # `agents` on the plan avoids it if you encounter the issue. # ========================================================================= - name: "D1 - Wait with while" categories: ["RPA"] @@ -277,10 +239,9 @@ plans: nodeName: "Count a backlog down to zero" children: - # The loop counts down a plan variable so this plan runs on any - # instance. In a real bot the body would call a keyword that asks - # the system, and `set` the answer - see the block above the plan - # for that form and its provisioning caveat. + # The loop counts down a plan variable to keep the sample + # self-contained. In a real bot the body would call a keyword that + # asks the system, and `set` the answer. # A number, not the string "3" - see plan A. - set: diff --git a/plans/rpa/06-session-and-scheduling/README.md b/plans/rpa/06-session-and-scheduling/README.md index 064f511..7646bfb 100644 --- a/plans/rpa/06-session-and-scheduling/README.md +++ b/plans/rpa/06-session-and-scheduling/README.md @@ -30,15 +30,15 @@ automation: the second keyword would not find the browser the first one opened. `session` wraps a block so every keyword inside uses the same token. Anything a keyword puts into its `session` map is then visible to the next keyword. -## Do not use the session's own `before` block for stateful setup +## Initialization and cleanup inside the session -A keyword in a **session's** `before` block runs on a **different -agent token** than the session body, so the body cannot see what `before` put into the -session. +A session's own `before` and `after` blocks run **outside** the session, so a keyword placed +there cannot share application state with the session body. -Put a `sequence` inside the session and use **its** `before` / `after` instead. Those steps -run within the session's token, and `after` still runs when the body fails — so cleanup is -still guaranteed: +If you need initialization or cleanup steps that run **within** the session — opening a +browser, logging in, closing it again — put a `sequence` inside the session and use **its** +`before` / `after`. Those steps run within the session, and `after` still runs when the body +fails, so cleanup is guaranteed: ```yaml - session: diff --git a/plans/rpa/06-session-and-scheduling/automation-package.yaml b/plans/rpa/06-session-and-scheduling/automation-package.yaml index 8d9c275..ecc96e6 100644 --- a/plans/rpa/06-session-and-scheduling/automation-package.yaml +++ b/plans/rpa/06-session-and-scheduling/automation-package.yaml @@ -25,17 +25,16 @@ plans: # Anything a keyword puts into its `session` map is then visible to the # next keyword. # - # WHERE TO PUT open/close - this bit is easy to get wrong. + # WHERE TO PUT INITIALIZATION AND CLEANUP. # - # Do NOT use the session's own `before` / `after` blocks to open and close a - # stateful resource. A keyword in the session's - # `before` block runs on a DIFFERENT agent token than the session body, so - # the body cannot see what `before` put into the session and fails with - # "No browser in this session". + # A session's own `before` / `after` blocks run OUTSIDE the session, so a + # keyword placed there cannot share application state with the session body. # - # Put a `sequence` INSIDE the session instead and use the sequence's - # before / after. Those steps run within the session's token, and `after` - # still runs when the body fails - so cleanup is still guaranteed. + # If you need initialization or cleanup steps that run WITHIN the session - + # opening a browser, logging in, closing it again - put a `sequence` inside + # the session and use the sequence's before / after. Those steps run within + # the session, and `after` still runs when the body fails, so cleanup is + # guaranteed. # ========================================================================= - name: "A - Stateful bot in one session" categories: ["RPA"] diff --git a/plans/rpa/07-composition-and-reuse/README.md b/plans/rpa/07-composition-and-reuse/README.md index c006968..4284f45 100644 --- a/plans/rpa/07-composition-and-reuse/README.md +++ b/plans/rpa/07-composition-and-reuse/README.md @@ -11,23 +11,44 @@ level: advanced # 07 — Composition and reuse Once you have more than one bot, the same sub-process shows up everywhere: *log in*, *look -up a customer*, *book an entry*. This sample shows the three ways Step lets you package a -piece of plan once and call it from many bots. +up a customer*, *book an entry*. This sample shows the two ways Step lets you package a piece +of plan once and call it from many bots — plus the controls for running several bots +together and for keeping them out of each other's way. ## What this sample shows - A **Composite keyword** — a plan that behaves like a keyword, with inputs and outputs - `callPlan` to delegate to another whole plan -- `testSet` to run several bots as one orchestrated batch +- `testSet` to run several test cases from one plan, several at a time - `synchronized` to serialise access to a single shared licence -## The three reuse mechanisms +## The two reuse mechanisms -| Mechanism | Returns outputs? | Use it when | -|-----------|------------------|-------------| -| **Composite keyword** | Yes, via `return` | The thing you are reusing is a *step* inside someone else's plan. **Reach for this first** | -| `callPlan` | No — it produces its own report branch | The thing you are reusing is a whole bot in its own right, with its own schedule | -| `testSet` | n/a | You want to run several independent bots as one batch | +| Mechanism | Use it when | +|-----------|-------------| +| **Composite keyword** | The reused part has to **return outputs**; or it needs a **declared input schema**; or you want it to appear in reports **like any other keyword call** | +| `callPlan` | The reused part is a complete plan that also runs on its own. It returns nothing to the caller and produces its own branch in the report | + +`testSet` also appears in this sample, but it is not a reuse mechanism — see +[below](#testset-running-several-test-cases). + +Values passed through `callPlan.input` reach the called plan under **`input.`** — the same +convention a Composite keyword uses. A bare `archiveFolder` is not bound: + +```yaml +- callPlan: + selectionAttributes: + - name: "Shared - Archive processed records" + input: + - archiveFolder: "/archive/2026-08" +``` + +```yaml +# in the called plan +- echo: + text: + expression: "'Archiving into ' + input.archiveFolder" +``` ## The Composite keyword @@ -54,7 +75,31 @@ keywords: ``` The caller invokes it exactly like any other keyword, and reads its `return` values with -`output.*`. +`output.*`. That sameness is the point — it is why a Composite is the right choice when the +reused part: + +- **has to return outputs** — `return` hands them back, and the caller reads them with + `output.` exactly as for a Groovy or Java keyword; +- **needs a declared input contract** — `schema` states which inputs it takes and which are + required, so callers do not have to read the plan to find out; +- **should report like a keyword** — one node in the execution tree with its internals nested + beneath it, rather than a separate report branch. + +The input schema is ordinary JSON Schema: + +```yaml +- Composite: + name: "Process One Record" + schema: + type: object + properties: + recordId: + type: string + required: + - recordId + plan: + # ... +``` Two details that are easy to get wrong: @@ -64,17 +109,38 @@ Two details that are easy to get wrong: no string interpolation — `"${confirmationId}"` would hand the caller that literal string. See [01-linear-bot](../01-linear-bot/). -## `forEach threads` vs `testSet threads` +## `testSet`: running several test cases + +`testSet` is not about reuse. It is a control that runs its children — usually `testCase` +nodes — as separate test cases, `threads` of them at a time. Plan C uses it for the "nightly +batch" shape: three independent bots in one plan and one report. + +Note what `threads` means in each place: + +- `threads` on a **loop** parallelises **row processing** within one test case. +- `threads` on a **`testSet`** parallelises its **test cases**. + +## Keeping a sub-plan out of a normal run + +`Shared - Archive processed records` exists to be called by plan B. It expects +`archiveFolder` from its caller, so running it on its own fails — and by default, executing +the package runs every plan in it, including this one. -- `threads` on a **loop** parallelises **rows** of one bot. -- `threads` on a **`testSet`** parallelises **whole bots**. +Mark it with a category and exclude that category at execution: -## A plan that is called is still a plan +```yaml +- name: "Shared - Archive processed records" + categories: ["RPA", "sub-plan"] +``` + +```bash +step ap execute -p . --excludeCategories=sub-plan +``` -`Shared - Archive processed records` exists to be called by plan B, but nothing marks it as -"a sub-plan" — so it is also executed on its own when the whole package runs. That is why it -defaults its input: a plan meant to be called by others should still stand up when launched -directly. +Categories are only labels — they change nothing about how `callPlan` reaches the plan, so +plan B still works. This is cleaner than giving the plan a default value for an input it +should really require: a default that exists only to keep a standalone run green hides a +missing input rather than reporting it. ## Key files @@ -88,5 +154,9 @@ directly. ## Running it ```bash -step ap execute -p . -u --token --projectName +step ap execute -p . -u --token --projectName --excludeCategories=sub-plan ``` + +`--excludeCategories=sub-plan` leaves out `Shared - Archive processed records`, which is +meant to be reached through `callPlan` rather than run on its own. Without the flag that +plan is executed too, and fails for want of the `archiveFolder` its caller supplies. diff --git a/plans/rpa/07-composition-and-reuse/automation-package.yaml b/plans/rpa/07-composition-and-reuse/automation-package.yaml index b7cd6ba..2de9bfc 100644 --- a/plans/rpa/07-composition-and-reuse/automation-package.yaml +++ b/plans/rpa/07-composition-and-reuse/automation-package.yaml @@ -3,14 +3,17 @@ # RPA sample 07 - Composition and reuse # # Once you have more than one bot, the same sub-process shows up everywhere: -# "log in", "look up a customer", "book an entry". This sample shows the three +# "log in", "look up a customer", "book an entry". This sample shows the two # ways Step lets you package a piece of plan once and call it from many bots. # -# Composite keyword a plan that behaves like a keyword - inputs, outputs, -# and it shows up in the keyword list. The cleanest -# reuse mechanism. -# callPlan call another plan by name -# testSet run several bot plans as one orchestrated batch +# Composite keyword a plan that behaves like a keyword: it returns outputs, +# can declare an input schema, and reports as a single +# keyword call +# callPlan call another plan by name; returns nothing to the +# caller and reports as its own branch +# +# It also uses `testSet`, which is not a reuse mechanism but the control for +# running several test cases from one plan. # --------------------------------------------------------------------------- version: "1.2.0" name: "rpa-07-composition-and-reuse" @@ -24,8 +27,9 @@ plans: # keyword: its implementation is a PLAN, not code. From here it is called # exactly like any other keyword - inputs go in, `return` sends outputs back. # - # This is the reuse mechanism to reach for first: the caller does not need to - # know the sub-process has five steps in it. + # Use a Composite when the reused part has to RETURN OUTPUTS, needs a + # declared INPUT SCHEMA, or should report as one keyword call rather than a + # separate branch. # ========================================================================= - name: "A - Use a composite keyword as a sub-bot" categories: ["RPA"] @@ -87,36 +91,40 @@ plans: - archiveFolder: "/archive/2026-08" # ------------------------------------------------------------------------- - # The plan that B calls. An ordinary plan - nothing marks it as "a sub-plan", - # which means it is also executed on its own when the whole package runs. + # The plan that B calls. It expects `archiveFolder` from its caller and has + # no reason to run on its own - so it carries a "sub-plan" category and is + # left out of a normal run: + # + # step ap execute -p . --excludeCategories=sub-plan # - # That is why it defaults its input: a plan meant to be called by others - # should still stand up when launched directly. + # Categories are just labels; they change nothing about how `callPlan` + # reaches this plan. + # + # NOTE how the input is read: values passed through `callPlan.input` arrive + # under `input.`, the same way a Composite keyword receives its inputs. + # A bare `archiveFolder` is NOT bound. # ------------------------------------------------------------------------- - name: "Shared - Archive processed records" - categories: ["RPA"] + categories: ["RPA", "sub-plan"] root: testCase: nodeName: "Archive what the bots processed" children: - - set: - key: archiveFolder - value: - expression: "binding.variables.containsKey('archiveFolder') ? archiveFolder : '/archive/default'" - echo: text: - expression: "'Archiving into ' + archiveFolder" + expression: "'Archiving into ' + input.archiveFolder" description: "Reads the value the caller passed through callPlan input." # ========================================================================= - # C. testSet - orchestrating several bots as one batch. + # C. testSet - running several test cases from one plan. # - # A `testSet` runs its children (usually testCases or callPlans) as separate - # test cases, `threads` of them at a time. This is the "nightly RPA batch" - # shape: five independent bots, two agents, one report. + # Not a reuse mechanism: `testSet` is simply the control that runs its + # children - usually testCases - as separate test cases, `threads` of them + # at a time. This is the "nightly RPA batch" shape: three independent bots, + # two at a time, one report. # - # Note the difference from `forEach threads`: forEach parallelises ROWS of - # one bot, testSet parallelises WHOLE BOTS. + # Note the difference from `forEach threads`: forEach parallelises ROWS + # within one test case, testSet parallelises the TEST CASES. # ========================================================================= - name: "C - Nightly batch of bots" categories: ["RPA"] @@ -202,6 +210,16 @@ keywords: - Composite: name: "Process One Record" description: "Reads a record and submits it. Reused by every bot in this package." + # `schema` declares the inputs this keyword accepts, the same way any + # other keyword does. Callers get them documented and validated instead + # of having to read the plan to find out what it expects. + schema: + type: object + properties: + recordId: + type: string + required: + - recordId plan: root: sequence: diff --git a/plans/rpa/README.md b/plans/rpa/README.md index 20c531d..30a7895 100644 --- a/plans/rpa/README.md +++ b/plans/rpa/README.md @@ -63,11 +63,20 @@ step ap deploy -p . -u --token --projectName