From 314098281fd9d477e8b53f1e75b2d2ac7d843343 Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:42:21 -0700 Subject: [PATCH 1/6] Return dropped search-attribute registration error in ebb_and_flow spawnWorkflowWithActivities ignored the error from RegisterDefaultSearchAttributes, so a registration failure was silently lost and surfaced later as an obscure workflow-start failure. --- scenarios/ebb_and_flow.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scenarios/ebb_and_flow.go b/scenarios/ebb_and_flow.go index a26451e6..c2e7f8b0 100644 --- a/scenarios/ebb_and_flow.go +++ b/scenarios/ebb_and_flow.go @@ -230,7 +230,9 @@ func (e *ebbAndFlowExecutor) Run(ctx context.Context, info loadgen.ScenarioInfo) config.MinBacklog, config.MaxBacklog, config.Period, e.Configuration.Duration) } - e.RegisterDefaultSearchAttributes(ctx) + if err := e.RegisterDefaultSearchAttributes(ctx); err != nil { + return fmt.Errorf("failed to register search attributes: %w", err) + } var started, completed, backlog, target int64 lastIteration := time.Now() From f8150153b08d133a60ecd2d4bac6d4de76909b5f Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:44:06 -0700 Subject: [PATCH 2/6] Surface failed ebbAndFlowTrack workflows instead of swallowing them spawnWorkflowWithActivities logged wf.Get failures and returned nil, so they never reached the MaxConsecutiveErrors guard and a failed workflow was still counted as completed (inflating TotalCompletedWorkflows, which post-scenario verification relies on). Return the error and gate the completion count on success; the backlog still drains either way so the controller's behavior is unchanged. --- scenarios/ebb_and_flow.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scenarios/ebb_and_flow.go b/scenarios/ebb_and_flow.go index c2e7f8b0..f3449f93 100644 --- a/scenarios/ebb_and_flow.go +++ b/scenarios/ebb_and_flow.go @@ -376,10 +376,12 @@ func (e *ebbAndFlowExecutor) spawnWorkflowWithActivities( // Wait for workflow completion var result ebbandflow.WorkflowOutput err = wf.Get(ctx, &result) + // The batch has settled either way, so drain it from the backlog the + // controller tracks; only a successful workflow counts as completed. + e.completedActivities.Add(activities) if err != nil { - e.Logger.Errorf("ebbAndFlowTrack workflow failed for iteration %d: %v", iteration, err) + return fmt.Errorf("ebbAndFlowTrack workflow failed for iteration %d: %w", iteration, err) } - e.completedActivities.Add(activities) e.incrementTotalCompletedWorkflow() return nil From 5e9e823ada33c54e59b186f30ccdb718d958b88e Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:44:40 -0700 Subject: [PATCH 3/6] Fix dropped terminate error and pointer log in ExecuteKitchenSinkWorkflow The client-actions goroutine logged the atomic.Pointer rather than the error value, and silently dropped the result of the cleanup TerminateWorkflow call. Log the actual error, and log the terminate failure (the goroutine has no return path for it). --- loadgen/scenario.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/loadgen/scenario.go b/loadgen/scenario.go index 8e13d75b..95b2caa6 100644 --- a/loadgen/scenario.go +++ b/loadgen/scenario.go @@ -546,14 +546,13 @@ func (r *Run) ExecuteKitchenSinkWorkflow(ctx context.Context, options *KitchenSi err := executor.ExecuteClientSequence(cancelCtx, clientSeq) if err != nil { clientActionsErrPtr.Store(&err) - r.Logger.Error("Client actions failed: ", clientActionsErrPtr) + r.Logger.Error("Client actions failed: ", err) cancel() // TODO: Remove or change to "always terminate when exiting early" flag - err := r.Client.TerminateWorkflow( - ctx, options.StartOptions.ID, "", "client actions failed", nil) - if err != nil { - return + if terminateErr := r.Client.TerminateWorkflow( + ctx, options.StartOptions.ID, "", "client actions failed", nil); terminateErr != nil { + r.Logger.Errorf("Failed to terminate workflow after client actions failure: %v", terminateErr) } } }() From 57b2affff90ab4494737dc7338b5711f388c6b03 Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:46:31 -0700 Subject: [PATCH 4/6] Stop double-handling terminal iteration errors in generic_executor The per-iteration goroutine logged the terminal failure and also sent it on doneCh, where Run returns it wrapped for its caller to report. Drop the goroutine log for the terminal branch (propagate only); keep the retry-branch log, whose transient error is superseded and never returned. --- loadgen/generic_executor.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/loadgen/generic_executor.go b/loadgen/generic_executor.go index 3757154a..773b0238 100644 --- a/loadgen/generic_executor.go +++ b/loadgen/generic_executor.go @@ -173,11 +173,14 @@ func (g *genericRun) Run(ctx context.Context) error { backoff, retry := run.ShouldRetry(err) if retry { + // Transient: this attempt is superseded by the retry and + // never propagated, so log it here as the only record. err = fmt.Errorf("iteration %v encountered error: %w", run.Iteration, err) g.logger.Error(err) } else { + // Terminal: propagated via doneCh and reported by Run's + // caller, so don't also log it here. err = fmt.Errorf("iteration %v failed: %w", run.Iteration, err) - g.logger.Error(err) break retryLoop } From 39366a308072d4d77105ab14a2a86c6b333eae85 Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:47:09 -0700 Subject: [PATCH 5/6] Branch on gRPC status code, not error string, in RegisterDefaultSearchAttributes Replace strings.Contains(err.Error(), ...) with status.Code(err) == codes.AlreadyExists, matching ensureNexusEndpoint in the same file. The second match string was a typo ("unavailble") that never matched, so no working behavior is lost. --- loadgen/scenario.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/loadgen/scenario.go b/loadgen/scenario.go index 95b2caa6..8656a865 100644 --- a/loadgen/scenario.go +++ b/loadgen/scenario.go @@ -404,22 +404,10 @@ func (s *ScenarioInfo) RegisterDefaultSearchAttributes(ctx context.Context) erro }, Namespace: s.Namespace, }) - // Throw an error if the attributes could not be registered, but ignore already exists errs - alreadyExistsStrings := []string{ - "already exists", - "attributes mapping unavailble", - } - if err != nil { - isAlreadyExistsErr := false - for _, s := range alreadyExistsStrings { - if strings.Contains(err.Error(), s) { - isAlreadyExistsErr = true - break - } - } - if !isAlreadyExistsErr { - return fmt.Errorf("failed to register search attributes: %w", err) - } + // Throw an error if the attributes could not be registered, but ignore the + // case where they already exist (re-registration on a reused namespace). + if err != nil && status.Code(err) != codes.AlreadyExists { + return fmt.Errorf("failed to register search attributes: %w", err) } return nil } From 6d9c34aaecfe159c1a2c6308fa90f4ff5ddd7c0e Mon Sep 17 00:00:00 2001 From: lilydoar Date: Thu, 25 Jun 2026 15:47:23 -0700 Subject: [PATCH 6/6] Wrap cert-load error with %w in loadTLSConfig Preserve the error chain at this library boundary with %w, matching the sibling wraps in the same function (lines 93, 118). --- clioptions/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clioptions/client.go b/clioptions/client.go index 9bfbd05d..fd2c2a2c 100644 --- a/clioptions/client.go +++ b/clioptions/client.go @@ -64,7 +64,7 @@ func (c *ClientOptions) loadTLSConfig() (*tls.Config, error) { } cert, err := tls.LoadX509KeyPair(c.ClientCertPath, c.ClientKeyPath) if err != nil { - return nil, fmt.Errorf("failed to load certs: %s", err) + return nil, fmt.Errorf("failed to load certs: %w", err) } tlsConfig.Certificates = []tls.Certificate{cert} return tlsConfig, nil