k8s: keep credential plugins off the terminal

docs/cluster-setup.md

@@ -147,6 +147,60 @@ A `401`/`403` is **not** treated as "no cluster": that is a live cluster
 refusing your credentials, a different problem with a different fix, and
 k10s connects anyway so you can see whatever you are allowed to see.
 
+A third failure gets its own wording, because its fix is different again:
+
+3. **The credential plugin failed.** See the next section.
+
+## Credential plugins (tsh, aws, gke-gcloud-auth-plugin, kubelogin)
+
+Many kubeconfigs do not hold a token. The user entry names a program
+(`exec:`) that fetches one, and client-go runs it before the first request:
+`tsh kube credentials` for Teleport, `aws eks get-token`, `kubelogin` for
+AKS, `gke-gcloud-auth-plugin`.
+
+Under `kubectl` that program shares the terminal, so it can print "tap your
+security key" and read an MFA code. Under a TUI it cannot: k10s owns the
+keyboard, a prompt would swallow keystrokes meant for k10s, and whatever the
+plugin printed would land on top of the frame. That is where k9s hangs
+([derailed/k9s#2048](https://github.com/derailed/k9s/issues/2048)), with
+`kill -9` as the way out.
+
+k10s does two things instead:
+
+- **The plugin never gets the terminal.** It runs with stdin marked
+  unavailable — what `kubectl exec -i` does — so a plugin that needs to
+  prompt fails at once rather than waiting for input nobody can type. One
+  that declares `interactiveMode: Always` is refused before it runs.
+- **What it said is what you see.** The plugin's stderr goes to k10s, not
+  the screen, and its last line is the reason under **No cluster**:
+
+  ```
+  credential plugin tsh: Having problems with relogin, please use 'tsh login/tsh kube login' manually
+  ```
+
+The fix is in another shell: `tsh login` or `tsh kube login <cluster>`,
+`aws sso login`, `az login` — whatever the message names — then `r`.
+Credentials the plugin caches (tsh keeps certificates under `~/.tsh`) are
+picked up on the retry.
+
+A plugin that takes a while (minting a certificate, calling STS) does not
+cost you the connection: its run time is spent inside the first request's
+timeout, which client-go cannot interrupt, so k10s retries that one request
+once with the credentials now cached.
+
+**Teleport.** Two setups do not fit the exec plugin at all, and `tsh` says
+so: per-session MFA, whose certificates last about a minute, and a proxy
+behind an L7 load balancer, which refuses direct Kubernetes clients. Both
+are what `tsh proxy kube` is for. It keeps the session in its own terminal
+and writes a kubeconfig into `$KUBECONFIG`:
+
+```bash
+tsh proxy kube <cluster> --exec   # opens a shell with KUBECONFIG set
+k10s                              # inside that shell
+```
+
+`:ctx` then lists only what that kubeconfig holds.
+
 ## Related
 
 - [backends.md](backends.md) — the live backend, the demo backend, and what

internal/k8s/client.go

@@ -6,6 +6,7 @@ package k8s
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"time"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -96,6 +97,9 @@ func New(path, context string) (*Client, error) {
 	}
 	restCfg.QPS = 50
 	restCfg.Burst = 100
+	// A plugin that prompts would be prompting a terminal the TUI owns. See
+	// credplugin.go.
+	nonInteractive(restCfg.ExecProvider)
 	// Before any client is built from it, so every one of them, and the
 	// discovery copy below, carries the guard.
 	if readOnly.Load() {
@@ -116,7 +120,7 @@ func New(path, context string) (*Client, error) {
 	// itself: that config also backs the informers' long-lived list-watch
 	// connections, which a blanket per-request timeout would kill.
 	discoCfg := *restCfg
-	discoCfg.Timeout = 10 * time.Second
+	discoCfg.Timeout = discoveryTimeout
 	disco, err := discovery.NewDiscoveryClientForConfig(&discoCfg)
 	if err != nil {
 		return nil, fmt.Errorf("discovery client: %w", err)
@@ -146,12 +150,49 @@ func New(path, context string) (*Client, error) {
 		CurrentContext: cur,
 		Server:         restCfg.Host,
 	}
-	if v, err := disco.ServerVersion(); err == nil {
+	c.probe(disco)
+	return c, nil
+}
+
+// discoveryTimeout bounds the one-shot discovery GETs. A variable so a test
+// can make it short.
+var discoveryTimeout = 10 * time.Second
+
+// probe asks the API server its version: the one request New makes, and so
+// the only evidence that there is a cluster at the other end at all (see
+// Reachable).
+//
+// A credential plugin runs inside that request's timeout. client-go runs it
+// before sending and cannot interrupt it, so a plugin that takes its time —
+// tsh minting a certificate, aws calling STS — spends the budget, and the
+// request itself then fails with "context deadline exceeded" even though
+// the plugin succeeded. Its credentials are cached by then, so one more try
+// is the network alone. Without it the user sees No cluster and presses r,
+// which does the same thing by hand (gravitational/teleport#49622 is k9s
+// doing exactly this).
+//
+// When the plugin is what failed, the error says so, in the plugin's own
+// words where it had any: that is a different problem from the API server
+// not answering, with a different fix.
+func (c *Client) probe(disco discovery.DiscoveryInterface) {
+	plugin := c.RestConfig.ExecProvider
+	started := time.Now()
+	v, err := disco.ServerVersion()
+	if err != nil && plugin != nil && requestTimedOut(err) {
+		v, err = disco.ServerVersion()
+	}
+	switch {
+	case err == nil:
 		c.Version = v.GitVersion
-	} else {
+	case plugin != nil && credentialPluginFailed(err):
+		c.versionErr = &CredentialPluginError{
+			Plugin: filepath.Base(plugin.Command),
+			Said:   tap.said(started),
+			Err:    err,
+		}
+	default:
 		c.versionErr = err
 	}
-	return c, nil
 }
 
 // Reachable reports whether there is actually a cluster behind this

internal/k8s/credplugin.go

@@ -0,0 +1,238 @@
+package k8s
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"net"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/client-go/tools/clientcmd"
+	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+)
+
+// Credential plugins and the terminal.
+//
+// A kubeconfig user is often not a token but a program — tsh (Teleport),
+// aws, gke-gcloud-auth-plugin, kubelogin — that client-go runs to get one.
+// Under kubectl that program shares the terminal: it can print "tap your
+// security key" and read an MFA code from stdin. Under a TUI it cannot.
+// Bubble Tea owns stdin, so a prompt swallows keystrokes meant for k10s, and
+// whatever the plugin prints lands on top of the frame. That is the k9s
+// failure (derailed/k9s#2048): a session that looks hung, with kill -9 as
+// the only way out.
+//
+// Two rules follow, and this file is both of them.
+//
+//  1. The plugin never gets the terminal while the TUI is up. New marks
+//     stdin unavailable — what kubectl does when stdin is spoken for, as in
+//     `kubectl exec -i` — so client-go runs the plugin without it and the
+//     plugin fails fast instead of waiting for input nobody can type.
+//     Logging in is done outside k10s, in a shell, then `r`.
+//
+//  2. What the plugin says is kept, not painted over. client-go hands every
+//     plugin the os.Stderr of the moment the first client is built, so
+//     CaptureStderr swaps a pipe in before that, and the last line a plugin
+//     wrote becomes the error the user reads: "Having problems with relogin,
+//     please use 'tsh login'" instead of "exit code 1".
+
+// stdinUnavailableMessage is the reason client-go gives when a plugin that
+// declares `interactiveMode: Always` is refused before it runs.
+const stdinUnavailableMessage = "k10s owns the terminal, so the credential plugin cannot prompt; " +
+	"log in with it in another shell, then press r"
+
+// nonInteractive keeps an exec plugin off the terminal (rule 1 above).
+// Nothing else about the plugin changes, so its own declaration still
+// decides between running without stdin (Never, IfAvailable) and refusing
+// to run at all (Always).
+func nonInteractive(ep *clientcmdapi.ExecConfig) {
+	if ep == nil {
+		return
+	}
+	ep.StdinUnavailable = true
+	ep.StdinUnavailableMessage = stdinUnavailableMessage
+}
+
+// stderrTap is os.Stderr rerouted through a pipe k10s reads (rule 2). It
+// remembers only the last non-empty line and when it arrived: plugins say
+// one useful thing, at the end, and a message from a different attempt
+// must not be pinned on this one.
+type stderrTap struct {
+	mu       sync.Mutex
+	terminal *os.File // what os.Stderr was before the swap
+	pipe     *os.File // the write end now in os.Stderr
+	last     string
+	at       time.Time
+}
+
+var (
+	captureOnce sync.Once
+	tap         = &stderrTap{}
+)
+
+// start swaps the pipe in and drains it for as long as the write end lives.
+func (t *stderrTap) start() error {
+	r, w, err := os.Pipe()
+	if err != nil {
+		return err
+	}
+	t.terminal, t.pipe = os.Stderr, w
+	os.Stderr = w
+	go func() {
+		sc := bufio.NewScanner(r)
+		sc.Buffer(make([]byte, 0, 64*1024), 1<<20)
+		for sc.Scan() {
+			line := strings.TrimSpace(sc.Text())
+			if line == "" {
+				continue
+			}
+			t.mu.Lock()
+			t.last, t.at = line, time.Now()
+			t.mu.Unlock()
+		}
+		// A line too long for the buffer ends the scan; nothing to do about
+		// it but stop reading, and nothing is lost that mattered.
+		_ = sc.Err()
+		_ = r.Close()
+	}()
+	return nil
+}
+
+// said returns the last line written at or after since, "" if none. The
+// pipe is read on another goroutine, so a line the plugin wrote just before
+// exiting may still be in flight when its error arrives; said waits a
+// moment for it rather than reporting silence that is not.
+func (t *stderrTap) said(since time.Time) string {
+	deadline := time.Now().Add(200 * time.Millisecond)
+	for {
+		t.mu.Lock()
+		line, at := t.last, t.at
+		t.mu.Unlock()
+		if !at.Before(since) {
+			// The prefix is the plugin's; the error carries its own.
+			line = strings.TrimPrefix(line, "ERROR: ")
+			return strings.TrimPrefix(line, "error: ")
+		}
+		if time.Now().After(deadline) {
+			return ""
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+}
+
+// CaptureStderr routes os.Stderr through k10s for the rest of the process
+// and returns what it was, for the few things still allowed to write to the
+// terminal: the editor, a foreground plugin, main's own last words. Call it
+// once, before the first client is built — client-go copies os.Stderr for
+// the plugin at that point and never looks again. If the pipe cannot be
+// made nothing is captured and the terminal stays the plugin's stderr, as
+// it always was.
+func CaptureStderr() (terminal *os.File) {
+	captureOnce.Do(func() {
+		if err := tap.start(); err != nil {
+			tap.terminal = os.Stderr
+		}
+	})
+	return tap.terminal
+}
+
+// ReleaseStderr puts the terminal back as os.Stderr. For after the TUI has
+// exited: anything said then belongs on the screen, not in the pipe.
+func ReleaseStderr() {
+	if tap.terminal != nil {
+		os.Stderr = tap.terminal
+	}
+}
+
+// TerminalStderr is the terminal's stderr whether or not CaptureStderr has
+// run: the right destination for a process that takes the terminal over.
+func TerminalStderr() *os.File {
+	if tap.terminal != nil {
+		return tap.terminal
+	}
+	return os.Stderr
+}
+
+// CredentialPlugin names the exec credential plugin a kubeconfig context
+// signs in with — "tsh", "aws", "kubelogin" — or "" when the context uses
+// none. It reads kubeconfig only, so it can be asked before connecting,
+// which is when the UI wants to say who it is waiting on.
+func CredentialPlugin(path, context string) string {
+	rules := clientcmd.NewDefaultClientConfigLoadingRules()
+	if path != "" {
+		rules.ExplicitPath = path
+	}
+	raw, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, &clientcmd.ConfigOverrides{}).RawConfig()
+	if err != nil {
+		return ""
+	}
+	return credentialPlugin(raw, context)
+}
+
+func credentialPlugin(raw clientcmdapi.Config, context string) string {
+	if context == "" {
+		context = raw.CurrentContext
+	}
+	ctx := raw.Contexts[context]
+	if ctx == nil {
+		return ""
+	}
+	auth := raw.AuthInfos[ctx.AuthInfo]
+	if auth == nil || auth.Exec == nil {
+		return ""
+	}
+	return filepath.Base(auth.Exec.Command)
+}
+
+// CredentialPluginError is the context's exec credential plugin failing,
+// which is a different situation from the API server not answering: the
+// cluster may be fine, and the fix is to log in again, outside k10s.
+type CredentialPluginError struct {
+	Plugin string // the command's base name: tsh, aws, kubelogin
+	Said   string // the last line it wrote to stderr, "" if it was silent
+	Err    error  // client-go's own error, for errors.Is and errors.As
+}
+
+func (e *CredentialPluginError) Error() string {
+	what := e.Said
+	if what == "" {
+		what = pluginClause(e.Err)
+	}
+	return "credential plugin " + e.Plugin + ": " + what
+}
+
+func (e *CredentialPluginError) Unwrap() error { return e.Err }
+
+// credentialPluginFailed reports whether err is the plugin failing rather
+// than the request it was fetching credentials for. client-go formats the
+// plugin's error into the request's with %v, so the chain is cut there and
+// the text is all there is to go on.
+func credentialPluginFailed(err error) bool {
+	return err != nil && strings.Contains(err.Error(), "getting credentials: ")
+}
+
+// pluginClause cuts client-go's error down to the part about the plugin:
+// `Get "https://…/version": getting credentials: exec: executable tsh
+// failed with exit code 1` → `exec: executable tsh failed with exit code 1`.
+func pluginClause(err error) string {
+	msg := err.Error()
+	if i := strings.LastIndex(msg, "getting credentials: "); i >= 0 {
+		msg = msg[i+len("getting credentials: "):]
+	}
+	return msg
+}
+
+// requestTimedOut reports whether err is a request's own budget running
+// out: rest.Config.Timeout fires through the request context, and the
+// transport says so in a net.Error.
+func requestTimedOut(err error) bool {
+	if errors.Is(err, context.DeadlineExceeded) {
+		return true
+	}
+	var ne net.Error
+	return errors.As(err, &ne) && ne.Timeout()
+}