[rush-client-core] Reclaim provably stale daemon artifacts; make daemon stop/restart idempotent; add daemon stop --force

apps/rush-cli-client/README.md

@@ -247,19 +247,39 @@ attests a restart request, not completion of successor startup or success of a c
 
 `rush-client daemon stop` requires protocol >= 0.6 and waits for `shutdownAck`
 followed by EOF. It reports `state: "shutdownAccepted"` with exit code 0; this
-does not assert successful workspace disposal. An absent/unreachable daemon,
-unsupported protocol, missing acknowledgement, or timeout returns exit code 1.
-It does not auto-start anything.
+does not assert successful workspace disposal. Stop is idempotent: when nothing
+listens at the endpoint it reports `state: "notRunning"` with exit code 0. An
+unsupported protocol, missing acknowledgement, handshake failure, or timeout
+returns exit code 1. It does not auto-start anything.
+
+`rush-client daemon stop --force` stops a running daemon the same way, then waits
+(up to 15 seconds) for it to release its listener and ownership record and removes
+any remaining artifacts, such as an abandoned startup reservation, reporting them in
+`removedPaths`. When none is listening, it removes this workspace's leftover ownership record
+(`<key>.pid.json`), socket, and startup reservation (`.starting`), then reports
+`state: "reset"` and the `removedPaths` (or `state: "notRunning"` if nothing was
+left behind). It holds the start mutex, proves that no listener is bound, and
+refuses (exit 1) while the recorded owner PID still exists and cannot be shown to
+be a reused PID. It never kills a process. Automatic startup already reclaims
+the common leftovers on its own (see below); this is the documented escape hatch
+that every fail-closed startup message points to.
 
 `rush-client daemon restart` first verifies that the selected Rush version has a
 launcher and captures the original lock's PID/start timestamp, checking that it
 matches pong's positive PID and the selected endpoint, then performs acknowledged
 shutdown. It waits for original ownership release or a demonstrably dead owner
-before calling the existing locked starter. A live/reused owner fails closed at
+before calling the existing locked starter. A live owner fails closed at
 the startup deadline; no PID is killed and no live ownership record is deleted.
 A newly
 started/reused successor must pass hello/ping before reporting `state: "ready"`.
-An absent daemon must be started explicitly with `daemon start`.
+When nothing listens at the endpoint, restart starts a daemon exactly like `daemon start`.
+
+Automatic and explicit startup reclaim stale artifacts only when that is provably
+safe: while holding the start mutex with no `.starting` reservation, a socket
+without an ownership record, or an unreadable/corrupt record, is removed only after
+a connection attempt is refused (so no listener exists). On Linux, a record whose
+PID now belongs to a process that started after the record's `startedAt` (PID reuse)
+is treated as dead; other platforms fail closed and point to `daemon stop --force`.
 
 Restart is explicit even when automatic startup or CI execution routing is
 disabled, but conflicts with `--no-daemon`. The two-phase host retains ownership

apps/rush-cli-client/src/daemonCommands.ts

@@ -7,16 +7,23 @@ import {
   DaemonClient,
   connectOrStartDaemonAsync,
   requestDaemonShutdownAsync,
+  resetDaemonArtifactsAsync,
   type IConnectOrStartDaemonOptions
 } from '@rushstack/rush-client-core';
-import type { IDaemonLockfile } from '@rushstack/rush-daemon-transport';
+import {
+  DaemonTransportError,
+  DaemonTransportErrorCode,
+  type IDaemonLockfile
+} from '@rushstack/rush-daemon-transport';
 import type { IDaemonRequestAdmissionOptions } from '@rushstack/rush-daemon-protocol';
 
 import { getDaemonConnectionOptionsAsync } from './daemonConnectionOptions';
 import { printDaemonLogAsync } from './daemonLogs';
 import { executeDaemonGraphCommandAsync } from './daemonGraph';
 import { writeStreamAsync } from './writeStreamAsync';
 
+const FORCE_STOP_WAIT_MS: number = 15000;
+
 export interface IDaemonCommandOptions {
   readonly argv: ReadonlyArray<string>;
   readonly environment: Readonly<NodeJS.ProcessEnv>;
@@ -38,14 +45,18 @@ export async function executeDaemonCommandAsync(options: IDaemonCommandOptions):
   }
   if (
     (options.argv.length !== 1 &&
-      !(command === 'logs' && options.argv.length === 2 && options.argv[1] === '--follow')) ||
+      !(
+        options.argv.length === 2 &&
+        ((command === 'logs' && options.argv[1] === '--follow') ||
+          (command === 'stop' && options.argv[1] === '--force'))
+      )) ||
     (command !== 'start' &&
       command !== 'status' &&
       command !== 'stop' &&
       command !== 'restart' &&
       command !== 'logs')
   ) {
-    throw new Error('Usage: rush-client daemon start|status|stop|restart|logs [--follow]');
+    throw new Error('Usage: rush-client daemon start|status|stop [--force]|restart|logs [--follow]');
   }
   if (!options.rushJsonPath) throw new Error('Daemon management requires a repository containing rush.json.');
   const mayStart: boolean = command === 'start' || command === 'restart';
@@ -76,13 +87,52 @@ export async function executeDaemonCommandAsync(options: IDaemonCommandOptions):
   }
   // Status observes the selected endpoint, including a compatible daemon from a different client version.
   // It never starts a process or trusts a PID file as evidence of readiness.
-  const client: DaemonClient =
+  const client: DaemonClient | undefined =
     command === 'start'
       ? await connectOrStartDaemonAsync(connectionOptions)
-      : await DaemonClient.connectAsync({ socketPath: connectionOptions.paths.socketPath });
+      : await connectExistingAsync(connectionOptions, command !== 'status');
+  if (!client) {
+    if (command === 'restart') {
+      // Nothing to shut down: restart behaves like start.
+      const started: DaemonClient = await connectOrStartDaemonAsync(connectionOptions);
+      try {
+        await writeStatusAsync({
+          state: 'ready',
+          socketPath: connectionOptions.paths.socketPath,
+          ...(await started.status)
+        });
+      } finally {
+        await started.closeAsync();
+      }
+      return;
+    }
+    const { removedPaths } =
+      options.argv[1] === '--force'
+        ? await resetDaemonArtifactsAsync(connectionOptions.paths)
+        : { removedPaths: [] };
+    await writeStatusAsync({
+      state: removedPaths.length > 0 ? 'reset' : 'notRunning',
+      socketPath: connectionOptions.paths.socketPath,
+      ...(options.argv[1] === '--force' ? { removedPaths } : {})
+    });
+    return;
+  }
   try {
     if (command === 'stop') {
       await client.shutdownAsync();
+      if (options.argv[1] === '--force') {
+        // Wait for the acknowledged daemon to release its listener and record, then clear leftovers
+        // such as an abandoned startup reservation in the same invocation.
+        const { removedPaths } = await resetDaemonArtifactsAsync(connectionOptions.paths, {
+          waitTimeoutMs: FORCE_STOP_WAIT_MS
+        });
+        await writeStatusAsync({
+          state: 'shutdownAccepted',
+          socketPath: connectionOptions.paths.socketPath,
+          removedPaths
+        });
+        return;
+      }
       await writeStatusAsync({
         state: 'shutdownAccepted',
         socketPath: connectionOptions.paths.socketPath
@@ -105,6 +155,25 @@ export async function executeDaemonCommandAsync(options: IDaemonCommandOptions):
   }
 }
 
+/** Returns undefined when nothing listens at the endpoint and `allowAbsent` is set; other failures propagate. */
+async function connectExistingAsync(
+  options: IConnectOrStartDaemonOptions,
+  allowAbsent: boolean
+): Promise<DaemonClient | undefined> {
+  try {
+    return await DaemonClient.connectAsync({ socketPath: options.paths.socketPath });
+  } catch (error) {
+    if (
+      allowAbsent &&
+      error instanceof DaemonTransportError &&
+      error.code === DaemonTransportErrorCode.connectionRefused
+    ) {
+      return undefined;
+    }
+    throw error;
+  }
+}
+
 async function restartDaemonAsync(
   client: DaemonClient,
   options: IConnectOrStartDaemonOptions

apps/rush-cli-client/src/test/launchClient.test.ts

@@ -248,13 +248,82 @@ describe('standalone rushx fallback', () => {
     expect((await invokeAsync(true, false, false, ['daemon', 'logs', '--follow', 'extra'])).code).toBe(1);
   });
 
-  it('does not start an absent daemon when stop or restart cannot be acknowledged', async () => {
-    for (const verb of ['stop', 'restart']) {
-      const result: IInvocationResult = await invokeAsync(true, false, false, ['daemon', verb]);
-      expect(result.code).toBe(1);
-      expect(result.stderr).toContain('Could not connect to daemon');
+  it('treats stop as idempotent and restart as start when no daemon is running', async () => {
+    const { paths } = getDaemonConnectionOptions(folder, Rush.version, {}, false);
+    for (const args of [
+      ['daemon', 'stop'],
+      ['daemon', 'stop', '--force']
+    ]) {
+      const result: IInvocationResult = await invokeAsync(true, false, false, args);
+      expect(result).toMatchObject({ code: 0, stderr: '' });
+      expect(JSON.parse(result.stdout)).toEqual({
+        state: 'notRunning',
+        socketPath: paths.socketPath,
+        ...(args[2] ? { removedPaths: [] } : {})
+      });
     }
-  });
+    try {
+      const restarted: IInvocationResult = await invokeAsync(true, false, false, ['daemon', 'restart']);
+      expect(restarted.stderr).toBe('');
+      expect(restarted.code).toBe(0);
+      expect(JSON.parse(restarted.stdout)).toMatchObject({ state: 'ready', socketPath: paths.socketPath });
+      const stopped: IInvocationResult = await invokeAsync(true, false, false, ['daemon', 'stop']);
+      expect(stopped.code).toBe(0);
+      expect(JSON.parse(stopped.stdout)).toMatchObject({ state: 'shutdownAccepted' });
+    } finally {
+      const deadline: number = Date.now() + 7000;
+      while (fs.existsSync(paths.lockfilePath) && Date.now() < deadline) await delayAsync(50);
+      expect(fs.existsSync(paths.lockfilePath)).toBe(false);
+    }
+  }, 30000);
+
+  it('stop --force clears an abandoned startup reservation next to a running daemon', async () => {
+    const { paths } = getDaemonConnectionOptions(folder, Rush.version, {}, false);
+    const reservation: string = `${paths.lockfilePath}.starting`;
+    try {
+      expect((await invokeAsync(true, false, false, ['daemon', 'start'])).code).toBe(0);
+      fs.writeFileSync(reservation, 'abandoned');
+      const result: IInvocationResult = await invokeAsync(true, false, false, ['daemon', 'stop', '--force']);
+      expect(result).toMatchObject({ code: 0, stderr: '' });
+      expect(JSON.parse(result.stdout)).toEqual({
+        state: 'shutdownAccepted',
+        socketPath: paths.socketPath,
+        removedPaths: [reservation]
+      });
+      expect(fs.existsSync(reservation)).toBe(false);
+      expect(fs.existsSync(paths.lockfilePath)).toBe(false);
+    } finally {
+      const deadline: number = Date.now() + 7000;
+      while (fs.existsSync(paths.lockfilePath) && Date.now() < deadline) await delayAsync(50);
+    }
+  }, 30000);
+
+  (process.platform === 'win32' ? it.skip : it)(
+    'stop --force removes stale artifacts left by a killed daemon',
+    async () => {
+      const { paths } = getDaemonConnectionOptions(folder, Rush.version, {}, false);
+      fs.mkdirSync(path.dirname(paths.lockfilePath), { recursive: true, mode: 0o700 });
+      const listener: ChildProcess = spawn(
+        process.execPath,
+        [
+          '-e',
+          `require('net').createServer().listen(${JSON.stringify(paths.socketPath)}, () => process.kill(process.pid, 'SIGKILL'))`
+        ],
+        { stdio: 'ignore' }
+      );
+      await once(listener, 'close');
+      fs.writeFileSync(paths.lockfilePath, 'garbage{');
+      const result: IInvocationResult = await invokeAsync(true, false, false, ['daemon', 'stop', '--force']);
+      expect(result).toMatchObject({ code: 0, stderr: '' });
+      expect(JSON.parse(result.stdout)).toEqual({
+        state: 'reset',
+        socketPath: paths.socketPath,
+        removedPaths: [paths.lockfilePath, paths.socketPath]
+      });
+      expect(fs.existsSync(paths.lockfilePath)).toBe(false);
+      expect(fs.existsSync(paths.socketPath)).toBe(false);
+    }
+  );
 
   it.each([false, true])(
     'restarts after ownership release and stops the successor (embedded: %s)',

common/changes/@rushstack/rush-cli-client/daemon-recovery_2026-09-23.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/rush-cli-client",
+      "comment": "Make `rush-client daemon stop` idempotent (state notRunning, exit 0), make `daemon restart` start a daemon when none is running, and add `daemon stop --force` to remove stale workspace daemon artifacts.",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@rushstack/rush-cli-client"
+}

common/changes/@rushstack/rush-client-core/daemon-recovery_2026-09-23.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/rush-client-core",
+      "comment": "Reclaim provably stale daemon artifacts (a socket without an ownership record, a corrupt record, or a Linux PID reused since the record was written) instead of permanently disabling the daemon, and add resetDaemonArtifactsAsync() as the explicit recovery path referenced by fail-closed messages.",
+      "type": "patch"
+    }
+  ],
+  "packageName": "@rushstack/rush-client-core"
+}

common/reviews/api/rush-client-core.api.md

@@ -80,6 +80,16 @@ export interface IConnectOrStartDaemonOptions extends Omit<IDaemonClientConnectO
     readonly startupTimeoutMs?: number;
 }
 
+// @beta
+export interface IDaemonArtifactResetOptions {
+    readonly waitTimeoutMs?: number;
+}
+
+// @beta
+export interface IDaemonArtifactResetResult {
+    readonly removedPaths: ReadonlyArray<string>;
+}
+
 // @beta
 export interface IDaemonClientConnectOptions {
     // (undocumented)
@@ -130,4 +140,7 @@ export interface IDaemonStartCommand {
 // @beta
 export function requestDaemonShutdownAsync(client: DaemonClient, paths: IDaemonPaths, timeoutMs?: number): Promise<Pick<IDaemonLockfile, 'pid' | 'startedAt'>>;
 
+// @beta
+export function resetDaemonArtifactsAsync(paths: IDaemonPaths, options?: IDaemonArtifactResetOptions): Promise<IDaemonArtifactResetResult>;
+
 ```

libraries/rush-client-core/README.md

@@ -48,7 +48,13 @@ handing the explicit command to a detached startup helper. The helper spawns wit
 a shell and retains that reservation until the daemon completes hello/ping readiness,
 independently of whether the requesting client survives. Clients still await
 hello/pong under bounded backoff. Stdout/stderr go to `<lockfilePath>.log`. No PID
-is killed; a live (possibly reused) PID with an unreachable socket fails closed.
+is killed. While holding the mutex with no startup reservation, stale leftovers are
+reclaimed only when provably safe: a socket without an ownership record, or a corrupt
+record, once a connection attempt is refused (no listener exists); and, on Linux, a
+record whose PID now belongs to a process that started after the record's `startedAt`
+(PID reuse, detected from `/proc`). Any other live PID with an unreachable socket fails
+closed, pointing to `resetDaemonArtifactsAsync()` (`rush-client daemon stop --force`),
+which removes the record, socket and reservation after the same no-listener/no-live-owner checks.
 The helper uses a stable tool cwd, and the starting client awaits its exit after
 readiness. The explicit launcher's cwd is unchanged.
 
@@ -90,7 +96,8 @@ identifies the original ownership record by its `pid` and `startedAt`, captured
 before sending shutdown. Startup waits until that record disappears, another
 owner replaces it, or its owner is demonstrably dead. A new owner is checked by
 hello/ping; it is never blindly reclaimed. Signal 0 is only a liveness probe; no
-process is killed. A live/reused owner times out conservatively, while corrupt or
+process is killed. A live owner times out conservatively (a Linux PID provably reused
+since `startedAt` counts as dead), while corrupt or
 unreadable metadata fails closed.
 During a captured predecessor handoff, transient Windows sharing-denied reads stay
 unknown and are retried only within the existing startup deadline. They never