PoC: Automatically document command JSON output schemas

.changeset/automatic-json-output-help.md

@@ -0,0 +1,5 @@
+---
+'@shopify/cli-kit': minor
+---
+
+Automatically append declared JSON output types to command help.

packages/cli-kit/src/public/node/base-command.test.ts

@@ -6,6 +6,8 @@ import {inTemporaryDirectory, mkdir, writeFile} from './fs.js'
 import {joinPath, resolvePath, cwd} from './path.js'
 import {mockAndCaptureOutput} from './testing/output.js'
 import {unstyled} from './output.js'
+import {defineJsonOutputSchema} from './json-output-schema.js'
+import {zod} from './schema.js'
 import {afterEach, beforeEach, describe, expect, test, vi} from 'vitest'
 import {Flags} from '@oclif/core'
 
@@ -207,6 +209,39 @@ const allEnvironments: Environments = {
   },
 }
 
+describe('command descriptions', () => {
+  test('automatically appends a JSON output schema', () => {
+    class CommandWithJsonOutput extends Command {
+      static get jsonOutputSchema() {
+        return defineJsonOutputSchema({
+          name: 'CommandResult',
+          schema: zod.object({value: zod.string()}),
+        })
+      }
+
+      static descriptionWithMarkdown = 'Returns a value.'
+
+      static description = this.descriptionWithoutMarkdown()
+
+      public async run(): Promise<void> {}
+    }
+
+    expect(CommandWithJsonOutput.description).toBe(`Returns a value.
+
+With \`--json\`, the command returns \`CommandResult\`, described by these TypeScript types:
+
+\`\`\`ts
+interface CommandResult {
+  value: string
+}
+\`\`\``)
+    expect(CommandWithJsonOutput.descriptionWithMarkdown).toBe(CommandWithJsonOutput.description)
+
+    CommandWithJsonOutput.descriptionWithoutMarkdown()
+    expect(CommandWithJsonOutput.descriptionWithMarkdown?.match(/interface CommandResult/g)).toHaveLength(1)
+  })
+})
+
 describe('applying environments', async () => {
   const runTestInTmpDir = (testName: string, testFunc: (tmpDir: string) => Promise<void>) => {
     test(testName, async () => {

packages/cli-kit/src/public/node/base-command.ts

@@ -7,6 +7,7 @@ import {terminalSupportsPrompting} from './system.js'
 import {hashString} from './crypto.js'
 import {isTruthy} from './context/utilities.js'
 import {setCurrentCommandId} from './global-context.js'
+import {renderJsonOutputSchema, type JsonOutputSchema} from './json-output-schema.js'
 import {JsonMap} from '../../private/common/json.js'
 import {underscore} from '../common/string.js'
 import {Command, Config, Errors} from '@oclif/core'
@@ -32,15 +33,24 @@ interface EnvironmentFlags {
 
 abstract class BaseCommand extends Command {
   static baseFlags: FlagInput<{}> = {}
+  static descriptionWithMarkdown?: string
+
+  public static get jsonOutputSchema(): JsonOutputSchema | undefined {
+    return undefined
+  }
 
   public static nonTTYFlagRequirements(_flags: FlagOutput): NonTTYFlagRequirement[] {
     return []
   }
 
   // Replace markdown links to plain text like: "link label" (url)
   public static descriptionWithoutMarkdown(): string | undefined {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    return ((this as any).descriptionWithMarkdown ?? '').replace(/(\[)(.*?)(])(\()(.*?)(\))/gm, '"$2" ($5)')
+    const descriptionWithJsonOutputSchema = appendJsonOutputSchema(
+      this.descriptionWithMarkdown ?? '',
+      this.jsonOutputSchema,
+    )
+    this.descriptionWithMarkdown = descriptionWithJsonOutputSchema
+    return descriptionWithJsonOutputSchema.replace(/(\[)(.*?)(])(\()(.*?)(\))/gm, '"$2" ($5)')
   }
 
   public static analyticsNameOverride(): string | undefined {
@@ -388,6 +398,18 @@ function commandSupportsFlag(flags: FlagInput | undefined, flagName: string): bo
   return Boolean(flags) && Object.prototype.hasOwnProperty.call(flags, flagName)
 }
 
+function appendJsonOutputSchema(description: string, outputSchema: JsonOutputSchema | undefined): string {
+  if (!outputSchema) return description
+
+  const jsonOutputDescription = `With \`--json\`, the command returns \`${outputSchema.name}\`, described by these TypeScript types:
+
+\`\`\`ts
+${renderJsonOutputSchema(outputSchema)}
+\`\`\``
+
+  return description.includes(jsonOutputDescription) ? description : `${description}\n\n${jsonOutputDescription}`
+}
+
 async function removeDuplicatedPlugins(config: Config): Promise<void> {
   const plugins = Array.from(config.plugins.values())
   const bundlePlugins = ['@shopify/app', '@shopify/plugin-cloudflare']

packages/cli-kit/src/public/node/json-output-schema.test.ts

@@ -0,0 +1,42 @@
+import {defineJsonOutputSchema, renderJsonOutputSchema} from './json-output-schema.js'
+import {zod} from './schema.js'
+import {describe, expect, test} from 'vitest'
+
+describe('JSON output schemas', () => {
+  test('renders named object schemas as TypeScript interfaces', () => {
+    const ItemSchema = zod.object({
+      id: zod.string().optional(),
+      state: zod.enum(['ready', 'pending']),
+    })
+    const ResultSchema = zod.object({
+      items: zod.array(ItemSchema),
+      cursor: zod.string().nullable().optional(),
+    })
+    const outputSchema = defineJsonOutputSchema({
+      name: 'Result',
+      schema: ResultSchema,
+      definitions: {Item: ItemSchema},
+    })
+
+    expect(renderJsonOutputSchema(outputSchema)).toBe(`interface Result {
+  items: Item[]
+  cursor?: string | null
+}
+
+interface Item {
+  id?: string
+  state: "ready" | "pending"
+}`)
+  })
+
+  test('requires nested object schemas to be named', () => {
+    const outputSchema = defineJsonOutputSchema({
+      name: 'Result',
+      schema: zod.object({item: zod.object({id: zod.string()})}),
+    })
+
+    expect(() => renderJsonOutputSchema(outputSchema)).toThrow(
+      'Nested JSON output object schemas must be included in definitions.',
+    )
+  })
+})

packages/cli-kit/src/public/node/json-output-schema.ts

@@ -0,0 +1,108 @@
+import {
+  ZodArray,
+  ZodBoolean,
+  ZodEnum,
+  ZodLiteral,
+  ZodNull,
+  ZodNullable,
+  ZodNumber,
+  ZodObject,
+  ZodOptional,
+  ZodRecord,
+  ZodString,
+  ZodTypeAny,
+  ZodUnion,
+  type z,
+} from 'zod'
+
+export interface JsonOutputSchema<TSchema extends ZodTypeAny = ZodTypeAny> {
+  readonly name: string
+  readonly schema: TSchema
+  readonly definitions: Readonly<Record<string, ZodTypeAny>>
+}
+
+export type InferJsonOutputSchema<TOutputSchema extends JsonOutputSchema> = z.infer<TOutputSchema['schema']>
+
+interface DefineJsonOutputSchemaOptions<TSchema extends ZodTypeAny> {
+  name: string
+  schema: TSchema
+  definitions?: Readonly<Record<string, ZodTypeAny>>
+}
+
+/**
+ * Defines the runtime schema and named types for a command's JSON output.
+ *
+ * @param options - The root schema name, schema, and any named nested schemas.
+ * @returns Command metadata that can also be used to infer and validate the output type.
+ */
+export function defineJsonOutputSchema<TSchema extends ZodTypeAny>(
+  options: DefineJsonOutputSchemaOptions<TSchema>,
+): JsonOutputSchema<TSchema> {
+  const {name, schema, definitions = {}} = options
+  return {name, schema, definitions}
+}
+
+/**
+ * Renders a command JSON output schema as TypeScript interfaces for help text.
+ *
+ * @param outputSchema - The command's JSON output schema metadata.
+ * @returns TypeScript interfaces describing the command's JSON output.
+ */
+export function renderJsonOutputSchema(outputSchema: JsonOutputSchema): string {
+  const namedSchemas = new Map<ZodTypeAny, string>([
+    [outputSchema.schema, outputSchema.name],
+    ...Object.entries(outputSchema.definitions).map(([name, schema]) => [schema, name] as const),
+  ])
+
+  return [
+    renderInterface(outputSchema.name, outputSchema.schema, namedSchemas),
+    ...Object.entries(outputSchema.definitions).map(([name, schema]) => renderInterface(name, schema, namedSchemas)),
+  ].join('\n\n')
+}
+
+function renderInterface(name: string, schema: ZodTypeAny, namedSchemas: ReadonlyMap<ZodTypeAny, string>): string {
+  if (!(schema instanceof ZodObject)) {
+    throw new TypeError(`JSON output type ${name} must be an object schema.`)
+  }
+
+  const properties = Object.entries(schema.shape).map(([propertyName, propertySchema]) => {
+    const optional = propertySchema instanceof ZodOptional
+    const type = renderType(propertySchema as ZodTypeAny, namedSchemas)
+    return `  ${propertyName}${optional ? '?' : ''}: ${type}`
+  })
+
+  return [`interface ${name} {`, ...properties, '}'].join('\n')
+}
+
+function renderType(schema: ZodTypeAny, namedSchemas: ReadonlyMap<ZodTypeAny, string>): string {
+  if (schema instanceof ZodOptional || schema instanceof ZodNullable) {
+    const type = renderType(schema.unwrap(), namedSchemas)
+    return schema instanceof ZodNullable ? `${type} | null` : type
+  }
+
+  const namedType = namedSchemas.get(schema)
+  if (namedType) return namedType
+
+  if (schema instanceof ZodString) return 'string'
+  if (schema instanceof ZodNumber) return 'number'
+  if (schema instanceof ZodBoolean) return 'boolean'
+  if (schema instanceof ZodNull) return 'null'
+  if (schema instanceof ZodLiteral) return JSON.stringify(schema.value)
+  if (schema instanceof ZodEnum) return schema.options.map((value: string) => JSON.stringify(value)).join(' | ')
+  if (schema instanceof ZodArray) return `${renderArrayElementType(schema.element, namedSchemas)}[]`
+  if (schema instanceof ZodRecord) return `Record<string, ${renderType(schema.valueSchema, namedSchemas)}>`
+  if (schema instanceof ZodUnion) {
+    return schema.options.map((option: ZodTypeAny) => renderType(option, namedSchemas)).join(' | ')
+  }
+
+  if (schema instanceof ZodObject) {
+    throw new TypeError('Nested JSON output object schemas must be included in definitions.')
+  }
+
+  throw new TypeError(`Unsupported JSON output schema type: ${schema.constructor.name}.`)
+}
+
+function renderArrayElementType(schema: ZodTypeAny, namedSchemas: ReadonlyMap<ZodTypeAny, string>): string {
+  const type = renderType(schema, namedSchemas)
+  return schema instanceof ZodUnion || schema instanceof ZodNullable ? `(${type})` : type
+}

packages/cli/src/cli/help.test.ts

@@ -14,7 +14,55 @@ function renderFlags(flags: Command.Flag.Any[]): [string, string | undefined][]
   return (rows ?? []).map(([left, right]) => [stripAnsi(left), right === undefined ? undefined : stripAnsi(right)])
 }
 
+function renderDescription(command: Partial<Command.Loadable>, maxWidth = 80): string | undefined {
+  const help = new ShopifyCommandHelp(
+    command as Command.Loadable,
+    {} as Interfaces.Config,
+    {maxWidth} as Interfaces.HelpOptions,
+  )
+  return (help as unknown as {description: () => string | undefined}).description()
+}
+
 describe('ShopifyCommandHelp', () => {
+  test('wraps prose and preserves indentation in fenced code blocks', () => {
+    const description = renderDescription(
+      {
+        summary: 'Return a value.',
+        description: `With \`--json\`, the command returns \`Result\`, described by these TypeScript types:
+
+\`\`\`ts
+interface Result {
+  value: string
+}
+\`\`\``,
+      },
+      50,
+    )
+
+    expect(description).toBe(`Return a value.
+
+With \`--json\`, the command returns \`Result\`,
+described by these TypeScript types:
+
+\`\`\`ts
+interface Result {
+  value: string
+}
+\`\`\``)
+  })
+
+  test('uses the default description formatting without generated JSON types', () => {
+    const command = {summary: 'Return a value.', description: 'A regular command description.'}
+    const defaultHelp = new CommandHelp(
+      command as Command.Loadable,
+      {} as Interfaces.Config,
+      {maxWidth: 80} as Interfaces.HelpOptions,
+    )
+    const defaultDescription = (defaultHelp as unknown as {description: () => string | undefined}).description()
+
+    expect(renderDescription(command)).toBe(defaultDescription)
+  })
+
   test('moves the env metadata to the end of a boolean flag description', () => {
     // Given
     const flags = [

packages/cli/src/cli/help.ts

@@ -3,6 +3,8 @@ import type {Command} from '@oclif/core'
 
 type HelpSectionBody = Parameters<CommandHelp['section']>[1]
 type HelpList = [string, string | undefined][]
+const jsonOutputDescriptionPrefix = 'With `--json`, the command returns `'
+const indentationPlaceholder = '\uE000'
 
 function isHelpList(body: HelpSectionBody): body is HelpList {
   return Array.isArray(body) && body.every((entry): entry is [string, string | undefined] => Array.isArray(entry))
@@ -46,6 +48,20 @@ export class ShopifyCommandHelp extends CommandHelp {
     return super.section(header, body)
   }
 
+  protected override description(): string | undefined {
+    const command = this.command
+    if (!command.description?.includes(jsonOutputDescriptionPrefix)) return super.description()
+
+    let description = command.description
+    if (this.opts.hideCommandSummaryInDescription) {
+      description = command.description.split(/\r?\n/).at(-1) ?? ''
+    } else if (command.summary) {
+      description = `${command.summary}\n\n${command.description}`
+    }
+
+    return this.wrap(protectJsonOutputIndentation(description)).split(indentationPlaceholder).join(' ')
+  }
+
   protected flags(flags: Command.Flag.Any[]): [string, string | undefined][] | undefined {
     const relocated = flags.map((flag) => {
       if (!flag.env) return flag
@@ -62,6 +78,25 @@ export class ShopifyCommandHelp extends CommandHelp {
   }
 }
 
+function protectJsonOutputIndentation(description: string): string {
+  let insideJsonOutputDescription = false
+  let insideCodeBlock = false
+
+  return description
+    .split(/\r?\n/)
+    .map((line) => {
+      if (line.startsWith(jsonOutputDescriptionPrefix)) insideJsonOutputDescription = true
+      if (insideJsonOutputDescription && line.trimStart().startsWith('```')) {
+        insideCodeBlock = !insideCodeBlock
+        return line
+      }
+      return insideCodeBlock
+        ? line.replace(/^ +/, (indentation) => indentationPlaceholder.repeat(indentation.length))
+        : line
+    })
+    .join('\n')
+}
+
 /**
  * Custom help class, wired up via `oclif.helpClass` in this package's
  * `package.json`. It only swaps in {@link ShopifyCommandHelp}; everything else