Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/apps/metrics-systems/__tests__/api.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { describe, it, expect, vi } from 'vitest'
import {
refreshMs,
bucketMs,
byHealthThenName,
containerDisplayName,
Expand Down Expand Up @@ -359,3 +360,11 @@ describe('formatBytes', () => {
expect(formatBytes(0)).toBe('0 B')
})
})

describe('refreshMs', () => {
it('keeps short ranges live and slows a week to five minutes', () => {
expect(refreshMs('30m')).toBe(30_000)
expect(refreshMs('1d')).toBe(30_000)
expect(refreshMs('7d')).toBe(300_000)
})
})
27 changes: 21 additions & 6 deletions src/apps/metrics-systems/api.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ export interface ServiceCatalog {
export interface StandardMetrics {
requests_total: number
rate_per_sec: number
success_count_5m: number
failure_count_5m: number
success_count: number
failure_count: number
error_rate_percent: number
avg_duration_microseconds: number
p95_duration_microseconds: number
Expand Down Expand Up @@ -54,6 +54,10 @@ export interface ServiceMetricsResponse {
// proxy, which is also the signal that `requests_total` is still a lifetime
// total rather than a windowed count — see requestsTotalLabel.
view?: MetricView
// The window every windowed tile was read over — the ?range= the request
// named, echoed as the PromQL duration the queries carried (MoonBase#1507).
// Absent from a proxy that still windows every tile at five minutes.
window?: string
custom: CustomMetricGroup[]
}

Expand All @@ -72,15 +76,26 @@ export function hasToggleableMetrics(response: ServiceMetricsResponse | null): b
// What to call the standard block's `requests_total` tile.
//
// MoonBase#1287 changed that field from sum(x) — cumulative since process
// start, and so reset by every deploy — to a 5-minute increase(). The JSON key
// start, and so reset by every deploy — to a windowed increase(). The JSON key
// could not change without breaking this UI, so the name still says "total"
// while the number no longer is one; calling the tile "Total" would repeat the
// same wrong label the API is stuck with.
//
// `view` is the tell: it is present exactly when the proxy carries that change,
// because both landed in the same commit. An older host therefore keeps the
// old label, which is still honest about the number it is actually sending.
// The window is whatever the proxy says it read over: the selected range once
// it takes one (MoonBase#1507), five minutes on the proxy before that, which
// echoes `view` but no `window`. A host older than both keeps the old label,
// which is still honest about the number it is actually sending.
// How often a service page refetches. A week's numbers are refetched every
// five minutes rather than every thirty seconds: at 7d every tile is an
// instant query with a week-long lookback (MoonBase#1507), a page of them
// every half minute is the one load on Prometheus this dashboard adds, and
// a week's count does not move in half a minute. Shorter ranges stay live.
export function refreshMs(timeRange: string): number {
return timeRange === '7d' ? 5 * 60_000 : 30_000
}

export function requestsTotalLabel(response: ServiceMetricsResponse | null): string {
if (response?.window) return `Req (${response.window})`
return response?.view ? 'Req (5m)' : 'Total'
}

Expand Down
30 changes: 19 additions & 11 deletions src/apps/metrics-systems/components/ServiceDashboard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
fetchJson,
fillWindow,
hasToggleableMetrics,
refreshMs,
requestsTotalLabel,
seriesWindow,
serviceDisplayName,
Expand Down Expand Up @@ -260,11 +261,14 @@ const ServiceDashboard = ({ service, onConnectionStateChange }: ServiceDashboard
// service name server-side, so this no longer pulls the whole host
// payload — every other container's stats — to use one row of it.
const [scalarData, seriesData, containerData] = await Promise.all([
// ?view= is safe against an older proxy: Go's mux ignores query
// parameters nothing reads, so the request still succeeds and comes
// back in the only form that proxy has. The switch stays hidden in
// that case anyway — see hasToggleableMetrics.
fetchJson<ServiceMetricsResponse>(`${METRICS_API_URL}/service/${service}?view=${view}`),
// ?view= and ?range= are safe against an older proxy: Go's mux
// ignores query parameters nothing reads, so the request still
// succeeds and comes back in the only form that proxy has. The
// switch stays hidden in that case anyway — see hasToggleableMetrics.
// The range is the tiles' window (MoonBase#1507), so a tile and the
// chart under it describe the same span; the label reads it back
// from the response — see requestsTotalLabel.
fetchJson<ServiceMetricsResponse>(`${METRICS_API_URL}/service/${service}?view=${view}&range=${timeRange}`),
// No ?view= here: the timeseries endpoint always answers with both
// request_rate and request_count, and the chart below picks between
// them locally — see STANDARD_SERIES.
Expand All @@ -289,7 +293,7 @@ const ServiceDashboard = ({ service, onConnectionStateChange }: ServiceDashboard
}

const start = setTimeout(fetchMetrics, 0)
const interval = setInterval(fetchMetrics, 30000)
const interval = setInterval(fetchMetrics, refreshMs(timeRange))
return () => {
active = false
clearTimeout(start)
Expand Down Expand Up @@ -324,9 +328,12 @@ const ServiceDashboard = ({ service, onConnectionStateChange }: ServiceDashboard
onChange={(e) => setTimeRange(e.target.value as '30m' | '1d' | '7d')}
className={styles.timeRangeSelect}
>
<option value="30m">30m</option>
<option value="1d">1d</option>
<option value="7d">7d</option>
{/* The one cue for the whole page: every chart and, from a
proxy that takes a range (MoonBase#1507), every tile but
Active reads over this span. */}
<option value="30m">last 30m</option>
<option value="1d">last 1d</option>
<option value="7d">last 7d</option>
</select>
{/* Only when the proxy is new enough to answer a view at all. Every
service's Serving chart has a toggleable Request Rate — even one
Expand Down Expand Up @@ -386,12 +393,13 @@ const ServiceDashboard = ({ service, onConnectionStateChange }: ServiceDashboard
<div className={styles.miniValue}>{((standard.p95_duration_microseconds || 0) / 1000).toFixed(1)}</div>
</div>
<div className={styles.miniCard}>
<div className={styles.miniLabel}>Active</div>
{/* A gauge, and the one tile not over the selected range. */}
<div className={styles.miniLabel}>Active (now)</div>
<div className={styles.miniValue}>{(standard.active_requests || 0).toFixed(0)}</div>
</div>
<div className={styles.miniCard}>
{/* Not "Total" once the proxy is new enough: the field still
says total, but MoonBase#1287 made it a 5-minute count. */}
says total, but MoonBase#1287 made it a windowed count. */}
<div className={styles.miniLabel}>{requestsTotalLabel(scalar)}</div>
<div className={styles.miniValue}>{formatValue(standard.requests_total || 0)}</div>
</div>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ const scalarResponse = {
standard: {
requests_total: 1234,
rate_per_sec: 1.25,
success_count_5m: 100,
failure_count_5m: 5,
success_count: 100,
failure_count: 5,
error_rate_percent: 4.8,
avg_duration_microseconds: 1500,
p95_duration_microseconds: 9500,
Expand All @@ -58,6 +58,11 @@ const rateScalarResponse = {
}

// A host still running a pre-MoonBase#1287 proxy: no `view`, no `toggleable`.
// A proxy that windows every tile by the request's range echoes which one
// (MoonBase#1507); the fixture above stands for the one before it, which
// echoed a view and read every tile over five minutes.
const rangedScalarResponse = { ...scalarResponse, window: '1d' }

const legacyScalarResponse = {
...scalarResponse,
view: undefined,
Expand Down Expand Up @@ -119,6 +124,7 @@ const containerDetail = (overrides: Record<string, unknown> = {}) => ({
// longer matches it. Compare the path and read the view separately.
const pathOf = (url: string) => url.split('?')[0]
const viewOf = (url: string) => new URLSearchParams(url.split('?')[1] ?? '').get('view') ?? 'count'
const rangeOf = (url: string) => new URLSearchParams(url.split('?')[1] ?? '').get('range')

const ok = (body: unknown) => Promise.resolve({ ok: true, text: () => Promise.resolve(JSON.stringify(body)) })
const notFound = () => Promise.resolve({ ok: false, text: () => Promise.resolve('') })
Expand Down Expand Up @@ -267,6 +273,9 @@ describe('ServiceDashboard', () => {
// toggle picks between them client-side — so plain endsWith matches it.
const urls = mockFetch.mock.calls.map((call) => String(call[0]))
expect(urls.some((url) => url.endsWith('/service/golf_hub/timeseries/30m'))).toBe(true)
// The tiles read over the same range as the charts (MoonBase#1507),
// on the same request that names the view.
expect(urls.some((url) => pathOf(url).endsWith('/service/golf_hub') && rangeOf(url) === '30m' && viewOf(url) === 'count')).toBe(true)
})

it('reports failure and keeps the page shape when the API is down', async () => {
Expand Down Expand Up @@ -469,6 +478,35 @@ describe('ServiceDashboard', () => {
}
})

it('refetches a week every five minutes, not every thirty seconds', async () => {
vi.useFakeTimers()
try {
const { container } = render(<ServiceDashboard service="golf_hub" onConnectionStateChange={vi.fn()} />)
await act(async () => {
await vi.advanceTimersByTimeAsync(0)
})
fireEvent.change(container.querySelector('select')!, { target: { value: '7d' } })
await act(async () => {
await vi.advanceTimersByTimeAsync(0)
})
const scalarFetches = () =>
mockFetch.mock.calls.filter((call) => pathOf(String(call[0])).endsWith('/service/golf_hub') && rangeOf(String(call[0])) === '7d').length
expect(scalarFetches()).toBe(1)
// A week-long lookback per tile is the load this dashboard adds to
// Prometheus; a week's count does not move in half a minute.
await act(async () => {
await vi.advanceTimersByTimeAsync(30000)
})
expect(scalarFetches()).toBe(1)
await act(async () => {
await vi.advanceTimersByTimeAsync(5 * 60000 - 30000)
})
expect(scalarFetches()).toBe(2)
} finally {
vi.useRealTimers()
}
})

it('falls back to a dash when the image carries no tag', async () => {
mockFetch.mockImplementation((url: string) => {
if (url.endsWith('/container/golf_hub')) return ok(containerDetail({ image: 'golf_hub', version: '' }))
Expand Down Expand Up @@ -712,6 +750,51 @@ describe('ServiceDashboard', () => {
expect(screen.queryByText('Total')).toBeNull()
})

it('names the requests tile by the window the proxy read it over', async () => {
mockFetch.mockImplementation((url: string) => {
if (url.includes('/service/golf_hub/timeseries/')) return ok(timeseriesResponse)
if (pathOf(url).endsWith('/service/golf_hub')) return ok({ ...rangedScalarResponse, window: rangeOf(url) })
if (url.endsWith('/container/golf_hub')) return ok(containerDetail())
return notFound()
})
const { container } = render(<ServiceDashboard service="golf_hub" onConnectionStateChange={vi.fn()} />)
await act(async () => {
await new Promise((resolve) => setTimeout(resolve, 100))
})
expect(screen.getByText('Req (1d)')).toBeTruthy()

// The label follows the range, read back from the response rather than
// from the select, so it never says a window the number was not: while
// the 7d answer is still out, the 1d number keeps its 1d caption.
let answer: (body: unknown) => void = () => {}
mockFetch.mockImplementation((url: string) => {
if (url.includes('/service/golf_hub/timeseries/')) return ok(timeseriesResponse)
if (pathOf(url).endsWith('/service/golf_hub')) {
return new Promise((resolve) => {
answer = (body) => resolve({ ok: true, text: () => Promise.resolve(JSON.stringify(body)) })
})
}
if (url.endsWith('/container/golf_hub')) return ok(containerDetail())
return notFound()
})
fireEvent.change(container.querySelector('select')!, { target: { value: '7d' } })
await act(async () => {
await new Promise((resolve) => setTimeout(resolve, 100))
})
expect(screen.getByText('Req (1d)')).toBeTruthy()
expect(screen.queryByText('Req (7d)')).toBeNull()
await act(async () => {
answer({ ...rangedScalarResponse, window: '7d' })
await new Promise((resolve) => setTimeout(resolve, 50))
})
expect(screen.getByText('Req (7d)')).toBeTruthy()
expect(screen.queryByText('Req (5m)')).toBeNull()
// The range select says the span once for every tile; the gauge says
// it is the exception.
expect(screen.getByRole('option', { name: 'last 7d' })).toBeTruthy()
expect(screen.getByText('Active (now)')).toBeTruthy()
})

it('keeps the old label for a proxy still sending a lifetime total', async () => {
mockFetch.mockImplementation((url: string) => {
if (url.includes('/service/golf_hub/timeseries/')) return ok(timeseriesResponse)
Expand Down
Loading