From 7fa11703ed4eb6bc34059b073722befdf511c17f Mon Sep 17 00:00:00 2001 From: nafees nazik Date: Thu, 19 Nov 2026 15:40:10 +0530 Subject: [PATCH 1/3] feat(help-center): search the documentation, and count what readers look for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The help center could only be browsed. A reader who knew the word they wanted had no way to ask for it, and a workspace had no way to learn which questions its documentation was failing to answer. Search is two signals behind one program. Lexical: a generated `tsvector` column (title over summary over body) with a GIN index, ranked by `ts_rank_cd`, with the query built in one module that keeps only letters and digits so a reader's punctuation cannot reach the `tsquery` parser, and prefix branches so a half-typed word still finds the page. Semantic: every readable article carries an embedding, written after each save by a detached best-effort job, and the semantic pass runs only when the lexical pass returns fewer than three pages. It is a rescue rather than a ranker — the two scores do not share a scale, and a public search box must not spend a model call on the common case, which is a query that matches. Two day-bucketed tables answer the questions the reader left behind: `help_center_article_daily` counts views and verdicts per article, and `help_center_search_daily` counts searches and how many found nothing. Both writes are advisory — a failed counter never fails the read that produced it — and the dashboard panel turns the second into a list of documentation gaps. The widget now offers published articles above its similar-posts panel, so a question that is already documented does not become a new post. Its endpoint is strict (no semantic pass) and answers `{ articles, enabled }` without failing, because it is unauthenticated and a distinguishable failure would confirm which workspaces exist. Portal links come from one builder in `site/url.ts`, shared with the email outbox, so an email and a widget cannot disagree about the host. The dashboard operation sees drafts; the public one applies the same readable-state filter as every other portal read, as a parameter of the repository call rather than something a caller remembers. --- .../dashboard/features/help-center/atoms.ts | 46 ++ .../components/help-center-analytics.tsx | 372 ++++++++++ .../components/help-center-index.tsx | 6 + .../components/help-center-search.tsx | 143 ++++ apps/web/src/portal/help-center/atoms.ts | 24 + .../portal/help-center/help-center-page.tsx | 253 +++++-- ...ticle-is-markdown-with-derived-counters.md | 2 +- ...ch-is-lexical-with-day-bucket-analytics.md | 42 ++ docs/help-center.md | 36 +- .../migration.sql | 29 + .../snapshot.json | 662 ++++++++++++++++++ packages/db/src/schema/help-center.ts | 148 ++++ packages/domain/src/email-outbox/content.ts | 22 +- .../handlers.test.ts | 38 + .../repository.ts | 26 + .../src/help-center-article/embedding.ts | 158 +++++ .../src/help-center-article/handlers.test.ts | 430 ++++++++++++ .../src/help-center-article/handlers.ts | 227 +++++- .../help-center-article/hybrid-search.test.ts | 278 ++++++++ .../src/help-center-article/hybrid-search.ts | 180 +++++ .../src/help-center-article/repository.ts | 419 ++++++++++- .../domain/src/help-center-article/rpcs.ts | 51 +- .../domain/src/help-center-article/schema.ts | 174 +++++ .../src/help-center-article/search.test.ts | 61 ++ .../domain/src/help-center-article/search.ts | 101 +++ packages/domain/src/site/url.ts | 93 +++ packages/domain/src/widget/api-contract.ts | 19 + packages/domain/src/widget/api-live.ts | 130 +++- packages/domain/src/widget/schema.ts | 51 ++ .../src/components/feedback-form/context.ts | 10 +- .../create-help-article-search.ts | 120 ++++ .../feedback-form/help-article-search.test.ts | 193 +++++ .../feedback-form/help-article-search.ts | 129 ++++ .../feedback-form/help-articles.tsx | 63 ++ .../src/components/feedback-form/index.tsx | 2 + .../src/components/feedback-form/provider.tsx | 36 +- packages/feedback-widget/src/lib/api.test.ts | 51 ++ packages/feedback-widget/src/lib/api.ts | 4 +- packages/feedback-widget/src/lib/requests.ts | 32 + packages/feedback-widget/src/routes/board.tsx | 1 + 40 files changed, 4754 insertions(+), 108 deletions(-) create mode 100644 apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx create mode 100644 apps/web/src/dashboard/features/help-center/components/help-center-search.tsx create mode 100644 docs/adr/0031-help-center-search-is-lexical-with-day-bucket-analytics.md create mode 100644 packages/db/src/migrations/20261119100728_help_center_search_and_analytics/migration.sql create mode 100644 packages/db/src/migrations/20261119100728_help_center_search_and_analytics/snapshot.json create mode 100644 packages/domain/src/help-center-article/embedding.ts create mode 100644 packages/domain/src/help-center-article/hybrid-search.test.ts create mode 100644 packages/domain/src/help-center-article/hybrid-search.ts create mode 100644 packages/domain/src/help-center-article/search.test.ts create mode 100644 packages/domain/src/help-center-article/search.ts create mode 100644 packages/domain/src/site/url.ts create mode 100644 packages/feedback-widget/src/components/feedback-form/create-help-article-search.ts create mode 100644 packages/feedback-widget/src/components/feedback-form/help-article-search.test.ts create mode 100644 packages/feedback-widget/src/components/feedback-form/help-article-search.ts create mode 100644 packages/feedback-widget/src/components/feedback-form/help-articles.tsx diff --git a/apps/web/src/dashboard/features/help-center/atoms.ts b/apps/web/src/dashboard/features/help-center/atoms.ts index 81e8c72e9..6f891b9f4 100644 --- a/apps/web/src/dashboard/features/help-center/atoms.ts +++ b/apps/web/src/dashboard/features/help-center/atoms.ts @@ -1,3 +1,4 @@ +import type { THelpCenterAnalyticsRange } from "@feeblo/domain/help-center-article/schema"; import * as Atom from "effect/reactivity/Atom"; import { DashboardClient, dashboardSWR } from "~/lib/atom-rpc"; @@ -61,3 +62,48 @@ export const updateHelpCenterArticleAtom = DashboardClient.mutation( export const deleteHelpCenterArticleAtom = DashboardClient.mutation( "DeleteHelpCenterArticle" ); + +export type HelpCenterAnalyticsAtomArgs = { + readonly organizationId: string; + readonly range: THelpCenterAnalyticsRange; +}; + +/** The window the panel opens on. */ +export const DEFAULT_HELP_CENTER_ANALYTICS_RANGE = + "30d" satisfies THelpCenterAnalyticsRange; + +/** + * One window of help center traffic, verdicts and searches. + * + * Keyed by the window as well as the workspace, so switching the toggle is a + * cache lookup rather than a refetch of the other two windows. It shares the + * article reactivity key because a verdict cast on the portal changes what this + * panel reports: the reader's own write should move the number they are looking + * at. + */ +export const helpCenterAnalyticsAtom = Atom.family( + (args: HelpCenterAnalyticsAtomArgs) => + DashboardClient.query("GetHelpCenterAnalytics", args, { + reactivityKeys: helpCenterArticleReactivityKeys(args.organizationId), + }).pipe(dashboardSWR("60 seconds"), Atom.setIdleTTL("5 minutes")) +); + +export type HelpCenterSearchAtomArgs = { + readonly organizationId: string; + readonly query: string; +}; + +/** + * The dashboard's article search. + * + * A family keyed by the query, like the portal's, so a keystroke is a cache + * lookup and a deleted character is an answer that already exists. It shares + * the article reactivity key: renaming an article should refresh a search that + * would have matched the old title. + */ +export const helpCenterSearchAtom = Atom.family( + (args: HelpCenterSearchAtomArgs) => + DashboardClient.query("SearchHelpCenterArticles", args, { + reactivityKeys: helpCenterArticleReactivityKeys(args.organizationId), + }).pipe(dashboardSWR("30 seconds"), Atom.setIdleTTL("1 minute")) +); diff --git a/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx b/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx new file mode 100644 index 000000000..2fd771da7 --- /dev/null +++ b/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx @@ -0,0 +1,372 @@ +import { useAtomValue } from "@effect/atom-react"; +import { + type THelpCenterAnalytics, + type THelpCenterAnalyticsRange, +} from "@feeblo/domain/help-center-article/schema"; +import { + Card, + CardDescription, + CardHeader, + CardPanel, + CardTitle, +} from "@feeblo/ui/card"; +import { Skeleton } from "@feeblo/ui/skeleton"; +import { ToggleGroup, ToggleGroupItem } from "@feeblo/ui/toggle-group"; +import { Link } from "@tanstack/react-router"; +import * as Result from "effect/reactivity/AsyncResult"; +import { useMemo, useState } from "react"; + +import { + DEFAULT_HELP_CENTER_ANALYTICS_RANGE, + type HelpCenterAnalyticsAtomArgs, + helpCenterAnalyticsAtom, +} from "../atoms"; + +const RANGE_LABELS = { + "7d": "7 days", + "30d": "30 days", + "90d": "90 days", +} as const satisfies Record; + +/** + * The help center's traffic, verdicts and searches. + * + * Three questions on one panel because they are read together: how much the + * documentation is used, whether it answered, and what it failed to answer. The + * third is the only one that names work to do, so it gets its own section and + * the page links straight to the editor. + * + * A panel rather than a route: the index is where a member arrives when they + * are thinking about the help center, and a separate analytics page would be a + * second place to check for a workspace with a dozen articles. + */ +export function HelpCenterAnalytics({ + organizationId, +}: { + organizationId: string; +}) { + const [range, setRange] = useState( + DEFAULT_HELP_CENTER_ANALYTICS_RANGE + ); + + const args = useMemo( + () => ({ organizationId, range }), + [organizationId, range] + ); + + const result = useAtomValue(helpCenterAnalyticsAtom(args)); + + return ( +
+
+
+

Documentation insights

+

+ Published and archived articles only. Reads and verdicts are counted + on the portal. +

+
+ { + const next = value[0]; + + if (next === "7d" || next === "30d" || next === "90d") { + setRange(next); + } + }} + size="sm" + value={[range]} + variant="outline" + > + 7 days + 30 days + 90 days + +
+ + {Result.builder(result) + .onInitial(() => ( +
+ + + +
+ )) + .onFailure(() => ( + + + Insights could not be loaded + + The help center itself is unaffected; only this panel failed. + + + + )) + .onSuccess((analytics) => ( + + )) + .exhaustive()} +
+ ); +} + +function AnalyticsContent({ + analytics, + organizationId, + range, +}: { + readonly analytics: THelpCenterAnalytics; + readonly organizationId: string; + readonly range: THelpCenterAnalyticsRange; +}) { + const { totals } = analytics; + + const hasTraffic = + totals.views > 0 || + totals.helpful > 0 || + totals.unhelpful > 0 || + analytics.queries.length > 0; + + if (!hasTraffic) { + return ( + + + No traffic in the last {RANGE_LABELS[range]} + + Insights appear once readers open an article on the portal. + + + + ); + } + + return ( +
+ + +
+ + +
+ + {analytics.queries.length === 0 ? null : ( + + )} +
+ ); +} + +/** Views and the two verdict counts, with what each number is a share of. */ +function TrafficTotals({ + analytics, + range, +}: { + readonly analytics: THelpCenterAnalytics; + readonly range: THelpCenterAnalyticsRange; +}) { + const { totals } = analytics; + const verdicts = totals.helpful + totals.unhelpful; + + return ( +
+ + + +
+ ); +} + +function ArticleTrafficCard({ + articles, + organizationId, +}: { + readonly articles: THelpCenterAnalytics["articles"]; + readonly organizationId: string; +}) { + return ( + + + Most-read articles + + Views, with the verdicts they collected. + + + +
    + {articles.map((article) => ( +
  • + + {article.title} + {article.status === "draft" ? ( + + draft + + ) : null} + + + {article.views} views · {article.helpful}↑ {article.unhelpful}↓ + +
  • + ))} +
+
+
+ ); +} + +function SearchGapsCard({ + emptyQueries, + queries, +}: { + readonly emptyQueries: THelpCenterAnalytics["emptyQueries"]; + readonly queries: THelpCenterAnalytics["queries"]; +}) { + return ( + + + Documentation gaps + + Searches that returned nothing. An article for one of these closes the + gap. + + + + {emptyQueries.length === 0 ? ( +

+ {queries.length === 0 + ? "Nobody has searched the help center yet." + : "Every search so far found something."} +

+ ) : ( +
    + {emptyQueries.map((query) => ( +
  • + {query.query} + + {query.emptySearches} of {query.searches} found nothing + +
  • + ))} +
+ )} +
+
+ ); +} + +function TopSearchesCard({ + queries, +}: { + readonly queries: THelpCenterAnalytics["queries"]; +}) { + return ( + + + Top searches + + What readers look for, most frequent first. + + + +
    + {queries.map((query) => ( +
  • + {query.query} + + {query.searches} + +
  • + ))} +
+
+
+ ); +} + +function Stat({ + detail, + label, + value, +}: { + readonly detail: string; + readonly label: string; + readonly value: number; +}) { + return ( + + + + {label} + + + {value.toLocaleString()} + + + {detail} + + + + ); +} + +/** + * A whole-number percentage, and a dash when there is nothing to divide. + * + * "0 of 0" is not 0% — it is no verdicts — and rendering it as 0% would read as + * "everyone hated it". + */ +function percentOf(part: number, whole: number): string { + if (whole <= 0) { + return "no verdicts"; + } + + return `${Math.round((part / whole) * 100)}%`; +} diff --git a/apps/web/src/dashboard/features/help-center/components/help-center-index.tsx b/apps/web/src/dashboard/features/help-center/components/help-center-index.tsx index 42ccfbad9..3f3f5a84a 100644 --- a/apps/web/src/dashboard/features/help-center/components/help-center-index.tsx +++ b/apps/web/src/dashboard/features/help-center/components/help-center-index.tsx @@ -38,6 +38,8 @@ import { useState } from "react"; import { usePlan } from "~/hooks/use-plan"; import { useDashboardCollections } from "~/providers/dashboard-collections-provider"; +import { HelpCenterAnalytics } from "./help-center-analytics"; +import { HelpCenterSearch } from "./help-center-search"; import { CreateHelpCenterTopicDialog, EditHelpCenterTopicDialog, @@ -199,6 +201,10 @@ export function HelpCenterIndex({ + + + + {topics.length === 0 ? ( diff --git a/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx b/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx new file mode 100644 index 000000000..fdbdd9730 --- /dev/null +++ b/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx @@ -0,0 +1,143 @@ +import { useAtomValue } from "@effect/atom-react"; +import type { THelpCenterArticleSearchHit } from "@feeblo/domain/help-center-article/schema"; +import { Badge } from "@feeblo/ui/badge"; +import { DebouncedInputGroupInput } from "@feeblo/ui/debounced-input"; +import { + InputGroup, + InputGroupAddon, + InputGroupText, +} from "@feeblo/ui/input-group"; +import { Skeleton } from "@feeblo/ui/skeleton"; +import { Search01Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; +import { Link } from "@tanstack/react-router"; +import * as Result from "effect/reactivity/AsyncResult"; +import { useMemo, useState } from "react"; + +import { helpCenterSearchAtom, type HelpCenterSearchAtomArgs } from "../atoms"; + +/** Shorter than this is "typing", not a search. Same floor the portal uses. */ +const MIN_SEARCH_LENGTH = 2; + +/** + * The dashboard's article search. + * + * It reads the dashboard operation rather than the public one because an editor + * searching for a page they half-wrote should find it, and because counting an + * editor's query as a reader's search would turn the portal's "top searches" + * into an inventory of the team's drafts. + * + * The results sit above the section list instead of replacing it: an editor who + * clears the box is back where they were, and the list behind the results is + * the context they were reading when they started typing. + */ +export function HelpCenterSearch({ + organizationId, +}: { + organizationId: string; +}) { + const [query, setQuery] = useState(""); + const trimmed = query.trim(); + + return ( +
+ + + + + + + + + {trimmed.length >= MIN_SEARCH_LENGTH ? ( + + ) : null} +
+ ); +} + +function SearchResults({ + organizationId, + query, +}: { + organizationId: string; + query: string; +}) { + const args = useMemo( + () => ({ organizationId, query }), + [organizationId, query] + ); + + const result = useAtomValue(helpCenterSearchAtom(args)); + const hits = Result.getOrElse(result, () => []); + + if (Result.isInitial(result)) { + return ( +
+ + +
+ ); + } + + if (hits.length === 0) { + return ( +

+ No article matches “{query}”. +

+ ); + } + + return ( +
    + {hits.map((hit) => ( + + ))} +
+ ); +} + +function SearchResultRow({ + hit, + organizationId, +}: { + hit: THelpCenterArticleSearchHit; + organizationId: string; +}) { + return ( +
  • + +
    +
    + {hit.title} + {hit.status === "draft" ? ( + Draft + ) : null} + {hit.status === "archived" ? ( + Archived + ) : null} +
    +

    + {hit.topicName} +

    +
    + +
  • + ); +} diff --git a/apps/web/src/portal/help-center/atoms.ts b/apps/web/src/portal/help-center/atoms.ts index 94c884a56..c1351c01f 100644 --- a/apps/web/src/portal/help-center/atoms.ts +++ b/apps/web/src/portal/help-center/atoms.ts @@ -45,6 +45,30 @@ export const helpCenterArticlesPublicAtom = Atom.family( ).pipe(dashboardSWR("30 seconds"), Atom.setIdleTTL("5 minutes")) ); +export type HelpCenterSearchAtomArgs = { + readonly organizationId: string; + readonly query: string; +}; + +/** + * The portal's search. + * + * A family keyed by the query, so typing is a series of cache lookups rather + * than a series of writes to one entry: a reader who deletes a character and + * retypes it gets the earlier answer back instead of a spinner. The idle TTL is + * shorter than the other reads because the key includes text that only exists + * for as long as somebody is typing it. + * + * Callers must memoize `args` — `Atom.family` keys on identity, so a fresh + * object literal per render is a fresh atom and a fresh request per render. + */ +export const helpCenterSearchPublicAtom = Atom.family( + (args: HelpCenterSearchAtomArgs) => + DashboardClient.query("SearchHelpCenterArticlesPublic", args, { + reactivityKeys: helpCenterPublicReactivityKeys(args.organizationId), + }).pipe(dashboardSWR("30 seconds"), Atom.setIdleTTL("1 minute")) +); + export type HelpCenterArticlePublicAtomArgs = { readonly organizationId: string; readonly slug: string; diff --git a/apps/web/src/portal/help-center/help-center-page.tsx b/apps/web/src/portal/help-center/help-center-page.tsx index c4f6d6bcc..00a72457a 100644 --- a/apps/web/src/portal/help-center/help-center-page.tsx +++ b/apps/web/src/portal/help-center/help-center-page.tsx @@ -2,6 +2,7 @@ import { useAtomValue } from "@effect/atom-react"; import { isFeatureEnabled } from "@feeblo/domain/feature-flags"; import { useSite } from "@feeblo/public-feature-board/providers/site-provider"; import { Badge } from "@feeblo/ui/badge"; +import { DebouncedInputGroupInput } from "@feeblo/ui/debounced-input"; import { Empty, EmptyDescription, @@ -9,16 +10,37 @@ import { EmptyTitle, } from "@feeblo/ui/empty"; import { TablerIcon } from "@feeblo/ui/icon-picker"; +import { + InputGroup, + InputGroupAddon, + InputGroupText, +} from "@feeblo/ui/input-group"; import { Skeleton } from "@feeblo/ui/skeleton"; import { helpCenterTopicIconName } from "@feeblo/web-shared/icons/defaults"; +import { Search01Icon } from "@hugeicons/core-free-icons"; +import { HugeiconsIcon } from "@hugeicons/react"; import { Link } from "@tanstack/react-router"; import * as Result from "effect/reactivity/AsyncResult"; +import { useMemo, useState } from "react"; import { helpCenterArticlesPublicAtom, + helpCenterSearchPublicAtom, helpCenterTopicsPublicAtom, + type HelpCenterSearchAtomArgs, } from "./atoms"; +/** + * How short a query has to be before it is not a search. + * + * One character matches most of a help center, so asking the server for it + * spends a round trip and a rate-limit token to return everything. The number + * is shared with the widget's deflection for the same reason it exists: both + * are the difference between "they are typing" and "they are looking for + * something". + */ +const MIN_SEARCH_LENGTH = 2; + /** * The portal's help center index. * @@ -27,12 +49,21 @@ import { * projection that shared the dashboard's list would have to filter drafts out * on the client — one forgotten filter away from leaking unreleased * documentation. + * + * The search lives on this page rather than behind its own route because the + * index is already the place a reader arrives at; a `/s/help/search?q=` URL + * would be a second page whose only content is a shorter version of this one. + * Results replace the topic list rather than filtering it: a topic whose + * articles all rank low is not useful context beside the ranked answer. */ export function HelpCenterPage() { const site = useSite(); const organizationId = site.organizationId; const enabled = isFeatureEnabled("HELP_CENTER"); + const [query, setQuery] = useState(""); + const trimmedQuery = query.trim(); + const topicsResult = useAtomValue(helpCenterTopicsPublicAtom(organizationId)); const articlesResult = useAtomValue( @@ -65,13 +96,161 @@ export function HelpCenterPage() { ); } - if (topics.length === 0) { + return ( +
    +
    +

    Help center

    +

    + Guides and answers for {site.name}. +

    +
    + +
    + + + + + + + +
    + + {trimmedQuery.length >= MIN_SEARCH_LENGTH ? ( + + ) : topics.length === 0 ? ( + + + No documentation yet + + Nothing has been published to the help center. + + + + ) : ( +
    + {topics.map((topic) => { + const topicArticles = articlesByTopic.get(topic.id) ?? []; + + return ( +
    +
    +

    + + {topic.name} +

    + {topic.description === null ? null : ( +

    + {topic.description} +

    + )} +
    +
      + {topicArticles.map((article) => ( + + ))} +
    +
    + ); + })} +
    + )} +
    + ); +} + +/** One article in a topic's list. */ +function ArticleRow({ + article, +}: { + article: { + readonly id: string; + readonly slug: string; + readonly title: string; + readonly excerpt: string; + readonly isArchived: boolean; + }; +}) { + return ( +
  • + +
    +
    + {article.title} + {article.isArchived ? ( + Out of date + ) : null} +
    + {article.excerpt.length === 0 ? null : ( +

    + {article.excerpt} +

    + )} +
    + +
  • + ); +} + +/** + * The ranked results for one query. + * + * A component of its own so the atom read only exists while there is a query to + * read: hooks cannot be conditional, and mounting this is the condition. The + * args are memoized because `helpCenterSearchPublicAtom` is a family keyed on + * identity. + */ +function HelpCenterSearchResults({ + organizationId, + query, +}: { + organizationId: string; + query: string; +}) { + const args = useMemo( + () => ({ organizationId, query }), + [organizationId, query] + ); + + const result = useAtomValue(helpCenterSearchPublicAtom(args)); + + const hits = Result.getOrElse(result, () => []); + + if (Result.isInitial(result)) { + return ( +
    + + + +
    + ); + } + + if (hits.length === 0) { return ( - No documentation yet + No matching articles - Nothing has been published to the help center. + Nothing in this help center matches “{query}”. Try a different + search, or share the question with the team. @@ -79,65 +258,15 @@ export function HelpCenterPage() { } return ( -
    -
    -

    Help center

    -

    - Guides and answers for {site.name}. -

    -
    - -
    - {topics.map((topic) => { - const topicArticles = articlesByTopic.get(topic.id) ?? []; - - return ( -
    -
    -

    - - {topic.name} -

    - {topic.description === null ? null : ( -

    - {topic.description} -

    - )} -
    -
      - {topicArticles.map((article) => ( -
    • - -
      -
      - - {article.title} - - {article.isArchived ? ( - Out of date - ) : null} -
      - {article.excerpt.length === 0 ? null : ( -

      - {article.excerpt} -

      - )} -
      - -
    • - ))} -
    -
    - ); - })} -
    +
    +

    + {hits.length === 1 ? "1 article" : `${hits.length} articles`} +

    +
      + {hits.map((hit) => ( + + ))} +
    ); } diff --git a/docs/adr/0029-a-help-center-article-is-markdown-with-derived-counters.md b/docs/adr/0029-a-help-center-article-is-markdown-with-derived-counters.md index 0b469c3ee..a042c639d 100644 --- a/docs/adr/0029-a-help-center-article-is-markdown-with-derived-counters.md +++ b/docs/adr/0029-a-help-center-article-is-markdown-with-derived-counters.md @@ -34,6 +34,6 @@ The cost is stated rather than hidden: the dashboard's editor is a Markdown text - **The list row is a summary and does not carry the body.** The editor fetches one article by id and writes through the create/update operations instead of through the collection, because a collection cannot round-trip a field its rows do not hold. The dashboard's collection therefore has `onDelete` only. - **Copy as Markdown, `llms.txt` and an MCP projection are routes over this data, not schema changes.** The first is free because of the body decision; the second follows `apps/web/src/routes/s/llms[.]txt.ts`; the third is ADR 0007/0008's operation registry. - **The GitHub-sync shape is reserved, not built.** `help_center_article.source` (`EDITOR | API | GITHUB | IMPORT`) exists from the first migration so a sync write is distinguishable from a human edit at the row level — the one thing a column cannot answer retroactively. The link table and the sync rule are additive and belong to that feature; they are _not_ `github_sync_rule`, which maps issue state to post status and has no analogue in a file sync. -- **Embedding columns ship unused.** `embedding`/`embedding_model`/`embedded_at` follow `post`'s triple with the same all-null-or-all-set check, at the shared 1536 dimensions, so semantic search is an `ALTER`-free follow-up. There is no `tsvector` anywhere in this codebase today and this ADR does not add one. +- **Embedding columns ship unused.** `embedding`/`embedding_model`/`embedded_at` follow `post`'s triple with the same all-null-or-all-set check, at the shared 1536 dimensions, so semantic search is an `ALTER`-free follow-up. This ADR did not add a `tsvector`; ADR 0031 adds the lexical index beside them, so the two rankers are complementary rather than competing. - **The `site` read grew a field and the write grew its partner.** `help_center_visibility` defaults to `PUBLIC`, and the settings toggle that flips it is part of the dashboard surface. - **The whole feature ships behind `HELP_CENTER`, default off.** ADR 0027 and the audit log both ship a surface ahead of its switch; this one gates every operation rather than one writer, because the tables are new and empty, so a read is as much a launch as a write. `helpCenterFeatureGate` is a public policy that fails with `PolicyDeniedError` — already in every operation's error union — and is applied last in each pipe so it refuses before a row or a rate-limit token is spent. Because no article can exist while the gate is closed, the `PUBLIC` default publishes nothing on the day the flag flips; turning it on is one line and no migration. diff --git a/docs/adr/0031-help-center-search-is-lexical-with-day-bucket-analytics.md b/docs/adr/0031-help-center-search-is-lexical-with-day-bucket-analytics.md new file mode 100644 index 000000000..cbd444b94 --- /dev/null +++ b/docs/adr/0031-help-center-search-is-lexical-with-day-bucket-analytics.md @@ -0,0 +1,42 @@ +# ADR 0031: Help-center search is lexical-first, and its analytics are day buckets + +## Decision + +Four choices, all additive to ADR 0029: + +- **Lexical first, semantic as a rescue.** `help_center_article.search_vector` is a generated `tsvector` (`title` weight `A`, `excerpt` `B`, `content` `C`, `english` configuration), indexed with GIN, ranked by `ts_rank_cd`. Articles also carry an embedding, and the semantic pass runs **only when the lexical pass returns fewer than three pages**; its hits are appended behind the lexical ones. +- **The query is built by one module and nothing else may produce it.** `help-center-article/search.ts` normalizes the text, keeps letters and digits, drops one-character terms and duplicates, caps the term count, and emits prefix branches. The repository binds that string and never a caller's. +- **A write schedules its own embedding, detached and best-effort.** The embedding columns are an index, not content: a model that is unreachable logs and leaves them null, and the save still succeeds. +- **Search and article traffic are counted in day buckets, written on the read path, and advisory.** `help_center_search_daily` and `help_center_article_daily` are keyed by `(organization, day, …)` and `(article, day)`; a failed counter write is logged and ignored, and never fails the read that produced it. + +## Why + +**Why a generated column.** The three inputs are columns on the same row, so Postgres can always compute the vector and no caller can forget to. A trigger has to be replayed by restores and bulk inserts and can silently miss a path; a write-path update is one more statement every future write has to remember. `to_tsvector('english'::regconfig, …)` is immutable because the configuration is a literal, which is what makes the column admissible at DDL time — a parameterized config is rejected by Postgres, and that constraint is the one thing to know before changing the expression. The cost is a second copy of the body in vector form; a vector is far smaller than the Markdown it indexes. + +**Why lexical first.** Lexical search is deterministic, explainable — an editor can see why a page ranked — costs nothing per query, and works in PGlite, which is what the tests run on. It is also the answer to the common query, which is a word the page actually contains. + +**Why the semantic pass is a rescue and not a ranker.** Two reasons, and neither is a preference. _Cost_: the search box is public and unauthenticated, so embedding every query would put a metered model call on a keystroke; rescuing only a thin result set spends it on the searches that were about to fail. _Ranking_: `ts_rank_cd` is unbounded and depends on the query's term count, cosine distance is bounded and model-specific, and any arithmetic combining them would be a constant nobody could defend with evidence. Exact-word evidence keeps its precedence, near-miss evidence fills the tail, and a learned ranker is a later decision with data behind it. Three is the threshold because it is the number of results a reader scans before deciding the page is useless: above it the append adds noise to a page that already answered, below it the search looks broken. + +**Why the embedding is written on save rather than backfilled.** The three inputs are on the row and the write paths are the only places an article changes, so a save can schedule the refresh and a detached job can do it. A backfill sweep would need its own runner, its own progress records and its own outage handling for a table whose unembedded rows degrade to "lexical only" — which is the shipped behaviour, not a broken state. The cost is stated rather than hidden: an article written before this shipped has no vector until it is next saved, and switching `EMBEDDING_MODEL` strands the old rows because the semantic query filters on the model. + +**Why `strict` disables the semantic pass.** A strict search exists for the widget's deflection, whose whole purpose is to show a page that shares the visitor's words. A semantic neighbour with no word overlap is exactly the suggestion that looks like a mistake beside a feedback form. Disabling it there also means a deployment with no embedding key still answers the deflection. + +**Why prefix branches and not `websearch_to_tsquery`.** `websearch_to_tsquery` tolerates arbitrary input but has no prefix operator, so it cannot answer a search box: a reader who has typed "export" would see nothing until the word is complete. `to_tsquery` does have `:*`, and it throws on malformed input — which is why the tokenizer is the only producer of the string. Everything that `to_tsquery` treats as syntax (`&`, `|`, `!`, parentheses, `:`, `'`) is a separator before it is an operator, so a reader's punctuation cannot reach the parser. Precision is the ranking's job: `ts_rank_cd` rewards coverage over a single mention, and the weights put a title match above a body match. + +**Why the same query serves both surfaces with a flag.** The dashboard must find a draft its editor is writing; the portal must never reveal one. Making the readable-state filter a **parameter of the repository call** rather than something each caller remembers is the difference between a search box and a leak, and it mirrors how every other public read is scoped. + +**Why advisory writes.** Analytics is not the product. A search that returns results and fails to count them is a working error; a search that fails because a counter could not be written is a broken one. The same reasoning applies to a verdict: the vote lands first, and its bucket is best-effort. Wrapping the read and its counter in one transaction would take a write lock on a read path, which is a worse trade than a lost count. + +**Why day buckets rather than raw events.** The questions are bounded — "which articles lately", "what do readers search for", "what do they search for and not find" — and a bucket answers each with one grouped query. Raw search events would be a text log with a retention problem and a privacy surface (a reader's search terms attached to a timestamp), and the first thing anyone would build on top of them is this table. A bucket is its own natural key, so there is no surrogate id to write the same fact twice under. + +**Why verdict buckets count verdicts cast.** A flip is a second judgment, not a correction of the first, and crediting today's bucket with the flip while leaving the earlier day intact is what makes the daily series mean "how readers judged pages this week". The standing distribution — the number on the article — is the feedback table's own `count(*)`, which is exact because those rows are never pruned. Storing a second, slower-moving copy of it here would create two numbers that disagree. + +## Consequences + +- **Search is English-stemmed for every workspace.** The text-search configuration is part of the index, so per-locale or per-workspace configurations are a different column and a different decision; a non-English help center searches by exact words with English stopwords dropped. +- **`help_center_search_daily` is the first table here that would need retention**, because its key includes the reader's text. Growth is bounded by distinct queries per workspace per day, which is human-entered text; if that assumption stops holding, a prune by `day` is the fix and the aggregates it feeds are already windowed. +- **The widget's deflection is public, unauthenticated and plan-free.** It answers `{ articles, enabled }` — `enabled: false` for a deployment with the flag off or a workspace with no public help center — because a distinguishable failure would confirm which workspaces exist. It is on every plan: the ceiling on articles already prices the feature, and a deflection that only some plans get would be a worse help center rather than a smaller one. +- **One portal-URL builder.** `site/url.ts` owns `` or `.`, and both the email outbox and the widget's article links go through it, because the widget iframe is embedded on the customer's own origin and a relative link would resolve against the host page. +- **The semantic half costs a model call per rescued search.** It is bounded by the thin-result rule, the public rate limiter, the 120-character query cap, and `strict` refusing to embed at all; a deployment with no `EMBEDDING_API_KEY` never runs it. +- **An article's body is truncated to 6,000 characters for its embedding.** The head of a page is what identifies it, and an unbounded body is an unbounded provider cost per save. +- **ADR 0029's "no `tsvector` anywhere in this codebase" and "the embedding columns ship unused" no longer hold.** This ADR adds the lexical index beside the vector one, and a writer for the vector. diff --git a/docs/help-center.md b/docs/help-center.md index 047d5f97f..6ecf06cb8 100644 --- a/docs/help-center.md +++ b/docs/help-center.md @@ -13,6 +13,8 @@ This document is the design record for what shipped. The decisions themselves an | `help_center_article_view_event` | One row per read, with the client's idempotency key | | `help_center_article_view_total` | The all-time counter that outlives event retention | | `help_center_article_feedback` | One reader's "was this helpful?" verdict, flippable in place | +| `help_center_article_daily` | One article's views and verdicts, bucketed by UTC day | +| `help_center_search_daily` | What readers searched for, and how often it found nothing, bucketed by UTC day | | `site.help_center_visibility` | Whether the portal serves the help center at all | Every table carries `organization_id` with `onDelete: cascade`, and slugs are unique **per workspace** (`help_center_article_organizationId_slug_uidx`). The article's topic reference is a composite foreign key on `(topic_id, organization_id)` with `ON DELETE RESTRICT`, so an article cannot name another workspace's section and deleting a section cannot silently delete documentation. @@ -53,6 +55,37 @@ A flip is an `UPDATE` of the same row. Flipping to helpful clears `reason`, beca **Portal routes.** `/s/help` and `/s/help/articles/$articleSlug`, which render the stored Markdown, offer "Copy as Markdown", fire the view beacon, and collect the verdict. +## Search + +Lexical first, semantic as a rescue. Two signals, one program (`help-center-article/hybrid-search.ts`), three surfaces. + +**Lexical.** `help_center_article.search_vector` is a **generated** `tsvector` column — `setweight(title,'A') || setweight(excerpt,'B') || setweight(content,'C')` against `english` — with a GIN index. Generated rather than a trigger or a write-path update because the three inputs live on the same row, so Postgres can always compute the value and no caller can forget to; that also means no backfill and no drift from a restored dump or a bulk insert. + +The query is built in `help-center-article/search.ts`, which is the only producer of the string `to_tsquery` ever parses. It lowercases, collapses whitespace, keeps runs of letters and digits, drops one-character terms and duplicates, caps the term count, and ORs prefix branches (`export:* | csv:*`). Nothing a reader types can reach the parser as syntax: `&`, `|`, `!`, `(`, `)`, `:` and `'` are separators before they are operators. Ranking is `ts_rank_cd`, so coverage beats a single passing mention, and the three weights decide title over summary over body. + +**Semantic.** Articles carry an embedding (title + the first 6,000 characters of the body) written after every readable create or update, detached and best-effort — a model that is unreachable leaves the columns null and never fails the save. `searchByVector` orders by cosine distance and filters on `embedding_model` as well as `embedding is not null`: a vector from a different model lives in a different space, and comparing across models returns confident nonsense rather than an error. + +**The rule that joins them.** The semantic pass runs only when the lexical pass returned fewer than three pages, and its hits are appended behind the lexical ones. It is a rescue, not a ranker: `ts_rank_cd` is unbounded and depends on the query's term count, cosine distance is bounded and model-specific, and any arithmetic combining them would be a constant nobody could defend. The rule also bounds cost — a public search box must not spend a model call on the common case, and the common case is a query that matches. + +**The surfaces.** One difference between the dashboard and the portal: the dashboard includes every lifecycle state — an editor searching for the draft they half-wrote should find it — and the public one applies the same readable-state filter every other portal read applies. Getting that wrong publishes a draft through a search box, so the flag is a parameter of the repository call rather than something a caller can forget. + +The widget's deflection passes `strict`, which restricts the lexical match to the title or summary **and disables the semantic pass**. A suggestion beside a feedback form is only worth showing if it names the thing being asked about; a semantic neighbour with no word overlap is exactly the suggestion that looks like a mistake there. It is also why the widget needs no embedding configuration. + +## Analytics + +The panel on the dashboard's help center index reads two day-bucketed tables over a 7, 30 or 90 day window: + +- `help_center_article_daily` counts views and verdicts per article per UTC day. The view counter moves inside the same transaction as the all-time counter, so the two cannot disagree. The verdict counters count verdicts **cast** that day, not the standing distribution — a flip adds an unhelpful to today and leaves the earlier helpful where it was. The standing distribution is the feedback table's own `count(*)`, and a second, slower-moving copy of it here would be a number that disagrees with the article page. +- `help_center_search_daily` counts searches and how many of them returned nothing, keyed on the normalized query. Both numbers live in one row so "what people look for" and "what we do not document" cannot disagree about how many searches there were. + +Writes to either table are **advisory**: a failure is logged and swallowed, because a reader's search must not fail because an analytics write did, and a verdict must not be lost because its bucket was. The search write is the only public read that writes at all. + +## Widget deflection + +`POST /api/widget/v1/help-articles` answers `{ articles, enabled }` for the title being typed in the feedback composer. `enabled: false` covers every case where the surface does not exist — the deployment flag is off, the workspace hides its help center, or it has none — and is a value rather than an error on purpose: this endpoint is unauthenticated and names the workspace in its body, so a distinguishable failure would confirm which workspaces exist. The widget latches the flag and stops asking. + +Each hit carries an absolute `url` built by `site/url.ts`, because the iframe is embedded on the customer's own origin and a relative link would resolve against the host page. That module is the single rule for `` or `.`; the email outbox's portal links go through it too. + ## Rollout The feature ships behind `HELP_CENTER` in `packages/domain/src/feature-flags.ts`, default **off**. Unlike the audit log — where only the recorder is gated, since a read of an empty table is harmless — every operation is gated, because a read is as much a launch as a write when the tables are new and empty. @@ -70,7 +103,8 @@ Turning it on is therefore a single-line change with no migration: no article ca ## Not built -- **Semantic search.** The embedding columns exist at the shared 1536 dimensions and nothing writes them. There is no `tsvector` anywhere in this codebase; adding one is its own decision. +- **Fused ranking.** The two halves are ordered by precedence, not by a shared score. A learned ranker or a weighted fusion needs data that does not exist yet. +- **Re-embedding on a model switch.** Changing `EMBEDDING_MODEL` stops the semantic pass matching old rows until they are written again; there is no backfill sweep. Lexical search is unaffected, which is why the feature degrades quietly rather than breaking. - **GitHub sync.** `help_center_article.source` reserves the provenance value. The link table, the sync rule, and the file front-matter contract belong to that feature. - **Content translation.** The repo's i18n is chrome-only (Paraglide). A per-locale body is a product surface, not a column. - **`llms.txt` for the help center.** The route precedent is `apps/web/src/routes/s/llms[.]txt.ts`; it reads published articles through the same public projection. diff --git a/packages/db/src/migrations/20261119100728_help_center_search_and_analytics/migration.sql b/packages/db/src/migrations/20261119100728_help_center_search_and_analytics/migration.sql new file mode 100644 index 000000000..2c4b938d6 --- /dev/null +++ b/packages/db/src/migrations/20261119100728_help_center_search_and_analytics/migration.sql @@ -0,0 +1,29 @@ +CREATE TABLE "help_center_article_daily" ( + "organization_id" text NOT NULL, + "article_id" text, + "day" date, + "views" integer DEFAULT 0 NOT NULL, + "helpful_verdicts" integer DEFAULT 0 NOT NULL, + "unhelpful_verdicts" integer DEFAULT 0 NOT NULL, + CONSTRAINT "help_center_article_daily_pkey" PRIMARY KEY("article_id","day"), + CONSTRAINT "help_center_article_daily_counts_chk" CHECK ("views" >= 0 and "helpful_verdicts" >= 0 and "unhelpful_verdicts" >= 0) +); +--> statement-breakpoint +CREATE TABLE "help_center_search_daily" ( + "organization_id" text, + "day" date, + "query" text, + "searches" integer DEFAULT 0 NOT NULL, + "empty_searches" integer DEFAULT 0 NOT NULL, + CONSTRAINT "help_center_search_daily_pkey" PRIMARY KEY("organization_id","day","query"), + CONSTRAINT "help_center_search_daily_counts_chk" CHECK ("searches" >= 0 and "empty_searches" >= 0 and "empty_searches" <= "searches") +); +--> statement-breakpoint +ALTER TABLE "help_center_article" ADD COLUMN "search_vector" tsvector GENERATED ALWAYS AS (setweight(to_tsvector('english'::regconfig, title), 'A') || setweight(to_tsvector('english'::regconfig, excerpt), 'B') || setweight(to_tsvector('english'::regconfig, content), 'C')) STORED;--> statement-breakpoint +CREATE INDEX "help_center_article_daily_organizationId_day_idx" ON "help_center_article_daily" ("organization_id","day");--> statement-breakpoint +CREATE INDEX "help_center_article_search_vector_idx" ON "help_center_article" USING gin ("search_vector");--> statement-breakpoint +CREATE INDEX "help_center_search_daily_organizationId_day_searches_idx" ON "help_center_search_daily" ("organization_id","day","searches" desc);--> statement-breakpoint +ALTER TABLE "help_center_article_daily" ADD CONSTRAINT "help_center_article_daily_organization_id_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organization"("id") ON DELETE CASCADE;--> statement-breakpoint +ALTER TABLE "help_center_article_daily" ADD CONSTRAINT "help_center_article_daily_pgDlPfJgnRiS_fkey" FOREIGN KEY ("article_id") REFERENCES "help_center_article"("id") ON DELETE CASCADE;--> statement-breakpoint +ALTER TABLE "help_center_article_daily" ADD CONSTRAINT "help_center_article_daily_article_organization_fk" FOREIGN KEY ("article_id","organization_id") REFERENCES "help_center_article"("id","organization_id") ON DELETE CASCADE;--> statement-breakpoint +ALTER TABLE "help_center_search_daily" ADD CONSTRAINT "help_center_search_daily_organization_id_organization_id_fkey" FOREIGN KEY ("organization_id") REFERENCES "organization"("id") ON DELETE CASCADE; \ No newline at end of file diff --git a/packages/db/src/migrations/20261119100728_help_center_search_and_analytics/snapshot.json b/packages/db/src/migrations/20261119100728_help_center_search_and_analytics/snapshot.json new file mode 100644 index 000000000..ee540e3f0 --- /dev/null +++ b/packages/db/src/migrations/20261119100728_help_center_search_and_analytics/snapshot.json @@ -0,0 +1,662 @@ +{ + "type": "diff", + "version": "9", + "dialect": "postgres", + "id": "4e1f4cd9-0ecc-47e1-bf8b-143d2a630181", + "prevIds": [ + "863854fe-d0d7-47c3-8467-1f3e8a03d65a" + ], + "footprints": [ + { + "statementType": "create_table", + "statementTarget": [ + "table", + "public", + "help_center_article_daily" + ], + "writes": [ + "s|tbl|public.help_center_article_daily", + "n|tbl|public.help_center_article_daily", + "e|tbl|public.help_center_article_daily", + "n|rel|public.help_center_article_daily", + "n|type|public.help_center_article_daily", + "n|rel|public.help_center_article_daily_pkey" + ], + "reads": [ + "n|sch|public" + ] + }, + { + "statementType": "create_table", + "statementTarget": [ + "table", + "public", + "help_center_search_daily" + ], + "writes": [ + "s|tbl|public.help_center_search_daily", + "n|tbl|public.help_center_search_daily", + "e|tbl|public.help_center_search_daily", + "n|rel|public.help_center_search_daily", + "n|type|public.help_center_search_daily", + "n|rel|public.help_center_search_daily_pkey" + ], + "reads": [ + "n|sch|public" + ] + }, + { + "statementType": "add_column", + "statementTarget": [ + "column", + "public", + "help_center_article", + "search_vector" + ], + "writes": [ + "s|col|public.help_center_article.search_vector", + "n|col|public.help_center_article.search_vector", + "e|col|public.help_center_article.search_vector" + ], + "reads": [ + "n|tbl|public.help_center_article", + "n|sch|public" + ] + }, + { + "statementType": "create_index", + "statementTarget": [ + "index", + "public", + "help_center_article_daily", + "help_center_article_daily_organizationId_day_idx" + ], + "writes": [ + "s|idx|public.help_center_article_daily_organizationId_day_idx", + "n|idx|public.help_center_article_daily_organizationId_day_idx", + "e|idx|public.help_center_article_daily_organizationId_day_idx", + "n|rel|public.help_center_article_daily_organizationId_day_idx" + ], + "reads": [ + "n|tbl|public.help_center_article_daily", + "n|sch|public", + "n|col|public.help_center_article_daily.organization_id", + "n|col|public.help_center_article_daily.day" + ] + }, + { + "statementType": "create_index", + "statementTarget": [ + "index", + "public", + "help_center_article", + "help_center_article_search_vector_idx" + ], + "writes": [ + "s|idx|public.help_center_article_search_vector_idx", + "n|idx|public.help_center_article_search_vector_idx", + "e|idx|public.help_center_article_search_vector_idx", + "n|rel|public.help_center_article_search_vector_idx" + ], + "reads": [ + "n|tbl|public.help_center_article", + "n|sch|public", + "n|col|public.help_center_article.search_vector" + ] + }, + { + "statementType": "create_index", + "statementTarget": [ + "index", + "public", + "help_center_search_daily", + "help_center_search_daily_organizationId_day_searches_idx" + ], + "writes": [ + "s|idx|public.help_center_search_daily_organizationId_day_searches_idx", + "n|idx|public.help_center_search_daily_organizationId_day_searches_idx", + "e|idx|public.help_center_search_daily_organizationId_day_searches_idx", + "n|rel|public.help_center_search_daily_organizationId_day_searches_idx" + ], + "reads": [ + "n|tbl|public.help_center_search_daily", + "n|sch|public", + "n|col|public.help_center_search_daily.organization_id", + "n|col|public.help_center_search_daily.day" + ] + }, + { + "statementType": "create_fk", + "statementTarget": [ + "table", + "public", + "help_center_article_daily" + ], + "writes": [ + "s|fk|public.help_center_article_daily.help_center_article_daily_organization_id_organization_id_fkey", + "n|fk|public.help_center_article_daily.help_center_article_daily_organization_id_organization_id_fkey", + "e|fk|public.help_center_article_daily.help_center_article_daily_organization_id_organization_id_fkey", + "n|con|public.help_center_article_daily.help_center_article_daily_organization_id_organization_id_fkey" + ], + "reads": [ + "n|tbl|public.help_center_article_daily", + "n|sch|public", + "n|tbl|public.organization", + "n|sch|public", + "n|col|public.help_center_article_daily.organization_id", + "n|col|public.organization.id", + "e|uniqueness|public.organization.id" + ] + }, + { + "statementType": "create_fk", + "statementTarget": [ + "table", + "public", + "help_center_article_daily" + ], + "writes": [ + "s|fk|public.help_center_article_daily.help_center_article_daily_pgDlPfJgnRiS_fkey", + "n|fk|public.help_center_article_daily.help_center_article_daily_pgDlPfJgnRiS_fkey", + "e|fk|public.help_center_article_daily.help_center_article_daily_pgDlPfJgnRiS_fkey", + "n|con|public.help_center_article_daily.help_center_article_daily_pgDlPfJgnRiS_fkey" + ], + "reads": [ + "n|tbl|public.help_center_article_daily", + "n|sch|public", + "n|tbl|public.help_center_article", + "n|sch|public", + "n|col|public.help_center_article_daily.article_id", + "n|col|public.help_center_article.id", + "e|uniqueness|public.help_center_article.id" + ] + }, + { + "statementType": "create_fk", + "statementTarget": [ + "table", + "public", + "help_center_article_daily" + ], + "writes": [ + "s|fk|public.help_center_article_daily.help_center_article_daily_article_organization_fk", + "n|fk|public.help_center_article_daily.help_center_article_daily_article_organization_fk", + "e|fk|public.help_center_article_daily.help_center_article_daily_article_organization_fk", + "n|con|public.help_center_article_daily.help_center_article_daily_article_organization_fk" + ], + "reads": [ + "n|tbl|public.help_center_article_daily", + "n|sch|public", + "n|tbl|public.help_center_article", + "n|sch|public", + "n|col|public.help_center_article_daily.article_id", + "n|col|public.help_center_article_daily.organization_id", + "n|col|public.help_center_article.id", + "n|col|public.help_center_article.organization_id", + "e|uniqueness|public.help_center_article.id.organization_id" + ] + }, + { + "statementType": "create_fk", + "statementTarget": [ + "table", + "public", + "help_center_search_daily" + ], + "writes": [ + "s|fk|public.help_center_search_daily.help_center_search_daily_organization_id_organization_id_fkey", + "n|fk|public.help_center_search_daily.help_center_search_daily_organization_id_organization_id_fkey", + "e|fk|public.help_center_search_daily.help_center_search_daily_organization_id_organization_id_fkey", + "n|con|public.help_center_search_daily.help_center_search_daily_organization_id_organization_id_fkey" + ], + "reads": [ + "n|tbl|public.help_center_search_daily", + "n|sch|public", + "n|tbl|public.organization", + "n|sch|public", + "n|col|public.help_center_search_daily.organization_id", + "n|col|public.organization.id", + "e|uniqueness|public.organization.id" + ] + } + ], + "deltas": [ + { + "op": "create", + "row": { + "schema": "public", + "name": "help_center_article_daily", + "isRlsEnabled": false, + "entityType": "tables" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "name": "help_center_search_daily", + "isRlsEnabled": false, + "entityType": "tables" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "organization_id", + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "article_id", + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "day", + "type": "date", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "views", + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "helpful_verdicts", + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "unhelpful_verdicts", + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article", + "name": "search_vector", + "type": "tsvector", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": { + "as": "setweight(to_tsvector('english'::regconfig, title), 'A') || setweight(to_tsvector('english'::regconfig, excerpt), 'B') || setweight(to_tsvector('english'::regconfig, content), 'C')", + "type": "stored" + }, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_search_daily", + "name": "organization_id", + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_search_daily", + "name": "day", + "type": "date", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_search_daily", + "name": "query", + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_search_daily", + "name": "searches", + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_search_daily", + "name": "empty_searches", + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "entityType": "columns" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "help_center_article_daily_organizationId_day_idx", + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "day", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "entityType": "indexes" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article", + "name": "help_center_article_search_vector_idx", + "nameExplicit": true, + "columns": [ + { + "value": "search_vector", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "entityType": "indexes" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_search_daily", + "name": "help_center_search_daily_organizationId_day_searches_idx", + "nameExplicit": true, + "columns": [ + { + "value": "organization_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "day", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "\"searches\" desc", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "entityType": "indexes" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "help_center_article_daily_organization_id_organization_id_fkey", + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "entityType": "fks" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "help_center_article_daily_pgDlPfJgnRiS_fkey", + "nameExplicit": false, + "columns": [ + "article_id" + ], + "schemaTo": "public", + "tableTo": "help_center_article", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "entityType": "fks" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "help_center_article_daily_article_organization_fk", + "nameExplicit": true, + "columns": [ + "article_id", + "organization_id" + ], + "schemaTo": "public", + "tableTo": "help_center_article", + "columnsTo": [ + "id", + "organization_id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "entityType": "fks" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_search_daily", + "name": "help_center_search_daily_organization_id_organization_id_fkey", + "nameExplicit": false, + "columns": [ + "organization_id" + ], + "schemaTo": "public", + "tableTo": "organization", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "entityType": "fks" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "help_center_article_daily_pkey", + "columns": [ + "article_id", + "day" + ], + "nameExplicit": false, + "entityType": "pks" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_search_daily", + "name": "help_center_search_daily_pkey", + "columns": [ + "organization_id", + "day", + "query" + ], + "nameExplicit": false, + "entityType": "pks" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_article_daily", + "name": "help_center_article_daily_counts_chk", + "value": "\"views\" >= 0 and \"helpful_verdicts\" >= 0 and \"unhelpful_verdicts\" >= 0", + "entityType": "checks" + } + }, + { + "op": "create", + "row": { + "schema": "public", + "table": "help_center_search_daily", + "name": "help_center_search_daily_counts_chk", + "value": "\"searches\" >= 0 and \"empty_searches\" >= 0 and \"empty_searches\" <= \"searches\"", + "entityType": "checks" + } + } + ] +} \ No newline at end of file diff --git a/packages/db/src/schema/help-center.ts b/packages/db/src/schema/help-center.ts index 44b200f8e..674555303 100644 --- a/packages/db/src/schema/help-center.ts +++ b/packages/db/src/schema/help-center.ts @@ -7,11 +7,14 @@ import { bigint, boolean, check, + customType, + date, foreignKey, index, integer, pgEnum, pgTable, + primaryKey, text, timestamp, uniqueIndex, @@ -25,6 +28,18 @@ export const helpCenterVisibilityEnum = pgEnum("help_center_visibility", [ "HIDDEN", ]); +/** + * A Postgres `tsvector`. + * + * Only ever read and written by Postgres: the article's column is generated and + * the query builds its own vector from a parameter, so no application code + * stores or compares a `tsvector` value. It exists here so the column can be + * declared, indexed and generated inside the schema the migrator diffs. + */ +const tsvector = customType<{ data: string; driverData: string }>({ + dataType: () => "tsvector", +}); + /** * A section of the help center, and the container every article belongs to. * @@ -125,6 +140,26 @@ export const helpCenterArticleTable = pgTable( * `position`, which every topic has. */ position: integer("position"), + /** + * The article as a full-text index, derived and never authored. + * + * A generated column rather than a trigger or a write-path update: the + * three inputs live on this row, so Postgres can always compute the value + * and no caller can forget to. That rules out the drift a trigger's + * `UPDATE` can miss (a restored dump, a bulk insert, a migration backfill) + * and needs no backfill pass when it ships. + * + * Weights are the ranking: a title match outranks a summary match, which + * outranks a body match. That is the whole relevance model — a help center + * is small, and a second signal would be a tuning knob nobody tunes. + * + * The `regconfig` is a literal, which is what makes the expression + * immutable and therefore admissible in a generated column. A parameterized + * config (`to_tsvector($1, …)`) would be rejected by Postgres at DDL time. + */ + searchVector: tsvector("search_vector").generatedAlwaysAs( + sql`setweight(to_tsvector('english'::regconfig, title), 'A') || setweight(to_tsvector('english'::regconfig, excerpt), 'B') || setweight(to_tsvector('english'::regconfig, content), 'C')` + ), embedding: embeddingVector(DEFAULT_EMBEDDING_DIMENSIONS)("embedding"), embeddingModel: text("embedding_model"), embeddedAt: timestamp("embedded_at", { withTimezone: true }), @@ -156,6 +191,14 @@ export const helpCenterArticleTable = pgTable( table.publishedAt ), index("help_center_article_creatorMemberId_idx").on(table.creatorMemberId), + // The search's only access path. GIN rather than GiST because the corpus is + // small, updates are rare and a read is the common case, which is the + // trade-off GIN is built for. + index("help_center_article_search_vector_idx").using( + "gin", + table.searchVector + ), + index("help_center_article_embedding_hnsw_idx") .using("hnsw", sql`${table.embedding} vector_cosine_ops`) .where(sql`${table.embedding} is not null`), @@ -342,6 +385,111 @@ export const helpCenterArticleFeedbackTable = pgTable( ] ); +/** + * One article's traffic and verdicts, bucketed by UTC day. + * + * The row *is* the bucket, so the natural key is the primary key: a surrogate + * id would be a second identity for the same fact and a second way to write it + * twice. `onConflictDoUpdate` on that key is the whole write path. + * + * The two verdict counters count verdicts **cast** that day, not the current + * distribution of them. Flipping a vote from helpful to unhelpful adds an + * unhelpful to today and leaves the helpful on the day it was cast. Aggregating + * today's standing distribution is what the feedback table's own `count(*)` + * answers, and it is already what the article reads carry; keeping a second, + * slower-moving copy of it here would be two numbers that disagree. + * + * The `>= 0` check earns its place because the counters move by delta. A + * negative bucket is not a rounding artefact, it is a bug in the caller, and + * the constraint turns it into a failed write rather than a chart that says a + * page was un-read. + */ +export const helpCenterArticleDailyTable = pgTable( + "help_center_article_daily", + { + organizationId: text("organization_id") + .notNull() + .references(() => organizationTable.id, { onDelete: "cascade" }), + articleId: text("article_id") + .notNull() + .references(() => helpCenterArticleTable.id, { onDelete: "cascade" }), + /** UTC calendar day, `YYYY-MM-DD`. */ + day: date("day", { mode: "string" }).notNull(), + views: integer("views").notNull().default(0), + helpfulVerdicts: integer("helpful_verdicts").notNull().default(0), + unhelpfulVerdicts: integer("unhelpful_verdicts").notNull().default(0), + }, + (table) => [ + primaryKey({ columns: [table.articleId, table.day] }), + // The workspace-wide read: every article's buckets over a window. + index("help_center_article_daily_organizationId_day_idx").on( + table.organizationId, + table.day + ), + check( + "help_center_article_daily_counts_chk", + sql`${table.views} >= 0 and ${table.helpfulVerdicts} >= 0 and ${table.unhelpfulVerdicts} >= 0` + ), + // The same composite guard the verdict table carries: a bucket cannot name + // an article in another workspace, whatever a handler does. + foreignKey({ + name: "help_center_article_daily_article_organization_fk", + columns: [table.articleId, table.organizationId], + foreignColumns: [ + helpCenterArticleTable.id, + helpCenterArticleTable.organizationId, + ], + }).onDelete("cascade"), + ] +); + +/** + * What readers searched the help center for, bucketed by UTC day. + * + * Two questions, two counters, one row. "What are people looking for" is + * `searches` descending; "what documentation do we not have" is + * `empty_searches` above zero, because a query with no answer is the gap. Both + * come from the same bucket so they cannot disagree about how many searches + * there were. + * + * `query` is normalized before it is stored: lowercased, whitespace collapsed, + * bounded to the same length the search input accepts. Case and spacing are not + * meaning, and a counter keyed on the raw text would hold `Export CSV` and + * `export csv` as two rows with one search each. + * + * The composite key includes `query`, so the table grows with distinct queries + * rather than with searches. A help center sees human-entered text, where + * distinct-by-day is bounded by what people actually ask; there is no retention + * pass, and this is the table that would need one first if that assumption ever + * stops holding. + */ +export const helpCenterSearchDailyTable = pgTable( + "help_center_search_daily", + { + organizationId: text("organization_id") + .notNull() + .references(() => organizationTable.id, { onDelete: "cascade" }), + /** UTC calendar day, `YYYY-MM-DD`. */ + day: date("day", { mode: "string" }).notNull(), + query: text("query").notNull(), + searches: integer("searches").notNull().default(0), + /** The subset of `searches` that returned nothing. */ + emptySearches: integer("empty_searches").notNull().default(0), + }, + (table) => [ + primaryKey({ columns: [table.organizationId, table.day, table.query] }), + index("help_center_search_daily_organizationId_day_searches_idx").on( + table.organizationId, + table.day, + sql`${table.searches} desc` + ), + check( + "help_center_search_daily_counts_chk", + sql`${table.searches} >= 0 and ${table.emptySearches} >= 0 and ${table.emptySearches} <= ${table.searches}` + ), + ] +); + export type HelpCenterTopicRow = typeof helpCenterTopicTable.$inferSelect; export type HelpCenterArticleRow = typeof helpCenterArticleTable.$inferSelect; diff --git a/packages/domain/src/email-outbox/content.ts b/packages/domain/src/email-outbox/content.ts index bc43b34d6..343fefd6a 100644 --- a/packages/domain/src/email-outbox/content.ts +++ b/packages/domain/src/email-outbox/content.ts @@ -5,6 +5,7 @@ import * as Effect from "effect/Effect"; import { PUBLISHED_CHANGELOG_STATUS } from "../changelog/machine"; import type { EmailSubscriptionTopic } from "../email-subscription/schema"; import { CLOSING_POST_STATUS_TYPE } from "../post/status-intent"; +import { buildPublicSiteUrl } from "../site/url"; import type { PostBoardVisibility } from "./access"; import type { ChangelogTemplatePayload, @@ -13,12 +14,6 @@ import type { NotificationTemplatePayload, } from "./schema"; -/** A custom domain is stored as a bare host, but a person may paste a URL. */ -const urlSchemePattern = /^https?:\/\//; - -/** The trailing slash the same paste may carry. */ -const trailingSlashPattern = /\/$/; - type ChangelogNotificationContent = { readonly template: "changelog"; readonly templatePayload: Omit; @@ -207,21 +202,6 @@ export const isChangelogPubliclyVisible = ( }) ); -const buildPublicSiteUrl = ( - site: { readonly subdomain: string; readonly customDomain: string | null }, - appRootDomain: string -): string => { - if (site.customDomain) { - const host = site.customDomain - .replace(urlSchemePattern, "") - .replace(trailingSlashPattern, ""); - - return `https://${host}`; - } - - return `https://${site.subdomain}.${appRootDomain}`; -}; - /** Resolves current product data into an immutable subscription-mail snapshot. */ export const resolveSubscriptionNotificationContent = ( appUrl: string, diff --git a/packages/domain/src/help-center-article-feedback/handlers.test.ts b/packages/domain/src/help-center-article-feedback/handlers.test.ts index 7868a92a6..dc699689d 100644 --- a/packages/domain/src/help-center-article-feedback/handlers.test.ts +++ b/packages/domain/src/help-center-article-feedback/handlers.test.ts @@ -241,6 +241,44 @@ describe("HelpCenterArticleFeedbackRpcHandlers", () => { }) ); + it.effect("counts each verdict on the day it was cast", () => + Effect.gen(function* () { + const handlers = yield* HelpCenterArticleFeedbackRpcHandlersEffect; + const db = yield* currentDb; + const fixture = yield* makeFixture(); + const visitorId = visitorIdFor(fixture); + + const cast = (helpful: boolean) => + handlers + .SetHelpCenterArticleFeedback({ + organizationId: fixture.organizationId, + articleId: fixture.articleId, + helpful, + reason: helpful ? null : "Wrong section.", + visitorId, + }) + .pipe(Effect.provideService(OptionalCurrentSession, Option.none())); + + yield* cast(true); + yield* cast(false); + + const rows = yield* db + .select({ + helpful: schema.helpCenterArticleDailyTable.helpfulVerdicts, + unhelpful: schema.helpCenterArticleDailyTable.unhelpfulVerdicts, + }) + .from(schema.helpCenterArticleDailyTable) + .where( + eq(schema.helpCenterArticleDailyTable.articleId, fixture.articleId) + ); + + // A flip is a second verdict cast, not a correction of the first: the + // bucket counts judgments made, which is what "was this helpful" + // measures. The standing distribution is the feedback table itself. + expect(rows).toStrictEqual([{ helpful: 1, unhelpful: 1 }]); + }) + ); + it.effect("clears the reason when the verdict flips back to helpful", () => Effect.gen(function* () { const handlers = yield* HelpCenterArticleFeedbackRpcHandlersEffect; diff --git a/packages/domain/src/help-center-article-feedback/repository.ts b/packages/domain/src/help-center-article-feedback/repository.ts index 5619e4449..d85f07828 100644 --- a/packages/domain/src/help-center-article-feedback/repository.ts +++ b/packages/domain/src/help-center-article-feedback/repository.ts @@ -14,6 +14,8 @@ const article = schema.helpCenterArticleTable; const viewTotal = schema.helpCenterArticleViewTotalTable; +const daily = schema.helpCenterArticleDailyTable; + /** The two counts plus this reader's verdict, for the widget. */ export type TFeedbackTally = THelpCenterArticleFeedbackPublic; @@ -191,6 +193,30 @@ const makeHelpCenterArticleFeedbackRepository = Effect.gen(function* () { }, }) .pipe(Effect.asVoid); + + // The day's bucket, which counts verdicts *cast* rather than the standing + // distribution: a flip adds one to today and leaves the earlier day's + // count alone. Deliberately not part of the verdict's own write path — a + // counter that could fail would be a vote the reader cannot cast, so a + // failed bucket is logged and the verdict stands. + yield* db + .insert(daily) + .values({ + articleId, + day: DateTime.formatIsoDateUtc(DateTime.fromDateUnsafe(now)), + helpfulVerdicts: helpful ? 1 : 0, + organizationId, + unhelpfulVerdicts: helpful ? 0 : 1, + views: 0, + }) + .onConflictDoUpdate({ + target: [daily.articleId, daily.day], + set: { + helpfulVerdicts: sql`${daily.helpfulVerdicts} + ${helpful ? 1 : 0}`, + unhelpfulVerdicts: sql`${daily.unhelpfulVerdicts} + ${helpful ? 0 : 1}`, + }, + }) + .pipe(Effect.asVoid, Effect.ignoreCause); }); /** diff --git a/packages/domain/src/help-center-article/embedding.ts b/packages/domain/src/help-center-article/embedding.ts new file mode 100644 index 000000000..25c53c55b --- /dev/null +++ b/packages/domain/src/help-center-article/embedding.ts @@ -0,0 +1,158 @@ +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +/** + * The article's search embedding: how it is built, stored and refreshed. + * + * The column triple (`embedding`, `embedding_model`, `embedded_at`) shipped with + * the help center and nothing wrote it; this module is the writer. It is a + * **best-effort index** rather than part of the article: a failure to embed + * logs and leaves the columns null, because an article that cannot be saved + * because a model is unreachable would be a worse product than an article that + * does not participate in the semantic half of search. + */ + +/** What the embedder answers with: the model that produced it, and the vector. */ +export interface HelpCenterEmbedding { + readonly model: string; + readonly vector: readonly number[]; +} + +/** + * The one capability this module needs. + * + * A port rather than `PostEmbeddingService` itself, so the article writer does + * not depend on the post module. The deployment supplies its post-named + * embedder, which is structurally this shape and is the right service to + * borrow: one model, one dimension, one API key, and one vector space shared by + * `post.embedding` and `help_center_article.embedding`. A second embedder with + * its own configuration would be a second source of truth about which vectors + * may be compared. + * + * Deliberately **infallible**. The concrete embedder fails with provider errors + * that no caller here can act on, and every caller in this module treats a + * failure exactly like "no embedding": log and carry on. Absorbing that at the + * boundary — the adapter that binds the deployment's service in `handlers.ts` + * — keeps one error channel instead of threading a provider's union through + * the write path and the search program, neither of which can do anything with + * it. + */ +export interface HelpCenterEmbedder { + readonly embed: ( + input: string + ) => Effect.Effect, never>; +} + +/** + * One job, two failure channels. + * + * They stay separate because they are unrelated — a model that is down and a + * statement that lost the database — and the scheduler absorbs both. Both are + * inferred at the call site, so no caller names either. + */ +export interface HelpCenterArticleEmbeddingJob { + readonly articleId: string; + readonly content: string; + readonly embedder: HelpCenterEmbedder; + readonly organizationId: string; + readonly repository: { + readonly saveEmbedding: (args: { + readonly articleId: string; + readonly content: string; + readonly model: string; + readonly now: Date; + readonly organizationId: string; + readonly title: string; + readonly vector: readonly number[]; + }) => Effect.Effect; + }; + readonly title: string; +} + +/** + * How much of an article the embedding sees. + * + * An article's body is bounded at 200,000 characters and an embedding model's + * context is not, so the head of the document is what identifies it: a question + * is answered by a page's opening, not by its twentieth paragraph. Truncating + * here rather than at the provider keeps the cost of an article bounded and the + * failure mode obvious — the vector describes the part of the page a reader + * sees first. + */ +export const HELP_CENTER_EMBEDDING_MAX_CHARS = 6000; + +/** + * The text that is embedded: the title, then the head of the body. + * + * The excerpt is deliberately absent. It is derived from the body, so including + * it would weight the same sentences twice, and the title already carries the + * weight the lexical index gives it. + */ +export const helpCenterArticleEmbeddingInput = ({ + content, + title, +}: { + readonly content: string; + readonly title: string; +}): string => + `${title.trim()}\n\n${content.trim().slice(0, HELP_CENTER_EMBEDDING_MAX_CHARS)}`; + +/** + * Generates and stores one article's embedding. + * + * Not forked: the caller owns the lifetime. `scheduleHelpCenterArticleEmbedding` + * is the detached entry point every write path uses. + */ +export const generateHelpCenterArticleEmbedding = Effect.fn( + "HelpCenterArticleEmbedding.generate" +)(function* (payload: HelpCenterArticleEmbeddingJob) { + const embedding = yield* payload.embedder.embed( + helpCenterArticleEmbeddingInput(payload) + ); + + if (Option.isNone(embedding)) { + // No API key, or a model that answered nothing: not an error, just an + // article that will not rank semantically. + return; + } + + const now = yield* DateTime.nowAsDate; + + yield* payload.repository.saveEmbedding({ + articleId: payload.articleId, + content: payload.content, + model: embedding.value.model, + now, + organizationId: payload.organizationId, + title: payload.title, + vector: embedding.value.vector, + }); +}); + +/** + * Runs an embedding job detached, and never lets it fail the write. + * + * Detached because the write is already durable: the reader who saved an + * article should not wait on a model call, and a provider outage should not + * turn an edit into a failed request. A failed job leaves the columns null, + * which is the state the semantic pass already handles by skipping the row. + */ +export const scheduleHelpCenterArticleEmbedding = ( + payload: HelpCenterArticleEmbeddingJob +) => + generateHelpCenterArticleEmbedding(payload).pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "Failed to generate help center article embedding", + cause + ).pipe( + Effect.annotateLogs({ + articleId: payload.articleId, + organizationId: payload.organizationId, + }) + ) + ), + Effect.forkDetach({ startImmediately: true }), + Effect.asVoid + ); diff --git a/packages/domain/src/help-center-article/handlers.test.ts b/packages/domain/src/help-center-article/handlers.test.ts index a0b1e0bac..613b3ef22 100644 --- a/packages/domain/src/help-center-article/handlers.test.ts +++ b/packages/domain/src/help-center-article/handlers.test.ts @@ -1,5 +1,6 @@ import { describe, expect, layer } from "@effect/vitest"; import { currentDb, Database, schema } from "@feeblo/db"; +import { DEFAULT_POST_EMBEDDING_DIMENSIONS } from "@feeblo/db"; import { HelpCenterArticleId, HelpCenterTopicId, @@ -7,13 +8,17 @@ import { SiteId, WorkspaceId, } from "@feeblo/id"; +import { eq } from "drizzle-orm"; import * as DateTime from "effect/DateTime"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import { EntitlementPolicy } from "../entitlement/policies"; import { HelpCenterTopicRepository } from "../help-center-topic/repository"; import { HelpCenterEnabled } from "../help-center/flag"; +import { PostEmbeddingService } from "../post/embedding-service"; import { CurrentSession, type Session } from "../session-middleware"; import { SitePolicy } from "../site/policies"; import { SiteRepository } from "../site/repository"; @@ -169,6 +174,24 @@ describe("HelpCenterArticleRpcHandlers", () => { * must differ between tests: the event's primary key is the id alone, and a * shared one would be deduplicated away as a retry. */ + /** + * A unit vector with a single non-zero component. + * + * Cosine distance between two of them is 0 when the components match and 1 + * when they differ, which is the whole range a test needs and cheaper to + * reason about than a random vector. + */ + const unitVector = (component: number) => + Array.from({ length: DEFAULT_POST_EMBEDDING_DIMENSIONS }, (_, index) => + index === component ? 1 : 0 + ); + + /** An embedder that answers one fixed model and vector. */ + const fixedEmbedder = (vector: readonly number[]) => ({ + embed: () => + Effect.succeedSome({ model: "test-embedding", vector: [...vector] }), + }); + const beaconId = (seed: number) => `00000000-0000-4000-8000-${String(seed).padStart(12, "0")}`; @@ -519,6 +542,160 @@ describe("HelpCenterArticleRpcHandlers", () => { }) ); + it.effect("searches the live documentation and not the drafts", () => + Effect.gen(function* () { + const handlers = yield* HelpCenterArticleRpcHandlersEffect; + const fixture = yield* makeFixture(); + + yield* create( + fixture, + yield* HelpCenterArticleId.generate, + "Exporting your board", + "exporting-your-board" + ); + + yield* handlers + .CreateHelpCenterArticle({ + id: yield* HelpCenterArticleId.generate, + organizationId: fixture.organizationId, + topicId: fixture.topicId, + title: "Exporting a draft", + slug: "exporting-a-draft", + content: "Not published yet", + status: "draft", + publishedAt: null, + position: null, + }) + .pipe(Effect.provideService(CurrentSession, makeSession(fixture))); + + const publicHits = yield* handlers.SearchHelpCenterArticlesPublic({ + organizationId: fixture.organizationId, + query: "export", + }); + + expect(publicHits.map((hit) => hit.slug)).toStrictEqual([ + "exporting-your-board", + ]); + + // The dashboard is looking for the thing it is writing, so it sees the + // draft the public search has to hide. + const dashboardHits = yield* handlers + .SearchHelpCenterArticles({ + organizationId: fixture.organizationId, + query: "export", + }) + .pipe(Effect.provideService(CurrentSession, makeSession(fixture))); + + expect( + dashboardHits + .map((hit) => hit.slug) + .sort((left, right) => left.localeCompare(right)) + ).toStrictEqual(["exporting-a-draft", "exporting-your-board"]); + }) + ); + + it.effect("counts each public search, and whether it found anything", () => + Effect.gen(function* () { + const handlers = yield* HelpCenterArticleRpcHandlersEffect; + const db = yield* currentDb; + const fixture = yield* makeFixture(); + + yield* create( + fixture, + yield* HelpCenterArticleId.generate, + "Resetting your password", + "resetting-your-password" + ); + + yield* handlers.SearchHelpCenterArticlesPublic({ + organizationId: fixture.organizationId, + query: "Password", + }); + + yield* handlers.SearchHelpCenterArticlesPublic({ + organizationId: fixture.organizationId, + query: "password", + }); + + yield* handlers.SearchHelpCenterArticlesPublic({ + organizationId: fixture.organizationId, + query: "invoicing", + }); + + const rows = yield* ( + db + .select({ + query: schema.helpCenterSearchDailyTable.query, + searches: schema.helpCenterSearchDailyTable.searches, + emptySearches: schema.helpCenterSearchDailyTable.emptySearches, + }) + .from(schema.helpCenterSearchDailyTable) + // The file shares one database, and the search tests above wrote + // their own rows: the read has to be scoped to this workspace, which + // is the same filter the analytics handler applies. + .where( + eq( + schema.helpCenterSearchDailyTable.organizationId, + fixture.organizationId + ) + ) + .orderBy(schema.helpCenterSearchDailyTable.query) + ); + + // Case is not a different search, and the miss is recorded as a miss. + expect(rows).toStrictEqual([ + { emptySearches: 1, query: "invoicing", searches: 1 }, + { emptySearches: 0, query: "password", searches: 2 }, + ]); + }) + ); + + it.effect("reports views and searches over the chosen window", () => + Effect.gen(function* () { + const handlers = yield* HelpCenterArticleRpcHandlersEffect; + const fixture = yield* makeFixture(); + const session = makeSession(fixture); + const id = yield* HelpCenterArticleId.generate; + + yield* create(fixture, id, "Getting started with boards", "boards"); + + yield* handlers.AddHelpCenterArticleView({ + organizationId: fixture.organizationId, + articleId: id, + id: beaconId(11), + }); + + yield* handlers.AddHelpCenterArticleView({ + organizationId: fixture.organizationId, + articleId: id, + id: beaconId(12), + }); + + yield* handlers.SearchHelpCenterArticlesPublic({ + organizationId: fixture.organizationId, + query: "boards", + }); + + const analytics = yield* handlers + .GetHelpCenterAnalytics({ + organizationId: fixture.organizationId, + range: "30d", + }) + .pipe(Effect.provideService(CurrentSession, session)); + + expect(analytics.range).toBe("30d"); + expect(analytics.totals.views).toBe(2); + expect(analytics.articles).toHaveLength(1); + expect(analytics.articles[0]?.id).toBe(id); + expect(analytics.articles[0]?.views).toBe(2); + expect(analytics.queries).toStrictEqual([ + { emptySearches: 0, query: "boards", searches: 1 }, + ]); + // A search that found something is not a documentation gap. + expect(analytics.emptyQueries).toStrictEqual([]); + }) + ); + it.effect("refuses every operation while the flag is off", () => Effect.gen(function* () { const handlers = yield* HelpCenterArticleRpcHandlersEffect; @@ -537,4 +714,257 @@ describe("HelpCenterArticleRpcHandlers", () => { }) ); }); + + describe("embeddings", () => { + /** + * Reads what the write path left behind, without waiting for the detached + * job: every assertion here is about what the request did *before* it + * returned. + */ + const readEmbeddingState = (id: string) => + Effect.gen(function* () { + const db = yield* currentDb; + + const [row] = yield* db + .select({ + embeddedAt: schema.helpCenterArticleTable.embeddedAt, + embedding: schema.helpCenterArticleTable.embedding, + embeddingModel: schema.helpCenterArticleTable.embeddingModel, + }) + .from(schema.helpCenterArticleTable) + .where(eq(schema.helpCenterArticleTable.id, id)); + + return row; + }); + + const readContent = (id: string) => + Effect.gen(function* () { + const db = yield* currentDb; + + const [row] = yield* db + .select({ + content: schema.helpCenterArticleTable.content, + title: schema.helpCenterArticleTable.title, + }) + .from(schema.helpCenterArticleTable) + .where(eq(schema.helpCenterArticleTable.id, id)); + + return row; + }); + + layer(TestLayer)("handlers", (it) => { + it.effect("asks for an embedding without making the write wait", () => + Effect.gen(function* () { + const fixture = yield* makeFixture(); + const id = yield* HelpCenterArticleId.generate; + const release = yield* Deferred.make(); + let calls = 0; + + const embedder: PostEmbeddingService["Service"] = { + embed: () => + Effect.sync(() => { + calls += 1; + }).pipe( + Effect.andThen(Deferred.await(release)), + Effect.as(Option.none()) + ), + }; + + // The handlers read their embedder when they are built, which is why + // the substitute wraps the construction rather than the call. + const handlers = yield* HelpCenterArticleRpcHandlersEffect.pipe( + Effect.provideService(PostEmbeddingService, embedder) + ); + + yield* handlers + .CreateHelpCenterArticle({ + id, + organizationId: fixture.organizationId, + topicId: fixture.topicId, + title: "Embedded article", + slug: "embedded-article", + content: "Body", + status: "published", + publishedAt: null, + position: null, + }) + .pipe(Effect.provideService(CurrentSession, makeSession(fixture))); + + yield* Effect.yieldNow; + + const state = yield* readEmbeddingState(id); + + expect(calls).toBe(1); + // The job is still blocked on the deferred, so an embedding here + // would mean the write awaited it. + expect(state?.embedding).toBeNull(); + expect(state?.embeddingModel).toBeNull(); + expect(state?.embeddedAt).toBeNull(); + + yield* Deferred.succeed(release, undefined); + }) + ); + + it.effect("does not spend an embedding on a draft", () => + Effect.gen(function* () { + const fixture = yield* makeFixture(); + const id = yield* HelpCenterArticleId.generate; + let calls = 0; + + const handlers = yield* HelpCenterArticleRpcHandlersEffect.pipe( + Effect.provideService(PostEmbeddingService, { + embed: () => + Effect.sync(() => { + calls += 1; + }).pipe(Effect.as(Option.none())), + }) + ); + + yield* handlers + .CreateHelpCenterArticle({ + id, + organizationId: fixture.organizationId, + topicId: fixture.topicId, + title: "Unpublished", + slug: "unpublished", + content: "Body", + status: "draft", + publishedAt: null, + position: null, + }) + .pipe(Effect.provideService(CurrentSession, makeSession(fixture))); + + yield* Effect.yieldNow; + + // A page no search may return is not worth a model call; the + // publication that makes it readable schedules its own job. + expect(calls).toBe(0); + }) + ); + + it.effect("stores a vector only for the revision it describes", () => + Effect.gen(function* () { + const repository = yield* HelpCenterArticleRepository; + const fixture = yield* makeFixture(); + const id = yield* HelpCenterArticleId.generate; + const now = yield* DateTime.nowAsDate; + const vector = unitVector(0); + + yield* create(fixture, id, "Original title", "original-title"); + + const current = yield* readContent(id); + + if (current === undefined) { + return yield* Effect.die(new Error("fixture article missing")); + } + + const stale = yield* repository.saveEmbedding({ + articleId: id, + content: current.content, + model: "test-embedding", + now, + organizationId: fixture.organizationId, + title: "A title the editor has since replaced", + vector, + }); + + // The job runs after the write, and an editor who saved again in + // that window must not end up with a vector for the old text. + expect(stale).toBe(false); + + const afterStale = yield* readEmbeddingState(id); + + expect(afterStale?.embedding).toBeNull(); + + const stored = yield* repository.saveEmbedding({ + articleId: id, + content: current.content, + model: "test-embedding", + now, + organizationId: fixture.organizationId, + title: current.title, + vector, + }); + + expect(stored).toBe(true); + + const state = yield* readEmbeddingState(id); + + expect(state?.embeddingModel).toBe("test-embedding"); + expect(state?.embeddedAt).not.toBeNull(); + }) + ); + + it.effect( + "answers from the semantic pass when the words match nothing", + () => + Effect.gen(function* () { + const repository = yield* HelpCenterArticleRepository; + const fixture = yield* makeFixture(); + const now = yield* DateTime.nowAsDate; + + const nearId = yield* HelpCenterArticleId.generate; + const farId = yield* HelpCenterArticleId.generate; + const bareId = yield* HelpCenterArticleId.generate; + + yield* create( + fixture, + nearId, + "Reports are empty", + "empty-reports" + ); + yield* create( + fixture, + farId, + "Change your plan", + "change-your-plan" + ); + yield* create(fixture, bareId, "Export a board", "export-a-board"); + + const embed = (id: string, component: number) => + Effect.gen(function* () { + const row = yield* readContent(id); + + if (row === undefined) { + return yield* Effect.die( + new Error("fixture article missing") + ); + } + + yield* repository.saveEmbedding({ + articleId: id, + content: row.content, + model: "test-embedding", + now, + organizationId: fixture.organizationId, + title: row.title, + vector: unitVector(component), + }); + }); + + // The near article sits at distance 0 from the query vector, the far + // one at 1, and `bareId` carries no vector at all. + yield* embed(nearId, 3); + yield* embed(farId, 9); + + const searching = yield* HelpCenterArticleRpcHandlersEffect.pipe( + Effect.provideService( + PostEmbeddingService, + fixedEmbedder(unitVector(3)) + ) + ); + + const hits = yield* searching.SearchHelpCenterArticlesPublic({ + organizationId: fixture.organizationId, + query: "why is this page blank", + }); + + // No word is shared with the near article's title, so this answer can + // only have come from the vector. The far article is beyond the + // threshold and the unembedded one is not a candidate. + expect(hits.map((hit) => hit.id)).toStrictEqual([nearId]); + }) + ); + }); + }); }); diff --git a/packages/domain/src/help-center-article/handlers.ts b/packages/domain/src/help-center-article/handlers.ts index ee930771c..c68fe0cf6 100644 --- a/packages/domain/src/help-center-article/handlers.ts +++ b/packages/domain/src/help-center-article/handlers.ts @@ -1,11 +1,14 @@ +import type { THelpCenterArticleState } from "@feeblo/domain-contracts/help-center-article"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Option from "effect/Option"; import { EntitlementPolicy } from "../entitlement/policies"; import { HelpCenterTopicRepository } from "../help-center-topic/repository"; import { helpCenterFeatureGate } from "../help-center/flag"; import * as Policy from "../policy"; +import { PostEmbeddingService } from "../post/embedding-service"; import * as RateLimit from "../rate-limit"; import { BadRequestError, @@ -15,24 +18,37 @@ import { import { SitePolicy } from "../site/policies"; import { SiteRepository } from "../site/repository"; import { WorkspaceRepository } from "../workspace/repository"; +import { + type HelpCenterEmbedder, + type HelpCenterEmbedding, + scheduleHelpCenterArticleEmbedding, +} from "./embedding"; import { FailedToDeleteHelpCenterArticleError, HelpCenterArticleChangedError, HelpCenterArticleSlugTakenError, } from "./errors"; -import { canTransitionHelpCenterArticle } from "./machine"; +import { makeHelpCenterSearch } from "./hybrid-search"; +import { + canTransitionHelpCenterArticle, + isHelpCenterArticleReadable, +} from "./machine"; import { HelpCenterArticlePolicy } from "./policies"; import { HelpCenterArticleRepository } from "./repository"; import { HelpCenterArticleRpcs } from "./rpcs"; -import type { - THelpCenterArticleCreate, - THelpCenterArticleDelete, - THelpCenterArticleGet, - THelpCenterArticleGetPublic, - THelpCenterArticleList, - THelpCenterArticleListPublic, - THelpCenterArticleUpdate, - THelpCenterArticleViewRecord, +import { + HELP_CENTER_ANALYTICS_RANGE_DAYS, + type THelpCenterAnalyticsPayload, + type THelpCenterArticleCreate, + type THelpCenterArticleDelete, + type THelpCenterArticleGet, + type THelpCenterArticleGetPublic, + type THelpCenterArticleList, + type THelpCenterArticleListPublic, + type THelpCenterArticleSearch, + type THelpCenterArticleSearchPublic, + type THelpCenterArticleUpdate, + type THelpCenterArticleViewRecord, } from "./schema"; import { normalizeArticleWrite } from "./write"; @@ -44,6 +60,83 @@ export const HelpCenterArticleRpcHandlersEffect = Effect.gen(function* () { const articlePolicy = yield* HelpCenterArticlePolicy; const sitePolicy = yield* SitePolicy; + /** + * The deployment's embedder, when it has one. + * + * `serviceOption` rather than a layer requirement: an embedding service is an + * enhancement, and a process that has no API key configured must still serve + * a help center. The service is the post module's only because there is one + * model, one dimension, one key and one vector space — the article column and + * `post.embedding` are the same width and compared with the same metric, so a + * second embedder would be a second source of truth about which vectors may + * be compared. + */ + const embeddingService = yield* Effect.serviceOption(PostEmbeddingService); + + /** + * The deployment's embedder behind this module's infallible port. + * + * The provider's errors — a dimension mismatch, a timeout, an AI-layer + * failure — are all "we could not embed this" from here, and the caller + * treats them the same way: log, and let the article save or the search + * answer without a vector. Absorbing them here is what keeps a provider's + * union out of the write path's error channel. + */ + const embeddings = Option.map( + embeddingService, + (service): HelpCenterEmbedder => ({ + embed: (input) => + service + .embed(input) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "Failed to generate a help center embedding", + cause + ).pipe(Effect.as(Option.none())) + ) + ), + }) + ); + + const search = makeHelpCenterSearch({ + embeddings, + lexical: repository.search, + semantic: repository.searchByVector, + }); + + /** + * Refresh an article's search embedding after a write, detached. + * + * Only for a state a reader can open: embedding a draft would spend a model + * call on a page no search may return, and the publication that makes it + * readable is itself an update, which schedules its own job. + */ + const scheduleEmbedding = (args: { + readonly articleId: string; + readonly content: string; + readonly organizationId: string; + readonly status: THelpCenterArticleState; + readonly title: string; + }) => + Effect.gen(function* () { + if ( + !isHelpCenterArticleReadable(args.status) || + Option.isNone(embeddings) + ) { + return; + } + + yield* scheduleHelpCenterArticleEmbedding({ + articleId: args.articleId, + content: args.content, + embedder: embeddings.value, + organizationId: args.organizationId, + repository, + title: args.title, + }); + }); + return { ListHelpCenterArticles: (args: THelpCenterArticleList) => repository @@ -113,6 +206,14 @@ export const HelpCenterArticleRpcHandlersEffect = Effect.gen(function* () { now, write, }); + + yield* scheduleEmbedding({ + articleId: args.id, + content: write.content, + organizationId: args.organizationId, + status: write.status, + title: write.title, + }); }).pipe( Policy.withPolicy(articlePolicy.canCreate(args)), withRemapDbErrors({ @@ -183,6 +284,14 @@ export const HelpCenterArticleRpcHandlersEffect = Effect.gen(function* () { if (updated.length === 0) { return yield* new HelpCenterArticleChangedError(); } + + yield* scheduleEmbedding({ + articleId: args.id, + content: write.content, + organizationId: args.organizationId, + status: write.status, + title: write.title, + }); }).pipe( Policy.withPolicy(articlePolicy.canUpdate(args.organizationId)), withRemapDbErrors({ @@ -236,6 +345,104 @@ export const HelpCenterArticleRpcHandlersEffect = Effect.gen(function* () { withRemapDbErrors("HelpCenterArticleView", "upsert"), Policy.withPublicPolicy(helpCenterFeatureGate) ), + + /** + * The dashboard's search. + * + * It counts no analytics: an editor searching their own workspace is not a + * reader looking for an answer, and counting it would make the "top + * searches" list an inventory of the team's drafts. + */ + SearchHelpCenterArticles: (args: THelpCenterArticleSearch) => + search({ + includeUnreadable: true, + limit: args.limit ?? 10, + organizationId: args.organizationId, + query: args.query, + }).pipe( + Policy.withPolicy(articlePolicy.canRead(args.organizationId)), + withRemapDbErrors("HelpCenterArticle", "select"), + Policy.withPublicPolicy(helpCenterFeatureGate) + ), + + /** + * The portal's search, and the only public read that writes. + * + * The counter is recorded after the rows are read, because "found" is the + * length of the answer. It is swallowed on failure: a reader's search must + * not fail because the analytics table is unavailable, and the read is the + * product. + */ + SearchHelpCenterArticlesPublic: (args: THelpCenterArticleSearchPublic) => + Effect.gen(function* () { + const hits = yield* search({ + includeUnreadable: false, + limit: args.limit ?? 10, + organizationId: args.organizationId, + query: args.query, + }); + + const now = yield* DateTime.now; + + yield* repository + .recordSearch({ + found: hits.length > 0, + now, + organizationId: args.organizationId, + query: args.query, + }) + .pipe(Effect.ignoreCause); + + return hits.map((hit) => ({ + excerpt: hit.excerpt, + id: hit.id, + isArchived: hit.status === "archived", + slug: hit.slug, + title: hit.title, + topicName: hit.topicName, + topicSlug: hit.topicSlug, + })); + }).pipe( + RateLimit.withPublicRpcRateLimit({ + level: "read", + name: "SearchHelpCenterArticlesPublic", + }), + Policy.withPublicPolicy( + sitePolicy.canViewHelpCenter(args.organizationId) + ), + withRemapDbErrors("HelpCenterArticle", "select"), + Policy.withPublicPolicy(helpCenterFeatureGate) + ), + + /** + * The dashboard's analytics. + * + * The window's first day is computed here rather than in the repository, so + * "now" is read through `Clock` once at the edge and a test can pin the + * range without reaching into the query. + */ + GetHelpCenterAnalytics: (args: THelpCenterAnalyticsPayload) => + Effect.gen(function* () { + const now = yield* DateTime.now; + + const since = DateTime.formatIsoDateUtc( + DateTime.subtract({ + days: HELP_CENTER_ANALYTICS_RANGE_DAYS[args.range], + })(now) + ); + + const result = yield* repository.analytics({ + limit: 10, + organizationId: args.organizationId, + sinceDay: since, + }); + + return { ...result, range: args.range, since }; + }).pipe( + Policy.withPolicy(articlePolicy.canRead(args.organizationId)), + withRemapDbErrors("HelpCenterArticleAnalytics", "select"), + Policy.withPublicPolicy(helpCenterFeatureGate) + ), }; }); diff --git a/packages/domain/src/help-center-article/hybrid-search.test.ts b/packages/domain/src/help-center-article/hybrid-search.test.ts new file mode 100644 index 000000000..3f9caf8af --- /dev/null +++ b/packages/domain/src/help-center-article/hybrid-search.test.ts @@ -0,0 +1,278 @@ +import { expect, it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import type { HelpCenterEmbedder, HelpCenterEmbedding } from "./embedding"; +import { + HELP_CENTER_SEMANTIC_FILL_BELOW, + HELP_CENTER_SEMANTIC_MAX_DISTANCE, + makeHelpCenterSearch, +} from "./hybrid-search"; + +/** + * The search program's own tests. + * + * They pin the contract the dashboard, the portal and the widget share: how + * thin a lexical result has to be before the semantic pass runs, what order the + * two halves land in, and that a model failure is a shorter answer rather than + * a failed search. A lexical search and a semantic search stand in as plain + * functions, so no database and no OpenAI client is involved; the surfaces' + * own suites still drive the real repository. + */ + +interface Hit { + readonly id: string; + readonly title: string; +} + +const hit = (id: string): Hit => ({ id, title: id }); + +const hits = (...ids: string[]): readonly Hit[] => ids.map(hit); + +/** + * A lexical stand-in that answers with a fixed list and records its calls, so a + * test can prove the semantic pass was skipped rather than merely unobservable. + */ +const lexicalReturning = (rows: readonly Hit[] = []) => { + const calls: { + readonly includeUnreadable: boolean; + readonly limit: number; + }[] = []; + + const lexical = (args: { + readonly includeUnreadable: boolean; + readonly limit: number; + }) => { + calls.push({ + includeUnreadable: args.includeUnreadable, + limit: args.limit, + }); + + return Effect.succeed(rows); + }; + + return { calls, lexical }; +}; + +/** + * A semantic stand-in keyed by distance, ordered the way the repository orders. + * + * Callers name an id and a distance; the title is filled in so a row satisfies + * the same `Hit` the lexical half returns, which is what makes the merge + * meaningful. + */ +const semanticReturning = ( + rows: readonly { readonly distance: number | null; readonly id: string }[] +) => { + let calls = 0; + + const semantic = () => { + calls += 1; + + return Effect.succeed(rows.map((row) => ({ ...row, title: row.id }))); + }; + + return { calls: () => calls, semantic }; +}; + +/** An embedder that answers a fixed model and vector, and counts its calls. */ +const embedderAnswering = ( + answer: Option.Option = Option.some({ + model: "test-embedding", + vector: [0.1, 0.2], + }) +) => { + let calls = 0; + + const embedder: HelpCenterEmbedder = { + embed: () => { + calls += 1; + + return Effect.succeed(answer); + }, + }; + + return { calls: () => calls, embedder }; +}; + +const searchWith = (deps: { + readonly embeddings: Option.Option; + readonly lexical: ReturnType["lexical"]; + readonly semantic: ReturnType["semantic"]; +}) => makeHelpCenterSearch(deps); + +it.effect( + "answers lexically and never embeds when the words already found enough", + () => + Effect.gen(function* () { + const lexical = lexicalReturning( + hits("a", "b", "c").slice(0, HELP_CENTER_SEMANTIC_FILL_BELOW) + ); + + const semantic = semanticReturning([]); + const embedder = embedderAnswering(); + + const search = searchWith({ + embeddings: Option.some(embedder.embedder), + lexical: lexical.lexical, + semantic: semantic.semantic, + }); + + const results = yield* search({ + includeUnreadable: false, + organizationId: "org", + query: "export", + }); + + expect(results.map((row) => row.id)).toStrictEqual(["a", "b", "c"]); + // The point of the threshold: a search that answered costs nothing. + expect(embedder.calls()).toBe(0); + expect(semantic.calls()).toBe(0); + }) +); + +it.effect("fills the tail from the semantic pass, after the lexical hits", () => + Effect.gen(function* () { + const lexical = lexicalReturning(hits("exact")); + + const semantic = semanticReturning([ + { distance: HELP_CENTER_SEMANTIC_MAX_DISTANCE / 2, id: "near" }, + // Already returned lexically: an exact match is also a near vector. + { distance: 0.01, id: "exact" }, + { distance: HELP_CENTER_SEMANTIC_MAX_DISTANCE / 3, id: "nearer" }, + ]); + + const embedder = embedderAnswering(); + + const results = yield* searchWith({ + embeddings: Option.some(embedder.embedder), + lexical: lexical.lexical, + semantic: semantic.semantic, + })({ + includeUnreadable: false, + organizationId: "org", + query: "export csv", + }); + + // Exact-word evidence keeps its precedence; the near-misses fill behind it, + // in the repository's distance order, without repeating the exact hit. + expect(results.map((row) => row.id)).toStrictEqual([ + "exact", + "near", + "nearer", + ]); + expect(embedder.calls()).toBe(1); + }) +); + +it.effect("drops a semantic hit that is too far away", () => + Effect.gen(function* () { + const embedder = embedderAnswering(); + + const results = yield* searchWith({ + embeddings: Option.some(embedder.embedder), + lexical: lexicalReturning([]).lexical, + semantic: semanticReturning([ + { distance: HELP_CENTER_SEMANTIC_MAX_DISTANCE + 0.01, id: "far" }, + { distance: null, id: "unembedded" }, + ]).semantic, + })({ includeUnreadable: false, organizationId: "org", query: "export" }); + + // A page the model could not score, and one it scored as unrelated, are + // not answers. + expect(results).toStrictEqual([]); + }) +); + +it.effect("never returns more than the caller asked for", () => + Effect.gen(function* () { + const embedder = embedderAnswering(); + + const results = yield* searchWith({ + embeddings: Option.some(embedder.embedder), + lexical: lexicalReturning(hits("exact")).lexical, + semantic: semanticReturning([ + { distance: 0.1, id: "one" }, + { distance: 0.2, id: "two" }, + { distance: 0.3, id: "three" }, + ]).semantic, + })({ + includeUnreadable: false, + limit: 2, + organizationId: "org", + query: "export", + }); + + expect(results.map((row) => row.id)).toStrictEqual(["exact", "one"]); + }) +); + +it.effect("answers lexically when the model answers nothing", () => + Effect.gen(function* () { + // What the adapter produces when the provider is down: no embedding, no + // error. The program cannot tell this apart from a deployment with no key. + const results = yield* searchWith({ + embeddings: Option.some(embedderAnswering(Option.none()).embedder), + lexical: lexicalReturning(hits("exact")).lexical, + semantic: semanticReturning([{ distance: 0.1, id: "near" }]).semantic, + })({ includeUnreadable: false, organizationId: "org", query: "export" }); + + // The reader gets the answer they would have got without a model rather + // than an error page. + expect(results.map((row) => row.id)).toStrictEqual(["exact"]); + }) +); + +it.effect("answers lexically when the deployment has no key", () => + Effect.gen(function* () { + const lexical = lexicalReturning([]); + + const results = yield* searchWith({ + embeddings: Option.none(), + lexical: lexical.lexical, + semantic: semanticReturning([{ distance: 0.1, id: "near" }]).semantic, + })({ includeUnreadable: false, organizationId: "org", query: "export" }); + + expect(results).toStrictEqual([]); + expect(lexical.calls).toHaveLength(1); + }) +); + +it.effect("does not embed for a strict search, even with no hits", () => + Effect.gen(function* () { + const embedder = embedderAnswering(); + const semantic = semanticReturning([{ distance: 0.1, id: "near" }]); + + const results = yield* searchWith({ + embeddings: Option.some(embedder.embedder), + lexical: lexicalReturning([]).lexical, + semantic: semantic.semantic, + })({ + includeUnreadable: false, + organizationId: "org", + query: "export", + strict: true, + }); + + // A strict search exists to show a page that shares the visitor's words; a + // semantic neighbour with no word overlap is the suggestion that looks like + // a mistake beside a feedback form. + expect(results).toStrictEqual([]); + expect(embedder.calls()).toBe(0); + expect(semantic.calls()).toBe(0); + }) +); + +it.effect("does not embed an empty query", () => + Effect.gen(function* () { + const embedder = embedderAnswering(); + + yield* searchWith({ + embeddings: Option.some(embedder.embedder), + lexical: lexicalReturning([]).lexical, + semantic: semanticReturning([]).semantic, + })({ includeUnreadable: false, organizationId: "org", query: " " }); + + expect(embedder.calls()).toBe(0); + }) +); diff --git a/packages/domain/src/help-center-article/hybrid-search.ts b/packages/domain/src/help-center-article/hybrid-search.ts new file mode 100644 index 000000000..3d349c4ce --- /dev/null +++ b/packages/domain/src/help-center-article/hybrid-search.ts @@ -0,0 +1,180 @@ +import * as Effect from "effect/Effect"; +import * as Option from "effect/Option"; + +import type { HelpCenterEmbedder } from "./embedding"; + +/** + * The two halves of a help center search, and the rule that decides whether the + * second one runs. + * + * Lexical search is the product: it is exact, explainable, free per query, and + * already ranked by the column's own weights. The embedding exists for the + * queries words cannot answer — a reader who describes the problem in their own + * vocabulary will never match a page written in the documentation's. + * + * So the semantic pass is a **rescue, not a ranker**. It runs only when the + * lexical pass found fewer than {@link HELP_CENTER_SEMANTIC_FILL_BELOW} pages, + * and its hits are appended after the lexical ones rather than merged into one + * score. That is a deliberate refusal to fuse two numbers that do not share a + * scale: `ts_rank_cd` is unbounded and depends on the query's term count, cosine + * distance is bounded and model-specific, and any arithmetic combining them + * would be a constant nobody could justify with evidence. Exact-word evidence + * keeps its precedence, the near-miss evidence fills the rest, and a learned + * ranker is a later decision with data behind it. + * + * The rescue rule also bounds the cost. A public, unauthenticated search box + * must not spend a model call on the common case, and the common case is a + * query that matches. Only a thin result set — the searches that were about to + * fail — pays for an embedding. + */ + +/** The hit shape both halves return, and the merge keys on. */ +export interface HelpCenterSearchHitLike { + readonly id: string; +} + +/** A lexical search request. The repository's `search` matches this shape. */ +export interface HelpCenterLexicalSearch { + readonly includeUnreadable: boolean; + readonly limit: number; + readonly organizationId: string; + readonly query: string; + readonly strict?: boolean | undefined; +} + +/** A semantic search request. The repository's `searchByVector` matches this. */ +export interface HelpCenterSemanticSearch { + readonly embedding: readonly number[]; + readonly embeddingModel: string; + readonly includeUnreadable: boolean; + readonly limit: number; + readonly organizationId: string; +} + +export type HelpCenterSearchRequest = Omit & { + readonly limit?: number | undefined; +}; + +export interface HelpCenterSearchDependencies { + /** + * The deployment's embedder, when it has one. + * + * Its own error is `unknown` because the program absorbs it: a model that + * fails is a search that answers lexically, not a search that fails, so the + * failure never reaches the caller's error channel. + */ + readonly embeddings: Option.Option; + readonly lexical: ( + args: HelpCenterLexicalSearch + ) => Effect.Effect; + readonly semantic: ( + args: HelpCenterSemanticSearch + ) => Effect.Effect< + readonly (Hit & { readonly distance: number | null })[], + E, + R + >; +} + +/** The result count when a caller names none. */ +export const HELP_CENTER_SEARCH_DEFAULT_LIMIT = 10; + +/** + * How thin a lexical result must be before the semantic pass runs. + * + * Three is the number of hits a reader scans before deciding the page is + * useless. Above it, appending near-misses adds noise to a page that already + * answered; below it, the search looks broken. + */ +export const HELP_CENTER_SEMANTIC_FILL_BELOW = 3; + +/** + * The furthest a page may sit from the query and still be offered. + * + * Stricter than the post-suggestion threshold, because the two mistakes are not + * equal: a wrong "this may already exist" post costs the reader a glance and + * can be merged later, while a wrong documentation link sends them to a page + * about something else and makes the help center look unreliable. + */ +export const HELP_CENTER_SEMANTIC_MAX_DISTANCE = 0.6; + +/** + * Ranks a workspace's documentation against a reader's text. + * + * One program, three surfaces: `SearchHelpCenterArticles` (dashboard), + * `SearchHelpCenterArticlesPublic` (portal), and the widget's deflection. The + * dashboard and portal differ only in `includeUnreadable`, which stays a + * parameter for the reason it always was — a draft must not be reachable from a + * search box. + * + * `strict` disables the semantic pass rather than narrowing it. A strict search + * is the widget's deflection, whose whole purpose is to show a page that shares + * the visitor's words; a semantic neighbour with no word overlap is exactly the + * suggestion that looks like a mistake beside a feedback form. + */ +export const makeHelpCenterSearch = ( + dependencies: HelpCenterSearchDependencies +) => + Effect.fn("HelpCenterSearch.search")(function* ( + request: HelpCenterSearchRequest + ) { + const limit = request.limit ?? HELP_CENTER_SEARCH_DEFAULT_LIMIT; + + const lexicalArgs: HelpCenterLexicalSearch = { + includeUnreadable: request.includeUnreadable, + limit, + organizationId: request.organizationId, + query: request.query, + strict: request.strict, + }; + + const lexicalHits = yield* dependencies.lexical(lexicalArgs); + + const worthRescuing = + request.strict !== true && + lexicalHits.length < HELP_CENTER_SEMANTIC_FILL_BELOW && + Option.isSome(dependencies.embeddings) && + request.query.trim().length > 0; + + if (!worthRescuing) { + return lexicalHits; + } + + // The port is infallible: a model that is down answers `none`, and the + // adapter that binds the deployment's service is where that decision is + // made and logged. + const queryEmbedding = yield* dependencies.embeddings.value.embed( + request.query.trim() + ); + + if (Option.isNone(queryEmbedding)) { + return lexicalHits; + } + + const semanticHits = yield* dependencies.semantic({ + embedding: queryEmbedding.value.vector, + embeddingModel: queryEmbedding.value.model, + includeUnreadable: request.includeUnreadable, + limit, + organizationId: request.organizationId, + }); + + const nearEnough = semanticHits + .filter( + (hit) => + hit.distance !== null && + hit.distance <= HELP_CENTER_SEMANTIC_MAX_DISTANCE + ) + .map(({ distance: _distance, ...hit }) => hit); + + const seen = new Set(lexicalHits.map((hit) => hit.id)); + + // `filter` rather than a loop with a `push`: the repository returns one row + // per article, so the only duplicates the merge can see are the ones the + // lexical pass already returned, and those are what `seen` holds. + const fill = nearEnough + .filter((hit) => !seen.has(hit.id)) + .slice(0, Math.max(0, limit - lexicalHits.length)); + + return [...lexicalHits, ...fill]; + }); diff --git a/packages/domain/src/help-center-article/repository.ts b/packages/domain/src/help-center-article/repository.ts index 5263ef747..adac7054d 100644 --- a/packages/domain/src/help-center-article/repository.ts +++ b/packages/domain/src/help-center-article/repository.ts @@ -1,6 +1,18 @@ import { currentDb, schema, withTransactionAbortRetry } from "@feeblo/db"; import type { THelpCenterArticleState } from "@feeblo/domain-contracts/help-center-article"; -import { and, asc, count, desc, eq, inArray, lt, sql } from "drizzle-orm"; +import { + and, + asc, + cosineDistance, + count, + desc, + eq, + gte, + inArray, + isNotNull, + lt, + sql, +} from "drizzle-orm"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -10,6 +22,11 @@ import { HELP_CENTER_ARTICLE_STATES, isHelpCenterArticleReadable, } from "./machine"; +import { + helpCenterSearchTerms, + helpCenterSearchTsQuery, + normalizeHelpCenterSearchQuery, +} from "./search"; import type { NormalizedHelpCenterArticleWrite } from "./write"; const article = schema.helpCenterArticleTable; @@ -22,6 +39,10 @@ const viewEvent = schema.helpCenterArticleViewEventTable; const feedback = schema.helpCenterArticleFeedbackTable; +const daily = schema.helpCenterArticleDailyTable; + +const searchDaily = schema.helpCenterSearchDailyTable; + const articleColumns = { id: article.id, organizationId: article.organizationId, @@ -88,6 +109,58 @@ const articleOrder = [ asc(article.title), ] as const; +/** + * The projection every search returns. + * + * One list for both halves of the search, so a lexical hit and a semantic hit + * are the same shape and a surface projects one type. `status` travels rather + * than a pre-computed `isArchived`, because the dashboard distinguishes a draft + * from an archived page and the portal collapses both into "readable". + */ +const searchHitColumns = { + id: article.id, + slug: article.slug, + title: article.title, + excerpt: article.excerpt, + status: article.status, + topicId: article.topicId, + topicSlug: topic.slug, + topicName: topic.name, +} as const; + +/** + * The `tsquery` for a set of terms, or `undefined` when there are none. + * + * `to_tsquery` throws on malformed input, so the value bound here is always + * one `helpCenterSearchTsQuery` produced. A term list that collapsed to + * nothing yields `undefined`, which `and(...)` drops — and the caller never + * asks, because `search` returns early for it. + */ +const tsQuery = (queryText: string | null) => + queryText === null ? undefined : sql`to_tsquery('english', ${queryText})`; + +/** + * Relevance, as the generated column's own weights describe it. + * + * `ts_rank_cd` rather than `ts_rank`: coverage density rewards an article that + * uses the terms repeatedly in a short space over one that mentions them once + * in a long body, which is the difference between the page someone wants and + * the page that happens to contain the word. + */ +const rankOf = (queryText: string) => + sql`ts_rank_cd(${article.searchVector}, ${tsQuery(queryText)})`; + +/** + * The title and summary as their own vector, for a strict search. + * + * Computed rather than stored because it is only used by the widget's + * deflection, where the corpus is one workspace's articles and a sequential + * scan is cheaper than a second generated column and its index. Its weights + * match the generated column's `A` and `B`, so a strict search and a full one + * cannot disagree about where a term was found. + */ +const titleExcerptVector = sql`to_tsvector('english'::regconfig, ${article.title} || ' ' || ${article.excerpt})`; + const makeHelpCenterArticleRepository = Effect.gen(function* () { const db = yield* currentDb; @@ -211,6 +284,169 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { .where(eq(article.organizationId, organizationId)) .pipe(Effect.map((rows) => rows[0]?.count ?? 0)); + /** + * Ranked full-text search. + * + * Two callers, one query. The dashboard sees every state — an editor + * searching for the draft they half-wrote should find it — and the portal and + * widget pass `includeUnreadable: false`, which is the same `readableWhere` + * every other public read uses. Getting that backwards would publish a draft + * through a search box. + * + * `strict` narrows the match to the title and summary. The widget uses it + * because a suggestion beside a feedback form is only worth showing if it + * names the thing being asked about; a body match on a passing mention is a + * worse suggestion than none. + * + * No terms means no round trip: `helpCenterSearchTsQuery` returns null for + * input that survived normalization with nothing in it, and a query that + * cannot match should not be sent. + */ + const search = ({ + organizationId, + query, + limit, + includeUnreadable, + strict = false, + }: { + organizationId: string; + query: string; + limit: number; + includeUnreadable: boolean; + strict?: boolean | undefined; + }) => { + const queryText = helpCenterSearchTsQuery(helpCenterSearchTerms(query)); + + if (queryText === null) { + return Effect.succeed([]); + } + + return ( + db + .select(searchHitColumns) + .from(article) + .innerJoin(topic, eq(topic.id, article.topicId)) + .where( + and( + eq(article.organizationId, organizationId), + includeUnreadable ? undefined : readableWhere, + sql`${article.searchVector} @@ ${tsQuery(queryText)}`, + strict + ? sql`${titleExcerptVector} @@ ${tsQuery(queryText)}` + : undefined + ) + ) + // `publishedAt` before `id` so the order is total without asking for a + // tiebreak column the caller would have to carry. + .orderBy( + desc(rankOf(queryText)), + desc(article.publishedAt), + asc(article.id) + ) + .limit(limit) + ); + }; + + /** + * Nearest articles by cosine distance, for the semantic half of a search. + * + * Rows are filtered on `embedding_model` as well as `is not null`: a vector + * from a different model lives in a different space, and comparing across + * models returns confident nonsense rather than an error. A deployment that + * switches `EMBEDDING_MODEL` therefore stops matching its old rows until they + * are re-embedded, which is the honest behaviour — the alternative is + * silently mixing two geometries. + * + * No ANN index is used here on purpose. `help_center_article` is bounded by + * the plan's article ceiling, so a sequential distance scan over one + * workspace is a few hundred rows; the HNSW index that exists for the column + * is there for a corpus that grows past that assumption. + */ + const searchByVector = ({ + organizationId, + embedding, + embeddingModel, + limit, + includeUnreadable, + }: { + organizationId: string; + embedding: readonly number[]; + embeddingModel: string; + limit: number; + includeUnreadable: boolean; + }) => { + const distance = cosineDistance(article.embedding, [...embedding]); + + return ( + db + .select({ + ...searchHitColumns, + distance: sql`${distance}`, + }) + .from(article) + .innerJoin(topic, eq(topic.id, article.topicId)) + .where( + and( + eq(article.organizationId, organizationId), + includeUnreadable ? undefined : readableWhere, + isNotNull(article.embedding), + eq(article.embeddingModel, embeddingModel) + ) + ) + // `id` breaks a tie so the order is total; two articles can sit at the + // same distance and a page that reorders between renders looks broken. + .orderBy(asc(distance), asc(article.id)) + .limit(limit) + ); + }; + + /** + * Store an article's embedding, but only if the article has not changed. + * + * The guard is title and content equality, the same one the post embedding + * job uses: the job runs after the write, and an editor who saves again in + * that window must not end up with a vector describing the text they + * replaced. A refresh that matches nothing is not an error — it is a write + * that arrived too late. + * + * `updatedAt` is deliberately not touched: this is an index refresh, not a + * change to the article, and bumping it would make every save look like two. + */ + const saveEmbedding = ({ + articleId, + content, + model, + now, + organizationId, + title, + vector, + }: { + articleId: string; + content: string; + model: string; + now: Date; + organizationId: string; + title: string; + vector: readonly number[]; + }) => + db + .update(article) + .set({ + embeddedAt: now, + embedding: [...vector], + embeddingModel: model, + }) + .where( + and( + eq(article.id, articleId), + eq(article.organizationId, organizationId), + eq(article.title, title), + eq(article.content, content) + ) + ) + .returning({ id: article.id }) + .pipe(Effect.map((rows) => rows.length > 0)); + /** The row's lifecycle state and publication date, for the write path. */ const findStateById = ({ organizationId, @@ -390,6 +626,23 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { }, }); + // The daily bucket, in the same transaction as the counter: a view + // that moved one and not the other would leave the dashboard + // disagreeing with the article page, and the retry helper replays + // the whole body so a retried beacon still counts once. + yield* db + .insert(daily) + .values({ + articleId, + day: DateTime.formatIsoDateUtc(DateTime.fromDateUnsafe(now)), + organizationId, + views: 1, + }) + .onConflictDoUpdate({ + target: [daily.articleId, daily.day], + set: { views: sql`${daily.views} + 1` }, + }); + return true; }) ) @@ -408,7 +661,167 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { const deleteEventsBefore = (before: Date) => db.delete(viewEvent).where(lt(viewEvent.occurredAt, before)); + /** + * Count one search, and whether it found anything. + * + * A query that normalized to nothing is not recorded: it is a box the reader + * cleared, not a question the documentation failed to answer, and counting it + * would make "searches that found nothing" mostly empty strings. + * + * The caller swallows a failure here (see the public search handler): + * analytics must never fail the request that produced it, which is the same + * contract the post-traffic ingest states. + */ + const recordSearch = ({ + organizationId, + query, + found, + now, + }: { + organizationId: string; + query: string; + found: boolean; + now: DateTime.DateTime; + }) => { + const normalized = normalizeHelpCenterSearchQuery(query); + + if (normalized.length === 0) { + return Effect.void; + } + + return db + .insert(searchDaily) + .values({ + day: DateTime.formatIsoDateUtc(now), + emptySearches: found ? 0 : 1, + organizationId, + query: normalized, + searches: 1, + }) + .onConflictDoUpdate({ + target: [ + searchDaily.organizationId, + searchDaily.day, + searchDaily.query, + ], + set: { + emptySearches: sql`${searchDaily.emptySearches} + ${found ? 0 : 1}`, + searches: sql`${searchDaily.searches} + 1`, + }, + }) + .pipe(Effect.asVoid); + }; + + /** + * The dashboard's analytics for one window. + * + * Four reads over the same two tables, run together. `sinceDay` is a UTC + * calendar day supplied by the caller rather than computed here, so the + * window is read through `Clock` at the edge and a test can pin it. + * + * `articles` keeps the verdict counters beside the views because the page + * asks two questions of one table: what is read, and what is read and rated + * unhelpful. A separate "lowest rated" query would be a fourth round trip to + * answer what a `helpful` column in the same row already shows. + */ + const analytics = ({ + organizationId, + sinceDay, + limit, + }: { + organizationId: string; + sinceDay: string; + limit: number; + }) => { + const inWindow = and( + eq(daily.organizationId, organizationId), + gte(daily.day, sinceDay) + ); + + const searchesInWindow = and( + eq(searchDaily.organizationId, organizationId), + gte(searchDaily.day, sinceDay) + ); + + return Effect.all( + { + totals: db + .select({ + views: sql`coalesce(sum(${daily.views}), 0)::int`, + helpful: sql`coalesce(sum(${daily.helpfulVerdicts}), 0)::int`, + unhelpful: sql`coalesce(sum(${daily.unhelpfulVerdicts}), 0)::int`, + }) + .from(daily) + .where(inWindow) + .pipe( + Effect.map( + (rows) => rows[0] ?? { helpful: 0, unhelpful: 0, views: 0 } + ) + ), + + articles: db + .select({ + id: article.id, + title: article.title, + slug: article.slug, + status: article.status, + views: sql`coalesce(sum(${daily.views}), 0)::int`, + helpful: sql`coalesce(sum(${daily.helpfulVerdicts}), 0)::int`, + unhelpful: sql`coalesce(sum(${daily.unhelpfulVerdicts}), 0)::int`, + }) + .from(daily) + .innerJoin(article, eq(article.id, daily.articleId)) + .where(inWindow) + .groupBy(article.id, article.title, article.slug, article.status) + .orderBy( + sql`coalesce(sum(${daily.views}), 0) desc`, + asc(article.title) + ) + .limit(limit), + + queries: db + .select({ + query: searchDaily.query, + searches: sql`coalesce(sum(${searchDaily.searches}), 0)::int`, + emptySearches: sql`coalesce(sum(${searchDaily.emptySearches}), 0)::int`, + }) + .from(searchDaily) + .where(searchesInWindow) + .groupBy(searchDaily.query) + .orderBy( + sql`coalesce(sum(${searchDaily.searches}), 0) desc`, + asc(searchDaily.query) + ) + .limit(limit), + + /** + * The documentation gap: queries that returned nothing, most frequent + * first. A subgroup of `queries` by construction, and a separate read + * because the window's most-searched phrase and its most-fruitless one + * are routinely different phrases. + */ + emptyQueries: db + .select({ + query: searchDaily.query, + searches: sql`coalesce(sum(${searchDaily.searches}), 0)::int`, + emptySearches: sql`coalesce(sum(${searchDaily.emptySearches}), 0)::int`, + }) + .from(searchDaily) + .where(searchesInWindow) + .groupBy(searchDaily.query) + .having(sql`sum(${searchDaily.emptySearches}) > 0`) + .orderBy( + sql`coalesce(sum(${searchDaily.emptySearches}), 0) desc`, + asc(searchDaily.query) + ) + .limit(limit), + }, + { concurrency: "unbounded" } + ); + }; + return { + analytics, deleteEventsBefore, findMany, findManyPublic, @@ -419,7 +832,11 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { create, update, remove, + recordSearch, recordView, + saveEmbedding, + search, + searchByVector, }; }); diff --git a/packages/domain/src/help-center-article/rpcs.ts b/packages/domain/src/help-center-article/rpcs.ts index 3a4acd8cd..7796595c6 100644 --- a/packages/domain/src/help-center-article/rpcs.ts +++ b/packages/domain/src/help-center-article/rpcs.ts @@ -6,6 +6,8 @@ import { PublicRpcRateLimitMiddleware, RateLimitErrors } from "../rate-limit"; import { AuthMiddleware, OptionalAuthMiddleware } from "../session-middleware"; import { HelpCenterArticleServiceErrors } from "./errors"; import { + HelpCenterAnalytics, + HelpCenterAnalyticsPayload, HelpCenterArticle, HelpCenterArticleCreate, HelpCenterArticleDelete, @@ -15,6 +17,10 @@ import { HelpCenterArticleListPublic, HelpCenterArticlePublic, HelpCenterArticlePublicSummary, + HelpCenterArticleSearch, + HelpCenterArticleSearchHit, + HelpCenterArticleSearchPublic, + HelpCenterArticleSearchResult, HelpCenterArticleSummary, HelpCenterArticleUpdate, HelpCenterArticleViewAccepted, @@ -87,5 +93,48 @@ export class HelpCenterArticleRpcs extends RpcGroup.make( error: Schema.Union([HelpCenterArticleServiceErrors, RateLimitErrors]), }) .middleware(OptionalAuthMiddleware) - .middleware(PublicRpcRateLimitMiddleware) + .middleware(PublicRpcRateLimitMiddleware), + + /** + * The dashboard's search: every state, including the draft being written. + * + * Behind the session middleware rather than the public limiter because an + * editor is already authenticated, and behind no extra permission because + * every member may read the documentation. + */ + Rpc.make("SearchHelpCenterArticles", { + payload: HelpCenterArticleSearch, + success: Schema.Array(HelpCenterArticleSearchHit), + error: HelpCenterArticleServiceErrors, + }).middleware(AuthMiddleware), + + /** + * The portal's search. + * + * It returns the same projection the public list does, and it is the one + * public read that also **writes**: the query is counted, with whether it + * found anything, because "what do readers look for and not find" is the + * only signal that says which article to write next. The write never fails + * the read. + */ + Rpc.make("SearchHelpCenterArticlesPublic", { + payload: HelpCenterArticleSearchPublic, + success: Schema.Array(HelpCenterArticleSearchResult), + error: Schema.Union([HelpCenterArticleServiceErrors, RateLimitErrors]), + }) + .middleware(OptionalAuthMiddleware) + .middleware(PublicRpcRateLimitMiddleware), + + /** + * The dashboard's analytics for one window. + * + * One operation for the whole panel, for the same reason the insights page is + * one: the page renders every number together, and a per-metric operation + * would make the panel's cost the sum of its round trips. + */ + Rpc.make("GetHelpCenterAnalytics", { + payload: HelpCenterAnalyticsPayload, + success: HelpCenterAnalytics, + error: HelpCenterArticleServiceErrors, + }).middleware(AuthMiddleware) ) {} diff --git a/packages/domain/src/help-center-article/schema.ts b/packages/domain/src/help-center-article/schema.ts index 4af3ec6ed..5cd25069d 100644 --- a/packages/domain/src/help-center-article/schema.ts +++ b/packages/domain/src/help-center-article/schema.ts @@ -10,6 +10,8 @@ import { import * as Effect from "effect/Effect"; import * as S from "effect/Schema"; +import { HELP_CENTER_SEARCH_MAX_QUERY_LENGTH } from "./search"; + const ArticleTitle = S.Trim.pipe( S.check(S.isMinLength(1)), S.check(S.isMaxLength(200)) @@ -160,6 +162,178 @@ export type THelpCenterArticleGetPublic = S.Schema.Type< typeof HelpCenterArticleGetPublic >; +/** + * The search box's query, on either surface. + * + * `query` is bounded by the same constant the stored analytics key uses, so + * the string a reader typed is the string the dashboard counts. It is trimmed + * for the same reason: a leading space is not a different search. + */ +const SearchQuery = S.Trim.pipe( + S.check(S.isMaxLength(HELP_CENTER_SEARCH_MAX_QUERY_LENGTH)) +); + +/** + * How many hits a search may return. + * + * A search box shows a handful and then stops; `20` is the ceiling rather than + * the page size, so a caller cannot turn a search into a table scan by asking + * for everything. + */ +const SEARCH_RESULT_LIMIT = 20; + +const SearchLimit = S.Int.pipe( + S.check(S.isGreaterThanOrEqualTo(1)), + S.check(S.isLessThanOrEqualTo(SEARCH_RESULT_LIMIT)) +); + +export const HelpCenterArticleSearch = S.Struct({ + organizationId: WorkspaceId.schema, + query: SearchQuery, + limit: S.optional(SearchLimit), +}); + +export type THelpCenterArticleSearch = S.Schema.Type< + typeof HelpCenterArticleSearch +>; + +/** + * A search hit on the dashboard. + * + * It carries `status` and no body: the list is a jump-to-the-editor affordance, + * so the editor navigates to the article and reads it from its own operation. + * + * There is no `rank` field. Order is the contract — the server decides it, and + * `ts_rank_cd` is one of two scores that produced it, neither of which is a + * number a caller could use for anything but re-sorting a list that is already + * sorted. + */ +export const HelpCenterArticleSearchHit = S.Struct({ + id: S.String, + slug: S.String, + title: S.String, + excerpt: S.String, + status: HelpCenterArticleState, + topicId: S.String, + topicSlug: S.String, + topicName: S.String, +}); + +export type THelpCenterArticleSearchHit = S.Schema.Type< + typeof HelpCenterArticleSearchHit +>; + +export const HelpCenterArticleSearchPublic = S.Struct({ + organizationId: WorkspaceId.schema, + query: SearchQuery, + limit: S.optional(SearchLimit), +}); + +export type THelpCenterArticleSearchPublic = S.Schema.Type< + typeof HelpCenterArticleSearchPublic +>; + +/** + * A search hit on the portal. + * + * The same projection the public list carries — slug, title, excerpt, topic — + * and no `status`: a reader needs "this page is out of date", not the + * workspace's internal lifecycle state (ADR 0004's rule, applied to a portal + * payload). + */ +export const HelpCenterArticleSearchResult = S.Struct({ + id: S.String, + slug: S.String, + title: S.String, + excerpt: S.String, + topicSlug: S.String, + topicName: S.String, + isArchived: S.Boolean, +}); + +export type THelpCenterArticleSearchResult = S.Schema.Type< + typeof HelpCenterArticleSearchResult +>; + +/** + * The windows the help center's analytics offers. + * + * The same three the insights page offers, deliberately: two pages that report + * "last week" and disagree about where it starts are a support ticket. They + * are declared here rather than imported from `insights` so a change to one + * page's windows is a decision about that page. + */ +export const HelpCenterAnalyticsRange = S.Literals(["7d", "30d", "90d"]); + +export type THelpCenterAnalyticsRange = S.Schema.Type< + typeof HelpCenterAnalyticsRange +>; + +/** Days back from "now" for each window, in UTC. */ +export const HELP_CENTER_ANALYTICS_RANGE_DAYS = { + "7d": 7, + "30d": 30, + "90d": 90, +} as const satisfies Record; + +export const HelpCenterAnalyticsPayload = S.Struct({ + organizationId: WorkspaceId.schema, + range: HelpCenterAnalyticsRange, +}); + +export type THelpCenterAnalyticsPayload = S.Schema.Type< + typeof HelpCenterAnalyticsPayload +>; + +/** + * One article's traffic and verdicts over the window. + * + * `views` and the two verdict counters answer different questions and are not + * expected to add up: a view is a read, a verdict is a judgment, and most + * readers leave neither. `status` travels so the dashboard can tell a quiet + * published article from a quiet draft. + */ +export const HelpCenterAnalyticsArticle = S.Struct({ + id: S.String, + title: S.String, + slug: S.String, + status: HelpCenterArticleState, + views: S.Finite, + helpful: S.Finite, + unhelpful: S.Finite, +}); + +export type THelpCenterAnalyticsArticle = S.Schema.Type< + typeof HelpCenterAnalyticsArticle +>; + +/** One query and how often it was asked, and how often it found nothing. */ +export const HelpCenterAnalyticsQuery = S.Struct({ + query: S.String, + searches: S.Finite, + emptySearches: S.Finite, +}); + +export type THelpCenterAnalyticsQuery = S.Schema.Type< + typeof HelpCenterAnalyticsQuery +>; + +export const HelpCenterAnalytics = S.Struct({ + range: HelpCenterAnalyticsRange, + /** `YYYY-MM-DD`, the first day the window includes, in UTC. */ + since: S.String, + totals: S.Struct({ + views: S.Finite, + helpful: S.Finite, + unhelpful: S.Finite, + }), + articles: S.Array(HelpCenterAnalyticsArticle), + queries: S.Array(HelpCenterAnalyticsQuery), + emptyQueries: S.Array(HelpCenterAnalyticsQuery), +}); + +export type THelpCenterAnalytics = S.Schema.Type; + const publishedAtWithDefault = S.NullOr(S.DateFromString).pipe( S.withDecodingDefaultKey(Effect.succeed(null)) ); diff --git a/packages/domain/src/help-center-article/search.test.ts b/packages/domain/src/help-center-article/search.test.ts new file mode 100644 index 000000000..4f656aa64 --- /dev/null +++ b/packages/domain/src/help-center-article/search.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "@effect/vitest"; + +import { + HELP_CENTER_SEARCH_MAX_QUERY_LENGTH, + HELP_CENTER_SEARCH_MAX_TERMS, + helpCenterSearchTerms, + helpCenterSearchTsQuery, + normalizeHelpCenterSearchQuery, +} from "./search"; + +describe("normalizeHelpCenterSearchQuery", () => { + it("folds case and spacing into one form", () => { + expect(normalizeHelpCenterSearchQuery(" Export CSV ")).toBe( + "export csv" + ); + }); + + it("bounds the stored query to what the box accepts", () => { + const normalized = normalizeHelpCenterSearchQuery("a".repeat(500)); + + expect(normalized).toHaveLength(HELP_CENTER_SEARCH_MAX_QUERY_LENGTH); + }); +}); + +describe("helpCenterSearchTerms", () => { + it("drops single characters and duplicates", () => { + expect(helpCenterSearchTerms("I can export export a board")).toStrictEqual([ + "can", + "export", + "board", + ]); + }); + + it("keeps only letters and digits, so nothing reaches the tsquery parser", () => { + // Every one of these characters is `to_tsquery` syntax. + expect(helpCenterSearchTerms("a&b|c!d(e)f:g'h")).toStrictEqual([]); + expect(helpCenterSearchTerms("export & import")).toStrictEqual([ + "export", + "import", + ]); + }); + + it("bounds the number of terms", () => { + const terms = helpCenterSearchTerms( + Array.from({ length: 40 }, (_, index) => `word${index}`).join(" ") + ); + + expect(terms).toHaveLength(HELP_CENTER_SEARCH_MAX_TERMS); + }); +}); + +describe("helpCenterSearchTsQuery", () => { + it("ORs prefix branches", () => { + expect(helpCenterSearchTsQuery(["export", "csv"])).toBe("export:* | csv:*"); + }); + + it("answers null when there is nothing to match on", () => { + expect(helpCenterSearchTsQuery([])).toBeNull(); + expect(helpCenterSearchTsQuery(helpCenterSearchTerms("!!!"))).toBeNull(); + }); +}); diff --git a/packages/domain/src/help-center-article/search.ts b/packages/domain/src/help-center-article/search.ts new file mode 100644 index 000000000..e1c25a5f1 --- /dev/null +++ b/packages/domain/src/help-center-article/search.ts @@ -0,0 +1,101 @@ +/** + * The help center's full-text search, as pure functions. + * + * The query reaches Postgres as a `tsquery`, and `to_tsquery` throws on + * malformed input. Everything that turns a reader's text into that form lives + * here so the only strings the repository ever binds are ones this module + * produced: letters, digits and the `:*` prefix marker, and nothing else. A + * raw `&`, `|`, `!`, `(`, `)`, `:` or `'` in a reader's question cannot reach + * the parser because the tokenizer drops it first. + * + * The stemmer and the stopword list are `to_tsvector`'s, not ours: the column + * is generated with the `english` configuration, so the query has to be built + * with the same one or the two would tokenize differently. That is also why + * there is no stopword list here — a query of only stopwords becomes an empty + * `tsquery`, which matches nothing, which is the honest answer to "the do of". + */ + +/** + * How many distinct words one search may match on. + * + * A bound rather than a quality heuristic: every extra term is another branch + * the GIN index walks, and a pasted paragraph is a denial-of-service shape + * rather than a search. Twelve covers a question with room to spare. + */ +export const HELP_CENTER_SEARCH_MAX_TERMS = 12; + +/** + * The longest query the search box accepts, and the longest string stored in + * `help_center_search_daily`. + * + * One constant for both because they are the same decision: the analytics + * reports the searches the box allowed, and a stored query longer than the box + * accepts could only come from a different client. + */ +export const HELP_CENTER_SEARCH_MAX_QUERY_LENGTH = 120; + +/** A run of letters or digits; everything else separates words. */ +const wordPattern = /[\p{L}\p{N}]+/gu; + +/** + * The form a query is compared and counted in: lowercased, single-spaced, + * trimmed and bounded. + * + * Case and spacing are not meaning, so ` Export CSV ` and `export csv` are + * one bucket in the daily counters. Accents are left alone — folding them is a + * language decision, and the `english` text-search configuration does not fold + * them either, so doing it here would make the analytics disagree with the + * results. + */ +export const normalizeHelpCenterSearchQuery = (input: string): string => + input + .toLowerCase() + .replace(/\s+/gu, " ") + .trim() + .slice(0, HELP_CENTER_SEARCH_MAX_QUERY_LENGTH); + +/** + * The distinct topic words of a query, in the order the reader wrote them. + * + * Single characters are dropped: `to_tsquery` accepts them, but a one-letter + * term matches most of a corpus and ranks on noise. Duplicates are dropped + * because `to_tsquery` ORs its branches, so a repeated word adds work without + * adding recall. + */ +export const helpCenterSearchTerms = (input: string): readonly string[] => { + const words = input.toLowerCase().match(wordPattern) ?? []; + const terms: string[] = []; + + for (const word of words) { + if (word.length < 2 || terms.includes(word)) { + continue; + } + + terms.push(word); + + if (terms.length === HELP_CENTER_SEARCH_MAX_TERMS) { + break; + } + } + + return terms; +}; + +/** + * The `to_tsquery` text for a set of terms, or `null` when there are none. + * + * Every branch is a prefix match (`:*`) AND-ed into nothing and OR-ed together, + * so partial words find articles: a reader who has typed "export" should see + * "Exporting your board" before the rest of the word arrives. Precision comes + * from the ranking, not from demanding the whole word — `ts_rank_cd` already + * puts a title match above a body match, and a whole-word requirement would + * turn a slightly different inflection into no results at all. + * + * `null` rather than an empty string: an empty `tsquery` is a valid value that + * matches nothing, and the caller should skip the database round trip instead + * of running a query guaranteed to return no rows. + */ +export const helpCenterSearchTsQuery = ( + terms: readonly string[] +): string | null => + terms.length === 0 ? null : terms.map((term) => `${term}:*`).join(" | "); diff --git a/packages/domain/src/site/url.ts b/packages/domain/src/site/url.ts new file mode 100644 index 000000000..285b7c2e3 --- /dev/null +++ b/packages/domain/src/site/url.ts @@ -0,0 +1,93 @@ +import * as Config from "effect/Config"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; + +/** + * The URL a workspace's public portal is served at. + * + * One rule, one place. The portal is reachable through a custom domain when the + * workspace verified one and through `.` otherwise, and the + * two surfaces that have to name it — a link inside an email and a link inside + * the widget iframe — must produce the same string. A second copy of this + * function is how an email and a widget start pointing at different hosts after + * a rename. + */ +export type PublicSiteHost = { + readonly subdomain: string; + readonly customDomain: string | null; +}; + +/** A scheme prefix and any trailing slashes are not part of a host. */ +const urlSchemePattern = /^[a-z][a-z0-9+.-]*:\/\//iu; + +const trailingSlashPattern = /\/+$/u; + +/** + * `https://`, or `https://.`. + * + * Always `https`, because both a custom domain and a subdomain are served over + * TLS; there is no plain-HTTP portal to point at, and a scheme carried in the + * stored value would be a second thing to validate. + */ +export const buildPublicSiteUrl = ( + site: PublicSiteHost, + appRootDomain: string +): string => { + if (site.customDomain) { + const host = site.customDomain + .replace(urlSchemePattern, "") + .replace(trailingSlashPattern, ""); + + return `https://${host}`; + } + + return `https://${site.subdomain}.${appRootDomain}`; +}; + +/** + * The deployment's root domain, and the app's own origin. + * + * `APP_ROOT_DOMAIN` is the tenant root (`feeblo.example`); when it is unset the + * host of `APP_URL` stands in, which is what keeps a local or single-host + * deployment's portal links usable instead of `.`. + * + * `APP_URL` is read leniently here and strictly in `auth/config.ts`, and the + * asymmetry is deliberate: authentication cannot work without it, so a + * deployment that is missing it has already failed loudly, while this service + * only builds links. Making a link builder strict would turn "the help center's + * suggestion has a wrong host" into "the widget API does not start", and it + * would require every test that builds the widget layer to configure an origin + * it does not use. + * + * `appUrl` is kept because not every public link is a portal link: a surface + * that has no site row to read still needs the app's origin, and reading it + * here means `APP_URL` is parsed once and validated as a URL. + */ +export class PortalUrlConfig extends Context.Service()( + "PortalUrlConfig", + { + make: Effect.gen(function* () { + const appUrl = yield* Config.schema(Schema.URLFromString, "APP_URL").pipe( + Config.withDefault(new URL("http://localhost")) + ); + + const configuredRootDomain = yield* Config.String("APP_ROOT_DOMAIN").pipe( + Config.withDefault("") + ); + + const trimmedRootDomain = configuredRootDomain + .trim() + .replace(trailingSlashPattern, ""); + + return { + appUrl: appUrl.href.replace(trailingSlashPattern, ""), + rootDomain: + trimmedRootDomain.length > 0 ? trimmedRootDomain : appUrl.host, + } as const; + }), + } +) { + static readonly layer = Layer.effect(this, this.make); +} diff --git a/packages/domain/src/widget/api-contract.ts b/packages/domain/src/widget/api-contract.ts index b2ec367f1..a3569e7ae 100644 --- a/packages/domain/src/widget/api-contract.ts +++ b/packages/domain/src/widget/api-contract.ts @@ -16,6 +16,8 @@ import { WidgetBoard, WidgetFeedbackCreate, WidgetFeedbackResponse, + WidgetHelpArticleRequest, + WidgetHelpArticles, WidgetSuggestion, WidgetSuggestionRequest, WidgetUpdate, @@ -52,6 +54,23 @@ export class WidgetApiGroup extends HttpApiGroup.make("WidgetApiGroup") .annotate(OpenApi.Title, "Suggest Posts") .annotate(OpenApi.Summary, "Find similar public posts") ) + .add( + HttpApiEndpoint.post("helpArticles", "/help-articles", { + payload: WidgetHelpArticleRequest, + success: WidgetHelpArticles, + error: Schema.Union([ + DataValidationError, + InternalServerError, + RateLimitErrors, + ]), + }) + .annotate(OpenApi.Title, "Help Articles") + .annotate(OpenApi.Summary, "Find help center articles that may answer it") + .annotate( + OpenApi.Description, + "Returns published help center articles that match the question being written, so the composer can offer existing documentation before a new post is created." + ) + ) .add( HttpApiEndpoint.post("createFeedback", "/feedback", { payload: WidgetFeedbackCreate, diff --git a/packages/domain/src/widget/api-live.ts b/packages/domain/src/widget/api-live.ts index 0fc2a0747..34b625fc9 100644 --- a/packages/domain/src/widget/api-live.ts +++ b/packages/domain/src/widget/api-live.ts @@ -21,6 +21,9 @@ import { DataValidationError } from "../contact/errors"; import { ContactRepository } from "../contact/repository"; import { parsePersonAttributes } from "../contact/utils"; import { EntitlementPolicy } from "../entitlement/policies"; +import { makeHelpCenterSearch } from "../help-center-article/hybrid-search"; +import { HelpCenterArticleRepository } from "../help-center-article/repository"; +import { HelpCenterEnabled } from "../help-center/flag"; import { Api } from "../http/api"; import { JwtSecretRepository } from "../jwt-secret/repository"; import { @@ -42,6 +45,8 @@ import { withRemapDbErrors, } from "../rpc-errors"; import { SitePolicy } from "../site/policies"; +import { SiteRepository } from "../site/repository"; +import { buildPublicSiteUrl, PortalUrlConfig } from "../site/url"; import { ParticipationPolicy } from "../user-ban/policies"; import { WidgetCreateFeedbackErrors } from "./api-contract"; import { @@ -123,6 +128,94 @@ export const listWidgetUpdates = Effect.fn("Widget.listUpdates")(function* ({ }); }); +/** + * Help-center articles that may already answer the question being written. + * + * Three answers, one shape. `enabled: false` covers every case where the + * surface does not exist — the feature is off for the deployment, the + * workspace is on a site with no help center, or the workspace has none — and + * the widget needs that as a value rather than a failure so it can stop asking + * instead of polling an endpoint that will keep returning nothing. + * + * The visibility check is the predicate `SitePolicy.canViewHelpCenter` + * applies, answered as "nothing" rather than an error. That difference is + * deliberate here: this endpoint is unauthenticated and names the workspace in + * its body, so a distinguishable failure would confirm which workspaces exist. + * The same reasoning keeps a hidden changelog answering `[]`. + * + * The search is `strict`, so a hit means the term is in an article's title or + * summary. A suggestion beside a feedback form is only worth showing if it + * names the thing being asked about; a body match on a passing mention would be + * a worse suggestion than none. + */ +const findWidgetHelpArticles = Effect.fn("Widget.helpArticles")(function* ({ + organizationId, + query, +}: { + organizationId: string; + query: string; +}) { + const enabled = yield* HelpCenterEnabled; + const siteRepository = yield* SiteRepository; + + if (!enabled) { + return { articles: [], enabled: false }; + } + + const site = yield* siteRepository + .findByOrganizationId({ organizationId }) + .pipe(Effect.map(Option.getOrUndefined)); + + if (site === undefined || site.helpCenterVisibility !== "PUBLIC") { + return { articles: [], enabled: false }; + } + + const repository = yield* HelpCenterArticleRepository; + const config = yield* PortalUrlConfig; + + // The same program the dashboard and portal run, built without an embedder: + // the deflection is strict and a strict search never reaches the semantic + // pass, so a deployment with no API key still answers it. Going through the + // program rather than straight to the repository is what keeps "strict" + // meaning one thing across the three surfaces. + const search = makeHelpCenterSearch({ + embeddings: Option.none(), + lexical: repository.search, + semantic: repository.searchByVector, + }); + + const hits = yield* search({ + includeUnreadable: false, + limit: 3, + organizationId, + query, + strict: true, + }); + + // Counted, because "the widget offered documentation and the reader posted + // anyway" and "the widget found nothing" are the two facts that say whether + // the deflection is working. A failed counter never fails the suggestion. + const now = yield* DateTime.now; + + yield* repository + .recordSearch({ found: hits.length > 0, now, organizationId, query }) + .pipe(Effect.ignoreCause); + + const portalUrl = buildPublicSiteUrl(site, config.rootDomain); + + return { + articles: hits.map((hit) => ({ + excerpt: hit.excerpt, + id: hit.id, + title: hit.title, + // Absolute, because the iframe is embedded on the customer's own origin: + // a relative link would resolve against the host page. + url: `${portalUrl}/s/help/articles/${encodeURIComponent(hit.slug)}`, + })), + enabled: true, + }; +}); + /** * Resolves the SSO customer a feedback submission is attributed to. * @@ -259,6 +352,9 @@ export const WidgetApiLive = HttpApiBuilder.group( const postRepository = yield* PostRepository; const postEmbeddings = yield* PostEmbeddingService; const participation = yield* ParticipationPolicy; + const helpCenterArticles = yield* HelpCenterArticleRepository; + const siteRepository = yield* SiteRepository; + const portalUrlConfig = yield* PortalUrlConfig; // The same suggestion program the dashboard and public portal RPCs run, // with the widget's public-only rule. Its collaborators are captured @@ -290,7 +386,13 @@ export const WidgetApiLive = HttpApiBuilder.group( Effect.provideService(JwtSecretRepository, jwtSecretRepository), Effect.provideService(OrganizationRepository, organizationRepository), Effect.provideService(PostStatusRepository, postStatusRepository), - Effect.provideService(SitePolicy, sitePolicy) + Effect.provideService(SitePolicy, sitePolicy), + Effect.provideService( + HelpCenterArticleRepository, + helpCenterArticles + ), + Effect.provideService(SiteRepository, siteRepository), + Effect.provideService(PortalUrlConfig, portalUrlConfig) ); return handlers @@ -328,6 +430,22 @@ export const WidgetApiLive = HttpApiBuilder.group( }) ) ) + .handle("helpArticles", ({ payload }) => + findWidgetHelpArticles(payload).pipe( + // A search that fails is answered as "nothing to offer", not as an + // error: the composer's primary job is creating feedback, and a + // documentation hint that cannot be fetched must not block it. + Effect.catchCause(() => + Effect.succeed({ articles: [], enabled: false }) + ), + provideHandlerEnvironment, + RateLimit.withPublicHttpRateLimit({ + name: "WidgetHelpArticles", + level: "read", + }), + withRemapDbErrors("HelpCenterArticle", "select") + ) + ) .handle("listBoards", ({ payload }) => Effect.gen(function* () { const repository = yield* BoardRepository; @@ -494,12 +612,20 @@ export const WidgetApiLive = HttpApiBuilder.group( ChangelogRepository.layer, CompanyRepository.layer, ContactRepository.layer, + HelpCenterArticleRepository.layer, JwtSecretRepository.layer, OrganizationRepository.layer, + PortalUrlConfig.layer, PostEmbeddingService.layer, PostRepository.layer, PostStatusRepository.layer, - SitePolicy.layer, + // Provided into the policy rather than beside it: `Layer.mergeAll` + // builds in parallel, so a layer that depends on a sibling in the same + // call is a layer whose dependency is not there yet. The repository is + // still in the merge for the handler's own lookup of the workspace's + // public host, which is why both appear. + SitePolicy.layer.pipe(Layer.provide(SiteRepository.layer)), + SiteRepository.layer, ParticipationPolicy.layer ) ) diff --git a/packages/domain/src/widget/schema.ts b/packages/domain/src/widget/schema.ts index 9c86e1037..0067640d4 100644 --- a/packages/domain/src/widget/schema.ts +++ b/packages/domain/src/widget/schema.ts @@ -118,6 +118,55 @@ export const WidgetSuggestion = S.Struct({ export type TWidgetSuggestion = S.Schema.Type; +/** + * The composer's help-center deflection. + * + * A separate request from `WidgetSuggestionRequest` because it answers a + * different question with a different corpus: that one asks "has someone + * already posted this?", this one asks "is this already documented?". The + * widget sends both for the same title, and the panels are distinct on screen + * because acting on one means upvoting and acting on the other means reading. + */ +export const WidgetHelpArticleRequest = S.Struct({ + organizationId: WorkspaceId.schema, + query: S.String.check(S.isMaxLength(WIDGET_TITLE_MAX_LENGTH)), +}); + +export type TWidgetHelpArticleRequest = S.Schema.Type< + typeof WidgetHelpArticleRequest +>; + +/** + * A documentation hit beside the feedback form. + * + * `url` is absolute and built server-side: the iframe is embedded on the + * customer's own origin, so a relative link would resolve against the host page + * instead of the portal. Only the portal's canonical help path is ever + * produced here. + */ +export const WidgetHelpArticle = S.Struct({ + id: S.String, + title: S.String, + excerpt: S.String, + url: S.String, +}); + +export type TWidgetHelpArticle = S.Schema.Type; + +/** + * The deflection's answer. + * + * `enabled` is the important half: the widget stops asking once a deployment or + * a workspace says no, instead of polling an endpoint that will keep returning + * nothing. + */ +export const WidgetHelpArticles = S.Struct({ + articles: S.Array(WidgetHelpArticle), + enabled: S.Boolean, +}); + +export type TWidgetHelpArticles = S.Schema.Type; + export const WidgetUpdate = S.Struct({ id: S.String, title: S.String, @@ -230,6 +279,8 @@ export const decodeWidgetSuggestions = S.decodeUnknownSync( S.Array(WidgetSuggestion) ); +export const decodeWidgetHelpArticles = S.decodeUnknownSync(WidgetHelpArticles); + export const decodeWidgetError = S.decodeUnknownSync(WidgetError); export const decodeWidgetBootEnv = S.decodeUnknownSync(WidgetBootEnv); diff --git a/packages/feedback-widget/src/components/feedback-form/context.ts b/packages/feedback-widget/src/components/feedback-form/context.ts index 78871177c..08da28a19 100644 --- a/packages/feedback-widget/src/components/feedback-form/context.ts +++ b/packages/feedback-widget/src/components/feedback-form/context.ts @@ -1,7 +1,11 @@ import type { useSubmission } from "@solidjs/router"; import { type Accessor, createContext, useContext } from "solid-js"; -import type { FeedbackResult, WidgetSuggestion } from "../../lib/api"; +import type { + FeedbackResult, + WidgetHelpArticle, + WidgetSuggestion, +} from "../../lib/api"; import type { Board } from "../../lib/boards"; type FeedbackFormSubmission = ReturnType< @@ -10,6 +14,10 @@ type FeedbackFormSubmission = ReturnType< export interface FeedbackFormState { content: Accessor; + helpArticles: Accessor; + /** False once the deployment or workspace has no help center to deflect to. */ + helpArticlesEnabled: Accessor; + helpArticlesPending: Accessor; submission: FeedbackFormSubmission; suggestions: Accessor; suggestionsPending: Accessor; diff --git a/packages/feedback-widget/src/components/feedback-form/create-help-article-search.ts b/packages/feedback-widget/src/components/feedback-form/create-help-article-search.ts new file mode 100644 index 000000000..c7fdb4493 --- /dev/null +++ b/packages/feedback-widget/src/components/feedback-form/create-help-article-search.ts @@ -0,0 +1,120 @@ +import { createSignal, onCleanup, untrack } from "solid-js"; + +import type { WidgetHelpArticles } from "../../lib/api"; +import { + helpArticleQuery, + HELP_ARTICLES_DEBOUNCE_MS, + initialHelpArticleSearchState, + reduceHelpArticleSearch, + type HelpArticleSearchEvent, + type HelpArticleSearchState, +} from "./help-article-search"; + +export type HelpArticleFetch = ( + query: string, + signal: AbortSignal +) => Promise; + +export type HelpArticleSearch = { + /** + * Reports the title field's current value. The trimming and the minimum + * length are applied here, so "nothing worth asking" never reaches a timer. + * + * Every call supersedes the previous one: the pending timer is cleared and + * the in-flight request aborted, so at most one request is ever outstanding. + * Once the server has said the workspace has no help center, this is a no-op. + */ + readonly input: (title: string) => void; + readonly state: () => HelpArticleSearchState; +}; + +/** + * Drives `reduceHelpArticleSearch` from Solid. + * + * The graph and every decision in it live in the reducer; this file is the + * timer, the abort controller and the signal, which is all that is left. The + * cleanup is registered against the calling owner, so a component that unmounts + * mid-debounce or mid-request cancels both. + */ +export function createHelpArticleSearch( + fetch: HelpArticleFetch +): HelpArticleSearch { + const [state, setState] = createSignal( + initialHelpArticleSearchState + ); + + let controller: AbortController | undefined; + let timer: number | undefined; + + const send = (event: HelpArticleSearchEvent) => { + setState((current) => reduceHelpArticleSearch(current, event)); + }; + + const stop = () => { + if (timer !== undefined) { + window.clearTimeout(timer); + timer = undefined; + } + + controller?.abort(); + controller = undefined; + }; + + const input = (title: string) => { + // The latch, at the only place a request can start. The reducer enforces it + // too, but returning here means a disabled workspace does not churn the + // signal, clear the panel, or arm a timer on every keystroke. + // + // Untracked on purpose: `input` is normally called from an effect that + // reports the title, and a tracked read here would subscribe that effect to + // the state this function writes — an effect that re-arms itself forever. + if (!untrack(state).enabled) { + return; + } + + // Abort before the reducer sees the event: the request is superseded either + // way, and stopping it early is what keeps a burst of keystrokes to one + // request instead of one request per keystroke. + stop(); + const query = helpArticleQuery(title); + send({ type: "INPUT", query }); + + if (query === null) { + return; + } + + timer = window.setTimeout(() => { + timer = undefined; + const request = new AbortController(); + controller = request; + send({ type: "DEBOUNCED" }); + fetch(query, request.signal) + .then((response) => { + // The reducer drops a late answer only while it is still + // `debouncing`; by the time a superseded request settles, the next + // one can already be `loading`, so the abort check is what keeps an + // old outcome from landing on the new query. + if (request.signal.aborted) { + return; + } + + send({ + articles: response.articles, + enabled: response.enabled, + type: "LOADED", + }); + }) + .catch(() => { + if (request.signal.aborted) { + return; + } + + send({ type: "FAILED" }); + }); + }, HELP_ARTICLES_DEBOUNCE_MS); + }; + + onCleanup(stop); + + return { input, state }; +} diff --git a/packages/feedback-widget/src/components/feedback-form/help-article-search.test.ts b/packages/feedback-widget/src/components/feedback-form/help-article-search.test.ts new file mode 100644 index 000000000..d12893978 --- /dev/null +++ b/packages/feedback-widget/src/components/feedback-form/help-article-search.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, it } from "vitest"; + +import type { WidgetHelpArticle } from "../../lib/api"; +import { + helpArticleQuery, + initialHelpArticleSearchState, + isHelpArticleSearchPending, + reduceHelpArticleSearch, + type HelpArticleSearchEvent, + type HelpArticleSearchState, +} from "./help-article-search"; + +/** + * The reducer is the whole graph, so these tests walk it event by event. The + * driver in `create-help-article-search.ts` adds only a timer and an abort + * controller; the minimum-length gate and the session latch are here. + */ + +const QUERY = "invite my team"; + +const article = (id: string): WidgetHelpArticle => ({ + excerpt: "excerpt", + id, + title: `Article ${id}`, + url: `https://acme.feeblo.test/help/${id}`, +}); + +const settle = ( + events: readonly HelpArticleSearchEvent[], + state: HelpArticleSearchState = initialHelpArticleSearchState +) => events.reduce(reduceHelpArticleSearch, state); + +const loaded = ( + enabled: boolean, + ...ids: string[] +): HelpArticleSearchEvent => ({ + articles: ids.map(article), + enabled, + type: "LOADED", +}); + +const ids = (state: HelpArticleSearchState) => + state.articles.map((entry) => entry.id); + +describe("help article search reducer", () => { + it("starts idle, enabled, with nothing to show", () => { + expect(initialHelpArticleSearchState).toStrictEqual({ + articles: [], + enabled: true, + status: "idle", + }); + }); + + it("asks only about a trimmed title of at least three characters", () => { + expect(helpArticleQuery("")).toBeNull(); + expect(helpArticleQuery(" ab ")).toBeNull(); + expect(helpArticleQuery(" abc ")).toBe("abc"); + expect(helpArticleQuery("Invite")).toBe("Invite"); + }); + + it("shows the answer to the query that is in flight", () => { + const state = settle([ + { query: QUERY, type: "INPUT" }, + { type: "DEBOUNCED" }, + loaded(true, "a", "b"), + ]); + + expect(state.status).toBe("ready"); + expect(ids(state)).toStrictEqual(["a", "b"]); + }); + + it("keeps the last result visible while a newer query is pending", () => { + // The flash this removes: the panel used to empty on every keystroke. + const settled = settle([ + { query: QUERY, type: "INPUT" }, + { type: "DEBOUNCED" }, + loaded(true, "a", "b"), + ]); + + const debouncing = reduceHelpArticleSearch(settled, { + query: "invite my team please", + type: "INPUT", + }); + + expect(ids(debouncing)).toStrictEqual(["a", "b"]); + expect(debouncing.status).toBe("debouncing"); + + const loading = reduceHelpArticleSearch(debouncing, { type: "DEBOUNCED" }); + expect(ids(loading)).toStrictEqual(["a", "b"]); + expect(loading.status).toBe("loading"); + }); + + it("drops an answer that arrives after the query changed", () => { + const inFlight = settle([ + { query: QUERY, type: "INPUT" }, + { type: "DEBOUNCED" }, + ]); + + const afterChange = reduceHelpArticleSearch(inFlight, { + query: "a different question", + type: "INPUT", + }); + + expect(reduceHelpArticleSearch(afterChange, loaded(true, "stale"))).toBe( + afterChange + ); + }); + + it("clears the panel when the title is too short to ask about", () => { + const state = settle([ + { query: QUERY, type: "INPUT" }, + { type: "DEBOUNCED" }, + loaded(true, "a"), + { query: null, type: "INPUT" }, + ]); + + expect(state).toStrictEqual({ + articles: [], + enabled: true, + status: "idle", + }); + }); + + it("empties the panel when the request fails", () => { + const state = settle([ + { query: QUERY, type: "INPUT" }, + { type: "DEBOUNCED" }, + loaded(true, "a"), + { query: "invite my team please", type: "INPUT" }, + { type: "DEBOUNCED" }, + { type: "FAILED" }, + ]); + + expect(state).toStrictEqual({ + articles: [], + enabled: true, + status: "failed", + }); + }); + + it("latches a disabled answer and never asks again", () => { + const disabled = settle([ + { query: QUERY, type: "INPUT" }, + { type: "DEBOUNCED" }, + // Articles alongside `enabled: false` are not documentation this + // workspace publishes, so the answer is the latch, not the payload. + loaded(false, "a"), + ]); + + expect(disabled).toStrictEqual({ + articles: [], + enabled: false, + status: "ready", + }); + + // The next keystroke and the debounce it would have armed are both no-ops: + // the workspace's answer outlives the query that asked for it. + const afterTyping = reduceHelpArticleSearch(disabled, { + query: "a different question", + type: "INPUT", + }); + + expect(afterTyping).toBe(disabled); + expect(reduceHelpArticleSearch(afterTyping, { type: "DEBOUNCED" })).toBe( + disabled + ); + expect(reduceHelpArticleSearch(afterTyping, loaded(true, "b"))).toBe( + disabled + ); + expect(reduceHelpArticleSearch(afterTyping, { type: "FAILED" })).toBe( + disabled + ); + }); + + it("reports pending only while a timer or a request is outstanding", () => { + const pendingFor = (events: readonly HelpArticleSearchEvent[]) => + isHelpArticleSearchPending(settle(events)); + + expect(pendingFor([])).toBe(false); + expect(pendingFor([{ query: QUERY, type: "INPUT" }])).toBe(true); + expect( + pendingFor([{ query: QUERY, type: "INPUT" }, { type: "DEBOUNCED" }]) + ).toBe(true); + expect( + pendingFor([ + { query: QUERY, type: "INPUT" }, + { type: "DEBOUNCED" }, + loaded(true, "a"), + ]) + ).toBe(false); + expect(pendingFor([{ query: null, type: "INPUT" }])).toBe(false); + }); +}); diff --git a/packages/feedback-widget/src/components/feedback-form/help-article-search.ts b/packages/feedback-widget/src/components/feedback-form/help-article-search.ts new file mode 100644 index 000000000..0b2da7193 --- /dev/null +++ b/packages/feedback-widget/src/components/feedback-form/help-article-search.ts @@ -0,0 +1,129 @@ +import type { WidgetHelpArticle } from "../../lib/api"; + +/** + * The lifecycle of the widget's "related documentation" panel, as a plain + * reducer. + * + * It sits beside `suggestion-search.ts` instead of reusing that reducer because + * the two graphs differ in more than their item type. This answer carries + * `enabled`, which is a property of the deployment and the workspace, not of + * the query, and it is terminal for the session: once the server says the + * workspace has no public help center, every later keystroke has to leave that + * value alone. The suggestion reducer's only durable state is the last result, + * and its `INPUT null` transition is explicitly allowed to clear all of it — + * folding this latch into that state would make a suggestion concern part of + * the help-center contract and vice versa. + * + * The parts that do carry over are the ones that matter on screen: + * + * - **The last settled result stays visible while the next one is pending**, so + * the panel does not flash on every keystroke. + * - **A result that arrives after the query changed is dropped.** `LOADED` and + * `FAILED` are only accepted from `loading`, so a superseded response has no + * transition to take. + */ + +/** How long the title must sit still before the help-center lookup is worth making. */ +export const HELP_ARTICLES_DEBOUNCE_MS = 450; + +/** Below this many characters there is nothing worth asking about. */ +export const HELP_ARTICLE_QUERY_MIN_LENGTH = 3; + +export type HelpArticleSearchStatus = + | "idle" + | "debouncing" + | "loading" + | "ready" + | "failed"; + +export type HelpArticleSearchState = { + /** The last settled result. Carried across a newer query so the panel does not flash. */ + readonly articles: readonly WidgetHelpArticle[]; + /** False once a response said there is no help center. Terminal for the session. */ + readonly enabled: boolean; + readonly status: HelpArticleSearchStatus; +}; + +export type HelpArticleSearchEvent = + /** The field changed. `query` is null when there is nothing worth asking. */ + | { readonly query: string | null; readonly type: "INPUT" } + /** The debounce elapsed; the driver has started the request. */ + | { readonly type: "DEBOUNCED" } + | { + readonly articles: readonly WidgetHelpArticle[]; + readonly enabled: boolean; + readonly type: "LOADED"; + } + | { readonly type: "FAILED" }; + +export const initialHelpArticleSearchState: HelpArticleSearchState = { + articles: [], + // Unknown until the server answers. Assuming "off" here would suppress the + // request that would have told us otherwise. + enabled: true, + status: "idle", +}; + +/** True while the panel is waiting on a timer or a request. */ +export const isHelpArticleSearchPending = ( + state: HelpArticleSearchState +): boolean => state.status === "debouncing" || state.status === "loading"; + +/** + * The title, trimmed, when it is long enough to be worth asking about. + * + * The gate lives here rather than in the provider so the rule is a pure value a + * test can check, and so the driver cannot arm a timer for a two-letter title. + */ +export const helpArticleQuery = (title: string): string | null => { + const query = title.trim(); + + return query.length < HELP_ARTICLE_QUERY_MIN_LENGTH ? null : query; +}; + +export const reduceHelpArticleSearch = ( + state: HelpArticleSearchState, + event: HelpArticleSearchEvent +): HelpArticleSearchState => { + // The latch is checked before every event rather than inside `INPUT`: once + // the answer is "there is no help center", no later keystroke may re-open + // the question, and no late settle may rewrite the answer. + if (!state.enabled) { + return state; + } + + switch (event.type) { + case "INPUT": + // Nothing to ask about: results that answer a question the visitor is no + // longer asking go, the one case where carrying them forward is wrong. + if (event.query === null) { + return state.status === "idle" && state.articles.length === 0 + ? state + : { articles: [], enabled: true, status: "idle" }; + } + + return { articles: state.articles, enabled: true, status: "debouncing" }; + case "DEBOUNCED": + return state.status === "debouncing" + ? { articles: state.articles, enabled: true, status: "loading" } + : state; + // The guard is the staleness check: a request whose query was superseded + // left the state in `debouncing`, so its answer has nowhere to land. + case "LOADED": + return state.status === "loading" + ? { + // A disabled answer is the whole answer: anything else the payload + // carries is not documentation this workspace publishes. + articles: event.enabled ? event.articles : [], + enabled: event.enabled, + status: "ready", + } + : state; + // A failure clears the panel: showing the previous query's results under a + // query the server just refused would look like an answer. + case "FAILED": + return state.status === "loading" + ? { articles: [], enabled: true, status: "failed" } + : state; + } +}; diff --git a/packages/feedback-widget/src/components/feedback-form/help-articles.tsx b/packages/feedback-widget/src/components/feedback-form/help-articles.tsx new file mode 100644 index 000000000..125a59314 --- /dev/null +++ b/packages/feedback-widget/src/components/feedback-form/help-articles.tsx @@ -0,0 +1,63 @@ +import { For, Show } from "solid-js"; + +import { Spinner } from "../ui/spinner"; +import { useFeedbackForm } from "./context"; + +/** + * The help-center half of the composer's deflection: documentation that may + * already answer the question, shown above the similar-posts panel. + * + * Each link opens the portal in a new tab. The widget is an iframe on the + * customer's own page, so navigating the iframe would strand the embedding page + * on a portal route; `noreferrer` also keeps the portal from learning which + * customer page the visitor came from. + */ +export function FeedbackFormHelpArticles() { + const { state } = useFeedbackForm(); + + return ( + // The latch is part of the visibility: once the workspace has no help + // center, a stale payload must not resurrect the panel for this session. + 0}> +
    +
    +
    +

    Related documentation

    +

    + This may already be answered in the help center. +

    +
    + + + +
    + +
    +
    + ); +} diff --git a/packages/feedback-widget/src/components/feedback-form/index.tsx b/packages/feedback-widget/src/components/feedback-form/index.tsx index e8444c2a2..58fe85143 100644 --- a/packages/feedback-widget/src/components/feedback-form/index.tsx +++ b/packages/feedback-widget/src/components/feedback-form/index.tsx @@ -4,6 +4,7 @@ import { FeedbackFormFrame } from "./feedback-form"; import { FeedbackFormFields } from "./fields"; import { FeedbackFormError } from "./form-error"; import { FeedbackFormHeader } from "./header"; +import { FeedbackFormHelpArticles } from "./help-articles"; import { BoardNotFound } from "./not-found"; import { FeedbackFormProvider } from "./provider"; import { FeedbackFormSubmitButton } from "./submit-button"; @@ -18,6 +19,7 @@ export const FeedbackForm = { Fields: FeedbackFormFields, TitleField: FeedbackFormTitleField, ContentField: FeedbackFormContentField, + HelpArticles: FeedbackFormHelpArticles, Suggestions: FeedbackFormSuggestions, Actions: FeedbackFormActions, SubmitButton: FeedbackFormSubmitButton, diff --git a/packages/feedback-widget/src/components/feedback-form/provider.tsx b/packages/feedback-widget/src/components/feedback-form/provider.tsx index c513e5a72..efd64cd68 100644 --- a/packages/feedback-widget/src/components/feedback-form/provider.tsx +++ b/packages/feedback-widget/src/components/feedback-form/provider.tsx @@ -1,10 +1,16 @@ import { useSubmission } from "@solidjs/router"; import { createEffect, createSignal, type JSX } from "solid-js"; -import { createFeedBackAction, fetchSuggestions } from "../../lib/api"; +import { + createFeedBackAction, + fetchSuggestions, + requestHelpArticles, +} from "../../lib/api"; import type { Board } from "../../lib/boards"; import { FeedbackFormContext, type FeedbackFormContextValue } from "./context"; +import { createHelpArticleSearch } from "./create-help-article-search"; import { createSuggestionSearch } from "./create-suggestion-search"; +import { isHelpArticleSearchPending } from "./help-article-search"; import { isSuggestionSearchPending, SUGGESTION_QUERY_MIN_LENGTH, @@ -12,14 +18,12 @@ import { /** * The only place that knows how the feedback form state is managed: - * title/content signals, the suggestion search, and the router submission. + * title/content signals, the two searches, and the router submission. * Consumers only see the context interface. * - * The suggestion lifecycle is not in this file. It used to be a `createEffect` - * holding a timer, an `AbortController` and an `isCurrent` flag; it is now - * `suggestion-search.ts` (the graph, as a pure reducer) and - * `create-suggestion-search.ts` (the timer and the abort). What is left here is - * the one line that reports the field. + * Neither search lifecycle is in this file. Each is a reducer holding the + * graph plus a driver holding the timer and the abort. What is left here is + * the one line that reports the field to each. */ export function FeedbackFormProvider(props: { board: Board; @@ -30,6 +34,20 @@ export function FeedbackFormProvider(props: { const [content, setContent] = createSignal(""); const suggestionSearch = createSuggestionSearch(fetchSuggestions); + const helpArticleSearch = createHelpArticleSearch((query, signal) => + requestHelpArticles( + { organizationId: props.board.organizationId, query }, + signal + ) + ); + + createEffect(() => { + // Separate effect from the suggestion search on purpose: that one also + // reads the content field, and sharing it would reset this debounce (and + // abort its request) on every content keystroke. + helpArticleSearch.input(title()); + }); + createEffect(() => { const nextTitle = title().trim(); const nextContent = content(); @@ -47,6 +65,10 @@ export function FeedbackFormProvider(props: { state: { title, content, + helpArticles: () => helpArticleSearch.state().articles, + helpArticlesEnabled: () => helpArticleSearch.state().enabled, + helpArticlesPending: () => + isHelpArticleSearchPending(helpArticleSearch.state()), suggestions: () => suggestionSearch.state().suggestions, suggestionsPending: () => isSuggestionSearchPending(suggestionSearch.state()), diff --git a/packages/feedback-widget/src/lib/api.test.ts b/packages/feedback-widget/src/lib/api.test.ts index ca42fb395..c670c6d82 100644 --- a/packages/feedback-widget/src/lib/api.test.ts +++ b/packages/feedback-widget/src/lib/api.test.ts @@ -6,6 +6,7 @@ import { fetchSuggestions, getOrganizationId, requestBoards, + requestHelpArticles, requestUpdates, submitFeedback, } from "./requests"; @@ -51,6 +52,13 @@ const suggestion = { slug: "similar", }; +const helpArticle = { + id: "hca_1", + title: "Invite your team", + excerpt: "Add teammates from workspace settings.", + url: "https://acme.feeblo.test/help/invite-your-team", +}; + const stubBootEnv = () => { vi.stubGlobal("global", { __ENV: { API_URL: apiUrl, organizationId } }); }; @@ -118,6 +126,49 @@ describe("widget api boundary", () => { expect(suggestions).toEqual([suggestion]); }); + it("posts a help-article query and decodes the answer", async () => { + stubBootEnv(); + let sentUrl: string | undefined; + let sentBody: Json | undefined; + + const fetchSpy = vi.fn((url: string, init?: RequestInit) => { + sentUrl = url; + const body = init?.body; + sentBody = isString(body) ? JSON.parse(body) : undefined; + + return Promise.resolve( + new Response( + JSON.stringify({ articles: [helpArticle], enabled: true }), + { status: 200 } + ) + ); + }); + + vi.stubGlobal("fetch", fetchSpy); + + const result = await requestHelpArticles( + { organizationId, query: "invite team" }, + new AbortController().signal + ); + + expect(fetchSpy).toHaveBeenCalledOnce(); + expect(sentUrl).toBe(`${apiUrl}//api/widget/v1/help-articles`); + expect(sentBody).toEqual({ organizationId, query: "invite team" }); + expect(result).toEqual({ articles: [helpArticle], enabled: true }); + }); + + it("decodes a disabled help-center answer instead of treating it as a failure", async () => { + stubBootEnv(); + stubFetch({ articles: [], enabled: false }); + + const result = await requestHelpArticles( + { organizationId, query: "invite team" }, + new AbortController().signal + ); + + expect(result).toEqual({ articles: [], enabled: false }); + }); + it("reads the organization id through the boot contract", () => { stubBootEnv(); diff --git a/packages/feedback-widget/src/lib/api.ts b/packages/feedback-widget/src/lib/api.ts index 6df7eb6a7..862ff3216 100644 --- a/packages/feedback-widget/src/lib/api.ts +++ b/packages/feedback-widget/src/lib/api.ts @@ -4,11 +4,13 @@ import { requestBoards, requestUpdates, submitFeedback } from "./requests"; export type { FeedbackResult, + WidgetHelpArticle, + WidgetHelpArticles, WidgetSuggestion, WidgetUpdate, } from "./requests"; -export { fetchSuggestions } from "./requests"; +export { fetchSuggestions, requestHelpArticles } from "./requests"; /** * The router-facing half of the widget's HTTP boundary. diff --git a/packages/feedback-widget/src/lib/requests.ts b/packages/feedback-widget/src/lib/requests.ts index 96a47b775..84aedff06 100644 --- a/packages/feedback-widget/src/lib/requests.ts +++ b/packages/feedback-widget/src/lib/requests.ts @@ -2,11 +2,14 @@ import { decodeWidgetBoards, decodeWidgetBootEnv, decodeWidgetError, + decodeWidgetHelpArticles, decodeWidgetSuggestions, decodeWidgetUpdates, type TWidgetBoard, type TWidgetBootEnv, type TWidgetFeedbackCreateWire, + type TWidgetHelpArticle, + type TWidgetHelpArticles, type TWidgetSuggestion, type TWidgetUpdate, } from "@feeblo/domain/widget/schema"; @@ -29,6 +32,10 @@ export type WidgetBoard = TWidgetBoard; export type WidgetSuggestion = TWidgetSuggestion; +export type WidgetHelpArticle = TWidgetHelpArticle; + +export type WidgetHelpArticles = TWidgetHelpArticles; + export type WidgetUpdate = TWidgetUpdate; export type FeedbackResult = { ok: true } | { ok: false; message: string }; @@ -97,6 +104,31 @@ export async function fetchSuggestions( return decodeWidgetSuggestions(await response.json()); } +/** + * Asks whether the help center already answers the question being written. + * + * Separate from `fetchSuggestions` rather than a widened one: the answer also + * carries `enabled`, which is a property of the deployment and the workspace + * instead of of the query, and the widget has to remember it past this request. + */ +export async function requestHelpArticles( + input: { organizationId: string; query: string }, + signal: AbortSignal +): Promise { + const response = await fetch(`${getApiBaseUrl()}/help-articles`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(input), + signal, + }); + + if (!response.ok) { + throw new Error(`Failed to fetch help articles: ${response.status}`); + } + + return decodeWidgetHelpArticles(await response.json()); +} + /** * Submits a feedback post. * diff --git a/packages/feedback-widget/src/routes/board.tsx b/packages/feedback-widget/src/routes/board.tsx index debb922b1..5939c34eb 100644 --- a/packages/feedback-widget/src/routes/board.tsx +++ b/packages/feedback-widget/src/routes/board.tsx @@ -50,6 +50,7 @@ function FeedbackFormScreen() { + From e9752f5fefd5c9f9d5b38789651781c0fef3b32c Mon Sep 17 00:00:00 2001 From: nafees nazik Date: Thu, 19 Nov 2026 16:24:36 +0530 Subject: [PATCH 2/3] fix(help-center): close the review findings on search and analytics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A round of review on the search change; all of it behaviour, not style. - **The analytics window subtracted the whole range**, so "7 days" covered eight dates. It subtracts one fewer now, and a test pins the boundary by writing a bucket on the first included day and one on the day before it. - **The panel's "Across N articles" counted the top-ten list**, which reads as the workspace's article count. The views total is the whole window, so it keeps its caption and the count is gone. - **A failed search rendered as "No article matches"**, which is a different claim. Both search surfaces show a failure state with a retry; an empty success still says nothing matched. - **The hub E2E spec was failing for a fixture reason.** Its mocked board omitted `icon`, which the widget's decoder requires, so the iframe rendered its error boundary and the assertion saw a missing headline. The fixture carries the field now, and the spec asserts the form header the hub actually shows rather than the board list's heading. - **A failed deflection latched the widget off.** Swallowing the error into `enabled: false` tells the composer the workspace has no help center, permanently, on a transient database error. It propagates now, and the client's own failed state clears the panel and retries on the next keystroke. - **Search buckets had no prune.** The help center's daily retention pass now deletes them by day after 90 days — one day longer than the longest analytics window, and `day < cutoff`, so the cut day survives. The schema comment states the growth bound and the prune together. - **A query with no searchable term is no longer counted.** Punctuation can never match, so it was noise in both the top-searches list and the gaps list that someone is expected to act on. - **Editing an article's text clears its vector in the same statement**, so a search cannot match words the page no longer has; reordering, retitling the section or moving the status keeps it. - **The portal's search atom no longer revalidates on a timer or on focus.** Those reads answer "what is in the help center"; this one answers "what matches this text", and only the text changing should ask again. - **The atom argument types derive from the operation payloads** instead of restating their fields, so a field an operation gains is a field the panel sends. --- apps/server/src/app/program.ts | 7 +- .../dashboard/features/help-center/atoms.ts | 42 ++++-- .../components/help-center-analytics.tsx | 12 +- .../components/help-center-search.tsx | 26 +++- apps/web/src/portal/help-center/atoms.ts | 23 ++- .../portal/help-center/help-center-page.tsx | 51 +++++-- ...ch-is-lexical-with-day-bucket-analytics.md | 3 +- docs/help-center.md | 6 +- e2e/tests/widget-modes.spec.ts | 10 +- packages/db/src/schema/help-center.ts | 10 +- .../src/help-center-article/handlers.test.ts | 134 ++++++++++++++++++ .../src/help-center-article/handlers.ts | 5 +- .../src/help-center-article/repository.ts | 40 +++++- .../src/help-center-article/retention.test.ts | 90 ++++++++++++ .../src/help-center-article/retention.ts | 37 ++++- packages/domain/src/widget/api-live.ts | 12 +- 16 files changed, 445 insertions(+), 63 deletions(-) create mode 100644 packages/domain/src/help-center-article/retention.test.ts diff --git a/apps/server/src/app/program.ts b/apps/server/src/app/program.ts index b51a32c14..d0496bdca 100644 --- a/apps/server/src/app/program.ts +++ b/apps/server/src/app/program.ts @@ -17,7 +17,7 @@ import { subscriptionRevocationMaintenance } from "@feeblo/domain/billing/revoca import { PolarService } from "@feeblo/domain/billing/service"; import { isFeatureEnabled } from "@feeblo/domain/feature-flags"; import { HelpCenterArticleRepository } from "@feeblo/domain/help-center-article/repository"; -import { helpCenterArticleViewRetention } from "@feeblo/domain/help-center-article/retention"; +import { helpCenterRetention } from "@feeblo/domain/help-center-article/retention"; import { WebhookIntegrationConfig } from "@feeblo/domain/integration/config"; import { DiscordIntegrationConfig } from "@feeblo/domain/integration/discord/config"; import { ExternalResourceServiceLive } from "@feeblo/domain/integration/external-resource/live"; @@ -238,7 +238,8 @@ export const program = Effect.gen(function* () { // The help center's read events age out on the same terms, for the same // reason: the counter beside them is the number a customer reads and is never - // pruned, so the events only need to answer the recent window. + // pruned, so the events only need to answer the recent window. The same pass + // prunes the search buckets, whose key is the reader's text. // // Off means nothing writes a read event, so the sweep is not scheduled at // all — the same rule the audit-log sweep below applies. @@ -247,7 +248,7 @@ export const program = Effect.gen(function* () { HelpCenterArticleRepository.layer ); - yield* helpCenterArticleViewRetention.pipe( + yield* helpCenterRetention.pipe( Effect.provide(helpCenterTrafficRuntime), Effect.forkScoped ); diff --git a/apps/web/src/dashboard/features/help-center/atoms.ts b/apps/web/src/dashboard/features/help-center/atoms.ts index 6f891b9f4..036dcc6ae 100644 --- a/apps/web/src/dashboard/features/help-center/atoms.ts +++ b/apps/web/src/dashboard/features/help-center/atoms.ts @@ -1,8 +1,25 @@ -import type { THelpCenterAnalyticsRange } from "@feeblo/domain/help-center-article/schema"; +import type { + THelpCenterAnalyticsPayload, + THelpCenterAnalyticsRange, + THelpCenterArticleSearch, +} from "@feeblo/domain/help-center-article/schema"; import * as Atom from "effect/reactivity/Atom"; import { DashboardClient, dashboardSWR } from "~/lib/atom-rpc"; +/** + * The client's view of an operation's payload. + * + * Derived from the domain type with the branded id widened to `string`: the + * brand exists so a server route cannot pass one id where another belongs, and + * a React prop is not that boundary. Every other field comes from the + * operation, so a field it gains is a field this atom sends rather than one it + * silently omits. + */ +type ClientPayload = Readonly< + Omit & { readonly organizationId: string } +>; + /** * The editor's single-article read is Effect Atom, not `useQuery`. * @@ -63,10 +80,16 @@ export const deleteHelpCenterArticleAtom = DashboardClient.mutation( "DeleteHelpCenterArticle" ); -export type HelpCenterAnalyticsAtomArgs = { - readonly organizationId: string; - readonly range: THelpCenterAnalyticsRange; -}; +/** + * The analytics read's arguments, derived from the operation's payload. + * + * `Codec.Encoded` rather than the payload's decoded type: this is what the + * client actually sends, and the decoded side brands `organizationId`, which a + * React prop is not. Deriving it means a field the operation gains is a field + * this panel sends, instead of one it silently leaves out. + */ +export type HelpCenterAnalyticsAtomArgs = + ClientPayload; /** The window the panel opens on. */ export const DEFAULT_HELP_CENTER_ANALYTICS_RANGE = @@ -88,10 +111,11 @@ export const helpCenterAnalyticsAtom = Atom.family( }).pipe(dashboardSWR("60 seconds"), Atom.setIdleTTL("5 minutes")) ); -export type HelpCenterSearchAtomArgs = { - readonly organizationId: string; - readonly query: string; -}; +/** The same rule as the analytics args: derived from the operation. */ +export type HelpCenterSearchAtomArgs = Pick< + ClientPayload, + "organizationId" | "query" +>; /** * The dashboard's article search. diff --git a/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx b/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx index 2fd771da7..7c03ea577 100644 --- a/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx +++ b/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx @@ -180,12 +180,14 @@ function TrafficTotals({ return (
    + {/* + * No article count here. `articles` is the top ten by views, so a count + * of it would read as "the help center has ten articles" on any + * workspace bigger than that. The views total is the whole window and + * needs no such qualifier. + */} diff --git a/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx b/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx index fdbdd9730..db409ae19 100644 --- a/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx +++ b/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx @@ -1,6 +1,7 @@ -import { useAtomValue } from "@effect/atom-react"; +import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; import type { THelpCenterArticleSearchHit } from "@feeblo/domain/help-center-article/schema"; import { Badge } from "@feeblo/ui/badge"; +import { Button } from "@feeblo/ui/button"; import { DebouncedInputGroupInput } from "@feeblo/ui/debounced-input"; import { InputGroup, @@ -76,8 +77,9 @@ function SearchResults({ [organizationId, query] ); - const result = useAtomValue(helpCenterSearchAtom(args)); - const hits = Result.getOrElse(result, () => []); + const atom = helpCenterSearchAtom(args); + const result = useAtomValue(atom); + const refresh = useAtomRefresh(atom); if (Result.isInitial(result)) { return ( @@ -88,6 +90,24 @@ function SearchResults({ ); } + // A failed request is not an empty result. Reporting "no article matches" + // for a request that never completed would send an editor off to write a + // page that already exists. + if (Result.isFailure(result)) { + return ( +
    +

    + Search could not be loaded. +

    + +
    + ); + } + + const hits = Result.getOrElse(result, () => []); + if (hits.length === 0) { return (

    diff --git a/apps/web/src/portal/help-center/atoms.ts b/apps/web/src/portal/help-center/atoms.ts index c1351c01f..10c76b97f 100644 --- a/apps/web/src/portal/help-center/atoms.ts +++ b/apps/web/src/portal/help-center/atoms.ts @@ -1,3 +1,4 @@ +import type { THelpCenterArticleSearchPublic } from "@feeblo/domain/help-center-article/schema"; import * as Atom from "effect/reactivity/Atom"; import { DashboardClient, dashboardSWR } from "@/lib/atom-rpc"; @@ -45,10 +46,17 @@ export const helpCenterArticlesPublicAtom = Atom.family( ).pipe(dashboardSWR("30 seconds"), Atom.setIdleTTL("5 minutes")) ); -export type HelpCenterSearchAtomArgs = { - readonly organizationId: string; - readonly query: string; -}; +/** + * The portal's search arguments. + * + * Derivable from `THelpCenterArticleSearchPublic` the way the dashboard's are, + * but declared plainly here: the portal has one such atom, and a shared helper + * for one call would be more indirection than the derivation is worth. + */ +export type HelpCenterSearchAtomArgs = Readonly<{ + organizationId: string; + query: string; +}>; /** * The portal's search. @@ -64,9 +72,14 @@ export type HelpCenterSearchAtomArgs = { */ export const helpCenterSearchPublicAtom = Atom.family( (args: HelpCenterSearchAtomArgs) => + // No stale-while-revalidate here, unlike the reads above. Those answer + // "what is in the help center", so a revalidation on focus keeps them + // current; this one answers "what matches this text", and only the text + // changing should ask again. The idle TTL still drops an entry once the + // reader has stopped typing, which is the only invalidation it needs. DashboardClient.query("SearchHelpCenterArticlesPublic", args, { reactivityKeys: helpCenterPublicReactivityKeys(args.organizationId), - }).pipe(dashboardSWR("30 seconds"), Atom.setIdleTTL("1 minute")) + }).pipe(Atom.setIdleTTL("1 minute")) ); export type HelpCenterArticlePublicAtomArgs = { diff --git a/apps/web/src/portal/help-center/help-center-page.tsx b/apps/web/src/portal/help-center/help-center-page.tsx index 00a72457a..67abc20c4 100644 --- a/apps/web/src/portal/help-center/help-center-page.tsx +++ b/apps/web/src/portal/help-center/help-center-page.tsx @@ -1,7 +1,9 @@ -import { useAtomValue } from "@effect/atom-react"; +import { useAtomRefresh, useAtomValue } from "@effect/atom-react"; import { isFeatureEnabled } from "@feeblo/domain/feature-flags"; +import type { THelpCenterArticleSearchResult } from "@feeblo/domain/help-center-article/schema"; import { useSite } from "@feeblo/public-feature-board/providers/site-provider"; import { Badge } from "@feeblo/ui/badge"; +import { Button } from "@feeblo/ui/button"; import { DebouncedInputGroupInput } from "@feeblo/ui/debounced-input"; import { Empty, @@ -172,18 +174,20 @@ export function HelpCenterPage() { ); } +/** + * One article in a topic's list, or one search hit. + * + * The fields come from the public search projection rather than being restated: + * a list row is that same projection plus counters, so the two surfaces cannot + * drift into rendering different fields from the same article. + */ +type ArticleRowItem = Pick< + THelpCenterArticleSearchResult, + "excerpt" | "id" | "isArchived" | "slug" | "title" +>; + /** One article in a topic's list. */ -function ArticleRow({ - article, -}: { - article: { - readonly id: string; - readonly slug: string; - readonly title: string; - readonly excerpt: string; - readonly isArchived: boolean; - }; -}) { +function ArticleRow({ article }: { article: ArticleRowItem }) { return (

  • []); + const atom = helpCenterSearchPublicAtom(args); + const result = useAtomValue(atom); + const refresh = useAtomRefresh(atom); if (Result.isInitial(result)) { return ( @@ -243,6 +247,23 @@ function HelpCenterSearchResults({ ); } + // A failed request is not "nothing matches": telling a reader the answer does + // not exist because the network dropped is worse than saying so. + if (Result.isFailure(result)) { + return ( +
    +

    + Search could not be loaded. +

    + +
    + ); + } + + const hits = Result.getOrElse(result, () => []); + if (hits.length === 0) { return ( diff --git a/docs/adr/0031-help-center-search-is-lexical-with-day-bucket-analytics.md b/docs/adr/0031-help-center-search-is-lexical-with-day-bucket-analytics.md index cbd444b94..980ef51ff 100644 --- a/docs/adr/0031-help-center-search-is-lexical-with-day-bucket-analytics.md +++ b/docs/adr/0031-help-center-search-is-lexical-with-day-bucket-analytics.md @@ -34,7 +34,8 @@ Four choices, all additive to ADR 0029: ## Consequences - **Search is English-stemmed for every workspace.** The text-search configuration is part of the index, so per-locale or per-workspace configurations are a different column and a different decision; a non-English help center searches by exact words with English stopwords dropped. -- **`help_center_search_daily` is the first table here that would need retention**, because its key includes the reader's text. Growth is bounded by distinct queries per workspace per day, which is human-entered text; if that assumption stops holding, a prune by `day` is the fix and the aggregates it feeds are already windowed. +- **`help_center_search_daily` is pruned by day, because its key includes the reader's text.** The help center's daily retention pass deletes buckets whose `day` is older than 90 days and the raw view events on the same schedule; `day < cutoff` keeps the cut day, and the window is one day longer than the longest analytics range, so a prune can never remove a bucket a panel asks for. Growth is bounded by distinct queries per workspace per day in the meantime, which is human-entered text. +- **A failed deflection is a failure, not "no documentation".** The endpoint propagates the error and the widget's own `failed` state clears the panel without latching `enabled: false`; swallowing it into a disabled answer would tell the widget a workspace has no help center, permanently for that session, on a transient database error. - **The widget's deflection is public, unauthenticated and plan-free.** It answers `{ articles, enabled }` — `enabled: false` for a deployment with the flag off or a workspace with no public help center — because a distinguishable failure would confirm which workspaces exist. It is on every plan: the ceiling on articles already prices the feature, and a deflection that only some plans get would be a worse help center rather than a smaller one. - **One portal-URL builder.** `site/url.ts` owns `` or `.`, and both the email outbox and the widget's article links go through it, because the widget iframe is embedded on the customer's own origin and a relative link would resolve against the host page. - **The semantic half costs a model call per rescued search.** It is bounded by the thin-result rule, the public rate limiter, the 120-character query cap, and `strict` refusing to embed at all; a deployment with no `EMBEDDING_API_KEY` never runs it. diff --git a/docs/help-center.md b/docs/help-center.md index 6ecf06cb8..e81b39062 100644 --- a/docs/help-center.md +++ b/docs/help-center.md @@ -73,12 +73,12 @@ The widget's deflection passes `strict`, which restricts the lexical match to th ## Analytics -The panel on the dashboard's help center index reads two day-bucketed tables over a 7, 30 or 90 day window: +The panel on the dashboard's help center index reads two day-bucketed tables over a 7, 30 or 90 day window. A window of N days is N UTC dates **including today**, so its first day is `today - (N - 1)`; the boundary is pinned by a test that writes one bucket on the first included day and one on the day before it. - `help_center_article_daily` counts views and verdicts per article per UTC day. The view counter moves inside the same transaction as the all-time counter, so the two cannot disagree. The verdict counters count verdicts **cast** that day, not the standing distribution — a flip adds an unhelpful to today and leaves the earlier helpful where it was. The standing distribution is the feedback table's own `count(*)`, and a second, slower-moving copy of it here would be a number that disagrees with the article page. - `help_center_search_daily` counts searches and how many of them returned nothing, keyed on the normalized query. Both numbers live in one row so "what people look for" and "what we do not document" cannot disagree about how many searches there were. -Writes to either table are **advisory**: a failure is logged and swallowed, because a reader's search must not fail because an analytics write did, and a verdict must not be lost because its bucket was. The search write is the only public read that writes at all. +Writes to either table are **advisory**: a failure is logged and swallowed, because a reader's search must not fail because an analytics write did, and a verdict must not be lost because its bucket was. The search write is the only public read that writes at all. A query with no searchable term — punctuation, or a box the reader cleared — is not recorded: it can never match, so counting it would put noise in both "top searches" and the gaps list. ## Widget deflection @@ -92,7 +92,7 @@ The feature ships behind `HELP_CENTER` in `packages/domain/src/feature-flags.ts` - `help-center/flag.ts` exposes `HelpCenterEnabled`, a `Context.Reference` defaulting to the flag. Tests provide `true`; production reads the default. - `helpCenterFeatureGate` is a **public** policy (the portal has no session to check) that fails with `PolicyDeniedError`, which every operation already declares — so the flag adds no new error vocabulary at the RPC boundary. It is applied **last** in each handler's pipe, which makes it the outermost check: a deployment with the feature off refuses before it reads a row or spends a rate-limit token. -- The raw-event retention loop is forked only when the flag is on, matching the audit sweep: with the feature off nothing writes a read event, so a daily pass would only wake the worker to delete nothing. +- The retention loop is forked only when the flag is on, matching the audit sweep: with the feature off nothing writes a read event or a search, so a daily pass would only wake the worker to delete nothing. The same loop prunes both tables — raw read events older than 90 days, and search buckets whose day is older than 90 days. The day comparison is `day < cutoff`, so the cut day itself survives; with a 90-day window and a 90-day analytics range that starts 89 days back, the sweep can never delete a bucket a panel still reports on. - Every surface asks the same flag: the dashboard nav entry, the mobile tab, the settings section, the three dashboard routes and the settings route (an "unavailable" state for a deep link, not a 404), the portal pages, the portal nav tab, `sitemap.xml`, `llms.txt`, and the article page's SSR metadata. Turning it on is therefore a single-line change with no migration: no article can exist while the gate is closed, so `site.help_center_visibility` defaulting to `PUBLIC` cannot publish anything on the day the flag flips. diff --git a/e2e/tests/widget-modes.spec.ts b/e2e/tests/widget-modes.spec.ts index 7803c2c86..a98fb00b4 100644 --- a/e2e/tests/widget-modes.spec.ts +++ b/e2e/tests/widget-modes.spec.ts @@ -15,6 +15,11 @@ test("Feeblo Hub moves between updates and feedback inside one placed widget", a contentType: "application/json", body: JSON.stringify([ { + // `icon` is part of the board payload the widget decodes, so the + // fixture carries it even when the board has no override: a missing + // key fails the decode, which renders the boundary instead of the + // form. + icon: null, id: "board_features", name: "Feature requests", slug: "features", @@ -100,8 +105,11 @@ test("Feeblo Hub moves between updates and feedback inside one placed widget", a await launcher.click(); const widget = page.frameLocator("#feeblo-embed-container iframe"); + // The hub opens straight onto the only board's form, and the form's header + // names that board. "Give us feedback" is the *board list's* heading, which + // this fixture never shows — one board means one destination. await expect( - widget.getByText("Give us feedback", { exact: true }) + widget.getByText("Feature requests", { exact: true }) ).toBeVisible(); await widget.getByRole("link", { name: "Updates" }).click(); await expect(widget.getByText("A faster feedback editor")).toBeVisible(); diff --git a/packages/db/src/schema/help-center.ts b/packages/db/src/schema/help-center.ts index 674555303..f066bf6a3 100644 --- a/packages/db/src/schema/help-center.ts +++ b/packages/db/src/schema/help-center.ts @@ -458,10 +458,12 @@ export const helpCenterArticleDailyTable = pgTable( * `export csv` as two rows with one search each. * * The composite key includes `query`, so the table grows with distinct queries - * rather than with searches. A help center sees human-entered text, where - * distinct-by-day is bounded by what people actually ask; there is no retention - * pass, and this is the table that would need one first if that assumption ever - * stops holding. + * rather than with searches: a help center sees human-entered text, where + * distinct-by-day is bounded by what people actually ask. That bound is not a + * guarantee, so rows are pruned by day after `HELP_CENTER_SEARCH_RETENTION_DAYS` + * (90) by the help center's daily retention pass — the same window as the raw + * view events, and one day longer than the longest analytics range, so the + * sweep can never delete a bucket a panel still reports on. */ export const helpCenterSearchDailyTable = pgTable( "help_center_search_daily", diff --git a/packages/domain/src/help-center-article/handlers.test.ts b/packages/domain/src/help-center-article/handlers.test.ts index 613b3ef22..50af2b00e 100644 --- a/packages/domain/src/help-center-article/handlers.test.ts +++ b/packages/domain/src/help-center-article/handlers.test.ts @@ -14,6 +14,7 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import { TestClock } from "effect/testing"; import { EntitlementPolicy } from "../entitlement/policies"; import { HelpCenterTopicRepository } from "../help-center-topic/repository"; @@ -622,6 +623,13 @@ describe("HelpCenterArticleRpcHandlers", () => { query: "invoicing", }); + // Punctuation is not a search: there is no term a full-text index + // could match, so counting it would put noise in the gaps list. + yield* handlers.SearchHelpCenterArticlesPublic({ + organizationId: fixture.organizationId, + query: "!!!", + }); + const rows = yield* ( db .select({ @@ -650,6 +658,61 @@ describe("HelpCenterArticleRpcHandlers", () => { }) ); + it.effect("counts a window of N days as N dates ending today", () => + Effect.gen(function* () { + const handlers = yield* HelpCenterArticleRpcHandlersEffect; + const db = yield* currentDb; + const fixture = yield* makeFixture(); + const session = makeSession(fixture); + const id = yield* HelpCenterArticleId.generate; + + yield* create(fixture, id, "Windowed article", "windowed-article"); + + // Pin the clock: the handler reads "now" through `Clock`, and the + // assertion is about which days that puts inside the window. + yield* TestClock.setTime( + DateTime.toEpochMillis( + DateTime.makeUnsafe("2026-03-10T12:00:00.000Z") + ) + ); + + yield* db.insert(schema.helpCenterArticleDailyTable).values([ + // Seven days back is outside a window that starts six days back. + { + articleId: id, + day: "2026-03-03", + organizationId: fixture.organizationId, + views: 1, + }, + // The window's first day, inclusive. + { + articleId: id, + day: "2026-03-04", + organizationId: fixture.organizationId, + views: 2, + }, + { + articleId: id, + day: "2026-03-10", + organizationId: fixture.organizationId, + views: 4, + }, + ]); + + const analytics = yield* handlers + .GetHelpCenterAnalytics({ + organizationId: fixture.organizationId, + range: "7d", + }) + .pipe(Effect.provideService(CurrentSession, session)); + + // 10 March minus six days: the first of seven dates, today included. + expect(analytics.since).toBe("2026-03-04"); + expect(analytics.totals.views).toBe(6); + expect(analytics.articles[0]?.views).toBe(6); + }) + ); + it.effect("reports views and searches over the chosen window", () => Effect.gen(function* () { const handlers = yield* HelpCenterArticleRpcHandlersEffect; @@ -895,6 +958,77 @@ describe("HelpCenterArticleRpcHandlers", () => { }) ); + it.effect("clears the vector when the text changes, and only then", () => + Effect.gen(function* () { + const repository = yield* HelpCenterArticleRepository; + const handlers = yield* HelpCenterArticleRpcHandlersEffect; + const fixture = yield* makeFixture(); + const session = makeSession(fixture); + const id = yield* HelpCenterArticleId.generate; + const now = yield* DateTime.nowAsDate; + + yield* create(fixture, id, "Original title", "original-title"); + + const current = yield* readContent(id); + + if (current === undefined) { + return yield* Effect.die(new Error("fixture article missing")); + } + + yield* repository.saveEmbedding({ + articleId: id, + content: current.content, + model: "test-embedding", + now, + organizationId: fixture.organizationId, + title: current.title, + vector: unitVector(0), + }); + + // A reorder is not a text change: the vector still describes the + // words on the page, so it is kept. + yield* handlers + .UpdateHelpCenterArticle({ + content: current.content, + expectedStatus: "published", + id, + organizationId: fixture.organizationId, + position: 5, + publishedAt: null, + slug: "original-title", + status: "published", + title: current.title, + topicId: fixture.topicId, + }) + .pipe(Effect.provideService(CurrentSession, session)); + + const afterMove = yield* readEmbeddingState(id); + + expect(afterMove?.embedding).not.toBeNull(); + + yield* handlers + .UpdateHelpCenterArticle({ + content: `${current.content}\n\nOne more line.`, + expectedStatus: "published", + id, + organizationId: fixture.organizationId, + position: 5, + publishedAt: null, + slug: "original-title", + status: "published", + title: current.title, + topicId: fixture.topicId, + }) + .pipe(Effect.provideService(CurrentSession, session)); + + const afterEdit = yield* readEmbeddingState(id); + + // Keeping it would let a search match words the article no longer + // has, until the detached refresh lands. + expect(afterEdit?.embedding).toBeNull(); + }) + ); + it.effect( "answers from the semantic pass when the words match nothing", () => diff --git a/packages/domain/src/help-center-article/handlers.ts b/packages/domain/src/help-center-article/handlers.ts index c68fe0cf6..38033f1ee 100644 --- a/packages/domain/src/help-center-article/handlers.ts +++ b/packages/domain/src/help-center-article/handlers.ts @@ -425,9 +425,12 @@ export const HelpCenterArticleRpcHandlersEffect = Effect.gen(function* () { Effect.gen(function* () { const now = yield* DateTime.now; + // A window of N days is N dates *including* today, so the first day + // is `today - (N - 1)`. Subtracting N would make "7 days" an + // eight-date range and quietly widen every number the panel shows. const since = DateTime.formatIsoDateUtc( DateTime.subtract({ - days: HELP_CENTER_ANALYTICS_RANGE_DAYS[args.range], + days: HELP_CENTER_ANALYTICS_RANGE_DAYS[args.range] - 1, })(now) ); diff --git a/packages/domain/src/help-center-article/repository.ts b/packages/domain/src/help-center-article/repository.ts index adac7054d..f1eddae86 100644 --- a/packages/domain/src/help-center-article/repository.ts +++ b/packages/domain/src/help-center-article/repository.ts @@ -522,8 +522,17 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { expectedStatus: THelpCenterArticleState; now: Date; write: NormalizedHelpCenterArticleWrite; - }) => - db + }) => { + /** + * Whether this statement is changing the text an embedding describes. + * + * A `SET` expression reads the row as it was, so this compares the stored + * title and body against the ones being written — one statement, no read + * before it and no race between the two. + */ + const textChanged = sql`${article.title} is distinct from ${write.title} or ${article.content} is distinct from ${write.content}`; + + return db .update(article) .set({ topicId, @@ -535,6 +544,16 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { status: write.status, publishedAt: write.publishedAt, updatedAt: now, + // A vector describes the text it was generated from. Clearing it here, + // in the same statement that changes the text, closes the window in + // which a search could match words the article no longer has; the + // detached job fills it again. Only a text change clears it — a + // reorder, a retitle of the topic or a status move leaves the + // article's meaning, and therefore its vector, alone. The three + // columns move together because the table's check requires them to. + embedding: sql`case when ${textChanged} then null else ${article.embedding} end`, + embeddingModel: sql`case when ${textChanged} then null else ${article.embeddingModel} end`, + embeddedAt: sql`case when ${textChanged} then null else ${article.embeddedAt} end`, }) .where( and( @@ -545,6 +564,7 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { ) .returning({ id: article.id }) .pipe(Effect.map((rows) => rows.map((row) => row.id))); + }; const remove = ({ organizationId, @@ -651,6 +671,15 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { return { accepted }; }); + /** + * Drop search buckets older than `before`. + * + * `before` is a `YYYY-MM-DD` day rather than an instant, because the key is a + * calendar day and the analytics windows are inclusive ranges of them. + */ + const deleteSearchesBefore = (before: string) => + db.delete(searchDaily).where(lt(searchDaily.day, before)); + /** * Drop the raw events older than `before`. * @@ -685,7 +714,11 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { }) => { const normalized = normalizeHelpCenterSearchQuery(query); - if (normalized.length === 0) { + // No terms means nothing a full-text index could ever match — a box the + // reader filled with punctuation, or one they cleared. Recording it would + // put noise in "top searches" and in "documentation gaps", and the + // potential gaps are the list someone is expected to act on. + if (helpCenterSearchTerms(normalized).length === 0) { return Effect.void; } @@ -823,6 +856,7 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { return { analytics, deleteEventsBefore, + deleteSearchesBefore, findMany, findManyPublic, findById, diff --git a/packages/domain/src/help-center-article/retention.test.ts b/packages/domain/src/help-center-article/retention.test.ts new file mode 100644 index 000000000..7c8e4570c --- /dev/null +++ b/packages/domain/src/help-center-article/retention.test.ts @@ -0,0 +1,90 @@ +import { expect, layer } from "@effect/vitest"; +import { currentDb, Database, schema } from "@feeblo/db"; +import { WorkspaceId } from "@feeblo/id"; +import { asc, eq } from "drizzle-orm"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; + +import { HelpCenterArticleRepository } from "./repository"; +import { HELP_CENTER_SEARCH_RETENTION_DAYS } from "./retention"; +import { HELP_CENTER_ANALYTICS_RANGE_DAYS } from "./schema"; + +const TestLayer = HelpCenterArticleRepository.layer.pipe( + Layer.provideMerge(Database.PgliteDatabaseLive) +); + +/** + * The search bucket's prune, tested at the repository the sweep calls. + * + * The sweep itself repeats forever and is not the thing under test; what + * matters is that deleting "before a day" leaves the cut day alone, because + * the analytics windows are inclusive ranges of days. + */ +layer(TestLayer)("search retention", (it) => { + const seed = (organizationId: string, days: readonly string[]) => + Effect.gen(function* () { + const db = yield* currentDb; + const now = yield* DateTime.nowAsDate; + + yield* db.insert(schema.organizationTable).values({ + id: organizationId, + name: "Retention", + slug: organizationId, + createdAt: now, + }); + + yield* db.insert(schema.helpCenterSearchDailyTable).values( + days.map((day) => ({ + day, + emptySearches: 0, + organizationId, + query: `query-${day}`, + searches: 1, + })) + ); + }); + + /** The days still stored for one workspace. The file shares a database. */ + const remainingDays = (organizationId: string) => + Effect.gen(function* () { + const db = yield* currentDb; + + const rows = yield* db + .select({ day: schema.helpCenterSearchDailyTable.day }) + .from(schema.helpCenterSearchDailyTable) + .where( + eq(schema.helpCenterSearchDailyTable.organizationId, organizationId) + ) + .orderBy(asc(schema.helpCenterSearchDailyTable.day)); + + return rows.map((row) => row.day); + }); + + it.effect("deletes the days before the cut and keeps the cut day", () => + Effect.gen(function* () { + const repository = yield* HelpCenterArticleRepository; + const organizationId = yield* WorkspaceId.generate; + + yield* seed(organizationId, ["2026-01-01", "2026-01-09", "2026-01-10"]); + + yield* repository.deleteSearchesBefore("2026-01-10"); + + const rows = yield* remainingDays(organizationId); + + // `day < before`, so the day named is retained. A window that starts on + // that day still has its first day. + expect(rows).toStrictEqual(["2026-01-10"]); + }) + ); + + it.effect("keeps at least the longest analytics window", () => + Effect.sync(() => { + // The panel's 90-day range includes today and the 89 before it; a + // retention window shorter than that would delete a day a page reports. + expect(HELP_CENTER_SEARCH_RETENTION_DAYS).toBeGreaterThanOrEqual( + HELP_CENTER_ANALYTICS_RANGE_DAYS["90d"] + ); + }) + ); +}); diff --git a/packages/domain/src/help-center-article/retention.ts b/packages/domain/src/help-center-article/retention.ts index 8e4eeefa5..328acd758 100644 --- a/packages/domain/src/help-center-article/retention.ts +++ b/packages/domain/src/help-center-article/retention.ts @@ -15,26 +15,55 @@ import { HelpCenterArticleRepository } from "./repository"; */ export const HELP_CENTER_ARTICLE_VIEW_RETENTION_DAYS = 90; +/** + * How long a search bucket is kept. + * + * Matches the article view window, and the analytics panel's longest range: + * 90 days plus today is every day a window can ask for, so the prune can never + * remove a row a page still reports on. + */ +export const HELP_CENTER_SEARCH_RETENTION_DAYS = 90; + /** Once a day is plenty for a table that grows at human page-view rate. */ const RETENTION_PASS_INTERVAL = "24 hours"; /** - * The raw-event retention pass. + * The help center's daily sweep: raw read events, and search buckets. + * + * Two tables with different shapes and the same question — how long do the + * rows behind a windowed read live — so one pass prunes both. The search + * buckets are why this exists at all: their key is the reader's text, which is + * the only table here that grows with what people type rather than with what + * the workspace publishes. * * A failed pass is logged rather than allowed to kill the scope: an unreachable * database is transient and the next pass is the retry. Interruption still * propagates, so the loop dies with the program that owns it — the same * contract as the post-traffic and audit-log sweeps. */ -export const helpCenterArticleViewRetention = Effect.gen(function* () { +export const helpCenterRetention = Effect.gen(function* () { const repository = yield* HelpCenterArticleRepository; const now = yield* DateTime.now; - const before = DateTime.toDate( + const eventsBefore = DateTime.toDate( DateTime.subtract({ days: HELP_CENTER_ARTICLE_VIEW_RETENTION_DAYS })(now) ); - yield* repository.deleteEventsBefore(before); + // A day, not an instant: the bucket's key is a UTC calendar day, and the + // comparison is `day < before`, so the cut day itself survives. With a + // 90-day window and a 90-day analytics range (today and the 89 before it), + // the prune can never reach a day a panel still asks for. + const searchesBefore = DateTime.formatIsoDateUtc( + DateTime.subtract({ days: HELP_CENTER_SEARCH_RETENTION_DAYS })(now) + ); + + yield* Effect.all( + [ + repository.deleteEventsBefore(eventsBefore), + repository.deleteSearchesBefore(searchesBefore), + ], + { concurrency: "unbounded" } + ); }).pipe( Effect.tap(() => Effect.logDebug("Ran the help-center retention pass")), Effect.catch((error) => diff --git a/packages/domain/src/widget/api-live.ts b/packages/domain/src/widget/api-live.ts index 34b625fc9..e249a1225 100644 --- a/packages/domain/src/widget/api-live.ts +++ b/packages/domain/src/widget/api-live.ts @@ -432,12 +432,12 @@ export const WidgetApiLive = HttpApiBuilder.group( ) .handle("helpArticles", ({ payload }) => findWidgetHelpArticles(payload).pipe( - // A search that fails is answered as "nothing to offer", not as an - // error: the composer's primary job is creating feedback, and a - // documentation hint that cannot be fetched must not block it. - Effect.catchCause(() => - Effect.succeed({ articles: [], enabled: false }) - ), + // A failure propagates, and the client's own `failed` state keeps + // the panel empty without latching `enabled: false`. Swallowing it + // into `enabled: false` would tell the widget the workspace has no + // help center — permanently, for that session — on a transient + // database error. Creating feedback is a separate request, so a + // failed hint cannot block it either way. provideHandlerEnvironment, RateLimit.withPublicHttpRateLimit({ name: "WidgetHelpArticles", From 984ef615f1e98a0f3d6c8610be890109ae56c707 Mon Sep 17 00:00:00 2001 From: nafees nazik Date: Thu, 19 Nov 2026 17:44:52 +0530 Subject: [PATCH 3/3] fix(help-center): key the search atoms by value and make the sweeps observable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The second review round, all of it in the places this change was quietly weaker than it looked. - **The atom families relied on object identity.** `Atom.family` memoizes on its argument with `MutableHashMap`, which compares objects by reference, so `{ organizationId, query }` was a new atom — and a new request — on every render unless the caller memoized it. Each read is two nested string-keyed families now: the same workspace and query always resolve to the same atom, returning to a prior value is the cached answer, and the `useMemo` call sites are gone along with the comment telling people not to forget it. - **The advisory writes failed silently.** The search counter, the widget's counter and the verdict bucket were discarded with `ignoreCause`, so a write that had been failing for a week looked exactly like one that had never been attempted. Each logs its cause first and still discards it. - **The retention pass was fail-fast.** One table's failed delete cancelled the other's, which is the wrong shape for two independent prunes. Each absorbs its own failure into an outcome, both run to completion, and the pass still reports failure with the tables that failed. - **A successful embedding refresh stamped `updatedAt`.** The column carries `$onUpdate`, so a statement that omits it is given the current time by Drizzle — the refresh was moving the date a reader sees on the page and invalidating every cache keyed on it. It is assigned to the column now, and the storage test asserts the timestamp is unchanged. - **The portal's search arguments derive from the operation**, with the branded id widened back to `string`, instead of restating `query`. --- .../dashboard/features/help-center/atoms.ts | 58 +++++++++++++------ .../components/help-center-analytics.tsx | 12 ++-- .../components/help-center-search.tsx | 12 ++-- apps/web/src/portal/help-center/atoms.ts | 52 +++++++++-------- .../portal/help-center/help-center-page.tsx | 12 ++-- .../repository.ts | 8 ++- .../src/help-center-article/handlers.test.ts | 8 ++- .../src/help-center-article/handlers.ts | 10 +++- .../src/help-center-article/repository.ts | 6 ++ .../src/help-center-article/retention.ts | 46 ++++++++++++++- packages/domain/src/widget/api-live.ts | 7 ++- 11 files changed, 159 insertions(+), 72 deletions(-) diff --git a/apps/web/src/dashboard/features/help-center/atoms.ts b/apps/web/src/dashboard/features/help-center/atoms.ts index 036dcc6ae..a257c00ca 100644 --- a/apps/web/src/dashboard/features/help-center/atoms.ts +++ b/apps/web/src/dashboard/features/help-center/atoms.ts @@ -98,19 +98,32 @@ export const DEFAULT_HELP_CENTER_ANALYTICS_RANGE = /** * One window of help center traffic, verdicts and searches. * - * Keyed by the window as well as the workspace, so switching the toggle is a - * cache lookup rather than a refetch of the other two windows. It shares the - * article reactivity key because a verdict cast on the portal changes what this - * panel reports: the reader's own write should move the number they are looking - * at. + * Two nested families rather than one family taking `{ organizationId, range }`. + * `Atom.family` memoizes on its argument with `MutableHashMap`, which compares + * object keys by identity and string keys by value — so a single family given an + * object literal is a new atom on every render unless the caller memoizes it, + * and a caller that forgets is a request per render. Composing one string-keyed + * family per field makes the cache value-based: the same workspace and window + * always produce the same atom, and switching the toggle back is the cached + * answer rather than a refetch. + * + * It shares the article reactivity key because a verdict cast on the portal + * changes what this panel reports: the reader's own write should move the number + * they are looking at. */ -export const helpCenterAnalyticsAtom = Atom.family( - (args: HelpCenterAnalyticsAtomArgs) => - DashboardClient.query("GetHelpCenterAnalytics", args, { - reactivityKeys: helpCenterArticleReactivityKeys(args.organizationId), - }).pipe(dashboardSWR("60 seconds"), Atom.setIdleTTL("5 minutes")) +const analyticsAtoms = Atom.family((organizationId: string) => + Atom.family((range: THelpCenterAnalyticsRange) => + DashboardClient.query( + "GetHelpCenterAnalytics", + { organizationId, range }, + { reactivityKeys: helpCenterArticleReactivityKeys(organizationId) } + ).pipe(dashboardSWR("60 seconds"), Atom.setIdleTTL("5 minutes")) + ) ); +export const helpCenterAnalyticsAtom = (args: HelpCenterAnalyticsAtomArgs) => + analyticsAtoms(args.organizationId)(args.range); + /** The same rule as the analytics args: derived from the operation. */ export type HelpCenterSearchAtomArgs = Pick< ClientPayload, @@ -120,14 +133,21 @@ export type HelpCenterSearchAtomArgs = Pick< /** * The dashboard's article search. * - * A family keyed by the query, like the portal's, so a keystroke is a cache - * lookup and a deleted character is an answer that already exists. It shares - * the article reactivity key: renaming an article should refresh a search that - * would have matched the old title. + * Nested families for the same reason as the analytics read: the query is a + * string key, so a keystroke is a cache lookup and a deleted character is an + * answer that already exists, with no `useMemo` to forget. It shares the article + * reactivity key: renaming an article should refresh a search that would have + * matched the old title. */ -export const helpCenterSearchAtom = Atom.family( - (args: HelpCenterSearchAtomArgs) => - DashboardClient.query("SearchHelpCenterArticles", args, { - reactivityKeys: helpCenterArticleReactivityKeys(args.organizationId), - }).pipe(dashboardSWR("30 seconds"), Atom.setIdleTTL("1 minute")) +const searchAtoms = Atom.family((organizationId: string) => + Atom.family((query: string) => + DashboardClient.query( + "SearchHelpCenterArticles", + { organizationId, query }, + { reactivityKeys: helpCenterArticleReactivityKeys(organizationId) } + ).pipe(dashboardSWR("30 seconds"), Atom.setIdleTTL("1 minute")) + ) ); + +export const helpCenterSearchAtom = (args: HelpCenterSearchAtomArgs) => + searchAtoms(args.organizationId)(args.query); diff --git a/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx b/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx index 7c03ea577..463e7a5f7 100644 --- a/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx +++ b/apps/web/src/dashboard/features/help-center/components/help-center-analytics.tsx @@ -14,11 +14,10 @@ import { Skeleton } from "@feeblo/ui/skeleton"; import { ToggleGroup, ToggleGroupItem } from "@feeblo/ui/toggle-group"; import { Link } from "@tanstack/react-router"; import * as Result from "effect/reactivity/AsyncResult"; -import { useMemo, useState } from "react"; +import { useState } from "react"; import { DEFAULT_HELP_CENTER_ANALYTICS_RANGE, - type HelpCenterAnalyticsAtomArgs, helpCenterAnalyticsAtom, } from "../atoms"; @@ -49,13 +48,12 @@ export function HelpCenterAnalytics({ DEFAULT_HELP_CENTER_ANALYTICS_RANGE ); - const args = useMemo( - () => ({ organizationId, range }), - [organizationId, range] + // No `useMemo`: the atom family keys on the workspace and the window by + // value, so a fresh object literal still resolves to the same atom. + const result = useAtomValue( + helpCenterAnalyticsAtom({ organizationId, range }) ); - const result = useAtomValue(helpCenterAnalyticsAtom(args)); - return (
    diff --git a/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx b/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx index db409ae19..a4f659147 100644 --- a/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx +++ b/apps/web/src/dashboard/features/help-center/components/help-center-search.tsx @@ -13,9 +13,9 @@ import { Search01Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { Link } from "@tanstack/react-router"; import * as Result from "effect/reactivity/AsyncResult"; -import { useMemo, useState } from "react"; +import { useState } from "react"; -import { helpCenterSearchAtom, type HelpCenterSearchAtomArgs } from "../atoms"; +import { helpCenterSearchAtom } from "../atoms"; /** Shorter than this is "typing", not a search. Same floor the portal uses. */ const MIN_SEARCH_LENGTH = 2; @@ -72,12 +72,8 @@ function SearchResults({ organizationId: string; query: string; }) { - const args = useMemo( - () => ({ organizationId, query }), - [organizationId, query] - ); - - const atom = helpCenterSearchAtom(args); + // No `useMemo`: the family keys on the workspace and the query by value. + const atom = helpCenterSearchAtom({ organizationId, query }); const result = useAtomValue(atom); const refresh = useAtomRefresh(atom); diff --git a/apps/web/src/portal/help-center/atoms.ts b/apps/web/src/portal/help-center/atoms.ts index 10c76b97f..bce6a2d71 100644 --- a/apps/web/src/portal/help-center/atoms.ts +++ b/apps/web/src/portal/help-center/atoms.ts @@ -49,39 +49,45 @@ export const helpCenterArticlesPublicAtom = Atom.family( /** * The portal's search arguments. * - * Derivable from `THelpCenterArticleSearchPublic` the way the dashboard's are, - * but declared plainly here: the portal has one such atom, and a shared helper - * for one call would be more indirection than the derivation is worth. + * Derived from the operation, with the branded id widened back to `string`: the + * decoded payload carries `WorkspaceId`, and a React prop does not. `limit` is + * dropped because the page never sends one — the server's default is the page + * size. */ -export type HelpCenterSearchAtomArgs = Readonly<{ - organizationId: string; - query: string; -}>; +export type HelpCenterSearchAtomArgs = Readonly< + Omit & { + readonly organizationId: string; + } +>; /** * The portal's search. * - * A family keyed by the query, so typing is a series of cache lookups rather - * than a series of writes to one entry: a reader who deletes a character and - * retypes it gets the earlier answer back instead of a spinner. The idle TTL is - * shorter than the other reads because the key includes text that only exists + * Two nested families, so the cache is keyed by value: `Atom.family` compares an + * object argument by identity, and a query string is exactly the kind of key + * that should be compared by value. Typing is then a series of cache lookups + * rather than a series of writes to one entry — a reader who deletes a character + * and retypes it gets the earlier answer back instead of a spinner. The idle TTL + * is shorter than the other reads because the key includes text that only exists * for as long as somebody is typing it. * - * Callers must memoize `args` — `Atom.family` keys on identity, so a fresh - * object literal per render is a fresh atom and a fresh request per render. + * No stale-while-revalidate here, unlike the reads above. Those answer "what is + * in the help center", so a revalidation on focus keeps them current; this one + * answers "what matches this text", and only the text changing should ask again. */ -export const helpCenterSearchPublicAtom = Atom.family( - (args: HelpCenterSearchAtomArgs) => - // No stale-while-revalidate here, unlike the reads above. Those answer - // "what is in the help center", so a revalidation on focus keeps them - // current; this one answers "what matches this text", and only the text - // changing should ask again. The idle TTL still drops an entry once the - // reader has stopped typing, which is the only invalidation it needs. - DashboardClient.query("SearchHelpCenterArticlesPublic", args, { - reactivityKeys: helpCenterPublicReactivityKeys(args.organizationId), - }).pipe(Atom.setIdleTTL("1 minute")) +const searchAtoms = Atom.family((organizationId: string) => + Atom.family((query: string) => + DashboardClient.query( + "SearchHelpCenterArticlesPublic", + { organizationId, query }, + { reactivityKeys: helpCenterPublicReactivityKeys(organizationId) } + ).pipe(Atom.setIdleTTL("1 minute")) + ) ); +export const helpCenterSearchPublicAtom = (args: HelpCenterSearchAtomArgs) => + searchAtoms(args.organizationId)(args.query); + export type HelpCenterArticlePublicAtomArgs = { readonly organizationId: string; readonly slug: string; diff --git a/apps/web/src/portal/help-center/help-center-page.tsx b/apps/web/src/portal/help-center/help-center-page.tsx index 67abc20c4..b9ccf8150 100644 --- a/apps/web/src/portal/help-center/help-center-page.tsx +++ b/apps/web/src/portal/help-center/help-center-page.tsx @@ -23,13 +23,12 @@ import { Search01Icon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { Link } from "@tanstack/react-router"; import * as Result from "effect/reactivity/AsyncResult"; -import { useMemo, useState } from "react"; +import { useState } from "react"; import { helpCenterArticlesPublicAtom, helpCenterSearchPublicAtom, helpCenterTopicsPublicAtom, - type HelpCenterSearchAtomArgs, } from "./atoms"; /** @@ -228,12 +227,9 @@ function HelpCenterSearchResults({ organizationId: string; query: string; }) { - const args = useMemo( - () => ({ organizationId, query }), - [organizationId, query] - ); - - const atom = helpCenterSearchPublicAtom(args); + // No `useMemo`: the family keys on the workspace and the query by value, so a + // fresh object resolves to the same atom. + const atom = helpCenterSearchPublicAtom({ organizationId, query }); const result = useAtomValue(atom); const refresh = useAtomRefresh(atom); diff --git a/packages/domain/src/help-center-article-feedback/repository.ts b/packages/domain/src/help-center-article-feedback/repository.ts index d85f07828..e430b4b93 100644 --- a/packages/domain/src/help-center-article-feedback/repository.ts +++ b/packages/domain/src/help-center-article-feedback/repository.ts @@ -216,7 +216,13 @@ const makeHelpCenterArticleFeedbackRepository = Effect.gen(function* () { unhelpfulVerdicts: sql`${daily.unhelpfulVerdicts} + ${helpful ? 0 : 1}`, }, }) - .pipe(Effect.asVoid, Effect.ignoreCause); + .pipe( + Effect.tapCause((cause) => + Effect.logWarning("Failed to record a help center verdict", cause) + ), + Effect.asVoid, + Effect.ignoreCause + ); }); /** diff --git a/packages/domain/src/help-center-article/handlers.test.ts b/packages/domain/src/help-center-article/handlers.test.ts index 50af2b00e..e7929cfc2 100644 --- a/packages/domain/src/help-center-article/handlers.test.ts +++ b/packages/domain/src/help-center-article/handlers.test.ts @@ -793,6 +793,7 @@ describe("HelpCenterArticleRpcHandlers", () => { embeddedAt: schema.helpCenterArticleTable.embeddedAt, embedding: schema.helpCenterArticleTable.embedding, embeddingModel: schema.helpCenterArticleTable.embeddingModel, + updatedAt: schema.helpCenterArticleTable.updatedAt, }) .from(schema.helpCenterArticleTable) .where(eq(schema.helpCenterArticleTable.id, id)); @@ -916,8 +917,9 @@ describe("HelpCenterArticleRpcHandlers", () => { yield* create(fixture, id, "Original title", "original-title"); const current = yield* readContent(id); + const before = yield* readEmbeddingState(id); - if (current === undefined) { + if (current === undefined || before === undefined) { return yield* Effect.die(new Error("fixture article missing")); } @@ -955,6 +957,10 @@ describe("HelpCenterArticleRpcHandlers", () => { expect(state?.embeddingModel).toBe("test-embedding"); expect(state?.embeddedAt).not.toBeNull(); + // The refresh is an index rebuild, not an edit. `updatedAt` carries + // `$onUpdate`, so a statement that omits it would silently stamp the + // current time and move the date a reader sees on the page. + expect(state?.updatedAt?.getTime()).toBe(before.updatedAt.getTime()); }) ); diff --git a/packages/domain/src/help-center-article/handlers.ts b/packages/domain/src/help-center-article/handlers.ts index 38033f1ee..208efbaae 100644 --- a/packages/domain/src/help-center-article/handlers.ts +++ b/packages/domain/src/help-center-article/handlers.ts @@ -391,7 +391,15 @@ export const HelpCenterArticleRpcHandlersEffect = Effect.gen(function* () { organizationId: args.organizationId, query: args.query, }) - .pipe(Effect.ignoreCause); + .pipe( + // Logged before it is discarded: the counter is advisory, but a + // write that has been failing for a week should be visible in the + // logs rather than absent from them. + Effect.tapCause((cause) => + Effect.logWarning("Failed to record a help center search", cause) + ), + Effect.ignoreCause + ); return hits.map((hit) => ({ excerpt: hit.excerpt, diff --git a/packages/domain/src/help-center-article/repository.ts b/packages/domain/src/help-center-article/repository.ts index f1eddae86..470e8d423 100644 --- a/packages/domain/src/help-center-article/repository.ts +++ b/packages/domain/src/help-center-article/repository.ts @@ -435,6 +435,12 @@ const makeHelpCenterArticleRepository = Effect.gen(function* () { embeddedAt: now, embedding: [...vector], embeddingModel: model, + // Assigned to the column itself, which is what pins it: `updatedAt` + // carries `$onUpdate`, so any statement that omits it is stamped with + // the current time by Drizzle. A refresh is an index rebuild, not an + // edit — without this an embedding job would move the date a reader + // sees on the page and invalidate every cache keyed on it. + updatedAt: article.updatedAt, }) .where( and( diff --git a/packages/domain/src/help-center-article/retention.ts b/packages/domain/src/help-center-article/retention.ts index 328acd758..a3678f1ea 100644 --- a/packages/domain/src/help-center-article/retention.ts +++ b/packages/domain/src/help-center-article/retention.ts @@ -1,3 +1,4 @@ +import * as Data from "effect/Data"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Schedule from "effect/Schedule"; @@ -27,6 +28,20 @@ export const HELP_CENTER_SEARCH_RETENTION_DAYS = 90; /** Once a day is plenty for a table that grows at human page-view rate. */ const RETENTION_PASS_INTERVAL = "24 hours"; +/** The tables the sweep prunes, so a failure names its own. */ +export type HelpCenterRetentionTarget = "article_view_events" | "searches"; + +/** + * A pass in which at least one table's prune failed. + * + * Each failure is already logged with its cause where it happened; this exists + * so the pass as a whole still reports failure, which is what keeps a broken + * prune from looking like a clean run for a week. + */ +export class HelpCenterRetentionError extends Data.TaggedError( + "HelpCenterRetentionError" +)<{ readonly failed: readonly HelpCenterRetentionTarget[] }> {} + /** * The help center's daily sweep: raw read events, and search buckets. * @@ -57,13 +72,38 @@ export const helpCenterRetention = Effect.gen(function* () { DateTime.subtract({ days: HELP_CENTER_SEARCH_RETENTION_DAYS })(now) ); - yield* Effect.all( + // One prune per table, each absorbing its own failure into an outcome. Run + // together they would be fail-fast: one table's bad day would cancel the + // other's delete, so a table that can never be pruned would keep the working + // one from ever being pruned either. + const prune = ( + target: HelpCenterRetentionTarget, + effect: Effect.Effect + ) => + effect.pipe( + Effect.tapCause((cause) => + Effect.logError(`Failed to prune ${target}`, cause) + ), + Effect.as(true), + Effect.catchCause(() => Effect.succeed(false)) + ); + + const [viewsPruned, searchesPruned] = yield* Effect.all( [ - repository.deleteEventsBefore(eventsBefore), - repository.deleteSearchesBefore(searchesBefore), + prune("article_view_events", repository.deleteEventsBefore(eventsBefore)), + prune("searches", repository.deleteSearchesBefore(searchesBefore)), ], { concurrency: "unbounded" } ); + + const failed = [ + ...(viewsPruned ? [] : ["article_view_events" as const]), + ...(searchesPruned ? [] : ["searches" as const]), + ]; + + if (failed.length > 0) { + return yield* new HelpCenterRetentionError({ failed }); + } }).pipe( Effect.tap(() => Effect.logDebug("Ran the help-center retention pass")), Effect.catch((error) => diff --git a/packages/domain/src/widget/api-live.ts b/packages/domain/src/widget/api-live.ts index e249a1225..4b9c15f4f 100644 --- a/packages/domain/src/widget/api-live.ts +++ b/packages/domain/src/widget/api-live.ts @@ -199,7 +199,12 @@ const findWidgetHelpArticles = Effect.fn("Widget.helpArticles")(function* ({ yield* repository .recordSearch({ found: hits.length > 0, now, organizationId, query }) - .pipe(Effect.ignoreCause); + .pipe( + Effect.tapCause((cause) => + Effect.logWarning("Failed to record a widget help center search", cause) + ), + Effect.ignoreCause + ); const portalUrl = buildPublicSiteUrl(site, config.rootDomain);