From 09c544a7353631ec58aa218c5793291d4dd96ead Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Sep 2026 09:28:36 +0530 Subject: [PATCH 1/2] chore(deps): bump js-yaml from 3.14.2 to 3.15.2 in /tests (#742) Bumps [js-yaml](https://github.com/nodeca/js-yaml) from 3.14.2 to 3.15.2. - [Changelog](https://github.com/nodeca/js-yaml/blob/3.15.2/CHANGELOG.md) - [Commits](https://github.com/nodeca/js-yaml/compare/3.14.2...3.15.2) --- updated-dependencies: - dependency-name: js-yaml dependency-version: 3.15.2 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- tests/package-lock.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/package-lock.json b/tests/package-lock.json index c5b16acca..59d649555 100644 --- a/tests/package-lock.json +++ b/tests/package-lock.json @@ -3767,9 +3767,9 @@ "license": "MIT" }, "node_modules/js-yaml": { - "version": "3.14.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz", - "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==", + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", + "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", "dev": true, "license": "MIT", "dependencies": { From 092161d9d82ac4e883a376195b309b1646bc6f8b Mon Sep 17 00:00:00 2001 From: netcon Date: Fri, 11 Sep 2026 14:21:00 +0800 Subject: [PATCH 2/2] feat: replace OSSInsight trending with GitHub Discovery (#743) * feat: add discovery worker * feat: use discovery adapter replace ossinsight --------- Co-authored-by: Siddhant Khare --- .prettierrc.js | 8 +- extensions/github1s/package.json | 2 +- .../src/adapters/discovery/data-source.ts | 49 ++++++ .../github1s/src/adapters/discovery/index.ts | 18 +++ .../src/adapters/discovery/router-parser.ts | 43 +++++ .../src/adapters/discovery/templates.ts | 152 ++++++++++++++++++ .../github1s/src/adapters/discovery/types.ts | 1 + extensions/github1s/src/adapters/index.ts | 4 +- .../src/adapters/ossinsight/interfaces.ts | 8 +- .../src/adapters/sourcegraph/data-source.ts | 7 +- extensions/github1s/src/commands/global.ts | 2 +- extensions/github1s/src/helpers/func.ts | 17 +- package-lock.json | 15 +- package.json | 4 +- scripts/run-discovery.ts | 31 ++++ src/config.ts | 6 +- workers/discovery/README.md | 3 + workers/discovery/src/collections.ts | 51 ++++++ workers/discovery/src/generate.ts | 148 +++++++++++++++++ workers/discovery/src/index.ts | 41 +++++ workers/discovery/src/snapshots.ts | 22 +++ workers/discovery/src/types.ts | 67 ++++++++ workers/discovery/wrangler.jsonc | 24 +++ workers/tsconfig.json | 16 ++ 24 files changed, 714 insertions(+), 25 deletions(-) create mode 100644 extensions/github1s/src/adapters/discovery/data-source.ts create mode 100644 extensions/github1s/src/adapters/discovery/index.ts create mode 100644 extensions/github1s/src/adapters/discovery/router-parser.ts create mode 100644 extensions/github1s/src/adapters/discovery/templates.ts create mode 100644 extensions/github1s/src/adapters/discovery/types.ts create mode 100644 scripts/run-discovery.ts create mode 100644 workers/discovery/README.md create mode 100644 workers/discovery/src/collections.ts create mode 100644 workers/discovery/src/generate.ts create mode 100644 workers/discovery/src/index.ts create mode 100644 workers/discovery/src/snapshots.ts create mode 100644 workers/discovery/src/types.ts create mode 100644 workers/discovery/wrangler.jsonc create mode 100644 workers/tsconfig.json diff --git a/.prettierrc.js b/.prettierrc.js index aa5eda3cd..a01e7efd3 100644 --- a/.prettierrc.js +++ b/.prettierrc.js @@ -6,10 +6,16 @@ export default { printWidth: 120, overrides: [ { - files: ['*.yml', '*.yaml', '*.json'], + files: ['*.yml', '*.yaml', '*.json', '*.jsonc'], options: { useTabs: false, }, }, + { + files: ['*.jsonc'], + options: { + trailingComma: 'none', + }, + }, ], }; diff --git a/extensions/github1s/package.json b/extensions/github1s/package.json index 00a5b8294..cec49ada6 100644 --- a/extensions/github1s/package.json +++ b/extensions/github1s/package.json @@ -18,7 +18,7 @@ "onFileSystem:gitlab1s", "onFileSystem:bitbucket1s", "onFileSystem:npmjs1s", - "onFileSystem:ossinsight", + "onFileSystem:discovery", "onCommand:remoteHub.openRepository" ], "browser": "./dist/extension", diff --git a/extensions/github1s/src/adapters/discovery/data-source.ts b/extensions/github1s/src/adapters/discovery/data-source.ts new file mode 100644 index 000000000..9dac96413 --- /dev/null +++ b/extensions/github1s/src/adapters/discovery/data-source.ts @@ -0,0 +1,49 @@ +import { getBrowserUrl } from '@/helpers/context'; +import { decorate, memorize } from '@/helpers/func'; +import { DataSource, Directory, File, FileType } from '../types'; +import { createDiscoveryPages } from './templates'; +import type { DiscoverySnapshot } from './types'; + +export class DiscoveryDataSource extends DataSource { + private readonly encoder = new TextEncoder(); + + @decorate(memorize) + private async getPages() { + const response = await fetch('https://discovery.github1s.com/github/latest.json', { + signal: AbortSignal.timeout(15000), + }); + if (!response.ok) { + throw new Error(`Unable to load Discovery: HTTP ${response.status}.`); + } + const snapshot: DiscoverySnapshot = await response.json(); + if ( + !Array.isArray(snapshot.collections) || + !snapshot.collections.every((collection) => collection?.kind === 'topic' || collection?.kind === 'curated') || + !snapshot.repositories || + typeof snapshot.repositories !== 'object' || + !Number.isFinite(Date.parse(snapshot.generated_at)) + ) { + throw new Error('Unable to load Discovery: invalid snapshot.'); + } + return getBrowserUrl().then((browserUrl: string) => createDiscoveryPages(snapshot, new URL(browserUrl).origin)); + } + + async provideDirectory(repo: string, ref: string, path: string, recursive = false): Promise { + const prefix = path.replace(/\/+$/, '') + '/'; + if (prefix !== '/' && prefix !== '/topics/') { + return null; + } + const entries: Directory['entries'] = prefix === '/' ? [{ path: '/topics', type: FileType.Directory }] : []; + for (const filePath of (await this.getPages()).keys()) { + if (filePath.startsWith(prefix) && (recursive || !filePath.slice(prefix.length).includes('/'))) { + entries.push({ path: filePath, type: FileType.File }); + } + } + return { truncated: false, entries }; + } + + async provideFile(repo: string, ref: string, path: string): Promise { + const render = (await this.getPages()).get(path); + return render ? { content: this.encoder.encode(render()) } : null; + } +} diff --git a/extensions/github1s/src/adapters/discovery/index.ts b/extensions/github1s/src/adapters/discovery/index.ts new file mode 100644 index 000000000..da275acf6 --- /dev/null +++ b/extensions/github1s/src/adapters/discovery/index.ts @@ -0,0 +1,18 @@ +import { Adapter, PlatformName } from '../types'; +import { DiscoveryDataSource } from './data-source'; +import { DiscoveryRouterParser } from './router-parser'; + +export class DiscoveryAdapter implements Adapter { + public readonly scheme = 'discovery'; + public readonly platformName = PlatformName.GitHub; + private readonly dataSource = new DiscoveryDataSource(); + private readonly routerParser = new DiscoveryRouterParser(); + + resolveDataSource() { + return this.dataSource; + } + + resolveRouterParser() { + return this.routerParser; + } +} diff --git a/extensions/github1s/src/adapters/discovery/router-parser.ts b/extensions/github1s/src/adapters/discovery/router-parser.ts new file mode 100644 index 000000000..607f51be8 --- /dev/null +++ b/extensions/github1s/src/adapters/discovery/router-parser.ts @@ -0,0 +1,43 @@ +/** + * @file router parser + * @author netcon + */ + +import { parsePath } from 'history'; +import queryString from 'query-string'; +import * as adapterTypes from '../types'; +import { GitHub1sRouterParser } from '../github1s/router-parser'; +import { normalizePath } from '@/helpers/util'; + +export class DiscoveryRouterParser extends GitHub1sRouterParser { + async parsePath(path: string): Promise { + const { path: pathsOrNull } = queryString.parse((parsePath(path).search || '').slice(1)); + const filePath = normalizePath((Array.isArray(pathsOrNull) ? pathsOrNull[0] : pathsOrNull) || ''); + const pageType = filePath.endsWith('.md') ? adapterTypes.PageType.Blob : adapterTypes.PageType.Tree; + return { pageType, repo: '', ref: '', filePath }; + } + + buildTreePath(repo: string, ref?: string, filePath?: string) { + return repo ? super.buildTreePath(repo, ref, filePath) : '/'; + } + + buildBlobPath(repo: string, ref: string, filePath: string, startLine?: number, endLine?: number) { + return repo ? super.buildBlobPath(repo, ref, filePath, startLine, endLine) : '/'; + } + + buildCommitListPath(repo: string) { + return repo ? super.buildCommitListPath(repo) : '/'; + } + + buildCommitPath(repo: string, commitSha: string) { + return repo ? super.buildCommitPath(repo, commitSha) : '/'; + } + + buildCodeReviewListPath(repo: string) { + return repo ? super.buildCodeReviewListPath(repo) : '/'; + } + + buildCodeReviewPath(repo: string, codeReviewId: string) { + return repo ? super.buildCodeReviewPath(repo, codeReviewId) : '/'; + } +} diff --git a/extensions/github1s/src/adapters/discovery/templates.ts b/extensions/github1s/src/adapters/discovery/templates.ts new file mode 100644 index 000000000..8e4170fa9 --- /dev/null +++ b/extensions/github1s/src/adapters/discovery/templates.ts @@ -0,0 +1,152 @@ +import type { DiscoveryCollection, DiscoverySnapshot, Repository } from './types'; + +interface CollectionPresentation { + title: string; + description: string; + icon: string; +} + +// Page copy can change independently of persisted collection snapshots. +const curatedPresentations: Record = { + 'new-projects': { + title: 'New Projects', + description: 'Recently created repositories.', + icon: '🌱', + }, + 'hidden-gems': { + title: 'Hidden Gems', + description: 'Lesser-known repositories with recent development activity.', + icon: '💎', + }, + 'most-starred': { + title: 'Most Starred', + description: 'Repositories with the highest total star counts on GitHub.', + icon: '⭐', + }, +}; + +const getCollectionPresentation = ({ id, kind }: DiscoveryCollection): CollectionPresentation => { + if (kind === 'topic') { + return { title: id, description: `Repositories tagged ${id} on GitHub.`, icon: '🏷️' }; + } + if (Object.prototype.hasOwnProperty.call(curatedPresentations, id)) { + return curatedPresentations[id]; + } + return { title: id, description: 'A curated selection of GitHub repositories.', icon: '📚' }; +}; + +// Repository descriptions are plain text, even when they contain Markdown or HTML. +const escapeText = (text: string) => + text + .replace(/\s+/g, ' ') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/[\\`*_{}[\]()#+.!|~-]/g, '\\$&'); + +const formatCount = (count: number) => count.toLocaleString('en-US'); +const collectionPath = (collection: DiscoveryCollection) => + `/${collection.kind === 'topic' ? 'topics/' : ''}${collection.id}.md`; +const collectionLink = (collection: DiscoveryCollection) => + `[${escapeText(getCollectionPresentation(collection).title)}](${collectionPath(collection)})`; +const homeLink = '[← Discovery](/README.md)'; + +// Keep content in Markdown; use CSS for typography and section boundaries. +const renderPage = (content: string) => ` + +${content} +`; + +const getRepositories = (snapshot: DiscoverySnapshot, collection: DiscoveryCollection): Repository[] => + collection.repository_ids.map((id) => snapshot.repositories[id]).filter((repo): repo is Repository => !!repo); +const renderUpdatedAt = (snapshot: DiscoverySnapshot) => + `Updated ${new Date(snapshot.generated_at).toISOString().slice(0, 16).replace('T', ' ')} UTC`; +const renderMeta = (text: string) => `${text}`; + +const renderRepository = (repo: Repository, origin: string, heading: '##' | '###') => { + const path = repo.full_name.split('/').map(encodeURIComponent).join('/'); + const metadata = [ + repo.language && `**${escapeText(repo.language)}**`, + repo.stargazers_count !== null && + `★ ${formatCount(repo.stargazers_count)} ${repo.stargazers_count === 1 ? 'star' : 'stars'}`, + repo.forks_count !== null && `${formatCount(repo.forks_count)} ${repo.forks_count === 1 ? 'fork' : 'forks'}`, + repo.pushed_at && `Last push: ${escapeText(repo.pushed_at.slice(0, 10))}`, + `[GitHub ↗](https://github.com/${path})`, + ].filter(Boolean); + return `${heading} [${escapeText(repo.full_name)}](${origin}/${path}) + +${repo.description ? escapeText(repo.description) : 'No description provided.'} +${renderMeta(metadata.join(' · '))}`; +}; + +const renderRepositoryList = (repos: Repository[], origin: string, heading: '##' | '###') => + repos.map((repo) => renderRepository(repo, origin, heading)).join('\n\n---\n\n') || + 'No repositories available in this collection.'; + +const renderCollectionPreview = (snapshot: DiscoverySnapshot, collection: DiscoveryCollection, origin: string) => { + const { title, description, icon } = getCollectionPresentation(collection); + const repos = getRepositories(snapshot, collection); + return `
+ +## ${icon} ${escapeText(title)} + +${escapeText(description)} + +${renderRepositoryList(repos.slice(0, 3), origin, '###')} + +[View ${escapeText(title)} →](${collectionPath(collection)}) · ${formatCount(repos.length)} repositories`; +}; + +const renderHome = (snapshot: DiscoverySnapshot, origin: string) => { + const curated = snapshot.collections.filter((collection) => collection.kind === 'curated'); + const topics = snapshot.collections.filter((collection) => collection.kind === 'topic'); + const sections = curated.map((collection) => renderCollectionPreview(snapshot, collection, origin)); + return renderPage(`# GitHub1s Discovery + +Discover GitHub projects and explore their source code in your browser. + +${renderMeta(`${formatCount(Object.keys(snapshot.repositories).length)} repositories · ${renderUpdatedAt(snapshot)}`)} + +${topics.map(collectionLink).join(' | ')} + +${sections.join('\n\n') || 'No collections available.'} + +--- + +${renderMeta('Source: GitHub Search. Star and fork counts reflect the values at the time of collection.')}`); +}; + +const renderCollection = (snapshot: DiscoverySnapshot, collection: DiscoveryCollection, origin: string) => { + const { title, description, icon } = getCollectionPresentation(collection); + const repos = getRepositories(snapshot, collection); + const sort = collection.sort === 'stars' ? 'Stars, descending' : 'Recently updated first'; + return renderPage(`${homeLink} + +# ${icon} ${escapeText(title)} + +${escapeText(description)} + +${renderMeta(`${formatCount(repos.length)} repositories · Sort: ${sort} · ${renderUpdatedAt(snapshot)}`)} + +${renderRepositoryList(repos, origin, '##')} + +--- + +${homeLink}`); +}; + +export const createDiscoveryPages = (snapshot: DiscoverySnapshot, origin: string): Map string> => { + const pages = new Map string>([['/README.md', () => renderHome(snapshot, origin)]]); + for (const collection of snapshot.collections) { + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(collection.id)) { + throw new Error(`Invalid Discovery collection ID: ${collection.id}`); + } + pages.set(collectionPath(collection), () => renderCollection(snapshot, collection, origin)); + } + return pages; +}; diff --git a/extensions/github1s/src/adapters/discovery/types.ts b/extensions/github1s/src/adapters/discovery/types.ts new file mode 100644 index 000000000..203760616 --- /dev/null +++ b/extensions/github1s/src/adapters/discovery/types.ts @@ -0,0 +1 @@ +export type { DiscoveryCollection, DiscoverySnapshot, Repository } from '../../../../../workers/discovery/src/types'; diff --git a/extensions/github1s/src/adapters/index.ts b/extensions/github1s/src/adapters/index.ts index 87fbe9425..db0ec4901 100644 --- a/extensions/github1s/src/adapters/index.ts +++ b/extensions/github1s/src/adapters/index.ts @@ -8,7 +8,7 @@ import { GitHub1sAdapter } from './github1s'; import { GitLab1sAdapter } from './gitlab1s'; import { BitbucketAdapter } from './bitbucket1s'; import { Npmjs1sAdapter } from './npmjs1s'; -import { OSSInsightAdapter } from './ossinsight'; +import { DiscoveryAdapter } from './discovery'; import { Adapter, DataSource, PlatformName, RouterParser } from './types'; const emptyAdapter = { @@ -25,7 +25,7 @@ export const registerAdapters = async (): Promise => { adapterManager.registerAdapter(new GitLab1sAdapter()), adapterManager.registerAdapter(new BitbucketAdapter()), adapterManager.registerAdapter(new Npmjs1sAdapter()), - adapterManager.registerAdapter(new OSSInsightAdapter()), + adapterManager.registerAdapter(new DiscoveryAdapter()), ]); }; diff --git a/extensions/github1s/src/adapters/ossinsight/interfaces.ts b/extensions/github1s/src/adapters/ossinsight/interfaces.ts index 8fb38b93f..9b3da59b2 100644 --- a/extensions/github1s/src/adapters/ossinsight/interfaces.ts +++ b/extensions/github1s/src/adapters/ossinsight/interfaces.ts @@ -55,9 +55,13 @@ export type CollectionItem = { name: string; }; -export const getCollections = memorize((): Promise => { +export const getCollections = memorize(async (): Promise => { const requestUrl = `${OSSInsightEndpoint}/v1/collections/`; - return fetch(requestUrl).then(resolveDataFromResponse); + const response = await fetch(requestUrl); + if (!response.ok) { + throw new Error(`Unable to load OSSInsight collections: HTTP ${response.status}.`); + } + return resolveDataFromResponse(response); }); export const getCollectionIdByName = async (collectionName: string): Promise => { diff --git a/extensions/github1s/src/adapters/sourcegraph/data-source.ts b/extensions/github1s/src/adapters/sourcegraph/data-source.ts index d46302920..1d42f437d 100644 --- a/extensions/github1s/src/adapters/sourcegraph/data-source.ts +++ b/extensions/github1s/src/adapters/sourcegraph/data-source.ts @@ -41,7 +41,6 @@ type SupportedPlatform = 'github' | 'gitlab' | 'bitbucket'; export class SourcegraphDataSource extends DataSource { private static instanceMap: Map = new Map(); private refsPromiseMap: Map> = new Map(); - private repositoryPromiseMap: Map> = new Map(); private fileTypeMap: Map = new Map(); // cache if path is a directory private matchedRefsMap: Map = new Map(); private textEncoder = new TextEncoder(); @@ -91,11 +90,9 @@ export class SourcegraphDataSource extends DataSource { return this.fileTypeMap.get(mapKey) || FileType.File; } + @decorate(memorize) async provideRepository(repo: string) { - if (!this.repositoryPromiseMap.has(repo)) { - this.repositoryPromiseMap.set(repo, getRepository(this.buildRepository(repo))); - } - return this.repositoryPromiseMap.get(repo); + return getRepository(this.buildRepository(repo)); } async provideFile(repo: string, ref: string, path: string): Promise { diff --git a/extensions/github1s/src/commands/global.ts b/extensions/github1s/src/commands/global.ts index ccaa79cdc..a395a8da0 100644 --- a/extensions/github1s/src/commands/global.ts +++ b/extensions/github1s/src/commands/global.ts @@ -68,7 +68,7 @@ export const commandOpenRepository = async () => { }; const commandOpenOnlineEditor = async () => { - const onlineEditorPath = ['github1s', 'ossinsight'].includes(getAdapter().scheme) ? '/editor' : '/'; + const onlineEditorPath = ['github1s', 'ossinsight', 'discovery'].includes(getAdapter().scheme) ? '/editor' : '/'; const targetLink = vscode.Uri.parse((await router.href()) || '').with({ path: onlineEditorPath }); return vscode.commands.executeCommand('vscode.open', targetLink); }; diff --git a/extensions/github1s/src/helpers/func.ts b/extensions/github1s/src/helpers/func.ts index b3f2e78c5..de5d930e9 100644 --- a/extensions/github1s/src/helpers/func.ts +++ b/extensions/github1s/src/helpers/func.ts @@ -76,13 +76,26 @@ export const memorize = any>( const result = func.call(this, ...args); cache.set(key, result); + // Retain pending and fulfilled results, but allow the next call to retry a failure. + if (result && typeof result.then === 'function') { + Promise.resolve(result).catch(() => cache.delete(key)); + } return result; }; }; export const decorate = unknown>(transformer: (func: F) => F) => { return (_target: T, _propertyKey: string, descriptor: PropertyDescriptor) => { - const originalMethod = descriptor.value; - descriptor.value = transformer(originalMethod as F); + const originalMethod: F = descriptor.value; + // Stateful transformers, such as memorize, need a separate wrapper for each instance. + const methods = new WeakMap(); + descriptor.value = function (this: object, ...args: Parameters): ReturnType { + let method = methods.get(this); + if (!method) { + method = transformer(originalMethod); + methods.set(this, method); + } + return method.apply(this, args); + }; }; }; diff --git a/package-lock.json b/package-lock.json index b35fcf191..80357c23f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,8 +10,9 @@ "hasInstallScript": true, "license": "ISC", "devDependencies": { - "@cloudflare/workers-types": "^4.20250109.0", + "@cloudflare/workers-types": "^5.20260910.1", "@github1s/vscode-web": "^0.30.2", + "@types/node": "^22.10.5", "chokidar": "^4.0.3", "clean-css": "^5.3.3", "copy-webpack-plugin": "^14.0.0", @@ -40,9 +41,9 @@ } }, "node_modules/@cloudflare/workers-types": { - "version": "4.20250109.0", - "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-4.20250109.0.tgz", - "integrity": "sha512-Y1zgSaEOOevl9ORpzgMcm4j535p3nK2lrblHHvYM2yxR50SBKGh+wvkRFAIxWRfjUGZEU+Fp6923EGioDBbobA==", + "version": "5.20260910.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workers-types/-/workers-types-5.20260910.1.tgz", + "integrity": "sha512-heMipAW1SEPWaXCcleaxGMOEpvq/F4UKdc02Gb3xiLx1La5mE6VPFGBmOTGMt7kn+qX4L8D2WsYCfmrC4kKxLw==", "dev": true, "license": "MIT OR Apache-2.0" }, @@ -7763,9 +7764,9 @@ } }, "node_modules/semver": { - "version": "7.6.3", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", - "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, "license": "ISC", "bin": { diff --git a/package.json b/package.json index 200493944..2d6460f37 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "build": "node scripts/build.js", + "discovery": "node scripts/run-discovery.ts", "watch": "rm -rf dist && run-p watch:*", "watch:dev-server": "webpack serve --mode=development", "watch:github1s-extension": "cd extensions/github1s && npm run watch", @@ -32,8 +33,9 @@ "author": "", "license": "ISC", "devDependencies": { - "@cloudflare/workers-types": "^4.20250109.0", + "@cloudflare/workers-types": "^5.20260910.1", "@github1s/vscode-web": "^0.30.2", + "@types/node": "^22.10.5", "chokidar": "^4.0.3", "clean-css": "^5.3.3", "copy-webpack-plugin": "^14.0.0", diff --git a/scripts/run-discovery.ts b/scripts/run-discovery.ts new file mode 100644 index 000000000..a1275266a --- /dev/null +++ b/scripts/run-discovery.ts @@ -0,0 +1,31 @@ +#!/usr/bin/env node + +import { mkdir, rename, rm, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { generateDiscovery } from '../workers/discovery/src/generate.ts'; + +// Preview the same collection logic without writing to Workers KV. +const main = async () => { + if (process.argv.length > 2) { + throw new Error('This preview takes no arguments; edit workers/discovery/src/collections.ts.'); + } + const snapshot = await generateDiscovery({ + token: process.env.GITHUB_TOKEN || process.env.GH_TOKEN || '', + onProgress: console.error, + }); + const output = resolve('out/discovery/github/latest.json'); + const temporary = `${output}.${process.pid}.tmp`; + await mkdir(dirname(output), { recursive: true }); + try { + await writeFile(temporary, `${JSON.stringify(snapshot)}\n`); + await rename(temporary, output); + } finally { + await rm(temporary, { force: true }); + } + console.log(`Previewed ${snapshot.collections.length} collections at ${output}`); +}; + +main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/src/config.ts b/src/config.ts index 1e7561548..998390225 100644 --- a/src/config.ts +++ b/src/config.ts @@ -107,9 +107,9 @@ export const createVSCodeWebConfig = (platform: Platform, repository: string): a const isOnlineEditor = repository === 'editor'; return { hideTextFileLabelDecorations: !isOnlineEditor, - workspace: !isOnlineEditor ? createFolderWorkspace(repository ? 'github1s' : 'ossinsight') : undefined, - workspaceId: !isOnlineEditor ? 'github1s:' + (repository || 'trending') : '', - workspaceLabel: repository || (isOnlineEditor ? '' : 'GitHub Trending'), + workspace: !isOnlineEditor ? createFolderWorkspace(repository ? 'github1s' : 'discovery') : undefined, + workspaceId: !isOnlineEditor ? 'github1s:' + (repository || 'discovery') : '', + workspaceLabel: repository || (isOnlineEditor ? '' : 'GitHub Discovery'), logo: { title: 'Open on GitHub', icon: githubLogoUrl, diff --git a/workers/discovery/README.md b/workers/discovery/README.md new file mode 100644 index 000000000..a108bc4e0 --- /dev/null +++ b/workers/discovery/README.md @@ -0,0 +1,3 @@ +# Repository Discovery Snapshots + +This Worker collects public repositories from GitHub Search on a schedule, stores complete JSON snapshots in Workers KV, and exposes a public endpoint for reading the latest snapshot. diff --git a/workers/discovery/src/collections.ts b/workers/discovery/src/collections.ts new file mode 100644 index 000000000..ce6029c56 --- /dev/null +++ b/workers/discovery/src/collections.ts @@ -0,0 +1,51 @@ +import type { DiscoveryCollection } from './types.ts'; + +export const REPOSITORIES_PER_COLLECTION = 100; + +// Curated coverage for the homepage, rather than a ranking of topic popularity. +export const TOPICS = [ + 'developer-tools', + 'machine-learning', + 'llm', + 'web-development', + 'database', + 'devops', + 'security', + 'cli', + 'compiler', + 'game-development', +]; + +export type CollectionDefinition = Pick; + +export const getCollections = (now: Date): CollectionDefinition[] => { + const dateBefore = (days: number) => new Date(now.getTime() - days * 86400000).toISOString().slice(0, 10); + return [ + { + id: 'new-projects', + kind: 'curated', + query: `created:>=${dateBefore(30)} stars:>=20`, + sort: 'stars', + }, + { + id: 'hidden-gems', + kind: 'curated', + query: `stars:50..999 pushed:>=${dateBefore(90)}`, + sort: 'updated', + }, + { + id: 'most-starred', + kind: 'curated', + query: 'stars:>=10000', + sort: 'stars', + }, + ...TOPICS.map( + (id): CollectionDefinition => ({ + id, + kind: 'topic', + query: `topic:${id} stars:>=100`, + sort: 'stars', + }), + ), + ]; +}; diff --git a/workers/discovery/src/generate.ts b/workers/discovery/src/generate.ts new file mode 100644 index 000000000..4f086bcc3 --- /dev/null +++ b/workers/discovery/src/generate.ts @@ -0,0 +1,148 @@ +import { REPOSITORIES_PER_COLLECTION, getCollections } from './collections.ts'; +import type { DiscoverySnapshot, Repository, RepositoryOwner, RepositoryLicense } from './types.ts'; + +export interface GenerateOptions { + token: string; + now?: Date; + fetchImpl?: (url: URL, init: { headers: Record; signal: AbortSignal }) => Promise; + onProgress?: (message: string) => void; +} + +// Each run is independent: all data comes from GitHub, with no stored baseline. +export const generateDiscovery = async ({ + token, + now = new Date(), + fetchImpl = fetch, + onProgress = () => {}, +}: GenerateOptions): Promise => { + if (!token?.trim()) throw new Error('GITHUB_TOKEN is required.'); + const snapshot: DiscoverySnapshot = { + generated_at: now.toISOString(), + repositories: {}, + collections: [], + }; + const headers = { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${token}`, + 'User-Agent': 'github1s-discovery', + }; + + // One page per collection, sequentially, to keep the request budget predictable. + for (const { id, kind, query, sort } of getCollections(now)) { + onProgress(`Fetching ${id}`); + const searchQuery = `${query} is:public fork:false`; + const url = new URL('https://api.github.com/search/repositories'); + url.search = new URLSearchParams({ + q: searchQuery, + sort, + order: 'desc', + per_page: String(REPOSITORIES_PER_COLLECTION), + }).toString(); + const response = await fetchImpl(url, { headers, signal: AbortSignal.timeout(30000) }); + if (!response.ok) { + const retryAfter = response.headers.get('retry-after'); + const reset = response.headers.get('x-ratelimit-reset'); + const hint = retryAfter ? ` Retry after ${retryAfter}s.` : reset ? ` Rate limit reset (Unix): ${reset}.` : ''; + throw new Error(`GitHub search for ${id} returned HTTP ${response.status}.${hint}`); + } + const result: unknown = await response.json(); + if (!isRecord(result) || !Array.isArray(result.items) || typeof result.incomplete_results !== 'boolean') { + throw new Error(`GitHub returned an invalid search response for ${id}.`); + } + if (result.incomplete_results) throw new Error(`GitHub returned incomplete search results for ${id}.`); + + const fetchedAt = new Date().toISOString(); + const repositoryIds = new Set(); + for (const item of result.items) { + const repository = parseRepository(item, fetchedAt); + snapshot.repositories[repository.id] = repository; + repositoryIds.add(repository.id); + } + snapshot.collections.push({ + id, + kind, + query: searchQuery, + sort, + order: 'desc', + per_page: REPOSITORIES_PER_COLLECTION, + total_count: countOrNull(result.total_count), + fetched_at: fetchedAt, + repository_ids: [...repositoryIds], + }); + } + + if (Object.keys(snapshot.repositories).length === 0) throw new Error('GitHub returned no repository candidates.'); + return snapshot; +}; + +const isRecord = (value: unknown): value is Record => + typeof value === 'object' && value !== null && !Array.isArray(value); +const isCount = (value: unknown): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0; +const stringOrNull = (value: unknown): string | null => (typeof value === 'string' ? value : null); +const booleanOrNull = (value: unknown): boolean | null => (typeof value === 'boolean' ? value : null); +const countOrNull = (value: unknown): number | null => (isCount(value) ? value : null); + +const parseOwner = (value: unknown): RepositoryOwner | null => { + if (!isRecord(value)) return null; + return { + id: countOrNull(value.id), + login: stringOrNull(value.login), + type: stringOrNull(value.type), + avatar_url: stringOrNull(value.avatar_url), + html_url: stringOrNull(value.html_url), + }; +}; + +const parseLicense = (value: unknown): RepositoryLicense | null => { + if (!isRecord(value)) return null; + return { + key: stringOrNull(value.key), + name: stringOrNull(value.name), + spdx_id: stringOrNull(value.spdx_id), + }; +}; + +const parseRepository = (value: unknown, fetchedAt: string): Repository => { + // Only identity is required to index a repository and link it to collections. + if ( + !isRecord(value) || + !isCount(value.id) || + value.id === 0 || + typeof value.full_name !== 'string' || + !value.full_name.trim() + ) { + throw new Error('GitHub returned invalid repository identity.'); + } + return { + id: value.id, + node_id: stringOrNull(value.node_id), + full_name: value.full_name, + owner: parseOwner(value.owner), + html_url: stringOrNull(value.html_url) || `https://github.com/${value.full_name}`, + description: stringOrNull(value.description), + homepage: stringOrNull(value.homepage), + default_branch: stringOrNull(value.default_branch), + language: stringOrNull(value.language), + topics: Array.isArray(value.topics) + ? value.topics.filter((topic): topic is string => typeof topic === 'string') + : null, + license: parseLicense(value.license), + stargazers_count: countOrNull(value.stargazers_count), + forks_count: countOrNull(value.forks_count), + open_issues_count: countOrNull(value.open_issues_count), + size: countOrNull(value.size), + archived: booleanOrNull(value.archived), + disabled: booleanOrNull(value.disabled), + is_template: booleanOrNull(value.is_template), + allow_forking: booleanOrNull(value.allow_forking), + has_issues: booleanOrNull(value.has_issues), + has_discussions: booleanOrNull(value.has_discussions), + has_wiki: booleanOrNull(value.has_wiki), + has_pages: booleanOrNull(value.has_pages), + created_at: stringOrNull(value.created_at), + updated_at: stringOrNull(value.updated_at), + pushed_at: stringOrNull(value.pushed_at), + fetched_at: fetchedAt, + }; +}; diff --git a/workers/discovery/src/index.ts b/workers/discovery/src/index.ts new file mode 100644 index 000000000..2dc69cccf --- /dev/null +++ b/workers/discovery/src/index.ts @@ -0,0 +1,41 @@ +import { generateDiscovery } from './generate.ts'; +import { LATEST_KEY, saveSnapshot } from './snapshots.ts'; + +export interface Env { + DISCOVERY_KV: Pick; + GITHUB_TOKEN: string; +} + +export default { + async fetch(request: Request, env: Env): Promise { + const headers = new Headers({ 'Access-Control-Allow-Origin': '*' }); + if (new URL(request.url).pathname !== '/github/latest.json') { + return new Response('Not found', { status: 404, headers }); + } + if (request.method !== 'GET' && request.method !== 'HEAD') { + headers.set('Allow', 'GET, HEAD'); + return new Response('Method not allowed', { status: 405, headers }); + } + try { + const json = await env.DISCOVERY_KV.get(LATEST_KEY); + if (json === null) { + return new Response(request.method === 'HEAD' ? null : 'Snapshot not available', { status: 404, headers }); + } + headers.set('Content-Type', 'application/json; charset=utf-8'); + headers.set('Cache-Control', 'public, max-age=3600'); + return new Response(request.method === 'HEAD' ? null : json, { headers }); + } catch (error) { + console.error('Unable to read discovery snapshot.', error); + return new Response(request.method === 'HEAD' ? null : 'Snapshot temporarily unavailable', { + status: 503, + headers, + }); + } + }, + + async scheduled(controller: ScheduledController, env: Env): Promise { + const snapshot = await generateDiscovery({ token: env.GITHUB_TOKEN, onProgress: console.log }); + await saveSnapshot(env.DISCOVERY_KV, snapshot, controller.scheduledTime); + console.log(`Published discovery snapshot ${snapshot.generated_at}.`); + }, +} satisfies ExportedHandler; diff --git a/workers/discovery/src/snapshots.ts b/workers/discovery/src/snapshots.ts new file mode 100644 index 000000000..9966ca5fe --- /dev/null +++ b/workers/discovery/src/snapshots.ts @@ -0,0 +1,22 @@ +import type { DiscoverySnapshot } from './types.ts'; + +export const LATEST_KEY = 'latest'; +export const SNAPSHOT_RETENTION_SECONDS = 90 * 24 * 60 * 60; + +// Use the scheduled UTC date so execution delays cannot change the archive day. +export const getHistoryKey = (scheduledTime: number): string => + `history:${new Date(scheduledTime).toISOString().slice(0, 10)}`; + +export const saveSnapshot = async ( + kv: Pick, + snapshot: DiscoverySnapshot, + scheduledTime = Date.parse(snapshot.generated_at), +): Promise => { + const expiration = Math.floor(Date.parse(snapshot.generated_at) / 1000) + SNAPSHOT_RETENTION_SECONDS; + const json = JSON.stringify(snapshot); + // Reruns replace the same day. KV uses last-write-wins, not conditional creation. + await kv.put(getHistoryKey(scheduledTime), json, { expiration }); + // Preserve the previous latest snapshot if collection or archival fails. + // Latest also expires so a stopped collector cannot retain data past 90 days. + await kv.put(LATEST_KEY, json, { expiration }); +}; diff --git a/workers/discovery/src/types.ts b/workers/discovery/src/types.ts new file mode 100644 index 000000000..193c2368b --- /dev/null +++ b/workers/discovery/src/types.ts @@ -0,0 +1,67 @@ +export interface RepositoryOwner { + id: number | null; + login: string | null; + type: string | null; + avatar_url: string | null; + html_url: string | null; +} + +export interface RepositoryLicense { + key: string | null; + name: string | null; + spdx_id: string | null; +} + +// Select public metadata for discovery, filtering, and historical comparisons. +export interface Repository { + id: number; + node_id: string | null; + full_name: string; + owner: RepositoryOwner | null; + html_url: string; + description: string | null; + homepage: string | null; + default_branch: string | null; + language: string | null; + // Missing metadata stays unknown instead of becoming zero, an empty list, or false. + topics: string[] | null; + license: RepositoryLicense | null; + stargazers_count: number | null; + forks_count: number | null; + // GitHub includes both issues and pull requests in this count. + open_issues_count: number | null; + // Repository size in KB, not source lines or working-tree size. + size: number | null; + archived: boolean | null; + disabled: boolean | null; + is_template: boolean | null; + allow_forking: boolean | null; + has_issues: boolean | null; + has_discussions: boolean | null; + has_wiki: boolean | null; + has_pages: boolean | null; + created_at: string | null; + updated_at: string | null; + pushed_at: string | null; + // Local observation time for these counts, not a GitHub activity timestamp. + fetched_at: string; +} + +export interface DiscoveryCollection { + id: string; + kind: 'topic' | 'curated'; + query: string; + sort: 'stars' | 'updated'; + order: 'desc'; + per_page: number; + total_count: number | null; + fetched_at: string; + repository_ids: number[]; +} + +export interface DiscoverySnapshot { + generated_at: string; + // JSON object keys are GitHub repository IDs, shared by all collections. + repositories: Record; + collections: DiscoveryCollection[]; +} diff --git a/workers/discovery/wrangler.jsonc b/workers/discovery/wrangler.jsonc new file mode 100644 index 000000000..7b4e07a70 --- /dev/null +++ b/workers/discovery/wrangler.jsonc @@ -0,0 +1,24 @@ +{ + "name": "discovery", + "main": "src/index.ts", + "compatibility_date": "2026-09-09", + "workers_dev": false, + "routes": [ + { + "pattern": "discovery.github1s.com", + "custom_domain": true + } + ], + "cache": { + "enabled": true + }, + "triggers": { + // Run at 12:15 UTC every day + "crons": ["15 12 * * *"] + }, + "kv_namespaces": [ + { + "binding": "DISCOVERY_KV" + } + ] +} diff --git a/workers/tsconfig.json b/workers/tsconfig.json new file mode 100644 index 000000000..e08d12e04 --- /dev/null +++ b/workers/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "lib": ["ES2022"], + "types": ["@cloudflare/workers-types"], + "strict": true, + "noEmit": true, + "allowImportingTsExtensions": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true + }, + "include": ["**/*.ts"], + "exclude": ["**/*.test.ts", "**/node_modules/**", "**/dist/**", "**/out/**"] +}