diff --git a/AGENTS.md b/AGENTS.md index 73ee03eff12..f4a9850e099 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,8 @@ This repository (`google/osv.dev`) contains the backend services, database model It is structured as a multi-language monorepo: - `go/`: Go services and utilities (importers, exporter, internal libraries). **This is the primary target for active migrations from Python.** - `osv/`: Core Python library containing models, repository helpers, and ecosystem-specific logic. *Note: Some parts are deprecated as we migrate logic to Go.* -- `gcp/`: GCP deployment configurations, Cloud Functions, API server, and workers. (Website frontend uses `pnpm` and Hugo). +- `gcp/`: GCP deployment configurations, Cloud Functions, and workers. +- `website/`: Frontend assets for the osv.dev website (`frontend3` using `pnpm`) and blog posts (`blog` using Hugo). - `vulnfeeds/`: Vulnerability feed utilities (*independent Go module*). - `bindings/`: API bindings (*contains an independent Go module under `bindings/go`*). @@ -73,7 +74,7 @@ The project uses `poetry` for Python dependency management, `pnpm` for website f ``` - **Install Website Dependencies** (for frontend development): ```bash - cd gcp/website/frontend3 && pnpm install + cd website/frontend3 && pnpm install ``` - **Initialize Git Submodules**: ```bash @@ -302,9 +303,9 @@ Contains deployment setups, workers running in GKE, Cloud Functions, and the use - **Deployment Target**: **Google Cloud Run** (managed via Cloud Deploy pipeline `osv-api` deploying to `osv-grpc-backend`). - *Note*: Fully migrated from Python to Go. Protobuf definitions and descriptor files are located under `proto/v1/`. -### 2. Website (`go/cmd/website/`, `gcp/website/`) -- **Status**: **Active (Go / Python)**. -- Migrated to Go backend under `go/cmd/website/` and `go/internal/website/`. Frontend assets (Hugo blog, pnpm frontend3) are located under `website/` (symlinked to `gcp/website/`). +### 2. Website (`go/cmd/website/`, `website/`) +- **Status**: **Active (Go)**. +- Fully migrated to Go backend under `go/cmd/website/` and `go/internal/website/`. Frontend assets (Hugo blog, pnpm frontend3) are located under `website/`. - **Deployment Target**: **Google Cloud Run** (managed via Cloud Deploy pipeline `osv-website`). ### 3. Workers (`gcp/workers/`) diff --git a/Makefile b/Makefile index 6e805523d21..0634d065915 100644 --- a/Makefile +++ b/Makefile @@ -21,9 +21,6 @@ lib-tests: ## Run core Python library tests vanir-signatures-tests: ## Run Vanir signatures tests cd gcp/workers/vanir_signatures && ./run_tests.sh -website-tests: ## Run legacy Python website tests - cd gcp/website && ./run_tests.sh - vulnfeed-tests: ## Run Go vulnfeeds tests cd vulnfeeds && ./run_tests.sh @@ -118,7 +115,7 @@ run-api-server-test: @cd go && go build -o ./api-devserver ./cmd/api-devserver && (GOOGLE_CLOUD_PROJECT=oss-vdb-test OSV_VULNERABILITIES_BUCKET=osv-test-vulnerabilities ./api-devserver $(ARGS); EXIT_CODE=$$?; rm -f ./api-devserver; exit $$EXIT_CODE) # TODO: API integration tests. -all-tests: lib-tests website-tests vulnfeed-tests bindings-tests go-tests ## Run all tests +all-tests: lib-tests vulnfeed-tests bindings-tests go-tests ## Run all tests reimport-tui: ## Run the reimport TUI tool test -f $(HOME)/.config/gcloud/application_default_credentials.json || (echo "GCP Application Default Credentials not set, try 'gcloud auth application-default login'"; exit 1) diff --git a/README.md b/README.md index 3a87932e5e4..f2be6830541 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,12 @@ consists of: | `gcp/datastore` | The datastore index file (`index.yaml`) | | `gcp/functions` | The Cloud Function for publishing PyPI vulnerabilities (maintained, but not developed) | | `gcp/indexer` | The determine version `indexer` | -| `gcp/website` | Frontend assets for the osv.dev website (in `frontend3`) and blog posts (in `blog`) | | `gcp/workers/` | Python workers (`vanir_signatures` and `oss_fuzz_worker`) | | `go/` | Go module for shared libraries and commands (`cmd/api`, `cmd/website`, `cmd/importer`, `cmd/worker`, `cmd/exporter`, `cmd/recoverer`, `cmd/relations`, etc.) | | `osv/` | The core OSV Python library, used in basically all Python services
OSV ecosystem package versioning helpers in `ecosystems/`
Datastore model definitions in `models.py` | | `tools/` | Misc scripts/tools, mostly intended for development (datastore stuff, linting)
The `indexer-api-caller` for indexer calling | | `vulnfeeds/` | Go module for (mostly) the NVD CVE conversion
The Alpine feed converter (`cmd/alpine`)
The Debian feed converter (`tools/debian`, which is written in Python) | +| `website/` | Frontend assets for the osv.dev website (in `frontend3`) and blog posts (in `blog`) | You'll need to check out submodules as well for many local building steps to diff --git a/cloudbuild.yaml b/cloudbuild.yaml index dd82e954428..dd55a7453b4 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -27,11 +27,6 @@ steps: args: ['gcloud', 'builds', 'submit', '--region=${LOCATION}', '--config=gcp/workers/cloudbuild.yaml', '.'] waitFor: ['init'] -- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:slim' - id: 'website-tests' - args: ['gcloud', 'builds', 'submit', '--region=${LOCATION}', '--config=gcp/website/cloudbuild.yaml', '.'] - waitFor: ['init'] - - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:slim' id: 'vulnfeeds-tests' args: ['gcloud', 'builds', 'submit', '--region=${LOCATION}', '--config=vulnfeeds/cloudbuild.yaml', '.'] diff --git a/deployment/build-and-stage.yaml b/deployment/build-and-stage.yaml index ff14825c64e..7adcc171682 100644 --- a/deployment/build-and-stage.yaml +++ b/deployment/build-and-stage.yaml @@ -462,13 +462,13 @@ steps: corepack enable pnpm pnpm install --frozen-lockfile --ignore-scripts pnpm run build:prod - dir: 'gcp/website/frontend3' + dir: 'website/frontend3' id: 'build-frontend3' waitFor: ['setup'] - name: 'gcr.io/oss-vdb/ci' args: ['hugo', '--buildFuture', '-d', '../dist/static/blog'] - dir: 'gcp/website/blog' + dir: 'website/blog' id: 'build-hugo' waitFor: ['setup'] @@ -483,7 +483,7 @@ steps: '-t', 'gcr.io/oss-vdb/osv-website:latest', '-t', 'gcr.io/oss-vdb/osv-website:$COMMIT_SHA', '--target', 'website', '--build-context', 'bindings=../bindings', - '--build-context', 'website-dist=../gcp/website/dist', + '--build-context', 'website-dist=../website/dist', '--build-context', 'docs=../docs', '-f', 'Dockerfile', '--cache-from', 'gcr.io/oss-vdb/osv-website:latest', '--pull', '.'] dir: 'go' diff --git a/docs/contributing/architecture.md b/docs/contributing/architecture.md index 74250d74ab6..dd3230c79fd 100644 --- a/docs/contributing/architecture.md +++ b/docs/contributing/architecture.md @@ -67,7 +67,6 @@ The [API server](../api/index.md) (hosted at `api.osv.dev`, source code in [`go/ ## Website -The [main web UI](https://osv.dev) (source code in [`gcp/website`](../../gcp/website)) also runs on [Cloud Run], and is served through [Cloud Load Balancing]. +The [main web UI](https://osv.dev) (server source code in [`go/cmd/website`](../../go/cmd/website) and frontend assets in [`website`](../../website)) also runs on [Cloud Run], and is served through [Cloud Load Balancing]. -[Cloud Run]: https://cloud.google.com/run [Cloud Load Balancing]: https://cloud.google.com/load-balancing diff --git a/gcp/website/.gitignore b/gcp/website/.gitignore deleted file mode 100644 index 8e42b8e16ac..00000000000 --- a/gcp/website/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -dist/ -requirements.txt -app.yaml -cron-service.yaml -testing-app.yaml diff --git a/gcp/website/Dockerfile b/gcp/website/Dockerfile deleted file mode 100644 index 7374bbe4c5d..00000000000 --- a/gcp/website/Dockerfile +++ /dev/null @@ -1,69 +0,0 @@ -# Build the Javascript frontend -FROM node:24.20@sha256:be23f54a88d34e8824c741b19b91064094f92c1c97b194144bfc8b50d67258e2 AS FRONTEND3_BUILD -WORKDIR /build/frontend3 - -# Install dependencies first for better caching -RUN corepack enable pnpm -COPY gcp/website/frontend3/package.json gcp/website/frontend3/pnpm-lock.yaml ./ -RUN pnpm install --frozen-lockfile --ignore-scripts - -COPY gcp/website/frontend3/webpack.prod.js ./ -COPY gcp/website/frontend3/img img -COPY gcp/website/frontend3/src src - -RUN pnpm run build:prod - -# Build hugo blogs -# Use the ci image, since it already built the version of hugo we want from source -FROM gcr.io/oss-vdb/ci AS HUGO_BUILD - -WORKDIR /build/blog -COPY gcp/website/blog ./ - -RUN hugo --buildFuture -d ../dist/static/blog - -# OSV.dev site image -# Adapted from https://cloud.google.com/run/docs/quickstarts/build-and-deploy/deploy-python-service#writing -FROM python:3.13.3-slim@sha256:56a11364ffe0fee3bd60af6d6d5209eba8a99c2c16dc4c7c5861dc06261503cc - -# Generation 1 of cloud run overrides the HOME environment variable, causing -# poetry to run in the incorrect environment, as it defaults to using $HOME/.cache/virtualenvs/... -# -# This forces it to create the virtualenv in the same directory as the project, avoiding this issue. -ENV POETRY_VIRTUALENVS_IN_PROJECT=true -ENV POETRY_HOME "/opt/poetry" -COPY docker/poetry/requirements.txt ./poetry-requirements.txt -RUN python3 -m venv $POETRY_HOME && \ - $POETRY_HOME/bin/pip install --require-hashes -r ./poetry-requirements.txt && \ - ln -s $POETRY_HOME/bin/poetry /usr/local/bin/poetry - -# Allow statements and log messages to immediately appear in the logs -ENV PYTHONUNBUFFERED True -WORKDIR /osv/gcp/website - -# Install Python dependencies -COPY poetry.lock pyproject.toml README.md /osv/ -COPY osv /osv/osv -COPY gcp/website/poetry.lock gcp/website/pyproject.toml ./ -RUN poetry install - -# Website Python code -COPY gcp/website/*.py ./ - -# JS/hugo builds -COPY gcp/website/dist/public_keys dist/public_keys -COPY gcp/website/docs docs -# gcp/website/docs/docs/osv_service_v1.swagger.json is a symlink -COPY docs/osv_service_v1.swagger.json docs/ - -COPY --from=FRONTEND3_BUILD /build/dist/ dist/ -COPY --from=HUGO_BUILD /build/dist dist/ - -RUN poetry run python -m whitenoise.compress dist/static/ - -# Run the web service on container startup. Here we use the gunicorn -# webserver, with one worker process and 8 threads. -# For environments with multiple CPU cores, increase the number of workers -# to be equal to the cores available. -# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. -CMD poetry run gunicorn --bind :$PORT --workers 1 --threads 8 --timeout 0 main:app diff --git a/gcp/website/auth.py b/gcp/website/auth.py deleted file mode 100644 index 1bda4730ad0..00000000000 --- a/gcp/website/auth.py +++ /dev/null @@ -1,138 +0,0 @@ -"""Authentication handlers.""" -import os -import secrets -from functools import wraps -import requests - -from flask import Blueprint, request, session, redirect, url_for, jsonify -from google.oauth2 import id_token -from google.auth.transport import requests as google_requests - -blueprint = Blueprint('auth', __name__) - -GOOGLE_CLIENT_ID = os.environ.get("GOOGLE_OAUTH_CLIENT_ID") -GOOGLE_CLIENT_SECRET = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET") -GOOGLE_DISCOVERY_URL = ("https://accounts.google.com/" - ".well-known/openid-configuration") - -# Easy bypass for local development testing -BYPASS_OAUTH_FOR_LOCAL_DEV = os.environ.get("BYPASS_OAUTH_FOR_LOCAL_DEV", - "False").lower() in ("true", "1", - "t") - - -def get_google_provider_cfg(): - """Get Google provider configuration.""" - return requests.get(GOOGLE_DISCOVERY_URL, timeout=10).json() - - -@blueprint.route("/login") -def login(): - """Login route.""" - if BYPASS_OAUTH_FOR_LOCAL_DEV: - session['user_email'] = 'dev@google.com' - return redirect(url_for('triage_handlers.triage_index')) - - if not GOOGLE_CLIENT_ID or not GOOGLE_CLIENT_SECRET: - return jsonify({ - 'error': 'OAuth credentials not configured. ' - 'Set GOOGLE_OAUTH_CLIENT_ID and ' - 'GOOGLE_OAUTH_CLIENT_SECRET env vars or ' - 'enable BYPASS_OAUTH_FOR_LOCAL_DEV.' - }), 500 - - google_provider_cfg = get_google_provider_cfg() - authorization_endpoint = google_provider_cfg["authorization_endpoint"] - - state = secrets.token_urlsafe(16) - session['oauth_state'] = state - - redirect_uri = url_for('auth.callback', _external=True) - # Ensure redirect_uri uses https if the app is accessed over https - # (for environments behind load balancers without proxyfix) - if request.headers.get( - 'X-Forwarded-Proto', - 'http') == 'https' and redirect_uri.startswith('http://'): - redirect_uri = redirect_uri.replace('http://', 'https://', 1) - - request_uri = (f"{authorization_endpoint}?response_type=code" - f"&client_id={GOOGLE_CLIENT_ID}" - f"&redirect_uri={redirect_uri}" - f"&scope=openid%20email%20profile" - f"&state={state}" - f"&access_type=offline") - - return redirect(request_uri) - - -@blueprint.route("/auth/callback") -def callback(): - """Auth callback route.""" - if request.args.get('state') != session.get('oauth_state'): - return jsonify({'error': 'Invalid state parameter'}), 400 - - code = request.args.get("code") - google_provider_cfg = get_google_provider_cfg() - token_endpoint = google_provider_cfg["token_endpoint"] - - redirect_uri = url_for('auth.callback', _external=True) - if request.headers.get( - 'X-Forwarded-Proto', - 'http') == 'https' and redirect_uri.startswith('http://'): - redirect_uri = redirect_uri.replace('http://', 'https://', 1) - - token_url = token_endpoint - token_data = { - "code": code, - "client_id": GOOGLE_CLIENT_ID, - "client_secret": GOOGLE_CLIENT_SECRET, - "redirect_uri": redirect_uri, - "grant_type": "authorization_code", - } - - token_response = requests.post(token_url, data=token_data, timeout=10) - token_json = token_response.json() - - if "id_token" not in token_json: - return jsonify({ - 'error': 'Failed to obtain ID token. ' - 'Maybe credentials mismatch or invalid code.' - }), 400 - - token = token_json["id_token"] - - try: - # Verify the token - client_request = google_requests.Request() - id_info = id_token.verify_oauth2_token(token, client_request, - GOOGLE_CLIENT_ID) - - # The oauth only allows users to login with accounts that have - # access to the GCP project - session['user_email'] = id_info.get("email") - return redirect(url_for('triage_handlers.triage_index')) - - except ValueError: - return jsonify({'error': 'Invalid token'}), 400 - - -@blueprint.route("/logout") -def logout(): - session.pop('user_email', None) - return redirect(url_for('triage_handlers.triage_index')) - - -def require_google_account(f): - """Decorator to require Google account.""" - - @wraps(f) - def decorated_function(*args, **kwargs): - if BYPASS_OAUTH_FOR_LOCAL_DEV: - return f(*args, **kwargs) - - if 'user_email' not in session: - return redirect(url_for('auth.login')) - - return f(*args, **kwargs) - - return decorated_function diff --git a/gcp/website/build.sh b/gcp/website/build.sh deleted file mode 100755 index afa18998a6c..00000000000 --- a/gcp/website/build.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -x -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -cd ../.. && - docker build -t gcr.io/oss-vdb/osv-website:$1 -t gcr.io/oss-vdb/osv-website:latest -f gcp/website/Dockerfile --pull . diff --git a/gcp/website/cache.py b/gcp/website/cache.py deleted file mode 100644 index 95c22b0ac65..00000000000 --- a/gcp/website/cache.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2022 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Caching.""" - -import functools -import os -from typing import Callable -from concurrent.futures import ThreadPoolExecutor, Future - -import flask_caching - -import utils - -if utils.is_cloud_run(): - instance = flask_caching.Cache( - config={ - 'CACHE_TYPE': 'RedisCache', - 'CACHE_REDIS_HOST': os.environ.get('REDISHOST', '127.0.0.1'), - 'CACHE_REDIS_PORT': int(os.environ.get('REDISPORT', 6379)), - }) -else: - instance = flask_caching.Cache(config={ - 'CACHE_TYPE': 'SimpleCache', - }) - -_should_refresh_suffix = '__refresh_marker' -_executor_map: dict[str, ThreadPoolExecutor] = {} -_future_map: dict[str, Future] = {} - - -def smart_cache( - key: str, - hard_timeout: int, - soft_timeout: int, -) -> Callable: - """ - The decorated function will be cached with the given key. - - If the decorated function is called any time after the `soft_timeout` - of the cache, the cached value will still be returned, but the cache - will be refreshed in an asynchronous background thread. - - Currently this decorator does not support differing arguments. - """ - if key in _executor_map: - raise ValueError('key already exists') - - # Only require one background thread to run per cache key - # since we check whether an existing update task is already running - # before queuing another update. - _executor_map[key] = ThreadPoolExecutor(1) - - def decorator(f): - - @functools.wraps(f) - def decorated_function(*args, **kwargs): - - def update_func(): - result = f(*args, **kwargs) - instance.set(key, result, timeout=hard_timeout) - instance.set(key + _should_refresh_suffix, True, timeout=soft_timeout) - return result - - result = instance.get(key) - if result is None: # If the main cached value expires, refresh normally - return update_func() - - # Only refresh the cached value once instance times out - should_refresh_cached_value = not instance.has(key + - _should_refresh_suffix) - future = _future_map.get(key) - if should_refresh_cached_value and (future is None or future.done()): - # Store the future to make sure it executes. - # (Dropped futures are not executed) - # Also for reference to prevent queueing up multiple updates - _future_map[key] = _executor_map[key].submit(update_func) - - return result - - return decorated_function - - return decorator diff --git a/gcp/website/cloudbuild.yaml b/gcp/website/cloudbuild.yaml deleted file mode 100644 index 4fd73299be1..00000000000 --- a/gcp/website/cloudbuild.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# website test runner -# This should be triggered on changes to the following: -# gcp/website/poetry.lock -# gcp/website/run_tests.sh -# gcp/website/*.py -# gcp/website/**/*.py -# osv/*.py -# osv/**/*.py - -steps: -- name: 'gcr.io/cloud-builders/git' - id: 'init' - args: ['submodule', 'update', '--init'] - # if this is invoked from another cloud build, this will fail as it is not a git repo - # the invoking cloud build file should run this step. - allowFailure: true -- name: 'gcr.io/oss-vdb/ci' - id: 'sync' - dir: gcp/website - args: ['poetry', 'sync'] - waitFor: ['-'] - -- name: 'gcr.io/oss-vdb/ci' - id: 'website-tests' - dir: gcp/website - args: ['bash', '-ex', 'run_tests.sh'] - env: - # Each concurrent test that uses the datastore emulator must have a unique port number - - DATASTORE_EMULATOR_PORT=8004 - waitFor: ['init', 'sync'] - -timeout: 7200s -options: - env: - - CLOUDBUILD=1 diff --git a/gcp/website/docs/osv_service_v1.swagger.json b/gcp/website/docs/osv_service_v1.swagger.json deleted file mode 120000 index 3666a5d9c51..00000000000 --- a/gcp/website/docs/osv_service_v1.swagger.json +++ /dev/null @@ -1 +0,0 @@ -../../../docs/osv_service_v1.swagger.json \ No newline at end of file diff --git a/gcp/website/emulator_aliases.py b/gcp/website/emulator_aliases.py deleted file mode 100644 index 1766fd76848..00000000000 --- a/gcp/website/emulator_aliases.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""OSV alias computation.""" -import datetime -import logging - -from google.cloud import exceptions -from google.cloud import ndb - -import osv -from osv import gcs, pubsub - -ALIAS_GROUP_VULN_LIMIT = 32 -VULN_ALIASES_LIMIT = 5 - - -def _update_group(bug_ids: list[str], alias_group: osv.AliasGroup, - changed_vulns: dict[str, osv.AliasGroup | None]): - """Updates the alias group in the datastore.""" - if len(bug_ids) <= 1: - logging.info('Deleting alias group due to too few bugs: %s', bug_ids) - for vuln_id in bug_ids: - changed_vulns[vuln_id] = None - alias_group.key.delete() - return - if len(bug_ids) > ALIAS_GROUP_VULN_LIMIT: - logging.info('Deleting alias group due to too many bugs: %s', bug_ids) - for vuln_id in bug_ids: - changed_vulns[vuln_id] = None - alias_group.key.delete() - return - - if bug_ids == alias_group.bug_ids: - return - - alias_group.bug_ids = bug_ids - alias_group.last_modified = datetime.datetime.now(datetime.UTC) - alias_group.put() - for vuln_id in bug_ids: - changed_vulns[vuln_id] = alias_group - - -def _create_alias_group(bug_ids: list[str], - changed_vulns: dict[str, osv.AliasGroup | None]): - """Creates a new alias group in the datastore.""" - if len(bug_ids) <= 1: - logging.info('Skipping alias group creation due to too few bugs: %s', - bug_ids) - return - if len(bug_ids) > ALIAS_GROUP_VULN_LIMIT: - logging.info('Skipping alias group creation due to too many bugs: %s', - bug_ids) - return - - new_group = osv.AliasGroup(bug_ids=bug_ids) - new_group.last_modified = datetime.datetime.now(datetime.UTC) - new_group.put() - for vuln_id in bug_ids: - changed_vulns[vuln_id] = new_group - - -def _compute_aliases(bug_id: str, visited: set[str], - bug_aliases: dict[str, set[str]]) -> list[str]: - """Computes all aliases for the given bug ID. - The returned list contains the bug ID itself, all the IDs from the bug's - raw aliases, all the IDs of bugs that have the current bug as an alias, - and repeat for every bug encountered here.""" - to_visit = {bug_id} - bug_ids = [] - while to_visit: - bug_id = to_visit.pop() - if bug_id in visited: - continue - visited.add(bug_id) - bug_ids.append(bug_id) - - aliases = bug_aliases.get(bug_id, set()) - to_visit.update(aliases - visited) - - # Returns a sorted list of bug IDs, which ensures deterministic behaviour - # and avoids unnecessary updates to the groups. - return sorted(bug_ids) - - -def _update_vuln_with_group(vuln_id: str, alias_group: osv.AliasGroup | None): - """Updates the Vulnerability in Datastore & GCS with the new alias group. - If `alias_group` is None, assumes a preexisting AliasGroup was just deleted. - """ - # Get the existing vulnerability first, so we can recalculate search_indices - result = gcs.get_by_id_with_generation(vuln_id) - if result is None: - if osv.Vulnerability.get_by_id(vuln_id) is not None: - logging.error('vulnerability not in GCS - %s', vuln_id) - pubsub.publish_failure(b'', type='gcs_missing', id=vuln_id) - return - vuln_proto, generation = result - - def transaction(): - vuln: osv.Vulnerability = osv.Vulnerability.get_by_id(vuln_id) - if vuln is None: - logging.error('vulnerability not in Datastore - %s', vuln_id) - # TODO(michaelkedar): What to do in this case? - return - if alias_group is None: - modified = datetime.datetime.now(datetime.UTC) - aliases = [] - else: - modified = alias_group.last_modified - aliases = sorted(set(alias_group.bug_ids) - {vuln_id}) - vuln_proto.aliases[:] = aliases - vuln_proto.modified.FromDatetime(modified) - osv.ListedVulnerability.from_vulnerability(vuln_proto).put() - vuln.modified = modified - vuln.put() - - ndb.transaction(transaction) - try: - gcs.upload_vulnerability(vuln_proto, generation) - except exceptions.PreconditionFailed: - logging.error('Generation mismatch when writing aliases for %s', vuln_id) - osv.pubsub.publish_failure( - b'', type='gcs_gen_mismatch', id=vuln_id, field='aliases') - except Exception: - logging.error('Writing to bucket failed for %s', vuln_id) - osv.pubsub.publish_failure( - vuln_proto.SerializeToString(deterministic=True), type='gcs_retry') - - -def run(): - """Updates all alias groups in the datastore by re-computing existing - AliasGroups and creating new AliasGroups for un-computed bugs.""" - - # Query for all bugs that have aliases. - # Use (> '' OR < '') instead of (!= '') / (> '') to de-duplicate results - # and avoid datastore emulator problems, see issue #2093 - bugs = osv.Bug.query(ndb.OR(osv.Bug.aliases > '', osv.Bug.aliases < '')) - all_alias_group = osv.AliasGroup.query() - allow_list = { - allow_entry.bug_id for allow_entry in osv.AliasAllowListEntry.query() - } - deny_list = { - deny_entry.bug_id for deny_entry in osv.AliasDenyListEntry.query() - } - - # Mapping of ID to a set of all aliases for that bug, - # including its raw aliases and bugs that it is referenced in as an alias. - bug_aliases = {} - - # For each bug, add its aliases to the maps and ignore invalid bugs. - for bug in bugs: - if bug.db_id in deny_list: - continue - if len(bug.aliases) > VULN_ALIASES_LIMIT and bug.db_id not in allow_list: - logging.info('%s has too many listed aliases, skipping computation.', - bug.db_id) - continue - if bug.status != osv.BugStatus.PROCESSED: - continue - for alias in bug.aliases: - bug_aliases.setdefault(bug.db_id, set()).add(alias) - bug_aliases.setdefault(alias, set()).add(bug.db_id) - - visited = set() - - # Keep track of vulnerabilities that have been modified, to update GCS later. - # `None` means the AliasGroup has been removed. - changed_vulns: dict[str, osv.AliasGroup | None] = {} - - # For each alias group, re-compute the bug IDs in the group and update the - # group with the computed bug IDs. - for alias_group in all_alias_group: - bug_id = alias_group.bug_ids[0] # AliasGroups contain more than one bug. - # If the bug has already been counted in a different alias group, - # we delete the original one to merge two alias groups. - if bug_id in visited: - for vuln_id in alias_group.bug_ids: - if vuln_id not in changed_vulns: - changed_vulns[vuln_id] = None - alias_group.key.delete() - continue - bug_ids = _compute_aliases(bug_id, visited, bug_aliases) - _update_group(bug_ids, alias_group, changed_vulns) - - # For each bug ID that has not been visited, create new alias groups. - for bug_id in bug_aliases: - if bug_id not in visited: - bug_ids = _compute_aliases(bug_id, visited, bug_aliases) - _create_alias_group(bug_ids, changed_vulns) - - # For each updated vulnerability, update them in Datastore & GCS - for vuln_id, alias_group in changed_vulns.items(): - _update_vuln_with_group(vuln_id, alias_group) diff --git a/gcp/website/emulator_upstream.py b/gcp/website/emulator_upstream.py deleted file mode 100644 index 9a06771aaf3..00000000000 --- a/gcp/website/emulator_upstream.py +++ /dev/null @@ -1,243 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""OSV Upstream relation computation.""" - -from collections import defaultdict -import datetime -import json -import logging - -from google.cloud import exceptions -from google.cloud import ndb - -import osv -from osv import gcs, pubsub - - -def compute_upstream(target_bug, bugs: dict[str, set[str]]) -> list[str]: - """Computes all upstream vulnerabilities for the given bug ID. - The returned list contains all of the bug IDs that are upstream of the - target bug ID, including transitive upstreams.""" - visited = set() - - target_bug_upstream = target_bug - if not target_bug_upstream: - return [] - to_visit = set(target_bug_upstream) - while to_visit: - bug_id = to_visit.pop() - if bug_id in visited: - continue - visited.add(bug_id) - upstreams = set() - if bug_id in bugs.keys(): - bug = bugs.get(bug_id) - upstreams = set(bug) - - to_visit.update(upstreams - visited) - - # Returns a sorted list of bug IDs, which ensures deterministic behaviour - # and avoids unnecessary updates. - return sorted(visited) - - -def _create_group(bug_id: str, upstream_ids: list[str]) -> osv.UpstreamGroup: - """Creates a new upstream group in the datastore.""" - - new_group = osv.UpstreamGroup( - id=bug_id, - db_id=bug_id, - upstream_ids=upstream_ids, - last_modified=datetime.datetime.now(datetime.UTC)) - new_group.put() - _update_vuln_with_group(bug_id, new_group) - - return new_group - - -def _update_group(upstream_group: osv.UpstreamGroup, - upstream_ids: list[str]) -> osv.UpstreamGroup | None: - """Updates the upstream group in the datastore.""" - if len(upstream_ids) == 0: - logging.info('Deleting upstream group due to too few bugs: %s', - upstream_ids) - upstream_group.key.delete() - _update_vuln_with_group(upstream_group.db_id, None) - return None - - if upstream_ids == upstream_group.upstream_ids: - return None - - upstream_group.upstream_ids = upstream_ids - upstream_group.last_modified = datetime.datetime.now(datetime.UTC) - upstream_group.put() - _update_vuln_with_group(upstream_group.db_id, upstream_group) - return upstream_group - - -def _update_vuln_with_group(vuln_id: str, upstream: osv.UpstreamGroup | None): - """Updates the Vulnerability in Datastore & GCS with the new upstream group. - If `upstream` is None, assumes a preexisting UpstreamGroup was just deleted. - """ - # Get the existing vulnerability first, so we can recalculate search_indices - result = gcs.get_by_id_with_generation(vuln_id) - if result is None: - logging.error('vulnerability not in GCS - %s', vuln_id) - pubsub.publish_failure(b'', type='gcs_missing', id=vuln_id) - return - vuln_proto, generation = result - - def transaction(): - vuln: osv.Vulnerability = osv.Vulnerability.get_by_id(vuln_id) - if vuln is None: - logging.error('vulnerability not in Datastore - %s', vuln_id) - # TODO(michaelkedar): What to do in this case? - return - if upstream is None: - modified = datetime.datetime.now(datetime.UTC) - upstream_group = [] - else: - modified = upstream.last_modified - upstream_group = upstream.upstream_ids - vuln_proto.upstream[:] = upstream_group - vuln_proto.modified.FromDatetime(modified) - osv.ListedVulnerability.from_vulnerability(vuln_proto).put() - vuln.modified = modified - vuln.put() - - ndb.transaction(transaction) - try: - gcs.upload_vulnerability(vuln_proto, generation) - except exceptions.PreconditionFailed: - logging.error('Generation mismatch when writing upstream for %s', vuln_id) - osv.pubsub.publish_failure( - b'', type='gcs_gen_mismatch', id=vuln_id, field='upstream') - except Exception: - logging.error('Writing to bucket failed for %s', vuln_id) - osv.pubsub.publish_failure( - vuln_proto.SerializeToString(deterministic=True), type='gcs_retry') - - -def compute_upstream_hierarchy( - target_upstream_group: osv.UpstreamGroup, - all_upstream_groups: dict[str, osv.UpstreamGroup]) -> None: - """Computes all upstream vulnerabilities for the given bug ID. - The returned list contains all of the bug IDs that are upstream of the - target bug ID, including transitive upstreams in a map hierarchy. - upstream_group: - { db_id: bug id - upstream_ids: list of upstream bug ids - last_modified_date: date - upstream_hierarchy: JSON string of upstream hierarchy - } - """ - - # To convert to json, sets need to be converted to lists - # and sorting is done for a more consistent outcome. - def set_default(obj): - if isinstance(obj, set): - return list(sorted(obj)) - raise TypeError - - visited = set() - upstream_map = {} - to_visit = set([target_upstream_group.db_id]) - # BFS navigation through the upstream hierarchy of a given upstream group - while to_visit: - bug_id = to_visit.pop() - if bug_id in visited: - continue - visited.add(bug_id) - upstream_group = all_upstream_groups.get(bug_id) - if upstream_group is None: - continue - - upstreams = set(upstream_group.upstream_ids) - if not upstreams: - continue - for upstream in upstreams: - if upstream not in visited and upstream not in to_visit: - to_visit.add(upstream) - else: - if bug_id not in upstream_map: - upstream_map[bug_id] = set([upstream]) - else: - upstream_map[bug_id].add(upstream) - # Add the immediate upstreams of the bug to the dict - upstream_map[bug_id] = upstreams - to_visit.update(upstreams - visited) - - # Ensure there are no duplicate entries where transitive vulns appear - for k, v in upstream_map.items(): - if k is target_upstream_group.db_id: - continue - upstream_map[target_upstream_group - .db_id] = upstream_map[target_upstream_group.db_id] - v - - # Update the datastore entry if hierarchy has changed - if upstream_map: - upstream_json = json.dumps(upstream_map, default=set_default) - if upstream_json == target_upstream_group.upstream_hierarchy: - return - target_upstream_group.upstream_hierarchy = upstream_json - target_upstream_group.put() - - -def run(): - """Updates all upstream groups in the datastore by re-computing existing - UpstreamGroups and creating new UpstreamGroups for un-computed bugs.""" - - # Query for all bugs that have upstreams. - updated_bugs = [] - logging.info('Retrieving bugs...') - bugs_query = osv.Bug.query(osv.Bug.upstream_raw > '') - - bugs = defaultdict(set) - for bug in bugs_query.iter(projection=[osv.Bug.db_id, osv.Bug.upstream_raw]): - bugs[bug.db_id].add(bug.upstream_raw[0]) - logging.info('%s Bugs successfully retrieved', len(bugs)) - - logging.info('Retrieving upstream groups...') - upstream_groups = { - group.db_id: group for group in osv.UpstreamGroup.query().iter() - } - logging.info('Upstream Groups successfully retrieved') - - for bug_id, bug in bugs.items(): - # Get the specific upstream_group ID - upstream_group = upstream_groups.get(bug_id) - # Recompute the transitive upstreams and compare with the existing group - upstream_ids = compute_upstream(bug, bugs) - if upstream_group: - if upstream_ids == upstream_group.upstream_ids: - continue - # Update the existing UpstreamGroup - new_upstream_group = _update_group(upstream_group, upstream_ids) - if new_upstream_group is None: - continue - updated_bugs.append(new_upstream_group) - upstream_groups[bug_id] = new_upstream_group - logging.info('Upstream group updated for bug: %s', bug_id) - else: - # Create a new UpstreamGroup - new_upstream_group = _create_group(bug_id, upstream_ids) - logging.info('New upstream group created for bug: %s', bug_id) - updated_bugs.append(new_upstream_group) - upstream_groups[bug_id] = new_upstream_group - - for group in updated_bugs: - # Recompute the upstream hierarchies - compute_upstream_hierarchy(group, upstream_groups) - logging.info('Upstream hierarchy updated for bug: %s', group.db_id) diff --git a/gcp/website/frontend3/src/base.html b/gcp/website/frontend3/src/base.html deleted file mode 100644 index 229c09a64c1..00000000000 --- a/gcp/website/frontend3/src/base.html +++ /dev/null @@ -1,125 +0,0 @@ - - - - - - - - {% if disable_turbo_cache %} - - {% endif %} - - - - - - - - - - - OSV - Open Source Vulnerabilities - - - - {% block extra_head %}{% endblock %} - - - -
- {% block top_bar %} -
- - - - - - - - -
- {% endblock %} - {% block content %}{% endblock %} -
- - - diff --git a/gcp/website/frontend3/src/templates/404.html b/gcp/website/frontend3/src/templates/404.html deleted file mode 100644 index 0a4265942c2..00000000000 --- a/gcp/website/frontend3/src/templates/404.html +++ /dev/null @@ -1,54 +0,0 @@ -{% extends 'base.html' %} - -{% block extra_head %} - - - -{% endblock %} - -{% block top_bar %} {% endblock %} - -{% block content %} -{% if vuln_id %} -
-
-
-

{{ vuln_id }} Not Found!

-

- {% if has_importfindings %} - The record has not been successfully imported from its source
- {% endif %} - Please see our - FAQ for more information. -

-
- Back to Home -
-
-
- -
-{% else %} -
-
-
-

Oops! Page Not Found!

-

- While you're here, why not check out our - documentation or - FAQ? -

-
- Back to Home -
-
-
- -
-{% endif %} -{% endblock %} diff --git a/gcp/website/frontend3/src/templates/blog.html b/gcp/website/frontend3/src/templates/blog.html deleted file mode 100644 index 9c86dd302e7..00000000000 --- a/gcp/website/frontend3/src/templates/blog.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends 'base.html' %} -{% set active_section = 'blog' %} - -{% block content %} -
-
-
-
-

Blog

- {{ index|safe }} -
-
-
-
-{% endblock %} diff --git a/gcp/website/frontend3/src/templates/blog_post.html b/gcp/website/frontend3/src/templates/blog_post.html deleted file mode 100644 index cd9e69b8c0a..00000000000 --- a/gcp/website/frontend3/src/templates/blog_post.html +++ /dev/null @@ -1,15 +0,0 @@ -{% extends 'base.html' %} -{% set active_section = 'blog' %} -{% set disable_turbo_cache = 'true' %} - -{% block content %} -
-
-
-
- {{ content|safe }} -
-
-
-
-{% endblock %} diff --git a/gcp/website/frontend3/src/templates/home.html b/gcp/website/frontend3/src/templates/home.html deleted file mode 100644 index c6c984a0fbe..00000000000 --- a/gcp/website/frontend3/src/templates/home.html +++ /dev/null @@ -1,349 +0,0 @@ -{% extends 'base.html' %} -{% set active_section = 'home' %} - -{% block extra_head %} - - - - - - -{% endblock %} - -{% block content %} -
-
-
-

A distributed vulnerability database for Open Source

-
-

An open, precise, and distributed approach to producing and consuming vulnerability information for - open - source. -

-
-
- - search - Search Vulnerability Database - - Use the API -
-
- Vulnerability Scanner - Remediation Tools - GitHub Workflows -
-
-
-

Ecosystems

-
-
-
-
-
-
-
- {% if ecosystem_counts %} - {% set total = ecosystem_counts.values() | sum %} - {% for ecosystem, count in ecosystem_counts.items() %} - {% if count > 30 %} -
{{ ecosystem }}
-
- {% set radius = [(count | log) / (total | log) * 100, 30] | max %} - - - {{ count }} - - View {{ ecosystem }} vulnerabilities - - - -
- {% endif %} - {% endfor %} - {% endif %} -
-
-
- -
-

OSV schema

-

- All advisories in this database use the - OpenSSF OSV format, which - was developed in collaboration with open source communities. -

-

- The OSV schema provides a human and machine readable data format to - describe vulnerabilities in a way that precisely maps to open source - package versions or commit hashes. -

-
-
- 
{
-  "schema_version": "1.7.4",
-  "id": "GHSA-c3g4-w6cv-6v7h",
-  "modified": "2022-04-01T13:56:42Z",
-  "published": "2022-04-01T13:56:42Z",
-  "aliases": [ "CVE-2022-27651" ],
-  "summary": "Non-empty default inheritable capabilities for linux container in Buildah",
-  "details": "A bug was found in Buildah where containers were created ...",
-  "affected": [
-    {
-      "package": {
-        "ecosystem": "Go",
-        "name": "github.com/containers/buildah"
-      },
-      "ranges": [
-        {
-          "type": "SEMVER",
-          "events": [
-            {
-              "introduced": "0"
-            },
-            {
-              "fixed": "1.25.0"
-            }
-          ]
-        }
-      ]
-    }
-  ],
-  "references": [
-    {
-      "type": "WEB",
-      "url": "https://github.com/containers/buildah/commit/..."
-    },
-    {
-      "type": "PACKAGE",
-      "url": "https://github.com/containers/buildah"
-    }
-  ]
-}
-
-
-
- OSV Schema - Blog - post -
-
-
-

Data sources

-

- This infrastructure serves as an aggregator of vulnerability databases - that have adopted the OSV schema, including - GitHub Security Advisories, - PyPA, - RustSec, and - Global Security Database, and - more. -

-
- Learn more -
-
-
-

Use the API

-

- An easy-to-use API is available to query for all known vulnerabilities - by either a commit hash, or a package version. -

-
-
-

Query by commit hash

- 
curl -d \
-  '{"commit": "6879efc2c1596d11a6a6ad296f80063b558d5e0f"}' \
-  "https://api.osv.dev/v1/query"
- - - content_copy - - -
-
-

Query by version number

- 
curl -d \
-  '{"version": "2.4.1",
-    "package": {"name": "jinja2", "ecosystem": "PyPI"}}' \
-  "https://api.osv.dev/v1/query"
- - - content_copy - - -
-
-
-
- API - Documentation -
-
-
-
- -

Vulnerability Scanner

-
-
-
-

Install OSV‑Scanner

- 
-go install github.com/google/osv-scanner/v2/cmd/osv-scanner@v2
-
- - - content_copy - - -
-
-
-

Scan SBOM or Lockfiles

- 
-osv-scanner --sbom=cycloned-or-spdx-sbom.json
-osv-scanner --lockfile=package-lock.json
-
- - - content_copy - - -
-
-

Scan directory recursively

- 
-osv-scanner -r path/to/your/project
-
- - - content_copy - - -
-
-
-
- More details -
-
-
-
- -

Remediation Tools

-
-
-

Guided Remediation (basic)

- 
-osv-scanner fix --non-interactive --strategy=in-place -L path/to/package-lock.json
-osv-scanner fix --non-interactive --strategy=relock -M path/to/package.json -L path/to/package-lock.json
-
- - - content_copy - - -
-
-

Guided Remediation (interactive)

- 
-osv-scanner fix -M path/to/package.json -L path/to/package-lock.json
-
- - - content_copy - - -
- - -
-
-
- More details -
-
-
-
-

Container Image Scanning

-

- You can use - OSV-Scanner - to scan your container images for known vulnerabilities. -

- - -
-
-
-

Scan container image

- 
-osv-scanner scan image --serve alpine:3.12
-
- - - content_copy - - -
-
-
- - -
- Screenshot of container scan HTML output -
- - -
-

GitHub Workflows

-

OSV-Scanner also provides reusable GitHub workflows that can be easily - integrated into CI/CD pipelines to provide continuous vulnerability scanning coverage. This can scan - newly added dependencies in pull requests for introduced vulnerabilities, as well as perform regular - vulnerability scans for the entire project.

-
Screenshot of OSV-Scanner GitHub Action
-
Learn more
-
- -
-

Open source

-

This project is open source. If you - have any ideas or questions, please feel free to reach out by creating an issue!

- -
- Learn more -
-
-
-
- -
{% endblock %} \ No newline at end of file diff --git a/gcp/website/frontend3/src/templates/linter/index.html b/gcp/website/frontend3/src/templates/linter/index.html deleted file mode 100644 index da68812b2a1..00000000000 --- a/gcp/website/frontend3/src/templates/linter/index.html +++ /dev/null @@ -1,98 +0,0 @@ -{% set active_section = 'linter' %} {% block content %} - - - - - - - Vulnerability Linter Report - - - - - - - -
-
-
- -
-

Open Source Vulnerabilities

-
- sync -
-
- -
-
-
-
- Linter Report -
-
- -
-
- -
-
-
-
- Home Database -
-

-
- keyboard_arrow_down -
-
-
-
-
- Finding -
-

-
- keyboard_arrow_down -
-
-
-
-
- -
-
- search -
- -
- -
- - - - - - - - - - - -
Bug IDFindings - Modified unfold_more -
- -
-
-
-
-
- - - - -{% endblock %} \ No newline at end of file diff --git a/gcp/website/frontend3/src/templates/list.html b/gcp/website/frontend3/src/templates/list.html deleted file mode 100644 index 281bc3b7968..00000000000 --- a/gcp/website/frontend3/src/templates/list.html +++ /dev/null @@ -1,166 +0,0 @@ -{% extends 'base.html' %} -{% set active_section = 'vulnerabilities' %} -{% set disable_turbo_cache = 'true' %} - -{% macro table_header_cell(column_id, column_name, is_sortable, is_sorted, is_descending) %} - -
-
- {{ column_name }} -
- {% if is_sorted %} - - arrow_upward - - - {% endif %} -
- -{% endmacro %} - -{% block content %} -
-
-
-
-

Vulnerabilities

-
-
-
-
- - search - -
-
- - {% if ecosystem_counts %} -
- - - {% for ecosystem in ecosystem_counts %} - - - {% endfor %} -
- {% endif %} - - -
-
-
-
-
- -
-
-
- {{ table_header_cell('id', 'ID', is_sortable=False, is_sorted=False, is_descending=False) }} - {{ table_header_cell('package', 'Packages', is_sortable=False, is_sorted=False, is_descending=False) }} - {{ table_header_cell('summary', 'Summary', is_sortable=False, is_sorted=False, is_descending=False) }} - {{ table_header_cell('published', 'Published', is_sortable=True, is_sorted=True, is_descending=True) }} - {{ table_header_cell('attributes', 'Attributes', is_sortable=False, is_sorted=False, is_descending=False) }} -
-
-
- - {% for vulnerability in vulnerabilities %} -
- - {{ vulnerability.id - }} - - -
    - {% for package in vulnerability.packages[:5] %} -
  • {{ package }}
    • - {% endfor %} - {% if vulnerability.packages|length > 5 %} - {% set remainingPkgCount = vulnerability.packages|length - 5 %} -
  • ... {{ remainingPkgCount }} more
    • - {% elif vulnerability.packages|length == 0 %} -
  • Not specified
    • - {% endif %} -
- - - {% if vulnerability.summary %} - {{ vulnerability.summary }} - {% else %} - See record for full details - {% endif %} - - - - {{ vulnerability.published | relative_time }} - - - -
    -
  • - {%- if vulnerability.is_fixed -%} - Fix available - {%- else -%} - No fix available - {%- endif -%} -
    • - {%- if vulnerability.severity_score and vulnerability.severity_rating -%} -
  • - Severity - {{ - vulnerability.severity_score }} ({{ vulnerability.severity_rating }}) -
    • - {%- endif -%} -
- -
- {%- endfor -%} - {%- if vulnerabilities | length == 0 -%} - No results (check our FAQ if this is unexpected) - {%- endif -%} - {%- if page < total_pages -%} -
- - Load more... - {%- if total_pages - page <= 3 -%} ({{ total_pages - page }} page{% if - total_pages - page > 1 %}s{% endif %} left) - {%- endif -%} - - -
- - {%- endif -%} - -
-
- - - - -
-{% endblock %} diff --git a/gcp/website/frontend3/src/templates/triage.html b/gcp/website/frontend3/src/templates/triage.html deleted file mode 100644 index f031645bc47..00000000000 --- a/gcp/website/frontend3/src/templates/triage.html +++ /dev/null @@ -1,68 +0,0 @@ -{% extends 'base.html' %} -{% set active_section = 'triage' %} - -{% block content %} -
-
-

CVE Conversion Triager

- -
- - Load -
-
-
- {% for i in range(1, 4) %} -
-
- -
- -
-
-
- -
- 
Select a source to view content
-
- - - - -
-
- {% endfor %} -
-
-{% endblock %} diff --git a/gcp/website/frontend3/src/templates/vulnerability.html b/gcp/website/frontend3/src/templates/vulnerability.html deleted file mode 100644 index 6ffb9794f71..00000000000 --- a/gcp/website/frontend3/src/templates/vulnerability.html +++ /dev/null @@ -1,638 +0,0 @@ -{% extends 'base.html' -%} -{% set active_section = 'vulnerabilities' -%} - -{% block extra_head %} - -{% endblock %} - -{% block content -%} -
-
-
-
-

- {{ vulnerability.id }} -

- {% if vulnerability.human_source_link and vulnerability.human_source_link.startswith("https://github.com/advisories/") -%} - - Suggest an improvement - - {% elif vulnerability.human_source_link and not vulnerability.id.startswith("openSUSE-") -%} -
- - See a problem? - -
- Please try reporting it to the source first. -
- {% else -%} - - See a problem? - - {% endif -%} -
-
-
- {%- if vulnerability.human_source_link and not vulnerability.id.startswith("openSUSE-") -%} -
Source
-
{{ - vulnerability.human_source_link }} -
- {%- endif -%} -
Import Source
-
{{ - vulnerability.source }}
- -
JSON Data
-
- https://{{ api_url }}/v1/vulns/{{ vulnerability.id }} -
- {% if vulnerability.aliases -%} -
Aliases
-
-
    - {% for alias in vulnerability.aliases -%} -
  • - {% if alias in vulnerability.known_ids -%} - {{ alias }} - {% else -%} - {{ alias }} - {% endif -%} -
    • - {% endfor -%} -
-
- {% endif -%} - {%- if vulnerability.upstream_hierarchy -%} -
Upstream
-
{{ vulnerability.upstream_hierarchy | safe }}
- {%- endif -%} - {%- if vulnerability.downstream_hierarchy -%} -
Downstream
-
{{ vulnerability.downstream_hierarchy | safe }}
- {%- endif -%} - {% if vulnerability.related -%} -
Related
-
-
    - {% for related in vulnerability.related -%} -
  • - {% if related in vulnerability.known_ids -%} - {{ related }} - {% else -%} - {{ related }} - {% endif -%} -
    • - {% endfor -%} -
-
- {% endif -%} - {%- if vulnerability.withdrawn -%} -
- Withdrawn - -
-
{{ vulnerability.withdrawn }}
- {% endif -%} -
Published
-
{{ vulnerability.published }}
-
Modified
-
{{ vulnerability.modified }}
- {%- if vulnerability.severity -%} -
Severity
-
-
    - {% for item in vulnerability.severity -%} -
  • - {% if item | is_cvss %} - {{ item | display_severity_rating }} - {{ item.type }} - {{ item.score }} - {% if item.source %}(Source: {{ item.source }}){% endif %} - - CVSS Calculator - {% else %} - {{ item.type }} - {{ item.score }} - {% if item.source %}(Source: {{ item.source }}){% endif %} - {% endif %} -
    • - {% endfor -%} -
-
- {%- endif -%} -
Summary
-
- {% if vulnerability.summary %} - {{ vulnerability.summary }} - {% else %} - [none] - {% endif %} -
-
Details
-
- {{ vulnerability.details | markdown | safe -}} -
- {% if vulnerability.database_specific -%} -
- Database specific - -
- 
{{ vulnerability.database_specific | display_json }}
- {% endif %} -
References
-
- -
- {% if vulnerability.credits -%} -
Credits
-
-
    - {% for credit in vulnerability.credits -%} -
  • -
      -
    • {{ credit.name }}{% if 'type' in credit %} - {{ credit.type }}{% endif %}
      • - {%- if 'contact' in credit -%} -
    • -
        - {%- for item in credit.contact -%} -
      • {{ item }}
        • - {%- endfor -%} -
      -
      • - {%- endif -%} -
    -
    • - {% endfor -%} -
-
- {% endif %} -
-
-
-
-
-

Affected packages

- - {% if vulnerability.affected|should_collapse %} - {% set ecosystems = vulnerability.affected | group_by_ecosystem %} -
- {% for ecosystem_name, packages in ecosystems.items() -%} - {% set is_last_ecosystem = loop.last %} -
- - {{ ecosystem_name }} - -
- {% for affected in packages -%} -
- - {% if 'package' in affected %} - {{ affected.package.name }} - {% else %} - {% set affected_repo = affected.ranges | default([], true) | selectattr('repo') | map(attribute='repo') | first %} - {% if affected_repo %} - {{ affected_repo | strip_scheme }} - {% endif %} - {% endif %} - -
-
- {%- if 'package' in affected -%} -
-

Package

-
-
-
Name
-
{{ affected.package.name }}
- {%- if ecosystem_name | has_link_to_deps_dev -%} -
View open source insights on deps.dev
- {%- endif -%} - {%- if 'purl' in affected.package -%} -
Purl
-
{{ affected.package.purl }}
- {%- endif -%} -
-
-
- {%- endif -%} - {%- if 'severity' in affected -%} -
-

Severity

-
-
    - {% for item in affected.severity -%} -
  • - {% if item | is_cvss %} - {{ item | display_severity_rating }} - {{ item.type }} - {{ item.score }} - {% if item.source %}(Source: {{ item.source }}){% endif %} - - CVSS Calculator - {% else %} - {{ item.type }} - {{ item.score }} - {% if item.source %}(Source: {{ item.source }}){% endif %} - {% endif %} -
    • - {% endfor -%} -
-
-
- {%- endif -%} -
-

Affected ranges

-
- {% for range in affected.ranges -%} -
-
Type
-
{{ range.type -}}
- {%- if range.repo -%} -
Repo
-
{{ range.repo }}
- {%- endif -%} -
Events
-
-
- {% for event in range.events -%} -
{{ event | event_type -}}
-
- {% set link = event | event_link -%} - {% if link -%} - {{ event | event_value -}} - {% elif event | event_type == 'Introduced' and event | event_value == '0' -%} -
{{ event | event_value -}} - {% if range.type == 'GIT' %} - Unknown introduced commit / All previous commits are affected - {% else -%} - Unknown introduced version / All previous versions are affected - {% endif -%} -
- {% else -%} - {{ event | event_value -}} - {% endif -%} -
- {% endfor -%} -
-
- {%- if range.database_specific -%} -
Database specific
-
-
- Show details -
- 
{{ range.database_specific | display_json }}
-
-
-
- {%- endif -%} -
- {% endfor -%} -
-
- {% if affected.versions -%} -
-

Affected versions

-
- {% for group, versions in (affected.versions|group_versions(ecosystem_name)).items() -%} -
- {{ group }} -
- {% for version in versions -%} -
{{ version }}
- {% endfor -%} -
-
- {% endfor -%} -
-
- {% endif -%} - {% if affected.ecosystem_specific -%} -
-

Ecosystem specific

- 
{{ affected.ecosystem_specific | display_json }}
-
- {% endif -%} - {% if affected.database_specific -%} -
-

Database specific

-
- {% set db_specific = affected.database_specific %} - {% if db_specific is mapping %} -
- {% for key, value in db_specific.items() %} -
- {{ key }} -
- 
{{ value | display_json }}
-
-
- {% endfor %} -
- {% else %} - 
{{ db_specific | display_json }}
- {% endif %} -
-
- {% endif -%} -
-
-
- {% endfor -%} -
-
- {% endfor -%} -
- - {% else %} - - {% for affected in vulnerability.affected -%} - {% if 'package' in affected %} - {% set ecosystem = affected.package.ecosystem %} - {% set package = affected.package.name %} - {% else %} - {% set ecosystem = 'Git' %} - {% set affected_repo = affected.ranges | default([], true) | selectattr('repo') | map(attribute='repo') | first %} - {% if affected_repo %} - {% set package = affected_repo | strip_scheme %} - {% endif %} - {% endif %} -

- {{ ecosystem }} - / - {{ package }} -

-
- {%- if 'package' in affected -%} -
-

- Package -

-
-
-
Name
- {%- if affected.package | package_in_ecosystem -%} -
{{ - affected.package.name }}
- {%- else -%} -
{{ affected.package.name }}
- {%- endif -%} - {%- if ecosystem | has_link_to_deps_dev -%} -
- View open source insights on deps.dev
- {%- endif -%} - {%- if 'purl' in affected.package -%} -
Purl
-
{{ affected.package.purl }}
- {%- endif -%} -
-
-
- {%- endif -%} - {%- if 'severity' in affected -%} -
-

- Severity -

-
-
    - {% for item in affected.severity -%} -
  • - {% if item | is_cvss %} - {{ item | display_severity_rating }} - {{ item.type }} - {{ item.score }} - {% if item.source %}(Source: {{ item.source }}){% endif %} - - CVSS Calculator - {% else %} - {{ item.type }} - {{ item.score }} - {% if item.source %}(Source: {{ item.source }}){% endif %} - {% endif %} -
    • - {% endfor -%} -
-
-
- {%- endif -%} -
-

- Affected ranges - -

-
- {% for range in affected.ranges -%} -
-
Type
-
{{ range.type -}}
- - {%- if range.repo -%} -
Repo
-
{{ range.repo }}
- {%- endif -%} - -
Events
-
-
- {% for event in range.events -%} -
- {{ event | event_type -}} -
-
- {% set link = event | event_link -%} - {% if link -%} - - {{ event | event_value -}} - - {% elif event | event_type == 'Introduced' and event | event_value == '0' -%} -
- {{ event | event_value -}} - {% if range.type == 'GIT' %} - Unknown introduced commit / All previous commits are affected - {% else -%} - Unknown introduced version / All previous versions are affected - {% endif -%} -
- {% else -%} - {{ event | event_value -}} - {% endif -%} -
- {% endfor -%} -
-
- - {%- if range.database_specific -%} -
- Database specific - -
-
-
- Show details -
- 
{{ range.database_specific | display_json }}
-
-
-
- {%- endif -%} -
- {% endfor -%} -
-
- {% if affected.versions -%} -
-

- Affected versions - -

-
- {% for group, versions in (affected.versions|group_versions(ecosystem)).items() -%} -
- {{ group }} -
- {% for version in versions -%} -
{{ version }}
- {% endfor -%} -
-
- {% endfor -%} -
-
- {% endif -%} - {% if affected.ecosystem_specific -%} -
-

- Ecosystem specific - -

-
- 
{{ affected.ecosystem_specific | display_json }}
-
-
- {% endif -%} - {% if affected.database_specific -%} -
-

- Database specific - -

-
- {% set db_specific = affected.database_specific %} - {% if db_specific is mapping %} -
- {% for key, value in db_specific.items() %} -
- {{ key }} -
- 
{{ value | display_json }}
-
-
- {% endfor %} -
- {% else %} - 
{{ db_specific | display_json }}
- {% endif %} -
-
- {% endif -%} -
- {% endfor -%} - - {% endif %} -
-
- - - - - -{% endblock -%} diff --git a/gcp/website/frontend_emulator.py b/gcp/website/frontend_emulator.py deleted file mode 100644 index 9431afb125f..00000000000 --- a/gcp/website/frontend_emulator.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright 2025 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Frontend emulator to mock local data instead of from GCP""" -import os -import json -import yaml -from osv import tests -from osv import sources -from osv import vulnerability_pb2 -from google.cloud import ndb -import osv -import datetime - -import emulator_aliases -import emulator_upstream - -if __name__ == '__main__': - # The datastore emulator needs to be started before main is imported - # to make the global ndb client use the emulator. - with tests.datastore_emulator() as em: - import main - with ndb.Client().context() as context: - context.set_memcache_policy(False) - context.set_cache_policy(False) - - # Load OSV files from the repo testdata directory. - def _load_osv_vuln(path: str) -> vulnerability_pb2.Vulnerability | None: - """Parse a single vulnerability from an OSV file.""" - data = _read_osv_file(path) - vulnerability = _dict_to_vuln(data, path) - if vulnerability: - return vulnerability - - print(f'[emulator] No valid OSV record found in {path}') - return None - - def _vulnerability_to_bug( - vulnerability: vulnerability_pb2.Vulnerability) -> osv.Bug: - """Convert a parsed vulnerability into a datastore Bug entity.""" - bug_entity = osv.Bug( - id=vulnerability.id, - db_id=vulnerability.id, - public=True, - source='test', - status=osv.BugStatus.PROCESSED) - bug_entity.update_from_vulnerability(vulnerability) - if vulnerability.HasField('modified'): - bug_entity.import_last_modified = vulnerability.modified.ToDatetime( - datetime.UTC) - if not bug_entity.timestamp: - bug_entity.timestamp = vulnerability.modified.ToDatetime( - datetime.UTC) - if not bug_entity.timestamp: - bug_entity.timestamp = datetime.datetime.now(datetime.UTC) - return bug_entity - - def _read_osv_file(path: str) -> object | None: - """Read and parse raw data from an OSV file.""" - try: - with open(path) as fh: - ext = os.path.splitext(path)[1].lower() - if ext in sources.YAML_EXTENSIONS: - return yaml.load(fh, Loader=sources.NoDatesSafeLoader) - if ext in sources.JSON_EXTENSIONS: - return json.load(fh) - except Exception as error: - print(f'[emulator] Failed to read {path}: {error}') - return None - - print(f'[emulator] Unsupported file extension for {path}') - return None - - def _dict_to_vuln(data: object, - path: str) -> vulnerability_pb2.Vulnerability | None: - """Convert raw dict data into a Vulnerability proto.""" - if not isinstance(data, dict): - return None - - vuln_id = data.get('id') - if not vuln_id: - return None - - try: - vulnerability = sources.parse_vulnerability_from_dict( - data, strict=False) - except Exception as error: - print(f'[emulator] Failed to convert entry in {path}: {error}') - return None - - return vulnerability if vulnerability.id else None - - def _load_dir(emulator, dir_path: str): - """Load all OSV files from a directory into the emulator.""" - emulator.reset() - for root, _, files in os.walk(dir_path): - for fname in files: - ext = os.path.splitext(fname)[1].lower() - if ext not in (*sources.YAML_EXTENSIONS, *sources.JSON_EXTENSIONS): - continue - fpath = os.path.join(root, fname) - vulnerability = _load_osv_vuln(fpath) - if not vulnerability: - continue - - bug = _vulnerability_to_bug(vulnerability) - bug.put() - - # Compute upstream/alias groups based on loaded bugs. - emulator_aliases.run() - emulator_upstream.run() - - for b in osv.Bug.query(): - b.put() - - testdata_dir = os.path.join(os.path.dirname(__file__), 'testdata', 'osv') - if os.path.isdir(testdata_dir): - _load_dir(em, testdata_dir) - else: - print('No testdata found; starting emulator with empty dataset.') - main.app.run(host='127.0.0.1', port=8000, debug=False) diff --git a/gcp/website/frontend_handlers.py b/gcp/website/frontend_handlers.py deleted file mode 100644 index 7df522cefb7..00000000000 --- a/gcp/website/frontend_handlers.py +++ /dev/null @@ -1,1316 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Handlers for the OSV web frontend.""" - -import json -import logging -import math -import os -import re -from typing import Any, NamedTuple - -from flask import abort -from flask import current_app -from flask import Blueprint -from flask import make_response -from flask import redirect -from flask import render_template, render_template_string -from flask import request -from flask import url_for -from flask import send_from_directory -from werkzeug.security import safe_join -from werkzeug import exceptions -from collections import OrderedDict -from collections import defaultdict -from google.cloud import exceptions as gexceptions -from google.cloud import ndb -from cvss import CVSS2, CVSS3, CVSS4 - -import markdown2 -from markupsafe import Markup, escape -from urllib import parse - -import cache -import osv -import rate_limiter -import source_mapper -import utils - -blueprint = Blueprint('frontend_handlers', __name__) - -_PAGE_SIZE = 16 -_PAGE_LOOKAHEAD = 4 -_REQUESTS_PER_MIN = 30 -_WORD_CHARACTERS_OR_DASH = re.compile(r'^[+\w-]+$') -_WORD_CHARACTERS_OR_DASH_OR_COLON = re.compile(r'^[+\w:-]+$') -_VALID_BLOG_NAME = _WORD_CHARACTERS_OR_DASH -_VALID_VULN_ID = _WORD_CHARACTERS_OR_DASH_OR_COLON -_BLOG_CONTENTS_DIR = 'blog' -_LINTER_CONTENTS_DIR = 'linter' -_DEPS_BASE_URL = 'https://deps.dev' -_FIRST_CVSS_CALCULATOR_BASE_URL = 'https://www.first.org/cvss/calculator' -_GO_VANITY_METADATA = \ - ('') -MAX_SUGGESTIONS = 10 - -_ndb_client = ndb.Client() - - -class VulnerabilityNotFound(exceptions.NotFound): - """Plumb the vulnerability ID not found to the error handler.""" - - def __init__(self, vuln_id: str) -> None: - super().__init__() - self.vuln_id = vuln_id - - -class VulnerabilityNotImported(VulnerabilityNotFound): - """Plumb the vulnerability ID with import findings to the error handler.""" - - -if utils.is_cloud_run(): - redis_host = os.environ.get('REDISHOST', 'localhost') - redis_port = int(os.environ.get('REDISPORT', 6379)) - limiter = rate_limiter.RateLimiter( - redis_host, redis_port, requests_per_min=_REQUESTS_PER_MIN) - - @blueprint.before_request - def check_rate_limit(): - ip_addr = request.headers.get('X-Forwarded-For', 'unknown').split(',')[0] - if not limiter.check_request(ip_addr): - abort(429) - - -def _load_blog_content(name): - """Load blog content.""" - path = os.path.join(current_app.static_folder, _BLOG_CONTENTS_DIR, name) - if not os.path.exists(path): - abort(404) - return None - - with open(path) as handle: - return handle.read() - - -@blueprint.before_request -def check_cors_preflight(): - """Handle CORS preflight requests.""" - if request.method != 'OPTIONS': - return None - - response = make_response() - response.headers.add('Access-Control-Allow-Origin', 'http://localhost:8080') - response.headers.add('Access-Control-Allow-Methods', '*') - response.headers.add('Access-Control-Allow-Headers', '*') - return response - - -@blueprint.after_request -def add_cors_headers(response): - """Add CORS headers.""" - response.headers.add('Access-Control-Allow-Origin', 'http://localhost:8080') - return response - - -@blueprint.route('/v2/') -def index_v2(): - return redirect('/') - - -@blueprint.route('/v2/') -def index_v2_with_subpath(subpath): - return redirect('/' + subpath) - - -@blueprint.route('/bindings/go') -def go_bindings_vanity(): - if request.args.get('go-get', 0) == '1': - return _GO_VANITY_METADATA - - return redirect('https://pkg.go.dev/osv.dev/bindings/go') - - -@blueprint.route('/') -def index(): - # Go will request the root url to make sure we own this domain - if request.args.get('go-get', 0) == '1': - return _GO_VANITY_METADATA - - return render_template( - 'home.html', ecosystem_counts=osv_get_ecosystem_counts_cached()) - - -@blueprint.route('/robots.txt') -def robots(): - response = make_response(f'Sitemap: {request.host_url}sitemap_index.xml\n') - response.mimetype = 'text/plain' - return response - - -@blueprint.route('/favicon.ico') -def favicon(): - return current_app.send_static_file('img/favicon-32x32.png') - - -@blueprint.route('/blog/', strict_slashes=False) -def blog(): - return render_template('blog.html', index=_load_blog_content('index.html')) - - -@blueprint.route('/blog/index.xml') -def blog_rss(): - return current_app.send_static_file( - os.path.join(_BLOG_CONTENTS_DIR, 'index.xml')) - - -@blueprint.route('/blog/posts//', strict_slashes=False) -def blog_post(blog_name): - if not _VALID_BLOG_NAME.match(blog_name): - abort(404) - - path = safe_join('posts', blog_name, 'index.html') - if not path: - abort(404) - - return render_template( - 'blog_post.html', - content=_load_blog_content(safe_join('posts', blog_name, 'index.html'))) - - -@blueprint.route('/blog/posts//', strict_slashes=False) -def blog_post_static_files(blog_name: str, file_name: str): - """Return static files under blog post directories""" - if not _VALID_BLOG_NAME.match(blog_name): - abort(404) - - path = safe_join(current_app.static_folder, _BLOG_CONTENTS_DIR, 'posts', - blog_name) - if not path: - abort(404) - - return send_from_directory(path, file_name) - - -@blueprint.route('/about') -def about(): - return redirect('https://google.github.io/osv.dev/faq') - - -@blueprint.route('/faq') -def faq(): - return redirect('https://google.github.io/osv.dev/faq') - - -@blueprint.route('/docs', strict_slashes=False) -def docs(): - return redirect('https://google.github.io/osv.dev') - - -@blueprint.route('/linter', strict_slashes=False) -def linter(): - if request.host_url == 'osv.dev': - return redirect(request.url.replace('osv.dev', 'test.osv.dev'), code=302) - return render_template('linter.html') - - -@blueprint.route('/ecosystems') -def ecosystems(): - return redirect('https://storage.googleapis.com/osv-vulnerabilities/ecosystems.txt') # pylint: disable=line-too-long - - -_LIST_ARGS = ['q', 'ecosystem', 'page'] - - -@blueprint.route('/list') -def list_vulnerabilities(): - """Main page.""" - - # Remove unknown query parameters - args = {k: v for k, v in request.args.lists() if k in _LIST_ARGS} - - # Remove page parameter if not from turbo frame - is_turbo_frame = request.headers.get('Turbo-Frame') - if not is_turbo_frame and args.get('page', 1) != 1: - args.pop('page', None) - - # redirect if any query parameters were filtered - if args.keys() != request.args.keys(): - return redirect(url_for(request.endpoint, **args)) - - query = request.args.get('q', '') - # Remove leading and trailing spaces - query = query.strip() - page = int(request.args.get('page', 1)) - ecosystem = request.args.get('ecosystem') - - if page < 0: - args.pop('page', None) - return redirect(url_for(request.endpoint, **args)) - - results = osv_query(query, page, ecosystem) - - # Fetch ecosystems by default. As an optimization, skip when rendering page - # fragments. - ecosystem_counts = osv_get_ecosystem_counts_cached( - ) if not is_turbo_frame else None - - return render_template( - 'list.html', - page=page, - total_pages=math.ceil(results['total'] / _PAGE_SIZE), - query=query, - selected_ecosystem=ecosystem, - ecosystem_counts=ecosystem_counts, - vulnerabilities=results['items']) - - -@blueprint.route('/vulnerability/') -def vulnerability(vuln_id: str): - """Vulnerability page.""" - try: - vuln = osv_get_by_id(vuln_id) - except VulnerabilityNotImported: - # TODO: handle showing import findings - abort(404) - except VulnerabilityNotFound as e: - return redirect( - url_for('frontend_handlers.list_vulnerabilities', q=e.vuln_id)) - - api_url = utils.api_url() - - return render_template( - 'vulnerability.html', vulnerability=vuln, api_url=api_url) - - -@blueprint.route('/') -def vulnerability_redirector(potential_vuln_id: str): - """Convenience redirector for /VULN-ID to /vulnerability/VULN-ID.""" - # AlmaLinux have colons in their identifiers, which gets URL encoded. - potential_vuln_id = parse.unquote(potential_vuln_id) - if not _VALID_VULN_ID.match(potential_vuln_id): - abort(400) - return None - - # This may raise an exception directly or via abort() for failed retrievals. - try: - bug = osv_get_by_id(potential_vuln_id) - except VulnerabilityNotFound as e: - return redirect( - url_for('frontend_handlers.list_vulnerabilities', q=e.vuln_id)) - - path = safe_join('/vulnerability', bug.get('id', '')) - return redirect(path or '/vulnerability/') - - -@blueprint.route('/.json') -@blueprint.route('/vulnerability/.json') -def vulnerability_json_redirector(potential_vuln_id): - """Convenience redirector for /VULN-ID.json and /vulnerability/VULN-ID.json to - https://api.osv.dev/v1/vulns/VULN-ID. - """ - if not _VALID_VULN_ID.match(potential_vuln_id): - abort(400) - return None - - # This calls abort() on failed retrievals. - try: - bug = osv_get_by_id(potential_vuln_id) - except VulnerabilityNotFound: - return abort(404) - - api_url = utils.api_url() - return redirect(f'https://{api_url}/v1/vulns/{bug["id"]}') - - -def vuln_to_response(vuln): - """Convert a Vulnerability proto to a response object.""" - response = osv.vulnerability_to_dict(vuln) - add_cvss_score(response) - add_links(response) - add_source_info(response) - add_stream_info(response) - add_known_osv_bugs(response) - add_stream_strings(response) - return response - - -def add_known_osv_bugs(response: dict[str, Any]): - """Compute which vulns in aliases/related/upstream/downstream exist in OSV, - in order to add links to them. - - Note: This must be called after add_stream_info(). - """ - - # collect all the unique IDs referenced by aliases, related, up- & downstream. - ids = set(response.get('aliases', [])) | set(response.get('related', [])) - if computed := response.get('computed_upstream'): - tree = computed[1] - for k, v in tree.items(): - ids.add(k) - ids |= v - if computed := response.get('computed_downstream'): - tree = computed[1] - for k, v in tree.items(): - ids.add(k) - ids |= v - - # Somewhat asynchronously query all of the found IDs to check their existence. - # This is written to mimic how ndb.get_multi() is implemented. - exist_futures = [(vid, osv.Vulnerability.get_by_id_async(vid)) for vid in ids] - exist_ids = set(vid for vid, f in exist_futures if f.result()) - response['known_ids'] = exist_ids - - -def add_stream_info(response: dict[str, Any]): - """Add upstream hierarchy information to `response`.""" - vuln_id = response.get('id') - if not vuln_id: - return - # Check whether there are upstreams - upstreams = get_upstreams_of_vulnerability(vuln_id) - if upstreams: - response['computed_upstream'] = upstreams - # Check whether there are downstreams - downstreams = _get_downstreams_of_bug_query(vuln_id) - if downstreams: - response['computed_downstream'] = compute_downstream_hierarchy( - vuln_id, downstreams) - - -def add_stream_strings(response: dict[str, Any]): - """Add upstream hierarchy html strings to `response`.""" - vuln_id = response.get('id', '') - known_ids = response.get('known_ids', set()) - if computed := response.get('computed_upstream'): - response['upstream_hierarchy'] = construct_hierarchy_string( - vuln_id, computed, known_ids) - if computed := response.get('computed_downstream'): - response['downstream_hierarchy'] = construct_hierarchy_string( - vuln_id, computed, known_ids) - - -cvss_calculator = { - 'CVSS_V2': CVSS2, - 'CVSS_V3': CVSS3, - 'CVSS_V4': CVSS4, -} - - -def calculate_severity_details( - severity: dict) -> tuple[float | None, str | None]: - """Calculate score and rating of severity""" - type_ = severity.get('type') - score = severity.get('score') - - if not (type_ and score): - return None, None - - try: - c = cvss_calculator[type_](score) - severity_rating = c.severities()[0] - except Exception as e: - logging.error('Exception raised when parsing "%s" severity "%s": %s', type_, - score, e) - return None, None - - severity_score = c.base_score - return severity_score, severity_rating - - -def add_cvss_score(record: dict): - """Add severity score where possible.""" - severity_score = None - severity_rating = None - severity_type = None - - # The OSV record (vulnerability page) has the list of severities under the - # 'severity' field, while the ListedVulnerability (/list page) has it under - # the 'severities' field. - # This function is used for both, so check both. - sev = '' - if 'severity' in record: - sev = 'severity' - elif 'severities' in record: - sev = 'severities' - for severity in record.get(sev, []): - if not is_cvss(severity): - continue - type_ = severity.get('type') - if type_ and (not severity_type or type_ > severity_type): - severity_type = type_ - severity_score, severity_rating = calculate_severity_details(severity) - - record['severity_score'] = severity_score - record['severity_rating'] = severity_rating - - -def add_links(record: dict): - """Add VCS links where possible.""" - - first_repo_url = None - - for entry in record.get('affected', []): - for i, affected_range in enumerate(entry.get('ranges', [])): - affected_range['id'] = i - if affected_range.get('type') != 'GIT': - continue - - repo_url = affected_range.get('repo') - if not repo_url: - continue - - if not first_repo_url: - first_repo_url = repo_url - - for event in affected_range.get('events', []): - if event.get('introduced') and event['introduced'] != '0': - event['introduced_link'] = _commit_to_link(repo_url, - event['introduced']) - continue - - if event.get('last_affected'): - event['last_affected_link'] = _commit_to_link(repo_url, - event['last_affected']) - continue - - if event.get('fixed'): - event['fixed_link'] = _commit_to_link(repo_url, event['fixed']) - continue - - if event.get('limit'): - event['limit_link'] = _commit_to_link(repo_url, event['limit']) - continue - - if first_repo_url: - record['repo'] = first_repo_url - - -def add_source_info(response: dict): - """Add upstream provenance information to `response`.""" - # if bug.source_of_truth == osv.SourceOfTruth.INTERNAL: - # response['source'] = 'INTERNAL' - # return - vuln_id = response.get('id') - if not vuln_id: - logging.error('vulnerability does not have an id') - return - vuln = osv.Vulnerability.get_by_id(vuln_id) - if vuln is None: - logging.error('vulnerability %s in GCS but no Vulnerability entity exists', - vuln_id) - return - if vuln.source_id is None: - logging.error('Unexpected state for "%s": source_id is None', vuln_id) - return - source, _, source_path = vuln.source_id.partition(':') - source_repo = osv.get_source_repository(source) - if not source_repo or not source_repo.link: - logging.error( - 'Unexpected state for "%s": source repository/link not found for "%s"', - vuln_id, source) - return - - if source == 'oss-fuzz': - source_path = oss_fuzz_source_path(source_repo, response) - response['source'] = source_repo.link + source_path - response['source_link'] = response['source'] - if source_repo.human_link: - ecosystem_version = '' - if source.startswith('almalinux'): - splits = source_path.split('/') - if len(splits) >= 2: - ecosystem_version = splits[1].removeprefix('almalinux') - response['human_source_link'] = render_template_string( - source_repo.human_link, ECOSYSTEM_VER=ecosystem_version, BUG_ID=vuln_id) - - -def oss_fuzz_source_path(source_repo, response): - """Get the source path for an oss-fuzz vuln.""" - for aff in response.get('affected', []): - pkg = aff.get('package', {}) - if pkg.get('ecosystem') == 'OSS-Fuzz': - project = pkg.get('name') - if not project: - continue - path = os.path.join(project, - response.get('id', '') + source_repo.extension) - if source_repo.directory_path: - path = os.path.join(source_repo.directory_path, path) - return path - - return '' - - -def _commit_to_link(repo_url, commit): - """Convert commit to link.""" - vcs = source_mapper.get_vcs_viewer_for_url(repo_url) - if not vcs: - return None - - if ':' not in commit: - return vcs.get_source_url_for_revision(commit) - - commit_parts = commit.split(':') - if len(commit_parts) != 2: - return None - - start, end = commit_parts - if start == 'unknown': - return None - - return vcs.get_source_url_for_revision_diff(start, end) - - -def osv_get_ecosystems(): - """Get list of ecosystems.""" - query = osv.ListedVulnerability.query( - projection=[osv.ListedVulnerability.ecosystems], distinct=True) - return sorted([vuln.ecosystems[0] for vuln in query if vuln.ecosystems], - key=str.lower) - - -@cache.smart_cache( - "osv_get_ecosystem_counts", hard_timeout=24 * 60 * 60, soft_timeout=30 * 60) -def osv_get_ecosystem_counts_cached(): - """Get count of vulnerabilities per ecosystem, cached""" - # Check if we're already in ndb context, if not, put us in one - # We can sometimes not be in ndb context because caching - # runs in a separate thread - if ndb.get_context(raise_context_error=False) is None: - # IMPORTANT: Ensure this ndb.Client remains consistent - # with the one defined in main.py - with _ndb_client.context(): - return osv_get_ecosystem_counts() - - return osv_get_ecosystem_counts() - - -def osv_get_ecosystem_counts() -> dict[str, int]: - """Get count of vulnerabilities per ecosystem.""" - counts = {} - ecosystem_names = osv_get_ecosystems() - for ecosystem in ecosystem_names: - if ':' in ecosystem or ecosystem == '[EMPTY]': - # Count by the base ecosystem index. Otherwise we'll overcount as a - # single entry may refer to multiple sub-ecosystems. - continue - counts[ecosystem] = osv.ListedVulnerability.query( - osv.ListedVulnerability.ecosystems == ecosystem).count() - - filtered_counts = {key: elem for key, elem in counts.items() if elem > 0} - return filtered_counts - - -def listed_vuln_response(vuln: osv.ListedVulnerability) -> dict: - """Convert a ListedVulnerability to a dict for template rendering""" - resp = vuln.to_dict() - resp['id'] = vuln.key.id() - add_cvss_score(resp) - return resp - - -def osv_query(search_string, page, ecosystem): - """Run an OSV query.""" - query: ndb.Query = osv.ListedVulnerability.query() - if search_string and len(search_string) <= 300: - # Indexed value can only be 1500 bytes at most. - # The search string length is capped at 300 which should be enough for - # package names or bug IDs. - query = query.filter( - osv.ListedVulnerability.search_indices == search_string.lower()) - if ecosystem: - query = query.filter(osv.ListedVulnerability.ecosystems == ecosystem) - query = query.order(-osv.ListedVulnerability.published) - if not search_string: - # If no search string, use the cached ecosystem counts - total_future = ndb.Future() - total_future.set_result(get_vuln_count_for_ecosystem(ecosystem)) - else: - total_future = query.count_async() - result_items = [] - listed_vulns, _, _ = query.fetch_page( - page_size=_PAGE_SIZE, offset=(page - 1) * _PAGE_SIZE) - # FIXME(michaelkedar): This logic isn't currently possible with the - # ListedVulnerability entity. - # To support this, we'd need to add the alias/upstream/related fields to the - # ListedVulnerability entity. - # For now, just stick an exact ID match at the top of the list - # - # The following only works 100% as intended if all of these results appear in - # the first page - # if _VALID_VULN_ID.match(search_string): - # sorting_bugs = list(bugs) - # # yapf: disable - # bugs = sorted( - # sorting_bugs, - # key=lambda bug: ( - # 0 if bug.db_id == search_string else - # 1 if search_string in bug.aliases else - # 2 if search_string in bug.upstream_raw else - # 3 if search_string in bug.related else - # 4, # Default priority for items not matching specific criteria - # -bug.timestamp.timestamp())) - # # yapf: enable - if _VALID_VULN_ID.match(search_string): - sorting_vulns = list(listed_vulns) - listed_vulns = sorted( - sorting_vulns, - key=lambda v: - (0 if v.key.id() == search_string else -v.published.timestamp())) - result_items = [listed_vuln_response(v) for v in listed_vulns] - # FIXME(michaelkedar): This logic isn't currently possible with the - # ListedVulnerability entity. - # To support this, we'd need change the ListedVulnerability entity to include - # the ecosystems it's fixed in. - # Filter isFixed flag to apply only for selected ecosystem. - # if ecosystem: - # eco_variants = osv.ecosystems.add_matching_ecosystems({ecosystem}) - # eco_variants.add(osv.ecosystems.normalize(ecosystem)) - # for item, bug_obj in zip(result_items, bugs): - # item['isFixed'] = _is_fixed_in_ecosystem(bug_obj,eco_variants,ecosystem) - - results = { - 'total': total_future.get_result(), - 'items': result_items, - } - return results - - -def _is_fixed_in_ecosystem(bug: osv.Bug, - eco_variants: set[str], - base_ecosystem: str | None = None) -> bool: - """Determine if a bug has a fix within the specified ecosystem variants. - - Args: - bug: The Bug entity. - eco_variants: Set of ecosystem names (including variants) to match. - base_ecosystem: Base ecosystem name to match versioned variants. - - Returns: - True if any affected package in the ecosystem has a fixed/limit event. - """ - for affected_pkg in getattr(bug, 'affected_packages', []): - pkg = affected_pkg.package - - ecosystem_matches = False - if pkg: - ecosystem_matches |= ( - pkg.ecosystem in eco_variants or - (base_ecosystem and pkg.ecosystem.startswith(base_ecosystem + ":"))) - - if base_ecosystem == "GIT" or "GIT" in eco_variants: - # Handle Git ecosystems - ecosystem_matches |= any( - r.type == "GIT" for r in (affected_pkg.ranges or [])) - - if not ecosystem_matches: - continue - - for r in affected_pkg.ranges or []: - if any(evt.type in ('fixed', 'limit') for evt in (r.events or [])): - return True - return False - - -def get_vuln_count_for_ecosystem(ecosystem: str) -> int: - ecosystem_counts = osv_get_ecosystem_counts_cached() - if not ecosystem: - return sum(ecosystem_counts.values()) - - return ecosystem_counts.get(ecosystem, 0) - - -def osv_get_by_id(vuln_id: str) -> dict: - """Gets bug details from its id. If invalid, aborts the request.""" - if not vuln_id: - abort(400) - try: - vuln = osv.gcs.get_by_id(vuln_id) - except gexceptions.NotFound as exc: - alias = check_for_aliases(vuln_id) - if not alias: - if osv.ImportFinding.get_by_id(vuln_id): - raise VulnerabilityNotImported(vuln_id) from exc - raise VulnerabilityNotFound(vuln_id) from exc - vuln = alias - - return vuln_to_response(vuln) - - -def check_for_aliases( - vuln_id: str) -> osv.vulnerability_pb2.Vulnerability | None: - """ Search for aliases of a vuln if only one exists """ - alias_group = osv.AliasGroup.query(osv.AliasGroup.bug_ids == vuln_id).get() - if alias_group and len(alias_group.bug_ids) == 2: - # Assume if current ID doesn't exist, but an alias does, bug must exist. - alias = alias_group.bug_ids[0] - if alias == vuln_id: - alias = alias_group.bug_ids[1] - # Confirm bug exists - try: - vuln = osv.gcs.get_by_id(alias) - except gexceptions.NotFound: - return None - return vuln - return None - - -@blueprint.app_template_filter('event_type') -def event_type(event): - """Get the type from an event.""" - if event.get('introduced'): - return 'Introduced' - if event.get('fixed'): - return 'Fixed' - if event.get('limit'): - return 'Limit' - if event.get('last_affected'): - return 'Last affected' - - return None - - -@blueprint.app_template_filter('event_link') -def event_link(event): - """Get the link from an event.""" - if event.get('introduced_link'): - return event['introduced_link'] - if event.get('fixed_link'): - return event['fixed_link'] - if event.get('limit_link'): - return event['limit_link'] - if event.get('last_affected_link'): - return event['last_affected_link'] - - return None - - -@blueprint.app_template_filter('event_value') -def event_value(event): - """Get the value from an event.""" - if event.get('introduced'): - return event['introduced'] - if event.get('fixed'): - return event['fixed'] - if event.get('limit'): - return event['limit'] - if event.get('last_affected'): - return event['last_affected'] - - return None - - -@blueprint.app_template_filter('should_collapse') -def should_collapse(affected): - """Whether if we should collapse the package tab bar.""" - total_text_length_ecosystem = sum( - len(entry.get('package', {}).get('ecosystem', '')) for entry in affected) - total_text_length_package = sum( - len(entry.get('package', {}).get('name', '')) + - (len(ranges[0].get('repo', '')) if (ranges := entry.get('ranges')) else 0) - for entry in affected) - - max_total_length = max(total_text_length_ecosystem, total_text_length_package) - - return len(affected) > 5 or max_total_length > 100 - - -@blueprint.app_template_filter('group_versions') -def group_versions(versions, ecosystem): - """Group versions by prefix.""" - groups = {} - - for version in sort_versions(versions, ecosystem): - if '.' not in version: - groups.setdefault('Other', []).append(version) - continue - - label = version.split('.')[0] + '.*' - groups.setdefault(label, []).append(version) - - return groups - - -@blueprint.app_template_filter('group_by_ecosystem') -def group_by_ecosystem(affected_list): - """Groups a list of affected packages by their ecosystem.""" - grouped = defaultdict(list) - for affected in affected_list: - if 'package' in affected: - ecosystem = affected['package'].get('ecosystem', 'Unknown') - else: - ecosystem = 'Git' - grouped[ecosystem].append(affected) - - # Sort ecosystems alphabetically, but keep 'Git' at the end if it exists - sorted_ecosystems = sorted( - grouped.keys(), key=lambda x: (x == 'Git', x.lower())) - return {eco: grouped[eco] for eco in sorted_ecosystems} - - -def sort_versions(versions: list[str], ecosystem: str) -> list[str]: - """Sorts a list of version numbers in the given ecosystem's sorting order.""" - try: - return sorted(versions, key=osv.ecosystems.get(ecosystem).sort_key) - except (NotImplementedError, AttributeError): - # If the ecosystem doesn't support ordering, - # the versions are sorted lexicographically. - return sorted(versions) - - -# This is an increadibly fragile regex to catch links generated by markdown2 -# with incorrect sanitisation -# Temporary until markdown2 fixes url sanitisation. -# Replaces -# -# with -# -_URL_MARKDOWN_REPLACER = re.compile(r'()') -_ANCHOR_TAG_REPLACER = re.compile( - r'<a\s+[^>]*name=["\'][^"\']*["\'][^>]*>\s*</a>|<a\s+[^>]*name=["\'][^"\']*["\'][^/]*/?>', # pylint: disable=line-too-long - re.IGNORECASE) - - -@blueprint.app_template_filter('markdown') -def markdown(text): - """Render markdown.""" - if text: - try: - md = markdown2.markdown( - text, safe_mode='escape', extras=['fenced-code-blocks']) - # TODO(michaelkedar): Seems like there's a bug with markdown2 not escaping - # unclosed HTML comments more') - self.assertIn('<!--', result) - self.assertNotIn('