From 9911e1e9792a5c9387955c32278c943c93b1a7a2 Mon Sep 17 00:00:00 2001 From: Sagar-Talikoti Date: Mon, 21 Sep 2026 15:46:32 +0530 Subject: [PATCH 01/10] fix: initial commit for the status update command --- .secrets.baseline | 2 +- bin/mas-devops-feature-status-update.md | 799 ++++++++++++++++++++++++ mongodb_schemas/README.md | 528 ++++++++++++++++ 3 files changed, 1328 insertions(+), 1 deletion(-) create mode 100644 bin/mas-devops-feature-status-update.md create mode 100644 mongodb_schemas/README.md diff --git a/.secrets.baseline b/.secrets.baseline index 046b9045..dd518c23 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2026-07-03T11:02:31Z", + "generated_at": "2026-09-21T10:12:49Z", "plugins_used": [ { "name": "AWSKeyDetector" diff --git a/bin/mas-devops-feature-status-update.md b/bin/mas-devops-feature-status-update.md new file mode 100644 index 00000000..e181f7b1 --- /dev/null +++ b/bin/mas-devops-feature-status-update.md @@ -0,0 +1,799 @@ +# mas-devops-feature-status-update + +Writes MAS feature status records to the DevOps MongoDB (`mas_devops.feature_status` collection). + +## Prerequisites + +- Python 3 +- `pymongo` — `pip install pymongo` +- MongoDB 6+ (for local development — see [Local MongoDB](#local-mongodb)) + +## Local MongoDB + +Two options to run a local MongoDB instance for development and testing. + +### Option A — Docker (recommended) + +```bash +# Start a MongoDB 7 container, data persisted in a named volume +docker run -d \ + --name mongodb-local \ + -p 27017:27017 \ + -v mongodb-local-data:/data/db \ + mongo:7 + +# Verify it is running +docker ps --filter name=mongodb-local + +# Stop / restart +docker stop mongodb-local +docker start mongodb-local + +# Remove container and volume (destroys all data) +docker rm -f mongodb-local +docker volume rm mongodb-local-data +``` + +Connection URL: `mongodb://localhost:27017` + +### Option B — Homebrew (macOS) + +```bash +# Install +brew tap mongodb/brew +brew install mongodb-community + +# Start as a background service (auto-restarts on login) +brew services start mongodb-community + +# Or run in the foreground (current terminal only) +mongod --config /opt/homebrew/etc/mongod.conf + +# Stop +brew services stop mongodb-community +``` + +Connection URL: `mongodb://localhost:27017` + +--- + +### Initialize the `feature_dashboard` database + +Once MongoDB is running, initialize the schema and indexes from the repository root: + +```bash +mongosh "mongodb://localhost:27017/feature_dashboard" mongodb_schemas/init_db.js +``` + +Verify the collections were created: + +```bash +mongosh "mongodb://localhost:27017/feature_dashboard" --eval "db.getCollectionNames()" +# Expected: [ 'cluster_level_config', 'instance_level_config' ] +``` + +### Export the connection URL + +Export `MAS_FEATURE_STATUS_DB_URL` so every subsequent command picks it up automatically without needing `--db-url` or `--db-details`: + +```bash +export MAS_FEATURE_STATUS_DB_URL='mongodb://localhost:27017' +``` + +Then verify connectivity and indexes: + +```bash +mas-devops-feature-status-update prep --create-indexes +``` + +--- + +## Installation + +### From the package (recommended) + +Install the `mas-devops` package and the script is placed on `$PATH` automatically: + +```bash +# Install from PyPI +pip install mas-devops + +# Or install from source (editable) +git clone https://github.com/ibm-mas/python-devops.git +cd python-devops +pip install -e . +``` + +Once installed, run the script directly: + +```bash +mas-devops-feature-status-update [options] +``` + +### Run directly from source (without installing) + +```bash +# From the repository root +python bin/mas-devops-feature-status-update [options] + +# Or make the script executable and run it +chmod +x bin/mas-devops-feature-status-update +./bin/mas-devops-feature-status-update [options] +``` + +### Built-in help + +```bash +# Top-level help +mas-devops-feature-status-update --help + +# Sub-command help +mas-devops-feature-status-update prep --help +mas-devops-feature-status-update status-update --help +mas-devops-feature-status-update get --help +``` + +--- + +## Sub-commands + +### `prep` + +Verifies MongoDB connectivity and confirms that the required indexes exist on the collection. + +| Index name | Fields | +|------------------------|-----------------------------------------| +| `instance_config_level` | `region` + `instance_id` + `account` | +| `cluster_config_level` | `region` + `cluster` + `account` | + +Pass `--create-indexes` to create missing indexes automatically instead of exiting with an error. + +**Options** + +| Flag | Required | Description | +|------|----------|-------------| +| `--db-details JSON` | No† | JSON object with `url` and optional `credentials` keys | +| `--db-url URL` | No† | MongoDB connection URL (alternative to `--db-details`) | +| `--create-indexes` | No | Create missing indexes automatically | + +† At least one of `--db-details`, `--db-url`, or the `MAS_FEATURE_STATUS_DB_URL` environment variable is required. + +**Examples** + +```bash +# Verify using a db-details JSON blob (local MongoDB, no auth) +mas-devops-feature-status-update prep \ + --db-details '{"url": "mongodb://localhost:27017"}' + +# Verify using a db-details JSON blob (with credentials) +mas-devops-feature-status-update prep \ + --db-details '{"url": "mongodb://localhost:27017", "credentials": {"username": "user", "password": "pass -- pragma: allowlist secret", "authSource": "admin"}}' + +# Verify and auto-create missing indexes +mas-devops-feature-status-update prep \ + --db-url mongodb://localhost:27017 \ + --create-indexes +``` + +After a successful `prep` run the command prints the `export` statements needed to reuse the connection details in subsequent `status-update` calls. + +--- + +### `status-update` + +Upserts a feature status document. +Upsert key: `(region, instance_id, account, cluster, type)` — an existing document is updated in-place; a new document is inserted if no match is found. + +**Identity options** *(all required)* + +| Flag | Description | +|------|-------------| +| `--region` | AWS region (e.g. `us-east-2`) | +| `--instance-id` | MAS instance ID (e.g. `inst02`) | +| `--account` | GitOps account name (e.g. `fyre-noble10-dev`) | +| `--cluster` | GitOps cluster name (e.g. `noble10`) | +| `--subscription-id` | Subscription ID | + +**Feature options** *(all required)* + +| Flag | Description | +|------|-------------| +| `--type` | Feature type (e.g. `allow-list`) | +| `--feature-details JSON` | Type-specific JSON payload. `allow-list` requires an `ips` array. | + +**Status options** *(all required)* + +| Flag | Description | +|------|-------------| +| `--status` | One of `REQUESTED`, `IN_PROGRESS`, `ACTIVE`, `ERROR` | +| `--status-details JSON` | JSON object describing the outcome (see schema below) | + +**Timestamp options** *(all optional, default: current UTC time)* + +| Flag | Description | +|------|-------------| +| `--deployment-start ISO-8601` | Start of the deployment | +| `--deployment-end ISO-8601` | End of the deployment | +| `--created-at ISO-8601` | Overrides `created_at` on document insert only | +| `--updated-at ISO-8601` | Overrides `updated_at` | + +**Database connection options** *(one required)* + +| Flag | Description | +|------|-------------| +| `--db-details JSON` | JSON object with `url` and optional `credentials` keys | +| `--db-url URL` | MongoDB connection URL | + +**`--status-details` schema** + +*ACTIVE* +```json +{ + "message": "Allow list is active.", + "request_configuration": "2405:201:d000:9062::/64" +} +``` + +*ERROR* +```json +{ + "message": "sample error message", + "error_code": 401, + "error_source": { + "gitops_version": "8.6.0", + "filename": "cis_ip_allowlist.yml", + "line_no": 148, + "log_file": "/var/log/gitops/run-001.log", + "stacktrace": "Traceback (most recent call last): ..." + }, + "request_configuration": "2405:201:d000:9060::/64" +} +``` + +**Examples** + +```bash +# ACTIVE status +mas-devops-feature-status-update status-update \ + --region us-east-2 \ + --instance-id inst02 \ + --account fyre-noble10-dev \ + --cluster noble10 \ + --subscription-id sub-id01 \ + --type allow-list \ + --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' \ + --status ACTIVE \ + --status-details '{"message": "Allow list is active.", "request_configuration": "2405:201:d000:9062::/64"}' \ + --deployment-start 2026-09-11T11:48:42+00:00 \ + --deployment-end 2026-09-11T11:53:10+00:00 + +# ERROR status +mas-devops-feature-status-update status-update \ + --region us-east-2 \ + --instance-id inst02 \ + --account fyre-noble10-dev \ + --cluster noble10 \ + --subscription-id sub-id01 \ + --type allow-list \ + --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' \ + --status ERROR \ + --status-details '{ + "message": "sample error message", + "error_code": 401, + "error_source": { + "gitops_version": "8.6.0", + "filename": "cis_ip_allowlist.yml", + "line_no": 148, + "log_file": "/var/log/gitops/run-001.log", + "stacktrace": "Traceback (most recent call last): ..." + }, + "request_configuration": "2405:201:d000:9060::/64" + }' +``` + +--- + +### `get` + +Fetches a single feature status document by its ObjectId and prints it as formatted JSON. + +**Arguments** + +| Argument | Required | Description | +|----------|----------|-------------| +| `OBJECT_ID` | Yes | 24-character hex ObjectId (printed by `status-update` on success) | +| `--db-details JSON` | No† | JSON object with `url` and optional `credentials` keys | +| `--db-url URL` | No† | MongoDB connection URL | + +† At least one of `--db-details`, `--db-url`, or the `MAS_FEATURE_STATUS_DB_URL` environment variable is required. + +**Example** + +```bash +mas-devops-feature-status-update get 6ab0e70ee6d3a31faa808547 +``` + +**Sample output** + +```json +{ + "_id": "", + "schema_version": 1, + "region": "us-east-2", + "instance_id": "inst02", + "account": "fyre-noble10-dev", + "cluster": "noble10", + "subscription_id": "sub-id01", + "type": "allow-list", + "feature_details": { "ips": ["2405:201:d000:9062::/64"] }, + "status": "ACTIVE", + "status_details": { "message": "Allow list is active.", "request_configuration": "2405:201:d000:9062::/64" }, + "deployment_start": "2026-09-11 11:48:42+00:00", + "deployment_end": "2026-09-11 11:53:10+00:00", + "created_at": "2026-09-11 11:48:42+00:00", + "updated_at": "2026-09-11 11:48:42+00:00" +} +``` + +--- + +## Environment Variables + +Setting these avoids repeating `--db-details` / `--db-url` on every call. + +| Variable | Description | +|----------|-------------| +| `MAS_FEATURE_STATUS_DB_URL` | MongoDB connection URL | +| `MAS_FEATURE_STATUS_DB_CREDENTIALS` | JSON object with optional `username`, `password`, `authSource`, `tls` keys | + +**Precedence** (highest to lowest): `--db-details` → `--db-url` → environment variables. + +```bash +export MAS_FEATURE_STATUS_DB_URL='mongodb://user:pass@host:27017' #pragma: allowlist secret +export MAS_FEATURE_STATUS_DB_CREDENTIALS='{"username": "u", "password": "p"}' #pragma: allowlist secret + +mas-devops-feature-status-update status-update \ + --region us-east-2 \ + ... +``` + +--- + +## Database Setup + +The `feature_dashboard` MongoDB database must be initialised before this tool can write records. It holds two collections: + +| Collection | Cardinality | +|---|---| +| `cluster_level_config` | One document per `tenant_id × account × region × cluster` | +| `instance_level_config` | One document per `tenant_id × subscription_id × account × region × cluster × instance` | + +### Initialize + +Run `init_db.js` (which loads both schema files) against your MongoDB host: + +```bash +# mongosh (≥ 1.x, recommended) +mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/init_db.js + +# Legacy mongo shell +mongo "mongodb://:27017/feature_dashboard" mongodb_schemas/init_db.js +``` + +Or initialize each collection individually: + +```bash +mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/cluster_level_config.js +mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/instance_level_config.js +``` + +### Clear data (keep schema & indexes) + +```js +use feature_dashboard +db.cluster_level_config.deleteMany({}) +db.instance_level_config.deleteMany({}) +``` + +### Drop collections (removes schema & indexes) + +```js +use feature_dashboard +db.cluster_level_config.drop() +db.instance_level_config.drop() +``` + +> **Note:** `drop()` removes the collection, all documents, and all indexes. Re-run `init_db.js` to recreate them. + +### Indexes created by `init_db.js` + +**`cluster_level_config`** + +| Index name | Fields | Unique | +|---|---|---| +| `ux_cluster_level_config_tenant_account_region_cluster` | `tenant_id, account, region, cluster` | ✓ | +| `ix_cluster_level_config_tenant_account` | `tenant_id, account` | | + +**`instance_level_config`** + +| Index name | Fields | Unique | +|---|---|---| +| `ux_instance_level_config_tenant_sub_account_region_cluster_instance` | `tenant_id, subscription_id, account, region, cluster, instance` | ✓ | +| `ix_instance_level_config_tenant_sub_account_region_cluster` | `tenant_id, subscription_id, account, region, cluster` | | +| `ix_instance_level_config_feature_status` | `instance_level_features.status` | | +| `ix_instance_level_config_error_code` | `instance_level_features.status_details.error_code` (sparse) | | + +### Validation behaviour + +Both collections enforce: + +```js +validationLevel: "strict" // enforced on inserts AND updates +validationAction: "error" // rejects non-conforming writes outright +``` + +`additionalProperties: false` is set on every top-level and nested object (except `cluster_level_features[]` items, which allow extension fields). + +--- + +## MongoDB Document Schema + +Collection: `mas_devops.feature_status` + +```json +{ + "_id": "", + "schema_version": 1, + "region": "us-east-2", + "instance_id": "inst02", + "account": "fyre-noble10-dev", + "cluster": "noble10", + "subscription_id": "sub-id01", + "type": "allow-list", + "feature_details": { "ips": ["2405:201:d000:9062::/64"] }, + "status": "ACTIVE", + "status_details": { "message": "...", "request_configuration": "..." }, + "deployment_start": "", + "deployment_end": "", + "created_at": "", + "updated_at": "" +} +``` + +--- + +## Global Options + +| Flag | Default | Description | +|------|---------|-------------| +| `--log-level` | `WARNING` | Python logging level: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` | + +--- + +## Ansible Integration + +See the full sample playbook at [`playbooks/feature-status-update.yml`](../playbooks/feature-status-update.yml). + +### Minimal task — `prep` + +Verify connectivity before any write. Use `--create-indexes` on first run. + +```yaml +- name: Verify MongoDB connectivity and indexes + ansible.builtin.command: + cmd: >- + mas-devops-feature-status-update prep + --db-url {{ mas_mongo_url }} + --create-indexes + register: prep_result + changed_when: "'Creating missing indexes' in prep_result.stdout" +``` + +### Minimal task — `status-update` + +```yaml +- name: Upsert feature status (ACTIVE) + ansible.builtin.command: + cmd: >- + mas-devops-feature-status-update status-update + --db-url {{ mas_mongo_url }} + --region {{ mas_region }} + --instance-id {{ mas_instance_id }} + --account {{ mas_account }} + --cluster {{ mas_cluster }} + --subscription-id {{ mas_subscription_id }} + --type allow-list + --feature-details {{ '{"ips": ["2405:201:d000:9062::/64"]}' | quote }} + --status ACTIVE + --status-details {{ '{"message": "Allow list is active.", "request_configuration": "2405:201:d000:9062::/64"}' | quote }} + register: status_update_result + changed_when: "'written successfully' in status_update_result.stdout" +``` + +### Minimal task — `get` + +Extract the document ID from `status-update` output and fetch the written document: + +```yaml +- name: Extract document ID + ansible.builtin.set_fact: + mas_document_id: >- + {{ status_update_result.stdout + | regex_search('Document ID: ([a-f0-9]{24})', '\1') + | first }} + +- name: Fetch feature status document + ansible.builtin.command: + cmd: >- + mas-devops-feature-status-update get + --db-url {{ mas_mongo_url }} + {{ mas_document_id }} + register: get_result + changed_when: false + +- name: Display document + ansible.builtin.debug: + msg: "{{ get_result.stdout | from_json }}" +``` + +### Using environment variables instead of `--db-url` + +Set `MAS_FEATURE_STATUS_DB_URL` once (e.g. in `group_vars/all.yml` or a `block` `environment:`) to avoid repeating the flag on every task: + +```yaml +- name: Feature status tasks + environment: + MAS_FEATURE_STATUS_DB_URL: "mongodb://localhost:27017" + block: + - name: prep + ansible.builtin.command: + cmd: mas-devops-feature-status-update prep --create-indexes + + - name: status-update + ansible.builtin.command: + cmd: >- + mas-devops-feature-status-update status-update + --region us-east-2 + --instance-id inst02 + --account fyre-noble10-dev + --cluster noble10 + --subscription-id sub-id01 + --type allow-list + --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' + --status ACTIVE + --status-details '{"message": "Allow list is active.", "request_configuration": "2405:201:d000:9062::/64"}' +``` + +--- + +## MongoDB Query Reference + +Every query is a standalone `mongosh` command — replace `mongodb://localhost:27017` with your connection URL. + +--- + +### By document ID + +```bash +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.findOne({ _id: ObjectId("") })' +``` + +--- + +### By identity fields + +```bash +# Full identity match (region + instance + account + cluster + type) +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.findOne({ + region: "us-east-2", + instance_id: "inst02", + account: "fyre-noble10-dev", + cluster: "noble10", + type: "allow-list" + })' + +# All documents for a specific instance +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find( + { account: "fyre-noble10-dev", instance_id: "inst02" } + ).sort({ updated_at: -1 }).pretty()' + +# All documents for a cluster (all instances within it) +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find( + { account: "fyre-noble10-dev", cluster: "noble10" } + ).sort({ updated_at: -1 }).pretty()' + +# All documents for an account across all clusters +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find( + { account: "fyre-noble10-dev" } + ).sort({ cluster: 1, instance_id: 1 }).pretty()' + +# All documents for a region +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find( + { region: "us-east-2" } + ).sort({ account: 1, cluster: 1 }).pretty()' + +# All documents for a subscription ID +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find( + { subscription_id: "sub-id01" } + ).sort({ updated_at: -1 }).pretty()' +``` + +--- + +### By status + +```bash +# All documents in a specific status +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.find({ status: "ACTIVE" }).pretty()' + +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.find({ status: "ERROR" }).pretty()' + +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.find({ status: "IN_PROGRESS" }).pretty()' + +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.find({ status: "REQUESTED" }).pretty()' + +# Multiple statuses at once +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find( + { status: { $in: ["REQUESTED", "IN_PROGRESS"] } } + ).sort({ updated_at: 1 }).pretty()' + +# Count documents grouped by status +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.aggregate([ + { $group: { _id: "$status", count: { $sum: 1 } } }, + { $sort: { count: -1 } } + ])' +``` + +--- + +### By feature type and payload + +```bash +# All allow-list documents +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.find({ type: "allow-list" }).pretty()' + +# ACTIVE allow-list entries for a specific IP/CIDR +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find({ + type: "allow-list", + status: "ACTIVE", + "feature_details.ips": "2405:201:d000:9062::/64" + }).pretty()' + +# Any allow-list document whose IP array contains a given prefix (regex) +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find({ + type: "allow-list", + "feature_details.ips": { $regex: "^2405:201:" } + }).pretty()' +``` + +--- + +### By error details + +```bash +# All ERROR documents with a specific HTTP error code +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find({ + status: "ERROR", + "status_details.error_code": 401 + }).pretty()' + +# ERROR documents mentioning a keyword in the message (case-insensitive) +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find({ + status: "ERROR", + "status_details.message": { $regex: "timeout", $options: "i" } + }).pretty()' + +# ERROR documents from a specific GitOps version +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find({ + status: "ERROR", + "status_details.error_source.gitops_version": "8.6.0" + }).pretty()' +``` + +--- + +### By time + +```bash +# Documents updated in the last 24 hours +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find({ + updated_at: { $gte: new Date(Date.now() - 24 * 60 * 60 * 1000) } + }).sort({ updated_at: -1 }).pretty()' + +# Documents created in a specific date range +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find({ + created_at: { + $gte: new Date("2026-09-01T00:00:00Z"), + $lte: new Date("2026-09-30T23:59:59Z") + } + }).sort({ created_at: -1 }).pretty()' + +# Deployments that took longer than 5 minutes +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find({ + deployment_start: { $exists: true }, + deployment_end: { $exists: true }, + $expr: { + $gte: [ + { $dateDiff: { + startDate: { $dateFromString: { dateString: "$deployment_start" } }, + endDate: { $dateFromString: { dateString: "$deployment_end" } }, + unit: "minute" + }}, + 5 + ] + } + }).pretty()' + +# Most recently updated documents (last 10) +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.find().sort({ updated_at: -1 }).limit(10).pretty()' +``` + +--- + +### Projection — select specific fields only + +```bash +# Identity + status summary (no feature payload) +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find( + { account: "fyre-noble10-dev" }, + { region: 1, instance_id: 1, cluster: 1, subscription_id: 1, + type: 1, status: 1, updated_at: 1, _id: 0 } + ).sort({ updated_at: -1 }).pretty()' + +# Status and timestamps only +mongosh "mongodb://localhost:27017/mas_devops" --eval ' + db.feature_status.find( + { cluster: "noble10" }, + { status: 1, deployment_start: 1, deployment_end: 1, updated_at: 1, _id: 0 } + ).pretty()' +``` + +--- + +### Counting and diagnostics + +```bash +# Total document count +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.countDocuments()' + +# Count for a specific account + status +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.countDocuments({ account: "fyre-noble10-dev", status: "ACTIVE" })' + +# All distinct accounts +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.distinct("account")' + +# All distinct clusters for a region +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.distinct("cluster", { region: "us-east-2" })' + +# All distinct statuses present +mongosh "mongodb://localhost:27017/mas_devops" --eval \ + 'db.feature_status.distinct("status")' +``` diff --git a/mongodb_schemas/README.md b/mongodb_schemas/README.md new file mode 100644 index 00000000..59e31fd6 --- /dev/null +++ b/mongodb_schemas/README.md @@ -0,0 +1,528 @@ +# MongoDB Schemas — `feature_dashboard` + +MongoDB validator scripts for the `feature_dashboard` database. +Extracted from [`allowlisting-tdd.md`](../allowlisting-tdd.md) §7. + +## Collections + +The original `allowlisting_config` collection has been split into two flat collections — one per level of the hierarchy — to enable targeted writes and precise index coverage. + +| File | Collection | Cardinality | Description | +|---|---|---|---| +| [`cluster_level_config.js`](cluster_level_config.js) | `cluster_level_config` | 1 doc per `(tenant_id × account × region × cluster)` | Cluster-scoped feature entries (e.g. `dro`). | +| [`instance_level_config.js`](instance_level_config.js) | `instance_level_config` | 1 doc per `(tenant_id × account × region × cluster × instance)` | Instance-scoped IP/CIDR allowlisting entries with per-feature deployment lifecycle state. | +| [`init_db.js`](init_db.js) | *(all)* | — | Bootstrap runner — initialises both collections and all indexes. | + +### Key fields per collection + +**`cluster_level_config`** + +| Field | Type | Notes | +|---|---|---| +| `tenant_id` | string | Multi-tenancy isolation key | +| `account` | string | Account from cluster polling | +| `region` | string | e.g. `us-east-1` | +| `cluster` | string | Cluster name from polling | +| `cluster_level_features[]` | array | One element per cluster-level feature (e.g. `dro`) | + +**`instance_level_config`** + +| Field | Type | Notes | +|---|---|---| +| `tenant_id` | string | Multi-tenancy isolation key | +| `account` | string | Account from cluster polling | +| `region` | string | e.g. `us-east-1` | +| `cluster` | string | Cluster name from polling | +| `instance` | string | Specific instance within the cluster | +| `instance_level_features[]` | array | One element per feature — holds `allow_lists.ips[]`, `source`, `cluster_poll_status`, timestamps, audit fields | + +## Files + +| File | Collection | Description | +|---|---|---| +| [`cluster_level_config.js`](cluster_level_config.js) | `cluster_level_config` | Cluster-scoped feature entries. | +| [`instance_level_config.js`](instance_level_config.js) | `instance_level_config` | Instance-scoped IP/CIDR allowlisting entries. | +| [`init_db.js`](init_db.js) | *(all)* | Bootstrap runner — initialises all collections and indexes. | + +--- + +## Query reference + +All queries assume `use feature_dashboard` has been run first. Replace +``, ``, ``, ``, ``, and +`` with real values. + +--- + +### `cluster_level_config` queries + +#### List all cluster documents + +```js +db.cluster_level_config.find().pretty() +``` + +#### List all clusters for a tenant + +```js +db.cluster_level_config.find( + { tenant_id: "" }, + { _id: 0, account: 1, region: 1, cluster: 1 } +).pretty() +``` + +#### List all clusters for an account + +```js +db.cluster_level_config.find( + { tenant_id: "", account: "" }, + { _id: 0, region: 1, cluster: 1 } +).pretty() +``` + +#### List all clusters in a region + +```js +db.cluster_level_config.find( + { tenant_id: "", account: "", region: "" }, + { _id: 0, cluster: 1 } +).pretty() +``` + +#### Fetch a single cluster document + +```js +db.cluster_level_config.findOne({ + tenant_id: "", + account: "", + region: "", + cluster: "" +}) +``` + +#### List cluster-level features for a specific cluster + +```js +db.cluster_level_config.findOne( + { tenant_id: "", account: "", region: "", cluster: "" }, + { _id: 0, cluster_level_features: 1 } +) +``` + +#### Find all clusters that have a specific cluster-level feature enabled + +```js +db.cluster_level_config.find( + { + tenant_id: "", + "cluster_level_features.feature_key": "" + }, + { _id: 0, account: 1, region: 1, cluster: 1, cluster_level_features: 1 } +).pretty() +``` + +#### Find all clusters that have any cluster-level feature enabled (non-empty array) + +```js +db.cluster_level_config.find( + { tenant_id: "", "cluster_level_features.0": { $exists: true } }, + { _id: 0, account: 1, region: 1, cluster: 1 } +).pretty() +``` + +#### Find all clusters with no cluster-level features + +```js +db.cluster_level_config.find( + { tenant_id: "", cluster_level_features: { $size: 0 } }, + { _id: 0, account: 1, region: 1, cluster: 1 } +).pretty() +``` + +#### Count clusters per region (for a tenant) + +```js +db.cluster_level_config.aggregate([ + { $match: { tenant_id: "" } }, + { $group: { _id: { account: "$account", region: "$region" }, cluster_count: { $sum: 1 } } }, + { $sort: { "_id.account": 1, "_id.region": 1 } } +]) +``` + +#### Count clusters per account (for a tenant) + +```js +db.cluster_level_config.aggregate([ + { $match: { tenant_id: "" } }, + { $group: { _id: "$account", cluster_count: { $sum: 1 } } }, + { $sort: { _id: 1 } } +]) +``` + +--- + +### `instance_level_config` queries + +#### List all instance documents + +```js +db.instance_level_config.find().pretty() +``` + +#### List all instances for a tenant + +```js +db.instance_level_config.find( + { tenant_id: "" }, + { _id: 0, account: 1, region: 1, cluster: 1, instance: 1 } +).pretty() +``` + +#### List all instances for an account + +```js +db.instance_level_config.find( + { tenant_id: "", account: "" }, + { _id: 0, region: 1, cluster: 1, instance: 1 } +).pretty() +``` + +#### List all instances in a region + +```js +db.instance_level_config.find( + { tenant_id: "", account: "", region: "" }, + { _id: 0, cluster: 1, instance: 1 } +).pretty() +``` + +#### List all instances in a cluster + +```js +db.instance_level_config.find( + { tenant_id: "", account: "", region: "", cluster: "" }, + { _id: 0, instance: 1 } +).pretty() +``` + +#### Fetch a single instance document (full detail) + +```js +db.instance_level_config.findOne({ + tenant_id: "", + account: "", + region: "", + cluster: "", + instance: "" +}) +``` + +#### Fetch only the instance-level features for a specific instance + +```js +db.instance_level_config.findOne( + { + tenant_id: "", + account: "", + region: "", + cluster: "", + instance: "" + }, + { _id: 0, instance_level_features: 1 } +) +``` + +#### Fetch a single feature entry for a specific instance + +Returns just the matching element from `instance_level_features[]` using `$elemMatch`. + +```js +db.instance_level_config.findOne( + { + tenant_id: "", + account: "", + region: "", + cluster: "", + instance: "" + }, + { + _id: 0, + instance_level_features: { + $elemMatch: { feature_key: "" } + } + } +) +``` + +#### Find all instances that have a specific feature key + +```js +db.instance_level_config.find( + { + tenant_id: "", + "instance_level_features.feature_key": "" + }, + { _id: 0, account: 1, region: 1, cluster: 1, instance: 1 } +).pretty() +``` + +#### Find all instances for a feature key — include the matching feature entry only + +```js +db.instance_level_config.find( + { + tenant_id: "", + "instance_level_features.feature_key": "" + }, + { + _id: 0, + account: 1, region: 1, cluster: 1, instance: 1, + instance_level_features: { $elemMatch: { feature_key: "" } } + } +).pretty() +``` + +#### Find all instances that have any non-empty allow list for a feature + +```js +db.instance_level_config.find( + { + tenant_id: "", + instance_level_features: { + $elemMatch: { + feature_key: "", + "allow_lists.ips": { $exists: true, $not: { $size: 0 } } + } + } + }, + { _id: 0, account: 1, region: 1, cluster: 1, instance: 1 } +).pretty() +``` + +#### Find all instances whose allow list contains a specific IP or CIDR + +```js +db.instance_level_config.find( + { + tenant_id: "", + instance_level_features: { + $elemMatch: { + feature_key: "", + "allow_lists.ips": "" + } + } + }, + { _id: 0, account: 1, region: 1, cluster: 1, instance: 1 } +).pretty() +``` + +#### Find all stale instances (poll status = stale or unreachable) + +```js +db.instance_level_config.find( + { + tenant_id: "", + "instance_level_features.cluster_poll_status": { $in: ["stale", "unreachable"] } + }, + { _id: 0, account: 1, region: 1, cluster: 1, instance: 1, + instance_level_features: { + $elemMatch: { cluster_poll_status: { $in: ["stale", "unreachable"] } } + } + } +).pretty() +``` + +#### Find all instances not polled since a given timestamp + +```js +db.instance_level_config.find( + { + tenant_id: "", + "instance_level_features.cluster_last_polled_at": { + $lt: ISODate("") + } + }, + { _id: 0, account: 1, region: 1, cluster: 1, instance: 1 } +).pretty() +``` + +#### Find all instances never polled (cluster_last_polled_at absent) + +```js +db.instance_level_config.find( + { + tenant_id: "", + "instance_level_features.cluster_last_polled_at": { $exists: false } + }, + { _id: 0, account: 1, region: 1, cluster: 1, instance: 1 } +).pretty() +``` + +--- + +### Cross-collection queries + +#### List all distinct regions for an account + +```js +// From cluster_level_config (one query covers all clusters, hence all regions) +db.cluster_level_config.distinct("region", { + tenant_id: "", + account: "" +}) +``` + +#### List all distinct accounts for a tenant + +```js +db.cluster_level_config.distinct("account", { tenant_id: "" }) +``` + +#### List all distinct clusters in a region + +```js +db.cluster_level_config.distinct("cluster", { + tenant_id: "", + account: "", + region: "" +}) +``` + +#### List all distinct instances in a cluster + +```js +db.instance_level_config.distinct("instance", { + tenant_id: "", + account: "", + region: "", + cluster: "" +}) +``` + +#### Full cluster view — cluster features + all instances (aggregation join) + +Joins `cluster_level_config` and `instance_level_config` for a single cluster +using `$lookup`. + +```js +db.cluster_level_config.aggregate([ + { + $match: { + tenant_id: "", + account: "", + region: "", + cluster: "" + } + }, + { + $lookup: { + from: "instance_level_config", + localField: "cluster", + foreignField: "cluster", + let: { + t: "$tenant_id", + a: "$account", + r: "$region", + c: "$cluster" + }, + pipeline: [ + { + $match: { + $expr: { + $and: [ + { $eq: ["$tenant_id", "$$t"] }, + { $eq: ["$account", "$$a"] }, + { $eq: ["$region", "$$r"] }, + { $eq: ["$cluster", "$$c"] } + ] + } + } + }, + { $project: { _id: 0, instance: 1, instance_level_features: 1 } } + ], + as: "instances" + } + }, + { + $project: { + _id: 0, + account: 1, region: 1, cluster: 1, + cluster_level_features: 1, + instances: 1 + } + } +]) +``` + +#### Count instances per cluster across all clusters for a tenant + +```js +db.instance_level_config.aggregate([ + { $match: { tenant_id: "" } }, + { + $group: { + _id: { account: "$account", region: "$region", cluster: "$cluster" }, + instance_count: { $sum: 1 } + } + }, + { $sort: { "_id.account": 1, "_id.region": 1, "_id.cluster": 1 } } +]) +``` + +#### Count instances per region for an account + +```js +db.instance_level_config.aggregate([ + { $match: { tenant_id: "", account: "" } }, + { $group: { _id: "$region", instance_count: { $sum: 1 } } }, + { $sort: { _id: 1 } } +]) +``` + +--- + +## Running + +### Initialize the collections + +```bash +mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/init_db.js +``` + +### Clear the collections + +```js +use feature_dashboard +db.cluster_level_config.deleteMany({}) +db.instance_level_config.deleteMany({}) +``` + +### Drop the collections + +```js +use feature_dashboard +db.cluster_level_config.drop() +db.instance_level_config.drop() +``` + +> **Note:** `drop()` removes the collection, all its documents, and its indexes. Re-run `init_db.js` to recreate them. + +### Run a schema file directly + +```bash +mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/cluster_level_config.js +mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/instance_level_config.js +``` + +--- + +## Validation behaviour + +Both collections use: + +```js +validationLevel: "strict" // enforced on inserts AND updates +validationAction: "error" // rejects non-conforming writes outright +``` + +`additionalProperties: false` is set on every top-level object and nested object (except `cluster_level_features[]` items, which allow extension fields) to prevent undocumented fields from being silently stored. From 3040a70732e9924b937407816b0e8de41da1a572 Mon Sep 17 00:00:00 2001 From: Sagar-Talikoti Date: Mon, 21 Sep 2026 16:20:48 +0530 Subject: [PATCH 02/10] update for REQUESTED and IN_PROGRESS --- bin/mas-devops-feature-status-update.md | 54 ++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/bin/mas-devops-feature-status-update.md b/bin/mas-devops-feature-status-update.md index e181f7b1..fad10038 100644 --- a/bin/mas-devops-feature-status-update.md +++ b/bin/mas-devops-feature-status-update.md @@ -226,7 +226,23 @@ Upsert key: `(region, instance_id, account, cluster, type)` — an existing docu **`--status-details` schema** -*ACTIVE* +*REQUESTED* — pipeline has received the request but processing has not yet started. +```json +{ + "message": "Allow list request received.", + "request_configuration": "2405:201:d000:9062::/64" +} +``` + +*IN_PROGRESS* — pipeline is actively deploying the feature. +```json +{ + "message": "Allow list deployment in progress.", + "request_configuration": "2405:201:d000:9062::/64" +} +``` + +*ACTIVE* — deployment completed successfully. ```json { "message": "Allow list is active.", @@ -234,7 +250,7 @@ Upsert key: `(region, instance_id, account, cluster, type)` — an existing docu } ``` -*ERROR* +*ERROR* — deployment failed. ```json { "message": "sample error message", @@ -253,7 +269,33 @@ Upsert key: `(region, instance_id, account, cluster, type)` — an existing docu **Examples** ```bash -# ACTIVE status +# REQUESTED status — record that a request has been received +mas-devops-feature-status-update status-update \ + --region us-east-2 \ + --instance-id inst02 \ + --account fyre-noble10-dev \ + --cluster noble10 \ + --subscription-id sub-id01 \ + --type allow-list \ + --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' \ + --status REQUESTED \ + --status-details '{"message": "Allow list request received.", "request_configuration": "2405:201:d000:9062::/64"}' \ + --deployment-start 2026-09-11T11:48:42+00:00 + +# IN_PROGRESS status — record that deployment has started +mas-devops-feature-status-update status-update \ + --region us-east-2 \ + --instance-id inst02 \ + --account fyre-noble10-dev \ + --cluster noble10 \ + --subscription-id sub-id01 \ + --type allow-list \ + --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' \ + --status IN_PROGRESS \ + --status-details '{"message": "Allow list deployment in progress.", "request_configuration": "2405:201:d000:9062::/64"}' \ + --deployment-start 2026-09-11T11:48:42+00:00 + +# ACTIVE status — record successful completion mas-devops-feature-status-update status-update \ --region us-east-2 \ --instance-id inst02 \ @@ -267,7 +309,7 @@ mas-devops-feature-status-update status-update \ --deployment-start 2026-09-11T11:48:42+00:00 \ --deployment-end 2026-09-11T11:53:10+00:00 -# ERROR status +# ERROR status — record a failed deployment mas-devops-feature-status-update status-update \ --region us-east-2 \ --instance-id inst02 \ @@ -288,7 +330,9 @@ mas-devops-feature-status-update status-update \ "stacktrace": "Traceback (most recent call last): ..." }, "request_configuration": "2405:201:d000:9060::/64" - }' + }' \ + --deployment-start 2026-09-11T11:48:42+00:00 \ + --deployment-end 2026-09-11T11:53:10+00:00 ``` --- From 78579a6fb0859ef8461bff880b4f0ed79674886a Mon Sep 17 00:00:00 2001 From: Sagar-Talikoti Date: Tue, 22 Sep 2026 14:20:49 +0530 Subject: [PATCH 03/10] initial code --- bin/mas-devops-feature-status-update | 484 +++++++++++++++++++++++ mongodb_schemas/cluster_level_config.js | 101 +++++ mongodb_schemas/init_db.js | 25 ++ mongodb_schemas/instance_level_config.js | 286 ++++++++++++++ setup.py | 2 + src/mas/devops/feature_status.py | 328 +++++++++++++++ 6 files changed, 1226 insertions(+) create mode 100755 bin/mas-devops-feature-status-update create mode 100644 mongodb_schemas/cluster_level_config.js create mode 100644 mongodb_schemas/init_db.js create mode 100644 mongodb_schemas/instance_level_config.js create mode 100644 src/mas/devops/feature_status.py diff --git a/bin/mas-devops-feature-status-update b/bin/mas-devops-feature-status-update new file mode 100755 index 00000000..97887013 --- /dev/null +++ b/bin/mas-devops-feature-status-update @@ -0,0 +1,484 @@ +#!/usr/bin/env python3 + +# ***************************************************************************** +# Copyright (c) 2025 IBM Corporation and other Contributors. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# ***************************************************************************** +""" +mas-devops-feature-status-update — Write MAS feature status records to the +DevOps MongoDB (mas_devops.feature_status collection). + +Sub-commands +──────────── + + prep + Verify MongoDB connectivity and confirm that the required indexes + (instance_config_level, cluster_config_level) exist on the collection. + Pass --create-indexes to create them when absent. + + Example: + mas-devops-feature-status-update prep \\ + --db-details '{"url": "mongodb://:27017", "credentials": {"username": ""}}' #pragma: allowlist secret + + mas-devops-feature-status-update prep --db-url mongodb://host:27017 --create-indexes + + + status-update + Upsert a feature status document. + Upsert key: (region, instance_id, account, cluster, type). + + Example — ACTIVE: + mas-devops-feature-status-update status-update \\ + --region us-east-2 \\ + --instance-id inst02 \\ + --account fyre-noble10-dev \\ + --cluster noble10 \\ + --subscription-id sub-id01 \\ + --type allow-list \\ + --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' \\ + --status ACTIVE \\ + --status-details '{"message": "Allow list is active.", "request_configuration": "2405:201:d000:9062::/64"}' \\ + --deployment-start 2026-09-11T11:48:42+00:00 \\ + --deployment-end 2026-09-11T11:53:10+00:00 + + Example — ERROR: + mas-devops-feature-status-update status-update \\ + --region us-east-2 \\ + --instance-id inst02 \\ + --account fyre-noble10-dev \\ + --cluster noble10 \\ + --subscription-id sub-id01 \\ + --type allow-list \\ + --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' \\ + --status ERROR \\ + --status-details '{ + "message": "sample error message", + "error_code": 401, + "error_source": { + "gitops_version": "8.6.0", + "filename": "cis_ip_allowlist.yml", + "line_no": 148, + "log_file": "/var/log/gitops/run-001.log", + "stacktrace": "Traceback (most recent call last): ..." + }, + "request_configuration": "2405:201:d000:9060::/64" + }' + +Environment variables +───────────────────── + MAS_FEATURE_STATUS_DB_URL MongoDB connection URL + MAS_FEATURE_STATUS_DB_CREDENTIALS JSON object: username, password, + authSource, tls (all optional) +""" + +import argparse +import json +import logging +import os +import sys +from datetime import datetime, timezone +from typing import Optional + +# --------------------------------------------------------------------------- +# Env-var names used to persist / retrieve DB details between invocations +# --------------------------------------------------------------------------- + +_ENV_DB_URL = "MAS_FEATURE_STATUS_DB_URL" +_ENV_DB_CREDENTIALS = "MAS_FEATURE_STATUS_DB_CREDENTIALS" # pragma: allowlist secret + + +# --------------------------------------------------------------------------- +# Argument-parsing helpers +# --------------------------------------------------------------------------- + + +def _parse_json_relaxed(raw: str) -> dict: + """Parse a JSON string, accepting bare keys and single quotes (shell-friendly).""" + try: + return json.loads(raw) + except json.JSONDecodeError: + import re + + relaxed = re.sub(r"'([^']*)'", r'"\1"', raw) + relaxed = re.sub(r"([{,\[]\s*)([A-Za-z_][A-Za-z0-9_]*)\s*:", r'\1"\2":', relaxed) + relaxed = re.sub(r"^(\s*)([A-Za-z_][A-Za-z0-9_]*)\s*:", r'\1"\2":', relaxed) + try: + return json.loads(relaxed) + except json.JSONDecodeError: + raise ValueError(f"Could not parse value as JSON.\n" f" Input : {raw!r}\n" f' Hint : use double-quoted keys, e.g. {{"url": "mongodb://..."}}') + + +def _parse_json_arg(value: str, arg_name: str) -> dict: + try: + result = _parse_json_relaxed(value) + except ValueError as exc: + print(f"ERROR: --{arg_name}: {exc}", file=sys.stderr) + sys.exit(1) + if not isinstance(result, dict): + print(f"ERROR: --{arg_name} must be a JSON object (got {type(result).__name__})", file=sys.stderr) + sys.exit(1) + return result + + +def _parse_isodate(value: Optional[str], arg_name: str) -> Optional[datetime]: + """Parse an ISO-8601 datetime, stripping MongoDB ISODate() wrappers.""" + if value is None: + return None + import re + + m = re.match(r"ISODate\(['\"](.+?)['\"]\)", value.strip()) + if m: + value = m.group(1) + try: + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt + except ValueError: + print(f"ERROR: --{arg_name} must be an ISO-8601 datetime string, got: {value!r}", file=sys.stderr) + sys.exit(1) + + +def _resolve_db(args) -> tuple: + """Return (mongo_url, credentials) resolving from CLI args then env vars. + + Precedence: + 1. --db-details JSON + 2. --db-url + 3. MAS_FEATURE_STATUS_DB_URL / MAS_FEATURE_STATUS_DB_CREDENTIALS + """ + db_details = getattr(args, "db_details", None) + db_url_arg = getattr(args, "db_url", None) + + if db_details: + details = _parse_json_arg(db_details, "db-details") + url = details.get("url") or details.get("mongo_url") + if not url: + print("ERROR: --db-details must contain a 'url' key", file=sys.stderr) + sys.exit(1) + credentials = details.get("credentials") or None + return url, credentials + + if db_url_arg: + return db_url_arg, None + + env_url = os.environ.get(_ENV_DB_URL, "") + if env_url: + creds_raw = os.environ.get(_ENV_DB_CREDENTIALS, "") + credentials = json.loads(creds_raw) if creds_raw else None + return env_url, credentials + + print( + "ERROR: MongoDB connection details are required.\n" + " Provide one of:\n" + f' --db-details \'{{"url": "mongodb://..."}}\'\n' + f" --db-url \n" + f" {_ENV_DB_URL} environment variable", + file=sys.stderr, + ) + sys.exit(1) + + +# --------------------------------------------------------------------------- +# Sub-command: prep +# --------------------------------------------------------------------------- + + +def cmd_prep(args) -> int: + """Verify connection and confirm required indexes exist.""" + from mas.devops.feature_status import ( + _redact_url, + create_indexes, + verify_connection_and_indexes, + ) + + mongo_url, credentials = _resolve_db(args) + print(f"Connecting to: {_redact_url(mongo_url)}") + + try: + warnings = verify_connection_and_indexes(mongo_url, credentials) + except Exception as exc: + print(f"ERROR: Could not connect to MongoDB: {exc}", file=sys.stderr) + return 1 + + if warnings: + for w in warnings: + print(f"WARNING: {w}") + if args.create_indexes: + print("Creating missing indexes …") + try: + create_indexes(mongo_url, credentials) + print("Indexes created successfully.") + except Exception as exc: + print(f"ERROR: Failed to create indexes: {exc}", file=sys.stderr) + return 1 + else: + print( + "\nTip: re-run with --create-indexes to create missing indexes automatically.", + file=sys.stderr, + ) + return 1 + else: + print("All required indexes are present.") + + print("\n# To reuse these DB details in subsequent calls, export:") + print(f"# export {_ENV_DB_URL}='{mongo_url}'") + if credentials: + print(f"# export {_ENV_DB_CREDENTIALS}='{json.dumps(credentials)}'") + + return 0 + + +# --------------------------------------------------------------------------- +# Sub-command: get +# --------------------------------------------------------------------------- + + +def cmd_get(args) -> int: + """Fetch and pretty-print a feature status document by ObjectId.""" + from mas.devops.feature_status import get_feature_status_by_id + + mongo_url, credentials = _resolve_db(args) + + try: + doc = get_feature_status_by_id(mongo_url, args.id, credentials) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + except Exception as exc: + print(f"ERROR: Could not connect to MongoDB: {exc}", file=sys.stderr) + return 1 + + if doc is None: + print(f"No document found with ID: {args.id}", file=sys.stderr) + return 1 + + print(json.dumps(doc, indent=2, default=str)) + return 0 + + +# --------------------------------------------------------------------------- +# Sub-command: status-update +# --------------------------------------------------------------------------- + + +def cmd_status_update(args) -> int: + """Upsert a feature status document into MongoDB.""" + from mas.devops.feature_status import ( + VALID_STATUSES, + _redact_url, + upsert_feature_status, + validate_feature_details, + ) + + # Validate status enum + if args.status not in VALID_STATUSES: + print(f"ERROR: --status must be one of {sorted(VALID_STATUSES)}, got '{args.status}'", file=sys.stderr) + return 1 + + # Parse JSON arguments + feature_details = _parse_json_arg(args.feature_details, "feature-details") + status_details = _parse_json_arg(args.status_details, "status-details") + + # Type-specific feature_details validation + try: + validate_feature_details(args.type, feature_details) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + # Parse datetime arguments + deployment_start = _parse_isodate(args.deployment_start, "deployment-start") + deployment_end = _parse_isodate(args.deployment_end, "deployment-end") + created_at = _parse_isodate(args.created_at, "created-at") + updated_at = _parse_isodate(args.updated_at, "updated-at") + + # Resolve DB details + mongo_url, credentials = _resolve_db(args) + + print(f"Writing feature status: account={args.account} cluster={args.cluster} " f"instance={args.instance_id} type={args.type} status={args.status}") + print(f" MongoDB: {_redact_url(mongo_url)}") + + try: + doc_id = upsert_feature_status( + mongo_url, + region=args.region, + instance_id=args.instance_id, + account=args.account, + cluster=args.cluster, + subscription_id=args.subscription_id, + feature_type=args.type, + feature_details=feature_details, + status=args.status, + status_details=status_details, + deployment_start=deployment_start, + deployment_end=deployment_end, + created_at=created_at, + updated_at=updated_at, + credentials=credentials, # pragma: allowlist secret + ) + print(f"Feature status written successfully. Document ID: {doc_id}") + return 0 + except ValueError as exc: + print(f"ERROR: Validation failed — {exc}", file=sys.stderr) + return 1 + except Exception as exc: + print(f"ERROR: Failed to write feature status to MongoDB: {exc}", file=sys.stderr) + return 1 + + +# --------------------------------------------------------------------------- +# Argument parser +# --------------------------------------------------------------------------- + + +def _add_db_args(parser: argparse.ArgumentParser) -> None: + """Add the shared --db-details / --db-url arguments to a sub-parser.""" + g = parser.add_argument_group("database connection") + g.add_argument( + "--db-details", + required=False, + default=None, + metavar="JSON", + help=( + 'JSON object with "url" and optional "credentials" keys. ' + 'Example: \'{"url": "mongodb://host:27017", "credentials": {"username": "u", "password": "p"}}\'' # pragma: allowlist secret + ), + ) + g.add_argument( + "--db-url", + required=False, + default=None, + metavar="URL", + help=f"MongoDB connection URL (alternative to --db-details). Can also be set via {_ENV_DB_URL}.", + ) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="mas-devops-feature-status-update", + description="Write MAS feature status records to the DevOps MongoDB.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.add_argument( + "--log-level", + required=False, + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + default="WARNING", + help="Python logging level (default: WARNING)", + ) + + subparsers = parser.add_subparsers(dest="command", metavar="") + subparsers.required = True + + # ── prep ────────────────────────────────────────────────────────────────── + prep = subparsers.add_parser( + "prep", + help="Verify MongoDB connection and check/create required indexes.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + _add_db_args(prep) + prep.add_argument( + "--create-indexes", + action="store_true", + default=False, + help="Create missing indexes automatically instead of failing with a warning.", + ) + + # ── status-update ───────────────────────────────────────────────────────── + su = subparsers.add_parser( + "status-update", + help="Upsert a feature status document into mas_devops.feature_status.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + + identity = su.add_argument_group("identity") + identity.add_argument("--region", required=True, help="AWS region (e.g. us-east-2)") + identity.add_argument("--instance-id", required=True, dest="instance_id", help="MAS instance ID (e.g. inst02)") + identity.add_argument("--account", required=True, help="GitOps account name (e.g. fyre-noble10-dev)") + identity.add_argument("--cluster", required=True, help="GitOps cluster name (e.g. noble10)") + identity.add_argument("--subscription-id", required=True, dest="subscription_id", help="Subscription ID") + + feature = su.add_argument_group("feature") + feature.add_argument( + "--type", + required=True, + help="Feature type (e.g. allow-list). Drives feature_details validation.", + ) + feature.add_argument( + "--feature-details", + required=True, + dest="feature_details", + metavar="JSON", + help="JSON object with type-specific fields. allow-list requires 'ips'.", + ) + + status = su.add_argument_group("status") + status.add_argument( + "--status", + required=True, + choices=["REQUESTED", "IN_PROGRESS", "ACTIVE", "ERROR"], + help="Feature lifecycle status.", + ) + status.add_argument( + "--status-details", + required=True, + dest="status_details", + metavar="JSON", + help=( + "JSON object describing the outcome. " + "ACTIVE: {message, request_configuration}. " + "ERROR: {message, error_code, error_source, request_configuration}." + ), + ) + + timestamps = su.add_argument_group("timestamps (all optional, default: now)") + timestamps.add_argument("--deployment-start", required=False, default=None, dest="deployment_start", metavar="ISO-8601") + timestamps.add_argument("--deployment-end", required=False, default=None, dest="deployment_end", metavar="ISO-8601") + timestamps.add_argument("--created-at", required=False, default=None, dest="created_at", metavar="ISO-8601", help="Used only on document insert.") + timestamps.add_argument("--updated-at", required=False, default=None, dest="updated_at", metavar="ISO-8601") + + _add_db_args(su) + + # ── get ─────────────────────────────────────────────────────────────────── + get = subparsers.add_parser( + "get", + help="Fetch and print a feature status document by its ObjectId.", + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + get.add_argument( + "id", + metavar="OBJECT_ID", + help="24-character hex ObjectId of the document (e.g. 6ab0e70ee6d3a31faa808547).", + ) + _add_db_args(get) + + return parser + + +# --------------------------------------------------------------------------- +# Entry point +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + parser = build_parser() + args = parser.parse_args() + + log_level = getattr(logging, args.log_level) + logging.basicConfig(format="%(levelname)s %(name)s: %(message)s") + logging.getLogger("mas.devops.feature_status").setLevel(log_level) + + if args.command == "prep": + sys.exit(cmd_prep(args)) + elif args.command == "status-update": + sys.exit(cmd_status_update(args)) + elif args.command == "get": + sys.exit(cmd_get(args)) + else: + parser.print_help() + sys.exit(1) diff --git a/mongodb_schemas/cluster_level_config.js b/mongodb_schemas/cluster_level_config.js new file mode 100644 index 00000000..75e59936 --- /dev/null +++ b/mongodb_schemas/cluster_level_config.js @@ -0,0 +1,101 @@ +// ============================================================================= +// Collection: cluster_level_config +// Database: feature_dashboard +// Purpose: Stores cluster-scoped feature entries. Each document represents +// one cluster within a region/account/tenant and holds the list of +// cluster-level features enabled for that cluster (e.g. 'dro'). +// +// Split from allowlisting_config: cluster_level_features[] was +// previously embedded inside the deep +// account → regions[] → clusters[] hierarchy. +// Flattening to one document per cluster simplifies writes and +// allows targeted index coverage without touching instance data. +// +// Document cardinality: +// ONE document per (tenant_id × account × region × cluster). +// ============================================================================= + +db.createCollection("cluster_level_config", { + validator: { + $jsonSchema: { + bsonType: "object", + required: [ + "_id", "tenant_id", "account", "region", "cluster", + "cluster_level_features", "created_at", "updated_at" + ], + additionalProperties: false, + properties: { + + // ------------------------------------------------------------------ + // Document identity + // ------------------------------------------------------------------ + _id: { + bsonType: "objectId", + description: "MongoDB-generated document identifier." + }, + tenant_id: { + bsonType: "string", + description: "Tenant/customer identifier. All customer-scoped queries MUST filter on this field. This is the multi-tenancy isolation key." + }, + account: { + bsonType: "string", + description: "Account identifier as returned by the cluster polling mechanism." + }, + region: { + bsonType: "string", + description: "Cloud/geographic region identifier. Example: 'us-east-1'." + }, + cluster: { + bsonType: "string", + description: "Cluster name or identifier as returned by the cluster polling mechanism." + }, + + // ------------------------------------------------------------------ + // Cluster-level features (e.g. dro) + // ------------------------------------------------------------------ + cluster_level_features: { + bsonType: "array", + description: "List of cluster-scoped feature entries. Each element represents one feature enabled at the cluster level (e.g. 'dro').", + minItems: 0, + items: { + bsonType: "object", + additionalProperties: true + } + }, + + // ------------------------------------------------------------------ + // Document-level audit timestamps + // ------------------------------------------------------------------ + created_at: { + bsonType: "date", + description: "UTC timestamp when this document was first created." + }, + updated_at: { + bsonType: "date", + description: "UTC timestamp of the most recent modification to this document." + } + + } + } + }, + validationLevel: "strict", + validationAction: "error" +}); + +// --------------------------------------------------------------------------- +// Indexes +// --------------------------------------------------------------------------- + +// Compound unique index — enforces the one-document-per +// (tenant × account × region × cluster) invariant +db.cluster_level_config.createIndex( + { tenant_id: 1, account: 1, region: 1, cluster: 1 }, + { unique: true, name: "ux_cluster_level_config_tenant_account_region_cluster" } +); + +// Index for querying all clusters for a given tenant + account +db.cluster_level_config.createIndex( + { tenant_id: 1, account: 1 }, + { name: "ix_cluster_level_config_tenant_account" } +); + diff --git a/mongodb_schemas/init_db.js b/mongodb_schemas/init_db.js new file mode 100644 index 00000000..a282b6fb --- /dev/null +++ b/mongodb_schemas/init_db.js @@ -0,0 +1,25 @@ +// ============================================================================= +// init_db.js — Bootstrap script for the feature_dashboard database +// Database: feature_dashboard +// +// Initialises all collections and their indexes: +// • cluster_level_config — cluster-scoped feature entries +// • instance_level_config — instance-scoped allowlisting entries +// +// Usage (mongosh): +// mongosh "mongodb://:27017/feature_dashboard" init_db.js +// +// Usage (legacy mongo shell): +// mongo "mongodb://:27017/feature_dashboard" init_db.js +// ============================================================================= + +const scriptDir = __dirname ?? (function() { + const parts = __filename.split("/"); + parts.pop(); + return parts.join("/"); +})(); + +load(scriptDir + "/cluster_level_config.js"); +load(scriptDir + "/instance_level_config.js"); + +print("✅ feature_dashboard: cluster_level_config and instance_level_config collections and indexes initialized."); diff --git a/mongodb_schemas/instance_level_config.js b/mongodb_schemas/instance_level_config.js new file mode 100644 index 00000000..db4c9ab7 --- /dev/null +++ b/mongodb_schemas/instance_level_config.js @@ -0,0 +1,286 @@ +// ============================================================================= +// Collection: instance_level_config +// Database: feature_dashboard +// Purpose: Stores instance-scoped feature entries. Each document represents +// one instance within a cluster/region/account/tenant and holds the +// list of per-feature state for that instance (IP/CIDR lists, +// deployment lifecycle, audit metadata, etc.). +// +// Split from allowlisting_config: instance_level_features[] was +// previously embedded at the deepest leaf of the +// account → regions[] → clusters[] → instances[] hierarchy. +// Flattening to one document per instance enables efficient +// targeted upserts by ansible-devops and GitHub webhook writes, +// precise index coverage for status detection, and independent +// scaling of cluster and instance data. +// +// Document cardinality: +// ONE document per (tenant_id × subscription_id × account × region × cluster × instance). +// +// instance_level_features item shape (flattened — no nested allow_lists[]): +// +// SUCCESS scenario +// ---------------- +// { +// type: "allow-list", +// feature_details: { ips: ["2405:201:d000:9062::/64"] }, +// status: "ACTIVE", +// status_details: { +// message: "Allow list is active.", +// request_configuration: "2405:201:d000:9062::/64" +// }, +// deployment_start: "2026-09-11T11:48:42.863523+00:00", +// deployment_end: "2026-09-11T11:48:42.863523+00:00", +// source: "admin_ui", +// created_at: ISODate(...), +// updated_at: ISODate(...) +// } +// +// FAILURE scenario +// ---------------- +// { +// type: "allow-list", +// feature_details: { ips: ["2405:201:d000:9062::/64"] }, +// status: "ERROR", +// status_details: { +// message: "sample error message", +// error_code: 401, +// error_source: { +// gitops_version: "8.6.0", +// filename: "...", +// line_no: 1, +// log_file: "...", +// stacktrace: "..." +// }, +// request_configuration: "2405:201:d000:9060::/64" +// }, +// deployment_start: "2026-09-11T11:48:42.863523+00:00", +// deployment_end: "2026-09-11T11:48:42.863523+00:00", +// source: "admin_ui", +// created_at: ISODate(...), +// updated_at: ISODate(...) +// } +// ============================================================================= + +db.createCollection("instance_level_config", { + validator: { + $jsonSchema: { + bsonType: "object", + required: [ + "_id", "tenant_id", "subscription_id", "account", "region", + "cluster", "instance", "instance_level_features", "created_at", "updated_at" + ], + additionalProperties: false, + properties: { + + // ------------------------------------------------------------------ + // Document identity + // ------------------------------------------------------------------ + _id: { + bsonType: "objectId", + description: "MongoDB-generated document identifier." + }, + tenant_id: { + bsonType: "string", + description: "Tenant/customer identifier. All customer-scoped queries MUST filter on this field. This is the multi-tenancy isolation key." + }, + subscription_id: { + bsonType: "string", + description: "Subscription identifier associated with the tenant/instance. Example: 'sub-id01'." + }, + account: { + bsonType: "string", + description: "Account identifier as returned by the cluster polling mechanism." + }, + region: { + bsonType: "string", + description: "Cloud/geographic region identifier. Example: 'us-east-2'." + }, + cluster: { + bsonType: "string", + description: "Cluster name or identifier as returned by the cluster polling mechanism." + }, + instance: { + bsonType: "string", + description: "Specific instance identifier within the cluster." + }, + + // ------------------------------------------------------------------ + // Instance-level features — one entry per deployed feature + // ------------------------------------------------------------------ + instance_level_features: { + bsonType: "array", + description: "List of feature-level entries for this instance. Each element represents one deployed feature (e.g. allow-list) with its full deployment lifecycle state.", + minItems: 0, + items: { + bsonType: "object", + required: [ + "type", "feature_details", "status", + "source", "created_at", "updated_at" + ], + additionalProperties: false, + properties: { + + // Feature type discriminator + type: { + bsonType: "string", + enum: ["allow-list"], + description: "Feature type. Determines the shape of feature_details. Currently only 'allow-list' is supported." + }, + + // ---- feature payload ---------------------------------------- + feature_details: { + bsonType: "object", + description: "Feature-specific configuration payload. Shape depends on 'type'.", + required: ["ips"], + additionalProperties: false, + properties: { + ips: { + bsonType: "array", + description: "List of IPv4/IPv6 addresses or CIDR ranges to allowlist. Example: ['2405:201:d000:9060::/64', '10.0.0.0/8'].", + minItems: 1, + items: { + bsonType: "string", + description: "A single IPv4/IPv6 address or CIDR range." + } + } + } + }, + + // ---- deployment lifecycle ------------------------------------ + status: { + bsonType: "string", + enum: ["REQUESTED", "IN_PROGRESS", "ACTIVE", "ERROR"], + description: "Deployment lifecycle state of this feature entry." + }, + + status_details: { + bsonType: "object", + description: "Additional context for the current status. Present for ACTIVE and ERROR; may be omitted for REQUESTED/IN_PROGRESS.", + additionalProperties: false, + properties: { + + message: { + bsonType: "string", + description: "Human-readable status message. ACTIVE: confirmation text. ERROR: error description." + }, + + // ERROR-only fields + error_code: { + bsonType: "int", + description: "Numeric HTTP/application error code. Set only when status = ERROR. Example: 401." + }, + error_source: { + bsonType: "object", + description: "Machine-readable origin of the error. Set only when status = ERROR.", + additionalProperties: false, + properties: { + gitops_version: { + bsonType: "string", + description: "GitOps toolchain version that processed this entry. Example: '8.6.0'." + }, + filename: { + bsonType: "string", + description: "Source filename in the GitOps pipeline where the error originated." + }, + line_no: { + bsonType: "int", + description: "Line number within 'filename' where the error was raised." + }, + log_file: { + bsonType: "string", + description: "Path or reference to the log file capturing the error output." + }, + stacktrace: { + bsonType: "string", + description: "Full stacktrace string captured at the point of failure." + } + } + }, + + // Common field for ACTIVE and ERROR + request_configuration: { + bsonType: "string", + description: "Verbatim echo of the originally submitted IP/CIDR value that was processed. Aids reconciliation when the applied value differs from what was requested." + } + + } + }, + + deployment_start: { + bsonType: "string", + description: "ISO-8601 timestamp set when the deployment pipeline begins processing this feature entry (status → IN_PROGRESS)." + }, + deployment_end: { + bsonType: "string", + description: "ISO-8601 timestamp set when the pipeline completes, whether successfully (ACTIVE) or with failure (ERROR). Null while the pipeline is still running." + }, + + // ---- audit --------------------------------------------------- + source: { + bsonType: "string", + enum: ["ansible_devops", "github_webhook", "cluster_poll", "admin_ui"], + description: "The system that last wrote this feature entry. Used for audit traceability and reconciliation." + }, + created_at: { + bsonType: "date", + description: "UTC timestamp when this feature entry was first created." + }, + updated_at: { + bsonType: "date", + description: "UTC timestamp of the most recent modification to this feature entry." + } + + } // end instance_level_features item properties + } // end instance_level_features items + }, // end instance_level_features array + + // ------------------------------------------------------------------ + // Document-level audit timestamps + // ------------------------------------------------------------------ + created_at: { + bsonType: "date", + description: "UTC timestamp when this document was first created." + }, + updated_at: { + bsonType: "date", + description: "UTC timestamp of the most recent modification to this document." + } + + } + } + }, + validationLevel: "strict", + validationAction: "error" +}); + +// --------------------------------------------------------------------------- +// Indexes +// --------------------------------------------------------------------------- + +// Compound unique index — enforces the one-document-per +// (tenant × subscription × account × region × cluster × instance) invariant +db.instance_level_config.createIndex( + { tenant_id: 1, subscription_id: 1, account: 1, region: 1, cluster: 1, instance: 1 }, + { unique: true, name: "ux_instance_level_config_tenant_sub_account_region_cluster_instance" } +); + +// Index for ansible-devops / GitHub webhook upserts — primary lookup path +db.instance_level_config.createIndex( + { tenant_id: 1, subscription_id: 1, account: 1, region: 1, cluster: 1 }, + { name: "ix_instance_level_config_tenant_sub_account_region_cluster" } +); + +// Multikey index on feature status — supports finding all documents with +// at least one entry in a given lifecycle state (e.g. IN_PROGRESS or ERROR) +db.instance_level_config.createIndex( + { "instance_level_features.status": 1 }, + { name: "ix_instance_level_config_feature_status" } +); + +// Sparse index for error triage — only indexes documents that have at least +// one ERROR entry; avoids index bloat for the common ACTIVE/non-error case +db.instance_level_config.createIndex( + { "instance_level_features.status_details.error_code": 1 }, + { sparse: true, name: "ix_instance_level_config_error_code" } +); diff --git a/setup.py b/setup.py index fa0fcc1d..6049b278 100644 --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ def get_version(rel_path): "boto3", # Apache Software License "slack_sdk", # MIT License "packaging", # Apache Software License + "pymongo", # Apache Software License ], extras_require={ "dev": [ @@ -93,5 +94,6 @@ def get_version(rel_path): "bin/mas-devops-saas-job-cleaner", "bin/mas-devops-notify-slack", "bin/mas-devops-apply-preinstall-rbac-for-saas", + "bin/mas-devops-feature-status-update", ], ) diff --git a/src/mas/devops/feature_status.py b/src/mas/devops/feature_status.py new file mode 100644 index 00000000..357424c2 --- /dev/null +++ b/src/mas/devops/feature_status.py @@ -0,0 +1,328 @@ +# ***************************************************************************** +# Copyright (c) 2025 IBM Corporation and other Contributors. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# ***************************************************************************** +""" +feature_status.py — Write feature status records into the DevOps MongoDB. + +Supports two operations: + + prep — Verify the MongoDB connection and confirm the expected indexes + (instance_config_level, cluster_config_level) exist on the + target collection. Stores db-details for later use in an + environment variable so they do not need to be repeated on + every status-update call. + + status_update — Upsert a feature status document into + ``mas_devops.feature_status``. + +Collection: ``mas_devops.feature_status`` + +Document schema (mirrors the CIS allowlist status tracking design): + + { + "_id": , + "schema_version": 1, + "region": str, + "instance_id": str, + "account": str, + "cluster": str, + "subscription_id": str, + "type": str, # e.g. "allow-list" + "feature_details": dict, # type-specific payload + "status": str, # REQUESTED | IN_PROGRESS | ACTIVE | ERROR + "status_details": dict, # message, error_code, error_source, … + "deployment_start": datetime, + "deployment_end": datetime | None, + "created_at": datetime, + "updated_at": datetime, + } + +Indexes expected on the collection + • ``instance_config_level`` — compound: region + instance_id + account + • ``cluster_config_level`` — compound: region + cluster + account +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timezone +from typing import Any, Optional + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +COLLECTION = "feature_status" +DATABASE = "mas_devops" + +# Status enum values (matches the CIS allowlist lifecycle) +STATUS_REQUESTED = "REQUESTED" +STATUS_IN_PROGRESS = "IN_PROGRESS" +STATUS_ACTIVE = "ACTIVE" +STATUS_ERROR = "ERROR" + +VALID_STATUSES = {STATUS_REQUESTED, STATUS_IN_PROGRESS, STATUS_ACTIVE, STATUS_ERROR} + +# Required index names that must exist on the collection. +REQUIRED_INDEX_NAMES = {"instance_config_level", "cluster_config_level"} + +# --------------------------------------------------------------------------- +# Per-type feature_details validators +# --------------------------------------------------------------------------- + +# Each key maps to the set of field names that MUST be present in feature_details +# when --type matches that key. +_FEATURE_DETAILS_REQUIRED_FIELDS: dict[str, set[str]] = { + "allow-list": {"ips"}, +} + + +def validate_feature_details(feature_type: str, feature_details: dict) -> None: + """Validate that *feature_details* contains the required keys for *feature_type*. + + Raises: + ValueError: if required keys are missing or feature_details is not a dict. + """ + if not isinstance(feature_details, dict): + raise ValueError(f"feature_details must be a JSON object, got {type(feature_details).__name__}") + + required = _FEATURE_DETAILS_REQUIRED_FIELDS.get(feature_type) + if required is None: + # Unknown type — no field-level validation, but emit a warning. + logger.warning("No feature_details validation rules defined for type '%s'", feature_type) + return + + missing = required - set(feature_details.keys()) + if missing: + raise ValueError(f"feature_details is missing required field(s) for type '{feature_type}': {sorted(missing)}") + + +# --------------------------------------------------------------------------- +# MongoDB helpers +# --------------------------------------------------------------------------- + + +def _get_client(mongo_url: str, credentials: Optional[dict] = None): + """Return a pymongo MongoClient for *mongo_url*. + + Credentials dict may contain ``username`` and ``password`` keys. + If the URL already embeds credentials they take precedence. + """ + try: + from pymongo import MongoClient # type: ignore + except ImportError as exc: # pragma: no cover + raise ImportError("pymongo is required. Install it with: pip install pymongo") from exc + + kwargs: dict[str, Any] = {"serverSelectionTimeoutMS": 10_000} + if credentials: + if "username" in credentials: + kwargs["username"] = credentials["username"] + if "password" in credentials: + kwargs["password"] = credentials["password"] + if "authSource" in credentials: + kwargs["authSource"] = credentials["authSource"] + if "tls" in credentials: + kwargs["tls"] = credentials["tls"] + + return MongoClient(mongo_url, **kwargs) + + +def verify_connection_and_indexes(mongo_url: str, credentials: Optional[dict] = None) -> list[str]: + """Connect to MongoDB and check that the expected indexes exist. + + Returns a list of warning messages for any missing indexes. + Raises on connection failure. + """ + client = _get_client(mongo_url, credentials) + try: + # Ping — will raise if the server is unreachable. + client.admin.command("ping") + logger.info("MongoDB connection OK: %s", _redact_url(mongo_url)) + + db = client[DATABASE] + collection = db[COLLECTION] + + # Retrieve existing index names. + existing_index_names = {info["name"] for info in collection.list_indexes()} + + warnings = [] + for expected in REQUIRED_INDEX_NAMES: + if expected not in existing_index_names: + warnings.append( + f"Index '{expected}' not found on {DATABASE}.{COLLECTION}. " f"Run the index-creation script or use 'prep' with --create-indexes." + ) + return warnings + finally: + client.close() + + +def create_indexes(mongo_url: str, credentials: Optional[dict] = None) -> None: + """Create the required indexes on the feature_status collection if they do not exist.""" + try: + from pymongo import ASCENDING # type: ignore + except ImportError as exc: # pragma: no cover + raise ImportError("pymongo is required. Install it with: pip install pymongo") from exc + + client = _get_client(mongo_url, credentials) + try: + db = client[DATABASE] + collection = db[COLLECTION] + + collection.create_index( + [("region", ASCENDING), ("instance_id", ASCENDING), ("account", ASCENDING)], + name="instance_config_level", + background=True, + ) + logger.info("Index 'instance_config_level' ensured on %s.%s", DATABASE, COLLECTION) + + collection.create_index( + [("region", ASCENDING), ("cluster", ASCENDING), ("account", ASCENDING)], + name="cluster_config_level", + background=True, + ) + logger.info("Index 'cluster_config_level' ensured on %s.%s", DATABASE, COLLECTION) + finally: + client.close() + + +def upsert_feature_status( + mongo_url: str, + *, + region: str, + instance_id: str, + account: str, + cluster: str, + subscription_id: str, + feature_type: str, + feature_details: dict, + status: str, + status_details: dict, + deployment_start: Optional[datetime] = None, + deployment_end: Optional[datetime] = None, + created_at: Optional[datetime] = None, + updated_at: Optional[datetime] = None, + credentials: Optional[dict] = None, +) -> str: + """Upsert a feature status document. Returns the upserted / matched document ID as a string. + + The upsert key is ``(region, instance_id, account, cluster, type)``. + On insert ``created_at`` is set; ``updated_at`` is always refreshed. + """ + try: + from pymongo import ReturnDocument # type: ignore + except ImportError as exc: # pragma: no cover + raise ImportError("pymongo is required. Install it with: pip install pymongo") from exc + + if status not in VALID_STATUSES: + raise ValueError(f"Invalid status '{status}'. Must be one of {sorted(VALID_STATUSES)}") + + validate_feature_details(feature_type, feature_details) + + now = datetime.now(timezone.utc) + deployment_start = deployment_start or now + updated_at = updated_at or now + created_at = created_at or now + + filter_doc = { + "region": region, + "instance_id": instance_id, + "account": account, + "cluster": cluster, + "type": feature_type, + } + + update_doc = { + "$set": { + "subscription_id": subscription_id, + "feature_details": feature_details, + "status": status, + "status_details": status_details, + "deployment_start": deployment_start, + "deployment_end": deployment_end, + "updated_at": updated_at, + "schema_version": 1, + }, + "$setOnInsert": { + "created_at": created_at, + }, + } + + client = _get_client(mongo_url, credentials) + try: + db = client[DATABASE] + collection = db[COLLECTION] + result = collection.find_one_and_update( + filter_doc, + update_doc, + upsert=True, + return_document=ReturnDocument.AFTER, + ) + doc_id = str(result["_id"]) + logger.info( + "Feature status upserted [%s / %s / %s] status=%s id=%s", + account, + instance_id, + feature_type, + status, + doc_id, + ) + return doc_id + finally: + client.close() + + +def get_feature_status_by_id(mongo_url: str, doc_id: str, credentials: Optional[dict] = None) -> Optional[dict]: + """Fetch a single feature status document by its ObjectId string. + + Args: + mongo_url (str): MongoDB connection URL. + doc_id (str): Hex string ObjectId of the document to retrieve. + credentials (dict, optional): Optional credential overrides. Defaults to None. + + Returns: + dict: The document with ``_id`` serialised to a string, or None if not found. + + Raises: + ValueError: If *doc_id* is not a valid 24-character hex ObjectId. + pymongo.errors.ConnectionFailure: If the MongoDB server is unreachable. + """ + try: + from bson import ObjectId + from bson.errors import InvalidId + except ImportError as exc: # pragma: no cover + raise ImportError("pymongo is required. Install it with: pip install pymongo") from exc + + try: + oid = ObjectId(doc_id) + except InvalidId: + raise ValueError(f"'{doc_id}' is not a valid ObjectId (expected a 24-character hex string)") + + client = _get_client(mongo_url, credentials) + try: + doc = client[DATABASE][COLLECTION].find_one({"_id": oid}) + if doc is None: + return None + doc["_id"] = str(doc["_id"]) + return doc + finally: + client.close() + + +# --------------------------------------------------------------------------- +# URL redaction helper (keeps passwords out of logs) +# --------------------------------------------------------------------------- + + +def _redact_url(url: str) -> str: + """Replace the password component of a MongoDB connection URI with *****.""" + import re + + return re.sub(r"(mongodb(?:\+srv)?://[^:]+:)[^@]+(@)", r"\1*****\2", url) From e254d00ee09aefcffb6a0047331fccf1ee9cfb63 Mon Sep 17 00:00:00 2001 From: Sagar-Talikoti Date: Tue, 29 Sep 2026 22:49:26 +0530 Subject: [PATCH 04/10] fix for idempotent call --- bin/mas-devops-feature-status-update.md | 8 +++- mongodb_schemas/README.md | 57 +++++++++++++++++++++++++ src/mas/devops/feature_status.py | 2 - 3 files changed, 64 insertions(+), 3 deletions(-) diff --git a/bin/mas-devops-feature-status-update.md b/bin/mas-devops-feature-status-update.md index fad10038..249cb9cb 100644 --- a/bin/mas-devops-feature-status-update.md +++ b/bin/mas-devops-feature-status-update.md @@ -148,6 +148,8 @@ Verifies MongoDB connectivity and confirms that the required indexes exist on th Pass `--create-indexes` to create missing indexes automatically instead of exiting with an error. +**Idempotency:** Safe to run repeatedly. The connectivity check is read-only. When `--create-indexes` is passed, `create_index` is a no-op for any index that already exists — it will never drop or recreate an existing index. + **Options** | Flag | Required | Description | @@ -181,9 +183,11 @@ After a successful `prep` run the command prints the `export` statements needed ### `status-update` -Upserts a feature status document. +Upserts a feature status document. Upsert key: `(region, instance_id, account, cluster, type)` — an existing document is updated in-place; a new document is inserted if no match is found. +**Idempotency:** Safe to call multiple times with the same arguments. The underlying `find_one_and_update` with `upsert=True` guarantees that re-running with the same identity key produces the same final document state. `created_at` is set only on the first insert (`$setOnInsert`); subsequent calls update `updated_at` and all mutable fields without creating duplicate documents. + **Identity options** *(all required)* | Flag | Description | @@ -341,6 +345,8 @@ mas-devops-feature-status-update status-update \ Fetches a single feature status document by its ObjectId and prints it as formatted JSON. +**Idempotency:** Read-only. Safe to call any number of times with no side effects. + **Arguments** | Argument | Required | Description | diff --git a/mongodb_schemas/README.md b/mongodb_schemas/README.md index 59e31fd6..c05e15b5 100644 --- a/mongodb_schemas/README.md +++ b/mongodb_schemas/README.md @@ -489,6 +489,44 @@ db.instance_level_config.aggregate([ mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/init_db.js ``` +> **Idempotency:** `db.createCollection()` raises a `MongoServerError: Collection already exists` error if the collection is already present. The initialization scripts are **not safe to re-run** against an existing database. Use the safe re-initialization pattern below if you need to ensure indexes are up to date without dropping data. + +### Safe re-initialization (collections already exist) + +If the collections already exist and you only need to ensure indexes are up to date, run `createIndex` calls directly — they are no-ops when the index name and definition already match: + +```js +use feature_dashboard + +// cluster_level_config indexes +db.cluster_level_config.createIndex( + { tenant_id: 1, account: 1, region: 1, cluster: 1 }, + { unique: true, name: "ux_cluster_level_config_tenant_account_region_cluster" } +); +db.cluster_level_config.createIndex( + { tenant_id: 1, account: 1 }, + { name: "ix_cluster_level_config_tenant_account" } +); + +// instance_level_config indexes +db.instance_level_config.createIndex( + { tenant_id: 1, subscription_id: 1, account: 1, region: 1, cluster: 1, instance: 1 }, + { unique: true, name: "ux_instance_level_config_tenant_sub_account_region_cluster_instance" } +); +db.instance_level_config.createIndex( + { tenant_id: 1, subscription_id: 1, account: 1, region: 1, cluster: 1 }, + { name: "ix_instance_level_config_tenant_sub_account_region_cluster" } +); +db.instance_level_config.createIndex( + { "instance_level_features.status": 1 }, + { name: "ix_instance_level_config_feature_status" } +); +db.instance_level_config.createIndex( + { "instance_level_features.status_details.error_code": 1 }, + { sparse: true, name: "ix_instance_level_config_error_code" } +); +``` + ### Clear the collections ```js @@ -497,6 +535,8 @@ db.cluster_level_config.deleteMany({}) db.instance_level_config.deleteMany({}) ``` +> **Idempotency:** Safe to run repeatedly — `deleteMany({})` is a no-op when the collection is already empty. + ### Drop the collections ```js @@ -506,6 +546,7 @@ db.instance_level_config.drop() ``` > **Note:** `drop()` removes the collection, all its documents, and its indexes. Re-run `init_db.js` to recreate them. +> **Idempotency:** Not idempotent — `drop()` raises an error if the collection does not exist. Re-running `init_db.js` after a drop is safe because the collections no longer exist at that point. ### Run a schema file directly @@ -514,6 +555,22 @@ mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/cluster_level mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/instance_level_config.js ``` +> **Idempotency:** Same caveat as `init_db.js` — each file calls `db.createCollection()`, which fails if the collection already exists. Only run against a fresh or dropped database. + +--- + +## Idempotency reference + +| Operation | Idempotent | Notes | +|---|---|---| +| `init_db.js` (full init) | ❌ | `db.createCollection()` fails if the collection already exists. Only run against a fresh or dropped database. | +| `cluster_level_config.js` | ❌ | Same — calls `db.createCollection()`. | +| `instance_level_config.js` | ❌ | Same — calls `db.createCollection()`. | +| `createIndex` (standalone) | ✅ | No-op when an index with the same name and definition already exists. Safe to run on a live collection. | +| `deleteMany({})` (clear) | ✅ | No-op on an already-empty collection. | +| `drop()` | ❌ | Errors if the collection does not exist. | +| Document upserts via `mas-devops-feature-status-update status-update` | ✅ | Uses `find_one_and_update` with `upsert=True`. Repeated calls on the same identity key update in-place; `created_at` is protected by `$setOnInsert`. | + --- ## Validation behaviour diff --git a/src/mas/devops/feature_status.py b/src/mas/devops/feature_status.py index 357424c2..6336f9bc 100644 --- a/src/mas/devops/feature_status.py +++ b/src/mas/devops/feature_status.py @@ -179,14 +179,12 @@ def create_indexes(mongo_url: str, credentials: Optional[dict] = None) -> None: collection.create_index( [("region", ASCENDING), ("instance_id", ASCENDING), ("account", ASCENDING)], name="instance_config_level", - background=True, ) logger.info("Index 'instance_config_level' ensured on %s.%s", DATABASE, COLLECTION) collection.create_index( [("region", ASCENDING), ("cluster", ASCENDING), ("account", ASCENDING)], name="cluster_config_level", - background=True, ) logger.info("Index 'cluster_config_level' ensured on %s.%s", DATABASE, COLLECTION) finally: From 93e2ed5f4a6f131508ca255562dba9d519ef0474 Mon Sep 17 00:00:00 2001 From: Sagar-Talikoti Date: Tue, 29 Sep 2026 23:31:43 +0530 Subject: [PATCH 05/10] fix for get feature by criteria --- bin/mas-devops-feature-status-update | 79 ++++++++++++++++++++++--- bin/mas-devops-feature-status-update.md | 62 ++++++++++++++++--- src/mas/devops/feature_status.py | 49 +++++++++++++++ 3 files changed, 176 insertions(+), 14 deletions(-) diff --git a/bin/mas-devops-feature-status-update b/bin/mas-devops-feature-status-update index 97887013..5c5aae20 100755 --- a/bin/mas-devops-feature-status-update +++ b/bin/mas-devops-feature-status-update @@ -240,13 +240,47 @@ def cmd_prep(args) -> int: def cmd_get(args) -> int: - """Fetch and pretty-print a feature status document by ObjectId.""" - from mas.devops.feature_status import get_feature_status_by_id + """Fetch and pretty-print a feature status document by ObjectId or by criteria fields.""" + from mas.devops.feature_status import get_feature_status_by_criteria, get_feature_status_by_id mongo_url, credentials = _resolve_db(args) try: - doc = get_feature_status_by_id(mongo_url, args.id, credentials) + if args.id: + doc = get_feature_status_by_id(mongo_url, args.id, credentials) + not_found_msg = f"No document found with ID: {args.id}" + else: + # Criteria mode — all companion flags are required. + missing = [ + f + for f, v in [ + ("--instance-id", args.instance_id), + ("--account", args.account), + ("--cluster", args.cluster), + ("--subscription-id", args.subscription_id), + ("--type", args.type), + ] + if v is None + ] + if missing: + print(f"ERROR: the following arguments are required when using --region: {', '.join(missing)}", file=sys.stderr) + return 1 + + doc = get_feature_status_by_criteria( + mongo_url, + region=args.region, + instance_id=args.instance_id, + account=args.account, + cluster=args.cluster, + subscription_id=args.subscription_id, + feature_type=args.type, + credentials=credentials, # pragma: allowlist secret + ) + not_found_msg = ( + f"No document found for region={args.region} instance_id={args.instance_id} " + f"account={args.account} cluster={args.cluster} " + f"subscription_id={args.subscription_id} type={args.type}" + ) except ValueError as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 @@ -255,7 +289,7 @@ def cmd_get(args) -> int: return 1 if doc is None: - print(f"No document found with ID: {args.id}", file=sys.stderr) + print(not_found_msg, file=sys.stderr) return 1 print(json.dumps(doc, indent=2, default=str)) @@ -448,14 +482,45 @@ def build_parser() -> argparse.ArgumentParser: # ── get ─────────────────────────────────────────────────────────────────── get = subparsers.add_parser( "get", - help="Fetch and print a feature status document by its ObjectId.", + help="Fetch and print a feature status document by ObjectId or by criteria fields.", formatter_class=argparse.RawDescriptionHelpFormatter, + description=( + "Retrieve a feature status document using either its ObjectId (--id) or a\n" + "combination of criteria fields (--region, --instance-id, --account,\n" + "--cluster, --subscription-id, --type). Exactly one lookup mode must be\n" + "provided; the two modes are mutually exclusive.\n\n" + "Examples:\n" + " # by ObjectId\n" + " mas-devops-feature-status-update get --id 6ab0e70ee6d3a31faa808547\n\n" + " # by criteria\n" + " mas-devops-feature-status-update get \\\n" + " --region us-east-2 \\\n" + " --instance-id inst02 \\\n" + " --account fyre-noble10-dev \\\n" + " --cluster noble10 \\\n" + " --subscription-id sub-id01 \\\n" + " --type allow-list" + ), ) - get.add_argument( - "id", + get_lookup = get.add_mutually_exclusive_group(required=True) + get_lookup.add_argument( + "--id", + default=None, metavar="OBJECT_ID", help="24-character hex ObjectId of the document (e.g. 6ab0e70ee6d3a31faa808547).", ) + get_lookup.add_argument( + "--region", + default=None, + metavar="REGION", + help="AWS region (e.g. us-east-2). Use together with the other criteria flags.", + ) + get_criteria = get.add_argument_group("criteria (required when --region is used)") + get_criteria.add_argument("--instance-id", dest="instance_id", default=None, metavar="INSTANCE_ID", help="MAS instance ID (e.g. inst02)") + get_criteria.add_argument("--account", default=None, metavar="ACCOUNT", help="GitOps account name (e.g. fyre-noble10-dev)") + get_criteria.add_argument("--cluster", default=None, metavar="CLUSTER", help="GitOps cluster name (e.g. noble10)") + get_criteria.add_argument("--subscription-id", dest="subscription_id", default=None, metavar="SUBSCRIPTION_ID", help="Subscription ID") + get_criteria.add_argument("--type", default=None, metavar="TYPE", help="Feature type (e.g. allow-list)") _add_db_args(get) return parser diff --git a/bin/mas-devops-feature-status-update.md b/bin/mas-devops-feature-status-update.md index 249cb9cb..fe61a1cd 100644 --- a/bin/mas-devops-feature-status-update.md +++ b/bin/mas-devops-feature-status-update.md @@ -343,7 +343,12 @@ mas-devops-feature-status-update status-update \ ### `get` -Fetches a single feature status document by its ObjectId and prints it as formatted JSON. +Fetches a single feature status document and prints it as formatted JSON. + +Two mutually exclusive lookup modes are supported — exactly one must be provided: + +- **`--id`** — look up by ObjectId (the value printed by `status-update` on success). +- **`--region` + criteria flags** — look up by the document's identifying fields. **Idempotency:** Read-only. Safe to call any number of times with no side effects. @@ -351,16 +356,37 @@ Fetches a single feature status document by its ObjectId and prints it as format | Argument | Required | Description | |----------|----------|-------------| -| `OBJECT_ID` | Yes | 24-character hex ObjectId (printed by `status-update` on success) | +| `--id OBJECT_ID` | One of `--id` / `--region` | 24-character hex ObjectId | +| `--region REGION` | One of `--id` / `--region` | AWS region (e.g. `us-east-2`). Enables criteria-based lookup | +| `--instance-id INSTANCE_ID` | Yes (criteria mode) | MAS instance ID (e.g. `inst02`) | +| `--account ACCOUNT` | Yes (criteria mode) | GitOps account name (e.g. `fyre-noble10-dev`) | +| `--cluster CLUSTER` | Yes (criteria mode) | GitOps cluster name (e.g. `noble10`) | +| `--subscription-id SUBSCRIPTION_ID` | Yes (criteria mode) | Subscription ID | +| `--type TYPE` | Yes (criteria mode) | Feature type (e.g. `allow-list`) | | `--db-details JSON` | No† | JSON object with `url` and optional `credentials` keys | | `--db-url URL` | No† | MongoDB connection URL | † At least one of `--db-details`, `--db-url`, or the `MAS_FEATURE_STATUS_DB_URL` environment variable is required. -**Example** +**Example — by ObjectId** + +```bash +mas-devops-feature-status-update get \ + --id 6ab0e70ee6d3a31faa808547 \ + --db-url mongodb://localhost:27017 +``` + +**Example — by criteria** ```bash -mas-devops-feature-status-update get 6ab0e70ee6d3a31faa808547 +mas-devops-feature-status-update get \ + --region us-east-2 \ + --instance-id inst02 \ + --account fyre-noble10-dev \ + --cluster noble10 \ + --subscription-id sub-id01 \ + --type allow-list \ + --db-url mongodb://localhost:27017 ``` **Sample output** @@ -562,7 +588,7 @@ Verify connectivity before any write. Use `--create-indexes` on first run. ### Minimal task — `get` -Extract the document ID from `status-update` output and fetch the written document: +**By ObjectId** — extract the document ID from `status-update` output and fetch the written document: ```yaml - name: Extract document ID @@ -572,12 +598,34 @@ Extract the document ID from `status-update` output and fetch the written docume | regex_search('Document ID: ([a-f0-9]{24})', '\1') | first }} -- name: Fetch feature status document +- name: Fetch feature status document by ID + ansible.builtin.command: + cmd: >- + mas-devops-feature-status-update get + --id {{ mas_document_id }} + --db-url {{ mas_mongo_url }} + register: get_result + changed_when: false + +- name: Display document + ansible.builtin.debug: + msg: "{{ get_result.stdout | from_json }}" +``` + +**By criteria** — look up the document without needing to capture an ObjectId first: + +```yaml +- name: Fetch feature status document by criteria ansible.builtin.command: cmd: >- mas-devops-feature-status-update get + --region {{ mas_region }} + --instance-id {{ mas_instance_id }} + --account {{ mas_account }} + --cluster {{ mas_cluster }} + --subscription-id {{ mas_subscription_id }} + --type allow-list --db-url {{ mas_mongo_url }} - {{ mas_document_id }} register: get_result changed_when: false diff --git a/src/mas/devops/feature_status.py b/src/mas/devops/feature_status.py index 6336f9bc..8dcfa852 100644 --- a/src/mas/devops/feature_status.py +++ b/src/mas/devops/feature_status.py @@ -314,6 +314,55 @@ def get_feature_status_by_id(mongo_url: str, doc_id: str, credentials: Optional[ client.close() +def get_feature_status_by_criteria( + mongo_url: str, + *, + region: str, + instance_id: str, + account: str, + cluster: str, + subscription_id: str, + feature_type: str, + credentials: Optional[dict] = None, +) -> Optional[dict]: + """Fetch a single feature status document by its identifying criteria fields. + + Args: + mongo_url (str): MongoDB connection URL. + region (str): AWS region (e.g. us-east-2). + instance_id (str): MAS instance ID (e.g. inst02). + account (str): GitOps account name (e.g. fyre-noble10-dev). + cluster (str): GitOps cluster name (e.g. noble10). + subscription_id (str): Subscription ID. + feature_type (str): Feature type (e.g. allow-list). + credentials (dict, optional): Optional credential overrides. Defaults to None. + + Returns: + dict: The matching document with ``_id`` serialised to a string, or None if not found. + + Raises: + pymongo.errors.ConnectionFailure: If the MongoDB server is unreachable. + """ + filterDoc = { + "region": region, + "instance_id": instance_id, + "account": account, + "cluster": cluster, + "subscription_id": subscription_id, + "type": feature_type, + } + + client = _get_client(mongo_url, credentials) + try: + doc = client[DATABASE][COLLECTION].find_one(filterDoc) + if doc is None: + return None + doc["_id"] = str(doc["_id"]) + return doc + finally: + client.close() + + # --------------------------------------------------------------------------- # URL redaction helper (keeps passwords out of logs) # --------------------------------------------------------------------------- From c90efb5d97919ab1f9c5ef350a346f62fded94d3 Mon Sep 17 00:00:00 2001 From: Sagar-Talikoti Date: Wed, 30 Sep 2026 16:14:48 +0530 Subject: [PATCH 06/10] update cli to use DEVOPS_MONGO_URI --- bin/mas-devops-feature-status-update | 214 +++++++----------------- bin/mas-devops-feature-status-update.md | 123 ++------------ 2 files changed, 82 insertions(+), 255 deletions(-) diff --git a/bin/mas-devops-feature-status-update b/bin/mas-devops-feature-status-update index 5c5aae20..f966a7fb 100755 --- a/bin/mas-devops-feature-status-update +++ b/bin/mas-devops-feature-status-update @@ -16,20 +16,10 @@ DevOps MongoDB (mas_devops.feature_status collection). Sub-commands ──────────── - prep - Verify MongoDB connectivity and confirm that the required indexes - (instance_config_level, cluster_config_level) exist on the collection. - Pass --create-indexes to create them when absent. - - Example: - mas-devops-feature-status-update prep \\ - --db-details '{"url": "mongodb://:27017", "credentials": {"username": ""}}' #pragma: allowlist secret - - mas-devops-feature-status-update prep --db-url mongodb://host:27017 --create-indexes - - status-update Upsert a feature status document. + Required indexes (instance_config_level, cluster_config_level) are + created automatically on the first call if they do not already exist. Upsert key: (region, instance_id, account, cluster, type). Example — ACTIVE: @@ -69,11 +59,35 @@ Sub-commands "request_configuration": "2405:201:d000:9060::/64" }' + get + Fetch and pretty-print a feature status document by ObjectId or by + criteria fields (region, instance_id, account, cluster, type). + + Example — by ObjectId: + mas-devops-feature-status-update get --id 6ab0e70ee6d3a31faa808547 + + Example — by criteria: + mas-devops-feature-status-update get \\ + --region us-east-2 \\ + --instance-id inst02 \\ + --account fyre-noble10-dev \\ + --cluster noble10 \\ + --subscription-id sub-id01 \\ + --type allow-list + Environment variables ───────────────────── - MAS_FEATURE_STATUS_DB_URL MongoDB connection URL - MAS_FEATURE_STATUS_DB_CREDENTIALS JSON object: username, password, - authSource, tls (all optional) + DEVOPS_MONGO_URI (required) MongoDB connection URI with embedded credentials + and TLS options: + mongodb://user:password@host1:port1,host2:port2/admin?tls=true&tlsAllowInvalidCertificates=true # pragma: allowlist secret + +One-time initialisation +─────────────────────── + No separate setup step is required. The first call to status-update + automatically creates the required collection indexes if they are absent. + For manual initialisation of the database schema and validators, run: + + mongosh "" mongodb_schemas/init_db.js """ import argparse @@ -85,11 +99,10 @@ from datetime import datetime, timezone from typing import Optional # --------------------------------------------------------------------------- -# Env-var names used to persist / retrieve DB details between invocations +# Env-var name used to supply the MongoDB connection URI # --------------------------------------------------------------------------- -_ENV_DB_URL = "MAS_FEATURE_STATUS_DB_URL" -_ENV_DB_CREDENTIALS = "MAS_FEATURE_STATUS_DB_CREDENTIALS" # pragma: allowlist secret +_ENV_DEVOPS_MONGO_URI = "DEVOPS_MONGO_URI" # --------------------------------------------------------------------------- @@ -144,94 +157,22 @@ def _parse_isodate(value: Optional[str], arg_name: str) -> Optional[datetime]: sys.exit(1) -def _resolve_db(args) -> tuple: - """Return (mongo_url, credentials) resolving from CLI args then env vars. +def _resolve_db() -> str: + """Return the MongoDB connection URI from the DEVOPS_MONGO_URI environment variable. - Precedence: - 1. --db-details JSON - 2. --db-url - 3. MAS_FEATURE_STATUS_DB_URL / MAS_FEATURE_STATUS_DB_CREDENTIALS + Raises SystemExit(1) if the variable is unset or empty. """ - db_details = getattr(args, "db_details", None) - db_url_arg = getattr(args, "db_url", None) - - if db_details: - details = _parse_json_arg(db_details, "db-details") - url = details.get("url") or details.get("mongo_url") - if not url: - print("ERROR: --db-details must contain a 'url' key", file=sys.stderr) - sys.exit(1) - credentials = details.get("credentials") or None - return url, credentials - - if db_url_arg: - return db_url_arg, None - - env_url = os.environ.get(_ENV_DB_URL, "") - if env_url: - creds_raw = os.environ.get(_ENV_DB_CREDENTIALS, "") - credentials = json.loads(creds_raw) if creds_raw else None - return env_url, credentials - - print( - "ERROR: MongoDB connection details are required.\n" - " Provide one of:\n" - f' --db-details \'{{"url": "mongodb://..."}}\'\n' - f" --db-url \n" - f" {_ENV_DB_URL} environment variable", - file=sys.stderr, - ) - sys.exit(1) - - -# --------------------------------------------------------------------------- -# Sub-command: prep -# --------------------------------------------------------------------------- - - -def cmd_prep(args) -> int: - """Verify connection and confirm required indexes exist.""" - from mas.devops.feature_status import ( - _redact_url, - create_indexes, - verify_connection_and_indexes, - ) - - mongo_url, credentials = _resolve_db(args) - print(f"Connecting to: {_redact_url(mongo_url)}") - - try: - warnings = verify_connection_and_indexes(mongo_url, credentials) - except Exception as exc: - print(f"ERROR: Could not connect to MongoDB: {exc}", file=sys.stderr) - return 1 - - if warnings: - for w in warnings: - print(f"WARNING: {w}") - if args.create_indexes: - print("Creating missing indexes …") - try: - create_indexes(mongo_url, credentials) - print("Indexes created successfully.") - except Exception as exc: - print(f"ERROR: Failed to create indexes: {exc}", file=sys.stderr) - return 1 - else: - print( - "\nTip: re-run with --create-indexes to create missing indexes automatically.", - file=sys.stderr, - ) - return 1 - else: - print("All required indexes are present.") - - print("\n# To reuse these DB details in subsequent calls, export:") - print(f"# export {_ENV_DB_URL}='{mongo_url}'") - if credentials: - print(f"# export {_ENV_DB_CREDENTIALS}='{json.dumps(credentials)}'") - - return 0 + uri = os.environ.get(_ENV_DEVOPS_MONGO_URI, "") + if not uri: + print( + f"ERROR: {_ENV_DEVOPS_MONGO_URI} environment variable is required.\n" + f" Set it to a full MongoDB connection URI, e.g.:\n" + f" export {_ENV_DEVOPS_MONGO_URI}='mongodb://user:password@host:port/admin" # pragma: allowlist secret + f"?tls=true&tlsAllowInvalidCertificates=true'", + file=sys.stderr, + ) + sys.exit(1) + return uri # --------------------------------------------------------------------------- @@ -243,7 +184,8 @@ def cmd_get(args) -> int: """Fetch and pretty-print a feature status document by ObjectId or by criteria fields.""" from mas.devops.feature_status import get_feature_status_by_criteria, get_feature_status_by_id - mongo_url, credentials = _resolve_db(args) + mongo_url = _resolve_db() + credentials = None try: if args.id: @@ -302,10 +244,15 @@ def cmd_get(args) -> int: def cmd_status_update(args) -> int: - """Upsert a feature status document into MongoDB.""" + """Upsert a feature status document into MongoDB. + + Required indexes are created automatically if they do not already exist, + so no separate 'prep' step is needed. + """ from mas.devops.feature_status import ( VALID_STATUSES, _redact_url, + create_indexes, upsert_feature_status, validate_feature_details, ) @@ -332,12 +279,20 @@ def cmd_status_update(args) -> int: created_at = _parse_isodate(args.created_at, "created-at") updated_at = _parse_isodate(args.updated_at, "updated-at") - # Resolve DB details - mongo_url, credentials = _resolve_db(args) + # Resolve DB URI from environment + mongo_url = _resolve_db() + credentials = None print(f"Writing feature status: account={args.account} cluster={args.cluster} " f"instance={args.instance_id} type={args.type} status={args.status}") print(f" MongoDB: {_redact_url(mongo_url)}") + # Ensure required indexes exist (idempotent — no-op when already present). + try: + create_indexes(mongo_url, credentials) + except Exception as exc: + print(f"ERROR: Could not initialise collection indexes: {exc}", file=sys.stderr) + return 1 + try: doc_id = upsert_feature_status( mongo_url, @@ -371,28 +326,6 @@ def cmd_status_update(args) -> int: # --------------------------------------------------------------------------- -def _add_db_args(parser: argparse.ArgumentParser) -> None: - """Add the shared --db-details / --db-url arguments to a sub-parser.""" - g = parser.add_argument_group("database connection") - g.add_argument( - "--db-details", - required=False, - default=None, - metavar="JSON", - help=( - 'JSON object with "url" and optional "credentials" keys. ' - 'Example: \'{"url": "mongodb://host:27017", "credentials": {"username": "u", "password": "p"}}\'' # pragma: allowlist secret - ), - ) - g.add_argument( - "--db-url", - required=False, - default=None, - metavar="URL", - help=f"MongoDB connection URL (alternative to --db-details). Can also be set via {_ENV_DB_URL}.", - ) - - def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="mas-devops-feature-status-update", @@ -410,20 +343,6 @@ def build_parser() -> argparse.ArgumentParser: subparsers = parser.add_subparsers(dest="command", metavar="") subparsers.required = True - # ── prep ────────────────────────────────────────────────────────────────── - prep = subparsers.add_parser( - "prep", - help="Verify MongoDB connection and check/create required indexes.", - formatter_class=argparse.RawDescriptionHelpFormatter, - ) - _add_db_args(prep) - prep.add_argument( - "--create-indexes", - action="store_true", - default=False, - help="Create missing indexes automatically instead of failing with a warning.", - ) - # ── status-update ───────────────────────────────────────────────────────── su = subparsers.add_parser( "status-update", @@ -477,8 +396,6 @@ def build_parser() -> argparse.ArgumentParser: timestamps.add_argument("--created-at", required=False, default=None, dest="created_at", metavar="ISO-8601", help="Used only on document insert.") timestamps.add_argument("--updated-at", required=False, default=None, dest="updated_at", metavar="ISO-8601") - _add_db_args(su) - # ── get ─────────────────────────────────────────────────────────────────── get = subparsers.add_parser( "get", @@ -521,7 +438,6 @@ def build_parser() -> argparse.ArgumentParser: get_criteria.add_argument("--cluster", default=None, metavar="CLUSTER", help="GitOps cluster name (e.g. noble10)") get_criteria.add_argument("--subscription-id", dest="subscription_id", default=None, metavar="SUBSCRIPTION_ID", help="Subscription ID") get_criteria.add_argument("--type", default=None, metavar="TYPE", help="Feature type (e.g. allow-list)") - _add_db_args(get) return parser @@ -538,9 +454,7 @@ if __name__ == "__main__": logging.basicConfig(format="%(levelname)s %(name)s: %(message)s") logging.getLogger("mas.devops.feature_status").setLevel(log_level) - if args.command == "prep": - sys.exit(cmd_prep(args)) - elif args.command == "status-update": + if args.command == "status-update": sys.exit(cmd_status_update(args)) elif args.command == "get": sys.exit(cmd_get(args)) diff --git a/bin/mas-devops-feature-status-update.md b/bin/mas-devops-feature-status-update.md index fe61a1cd..4ed7504c 100644 --- a/bin/mas-devops-feature-status-update.md +++ b/bin/mas-devops-feature-status-update.md @@ -72,19 +72,16 @@ mongosh "mongodb://localhost:27017/feature_dashboard" --eval "db.getCollectionNa # Expected: [ 'cluster_level_config', 'instance_level_config' ] ``` -### Export the connection URL +### Set the connection URI -Export `MAS_FEATURE_STATUS_DB_URL` so every subsequent command picks it up automatically without needing `--db-url` or `--db-details`: +Set `DEVOPS_MONGO_URI` — all commands read it automatically. Credentials and TLS options are embedded directly in the URI, matching the convention used across all other DevOps pipeline scripts: ```bash -export MAS_FEATURE_STATUS_DB_URL='mongodb://localhost:27017' +export DEVOPS_MONGO_URI='mongodb://user:password@host1:port1,host2:port2/admin?tls=true&tlsAllowInvalidCertificates=true' # pragma: allowlist secret ``` -Then verify connectivity and indexes: - -```bash -mas-devops-feature-status-update prep --create-indexes -``` +No separate verification step is needed — the first `status-update` call will +create required indexes automatically if they are absent. --- @@ -128,7 +125,6 @@ chmod +x bin/mas-devops-feature-status-update mas-devops-feature-status-update --help # Sub-command help -mas-devops-feature-status-update prep --help mas-devops-feature-status-update status-update --help mas-devops-feature-status-update get --help ``` @@ -137,55 +133,13 @@ mas-devops-feature-status-update get --help ## Sub-commands -### `prep` - -Verifies MongoDB connectivity and confirms that the required indexes exist on the collection. - -| Index name | Fields | -|------------------------|-----------------------------------------| -| `instance_config_level` | `region` + `instance_id` + `account` | -| `cluster_config_level` | `region` + `cluster` + `account` | - -Pass `--create-indexes` to create missing indexes automatically instead of exiting with an error. - -**Idempotency:** Safe to run repeatedly. The connectivity check is read-only. When `--create-indexes` is passed, `create_index` is a no-op for any index that already exists — it will never drop or recreate an existing index. - -**Options** - -| Flag | Required | Description | -|------|----------|-------------| -| `--db-details JSON` | No† | JSON object with `url` and optional `credentials` keys | -| `--db-url URL` | No† | MongoDB connection URL (alternative to `--db-details`) | -| `--create-indexes` | No | Create missing indexes automatically | - -† At least one of `--db-details`, `--db-url`, or the `MAS_FEATURE_STATUS_DB_URL` environment variable is required. - -**Examples** - -```bash -# Verify using a db-details JSON blob (local MongoDB, no auth) -mas-devops-feature-status-update prep \ - --db-details '{"url": "mongodb://localhost:27017"}' - -# Verify using a db-details JSON blob (with credentials) -mas-devops-feature-status-update prep \ - --db-details '{"url": "mongodb://localhost:27017", "credentials": {"username": "user", "password": "pass -- pragma: allowlist secret", "authSource": "admin"}}' - -# Verify and auto-create missing indexes -mas-devops-feature-status-update prep \ - --db-url mongodb://localhost:27017 \ - --create-indexes -``` - -After a successful `prep` run the command prints the `export` statements needed to reuse the connection details in subsequent `status-update` calls. - ---- - ### `status-update` Upserts a feature status document. Upsert key: `(region, instance_id, account, cluster, type)` — an existing document is updated in-place; a new document is inserted if no match is found. +Required collection indexes (`instance_config_level`, `cluster_config_level`) are created automatically on the first call if they are absent — no separate setup step is needed. + **Idempotency:** Safe to call multiple times with the same arguments. The underlying `find_one_and_update` with `upsert=True` guarantees that re-running with the same identity key produces the same final document state. `created_at` is set only on the first insert (`$setOnInsert`); subsequent calls update `updated_at` and all mutable fields without creating duplicate documents. **Identity options** *(all required)* @@ -221,12 +175,7 @@ Upsert key: `(region, instance_id, account, cluster, type)` — an existing docu | `--created-at ISO-8601` | Overrides `created_at` on document insert only | | `--updated-at ISO-8601` | Overrides `updated_at` | -**Database connection options** *(one required)* - -| Flag | Description | -|------|-------------| -| `--db-details JSON` | JSON object with `url` and optional `credentials` keys | -| `--db-url URL` | MongoDB connection URL | +The MongoDB connection URI is read from the `DEVOPS_MONGO_URI` environment variable — no connection flags are needed on the command line. **`--status-details` schema** @@ -363,17 +312,14 @@ Two mutually exclusive lookup modes are supported — exactly one must be provid | `--cluster CLUSTER` | Yes (criteria mode) | GitOps cluster name (e.g. `noble10`) | | `--subscription-id SUBSCRIPTION_ID` | Yes (criteria mode) | Subscription ID | | `--type TYPE` | Yes (criteria mode) | Feature type (e.g. `allow-list`) | -| `--db-details JSON` | No† | JSON object with `url` and optional `credentials` keys | -| `--db-url URL` | No† | MongoDB connection URL | -† At least one of `--db-details`, `--db-url`, or the `MAS_FEATURE_STATUS_DB_URL` environment variable is required. +The MongoDB connection URI is read from `DEVOPS_MONGO_URI` — no connection flags are needed. **Example — by ObjectId** ```bash mas-devops-feature-status-update get \ - --id 6ab0e70ee6d3a31faa808547 \ - --db-url mongodb://localhost:27017 + --id 6ab0e70ee6d3a31faa808547 ``` **Example — by criteria** @@ -385,8 +331,7 @@ mas-devops-feature-status-update get \ --account fyre-noble10-dev \ --cluster noble10 \ --subscription-id sub-id01 \ - --type allow-list \ - --db-url mongodb://localhost:27017 + --type allow-list ``` **Sample output** @@ -415,22 +360,12 @@ mas-devops-feature-status-update get \ ## Environment Variables -Setting these avoids repeating `--db-details` / `--db-url` on every call. - -| Variable | Description | -|----------|-------------| -| `MAS_FEATURE_STATUS_DB_URL` | MongoDB connection URL | -| `MAS_FEATURE_STATUS_DB_CREDENTIALS` | JSON object with optional `username`, `password`, `authSource`, `tls` keys | - -**Precedence** (highest to lowest): `--db-details` → `--db-url` → environment variables. +| Variable | Required | Description | +|----------|----------|-------------| +| `DEVOPS_MONGO_URI` | Yes | Full MongoDB connection URI with embedded credentials and TLS options. | ```bash -export MAS_FEATURE_STATUS_DB_URL='mongodb://user:pass@host:27017' #pragma: allowlist secret -export MAS_FEATURE_STATUS_DB_CREDENTIALS='{"username": "u", "password": "p"}' #pragma: allowlist secret - -mas-devops-feature-status-update status-update \ - --region us-east-2 \ - ... +export DEVOPS_MONGO_URI='mongodb://user:password@host1:port1,host2:port2/admin?tls=true&tlsAllowInvalidCertificates=true' # pragma: allowlist secret ``` --- @@ -550,21 +485,6 @@ Collection: `mas_devops.feature_status` See the full sample playbook at [`playbooks/feature-status-update.yml`](../playbooks/feature-status-update.yml). -### Minimal task — `prep` - -Verify connectivity before any write. Use `--create-indexes` on first run. - -```yaml -- name: Verify MongoDB connectivity and indexes - ansible.builtin.command: - cmd: >- - mas-devops-feature-status-update prep - --db-url {{ mas_mongo_url }} - --create-indexes - register: prep_result - changed_when: "'Creating missing indexes' in prep_result.stdout" -``` - ### Minimal task — `status-update` ```yaml @@ -572,7 +492,6 @@ Verify connectivity before any write. Use `--create-indexes` on first run. ansible.builtin.command: cmd: >- mas-devops-feature-status-update status-update - --db-url {{ mas_mongo_url }} --region {{ mas_region }} --instance-id {{ mas_instance_id }} --account {{ mas_account }} @@ -603,7 +522,6 @@ Verify connectivity before any write. Use `--create-indexes` on first run. cmd: >- mas-devops-feature-status-update get --id {{ mas_document_id }} - --db-url {{ mas_mongo_url }} register: get_result changed_when: false @@ -625,7 +543,6 @@ Verify connectivity before any write. Use `--create-indexes` on first run. --cluster {{ mas_cluster }} --subscription-id {{ mas_subscription_id }} --type allow-list - --db-url {{ mas_mongo_url }} register: get_result changed_when: false @@ -634,19 +551,15 @@ Verify connectivity before any write. Use `--create-indexes` on first run. msg: "{{ get_result.stdout | from_json }}" ``` -### Using environment variables instead of `--db-url` +### Using `DEVOPS_MONGO_URI` -Set `MAS_FEATURE_STATUS_DB_URL` once (e.g. in `group_vars/all.yml` or a `block` `environment:`) to avoid repeating the flag on every task: +Set `DEVOPS_MONGO_URI` once (e.g. in `group_vars/all.yml` or a `block` `environment:`). All sub-commands read it automatically — no connection flag is required on any task: ```yaml - name: Feature status tasks environment: - MAS_FEATURE_STATUS_DB_URL: "mongodb://localhost:27017" + DEVOPS_MONGO_URI: "mongodb://{{ mas_mongo_user }}:{{ mas_mongo_password }}@{{ mas_mongo_host }}:{{ mas_mongo_port }}/admin?tls=true&tlsAllowInvalidCertificates=true" # pragma: allowlist secret block: - - name: prep - ansible.builtin.command: - cmd: mas-devops-feature-status-update prep --create-indexes - - name: status-update ansible.builtin.command: cmd: >- From 490a6e690dba06ea6a597268a5a0d19606cea0c5 Mon Sep 17 00:00:00 2001 From: Sagar-Talikoti Date: Wed, 30 Sep 2026 19:42:38 +0530 Subject: [PATCH 07/10] code update --- bin/mas-devops-feature-status-update | 304 ++++++++++---- mongodb_schemas/README.md | 18 +- mongodb_schemas/init_db.js | 10 +- src/mas/devops/feature_status.py | 573 ++++++++++++++++++++------- 4 files changed, 655 insertions(+), 250 deletions(-) diff --git a/bin/mas-devops-feature-status-update b/bin/mas-devops-feature-status-update index f966a7fb..0a4205ad 100755 --- a/bin/mas-devops-feature-status-update +++ b/bin/mas-devops-feature-status-update @@ -11,41 +11,55 @@ # ***************************************************************************** """ mas-devops-feature-status-update — Write MAS feature status records to the -DevOps MongoDB (mas_devops.feature_status collection). +DevOps MongoDB (mas_devops database). + +Collection routing +────────────────── + --instance-id supplied → mas_devops.instance_level_config + One document per (tenant_id × subscription_id × + account × region × cluster × instance). + The feature entry is embedded in + instance_level_features[]. + + --instance-id omitted → mas_devops.cluster_level_config + One document per (tenant_id × account × region × + cluster). + The feature entry is embedded in + cluster_level_features[]. Sub-commands ──────────── status-update - Upsert a feature status document. - Required indexes (instance_config_level, cluster_config_level) are - created automatically on the first call if they do not already exist. - Upsert key: (region, instance_id, account, cluster, type). + Upsert a feature status entry. + Required indexes are created automatically on the first call if they do + not already exist (idempotent). - Example — ACTIVE: + Example — instance-level, ACTIVE: mas-devops-feature-status-update status-update \\ - --region us-east-2 \\ + --tenant-id tenant-abc \\ + --region us-east-2 \\ --instance-id inst02 \\ - --account fyre-noble10-dev \\ - --cluster noble10 \\ + --account fyre-noble10-dev \\ + --cluster noble10 \\ --subscription-id sub-id01 \\ - --type allow-list \\ + --type allow-list \\ --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' \\ - --status ACTIVE \\ + --status ACTIVE \\ --status-details '{"message": "Allow list is active.", "request_configuration": "2405:201:d000:9062::/64"}' \\ --deployment-start 2026-09-11T11:48:42+00:00 \\ --deployment-end 2026-09-11T11:53:10+00:00 - Example — ERROR: + Example — cluster-level (no --instance-id), ERROR: mas-devops-feature-status-update status-update \\ - --region us-east-2 \\ - --instance-id inst02 \\ - --account fyre-noble10-dev \\ - --cluster noble10 \\ + --tenant-id tenant-abc \\ + --region us-east-2 \\ + --account fyre-noble10-dev \\ + --cluster noble10 \\ --subscription-id sub-id01 \\ - --type allow-list \\ + --type allow-list \\ --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' \\ - --status ERROR \\ + --status ERROR \\ --status-details '{ "message": "sample error message", "error_code": 401, @@ -60,20 +74,29 @@ Sub-commands }' get - Fetch and pretty-print a feature status document by ObjectId or by - criteria fields (region, instance_id, account, cluster, type). + Fetch and pretty-print a feature status entry by ObjectId or by + criteria fields (tenant_id, region, account, cluster[, instance_id], type). Example — by ObjectId: mas-devops-feature-status-update get --id 6ab0e70ee6d3a31faa808547 - Example — by criteria: + Example — instance-level by criteria: mas-devops-feature-status-update get \\ - --region us-east-2 \\ + --tenant-id tenant-abc \\ + --region us-east-2 \\ --instance-id inst02 \\ - --account fyre-noble10-dev \\ - --cluster noble10 \\ + --account fyre-noble10-dev \\ + --cluster noble10 \\ --subscription-id sub-id01 \\ - --type allow-list + --type allow-list + + Example — cluster-level by criteria (no --instance-id): + mas-devops-feature-status-update get \\ + --tenant-id tenant-abc \\ + --region us-east-2 \\ + --account fyre-noble10-dev \\ + --cluster noble10 \\ + --type allow-list Environment variables ───────────────────── @@ -181,8 +204,14 @@ def _resolve_db() -> str: def cmd_get(args) -> int: - """Fetch and pretty-print a feature status document by ObjectId or by criteria fields.""" - from mas.devops.feature_status import get_feature_status_by_criteria, get_feature_status_by_id + """Fetch and pretty-print a feature status entry by ObjectId or by criteria fields.""" + from mas.devops.feature_status import ( + INSTANCE_LEVEL, + get_cluster_feature_by_criteria, + get_feature_level, + get_feature_status_by_id, + get_instance_feature_by_criteria, + ) mongo_url = _resolve_db() credentials = None @@ -192,14 +221,13 @@ def cmd_get(args) -> int: doc = get_feature_status_by_id(mongo_url, args.id, credentials) not_found_msg = f"No document found with ID: {args.id}" else: - # Criteria mode — all companion flags are required. + # Criteria mode — tenant_id, region, account, cluster, type are always required. missing = [ f for f, v in [ - ("--instance-id", args.instance_id), + ("--tenant-id", args.tenant_id), ("--account", args.account), ("--cluster", args.cluster), - ("--subscription-id", args.subscription_id), ("--type", args.type), ] if v is None @@ -208,21 +236,55 @@ def cmd_get(args) -> int: print(f"ERROR: the following arguments are required when using --region: {', '.join(missing)}", file=sys.stderr) return 1 - doc = get_feature_status_by_criteria( - mongo_url, - region=args.region, - instance_id=args.instance_id, - account=args.account, - cluster=args.cluster, - subscription_id=args.subscription_id, - feature_type=args.type, - credentials=credentials, # pragma: allowlist secret - ) - not_found_msg = ( - f"No document found for region={args.region} instance_id={args.instance_id} " - f"account={args.account} cluster={args.cluster} " - f"subscription_id={args.subscription_id} type={args.type}" - ) + # Resolve collection routing from the feature type map. + try: + level = get_feature_level(args.type) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + if level == INSTANCE_LEVEL: + if not args.instance_id: + print(f"ERROR: --instance-id is required for instance-level feature type '{args.type}'", file=sys.stderr) + return 1 + if not args.subscription_id: + print("ERROR: --subscription-id is required for instance-level feature types", file=sys.stderr) + return 1 + doc = get_instance_feature_by_criteria( + mongo_url, + tenant_id=args.tenant_id, + region=args.region, + instance_id=args.instance_id, + account=args.account, + cluster=args.cluster, + subscription_id=args.subscription_id, + feature_type=args.type, + credentials=credentials, # pragma: allowlist secret + ) + not_found_msg = ( + f"No '{args.type}' feature entry found for " + f"tenant={args.tenant_id} region={args.region} " + f"instance_id={args.instance_id} account={args.account} " + f"cluster={args.cluster} subscription_id={args.subscription_id}" + ) + else: + if args.instance_id: + print(f"ERROR: --instance-id must not be supplied for cluster-level feature type '{args.type}'", file=sys.stderr) + return 1 + doc = get_cluster_feature_by_criteria( + mongo_url, + tenant_id=args.tenant_id, + region=args.region, + account=args.account, + cluster=args.cluster, + feature_type=args.type, + credentials=credentials, # pragma: allowlist secret + ) + not_found_msg = ( + f"No '{args.type}' feature entry found for " + f"tenant={args.tenant_id} region={args.region} " + f"account={args.account} cluster={args.cluster}" + ) except ValueError as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 @@ -244,16 +306,22 @@ def cmd_get(args) -> int: def cmd_status_update(args) -> int: - """Upsert a feature status document into MongoDB. + """Upsert a feature status entry into MongoDB. - Required indexes are created automatically if they do not already exist, - so no separate 'prep' step is needed. + Collection routing is determined by FEATURE_LEVEL_MAP: + INSTANCE_LEVEL → instance_level_config (--instance-id required) + otherwise → cluster_level_config (--instance-id must be absent) + + Required indexes are created automatically if they do not already exist. """ from mas.devops.feature_status import ( + INSTANCE_LEVEL, VALID_STATUSES, _redact_url, create_indexes, - upsert_feature_status, + get_feature_level, + upsert_cluster_feature, + upsert_instance_feature, validate_feature_details, ) @@ -262,6 +330,21 @@ def cmd_status_update(args) -> int: print(f"ERROR: --status must be one of {sorted(VALID_STATUSES)}, got '{args.status}'", file=sys.stderr) return 1 + # Resolve collection routing from the feature type map (fail fast before any I/O). + try: + level = get_feature_level(args.type) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + # Validate --instance-id consistency with the feature type's level. + if level == INSTANCE_LEVEL and not args.instance_id: + print(f"ERROR: --instance-id is required for instance-level feature type '{args.type}'", file=sys.stderr) + return 1 + if level != INSTANCE_LEVEL and args.instance_id: + print(f"ERROR: --instance-id must not be supplied for cluster-level feature type '{args.type}'", file=sys.stderr) + return 1 + # Parse JSON arguments feature_details = _parse_json_arg(args.feature_details, "feature-details") status_details = _parse_json_arg(args.status_details, "status-details") @@ -283,10 +366,18 @@ def cmd_status_update(args) -> int: mongo_url = _resolve_db() credentials = None - print(f"Writing feature status: account={args.account} cluster={args.cluster} " f"instance={args.instance_id} type={args.type} status={args.status}") + # Build the console label from the resolved level. + if level == INSTANCE_LEVEL: + collection_label = "instance_level_config" + target_label = f"account={args.account} cluster={args.cluster} " f"instance={args.instance_id} type={args.type} status={args.status}" + else: + collection_label = "cluster_level_config" + target_label = f"account={args.account} cluster={args.cluster} " f"type={args.type} status={args.status}" + + print(f"Writing feature status to mas_devops.{collection_label}: {target_label}") print(f" MongoDB: {_redact_url(mongo_url)}") - # Ensure required indexes exist (idempotent — no-op when already present). + # Ensure required indexes exist on both collections (idempotent — no-op when already present). try: create_indexes(mongo_url, credentials) except Exception as exc: @@ -294,23 +385,42 @@ def cmd_status_update(args) -> int: return 1 try: - doc_id = upsert_feature_status( - mongo_url, - region=args.region, - instance_id=args.instance_id, - account=args.account, - cluster=args.cluster, - subscription_id=args.subscription_id, - feature_type=args.type, - feature_details=feature_details, - status=args.status, - status_details=status_details, - deployment_start=deployment_start, - deployment_end=deployment_end, - created_at=created_at, - updated_at=updated_at, - credentials=credentials, # pragma: allowlist secret - ) + if level == INSTANCE_LEVEL: + doc_id = upsert_instance_feature( + mongo_url, + tenant_id=args.tenant_id, + subscription_id=args.subscription_id, + region=args.region, + account=args.account, + cluster=args.cluster, + instance=args.instance_id, + feature_type=args.type, + feature_details=feature_details, + status=args.status, + status_details=status_details, + deployment_start=deployment_start, + deployment_end=deployment_end, + created_at=created_at, + updated_at=updated_at, + credentials=credentials, # pragma: allowlist secret + ) + else: + doc_id = upsert_cluster_feature( + mongo_url, + tenant_id=args.tenant_id, + region=args.region, + account=args.account, + cluster=args.cluster, + feature_type=args.type, + feature_details=feature_details, + status=args.status, + status_details=status_details, + deployment_start=deployment_start, + deployment_end=deployment_end, + created_at=created_at, + updated_at=updated_at, + credentials=credentials, # pragma: allowlist secret + ) print(f"Feature status written successfully. Document ID: {doc_id}") return 0 except ValueError as exc: @@ -329,7 +439,7 @@ def cmd_status_update(args) -> int: def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="mas-devops-feature-status-update", - description="Write MAS feature status records to the DevOps MongoDB.", + description="Write MAS feature status records to the DevOps MongoDB (mas_devops).", formatter_class=argparse.RawDescriptionHelpFormatter, ) parser.add_argument( @@ -346,13 +456,22 @@ def build_parser() -> argparse.ArgumentParser: # ── status-update ───────────────────────────────────────────────────────── su = subparsers.add_parser( "status-update", - help="Upsert a feature status document into mas_devops.feature_status.", + help=( + "Upsert a feature status entry. " "Routes to instance_level_config when --instance-id is supplied, " "or cluster_level_config when it is omitted." + ), formatter_class=argparse.RawDescriptionHelpFormatter, ) identity = su.add_argument_group("identity") + identity.add_argument("--tenant-id", required=True, dest="tenant_id", help="Tenant/customer identifier (multi-tenancy isolation key)") identity.add_argument("--region", required=True, help="AWS region (e.g. us-east-2)") - identity.add_argument("--instance-id", required=True, dest="instance_id", help="MAS instance ID (e.g. inst02)") + identity.add_argument( + "--instance-id", + required=False, + default=None, + dest="instance_id", + help=("MAS instance ID (e.g. inst02). " "When supplied, writes to instance_level_config. " "When omitted, writes to cluster_level_config."), + ) identity.add_argument("--account", required=True, help="GitOps account name (e.g. fyre-noble10-dev)") identity.add_argument("--cluster", required=True, help="GitOps cluster name (e.g. noble10)") identity.add_argument("--subscription-id", required=True, dest="subscription_id", help="Subscription ID") @@ -399,24 +518,34 @@ def build_parser() -> argparse.ArgumentParser: # ── get ─────────────────────────────────────────────────────────────────── get = subparsers.add_parser( "get", - help="Fetch and print a feature status document by ObjectId or by criteria fields.", + help="Fetch and print a feature status entry by ObjectId or by criteria fields.", formatter_class=argparse.RawDescriptionHelpFormatter, description=( - "Retrieve a feature status document using either its ObjectId (--id) or a\n" - "combination of criteria fields (--region, --instance-id, --account,\n" - "--cluster, --subscription-id, --type). Exactly one lookup mode must be\n" - "provided; the two modes are mutually exclusive.\n\n" + "Retrieve a feature status entry using either its ObjectId (--id) or a\n" + "combination of criteria fields.\n\n" + "When --instance-id is supplied the lookup targets instance_level_config\n" + "and returns the matching entry from instance_level_features[].\n" + "When --instance-id is omitted the lookup targets cluster_level_config\n" + "and returns the matching entry from cluster_level_features[].\n\n" "Examples:\n" " # by ObjectId\n" " mas-devops-feature-status-update get --id 6ab0e70ee6d3a31faa808547\n\n" - " # by criteria\n" + " # instance-level by criteria\n" " mas-devops-feature-status-update get \\\n" - " --region us-east-2 \\\n" + " --tenant-id tenant-abc \\\n" + " --region us-east-2 \\\n" " --instance-id inst02 \\\n" - " --account fyre-noble10-dev \\\n" - " --cluster noble10 \\\n" + " --account fyre-noble10-dev \\\n" + " --cluster noble10 \\\n" " --subscription-id sub-id01 \\\n" - " --type allow-list" + " --type allow-list\n\n" + " # cluster-level by criteria (no --instance-id)\n" + " mas-devops-feature-status-update get \\\n" + " --tenant-id tenant-abc \\\n" + " --region us-east-2 \\\n" + " --account fyre-noble10-dev \\\n" + " --cluster noble10 \\\n" + " --type allow-list" ), ) get_lookup = get.add_mutually_exclusive_group(required=True) @@ -433,10 +562,19 @@ def build_parser() -> argparse.ArgumentParser: help="AWS region (e.g. us-east-2). Use together with the other criteria flags.", ) get_criteria = get.add_argument_group("criteria (required when --region is used)") - get_criteria.add_argument("--instance-id", dest="instance_id", default=None, metavar="INSTANCE_ID", help="MAS instance ID (e.g. inst02)") + get_criteria.add_argument("--tenant-id", dest="tenant_id", default=None, metavar="TENANT_ID", help="Tenant/customer identifier") + get_criteria.add_argument( + "--instance-id", + dest="instance_id", + default=None, + metavar="INSTANCE_ID", + help="MAS instance ID. When supplied routes to instance_level_config; when omitted routes to cluster_level_config.", + ) get_criteria.add_argument("--account", default=None, metavar="ACCOUNT", help="GitOps account name (e.g. fyre-noble10-dev)") get_criteria.add_argument("--cluster", default=None, metavar="CLUSTER", help="GitOps cluster name (e.g. noble10)") - get_criteria.add_argument("--subscription-id", dest="subscription_id", default=None, metavar="SUBSCRIPTION_ID", help="Subscription ID") + get_criteria.add_argument( + "--subscription-id", dest="subscription_id", default=None, metavar="SUBSCRIPTION_ID", help="Subscription ID (required when --instance-id is used)" + ) get_criteria.add_argument("--type", default=None, metavar="TYPE", help="Feature type (e.g. allow-list)") return parser diff --git a/mongodb_schemas/README.md b/mongodb_schemas/README.md index c05e15b5..0959d740 100644 --- a/mongodb_schemas/README.md +++ b/mongodb_schemas/README.md @@ -1,6 +1,6 @@ -# MongoDB Schemas — `feature_dashboard` +# MongoDB Schemas — `mas_devops` -MongoDB validator scripts for the `feature_dashboard` database. +MongoDB validator scripts for the `mas_devops` database. Extracted from [`allowlisting-tdd.md`](../allowlisting-tdd.md) §7. ## Collections @@ -48,7 +48,7 @@ The original `allowlisting_config` collection has been split into two flat colle ## Query reference -All queries assume `use feature_dashboard` has been run first. Replace +All queries assume `use mas_devops` has been run first. Replace ``, ``, ``, ``, ``, and `` with real values. @@ -486,7 +486,7 @@ db.instance_level_config.aggregate([ ### Initialize the collections ```bash -mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/init_db.js +mongosh "mongodb://:27017/mas_devops" mongodb_schemas/init_db.js ``` > **Idempotency:** `db.createCollection()` raises a `MongoServerError: Collection already exists` error if the collection is already present. The initialization scripts are **not safe to re-run** against an existing database. Use the safe re-initialization pattern below if you need to ensure indexes are up to date without dropping data. @@ -496,7 +496,7 @@ mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/init_db.js If the collections already exist and you only need to ensure indexes are up to date, run `createIndex` calls directly — they are no-ops when the index name and definition already match: ```js -use feature_dashboard +use mas_devops // cluster_level_config indexes db.cluster_level_config.createIndex( @@ -530,7 +530,7 @@ db.instance_level_config.createIndex( ### Clear the collections ```js -use feature_dashboard +use mas_devops db.cluster_level_config.deleteMany({}) db.instance_level_config.deleteMany({}) ``` @@ -540,7 +540,7 @@ db.instance_level_config.deleteMany({}) ### Drop the collections ```js -use feature_dashboard +use mas_devops db.cluster_level_config.drop() db.instance_level_config.drop() ``` @@ -551,8 +551,8 @@ db.instance_level_config.drop() ### Run a schema file directly ```bash -mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/cluster_level_config.js -mongosh "mongodb://:27017/feature_dashboard" mongodb_schemas/instance_level_config.js +mongosh "mongodb://:27017/mas_devops" mongodb_schemas/cluster_level_config.js +mongosh "mongodb://:27017/mas_devops" mongodb_schemas/instance_level_config.js ``` > **Idempotency:** Same caveat as `init_db.js` — each file calls `db.createCollection()`, which fails if the collection already exists. Only run against a fresh or dropped database. diff --git a/mongodb_schemas/init_db.js b/mongodb_schemas/init_db.js index a282b6fb..c45cb0fc 100644 --- a/mongodb_schemas/init_db.js +++ b/mongodb_schemas/init_db.js @@ -1,16 +1,16 @@ // ============================================================================= -// init_db.js — Bootstrap script for the feature_dashboard database -// Database: feature_dashboard +// init_db.js — Bootstrap script for the mas_devops database +// Database: mas_devops // // Initialises all collections and their indexes: // • cluster_level_config — cluster-scoped feature entries // • instance_level_config — instance-scoped allowlisting entries // // Usage (mongosh): -// mongosh "mongodb://:27017/feature_dashboard" init_db.js +// mongosh "mongodb://:27017/mas_devops" init_db.js // // Usage (legacy mongo shell): -// mongo "mongodb://:27017/feature_dashboard" init_db.js +// mongo "mongodb://:27017/mas_devops" init_db.js // ============================================================================= const scriptDir = __dirname ?? (function() { @@ -22,4 +22,4 @@ const scriptDir = __dirname ?? (function() { load(scriptDir + "/cluster_level_config.js"); load(scriptDir + "/instance_level_config.js"); -print("✅ feature_dashboard: cluster_level_config and instance_level_config collections and indexes initialized."); +print("✅ mas_devops: cluster_level_config and instance_level_config collections and indexes initialized."); diff --git a/src/mas/devops/feature_status.py b/src/mas/devops/feature_status.py index 8dcfa852..b388c10c 100644 --- a/src/mas/devops/feature_status.py +++ b/src/mas/devops/feature_status.py @@ -10,42 +10,70 @@ """ feature_status.py — Write feature status records into the DevOps MongoDB. -Supports two operations: +Database: mas_devops +Collections: + instance_level_config — one document per (tenant_id × subscription_id × + account × region × cluster × instance). + Feature entries are embedded in instance_level_features[]. + Used when --instance-id is supplied. - prep — Verify the MongoDB connection and confirm the expected indexes - (instance_config_level, cluster_config_level) exist on the - target collection. Stores db-details for later use in an - environment variable so they do not need to be repeated on - every status-update call. + cluster_level_config — one document per (tenant_id × account × region × cluster). + Feature entries are embedded in cluster_level_features[]. + Used when --instance-id is omitted. - status_update — Upsert a feature status document into - ``mas_devops.feature_status``. - -Collection: ``mas_devops.feature_status`` - -Document schema (mirrors the CIS allowlist status tracking design): +Document schema — instance_level_config top-level: { - "_id": , - "schema_version": 1, - "region": str, - "instance_id": str, - "account": str, - "cluster": str, + "_id": , + "tenant_id": str, "subscription_id": str, - "type": str, # e.g. "allow-list" - "feature_details": dict, # type-specific payload - "status": str, # REQUESTED | IN_PROGRESS | ACTIVE | ERROR - "status_details": dict, # message, error_code, error_source, … - "deployment_start": datetime, - "deployment_end": datetime | None, + "account": str, + "region": str, + "cluster": str, + "instance": str, + "instance_level_features": [ + { + "type": str, # e.g. "allow-list" + "feature_details": dict, # type-specific payload + "status": str, # REQUESTED | IN_PROGRESS | ACTIVE | ERROR + "status_details": dict, # message, error_code, error_source, … + "deployment_start": str, # ISO-8601 + "deployment_end": str|None, # ISO-8601 + "source": str, # "ansible_devops" + "created_at": datetime, + "updated_at": datetime, + }, + … + ], "created_at": datetime, "updated_at": datetime, } -Indexes expected on the collection - • ``instance_config_level`` — compound: region + instance_id + account - • ``cluster_config_level`` — compound: region + cluster + account +Document schema — cluster_level_config top-level: + + { + "_id": , + "tenant_id": str, + "account": str, + "region": str, + "cluster": str, + "cluster_level_features": [ + { + "type": str, + "feature_details": dict, + "status": str, + "status_details": dict, + "deployment_start": str, + "deployment_end": str|None, + "source": str, + "created_at": datetime, + "updated_at": datetime, + }, + … + ], + "created_at": datetime, + "updated_at": datetime, + } """ from __future__ import annotations @@ -60,10 +88,14 @@ # Constants # --------------------------------------------------------------------------- -COLLECTION = "feature_status" DATABASE = "mas_devops" +COLLECTION_INSTANCE = "instance_level_config" +COLLECTION_CLUSTER = "cluster_level_config" + +# Source tag written by this CLI tool into every feature entry. +FEATURE_SOURCE = "ansible_devops" -# Status enum values (matches the CIS allowlist lifecycle) +# Status enum values STATUS_REQUESTED = "REQUESTED" STATUS_IN_PROGRESS = "IN_PROGRESS" STATUS_ACTIVE = "ACTIVE" @@ -71,15 +103,43 @@ VALID_STATUSES = {STATUS_REQUESTED, STATUS_IN_PROGRESS, STATUS_ACTIVE, STATUS_ERROR} -# Required index names that must exist on the collection. -REQUIRED_INDEX_NAMES = {"instance_config_level", "cluster_config_level"} +# Feature-level constants +INSTANCE_LEVEL = "INSTANCE_LEVEL" +CLUSTER_LEVEL = "CLUSTER_LEVEL" + +# --------------------------------------------------------------------------- +# Feature type → level map +# +# Declares which collection a feature type belongs to. Add new feature types +# here; the routing logic in the CLI and the library functions will pick it up +# automatically. +# +# INSTANCE_LEVEL → mas_devops.instance_level_config (requires --instance-id) +# CLUSTER_LEVEL → mas_devops.cluster_level_config (no --instance-id needed) +# --------------------------------------------------------------------------- + +FEATURE_LEVEL_MAP: dict[str, str] = { + "allow-list": INSTANCE_LEVEL, +} + + +def get_feature_level(feature_type: str) -> str: + """Return the level constant (INSTANCE_LEVEL or CLUSTER_LEVEL) for *feature_type*. + + Raises: + ValueError: if *feature_type* is not registered in FEATURE_LEVEL_MAP. + """ + level = FEATURE_LEVEL_MAP.get(feature_type) + if level is None: + known = sorted(FEATURE_LEVEL_MAP.keys()) + raise ValueError(f"Unknown feature type '{feature_type}'. " f"Known types: {known}. " f"Add it to FEATURE_LEVEL_MAP in feature_status.py.") + return level + # --------------------------------------------------------------------------- # Per-type feature_details validators # --------------------------------------------------------------------------- -# Each key maps to the set of field names that MUST be present in feature_details -# when --type matches that key. _FEATURE_DETAILS_REQUIRED_FIELDS: dict[str, set[str]] = { "allow-list": {"ips"}, } @@ -96,7 +156,6 @@ def validate_feature_details(feature_type: str, feature_details: dict) -> None: required = _FEATURE_DETAILS_REQUIRED_FIELDS.get(feature_type) if required is None: - # Unknown type — no field-level validation, but emit a warning. logger.warning("No feature_details validation rules defined for type '%s'", feature_type) return @@ -111,11 +170,7 @@ def validate_feature_details(feature_type: str, feature_details: dict) -> None: def _get_client(mongo_url: str, credentials: Optional[dict] = None): - """Return a pymongo MongoClient for *mongo_url*. - - Credentials dict may contain ``username`` and ``password`` keys. - If the URL already embeds credentials they take precedence. - """ + """Return a pymongo MongoClient for *mongo_url*.""" try: from pymongo import MongoClient # type: ignore except ImportError as exc: # pragma: no cover @@ -135,70 +190,242 @@ def _get_client(mongo_url: str, credentials: Optional[dict] = None): return MongoClient(mongo_url, **kwargs) -def verify_connection_and_indexes(mongo_url: str, credentials: Optional[dict] = None) -> list[str]: - """Connect to MongoDB and check that the expected indexes exist. +def create_indexes(mongo_url: str, credentials: Optional[dict] = None) -> None: + """Create all required indexes on both collections (idempotent).""" + try: + from pymongo import ASCENDING # type: ignore + except ImportError as exc: # pragma: no cover + raise ImportError("pymongo is required. Install it with: pip install pymongo") from exc - Returns a list of warning messages for any missing indexes. - Raises on connection failure. - """ client = _get_client(mongo_url, credentials) try: - # Ping — will raise if the server is unreachable. - client.admin.command("ping") - logger.info("MongoDB connection OK: %s", _redact_url(mongo_url)) - db = client[DATABASE] - collection = db[COLLECTION] - - # Retrieve existing index names. - existing_index_names = {info["name"] for info in collection.list_indexes()} - - warnings = [] - for expected in REQUIRED_INDEX_NAMES: - if expected not in existing_index_names: - warnings.append( - f"Index '{expected}' not found on {DATABASE}.{COLLECTION}. " f"Run the index-creation script or use 'prep' with --create-indexes." - ) - return warnings + + # ── instance_level_config ──────────────────────────────────────────── + inst = db[COLLECTION_INSTANCE] + + inst.create_index( + [ + ("tenant_id", ASCENDING), + ("subscription_id", ASCENDING), + ("account", ASCENDING), + ("region", ASCENDING), + ("cluster", ASCENDING), + ("instance", ASCENDING), + ], + unique=True, + name="ux_instance_level_config_tenant_sub_account_region_cluster_instance", + ) + logger.info("Index 'ux_instance_level_config_tenant_sub_account_region_cluster_instance' ensured on %s.%s", DATABASE, COLLECTION_INSTANCE) + + inst.create_index( + [ + ("tenant_id", ASCENDING), + ("subscription_id", ASCENDING), + ("account", ASCENDING), + ("region", ASCENDING), + ("cluster", ASCENDING), + ], + name="ix_instance_level_config_tenant_sub_account_region_cluster", + ) + logger.info("Index 'ix_instance_level_config_tenant_sub_account_region_cluster' ensured on %s.%s", DATABASE, COLLECTION_INSTANCE) + + inst.create_index( + [("instance_level_features.status", ASCENDING)], + name="ix_instance_level_config_feature_status", + ) + logger.info("Index 'ix_instance_level_config_feature_status' ensured on %s.%s", DATABASE, COLLECTION_INSTANCE) + + inst.create_index( + [("instance_level_features.status_details.error_code", ASCENDING)], + sparse=True, + name="ix_instance_level_config_error_code", + ) + logger.info("Index 'ix_instance_level_config_error_code' ensured on %s.%s", DATABASE, COLLECTION_INSTANCE) + + # ── cluster_level_config ───────────────────────────────────────────── + clst = db[COLLECTION_CLUSTER] + + clst.create_index( + [ + ("tenant_id", ASCENDING), + ("account", ASCENDING), + ("region", ASCENDING), + ("cluster", ASCENDING), + ], + unique=True, + name="ux_cluster_level_config_tenant_account_region_cluster", + ) + logger.info("Index 'ux_cluster_level_config_tenant_account_region_cluster' ensured on %s.%s", DATABASE, COLLECTION_CLUSTER) + + clst.create_index( + [("tenant_id", ASCENDING), ("account", ASCENDING)], + name="ix_cluster_level_config_tenant_account", + ) + logger.info("Index 'ix_cluster_level_config_tenant_account' ensured on %s.%s", DATABASE, COLLECTION_CLUSTER) + finally: client.close() -def create_indexes(mongo_url: str, credentials: Optional[dict] = None) -> None: - """Create the required indexes on the feature_status collection if they do not exist.""" +# --------------------------------------------------------------------------- +# Write helpers — shared feature-entry builder +# --------------------------------------------------------------------------- + + +def _build_feature_entry( + feature_type: str, + feature_details: dict, + status: str, + status_details: dict, + deployment_start: Optional[datetime], + deployment_end: Optional[datetime], + now: datetime, + created_at: Optional[datetime], + updated_at: Optional[datetime], +) -> dict: + """Build a single feature entry dict for embedding in the features array.""" + return { + "type": feature_type, + "feature_details": feature_details, + "status": status, + "status_details": status_details, + "deployment_start": (deployment_start or now).isoformat(), + "deployment_end": deployment_end.isoformat() if deployment_end else None, + "source": FEATURE_SOURCE, + "created_at": created_at or now, + "updated_at": updated_at or now, + } + + +# --------------------------------------------------------------------------- +# instance_level_config — upsert +# --------------------------------------------------------------------------- + + +def upsert_instance_feature( + mongo_url: str, + *, + tenant_id: str, + subscription_id: str, + region: str, + account: str, + cluster: str, + instance: str, + feature_type: str, + feature_details: dict, + status: str, + status_details: dict, + deployment_start: Optional[datetime] = None, + deployment_end: Optional[datetime] = None, + created_at: Optional[datetime] = None, + updated_at: Optional[datetime] = None, + credentials: Optional[dict] = None, +) -> str: + """Upsert a feature entry inside instance_level_config. + + The parent document is identified by + (tenant_id, subscription_id, account, region, cluster, instance). + If the parent does not exist it is created with an empty features array + and then the entry is pushed. If a feature entry with the same *type* + already exists it is updated in-place via arrayFilters; otherwise the + entry is appended. + + Returns the parent document _id as a string. + """ try: - from pymongo import ASCENDING # type: ignore + from pymongo import ReturnDocument # type: ignore except ImportError as exc: # pragma: no cover raise ImportError("pymongo is required. Install it with: pip install pymongo") from exc + if status not in VALID_STATUSES: + raise ValueError(f"Invalid status '{status}'. Must be one of {sorted(VALID_STATUSES)}") + validate_feature_details(feature_type, feature_details) + + now = datetime.now(timezone.utc) + entry = _build_feature_entry(feature_type, feature_details, status, status_details, deployment_start, deployment_end, now, created_at, updated_at) + + parent_filter = { + "tenant_id": tenant_id, + "subscription_id": subscription_id, + "account": account, + "region": region, + "cluster": cluster, + "instance": instance, + } + client = _get_client(mongo_url, credentials) try: - db = client[DATABASE] - collection = db[COLLECTION] - - collection.create_index( - [("region", ASCENDING), ("instance_id", ASCENDING), ("account", ASCENDING)], - name="instance_config_level", + collection = client[DATABASE][COLLECTION_INSTANCE] + + # Step 1 — ensure the parent document exists. + collection.update_one( + parent_filter, + { + "$setOnInsert": { + **parent_filter, + "instance_level_features": [], + "created_at": created_at or now, + }, + "$set": {"updated_at": updated_at or now}, + }, + upsert=True, ) - logger.info("Index 'instance_config_level' ensured on %s.%s", DATABASE, COLLECTION) - collection.create_index( - [("region", ASCENDING), ("cluster", ASCENDING), ("account", ASCENDING)], - name="cluster_config_level", + # Step 2 — check whether a feature entry for this type already exists. + existing = collection.find_one({**parent_filter, "instance_level_features.type": feature_type}) + + if existing: + # Update the matching array element in-place. + result = collection.find_one_and_update( + parent_filter, + { + "$set": { + "updated_at": updated_at or now, + **{f"instance_level_features.$[elem].{k}": v for k, v in entry.items() if k != "created_at"}, + } + }, + array_filters=[{"elem.type": feature_type}], + return_document=ReturnDocument.AFTER, + ) + else: + # Append a brand-new feature entry. + result = collection.find_one_and_update( + parent_filter, + { + "$push": {"instance_level_features": entry}, + "$set": {"updated_at": updated_at or now}, + }, + return_document=ReturnDocument.AFTER, + ) + + doc_id = str(result["_id"]) + logger.info( + "Instance feature upserted [%s / %s / %s / %s] status=%s id=%s", + account, + instance, + feature_type, + status, + status, + doc_id, ) - logger.info("Index 'cluster_config_level' ensured on %s.%s", DATABASE, COLLECTION) + return doc_id finally: client.close() -def upsert_feature_status( +# --------------------------------------------------------------------------- +# cluster_level_config — upsert +# --------------------------------------------------------------------------- + + +def upsert_cluster_feature( mongo_url: str, *, + tenant_id: str, region: str, - instance_id: str, account: str, cluster: str, - subscription_id: str, feature_type: str, feature_details: dict, status: str, @@ -209,10 +436,12 @@ def upsert_feature_status( updated_at: Optional[datetime] = None, credentials: Optional[dict] = None, ) -> str: - """Upsert a feature status document. Returns the upserted / matched document ID as a string. + """Upsert a feature entry inside cluster_level_config. - The upsert key is ``(region, instance_id, account, cluster, type)``. - On insert ``created_at`` is set; ``updated_at`` is always refreshed. + The parent document is identified by (tenant_id, account, region, cluster). + Same two-step upsert pattern as upsert_instance_feature. + + Returns the parent document _id as a string. """ try: from pymongo import ReturnDocument # type: ignore @@ -221,53 +450,66 @@ def upsert_feature_status( if status not in VALID_STATUSES: raise ValueError(f"Invalid status '{status}'. Must be one of {sorted(VALID_STATUSES)}") - validate_feature_details(feature_type, feature_details) now = datetime.now(timezone.utc) - deployment_start = deployment_start or now - updated_at = updated_at or now - created_at = created_at or now + entry = _build_feature_entry(feature_type, feature_details, status, status_details, deployment_start, deployment_end, now, created_at, updated_at) - filter_doc = { - "region": region, - "instance_id": instance_id, + parent_filter = { + "tenant_id": tenant_id, "account": account, + "region": region, "cluster": cluster, - "type": feature_type, - } - - update_doc = { - "$set": { - "subscription_id": subscription_id, - "feature_details": feature_details, - "status": status, - "status_details": status_details, - "deployment_start": deployment_start, - "deployment_end": deployment_end, - "updated_at": updated_at, - "schema_version": 1, - }, - "$setOnInsert": { - "created_at": created_at, - }, } client = _get_client(mongo_url, credentials) try: - db = client[DATABASE] - collection = db[COLLECTION] - result = collection.find_one_and_update( - filter_doc, - update_doc, + collection = client[DATABASE][COLLECTION_CLUSTER] + + # Step 1 — ensure the parent document exists. + collection.update_one( + parent_filter, + { + "$setOnInsert": { + **parent_filter, + "cluster_level_features": [], + "created_at": created_at or now, + }, + "$set": {"updated_at": updated_at or now}, + }, upsert=True, - return_document=ReturnDocument.AFTER, ) + + # Step 2 — check whether a feature entry for this type already exists. + existing = collection.find_one({**parent_filter, "cluster_level_features.type": feature_type}) + + if existing: + result = collection.find_one_and_update( + parent_filter, + { + "$set": { + "updated_at": updated_at or now, + **{f"cluster_level_features.$[elem].{k}": v for k, v in entry.items() if k != "created_at"}, + } + }, + array_filters=[{"elem.type": feature_type}], + return_document=ReturnDocument.AFTER, + ) + else: + result = collection.find_one_and_update( + parent_filter, + { + "$push": {"cluster_level_features": entry}, + "$set": {"updated_at": updated_at or now}, + }, + return_document=ReturnDocument.AFTER, + ) + doc_id = str(result["_id"]) logger.info( - "Feature status upserted [%s / %s / %s] status=%s id=%s", + "Cluster feature upserted [%s / %s / %s] status=%s id=%s", account, - instance_id, + cluster, feature_type, status, doc_id, @@ -277,20 +519,21 @@ def upsert_feature_status( client.close() +# --------------------------------------------------------------------------- +# get helpers +# --------------------------------------------------------------------------- + + def get_feature_status_by_id(mongo_url: str, doc_id: str, credentials: Optional[dict] = None) -> Optional[dict]: - """Fetch a single feature status document by its ObjectId string. + """Fetch a parent document by its ObjectId from either collection. - Args: - mongo_url (str): MongoDB connection URL. - doc_id (str): Hex string ObjectId of the document to retrieve. - credentials (dict, optional): Optional credential overrides. Defaults to None. + Tries instance_level_config first, then cluster_level_config. Returns: - dict: The document with ``_id`` serialised to a string, or None if not found. + dict with ``_id`` serialised to a string, or None if not found in either collection. Raises: - ValueError: If *doc_id* is not a valid 24-character hex ObjectId. - pymongo.errors.ConnectionFailure: If the MongoDB server is unreachable. + ValueError: if *doc_id* is not a valid 24-character hex ObjectId. """ try: from bson import ObjectId @@ -305,18 +548,20 @@ def get_feature_status_by_id(mongo_url: str, doc_id: str, credentials: Optional[ client = _get_client(mongo_url, credentials) try: - doc = client[DATABASE][COLLECTION].find_one({"_id": oid}) - if doc is None: - return None - doc["_id"] = str(doc["_id"]) - return doc + for col_name in (COLLECTION_INSTANCE, COLLECTION_CLUSTER): + doc = client[DATABASE][col_name].find_one({"_id": oid}) + if doc is not None: + doc["_id"] = str(doc["_id"]) + return doc + return None finally: client.close() -def get_feature_status_by_criteria( +def get_instance_feature_by_criteria( mongo_url: str, *, + tenant_id: str, region: str, instance_id: str, account: str, @@ -325,40 +570,62 @@ def get_feature_status_by_criteria( feature_type: str, credentials: Optional[dict] = None, ) -> Optional[dict]: - """Fetch a single feature status document by its identifying criteria fields. - - Args: - mongo_url (str): MongoDB connection URL. - region (str): AWS region (e.g. us-east-2). - instance_id (str): MAS instance ID (e.g. inst02). - account (str): GitOps account name (e.g. fyre-noble10-dev). - cluster (str): GitOps cluster name (e.g. noble10). - subscription_id (str): Subscription ID. - feature_type (str): Feature type (e.g. allow-list). - credentials (dict, optional): Optional credential overrides. Defaults to None. - - Returns: - dict: The matching document with ``_id`` serialised to a string, or None if not found. + """Fetch the feature entry for *feature_type* from instance_level_config. - Raises: - pymongo.errors.ConnectionFailure: If the MongoDB server is unreachable. + Returns the matching feature entry dict (not the full parent document), + or None if the parent or the feature entry does not exist. """ - filterDoc = { - "region": region, - "instance_id": instance_id, - "account": account, - "cluster": cluster, - "subscription_id": subscription_id, - "type": feature_type, - } + client = _get_client(mongo_url, credentials) + try: + filter_doc = { + "tenant_id": tenant_id, + "subscription_id": subscription_id, + "account": account, + "region": region, + "cluster": cluster, + "instance": instance_id, + } + doc = client[DATABASE][COLLECTION_INSTANCE].find_one(filter_doc) + if doc is None: + return None + for entry in doc.get("instance_level_features", []): + if entry.get("type") == feature_type: + return entry + return None + finally: + client.close() + +def get_cluster_feature_by_criteria( + mongo_url: str, + *, + tenant_id: str, + region: str, + account: str, + cluster: str, + feature_type: str, + credentials: Optional[dict] = None, +) -> Optional[dict]: + """Fetch the feature entry for *feature_type* from cluster_level_config. + + Returns the matching feature entry dict (not the full parent document), + or None if the parent or the feature entry does not exist. + """ client = _get_client(mongo_url, credentials) try: - doc = client[DATABASE][COLLECTION].find_one(filterDoc) + filter_doc = { + "tenant_id": tenant_id, + "account": account, + "region": region, + "cluster": cluster, + } + doc = client[DATABASE][COLLECTION_CLUSTER].find_one(filter_doc) if doc is None: return None - doc["_id"] = str(doc["_id"]) - return doc + for entry in doc.get("cluster_level_features", []): + if entry.get("type") == feature_type: + return entry + return None finally: client.close() From 56654a3e60bf47f999929a10bfd5f663f8304d6a Mon Sep 17 00:00:00 2001 From: Sagar-Talikoti Date: Wed, 30 Sep 2026 23:36:19 +0530 Subject: [PATCH 08/10] code update --- bin/mas-devops-feature-status-update | 45 +++-- mongodb_schemas/instance_level_config.js | 2 +- src/mas/devops/feature_status.py | 224 ++++++++++++----------- 3 files changed, 148 insertions(+), 123 deletions(-) diff --git a/bin/mas-devops-feature-status-update b/bin/mas-devops-feature-status-update index 0a4205ad..b4f1409a 100755 --- a/bin/mas-devops-feature-status-update +++ b/bin/mas-devops-feature-status-update @@ -100,9 +100,15 @@ Sub-commands Environment variables ───────────────────── - DEVOPS_MONGO_URI (required) MongoDB connection URI with embedded credentials - and TLS options: - mongodb://user:password@host1:port1,host2:port2/admin?tls=true&tlsAllowInvalidCertificates=true # pragma: allowlist secret + DEVOPS_MONGO_URI (required) Full MongoDB connection URI. All options — including + TLS settings — are passed as query parameters in the URI and are + handled directly by pymongo. + + With TLS enabled (production): + export DEVOPS_MONGO_URI="mongodb://user:pass@host1:port1,host2:port2/admin?tls=true" # pragma: allowlist secret + + With TLS disabled or self-signed certs (local / dev): + export DEVOPS_MONGO_URI="mongodb://user:pass@localhost:27017/admin?tls=false&tlsAllowInvalidCertificates=true" # pragma: allowlist secret One-time initialisation ─────────────────────── @@ -185,13 +191,13 @@ def _resolve_db() -> str: Raises SystemExit(1) if the variable is unset or empty. """ - uri = os.environ.get(_ENV_DEVOPS_MONGO_URI, "") + uri = os.getenv(_ENV_DEVOPS_MONGO_URI, "") if not uri: print( f"ERROR: {_ENV_DEVOPS_MONGO_URI} environment variable is required.\n" f" Set it to a full MongoDB connection URI, e.g.:\n" - f" export {_ENV_DEVOPS_MONGO_URI}='mongodb://user:password@host:port/admin" # pragma: allowlist secret - f"?tls=true&tlsAllowInvalidCertificates=true'", + f' export {_ENV_DEVOPS_MONGO_URI}="mongodb://user:pass@localhost:27017/admin' # pragma: allowlist secret + f'?tls=false&tlsAllowInvalidCertificates=true"', file=sys.stderr, ) sys.exit(1) @@ -214,11 +220,10 @@ def cmd_get(args) -> int: ) mongo_url = _resolve_db() - credentials = None try: if args.id: - doc = get_feature_status_by_id(mongo_url, args.id, credentials) + doc = get_feature_status_by_id(mongo_url, args.id) not_found_msg = f"No document found with ID: {args.id}" else: # Criteria mode — tenant_id, region, account, cluster, type are always required. @@ -259,7 +264,6 @@ def cmd_get(args) -> int: cluster=args.cluster, subscription_id=args.subscription_id, feature_type=args.type, - credentials=credentials, # pragma: allowlist secret ) not_found_msg = ( f"No '{args.type}' feature entry found for " @@ -278,7 +282,6 @@ def cmd_get(args) -> int: account=args.account, cluster=args.cluster, feature_type=args.type, - credentials=credentials, # pragma: allowlist secret ) not_found_msg = ( f"No '{args.type}' feature entry found for " @@ -323,6 +326,7 @@ def cmd_status_update(args) -> int: upsert_cluster_feature, upsert_instance_feature, validate_feature_details, + validate_status_details, ) # Validate status enum @@ -349,9 +353,10 @@ def cmd_status_update(args) -> int: feature_details = _parse_json_arg(args.feature_details, "feature-details") status_details = _parse_json_arg(args.status_details, "status-details") - # Type-specific feature_details validation + # Type-specific validation try: validate_feature_details(args.type, feature_details) + validate_status_details(args.type, args.status, status_details) except ValueError as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 @@ -364,7 +369,6 @@ def cmd_status_update(args) -> int: # Resolve DB URI from environment mongo_url = _resolve_db() - credentials = None # Build the console label from the resolved level. if level == INSTANCE_LEVEL: @@ -379,7 +383,7 @@ def cmd_status_update(args) -> int: # Ensure required indexes exist on both collections (idempotent — no-op when already present). try: - create_indexes(mongo_url, credentials) + create_indexes(mongo_url) except Exception as exc: print(f"ERROR: Could not initialise collection indexes: {exc}", file=sys.stderr) return 1 @@ -402,7 +406,6 @@ def cmd_status_update(args) -> int: deployment_end=deployment_end, created_at=created_at, updated_at=updated_at, - credentials=credentials, # pragma: allowlist secret ) else: doc_id = upsert_cluster_feature( @@ -419,7 +422,6 @@ def cmd_status_update(args) -> int: deployment_end=deployment_end, created_at=created_at, updated_at=updated_at, - credentials=credentials, # pragma: allowlist secret ) print(f"Feature status written successfully. Document ID: {doc_id}") return 0 @@ -512,7 +514,18 @@ def build_parser() -> argparse.ArgumentParser: timestamps = su.add_argument_group("timestamps (all optional, default: now)") timestamps.add_argument("--deployment-start", required=False, default=None, dest="deployment_start", metavar="ISO-8601") timestamps.add_argument("--deployment-end", required=False, default=None, dest="deployment_end", metavar="ISO-8601") - timestamps.add_argument("--created-at", required=False, default=None, dest="created_at", metavar="ISO-8601", help="Used only on document insert.") + timestamps.add_argument( + "--created-at", + required=False, + default=None, + dest="created_at", + metavar="ISO-8601", + help=( + "Timestamp recorded as the entry's creation time. " + "Only applied when the feature entry does not yet exist; " + "ignored on updates to preserve the original value." + ), + ) timestamps.add_argument("--updated-at", required=False, default=None, dest="updated_at", metavar="ISO-8601") # ── get ─────────────────────────────────────────────────────────────────── diff --git a/mongodb_schemas/instance_level_config.js b/mongodb_schemas/instance_level_config.js index db4c9ab7..b8c55bff 100644 --- a/mongodb_schemas/instance_level_config.js +++ b/mongodb_schemas/instance_level_config.js @@ -213,7 +213,7 @@ db.createCollection("instance_level_config", { }, deployment_end: { bsonType: "string", - description: "ISO-8601 timestamp set when the pipeline completes, whether successfully (ACTIVE) or with failure (ERROR). Null while the pipeline is still running." + description: "ISO-8601 timestamp set when the pipeline completes, whether successfully (ACTIVE) or with failure (ERROR). Omitted (field absent) while the pipeline is still running (REQUESTED/IN_PROGRESS)." }, // ---- audit --------------------------------------------------- diff --git a/src/mas/devops/feature_status.py b/src/mas/devops/feature_status.py index b388c10c..a3a095c4 100644 --- a/src/mas/devops/feature_status.py +++ b/src/mas/devops/feature_status.py @@ -80,7 +80,9 @@ import logging from datetime import datetime, timezone -from typing import Any, Optional +from typing import Optional + +from pymongo import MongoClient # type: ignore logger = logging.getLogger(__name__) @@ -165,39 +167,50 @@ def validate_feature_details(feature_type: str, feature_details: dict) -> None: # --------------------------------------------------------------------------- -# MongoDB helpers +# Per-type status_details validators # --------------------------------------------------------------------------- +_STATUS_DETAILS_REQUIRED_FIELDS: dict[str, dict[str, set[str]]] = { + "allow-list": { + "ACTIVE": {"message", "request_configuration"}, + "ERROR": {"message", "error_code", "error_source", "request_configuration"}, + } +} -def _get_client(mongo_url: str, credentials: Optional[dict] = None): - """Return a pymongo MongoClient for *mongo_url*.""" - try: - from pymongo import MongoClient # type: ignore - except ImportError as exc: # pragma: no cover - raise ImportError("pymongo is required. Install it with: pip install pymongo") from exc - kwargs: dict[str, Any] = {"serverSelectionTimeoutMS": 10_000} - if credentials: - if "username" in credentials: - kwargs["username"] = credentials["username"] - if "password" in credentials: - kwargs["password"] = credentials["password"] - if "authSource" in credentials: - kwargs["authSource"] = credentials["authSource"] - if "tls" in credentials: - kwargs["tls"] = credentials["tls"] +def validate_status_details(feature_type: str, status: str, status_details: dict) -> None: + """Validate that *status_details* contains the required keys for *feature_type* and *status*. + + Raises: + ValueError: if required keys are missing or status_details is not a dict. + """ + if not isinstance(status_details, dict): + raise ValueError(f"status_details must be a JSON object, got {type(status_details).__name__}") + + required = _STATUS_DETAILS_REQUIRED_FIELDS.get(feature_type, {}).get(status) + if required is None: + logger.warning( + "No status_details validation rules defined for type='%s' status='%s'", + feature_type, + status, + ) + return + + missing = required - set(status_details.keys()) + if missing: + raise ValueError(f"status_details is missing required field(s) for " f"type='{feature_type}' status='{status}': {sorted(missing)}") - return MongoClient(mongo_url, **kwargs) + +# --------------------------------------------------------------------------- +# MongoDB helpers +# --------------------------------------------------------------------------- -def create_indexes(mongo_url: str, credentials: Optional[dict] = None) -> None: +def create_indexes(mongo_url: str) -> None: """Create all required indexes on both collections (idempotent).""" - try: - from pymongo import ASCENDING # type: ignore - except ImportError as exc: # pragma: no cover - raise ImportError("pymongo is required. Install it with: pip install pymongo") from exc + from pymongo import ASCENDING # type: ignore - client = _get_client(mongo_url, credentials) + client = MongoClient(mongo_url) try: db = client[DATABASE] @@ -285,17 +298,21 @@ def _build_feature_entry( updated_at: Optional[datetime], ) -> dict: """Build a single feature entry dict for embedding in the features array.""" - return { + entry = { "type": feature_type, "feature_details": feature_details, "status": status, "status_details": status_details, "deployment_start": (deployment_start or now).isoformat(), - "deployment_end": deployment_end.isoformat() if deployment_end else None, "source": FEATURE_SOURCE, "created_at": created_at or now, "updated_at": updated_at or now, } + # deployment_end is omitted entirely when not yet known (REQUESTED/IN_PROGRESS). + # Writing null would violate the JSON schema (bsonType: "string"). + if deployment_end is not None: + entry["deployment_end"] = deployment_end.isoformat() + return entry # --------------------------------------------------------------------------- @@ -320,23 +337,18 @@ def upsert_instance_feature( deployment_end: Optional[datetime] = None, created_at: Optional[datetime] = None, updated_at: Optional[datetime] = None, - credentials: Optional[dict] = None, ) -> str: """Upsert a feature entry inside instance_level_config. The parent document is identified by (tenant_id, subscription_id, account, region, cluster, instance). - If the parent does not exist it is created with an empty features array - and then the entry is pushed. If a feature entry with the same *type* - already exists it is updated in-place via arrayFilters; otherwise the - entry is appended. + If a feature entry with the same *type* already exists it is updated + in-place via a single atomic find_one_and_update with arrayFilters; + otherwise the entry is appended (with parent upsert if needed). Returns the parent document _id as a string. """ - try: - from pymongo import ReturnDocument # type: ignore - except ImportError as exc: # pragma: no cover - raise ImportError("pymongo is required. Install it with: pip install pymongo") from exc + from pymongo import ReturnDocument # type: ignore if status not in VALID_STATUSES: raise ValueError(f"Invalid status '{status}'. Must be one of {sorted(VALID_STATUSES)}") @@ -354,59 +366,61 @@ def upsert_instance_feature( "instance": instance, } - client = _get_client(mongo_url, credentials) + client = MongoClient(mongo_url) try: collection = client[DATABASE][COLLECTION_INSTANCE] - # Step 1 — ensure the parent document exists. - collection.update_one( - parent_filter, + # Step 1 — attempt an atomic in-place update of an existing feature entry. + # Matches only when the parent document AND a feature entry with this type exist. + result = collection.find_one_and_update( + {**parent_filter, "instance_level_features.type": feature_type}, { - "$setOnInsert": { - **parent_filter, - "instance_level_features": [], - "created_at": created_at or now, - }, - "$set": {"updated_at": updated_at or now}, + "$set": { + "updated_at": updated_at or now, + **{f"instance_level_features.$[elem].{k}": v for k, v in entry.items() if k != "created_at"}, + } }, - upsert=True, + array_filters=[{"elem.type": feature_type}], + return_document=ReturnDocument.AFTER, ) - # Step 2 — check whether a feature entry for this type already exists. - existing = collection.find_one({**parent_filter, "instance_level_features.type": feature_type}) - - if existing: - # Update the matching array element in-place. - result = collection.find_one_and_update( - parent_filter, - { - "$set": { - "updated_at": updated_at or now, - **{f"instance_level_features.$[elem].{k}": v for k, v in entry.items() if k != "created_at"}, - } - }, - array_filters=[{"elem.type": feature_type}], - return_document=ReturnDocument.AFTER, - ) - else: - # Append a brand-new feature entry. + if result is None: + # No existing feature entry for this type — warn if --created-at would be discarded + # on a subsequent call, then append (upsert parent if it doesn't exist yet). + if created_at is not None: + logger.warning( + "--created-at is only applied on the initial insert of a feature entry " + "(type=%s). It is ignored when updating an existing entry to preserve " + "the original created_at.", + feature_type, + ) result = collection.find_one_and_update( parent_filter, { + "$setOnInsert": { + **parent_filter, + "created_at": created_at or now, + }, "$push": {"instance_level_features": entry}, "$set": {"updated_at": updated_at or now}, }, + upsert=True, return_document=ReturnDocument.AFTER, ) + else: + if created_at is not None: + logger.warning( + "--created-at is ignored when updating an existing feature entry " "(type=%s). The original created_at is preserved.", + feature_type, + ) doc_id = str(result["_id"]) logger.info( - "Instance feature upserted [%s / %s / %s / %s] status=%s id=%s", + "Instance feature upserted [%s / %s / %s] status=%s id=%s", account, instance, feature_type, status, - status, doc_id, ) return doc_id @@ -434,19 +448,15 @@ def upsert_cluster_feature( deployment_end: Optional[datetime] = None, created_at: Optional[datetime] = None, updated_at: Optional[datetime] = None, - credentials: Optional[dict] = None, ) -> str: """Upsert a feature entry inside cluster_level_config. The parent document is identified by (tenant_id, account, region, cluster). - Same two-step upsert pattern as upsert_instance_feature. + Same atomic two-step pattern as upsert_instance_feature. Returns the parent document _id as a string. """ - try: - from pymongo import ReturnDocument # type: ignore - except ImportError as exc: # pragma: no cover - raise ImportError("pymongo is required. Install it with: pip install pymongo") from exc + from pymongo import ReturnDocument # type: ignore if status not in VALID_STATUSES: raise ValueError(f"Invalid status '{status}'. Must be one of {sorted(VALID_STATUSES)}") @@ -462,48 +472,52 @@ def upsert_cluster_feature( "cluster": cluster, } - client = _get_client(mongo_url, credentials) + client = MongoClient(mongo_url) try: collection = client[DATABASE][COLLECTION_CLUSTER] - # Step 1 — ensure the parent document exists. - collection.update_one( - parent_filter, + # Step 1 — attempt an atomic in-place update of an existing feature entry. + # Matches only when the parent document AND a feature entry with this type exist. + result = collection.find_one_and_update( + {**parent_filter, "cluster_level_features.type": feature_type}, { - "$setOnInsert": { - **parent_filter, - "cluster_level_features": [], - "created_at": created_at or now, - }, - "$set": {"updated_at": updated_at or now}, + "$set": { + "updated_at": updated_at or now, + **{f"cluster_level_features.$[elem].{k}": v for k, v in entry.items() if k != "created_at"}, + } }, - upsert=True, + array_filters=[{"elem.type": feature_type}], + return_document=ReturnDocument.AFTER, ) - # Step 2 — check whether a feature entry for this type already exists. - existing = collection.find_one({**parent_filter, "cluster_level_features.type": feature_type}) - - if existing: - result = collection.find_one_and_update( - parent_filter, - { - "$set": { - "updated_at": updated_at or now, - **{f"cluster_level_features.$[elem].{k}": v for k, v in entry.items() if k != "created_at"}, - } - }, - array_filters=[{"elem.type": feature_type}], - return_document=ReturnDocument.AFTER, - ) - else: + if result is None: + # No existing feature entry for this type — append (upsert parent if needed). + if created_at is not None: + logger.warning( + "--created-at is only applied on the initial insert of a feature entry " + "(type=%s). It is ignored when updating an existing entry to preserve " + "the original created_at.", + feature_type, + ) result = collection.find_one_and_update( parent_filter, { + "$setOnInsert": { + **parent_filter, + "created_at": created_at or now, + }, "$push": {"cluster_level_features": entry}, "$set": {"updated_at": updated_at or now}, }, + upsert=True, return_document=ReturnDocument.AFTER, ) + else: + if created_at is not None: + logger.warning( + "--created-at is ignored when updating an existing feature entry " "(type=%s). The original created_at is preserved.", + feature_type, + ) doc_id = str(result["_id"]) logger.info( @@ -524,7 +538,7 @@ def upsert_cluster_feature( # --------------------------------------------------------------------------- -def get_feature_status_by_id(mongo_url: str, doc_id: str, credentials: Optional[dict] = None) -> Optional[dict]: +def get_feature_status_by_id(mongo_url: str, doc_id: str) -> Optional[dict]: """Fetch a parent document by its ObjectId from either collection. Tries instance_level_config first, then cluster_level_config. @@ -546,7 +560,7 @@ def get_feature_status_by_id(mongo_url: str, doc_id: str, credentials: Optional[ except InvalidId: raise ValueError(f"'{doc_id}' is not a valid ObjectId (expected a 24-character hex string)") - client = _get_client(mongo_url, credentials) + client = MongoClient(mongo_url) try: for col_name in (COLLECTION_INSTANCE, COLLECTION_CLUSTER): doc = client[DATABASE][col_name].find_one({"_id": oid}) @@ -568,14 +582,13 @@ def get_instance_feature_by_criteria( cluster: str, subscription_id: str, feature_type: str, - credentials: Optional[dict] = None, ) -> Optional[dict]: """Fetch the feature entry for *feature_type* from instance_level_config. Returns the matching feature entry dict (not the full parent document), or None if the parent or the feature entry does not exist. """ - client = _get_client(mongo_url, credentials) + client = MongoClient(mongo_url) try: filter_doc = { "tenant_id": tenant_id, @@ -604,14 +617,13 @@ def get_cluster_feature_by_criteria( account: str, cluster: str, feature_type: str, - credentials: Optional[dict] = None, ) -> Optional[dict]: """Fetch the feature entry for *feature_type* from cluster_level_config. Returns the matching feature entry dict (not the full parent document), or None if the parent or the feature entry does not exist. """ - client = _get_client(mongo_url, credentials) + client = MongoClient(mongo_url) try: filter_doc = { "tenant_id": tenant_id, From 1a2ed5d243d21f2394f803bdba9f28df1e87133a Mon Sep 17 00:00:00 2001 From: Sagar-Talikoti Date: Thu, 1 Oct 2026 18:37:27 +0530 Subject: [PATCH 09/10] code update --- bin/mas-devops-feature-status-update | 29 +++++-------------------- bin/mas-devops-feature-status-update.md | 12 +++++----- 2 files changed, 12 insertions(+), 29 deletions(-) diff --git a/bin/mas-devops-feature-status-update b/bin/mas-devops-feature-status-update index b4f1409a..b3faa12c 100755 --- a/bin/mas-devops-feature-status-update +++ b/bin/mas-devops-feature-status-update @@ -16,13 +16,13 @@ DevOps MongoDB (mas_devops database). Collection routing ────────────────── --instance-id supplied → mas_devops.instance_level_config - One document per (tenant_id × subscription_id × + One document per (subscription_id × account × region × cluster × instance). The feature entry is embedded in instance_level_features[]. --instance-id omitted → mas_devops.cluster_level_config - One document per (tenant_id × account × region × + One document per (account × region × cluster). The feature entry is embedded in cluster_level_features[]. @@ -37,7 +37,6 @@ Sub-commands Example — instance-level, ACTIVE: mas-devops-feature-status-update status-update \\ - --tenant-id tenant-abc \\ --region us-east-2 \\ --instance-id inst02 \\ --account fyre-noble10-dev \\ @@ -52,7 +51,6 @@ Sub-commands Example — cluster-level (no --instance-id), ERROR: mas-devops-feature-status-update status-update \\ - --tenant-id tenant-abc \\ --region us-east-2 \\ --account fyre-noble10-dev \\ --cluster noble10 \\ @@ -75,14 +73,13 @@ Sub-commands get Fetch and pretty-print a feature status entry by ObjectId or by - criteria fields (tenant_id, region, account, cluster[, instance_id], type). + criteria fields (region, account, cluster[, instance_id], type). Example — by ObjectId: mas-devops-feature-status-update get --id 6ab0e70ee6d3a31faa808547 Example — instance-level by criteria: mas-devops-feature-status-update get \\ - --tenant-id tenant-abc \\ --region us-east-2 \\ --instance-id inst02 \\ --account fyre-noble10-dev \\ @@ -92,7 +89,6 @@ Sub-commands Example — cluster-level by criteria (no --instance-id): mas-devops-feature-status-update get \\ - --tenant-id tenant-abc \\ --region us-east-2 \\ --account fyre-noble10-dev \\ --cluster noble10 \\ @@ -226,11 +222,10 @@ def cmd_get(args) -> int: doc = get_feature_status_by_id(mongo_url, args.id) not_found_msg = f"No document found with ID: {args.id}" else: - # Criteria mode — tenant_id, region, account, cluster, type are always required. + # Criteria mode — region, account, cluster, type are always required. missing = [ f for f, v in [ - ("--tenant-id", args.tenant_id), ("--account", args.account), ("--cluster", args.cluster), ("--type", args.type), @@ -257,7 +252,6 @@ def cmd_get(args) -> int: return 1 doc = get_instance_feature_by_criteria( mongo_url, - tenant_id=args.tenant_id, region=args.region, instance_id=args.instance_id, account=args.account, @@ -267,7 +261,7 @@ def cmd_get(args) -> int: ) not_found_msg = ( f"No '{args.type}' feature entry found for " - f"tenant={args.tenant_id} region={args.region} " + f"region={args.region} " f"instance_id={args.instance_id} account={args.account} " f"cluster={args.cluster} subscription_id={args.subscription_id}" ) @@ -277,17 +271,12 @@ def cmd_get(args) -> int: return 1 doc = get_cluster_feature_by_criteria( mongo_url, - tenant_id=args.tenant_id, region=args.region, account=args.account, cluster=args.cluster, feature_type=args.type, ) - not_found_msg = ( - f"No '{args.type}' feature entry found for " - f"tenant={args.tenant_id} region={args.region} " - f"account={args.account} cluster={args.cluster}" - ) + not_found_msg = f"No '{args.type}' feature entry found for " f"region={args.region} " f"account={args.account} cluster={args.cluster}" except ValueError as exc: print(f"ERROR: {exc}", file=sys.stderr) return 1 @@ -392,7 +381,6 @@ def cmd_status_update(args) -> int: if level == INSTANCE_LEVEL: doc_id = upsert_instance_feature( mongo_url, - tenant_id=args.tenant_id, subscription_id=args.subscription_id, region=args.region, account=args.account, @@ -410,7 +398,6 @@ def cmd_status_update(args) -> int: else: doc_id = upsert_cluster_feature( mongo_url, - tenant_id=args.tenant_id, region=args.region, account=args.account, cluster=args.cluster, @@ -465,7 +452,6 @@ def build_parser() -> argparse.ArgumentParser: ) identity = su.add_argument_group("identity") - identity.add_argument("--tenant-id", required=True, dest="tenant_id", help="Tenant/customer identifier (multi-tenancy isolation key)") identity.add_argument("--region", required=True, help="AWS region (e.g. us-east-2)") identity.add_argument( "--instance-id", @@ -545,7 +531,6 @@ def build_parser() -> argparse.ArgumentParser: " mas-devops-feature-status-update get --id 6ab0e70ee6d3a31faa808547\n\n" " # instance-level by criteria\n" " mas-devops-feature-status-update get \\\n" - " --tenant-id tenant-abc \\\n" " --region us-east-2 \\\n" " --instance-id inst02 \\\n" " --account fyre-noble10-dev \\\n" @@ -554,7 +539,6 @@ def build_parser() -> argparse.ArgumentParser: " --type allow-list\n\n" " # cluster-level by criteria (no --instance-id)\n" " mas-devops-feature-status-update get \\\n" - " --tenant-id tenant-abc \\\n" " --region us-east-2 \\\n" " --account fyre-noble10-dev \\\n" " --cluster noble10 \\\n" @@ -575,7 +559,6 @@ def build_parser() -> argparse.ArgumentParser: help="AWS region (e.g. us-east-2). Use together with the other criteria flags.", ) get_criteria = get.add_argument_group("criteria (required when --region is used)") - get_criteria.add_argument("--tenant-id", dest="tenant_id", default=None, metavar="TENANT_ID", help="Tenant/customer identifier") get_criteria.add_argument( "--instance-id", dest="instance_id", diff --git a/bin/mas-devops-feature-status-update.md b/bin/mas-devops-feature-status-update.md index 4ed7504c..3ee8c6d1 100644 --- a/bin/mas-devops-feature-status-update.md +++ b/bin/mas-devops-feature-status-update.md @@ -376,8 +376,8 @@ The `feature_dashboard` MongoDB database must be initialised before this tool ca | Collection | Cardinality | |---|---| -| `cluster_level_config` | One document per `tenant_id × account × region × cluster` | -| `instance_level_config` | One document per `tenant_id × subscription_id × account × region × cluster × instance` | +| `cluster_level_config` | One document per `account × region × cluster` | +| `instance_level_config` | One document per `subscription_id × account × region × cluster × instance` | ### Initialize @@ -422,15 +422,15 @@ db.instance_level_config.drop() | Index name | Fields | Unique | |---|---|---| -| `ux_cluster_level_config_tenant_account_region_cluster` | `tenant_id, account, region, cluster` | ✓ | -| `ix_cluster_level_config_tenant_account` | `tenant_id, account` | | +| `ux_cluster_level_config_account_region_cluster` | `account, region, cluster` | ✓ | +| `ix_cluster_level_config_account` | `account` | | **`instance_level_config`** | Index name | Fields | Unique | |---|---|---| -| `ux_instance_level_config_tenant_sub_account_region_cluster_instance` | `tenant_id, subscription_id, account, region, cluster, instance` | ✓ | -| `ix_instance_level_config_tenant_sub_account_region_cluster` | `tenant_id, subscription_id, account, region, cluster` | | +| `ux_instance_level_config_sub_account_region_cluster_instance` | `subscription_id, account, region, cluster, instance` | ✓ | +| `ix_instance_level_config_sub_account_region_cluster` | `subscription_id, account, region, cluster` | | | `ix_instance_level_config_feature_status` | `instance_level_features.status` | | | `ix_instance_level_config_error_code` | `instance_level_features.status_details.error_code` (sparse) | | From bee35dca8ef862fb777d2f1b702aed4fdbb173c6 Mon Sep 17 00:00:00 2001 From: Sagar-Talikoti Date: Thu, 1 Oct 2026 19:59:56 +0530 Subject: [PATCH 10/10] code updates --- bin/mas-devops-feature-status-update | 110 +++++++++++---- bin/mas-devops-feature-status-update.md | 122 ++++++++++++---- mongodb_schemas/README.md | 169 ++++++++--------------- mongodb_schemas/cluster_level_config.js | 22 ++- mongodb_schemas/instance_level_config.js | 18 +-- src/mas/devops/feature_status.py | 39 ++---- 6 files changed, 267 insertions(+), 213 deletions(-) diff --git a/bin/mas-devops-feature-status-update b/bin/mas-devops-feature-status-update index b3faa12c..faa61411 100755 --- a/bin/mas-devops-feature-status-update +++ b/bin/mas-devops-feature-status-update @@ -35,7 +35,7 @@ Sub-commands Required indexes are created automatically on the first call if they do not already exist (idempotent). - Example — instance-level, ACTIVE: + Example — instance-level, ACTIVE (single IP): mas-devops-feature-status-update status-update \\ --region us-east-2 \\ --instance-id inst02 \\ @@ -49,27 +49,46 @@ Sub-commands --deployment-start 2026-09-11T11:48:42+00:00 \\ --deployment-end 2026-09-11T11:53:10+00:00 - Example — cluster-level (no --instance-id), ERROR: + Example — instance-level, ACTIVE (multiple IPs): mas-devops-feature-status-update status-update \\ --region us-east-2 \\ + --instance-id inst02 \\ --account fyre-noble10-dev \\ --cluster noble10 \\ --subscription-id sub-id01 \\ --type allow-list \\ - --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' \\ + --feature-details '{"ips": ["1.2.3.4/32", "2405:201:d000:9062::/64"]}' \\ + --status ACTIVE \\ + --status-details '{"message": "Allow list is active.", "request_configuration": "1.2.3.4/32, 2405:201:d000:9062::/64"}' \\ + --deployment-start 2026-09-11T11:48:42+00:00 \\ + --deployment-end 2026-09-11T11:53:10+00:00 + + Example — instance-level, ERROR (using --status-details-file to avoid shell quoting issues): + cat > /tmp/status-details.json <<'EOF' + { + "message": "sample error message", + "error_code": 401, + "error_source": { + "gitops_version": "8.6.0", + "filename": "cis_ip_allowlist.yml", + "log_file": "/var/log/gitops/run-001.log", + "stacktrace": "Traceback (most recent call last): ..." + }, + "request_configuration": "2405:201:d000:9060::/64" + } + EOF + mas-devops-feature-status-update status-update \\ + --region us-east-2 \\ + --instance-id inst02 \\ + --account fyre-noble10-dev \\ + --cluster noble10 \\ + --subscription-id sub-id01 \\ + --type allow-list \\ + --feature-details '{"ips": ["2405:201:d000:9060::/64"]}' \\ --status ERROR \\ - --status-details '{ - "message": "sample error message", - "error_code": 401, - "error_source": { - "gitops_version": "8.6.0", - "filename": "cis_ip_allowlist.yml", - "line_no": 148, - "log_file": "/var/log/gitops/run-001.log", - "stacktrace": "Traceback (most recent call last): ..." - }, - "request_configuration": "2405:201:d000:9060::/64" - }' + --status-details-file /tmp/status-details.json \\ + --deployment-start 2026-09-11T11:48:42+00:00 \\ + --deployment-end 2026-09-11T11:53:10+00:00 get Fetch and pretty-print a feature status entry by ObjectId or by @@ -163,6 +182,19 @@ def _parse_json_arg(value: str, arg_name: str) -> dict: return result +def _read_file_arg(path: str, arg_name: str) -> str: + """Read and return the contents of *path* for use as a CLI argument value. + + Raises SystemExit(1) if the file cannot be read. + """ + try: + with open(path) as fh: + return fh.read() + except OSError as exc: + print(f"ERROR: --{arg_name}: cannot read file '{path}': {exc}", file=sys.stderr) + sys.exit(1) + + def _parse_isodate(value: Optional[str], arg_name: str) -> Optional[datetime]: """Parse an ISO-8601 datetime, stripping MongoDB ISODate() wrappers.""" if value is None: @@ -301,8 +333,8 @@ def cmd_status_update(args) -> int: """Upsert a feature status entry into MongoDB. Collection routing is determined by FEATURE_LEVEL_MAP: - INSTANCE_LEVEL → instance_level_config (--instance-id required) - otherwise → cluster_level_config (--instance-id must be absent) + INSTANCE_LEVEL → instance_level_config (--instance-id and --subscription-id required) + otherwise → cluster_level_config (--instance-id and --subscription-id must be absent) Required indexes are created automatically if they do not already exist. """ @@ -330,17 +362,24 @@ def cmd_status_update(args) -> int: print(f"ERROR: {exc}", file=sys.stderr) return 1 - # Validate --instance-id consistency with the feature type's level. + # Validate --instance-id / --subscription-id consistency with the feature type's level. if level == INSTANCE_LEVEL and not args.instance_id: print(f"ERROR: --instance-id is required for instance-level feature type '{args.type}'", file=sys.stderr) return 1 + if level == INSTANCE_LEVEL and not args.subscription_id: + print(f"ERROR: --subscription-id is required for instance-level feature type '{args.type}'", file=sys.stderr) + return 1 if level != INSTANCE_LEVEL and args.instance_id: print(f"ERROR: --instance-id must not be supplied for cluster-level feature type '{args.type}'", file=sys.stderr) return 1 - # Parse JSON arguments + # Parse JSON arguments — --status-details-file takes precedence over --status-details feature_details = _parse_json_arg(args.feature_details, "feature-details") - status_details = _parse_json_arg(args.status_details, "status-details") + if args.status_details_file: + status_details_raw = _read_file_arg(args.status_details_file, "status-details-file") + status_details = _parse_json_arg(status_details_raw, "status-details-file") + else: + status_details = _parse_json_arg(args.status_details, "status-details") # Type-specific validation try: @@ -462,7 +501,13 @@ def build_parser() -> argparse.ArgumentParser: ) identity.add_argument("--account", required=True, help="GitOps account name (e.g. fyre-noble10-dev)") identity.add_argument("--cluster", required=True, help="GitOps cluster name (e.g. noble10)") - identity.add_argument("--subscription-id", required=True, dest="subscription_id", help="Subscription ID") + identity.add_argument( + "--subscription-id", + required=False, + default=None, + dest="subscription_id", + help="Subscription ID. Required for instance-level feature types; must be omitted for cluster-level types.", + ) feature = su.add_argument_group("feature") feature.add_argument( @@ -485,15 +530,30 @@ def build_parser() -> argparse.ArgumentParser: choices=["REQUESTED", "IN_PROGRESS", "ACTIVE", "ERROR"], help="Feature lifecycle status.", ) - status.add_argument( + status_details_group = status.add_mutually_exclusive_group(required=True) + status_details_group.add_argument( "--status-details", - required=True, + default=None, dest="status_details", metavar="JSON", help=( "JSON object describing the outcome. " - "ACTIVE: {message, request_configuration}. " - "ERROR: {message, error_code, error_source, request_configuration}." + "ACTIVE/REQUESTED/IN_PROGRESS: {message, request_configuration}. " + "ERROR: {message, error_code, error_source, request_configuration}. " + "request_configuration may be a single CIDR or a comma/space-separated list. " + "Mutually exclusive with --status-details-file." + ), + ) + status_details_group.add_argument( + "--status-details-file", + default=None, + dest="status_details_file", + metavar="FILE", + help=( + "Path to a JSON file containing the status-details object. " + "Useful for ERROR payloads whose message or stacktrace contains quotes " + "that would break inline shell quoting. " + "Mutually exclusive with --status-details." ), ) diff --git a/bin/mas-devops-feature-status-update.md b/bin/mas-devops-feature-status-update.md index 3ee8c6d1..f4b44a16 100644 --- a/bin/mas-devops-feature-status-update.md +++ b/bin/mas-devops-feature-status-update.md @@ -142,15 +142,15 @@ Required collection indexes (`instance_config_level`, `cluster_config_level`) ar **Idempotency:** Safe to call multiple times with the same arguments. The underlying `find_one_and_update` with `upsert=True` guarantees that re-running with the same identity key produces the same final document state. `created_at` is set only on the first insert (`$setOnInsert`); subsequent calls update `updated_at` and all mutable fields without creating duplicate documents. -**Identity options** *(all required)* +**Identity options** -| Flag | Description | -|------|-------------| -| `--region` | AWS region (e.g. `us-east-2`) | -| `--instance-id` | MAS instance ID (e.g. `inst02`) | -| `--account` | GitOps account name (e.g. `fyre-noble10-dev`) | -| `--cluster` | GitOps cluster name (e.g. `noble10`) | -| `--subscription-id` | Subscription ID | +| Flag | Required | Description | +|------|----------|-------------| +| `--region` | Yes | AWS region (e.g. `us-east-2`) | +| `--instance-id` | Instance-level types | MAS instance ID (e.g. `inst02`). When supplied routes to `instance_level_config`; when omitted routes to `cluster_level_config`. | +| `--account` | Yes | GitOps account name (e.g. `fyre-noble10-dev`) | +| `--cluster` | Yes | GitOps cluster name (e.g. `noble10`) | +| `--subscription-id` | Instance-level types | Subscription ID. Required when `--instance-id` is supplied; must be omitted for cluster-level feature types. | **Feature options** *(all required)* @@ -164,7 +164,8 @@ Required collection indexes (`instance_config_level`, `cluster_config_level`) ar | Flag | Description | |------|-------------| | `--status` | One of `REQUESTED`, `IN_PROGRESS`, `ACTIVE`, `ERROR` | -| `--status-details JSON` | JSON object describing the outcome (see schema below) | +| `--status-details JSON` | JSON object describing the outcome (see schema below). Mutually exclusive with `--status-details-file`. | +| `--status-details-file FILE` | Path to a JSON file containing the status-details object. Use this for `ERROR` payloads whose `message` or `stacktrace` contains quote characters that would break inline shell interpolation. Mutually exclusive with `--status-details`. | **Timestamp options** *(all optional, default: current UTC time)* @@ -179,6 +180,8 @@ The MongoDB connection URI is read from the `DEVOPS_MONGO_URI` environment varia **`--status-details` schema** +`request_configuration` is a free-form string describing what was requested — it may be a single CIDR, a comma-separated list, or a space-separated list. The validator does not enforce format. + *REQUESTED* — pipeline has received the request but processing has not yet started. ```json { @@ -195,15 +198,15 @@ The MongoDB connection URI is read from the `DEVOPS_MONGO_URI` environment varia } ``` -*ACTIVE* — deployment completed successfully. +*ACTIVE* — deployment completed successfully. Multiple IPs may be comma-separated. ```json { "message": "Allow list is active.", - "request_configuration": "2405:201:d000:9062::/64" + "request_configuration": "1.2.3.4/32, 2405:201:d000:9062::/64" } ``` -*ERROR* — deployment failed. +*ERROR* — deployment failed. `error_source` fields are all optional except that the object itself is required. Use `--status-details-file` when `message` or `stacktrace` may contain quote characters. ```json { "message": "sample error message", @@ -211,7 +214,6 @@ The MongoDB connection URI is read from the `DEVOPS_MONGO_URI` environment varia "error_source": { "gitops_version": "8.6.0", "filename": "cis_ip_allowlist.yml", - "line_no": 148, "log_file": "/var/log/gitops/run-001.log", "stacktrace": "Traceback (most recent call last): ..." }, @@ -248,7 +250,7 @@ mas-devops-feature-status-update status-update \ --status-details '{"message": "Allow list deployment in progress.", "request_configuration": "2405:201:d000:9062::/64"}' \ --deployment-start 2026-09-11T11:48:42+00:00 -# ACTIVE status — record successful completion +# ACTIVE status — single IP mas-devops-feature-status-update status-update \ --region us-east-2 \ --instance-id inst02 \ @@ -262,7 +264,7 @@ mas-devops-feature-status-update status-update \ --deployment-start 2026-09-11T11:48:42+00:00 \ --deployment-end 2026-09-11T11:53:10+00:00 -# ERROR status — record a failed deployment +# ACTIVE status — multiple IPs (request_configuration is comma-separated) mas-devops-feature-status-update status-update \ --region us-east-2 \ --instance-id inst02 \ @@ -270,20 +272,36 @@ mas-devops-feature-status-update status-update \ --cluster noble10 \ --subscription-id sub-id01 \ --type allow-list \ - --feature-details '{"ips": ["2405:201:d000:9062::/64"]}' \ + --feature-details '{"ips": ["1.2.3.4/32", "2405:201:d000:9062::/64"]}' \ + --status ACTIVE \ + --status-details '{"message": "Allow list is active.", "request_configuration": "1.2.3.4/32, 2405:201:d000:9062::/64"}' \ + --deployment-start 2026-09-11T11:48:42+00:00 \ + --deployment-end 2026-09-11T11:53:10+00:00 + +# ERROR status — use --status-details-file to avoid shell quoting issues with error text +cat > /tmp/status-details.json <<'EOF' +{ + "message": "sample error message", + "error_code": 401, + "error_source": { + "gitops_version": "8.6.0", + "filename": "cis_ip_allowlist.yml", + "log_file": "/var/log/gitops/run-001.log", + "stacktrace": "Traceback (most recent call last): ..." + }, + "request_configuration": "2405:201:d000:9060::/64" +} +EOF +mas-devops-feature-status-update status-update \ + --region us-east-2 \ + --instance-id inst02 \ + --account fyre-noble10-dev \ + --cluster noble10 \ + --subscription-id sub-id01 \ + --type allow-list \ + --feature-details '{"ips": ["2405:201:d000:9060::/64"]}' \ --status ERROR \ - --status-details '{ - "message": "sample error message", - "error_code": 401, - "error_source": { - "gitops_version": "8.6.0", - "filename": "cis_ip_allowlist.yml", - "line_no": 148, - "log_file": "/var/log/gitops/run-001.log", - "stacktrace": "Traceback (most recent call last): ..." - }, - "request_configuration": "2405:201:d000:9060::/64" - }' \ + --status-details-file /tmp/status-details.json \ --deployment-start 2026-09-11T11:48:42+00:00 \ --deployment-end 2026-09-11T11:53:10+00:00 ``` @@ -414,6 +432,16 @@ db.cluster_level_config.drop() db.instance_level_config.drop() ``` +### Drop collections (removes schema & indexes) with auth +``` +mongosh "mongodb://mas_devops_user:mas_devops_password@localhost:27017/mas_devops?authSource=mas_devops&tls=false&tlsAllowInvalidCertificates=true" \ # pragma: allowlist secret + --eval " +db.cluster_level_config.drop() +db.instance_level_config.drop() +print('collections dropped') +" +``` + > **Note:** `drop()` removes the collection, all documents, and all indexes. Re-run `init_db.js` to recreate them. ### Indexes created by `init_db.js` @@ -505,6 +533,44 @@ See the full sample playbook at [`playbooks/feature-status-update.yml`](../playb changed_when: "'written successfully' in status_update_result.stdout" ``` +For `ERROR` status, write the payload to a file first to avoid shell quoting problems with error text: + +```yaml +- name: Write ERROR status-details to file + ansible.builtin.copy: + dest: /tmp/mas-status-details.json + content: | + { + "message": "{{ _error_msg | replace('\\', '\\\\') | replace('"', '\\"') }}", + "error_code": {{ _error_code }}, + "error_source": { + "gitops_version": "{{ lookup('env', 'GITOPS_VERSION') | default('', true) }}", + "filename": "{{ _error_filename }}", + "log_file": "{{ lookup('env', 'JUNIT_OUTPUT_DIR') | default('/var/log/gitops', true) }}/run.log", + "stacktrace": "{{ _error_msg | replace('\\', '\\\\') | replace('"', '\\"') }}" + }, + "request_configuration": "{{ _request_configuration }}" + } + +- name: Upsert feature status (ERROR) + ansible.builtin.command: + cmd: >- + mas-devops-feature-status-update status-update + --region {{ mas_region }} + --instance-id {{ mas_instance_id }} + --account {{ mas_account }} + --cluster {{ mas_cluster }} + --subscription-id {{ mas_subscription_id }} + --type allow-list + --feature-details {{ ('{"ips": ' + _normalised_ips | to_json + '}') | quote }} + --status ERROR + --status-details-file /tmp/mas-status-details.json + --deployment-start {{ _deployment_start }} + --deployment-end {{ _deployment_end }} + register: status_update_result + changed_when: "'written successfully' in status_update_result.stdout" +``` + ### Minimal task — `get` **By ObjectId** — extract the document ID from `status-update` output and fetch the written document: diff --git a/mongodb_schemas/README.md b/mongodb_schemas/README.md index 0959d740..4044e931 100644 --- a/mongodb_schemas/README.md +++ b/mongodb_schemas/README.md @@ -9,8 +9,8 @@ The original `allowlisting_config` collection has been split into two flat colle | File | Collection | Cardinality | Description | |---|---|---|---| -| [`cluster_level_config.js`](cluster_level_config.js) | `cluster_level_config` | 1 doc per `(tenant_id × account × region × cluster)` | Cluster-scoped feature entries (e.g. `dro`). | -| [`instance_level_config.js`](instance_level_config.js) | `instance_level_config` | 1 doc per `(tenant_id × account × region × cluster × instance)` | Instance-scoped IP/CIDR allowlisting entries with per-feature deployment lifecycle state. | +| [`cluster_level_config.js`](cluster_level_config.js) | `cluster_level_config` | 1 doc per `(account × region × cluster)` | Cluster-scoped feature entries (e.g. `dro`). | +| [`instance_level_config.js`](instance_level_config.js) | `instance_level_config` | 1 doc per `(account × region × cluster × instance)` | Instance-scoped IP/CIDR allowlisting entries with per-feature deployment lifecycle state. | | [`init_db.js`](init_db.js) | *(all)* | — | Bootstrap runner — initialises both collections and all indexes. | ### Key fields per collection @@ -19,7 +19,6 @@ The original `allowlisting_config` collection has been split into two flat colle | Field | Type | Notes | |---|---|---| -| `tenant_id` | string | Multi-tenancy isolation key | | `account` | string | Account from cluster polling | | `region` | string | e.g. `us-east-1` | | `cluster` | string | Cluster name from polling | @@ -29,7 +28,6 @@ The original `allowlisting_config` collection has been split into two flat colle | Field | Type | Notes | |---|---|---| -| `tenant_id` | string | Multi-tenancy isolation key | | `account` | string | Account from cluster polling | | `region` | string | e.g. `us-east-1` | | `cluster` | string | Cluster name from polling | @@ -49,7 +47,7 @@ The original `allowlisting_config` collection has been split into two flat colle ## Query reference All queries assume `use mas_devops` has been run first. Replace -``, ``, ``, ``, ``, and +``, ``, ``, ``, and `` with real values. --- @@ -62,20 +60,11 @@ All queries assume `use mas_devops` has been run first. Replace db.cluster_level_config.find().pretty() ``` -#### List all clusters for a tenant - -```js -db.cluster_level_config.find( - { tenant_id: "" }, - { _id: 0, account: 1, region: 1, cluster: 1 } -).pretty() -``` - #### List all clusters for an account ```js db.cluster_level_config.find( - { tenant_id: "", account: "" }, + { account: "" }, { _id: 0, region: 1, cluster: 1 } ).pretty() ``` @@ -84,7 +73,7 @@ db.cluster_level_config.find( ```js db.cluster_level_config.find( - { tenant_id: "", account: "", region: "" }, + { account: "", region: "" }, { _id: 0, cluster: 1 } ).pretty() ``` @@ -93,10 +82,9 @@ db.cluster_level_config.find( ```js db.cluster_level_config.findOne({ - tenant_id: "", - account: "", - region: "", - cluster: "" + account: "", + region: "", + cluster: "" }) ``` @@ -104,7 +92,7 @@ db.cluster_level_config.findOne({ ```js db.cluster_level_config.findOne( - { tenant_id: "", account: "", region: "", cluster: "" }, + { account: "", region: "", cluster: "" }, { _id: 0, cluster_level_features: 1 } ) ``` @@ -113,10 +101,7 @@ db.cluster_level_config.findOne( ```js db.cluster_level_config.find( - { - tenant_id: "", - "cluster_level_features.feature_key": "" - }, + { "cluster_level_features.feature_key": "" }, { _id: 0, account: 1, region: 1, cluster: 1, cluster_level_features: 1 } ).pretty() ``` @@ -125,7 +110,7 @@ db.cluster_level_config.find( ```js db.cluster_level_config.find( - { tenant_id: "", "cluster_level_features.0": { $exists: true } }, + { "cluster_level_features.0": { $exists: true } }, { _id: 0, account: 1, region: 1, cluster: 1 } ).pretty() ``` @@ -134,26 +119,25 @@ db.cluster_level_config.find( ```js db.cluster_level_config.find( - { tenant_id: "", cluster_level_features: { $size: 0 } }, + { cluster_level_features: { $size: 0 } }, { _id: 0, account: 1, region: 1, cluster: 1 } ).pretty() ``` -#### Count clusters per region (for a tenant) +#### Count clusters per region for an account ```js db.cluster_level_config.aggregate([ - { $match: { tenant_id: "" } }, - { $group: { _id: { account: "$account", region: "$region" }, cluster_count: { $sum: 1 } } }, - { $sort: { "_id.account": 1, "_id.region": 1 } } + { $match: { account: "" } }, + { $group: { _id: "$region", cluster_count: { $sum: 1 } } }, + { $sort: { _id: 1 } } ]) ``` -#### Count clusters per account (for a tenant) +#### Count clusters per account ```js db.cluster_level_config.aggregate([ - { $match: { tenant_id: "" } }, { $group: { _id: "$account", cluster_count: { $sum: 1 } } }, { $sort: { _id: 1 } } ]) @@ -169,20 +153,11 @@ db.cluster_level_config.aggregate([ db.instance_level_config.find().pretty() ``` -#### List all instances for a tenant - -```js -db.instance_level_config.find( - { tenant_id: "" }, - { _id: 0, account: 1, region: 1, cluster: 1, instance: 1 } -).pretty() -``` - #### List all instances for an account ```js db.instance_level_config.find( - { tenant_id: "", account: "" }, + { account: "" }, { _id: 0, region: 1, cluster: 1, instance: 1 } ).pretty() ``` @@ -191,7 +166,7 @@ db.instance_level_config.find( ```js db.instance_level_config.find( - { tenant_id: "", account: "", region: "" }, + { account: "", region: "" }, { _id: 0, cluster: 1, instance: 1 } ).pretty() ``` @@ -200,7 +175,7 @@ db.instance_level_config.find( ```js db.instance_level_config.find( - { tenant_id: "", account: "", region: "", cluster: "" }, + { account: "", region: "", cluster: "" }, { _id: 0, instance: 1 } ).pretty() ``` @@ -209,11 +184,10 @@ db.instance_level_config.find( ```js db.instance_level_config.findOne({ - tenant_id: "", - account: "", - region: "", - cluster: "", - instance: "" + account: "", + region: "", + cluster: "", + instance: "" }) ``` @@ -222,11 +196,10 @@ db.instance_level_config.findOne({ ```js db.instance_level_config.findOne( { - tenant_id: "", - account: "", - region: "", - cluster: "", - instance: "" + account: "", + region: "", + cluster: "", + instance: "" }, { _id: 0, instance_level_features: 1 } ) @@ -239,11 +212,10 @@ Returns just the matching element from `instance_level_features[]` using `$elemM ```js db.instance_level_config.findOne( { - tenant_id: "", - account: "", - region: "", - cluster: "", - instance: "" + account: "", + region: "", + cluster: "", + instance: "" }, { _id: 0, @@ -258,10 +230,7 @@ db.instance_level_config.findOne( ```js db.instance_level_config.find( - { - tenant_id: "", - "instance_level_features.feature_key": "" - }, + { "instance_level_features.feature_key": "" }, { _id: 0, account: 1, region: 1, cluster: 1, instance: 1 } ).pretty() ``` @@ -270,10 +239,7 @@ db.instance_level_config.find( ```js db.instance_level_config.find( - { - tenant_id: "", - "instance_level_features.feature_key": "" - }, + { "instance_level_features.feature_key": "" }, { _id: 0, account: 1, region: 1, cluster: 1, instance: 1, @@ -287,7 +253,6 @@ db.instance_level_config.find( ```js db.instance_level_config.find( { - tenant_id: "", instance_level_features: { $elemMatch: { feature_key: "", @@ -304,7 +269,6 @@ db.instance_level_config.find( ```js db.instance_level_config.find( { - tenant_id: "", instance_level_features: { $elemMatch: { feature_key: "", @@ -321,7 +285,6 @@ db.instance_level_config.find( ```js db.instance_level_config.find( { - tenant_id: "", "instance_level_features.cluster_poll_status": { $in: ["stale", "unreachable"] } }, { _id: 0, account: 1, region: 1, cluster: 1, instance: 1, @@ -337,7 +300,6 @@ db.instance_level_config.find( ```js db.instance_level_config.find( { - tenant_id: "", "instance_level_features.cluster_last_polled_at": { $lt: ISODate("") } @@ -350,10 +312,7 @@ db.instance_level_config.find( ```js db.instance_level_config.find( - { - tenant_id: "", - "instance_level_features.cluster_last_polled_at": { $exists: false } - }, + { "instance_level_features.cluster_last_polled_at": { $exists: false } }, { _id: 0, account: 1, region: 1, cluster: 1, instance: 1 } ).pretty() ``` @@ -365,26 +324,21 @@ db.instance_level_config.find( #### List all distinct regions for an account ```js -// From cluster_level_config (one query covers all clusters, hence all regions) -db.cluster_level_config.distinct("region", { - tenant_id: "", - account: "" -}) +db.cluster_level_config.distinct("region", { account: "" }) ``` -#### List all distinct accounts for a tenant +#### List all distinct accounts ```js -db.cluster_level_config.distinct("account", { tenant_id: "" }) +db.cluster_level_config.distinct("account") ``` #### List all distinct clusters in a region ```js db.cluster_level_config.distinct("cluster", { - tenant_id: "", - account: "", - region: "" + account: "", + region: "" }) ``` @@ -392,10 +346,9 @@ db.cluster_level_config.distinct("cluster", { ```js db.instance_level_config.distinct("instance", { - tenant_id: "", - account: "", - region: "", - cluster: "" + account: "", + region: "", + cluster: "" }) ``` @@ -408,10 +361,9 @@ using `$lookup`. db.cluster_level_config.aggregate([ { $match: { - tenant_id: "", - account: "", - region: "", - cluster: "" + account: "", + region: "", + cluster: "" } }, { @@ -420,7 +372,6 @@ db.cluster_level_config.aggregate([ localField: "cluster", foreignField: "cluster", let: { - t: "$tenant_id", a: "$account", r: "$region", c: "$cluster" @@ -430,10 +381,9 @@ db.cluster_level_config.aggregate([ $match: { $expr: { $and: [ - { $eq: ["$tenant_id", "$$t"] }, - { $eq: ["$account", "$$a"] }, - { $eq: ["$region", "$$r"] }, - { $eq: ["$cluster", "$$c"] } + { $eq: ["$account", "$$a"] }, + { $eq: ["$region", "$$r"] }, + { $eq: ["$cluster", "$$c"] } ] } } @@ -454,11 +404,10 @@ db.cluster_level_config.aggregate([ ]) ``` -#### Count instances per cluster across all clusters for a tenant +#### Count instances per cluster across all clusters ```js db.instance_level_config.aggregate([ - { $match: { tenant_id: "" } }, { $group: { _id: { account: "$account", region: "$region", cluster: "$cluster" }, @@ -473,7 +422,7 @@ db.instance_level_config.aggregate([ ```js db.instance_level_config.aggregate([ - { $match: { tenant_id: "", account: "" } }, + { $match: { account: "" } }, { $group: { _id: "$region", instance_count: { $sum: 1 } } }, { $sort: { _id: 1 } } ]) @@ -500,22 +449,22 @@ use mas_devops // cluster_level_config indexes db.cluster_level_config.createIndex( - { tenant_id: 1, account: 1, region: 1, cluster: 1 }, - { unique: true, name: "ux_cluster_level_config_tenant_account_region_cluster" } + { account: 1, region: 1, cluster: 1 }, + { unique: true, name: "ux_cluster_level_config_account_region_cluster" } ); db.cluster_level_config.createIndex( - { tenant_id: 1, account: 1 }, - { name: "ix_cluster_level_config_tenant_account" } + { account: 1 }, + { name: "ix_cluster_level_config_account" } ); // instance_level_config indexes db.instance_level_config.createIndex( - { tenant_id: 1, subscription_id: 1, account: 1, region: 1, cluster: 1, instance: 1 }, - { unique: true, name: "ux_instance_level_config_tenant_sub_account_region_cluster_instance" } + { subscription_id: 1, account: 1, region: 1, cluster: 1, instance: 1 }, + { unique: true, name: "ux_instance_level_config_sub_account_region_cluster_instance" } ); db.instance_level_config.createIndex( - { tenant_id: 1, subscription_id: 1, account: 1, region: 1, cluster: 1 }, - { name: "ix_instance_level_config_tenant_sub_account_region_cluster" } + { subscription_id: 1, account: 1, region: 1, cluster: 1 }, + { name: "ix_instance_level_config_sub_account_region_cluster" } ); db.instance_level_config.createIndex( { "instance_level_features.status": 1 }, diff --git a/mongodb_schemas/cluster_level_config.js b/mongodb_schemas/cluster_level_config.js index 75e59936..5b2c9f17 100644 --- a/mongodb_schemas/cluster_level_config.js +++ b/mongodb_schemas/cluster_level_config.js @@ -2,7 +2,7 @@ // Collection: cluster_level_config // Database: feature_dashboard // Purpose: Stores cluster-scoped feature entries. Each document represents -// one cluster within a region/account/tenant and holds the list of +// one cluster within a region/account and holds the list of // cluster-level features enabled for that cluster (e.g. 'dro'). // // Split from allowlisting_config: cluster_level_features[] was @@ -12,7 +12,7 @@ // allows targeted index coverage without touching instance data. // // Document cardinality: -// ONE document per (tenant_id × account × region × cluster). +// ONE document per (account × region × cluster). // ============================================================================= db.createCollection("cluster_level_config", { @@ -20,7 +20,7 @@ db.createCollection("cluster_level_config", { $jsonSchema: { bsonType: "object", required: [ - "_id", "tenant_id", "account", "region", "cluster", + "_id", "account", "region", "cluster", "cluster_level_features", "created_at", "updated_at" ], additionalProperties: false, @@ -33,10 +33,6 @@ db.createCollection("cluster_level_config", { bsonType: "objectId", description: "MongoDB-generated document identifier." }, - tenant_id: { - bsonType: "string", - description: "Tenant/customer identifier. All customer-scoped queries MUST filter on this field. This is the multi-tenancy isolation key." - }, account: { bsonType: "string", description: "Account identifier as returned by the cluster polling mechanism." @@ -87,15 +83,15 @@ db.createCollection("cluster_level_config", { // --------------------------------------------------------------------------- // Compound unique index — enforces the one-document-per -// (tenant × account × region × cluster) invariant +// (account × region × cluster) invariant db.cluster_level_config.createIndex( - { tenant_id: 1, account: 1, region: 1, cluster: 1 }, - { unique: true, name: "ux_cluster_level_config_tenant_account_region_cluster" } + { account: 1, region: 1, cluster: 1 }, + { unique: true, name: "ux_cluster_level_config_account_region_cluster" } ); -// Index for querying all clusters for a given tenant + account +// Index for querying all clusters for a given account db.cluster_level_config.createIndex( - { tenant_id: 1, account: 1 }, - { name: "ix_cluster_level_config_tenant_account" } + { account: 1 }, + { name: "ix_cluster_level_config_account" } ); diff --git a/mongodb_schemas/instance_level_config.js b/mongodb_schemas/instance_level_config.js index b8c55bff..38bbdf8a 100644 --- a/mongodb_schemas/instance_level_config.js +++ b/mongodb_schemas/instance_level_config.js @@ -15,7 +15,7 @@ // scaling of cluster and instance data. // // Document cardinality: -// ONE document per (tenant_id × subscription_id × account × region × cluster × instance). +// ONE document per (subscription_id × account × region × cluster × instance). // // instance_level_features item shape (flattened — no nested allow_lists[]): // @@ -67,7 +67,7 @@ db.createCollection("instance_level_config", { $jsonSchema: { bsonType: "object", required: [ - "_id", "tenant_id", "subscription_id", "account", "region", + "_id", "subscription_id", "account", "region", "cluster", "instance", "instance_level_features", "created_at", "updated_at" ], additionalProperties: false, @@ -80,10 +80,6 @@ db.createCollection("instance_level_config", { bsonType: "objectId", description: "MongoDB-generated document identifier." }, - tenant_id: { - bsonType: "string", - description: "Tenant/customer identifier. All customer-scoped queries MUST filter on this field. This is the multi-tenancy isolation key." - }, subscription_id: { bsonType: "string", description: "Subscription identifier associated with the tenant/instance. Example: 'sub-id01'." @@ -259,16 +255,16 @@ db.createCollection("instance_level_config", { // --------------------------------------------------------------------------- // Compound unique index — enforces the one-document-per -// (tenant × subscription × account × region × cluster × instance) invariant +// (subscription × account × region × cluster × instance) invariant db.instance_level_config.createIndex( - { tenant_id: 1, subscription_id: 1, account: 1, region: 1, cluster: 1, instance: 1 }, - { unique: true, name: "ux_instance_level_config_tenant_sub_account_region_cluster_instance" } + { subscription_id: 1, account: 1, region: 1, cluster: 1, instance: 1 }, + { unique: true, name: "ux_instance_level_config_sub_account_region_cluster_instance" } ); // Index for ansible-devops / GitHub webhook upserts — primary lookup path db.instance_level_config.createIndex( - { tenant_id: 1, subscription_id: 1, account: 1, region: 1, cluster: 1 }, - { name: "ix_instance_level_config_tenant_sub_account_region_cluster" } + { subscription_id: 1, account: 1, region: 1, cluster: 1 }, + { name: "ix_instance_level_config_sub_account_region_cluster" } ); // Multikey index on feature status — supports finding all documents with diff --git a/src/mas/devops/feature_status.py b/src/mas/devops/feature_status.py index a3a095c4..f56eef9d 100644 --- a/src/mas/devops/feature_status.py +++ b/src/mas/devops/feature_status.py @@ -12,12 +12,12 @@ Database: mas_devops Collections: - instance_level_config — one document per (tenant_id × subscription_id × + instance_level_config — one document per (subscription_id × account × region × cluster × instance). Feature entries are embedded in instance_level_features[]. Used when --instance-id is supplied. - cluster_level_config — one document per (tenant_id × account × region × cluster). + cluster_level_config — one document per (account × region × cluster). Feature entries are embedded in cluster_level_features[]. Used when --instance-id is omitted. @@ -25,7 +25,6 @@ { "_id": , - "tenant_id": str, "subscription_id": str, "account": str, "region": str, @@ -53,7 +52,6 @@ { "_id": , - "tenant_id": str, "account": str, "region": str, "cluster": str, @@ -219,7 +217,6 @@ def create_indexes(mongo_url: str) -> None: inst.create_index( [ - ("tenant_id", ASCENDING), ("subscription_id", ASCENDING), ("account", ASCENDING), ("region", ASCENDING), @@ -227,21 +224,20 @@ def create_indexes(mongo_url: str) -> None: ("instance", ASCENDING), ], unique=True, - name="ux_instance_level_config_tenant_sub_account_region_cluster_instance", + name="ux_instance_level_config_sub_account_region_cluster_instance", ) - logger.info("Index 'ux_instance_level_config_tenant_sub_account_region_cluster_instance' ensured on %s.%s", DATABASE, COLLECTION_INSTANCE) + logger.info("Index 'ux_instance_level_config_sub_account_region_cluster_instance' ensured on %s.%s", DATABASE, COLLECTION_INSTANCE) inst.create_index( [ - ("tenant_id", ASCENDING), ("subscription_id", ASCENDING), ("account", ASCENDING), ("region", ASCENDING), ("cluster", ASCENDING), ], - name="ix_instance_level_config_tenant_sub_account_region_cluster", + name="ix_instance_level_config_sub_account_region_cluster", ) - logger.info("Index 'ix_instance_level_config_tenant_sub_account_region_cluster' ensured on %s.%s", DATABASE, COLLECTION_INSTANCE) + logger.info("Index 'ix_instance_level_config_sub_account_region_cluster' ensured on %s.%s", DATABASE, COLLECTION_INSTANCE) inst.create_index( [("instance_level_features.status", ASCENDING)], @@ -261,21 +257,20 @@ def create_indexes(mongo_url: str) -> None: clst.create_index( [ - ("tenant_id", ASCENDING), ("account", ASCENDING), ("region", ASCENDING), ("cluster", ASCENDING), ], unique=True, - name="ux_cluster_level_config_tenant_account_region_cluster", + name="ux_cluster_level_config_account_region_cluster", ) - logger.info("Index 'ux_cluster_level_config_tenant_account_region_cluster' ensured on %s.%s", DATABASE, COLLECTION_CLUSTER) + logger.info("Index 'ux_cluster_level_config_account_region_cluster' ensured on %s.%s", DATABASE, COLLECTION_CLUSTER) clst.create_index( - [("tenant_id", ASCENDING), ("account", ASCENDING)], - name="ix_cluster_level_config_tenant_account", + [("account", ASCENDING)], + name="ix_cluster_level_config_account", ) - logger.info("Index 'ix_cluster_level_config_tenant_account' ensured on %s.%s", DATABASE, COLLECTION_CLUSTER) + logger.info("Index 'ix_cluster_level_config_account' ensured on %s.%s", DATABASE, COLLECTION_CLUSTER) finally: client.close() @@ -323,7 +318,6 @@ def _build_feature_entry( def upsert_instance_feature( mongo_url: str, *, - tenant_id: str, subscription_id: str, region: str, account: str, @@ -341,7 +335,7 @@ def upsert_instance_feature( """Upsert a feature entry inside instance_level_config. The parent document is identified by - (tenant_id, subscription_id, account, region, cluster, instance). + (subscription_id, account, region, cluster, instance). If a feature entry with the same *type* already exists it is updated in-place via a single atomic find_one_and_update with arrayFilters; otherwise the entry is appended (with parent upsert if needed). @@ -358,7 +352,6 @@ def upsert_instance_feature( entry = _build_feature_entry(feature_type, feature_details, status, status_details, deployment_start, deployment_end, now, created_at, updated_at) parent_filter = { - "tenant_id": tenant_id, "subscription_id": subscription_id, "account": account, "region": region, @@ -436,7 +429,6 @@ def upsert_instance_feature( def upsert_cluster_feature( mongo_url: str, *, - tenant_id: str, region: str, account: str, cluster: str, @@ -451,7 +443,7 @@ def upsert_cluster_feature( ) -> str: """Upsert a feature entry inside cluster_level_config. - The parent document is identified by (tenant_id, account, region, cluster). + The parent document is identified by (account, region, cluster). Same atomic two-step pattern as upsert_instance_feature. Returns the parent document _id as a string. @@ -466,7 +458,6 @@ def upsert_cluster_feature( entry = _build_feature_entry(feature_type, feature_details, status, status_details, deployment_start, deployment_end, now, created_at, updated_at) parent_filter = { - "tenant_id": tenant_id, "account": account, "region": region, "cluster": cluster, @@ -575,7 +566,6 @@ def get_feature_status_by_id(mongo_url: str, doc_id: str) -> Optional[dict]: def get_instance_feature_by_criteria( mongo_url: str, *, - tenant_id: str, region: str, instance_id: str, account: str, @@ -591,7 +581,6 @@ def get_instance_feature_by_criteria( client = MongoClient(mongo_url) try: filter_doc = { - "tenant_id": tenant_id, "subscription_id": subscription_id, "account": account, "region": region, @@ -612,7 +601,6 @@ def get_instance_feature_by_criteria( def get_cluster_feature_by_criteria( mongo_url: str, *, - tenant_id: str, region: str, account: str, cluster: str, @@ -626,7 +614,6 @@ def get_cluster_feature_by_criteria( client = MongoClient(mongo_url) try: filter_doc = { - "tenant_id": tenant_id, "account": account, "region": region, "cluster": cluster,