From cd5fe306cf56f64574a3b7f6382f2a387e72a1bf Mon Sep 17 00:00:00 2001 From: exec-unit Date: Sun, 15 Feb 2026 13:19:16 +0700 Subject: [PATCH 1/3] ci(scripts): add python utilities for yc registry cleanup and vault generation --- .github/scripts/cleanup-registry-yc.py | 236 +++++++++++++++++++++++++ .github/scripts/generate-vault.py | 96 ++++++++++ 2 files changed, 332 insertions(+) create mode 100644 .github/scripts/cleanup-registry-yc.py create mode 100644 .github/scripts/generate-vault.py diff --git a/.github/scripts/cleanup-registry-yc.py b/.github/scripts/cleanup-registry-yc.py new file mode 100644 index 0000000..1d1475d --- /dev/null +++ b/.github/scripts/cleanup-registry-yc.py @@ -0,0 +1,236 @@ +""" +Yandex Container Registry Cleanup Script + +Strategy: +- Delete staging images (sha-*) older than 14 days WITHOUT 'latest' tag +- Keep all production images (v*.*.* tags) +- Keep all images with 'latest' tag +""" + +import argparse +import json +import sys +import subprocess +from datetime import datetime, timedelta, timezone +from typing import List, Dict +import re + + +def run_yc_command(cmd: List[str]) -> Dict: + """Execute yc CLI command and return JSON result.""" + full_cmd = ["yc"] + cmd + ["--format", "json"] + try: + result = subprocess.run( + full_cmd, + capture_output=True, + text=True, + check=True + ) + return json.loads(result.stdout) if result.stdout else {} + except subprocess.CalledProcessError as e: + print(f"Error running yc command: {e.stderr}", file=sys.stderr) + return {} + except json.JSONDecodeError as e: + print(f"Error parsing yc output: {e}", file=sys.stderr) + return {} + + +def get_registry_images(registry_id: str) -> List[Dict]: + """Get all images from registry using yc CLI.""" + print(f"Fetching images from registry {registry_id}...") + + # List all repositories + repos = run_yc_command([ + "container", "repository", "list", + "--registry-id", registry_id + ]) + + if not repos: + return [] + + all_images = [] + + for repo in repos: + repo_name = repo.get('name', '') + print(f" Scanning repository: {repo_name}") + + # List images in repository + images = run_yc_command([ + "container", "image", "list", + "--repository-name", repo_name + ]) + + for img in images: + all_images.append({ + 'id': img.get('id'), + 'repository': repo_name.split('/')[-1], + 'tags': img.get('tags', []), + 'created_at': img.get('created_at'), + 'size': img.get('compressed_size', 0), + 'digest': img.get('digest', '') + }) + + return all_images + + +def is_production_tag(tag: str) -> bool: + """Check if tag is a production tag.""" + if tag == 'latest': + return True + if re.match(r'^v\d+\.\d+\.\d+', tag): + return True + return False + + +def is_staging_tag(tag: str) -> bool: + """Check if tag is a staging tag.""" + return tag.startswith('sha-') + + +def parse_iso_date(date_str: str) -> datetime: + """Parse ISO 8601 date string.""" + try: + # Remove microseconds if present + if '.' in date_str: + date_str = date_str.split('.')[0] + 'Z' + return datetime.fromisoformat(date_str.replace('Z', '+00:00')) + except Exception as e: + print(f"Warning: Could not parse date '{date_str}': {e}", file=sys.stderr) + return datetime(2020, 1, 1, tzinfo=timezone.utc) + + +def cleanup_registry( + registry_id: str, + keep_days: int = 14, + dry_run: bool = True +): + """Clean up old staging images from registry.""" + cutoff_date = datetime.now(timezone.utc) - timedelta(days=keep_days) + + print(f"Registry ID: {registry_id}") + print(f"Delete staging images older than: {cutoff_date.isoformat()}") + print(f"Dry run: {dry_run}") + print() + + images = get_registry_images(registry_id) + + if not images: + print("No images found in registry") + return + + print(f"Found {len(images)} images") + print() + + to_delete = [] + to_keep = [] + total_size_freed = 0 + + for img in images: + tags = img.get('tags', []) + img_id = img['id'] + repo = img['repository'] + created = parse_iso_date(img['created_at']) + age_days = (datetime.now(timezone.utc) - created).days + size_mb = img.get('size', 0) / (1024 * 1024) + + display_tags = ', '.join(tags) if tags else '' + full_name = f"{repo}:{display_tags}" + + # Keep all production images (v*.*.*) + has_production_tag = any(is_production_tag(tag) for tag in tags) + if has_production_tag: + to_keep.append(full_name) + print(f"KEEP (production): {full_name}") + continue + + # Keep all images with 'latest' tag + if 'latest' in tags: + to_keep.append(full_name) + print(f"KEEP (latest): {full_name}") + continue + + # Delete old staging images (sha-*) older than 14 days + has_staging_tag = any(is_staging_tag(tag) for tag in tags) + if has_staging_tag and created <= cutoff_date: + to_delete.append({ + 'id': img_id, + 'name': full_name, + 'size_mb': size_mb + }) + total_size_freed += size_mb + print(f"DELETE (old staging, {age_days}d): {full_name}") + else: + to_keep.append(full_name) + print(f"KEEP (recent or other): {full_name}") + + print() + print(f"Summary: {len(to_keep)} to keep, {len(to_delete)} to delete") + print(f"Space to free: {total_size_freed:.1f} MB") + print() + + deleted_count = 0 + if not dry_run and to_delete: + print("Deleting images...") + for img in to_delete: + print(f" Deleting {img['name']}...") + try: + subprocess.run( + ["yc", "container", "image", "delete", "--id", img['id']], + check=True, + capture_output=True + ) + deleted_count += 1 + except subprocess.CalledProcessError as e: + print(f" Failed to delete {img['name']}: {e.stderr.decode()}", file=sys.stderr) + + print() + if dry_run: + print("DRY RUN - No images were actually deleted") + print(f"Would delete {len(to_delete)} images ({total_size_freed:.1f} MB)") + else: + print(f"Deleted {deleted_count}/{len(to_delete)} images ({total_size_freed:.1f} MB)") + + +def main(): + parser = argparse.ArgumentParser( + description='Clean up old staging images from Yandex Container Registry' + ) + parser.add_argument( + '--registry-id', + required=True, + help='Yandex Container Registry ID' + ) + parser.add_argument( + '--keep-days', + type=int, + default=14, + help='Keep staging images newer than N days (default: 14)' + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Preview deletions without executing' + ) + + args = parser.parse_args() + + try: + cleanup_registry( + registry_id=args.registry_id, + keep_days=args.keep_days, + dry_run=args.dry_run + ) + + print() + print("Cleanup completed successfully") + sys.exit(0) + + except Exception as e: + print(f"Error during cleanup: {e}", file=sys.stderr) + import traceback + traceback.print_exc() + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/.github/scripts/generate-vault.py b/.github/scripts/generate-vault.py new file mode 100644 index 0000000..c00032e --- /dev/null +++ b/.github/scripts/generate-vault.py @@ -0,0 +1,96 @@ +""" +Generate Ansible vault.yml from GitHub Secrets (JSON format). + +This script safely processes secrets without shell injection risks. +It uses PyYAML for proper YAML escaping of special characters. + +Usage: + SECRETS_JSON='{"KEY":"value"}' python3 generate-vault.py +""" + +import os +import sys +import json +import yaml +from typing import Dict, Any + + +def parse_multiline_env(content: str) -> Dict[str, str]: + """Parse KEY=VALUE format from multiline string.""" + result = {} + for line in content.splitlines(): + line = line.strip() + if line and '=' in line: + key, value = line.split('=', 1) + result[key.strip()] = value.strip() + return result + + +def generate_vault_yaml(secrets: Dict[str, Any], output_file: str) -> None: + """Generate vault.yml from secrets dictionary using PyYAML for safe serialization.""" + + vault_data = {} + + # Infrastructure secrets + if 'INFRA_ENV' in secrets and secrets['INFRA_ENV']: + infra_vars = parse_multiline_env(secrets['INFRA_ENV']) + for key, value in infra_vars.items(): + vault_key = f'vault_{key.lower()}' + vault_data[vault_key] = value + + # Consolidated service secrets + if 'SERVICES_ENV' in secrets and secrets['SERVICES_ENV']: + vault_data['vault_services_env'] = secrets['SERVICES_ENV'] + + # Per-service secrets (auto-detected) + per_service_secrets = { + k: v for k, v in secrets.items() + if k.endswith('_ENV') and k not in ['INFRA_ENV', 'SERVICES_ENV'] and v + } + + for secret_name in sorted(per_service_secrets.keys()): + vault_var_name = f'vault_{secret_name.lower()}' + vault_data[vault_var_name] = per_service_secrets[secret_name] + + # Registry credentials + if 'YC_REGISTRY_USERNAME' in secrets and secrets['YC_REGISTRY_USERNAME']: + vault_data['vault_yc_registry_username'] = secrets['YC_REGISTRY_USERNAME'] + + if 'YC_REGISTRY_PASSWORD' in secrets and secrets['YC_REGISTRY_PASSWORD']: + vault_data['vault_yc_registry_password'] = secrets['YC_REGISTRY_PASSWORD'] + + with open(output_file, 'w') as f: + f.write('---\n') + f.write('# ' + '=' * 72 + '\n') + f.write('# Generated dynamically by CI/CD from GitHub Secrets\n') + f.write('# DO NOT EDIT MANUALLY - this file is recreated on each deployment\n') + f.write('# ' + '=' * 72 + '\n') + f.write('\n') + + yaml.dump(vault_data, f, default_flow_style=False, allow_unicode=True, sort_keys=False) + + +def main(): + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + output_file = sys.argv[1] + secrets_json = os.environ.get('SECRETS_JSON', '{}') + + try: + secrets = json.loads(secrets_json) + except json.JSONDecodeError as e: + print(f"Error parsing SECRETS_JSON: {e}", file=sys.stderr) + sys.exit(1) + + try: + generate_vault_yaml(secrets, output_file) + print(f"✓ Generated {output_file}", file=sys.stderr) + except Exception as e: + print(f"Error generating vault file: {e}", file=sys.stderr) + sys.exit(1) + + +if __name__ == '__main__': + main() From ac7647cd56f6c4b245aa7fd8fd7862496426b5ff Mon Sep 17 00:00:00 2001 From: exec-unit Date: Sun, 15 Feb 2026 17:46:29 +0700 Subject: [PATCH 2/3] ci(actions): create reusable workflows for build, deployment and release preparation --- .github/workflows/_build-images.yml | 68 ++++++++++++ .github/workflows/_deploy-ansible.yml | 100 +++++++++++++++++ .github/workflows/_prepare-release.yml | 148 +++++++++++++++++++++++++ 3 files changed, 316 insertions(+) create mode 100644 .github/workflows/_build-images.yml create mode 100644 .github/workflows/_deploy-ansible.yml create mode 100644 .github/workflows/_prepare-release.yml diff --git a/.github/workflows/_build-images.yml b/.github/workflows/_build-images.yml new file mode 100644 index 0000000..c8749ec --- /dev/null +++ b/.github/workflows/_build-images.yml @@ -0,0 +1,68 @@ +# ============================================================================= +# Reusable Workflow: Build Docker Images +# Purpose: Build and push Docker images for changed services to container registry +# Called by: deploy-staging.yml, release-production.yml +# ============================================================================= +name: Build Images + +on: + workflow_call: + inputs: + environment: + required: true + type: string + description: "Environment: staging or production" + services: + required: true + type: string + description: "JSON array of services to build" + version: + required: true + type: string + description: "Docker image tag version" + max_parallel: + required: false + type: number + default: 0 + description: "Max parallel builds (0 = unlimited)" + secrets: + YC_REGISTRY_ID: + required: true + YC_REGISTRY_USERNAME: + required: true + YC_REGISTRY_PASSWORD: + required: true + +jobs: + build: + name: Build ${{ matrix.service }} + if: inputs.services != '[]' + environment: ${{ inputs.environment }} + strategy: + fail-fast: false + max-parallel: ${{ inputs.max_parallel > 0 && inputs.max_parallel || 999 }} + matrix: + service: ${{ fromJSON(inputs.services) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Yandex Container Registry + uses: docker/login-action@v3 + with: + registry: cr.yandex + username: ${{ secrets.YC_REGISTRY_USERNAME }} + password: ${{ secrets.YC_REGISTRY_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: ./services/${{ matrix.service }}/Dockerfile + push: true + tags: cr.yandex/${{ secrets.YC_REGISTRY_ID }}/${{ matrix.service }}:${{ inputs.version }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/_deploy-ansible.yml b/.github/workflows/_deploy-ansible.yml new file mode 100644 index 0000000..719c9a3 --- /dev/null +++ b/.github/workflows/_deploy-ansible.yml @@ -0,0 +1,100 @@ +# ============================================================================= +# Reusable Workflow: Deploy via Ansible +# Purpose: Execute Ansible playbook to deploy services to target environment +# Called by: deploy-staging.yml, release-production.yml +# Key features: +# - Dynamic vault.yml generation from GitHub Secrets (secure, no shell injection) +# - Automated inventory creation from secrets +# - Idempotent deployment using release_map +# ============================================================================= +name: Deploy via Ansible + +on: + workflow_call: + inputs: + environment: + required: true + type: string + description: "Environment: staging or production" + release_map: + required: true + type: string + description: "JSON map of services to versions" + secrets: + ANSIBLE_SSH_KEY: + required: true + ANSIBLE_INVENTORY_HOSTS: + required: true + ANSIBLE_INVENTORY_ALL: + required: true + YC_REGISTRY_ID: + required: true + SECRETS_JSON: + required: true + description: "JSON string containing all secrets for vault.yml generation (INFRA_ENV, SERVICES_ENV, *_ENV)" + +jobs: + deploy: + name: Deploy to ${{ inputs.environment }} + runs-on: ubuntu-latest + environment: ${{ inputs.environment }} + steps: + - uses: actions/checkout@v6 + + - name: Setup SSH for Ansible + uses: webfactory/ssh-agent@a6f90b1f127823b31d4d4a8d96047790581349bd + with: + ssh-private-key: ${{ secrets.ANSIBLE_SSH_KEY }} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: "pip" + cache-dependency-path: infrastructure/ansible/requirements-pip.txt + + - name: Install Ansible and dependencies + run: | + sudo apt-get update + sudo apt-get install -y rsync + pip install -r infrastructure/ansible/requirements-pip.txt + + - name: Create inventory directory structure + run: | + mkdir -p infrastructure/ansible/inventories/${{ inputs.environment }}/group_vars + + - name: Create hosts.yml from secret + run: | + echo "${{ secrets.ANSIBLE_INVENTORY_HOSTS }}" > infrastructure/ansible/inventories/${{ inputs.environment }}/hosts.yml + + - name: Create all.yml from secret + run: | + echo "${{ secrets.ANSIBLE_INVENTORY_ALL }}" > infrastructure/ansible/inventories/${{ inputs.environment }}/group_vars/all.yml + + - name: Generate vault.yml dynamically from secrets + env: + SECRETS_JSON: ${{ secrets.SECRETS_JSON }} + run: | + VAULT_FILE="infrastructure/ansible/inventories/${{ inputs.environment }}/group_vars/vault.yml" + python3 .github/scripts/generate-vault.py "$VAULT_FILE" + + - name: Verify inventory files + run: | + echo "Checking inventory files..." + ls -la infrastructure/ansible/inventories/${{ inputs.environment }}/ + ls -la infrastructure/ansible/inventories/${{ inputs.environment }}/group_vars/ + echo "" + echo "Inventory files created successfully" + [ -f infrastructure/ansible/inventories/${{ inputs.environment }}/hosts.yml ] && echo "✓ hosts.yml" + [ -f infrastructure/ansible/inventories/${{ inputs.environment }}/group_vars/all.yml ] && echo "✓ all.yml" + [ -f infrastructure/ansible/inventories/${{ inputs.environment }}/group_vars/vault.yml ] && echo "✓ vault.yml" + + - name: Deploy via Ansible + env: + RELEASE_MAP: ${{ inputs.release_map }} + run: | + cd infrastructure/ansible + ansible-playbook -i inventories/${{ inputs.environment }}/hosts.yml \ + playbooks/release.yml \ + --extra-vars "release_map='$RELEASE_MAP'" \ + --extra-vars "registry_id=${{ secrets.YC_REGISTRY_ID }}" diff --git a/.github/workflows/_prepare-release.yml b/.github/workflows/_prepare-release.yml new file mode 100644 index 0000000..43a81ae --- /dev/null +++ b/.github/workflows/_prepare-release.yml @@ -0,0 +1,148 @@ +# ============================================================================= +# Reusable Workflow: Prepare Release +# Purpose: Detect changed services and generate release metadata +# Called by: deploy-staging.yml, release-production.yml +# Outputs: +# - version: Deployment version (commit SHA for staging, Git tag for production) +# - services: JSON array of changed services +# - release_map: JSON mapping of service -> version for Ansible +# - compose_changed: Boolean indicating if compose files changed +# ============================================================================= +name: Prepare Release + +on: + workflow_call: + inputs: + environment: + required: true + type: string + description: "Environment: staging or production" + version_override: + required: false + type: string + description: "Override version (for manual dispatch)" + outputs: + version: + description: "Release version" + value: ${{ jobs.prepare.outputs.version }} + release_map: + description: "JSON map of services to versions" + value: ${{ jobs.prepare.outputs.release_map }} + services: + description: "List of changed services" + value: ${{ jobs.prepare.outputs.services }} + compose_changed: + description: "Whether compose files changed" + value: ${{ jobs.prepare.outputs.compose_changed }} + +jobs: + prepare: + name: Prepare Release + runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.version }} + release_map: ${{ steps.release-map.outputs.map }} + services: ${{ steps.services-list.outputs.services }} + compose_changed: ${{ steps.check-compose.outputs.changed }} + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + + # Generate version (depends on environment) + - name: Generate version + id: version + run: | + if [ -n "${{ inputs.version_override }}" ]; then + VERSION="${{ inputs.version_override }}" + elif [ "${{ inputs.environment }}" = "staging" ]; then + # Staging: commit SHA + VERSION="sha-$(git rev-parse --short HEAD)" + else + # Production: Git tag + VERSION="${GITHUB_REF#refs/tags/}" + fi + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Version: $VERSION" + + # Determine comparison base + - name: Get comparison base + id: base + run: | + if [ "${{ inputs.environment }}" = "staging" ]; then + # Staging: use github.event.before for batch push support + # Fallback to HEAD~1 for new branches, manual dispatch, or amend commits + BEFORE_SHA="${{ github.event.before }}" + + if [ -z "$BEFORE_SHA" ] || [ "$BEFORE_SHA" = "0000000000000000000000000000000000000000" ]; then + # Empty or null SHA (new branch or manual dispatch) + BASE="HEAD~1" + echo "Using HEAD~1 (no previous commit available)" + else + if git cat-file -e "$BEFORE_SHA^{commit}" 2>/dev/null; then + BASE="$BEFORE_SHA" + echo "Using github.event.before: $BEFORE_SHA" + else + BASE="HEAD~1" + echo "Falling back to HEAD~1" + fi + fi + else + # Production: previous tag + CURRENT_TAG="${{ steps.version.outputs.version }}" + PREV_TAG=$(git tag --sort=-v:refname | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' | grep -v "^${CURRENT_TAG}$" | head -1) + if [ -z "$PREV_TAG" ]; then + PREV_TAG=$(git rev-list --max-parents=0 HEAD) + fi + BASE="$PREV_TAG" + fi + + echo "base=$BASE" >> $GITHUB_OUTPUT + echo "Final comparison base: $BASE" + + # Check compose changes + - name: Check compose changes + id: check-compose + run: | + BASE="${{ steps.base.outputs.base }}" + if git diff "$BASE" --name-only | grep -qE '^(docker-compose\.yml|compose/|Makefile|makefiles/|scripts/)'; then + echo "changed=true" >> $GITHUB_OUTPUT + echo "Compose files changed" + else + echo "changed=false" >> $GITHUB_OUTPUT + fi + + # Detect changed services + - name: Detect changed services + id: detect + uses: step-security/changed-files@60967b822d3001fa82242f8d6b4ed46bc3600a68 + with: + files: services/** + dir_names: true + dir_names_max_depth: 2 + base_sha: ${{ steps.base.outputs.base }} + + - name: Build services list + id: services-list + run: | + if [ -n "${{ steps.detect.outputs.all_changed_files }}" ]; then + # Extract service names: remove 'services/' prefix and filter out 'services' itself + SERVICES=$(echo "${{ steps.detect.outputs.all_changed_files }}" | tr ' ' '\n' | \ + sed 's|^services/||' | grep -v '^services$' | sort -u | \ + jq -R -s -c 'split("\n") | map(select(length > 0))') + else + SERVICES="[]" + fi + echo "services=$SERVICES" >> $GITHUB_OUTPUT + echo "Changed services: $SERVICES" + + # Create Release Map + - name: Generate release map + id: release-map + run: | + VERSION="${{ steps.version.outputs.version }}" + SERVICES='${{ steps.services-list.outputs.services }}' + RELEASE_MAP=$(echo "$SERVICES" | jq -c --arg ver "$VERSION" 'map({(.): $ver}) | add // {}') + + echo "Release Map (compact): $RELEASE_MAP" + echo "map=$RELEASE_MAP" >> $GITHUB_OUTPUT From d3e3d5b302dbbbae72b49a94fd51cf59dd412093 Mon Sep 17 00:00:00 2001 From: exec-unit Date: Tue, 17 Feb 2026 10:04:41 +0700 Subject: [PATCH 3/3] ci(actions): configure main ci/cd pipelines for staging and production environments --- .github/workflows/ci.yml | 193 +++++++++++++++++++++++ .github/workflows/deploy-staging.yml | 96 +++++++++++ .github/workflows/release-production.yml | 55 +++++++ 3 files changed, 344 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/deploy-staging.yml create mode 100644 .github/workflows/release-production.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..52ffaad --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,193 @@ +# ============================================================================= +# CI Pipeline: Path-based validation for services and infrastructure +# Triggers: Pull requests to main branch OR manual dispatch +# Strategy: Only run jobs for changed components (cost optimization) +# ============================================================================= +name: CI + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +permissions: + pull-requests: read + contents: read + security-events: write + +on: + pull_request: + branches: ["main"] + # Manual trigger: allows running CI on any branch without creating PR + workflow_dispatch: + inputs: + run_all: + description: "Run all jobs (ignore path filters)" + required: false + type: boolean + default: false + +jobs: + # Detect which parts of monorepo changed + detect-changes: + name: Detect Changed Components + runs-on: ubuntu-latest + outputs: + services: ${{ steps.filter.outputs.services }} + iac: ${{ steps.filter.outputs.iac }} + services-list: ${{ steps.set-matrix.outputs.services }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Detect path changes + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 + id: filter + with: + filters: | + services: + - 'services/**' + iac: + - 'infrastructure/ansible/**' + - 'compose/**' + - 'config/**' + + - name: Get changed files + if: steps.filter.outputs.services == 'true' + id: changed-files + uses: step-security/changed-files@60967b822d3001fa82242f8d6b4ed46bc3600a68 + with: + files: services/** + dir_names: true + dir_names_max_depth: 2 + + - name: Discover changed services + id: set-matrix + if: steps.filter.outputs.services == 'true' + run: | + if [ -n "${{ steps.changed-files.outputs.all_changed_files }}" ]; then + # Extract service names: remove 'services/' prefix and filter out 'services' itself + SERVICES=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | \ + sed 's|^services/||' | grep -v '^services$' | sort -u | \ + jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "Changed services: $SERVICES" + echo "services=$SERVICES" >> $GITHUB_OUTPUT + else + echo "services=[]" >> $GITHUB_OUTPUT + fi + + # Validate Infrastructure as Code (Ansible) + lint-iac: + name: Lint IaC + needs: detect-changes + if: needs.detect-changes.outputs.iac == 'true' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Generate cache key dependencies + run: | + echo "ansible==13.0.0" >> temp-deps.txt + echo "ansible-lint==25.11.1" >> temp-deps.txt + + - name: Cache pip packages + uses: actions/cache@v5 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('temp-deps.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install linting tools + run: pip install ansible==13.0.0 ansible-lint==25.11.1 + + - name: Run ansible-lint + run: cd infrastructure/ansible && ansible-lint roles/ --exclude 'roles/*/molecule/' + + # Build and test changed services (parallel execution) + build-test-services: + name: Build & Test ${{ matrix.service }} + needs: detect-changes + if: needs.detect-changes.outputs.services == 'true' && needs.detect-changes.outputs.services-list != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + service: ${{ fromJSON(needs.detect-changes.outputs.services-list) }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + java-version: "21" + distribution: "temurin" + cache: "maven" + + # Verify Maven wrapper exists + - name: Check Maven wrapper + run: | + if [ ! -f "./mvnw" ]; then + echo "Error: Maven wrapper (mvnw) not found in repository root" + echo "Please run: mvn wrapper:wrapper in the root directory" + exit 1 + fi + + - name: Build and test with Maven wrapper + run: ./mvnw -f services/${{ matrix.service }}/pom.xml --batch-mode --update-snapshots verify + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v6 + with: + name: test-results-${{ matrix.service }} + path: services/${{ matrix.service }}/target/surefire-reports/ + + # Security scan with Trivy (CRITICAL/HIGH vulnerabilities) + scan-services: + name: Scan ${{ matrix.service }} + needs: [detect-changes, build-test-services] + if: needs.detect-changes.outputs.services == 'true' && needs.detect-changes.outputs.services-list != '[]' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + service: ${{ fromJSON(needs.detect-changes.outputs.services-list) }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image for scanning + uses: docker/build-push-action@v6 + with: + context: . + file: ./services/${{ matrix.service }}/Dockerfile + push: false + load: true + tags: ${{ matrix.service }}:pr-${{ github.event.pull_request.number }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Scan for vulnerabilities + uses: aquasecurity/trivy-action@b6643a29fecd7f34b3597bc6acb0a98b03d33ff8 + with: + image-ref: ${{ matrix.service }}:pr-${{ github.event.pull_request.number }} + format: sarif + output: trivy-results-${{ matrix.service }}.sarif + severity: CRITICAL,HIGH + exit-code: 1 + + - name: Upload scan results + uses: github/codeql-action/upload-sarif@v4.31.10 + if: always() + with: + sarif_file: trivy-results-${{ matrix.service }}.sarif diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 0000000..34c9a9b --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,96 @@ +# ============================================================================= +# Continuous Deployment: Staging Environment +# Triggers: Push to main OR manual dispatch +# Strategy: Deploy only changed services (cost + time optimization) +# ============================================================================= +name: CD Staging + +concurrency: + group: deploy-staging + cancel-in-progress: false + +on: + push: + branches: ["main"] + paths: + - "services/**" + - "docker-compose.yml" + - "compose/**" + - "Makefile" + - "makefiles/**" + - "scripts/**" + workflow_dispatch: + inputs: + commit_sha: + description: "Commit SHA to deploy (leave empty for latest)" + required: false + type: string + +jobs: + prepare-release: + uses: ./.github/workflows/_prepare-release.yml + with: + environment: "staging" + version_override: ${{ inputs.commit_sha && format('sha-{0}', inputs.commit_sha) || '' }} + + build-images: + needs: prepare-release + uses: ./.github/workflows/_build-images.yml + with: + environment: "staging" + services: ${{ needs.prepare-release.outputs.services }} + version: ${{ needs.prepare-release.outputs.version }} + max_parallel: 0 + secrets: + YC_REGISTRY_ID: ${{ secrets.YC_REGISTRY_ID }} + YC_REGISTRY_USERNAME: ${{ secrets.YC_REGISTRY_USERNAME }} + YC_REGISTRY_PASSWORD: ${{ secrets.YC_REGISTRY_PASSWORD }} + + deploy: + needs: [prepare-release, build-images] + if: always() && (needs.build-images.result == 'success' || needs.prepare-release.outputs.compose_changed == 'true') + uses: ./.github/workflows/_deploy-ansible.yml + with: + environment: "staging" + release_map: ${{ needs.prepare-release.outputs.release_map }} + secrets: + ANSIBLE_SSH_KEY: ${{ secrets.ANSIBLE_SSH_KEY }} + ANSIBLE_INVENTORY_HOSTS: ${{ secrets.ANSIBLE_INVENTORY_HOSTS }} + ANSIBLE_INVENTORY_ALL: ${{ secrets.ANSIBLE_INVENTORY_ALL }} + YC_REGISTRY_ID: ${{ secrets.YC_REGISTRY_ID }} + SECRETS_JSON: ${{ toJSON(secrets) }} + + cleanup-registry: + name: Cleanup Old Staging Images + needs: deploy + if: success() + environment: staging + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install Yandex Cloud CLI + run: | + curl -sSL https://storage.yandexcloud.net/yandexcloud-yc/install.sh | bash + echo "$HOME/yandex-cloud/bin" >> $GITHUB_PATH + + - name: Authenticate with Yandex Cloud + env: + YC_TOKEN: ${{ secrets.YC_REGISTRY_PASSWORD }} + run: | + yc config profile create cleanup-profile + yc config set token "$YC_TOKEN" + + - name: Run cleanup script + env: + YC_REGISTRY_ID: ${{ secrets.YC_REGISTRY_ID }} + run: | + python3 .github/scripts/cleanup-registry-yc.py \ + --registry-id "$YC_REGISTRY_ID" \ + --keep-days 14 diff --git a/.github/workflows/release-production.yml b/.github/workflows/release-production.yml new file mode 100644 index 0000000..4fa0459 --- /dev/null +++ b/.github/workflows/release-production.yml @@ -0,0 +1,55 @@ +# ============================================================================= +# Continuous Deployment: Production Environment +# Triggers: Git tags (v1.2.3) OR manual dispatch with tag selection +# Strategy: Sequential deployment (max-parallel: 1) for safety +# ============================================================================= +name: CD Production + +concurrency: + group: deploy-production + cancel-in-progress: false + +on: + push: + tags: + - "v*.*.*" + workflow_dispatch: + inputs: + tag: + description: "Git tag to deploy (e.g., v1.2.3)" + required: true + type: string + +jobs: + prepare-release: + uses: ./.github/workflows/_prepare-release.yml + with: + environment: "production" + version_override: ${{ inputs.tag || '' }} + + build-images: + needs: prepare-release + uses: ./.github/workflows/_build-images.yml + with: + environment: "production" + services: ${{ needs.prepare-release.outputs.services }} + version: ${{ needs.prepare-release.outputs.version }} + max_parallel: 1 + secrets: + YC_REGISTRY_ID: ${{ secrets.YC_REGISTRY_ID }} + YC_REGISTRY_USERNAME: ${{ secrets.YC_REGISTRY_USERNAME }} + YC_REGISTRY_PASSWORD: ${{ secrets.YC_REGISTRY_PASSWORD }} + + deploy: + needs: [prepare-release, build-images] + if: always() && (needs.build-images.result == 'success' || needs.prepare-release.outputs.compose_changed == 'true') + uses: ./.github/workflows/_deploy-ansible.yml + with: + environment: "production" + release_map: ${{ needs.prepare-release.outputs.release_map }} + secrets: + ANSIBLE_SSH_KEY: ${{ secrets.ANSIBLE_SSH_KEY }} + ANSIBLE_INVENTORY_HOSTS: ${{ secrets.ANSIBLE_INVENTORY_HOSTS }} + ANSIBLE_INVENTORY_ALL: ${{ secrets.ANSIBLE_INVENTORY_ALL }} + YC_REGISTRY_ID: ${{ secrets.YC_REGISTRY_ID }} + SECRETS_JSON: ${{ toJSON(secrets) }}