From ef4f6438a7449ca84f7b525d2dabb635a742d226 Mon Sep 17 00:00:00 2001 From: Ljubo Nikolic Date: Wed, 12 Nov 2025 21:54:29 +0100 Subject: [PATCH 01/15] test env --- .github/workflows/bulk-import.yaml | 6 +- .github/workflows/drift-check.yaml | 4 +- .github/workflows/import.yaml | 6 +- .../workflows/promote-imported-configs.yaml | 4 +- .github/workflows/tf-apply.yaml | 8 +- .github/workflows/tf-plan.yaml | 4 +- .gitignore | 11 +- README.md | 2 +- .../DEVELOPERS_GUIDE.md | 40 +- docs/LOCAL_DEVELOPMENT_SETUP.md | 262 +++++++++++ docs/feature_github_environment.md | 147 ++++++ docs/flow.md | 185 ++++++++ .../github-repo-importer/ADDING_FEATURES.md | 244 ++++++++++ feature/github-repo-importer/Justfile | 2 +- .../github-repo-importer/LOCAL_DEVELOPMENT.md | 426 ++++++++++++++++++ feature/github-repo-importer/QUICK_START.md | 41 ++ .../github-repo-importer/cmd/bulk-import.go | 27 -- feature/github-repo-importer/cmd/config.go | 36 ++ feature/github-repo-importer/cmd/import.go | 55 ++- .../github-repo-importer/pkg/github/config.go | 17 +- .../pkg/github/constants.go | 9 + .../github-repo-importer/pkg/github/github.go | 305 ++++++++++++- .../pkg/github/github_test.go | 2 +- .../pkg/github/repositories.go | 27 ++ feature/github-repo-provisioning/Justfile | 2 +- .../github-repo-provisioning/QUICK_START.md | 58 +++ feature/github-repo-provisioning/main.tf | 282 +++++++----- .../terraform-github-repository/main.tf | 147 +++++- .../terraform-github-repository/outputs.tf | 15 + .../terraform-github-repository/variables.tf | 48 +- 30 files changed, 2196 insertions(+), 226 deletions(-) rename DEVELOPERS_GUIDE.md => docs/DEVELOPERS_GUIDE.md (90%) create mode 100644 docs/LOCAL_DEVELOPMENT_SETUP.md create mode 100644 docs/feature_github_environment.md create mode 100644 docs/flow.md create mode 100644 feature/github-repo-importer/ADDING_FEATURES.md create mode 100644 feature/github-repo-importer/LOCAL_DEVELOPMENT.md create mode 100644 feature/github-repo-importer/QUICK_START.md create mode 100644 feature/github-repo-importer/cmd/config.go create mode 100644 feature/github-repo-provisioning/QUICK_START.md diff --git a/.github/workflows/bulk-import.yaml b/.github/workflows/bulk-import.yaml index a85ed23..e31749a 100644 --- a/.github/workflows/bulk-import.yaml +++ b/.github/workflows/bulk-import.yaml @@ -32,7 +32,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: G-Research/github-terraformer + repository: ljubon-org/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -50,7 +50,7 @@ jobs: - name: Setup vars id: setup-vars run: | - date_suffix=$(date +%Y%m%d) + date_suffix=$(date +%Y%m%d%H%M) echo "automation_branch_name=import/${{ github.ref_name }}/${{ github.repository_owner }}/bulk-import/$date_suffix" >> $GITHUB_OUTPUT - name: Bulk import repos @@ -73,4 +73,4 @@ jobs: branch-name: ${{ steps.setup-vars.outputs.automation_branch_name }} pr-title: "Bulk import repos" commit-message: "Auto-generated changes. Bulk import repos" - github-token: ${{ steps.generate-token.outputs.token }} \ No newline at end of file + github-token: ${{ steps.generate-token.outputs.token }} diff --git a/.github/workflows/drift-check.yaml b/.github/workflows/drift-check.yaml index 43ced96..9e47ea5 100644 --- a/.github/workflows/drift-check.yaml +++ b/.github/workflows/drift-check.yaml @@ -26,7 +26,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: G-Research/github-terraformer + repository: ljubon-org/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -52,4 +52,4 @@ jobs: exit 1 else echo "No drift detected." - fi \ No newline at end of file + fi diff --git a/.github/workflows/import.yaml b/.github/workflows/import.yaml index d7f6b61..6a6ea5f 100644 --- a/.github/workflows/import.yaml +++ b/.github/workflows/import.yaml @@ -37,7 +37,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: G-Research/github-terraformer + repository: ljubon-org/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -55,7 +55,7 @@ jobs: - name: Setup vars id: setup-vars run: | - date_suffix=$(date +%Y%m%d) + date_suffix=$(date +%Y%m%d%H%M) echo "full_repo_name=${{ github.repository_owner }}/${{ inputs.repo_name }}" >> $GITHUB_OUTPUT echo "automation_branch_name=import/${{ github.repository_owner }}/${{ inputs.repo_name }}/$date_suffix" >> $GITHUB_OUTPUT @@ -78,4 +78,4 @@ jobs: branch-name: ${{ steps.setup-vars.outputs.automation_branch_name }} pr-title: "Import ${{ steps.setup-vars.outputs.full_repo_name }}" commit-message: "Auto-generated changes. Import ${{ steps.setup-vars.outputs.full_repo_name }} repository" - github-token: ${{ steps.generate-token.outputs.token }} \ No newline at end of file + github-token: ${{ steps.generate-token.outputs.token }} diff --git a/.github/workflows/promote-imported-configs.yaml b/.github/workflows/promote-imported-configs.yaml index 675e323..3484d9c 100644 --- a/.github/workflows/promote-imported-configs.yaml +++ b/.github/workflows/promote-imported-configs.yaml @@ -44,7 +44,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: G-Research/github-terraformer + repository: ljubon-org/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -108,4 +108,4 @@ jobs: date_suffix=$(date +%Y-%m-%d) git commit -m "Promote configs from PR #${{ inputs.pr_number }} on $date_suffix" git push origin "${{ github.ref_name }}" - } \ No newline at end of file + } diff --git a/.github/workflows/tf-apply.yaml b/.github/workflows/tf-apply.yaml index fab3cfd..042f99c 100644 --- a/.github/workflows/tf-apply.yaml +++ b/.github/workflows/tf-apply.yaml @@ -33,7 +33,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: G-Research/github-terraformer + repository: ljubon-org/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -58,8 +58,4 @@ jobs: env: TF_CLOUD_ORGANIZATION: ${{ inputs.tfc_org }} TF_WORKSPACE: ${{ vars.WORKSPACE }} - run: terraform apply -no-color -input=false ${{ steps.graformer.outputs.plan-file }} - - - name: Clean up plan file - if: always() && steps.graformer.outputs.plan-file != '' - run: rm -f "${{ steps.graformer.outputs.plan-file }}" \ No newline at end of file + run: terraform apply -no-color -input=false -auto-approve diff --git a/.github/workflows/tf-plan.yaml b/.github/workflows/tf-plan.yaml index 83fb82a..f294cc3 100644 --- a/.github/workflows/tf-plan.yaml +++ b/.github/workflows/tf-plan.yaml @@ -65,7 +65,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: G-Research/github-terraformer + repository: ljubon-org/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -234,4 +234,4 @@ jobs: summary: summary, text: text } - }); \ No newline at end of file + }); diff --git a/.gitignore b/.gitignore index fd871db..d8150cf 100644 --- a/.gitignore +++ b/.gitignore @@ -42,7 +42,16 @@ feature/github-repo-provisioning/.terraform feature/github-repo-provisioning/.terraform/* feature/github-repo-provisioning/app-list.yaml feature/github-repo-provisioning/import-config.yaml +feature/github-repo-provisioning/backend.tf +feature/github-repo-provisioning/gcss_config/* +!feature/github-repo-provisioning/gcss_config/.gitkeep + +feature/*/tfplan + +# Local development +feature/*/.env feature/github-repo-importer/dumps/* feature/github-repo-importer/configs/* -feature/github-repo-provisioning/backend.tf +feature/github-repo-importer/import-config.yaml +feature/github-repo-importer/app-list.yaml diff --git a/README.md b/README.md index eb79c32..5e45aca 100644 --- a/README.md +++ b/README.md @@ -43,4 +43,4 @@ To import a **forked** repository into the organization: > 📝 We are working on improving this so that the user has the same experience as when creating a new repo > [!IMPORTANT] -> All important attributes are documented in the [Developer's Guide](DEVELOPERS_GUIDE.md). \ No newline at end of file +> All important attributes are documented in the [Developer's Guide](docs/DEVELOPERS_GUIDE.md). diff --git a/DEVELOPERS_GUIDE.md b/docs/DEVELOPERS_GUIDE.md similarity index 90% rename from DEVELOPERS_GUIDE.md rename to docs/DEVELOPERS_GUIDE.md index 0b687cc..962ae05 100644 --- a/DEVELOPERS_GUIDE.md +++ b/docs/DEVELOPERS_GUIDE.md @@ -100,29 +100,39 @@ These are the primary configuration options for each repository. - **`vulnerability_alerts_enabled`**: *(optional, boolean)* If `true`, vulnerability alerts are enabled. -- **`branch_protections_v4`**: *(optional, object[] [BranchProtectionV4](#branch-protection-configuration-v4))* Configuration for branch protection rules. +- **`environments`**: *(optional, object[] [Environment](#environment-configuration))* Configuration for repository environments. Requires `feature_github_environment: true` in import config. When imported, environments are automatically managed by Terraform. -- **`high_integrity`**: *(optional, object [HighIntegrity](#high-integrity-configuration))* Expansion directives for high-integrity repositories. This field is consumed by the `expand` command and is **not** passed to Terraform — it is removed from the output after expansion. +- **`branch_protections_v4`**: *(optional, object[] [BranchProtectionV4](#branch-protection-configuration-v4))* Configuration for branch protection rules. -## High Integrity Configuration +## Environment Configuration -Options for enabling high-integrity mode on a repository. This block is a pre-processing directive consumed by the `expand` command — it is **not** forwarded to Terraform. +Configure GitHub deployment environments with protection rules and reviewers. -When `enabled` is `true`, the `expand` command automatically appends two rulesets to the repository's `rulesets` list: +**Import Control**: Set `feature_github_environment: true` in `import-config.yaml` to import environments. -- **Protect main branch** — an active branch ruleset targeting `~DEFAULT_BRANCH` that enforces deletion protection, no fast-forward pushes, linear history, and a pull request review policy (1 approver, stale review dismissal on push, last-push approval required). -- **Make tags immutable** — an active tag ruleset targeting `~ALL` that prevents deletion, non-fast-forward updates, and tag updates. +### Environment Fields -The `high_integrity` block is then removed from the expanded output. +- **`environment`**: *(required, string)* Environment name +- **`wait_timer`**: *(optional, int)* Delay in seconds (max 43200) +- **`can_admins_bypass`**: *(optional, bool)* Admin bypass allowed (default: true) +- **`prevent_self_review`**: *(optional, bool)* Prevent self-approval (default: false) +- **`reviewers`**: *(optional, object)* + - **`users`**: *(string[])* GitHub usernames (max 6 total) + - **`teams`**: *(string[])* Team slugs (max 6 total) -- **`enabled`**: *(required, boolean)* If `true`, the two high-integrity rulesets are injected during expansion. + > ⚠️ **IMPORTANT: Team Access Requirement** + > Teams specified as reviewers MUST have repository access first! + > - Manually grant access at: `https://github.com/{org}/{repo}/settings/access` + > - Verify team access at: `https://github.com/orgs/{org}/teams/{team}/repositories` + > + > **Without repository access, Terraform will apply successfully but teams won't be added as reviewers!** -Example: +- **`deployment_ref_policy`**: *(optional, object)* Controls which branches/tags can deploy + - **EITHER** `protected_branches_policy: true` (protected branches only) + - **OR** `selected_branches_or_tags_policy` with `branch_patterns` and/or `tag_patterns` + - **Note**: These options are mutually exclusive -```yaml -high_integrity: - enabled: true -``` +**📖 For complete guide with examples, see [feature_github_environment.md](feature_github_environment.md)** ## Template Configuration @@ -366,4 +376,4 @@ Options for configuring required status checks in V4. - **`strict`**: *(optional, boolean)* If `true`, strict status checks are enforced. -- **`contexts`**: *(optional, string[])* A list of required status check contexts. \ No newline at end of file +- **`contexts`**: *(optional, string[])* A list of required status check contexts. diff --git a/docs/LOCAL_DEVELOPMENT_SETUP.md b/docs/LOCAL_DEVELOPMENT_SETUP.md new file mode 100644 index 0000000..dfa2637 --- /dev/null +++ b/docs/LOCAL_DEVELOPMENT_SETUP.md @@ -0,0 +1,262 @@ +# Local Development Setup + +Quick setup guide for both GitHub Repo Importer and Terraform Provisioning. + +## Prerequisites + +- **Go 1.21+** (for importer) +- **Terraform 1.0+** (for provisioning) +- **GitHub Token** or **GitHub App credentials** + +## Quick Setup + +### 1. Clone Repositories + +```bash +git clone https://github.com/your-org/github-terraformer.git +git clone https://github.com/your-org/gcss-config-repo.git +``` + +### 2. Setup Symlinks + +```bash +#!/bin/bash +# Run from workspace root + +# For Terraform Provisioning +cd github-terraformer/feature/github-repo-provisioning +ln -sfn ../../../gcss-config-repo gcss_config +ln -sf gcss_config/config/app-list.yaml app-list.yaml + +# For GitHub Importer +cd ../github-repo-importer +ln -sf ../../../gcss-config-repo/config/import-config.yaml import-config.yaml +ln -sf ../../../gcss-config-repo/config/app-list.yaml app-list.yaml +ln -sf ../../../gcss-config-repo/importer_tmp_dir importer_tmp_dir + +echo "✅ Symlinks created" +``` + +### 3. Configure Environment + +#### For Importer (Go Tool) + +```bash +cd github-terraformer/feature/github-repo-importer + +cat > .env << 'EOF' +export GITHUB_TOKEN="ghp_your_token_here" +export OWNER="your-org" +EOF + +source .env +``` + +#### For Terraform + +```bash +cd github-terraformer/feature/github-repo-provisioning + +cat > .env << 'EOF' +# GitHub App credentials +export TF_VAR_app_id="123456" +export TF_VAR_app_installation_id="12345678" +export TF_VAR_app_private_key="$(cat ~/.secrets/github-app.pem)" + +# Required variables +export TF_VAR_owner="your-org" +export TF_VAR_environment_directory="gcss_config" +EOF + +source .env +``` + +## Common Operations + +### Import Repositories + +```bash +cd github-terraformer/feature/github-repo-importer +source .env + +# Single repository +just import-repo your-org/repo-name + +# All repositories +just import-repos + +# With custom output +go run main.go import your-org/repo -o ../custom-dir +``` + +### Provision with Terraform + +```bash +cd github-terraformer/feature/github-repo-provisioning +source .env + +# Initialize +terraform init + +# Plan changes +terraform plan + +# Apply changes +terraform apply + +# Target specific repo +terraform apply -target='module.repository["repo-name"]' +``` + +## Workflow + +### Creating New Repositories + +1. Create YAML in `gcss-config-repo/repos/new-repo.yaml`: +```yaml +description: "My new repository" +visibility: private +default_branch: main +has_issues: true +vulnerability_alerts_enabled: true + +# Optional environments +environments: + - environment: production + wait_timer: 300 + deployment_ref_policy: + protected_branches_policy: true +``` + +2. Apply with Terraform: +```bash +cd github-terraformer/feature/github-repo-provisioning +source .env && terraform apply +``` + +### Importing Existing Repositories + +1. Import with CLI tool: +```bash +cd github-terraformer/feature/github-repo-importer +source .env && just import-repo owner/repo +``` + +2. File appears in `gcss-config-repo/importer_tmp_dir/` + +3. Apply with Terraform to import: +```bash +cd ../github-repo-provisioning +source .env && terraform apply +``` + +4. Move to permanent location: +```bash +mv gcss_config/importer_tmp_dir/repo.yaml gcss_config/repos/ +``` + +## Configuration Files + +### import-config.yaml + +Controls import behavior: + +```yaml +# Ignore specific repos +ignored_repos: + - "your-org/gcss-config-repo" + - "your-org/github-terraformer" + +# Or select specific repos only +selected_repos: + - "your-org/repo1" + - "your-org/repo2" + +# Enable environment import +feature_github_environment: true +``` + +### app-list.yaml + +GitHub App IDs for ruleset bypass actors: + +```yaml +apps: + - name: dependabot + id: 12345 + - name: renovate + id: 67890 +``` + +## Testing + +```bash +# Importer tests +cd github-terraformer/feature/github-repo-importer +just test + +# Terraform validation +cd ../github-repo-provisioning +terraform validate +``` + +## Troubleshooting + +### Token/Authentication Issues + +```bash +# Test GitHub token +curl -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/user + +# Check rate limit +curl -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/rate_limit +``` + +### Symlink Issues + +```bash +# Verify targets exist +ls ../../../gcss-config-repo + +# Recreate if broken +rm -f symlink-name +ln -sf ../../../correct-path symlink-name +``` + +### Terraform Import Failures + +If "Cannot import non-existent remote object": +- Repository doesn't exist in GitHub +- Move YAML from `importer_tmp_dir/` to `repos/` to create it + +### Environment Variables Not Set + +Always run `source .env` before commands. + +## Local vs CI/CD + +### Local Development + +Use local Terraform state: +```bash +cp backend.tf.local backend.tf +terraform init -reconfigure +``` + +### CI/CD (GitHub Actions) + +Uses HCP Terraform backend - workflows handle this automatically. + +## Security Notes + +Never commit: +- `.env` files +- `*.pem` keys +- `terraform.tfvars` +- `*.tfstate` + +Store secrets in `~/.secrets/` with proper permissions: +```bash +mkdir -p ~/.secrets && chmod 700 ~/.secrets +mv *.pem ~/.secrets/ && chmod 600 ~/.secrets/*.pem +``` \ No newline at end of file diff --git a/docs/feature_github_environment.md b/docs/feature_github_environment.md new file mode 100644 index 0000000..1c479bf --- /dev/null +++ b/docs/feature_github_environment.md @@ -0,0 +1,147 @@ +# GitHub Environments Configuration Guide + +This guide explains how to configure GitHub repository environments using the YAML → Terraform workflow. + +## Quick Start + +### Enable Environment Import +```yaml +# gcss-config-repo/config/import-config.yaml +feature_github_environment: true # Required to import environments +``` + +### Environment Configuration + +```yaml +# repos/my-app.yaml +environments: + # Option 1: Protected branches only + - environment: production + wait_timer: 300 # 5 minutes wait before deployment + can_admins_bypass: false # Admins cannot bypass + prevent_self_review: true # Cannot approve own deployments + reviewers: + users: ["octocat"] + teams: ["platform-team"] + deployment_ref_policy: + protected_branches_policy: true + + # Option 2: Custom branch/tag patterns + - environment: staging + deployment_ref_policy: + protected_branches_policy: false # MUST be false for custom patterns + selected_branches_or_tags_policy: + branch_patterns: + - "release/*" + - "main" + tag_patterns: + - "v*" + + # Option 3: Any branch can deploy (no restrictions) + - environment: development + # No deployment_ref_policy = any branch can deploy +``` + +## ⚠️ Critical Rule: Deployment Policy Mutual Exclusivity + +**You MUST choose ONE of these options:** + +| Option | Configuration | Use Case | +|--------|--------------|----------| +| **Protected Branches** | `protected_branches_policy: true` | Production - only protected branches | +| **Custom Patterns** | `protected_branches_policy: false` + patterns | Staging - specific branches/tags | +| **Any Branch** | Omit `deployment_ref_policy` entirely | Development - no restrictions | + +**NEVER set both `protected_branches_policy: true` AND `selected_branches_or_tags_policy` together.** + +## Field Reference + +| Field | Type | Description | Default | +|-------|------|-------------|---------| +| `environment` | string | **Required** - Environment name | - | +| `wait_timer` | int | Wait time in seconds (max 43200) | 0 | +| `can_admins_bypass` | bool | Admins can bypass protections | true | +| `prevent_self_review` | bool | Prevent self-approval | false | +| `reviewers.users` | string[] | GitHub usernames (max 6 total with teams) | [] | +| `reviewers.teams` | string[] | Team slugs (max 6 total with users) | [] | +| `deployment_ref_policy.*` | object | Deployment restrictions | - | +| ↳ `protected_branches_policy` | bool | Only protected branches | - | +| ↳ `selected_branches_or_tags_policy.*` | object | Custom patterns | - | +| ↳↳ `branch_patterns` | string[] | Branch patterns (e.g., `release/*`) | [] | +| ↳↳ `tag_patterns` | string[] | Tag patterns (e.g., `v*`) | [] | + +## Pattern Matching + +Patterns support wildcards: +- `main` - Exact match +- `release/*` - Matches `release/1.0`, `release/2.0` +- `v*` - Matches `v1.0.0`, `v2.0.0` +- `*-final` - Matches `1.0-final`, `2.0-final` + +## Generated Terraform Resources + +The YAML configuration generates: + +1. **Environment Resource** +```hcl +resource "github_repository_environment" "environment" { + environment = "production" + repository = "my-app" + # ... other settings + + deployment_branch_policy { + protected_branches = true/false + custom_branch_policies = true/false + } +} +``` + +2. **Deployment Policies** (for custom patterns) +```hcl +resource "github_repository_environment_deployment_policy" "branch_policies" { + repository = "my-app" + environment = "staging" + branch_pattern = "release/*" +} + +resource "github_repository_environment_deployment_policy" "tag_policies" { + repository = "my-app" + environment = "staging" + tag_pattern = "v*" +} +``` + +## Complete Example + +```yaml +environments: + - environment: production + wait_timer: 300 + can_admins_bypass: false + prevent_self_review: true + reviewers: + teams: ["platform-team"] + deployment_ref_policy: + protected_branches_policy: true + + - environment: staging + prevent_self_review: true + deployment_ref_policy: + protected_branches_policy: false + selected_branches_or_tags_policy: + branch_patterns: ["release/*", "main"] + tag_patterns: ["v*", "rc-*"] + + - environment: development + # No restrictions - any branch can deploy +``` + +## Troubleshooting + +| Issue | Solution | +|-------|----------| +| "reviewers: must be 6 or fewer" | Combined users + teams must be ≤ 6 | +| Custom policies not working | Ensure `protected_branches_policy: false` | +| Deployment policies not created | Check `custom_branch_policies = true` in Terraform | + +For more configuration options, see [DEVELOPERS_GUIDE.md](DEVELOPERS_GUIDE.md) \ No newline at end of file diff --git a/docs/flow.md b/docs/flow.md new file mode 100644 index 0000000..fba129e --- /dev/null +++ b/docs/flow.md @@ -0,0 +1,185 @@ + +```mermaid +graph TD + %% Step 1: GitHub Import + GH[("🌐 GitHub API
(Existing Repos)")] + + GH -->|"go run main.go bulk-import"| IMP["🔧 github-repo-importer
(Go CLI Tool)"] + + %% Step 2: YAML Storage + IMP -->|"Creates YAML files"| CONFIGS["📁 configs/{owner}/*.yaml
(Temporary storage)"] + + %% Step 3: Justfile Processing + CONFIGS -->|"just import-repos"| JUST["📜 Justfile
(Bash script)"] + + JUST -->|"Check if exists"| DECISION{{"File exists in
repos/ ?"}} + + %% Step 4: File Distribution + DECISION -->|"NO: New import"| TMP["📁 importer_tmp_dir/
├── newRepo1232.yaml
└── 123123123.yaml"] + DECISION -->|"YES: Update"| REPOS["📁 repos/
├── test-env-111.yaml
└── existing-repo.yaml"] + + %% Step 5: Terraform Processing + TMP -->|"local.generated_repos"| TF_IMPORT["🔄 Terraform Import Blocks
import { for_each = local.generated_repos }"] + REPOS -->|"local.new_repos"| TF_CREATE["🏗️ Terraform Resources
(Create/Update)"] + + TF_IMPORT -->|"merge()"| MODULE["📦 module.repository
for_each = local.all_repos"] + TF_CREATE -->|"merge()"| MODULE + + %% Step 6: GitHub Operations + MODULE -->|"terraform apply"| GH_FINAL[("🌐 GitHub
(Creates/Updates/Imports)")] + + %% Step 7: Promotion + GH_FINAL -->|"After successful apply"| PROMOTE["🚀 Promote Workflow
(Move files)"] + PROMOTE -->|"mv importer_tmp_dir/* repos/"| REPOS_FINAL["📁 repos/
(Single source of truth)"] + + style GH fill:#e1f5fe + style GH_FINAL fill:#e1f5fe + style IMP fill:#fff3e0 + style JUST fill:#f3e5f5 + style TMP fill:#ffebee + style REPOS fill:#e8f5e9 + style REPOS_FINAL fill:#c8e6c9 + style MODULE fill:#fce4ec +``` + +# 🔍 Detailed Process Breakdown + +# 1️⃣ GitHub Import (github-repo-importer) + +Location: `feature/github-repo-importer/` + +```go +go run main.go bulk-import -c import-config.yaml +- Fetches repo data from GitHub API +- Creates YAML files in: configs/{owner}/*.yaml +``` + +# 2️⃣ YAML Storage Locations + + Execution Context: Where each command runs from +``` + | Stage | Full Path | Executed From Directory | + |---------------------------|---------------------------------------------------------------------------------------------------|-----------------------------------| + | 1. After Import | /home/.../github-terraformer/feature/github-repo-importer/configs/{owner}/*.yaml | feature/github-repo-importer/ | + | 2. Justfile Copy - New | /home/.../github-terraformer/feature/github-repo-provisioning/gcss_config/importer_tmp_dir/*.yaml | feature/github-repo-importer/ | + | 3. Justfile Copy - Update | /home/.../github-terraformer/feature/github-repo-provisioning/gcss_config/repos/*.yaml | feature/github-repo-importer/ | + | 4. Terraform Reads | gcss_config/importer_tmp_dir/*.yamlgcss_config/repos/*.yaml | feature/github-repo-provisioning/ | + | 5. Final (Promoted) | /home/.../gcss-config-repo/repos/*.yaml | N/A (separate repo) | +``` + +Directory Structure: + +```sh + github-terraformer/ + ├── feature/ + │ ├── github-repo-importer/ # 🔧 Import tool runs here + │ │ ├── Justfile + │ │ ├── main.go + │ │ └── configs/ # Step 1: Import creates YAMLs here + │ │ └── {owner}/ + │ │ ├── repo1.yaml + │ │ └── repo2.yaml + │ │ + │ └── github-repo-provisioning/ # 📦 Terraform runs here + │ ├── main.tf + │ └── gcss_config/ # This is actually gcss-config-repo checkout + │ ├── repos/ # Step 3: Existing repos updated here + │ │ └── existing.yaml + │ └── importer_tmp_dir/ # Step 2: New imports placed here + │ └── newRepo.yaml +``` + +Copies from: `configs/{owner}/*.yaml` + +Copies to: `../github-repo-provisioning/gcss_config/{repos or importer_tmp_dir}/` + +## Important Note: + + The gcss_config/ directory is actually a checkout of the gcss-config-repo (done by GitHub Actions), not a permanent part of github-terraformer! + +# 3️⃣ YAML → Terraform Transformation + +```hcl +# In main.tf - YAML becomes Terraform data +locals { +# Read YAML files and decode them + generated_repos = { + for file_path in fileset(..., "importer_tmp_dir/*.yaml") : + basename(file_path) => yamldecode(file(file_path)) # ← YAML → HCL + } +} +``` + +YAML structure becomes module variables + +```hcl +module "repository" { + for_each = local.all_repos + + # YAML fields map to module inputs + name = each.key # From filename + description = try(each.value.description, "") # From YAML content + visibility = try(each.value.visibility, "") # From YAML content + environments = try(each.value.environments, []) # From YAML content +} +``` + +# 4️⃣ Example YAML → Resource Flow + +```yaml +YAML File (123123123.yaml): +description: "My repo" +visibility: public +environments: +- environment: tesdt12 + wait_timer: 44 + reviewers: + users: + - ljubon +``` + +Becomes Terraform Resources: + +If in `importer_tmp_dir/` → Import block generated + +```hcl +import { + to = module.repository["123123123"].github_repository.repository + id = "123123123" +} + +import { + to = module.repository["123123123"].github_repository_environment.environment["tesdt12"] + id = "123123123:tesdt12" +} +``` + +Module creates actual resources + +```hcl +module "repository" { + # YAML filename → module key + for_each = { "123123123" = } + + # YAML fields → module variables + name = "123123123" + description = "My repo" + visibility = "public" + environments = [{ + environment = "tesdt12" + wait_timer = 44 + reviewers = { users = ["ljubon"] } + }] +} +``` + +# 5️⃣ Decision Tree + +```bash + Is repo already in repos/? + ├─ YES → Update existing file in repos/ + │ └─ Terraform updates resource + └─ NO → Place in importer_tmp_dir/ + ├─ Terraform imports from GitHub + └─ After success → Move to repos/ +``` diff --git a/feature/github-repo-importer/ADDING_FEATURES.md b/feature/github-repo-importer/ADDING_FEATURES.md new file mode 100644 index 0000000..1f7f815 --- /dev/null +++ b/feature/github-repo-importer/ADDING_FEATURES.md @@ -0,0 +1,244 @@ +# Adding New Features to the Importer + +This guide shows how to add new feature-gated functionality to the importer. The architecture is designed to be extremely simple and scalable. + +## Architecture Overview + +The importer uses a **config-driven feature flag system**: +- Features are controlled by `config/import-config.yaml` +- Same compiled binary works for all features +- Adding features requires **zero changes to CLI or command structure** +- Perfect for CI/CD workflows + +## Adding a New Feature: Step-by-Step Example + +Let's add a hypothetical `feature_github_webhooks` feature that imports GitHub repository webhooks. + +### Step 1: Add Feature Constant + +In `pkg/github/constants.go`, add your feature constant: + +```go +const ( + // Existing features + FeatureGithubEnvironment = "feature_github_environment" + + // Your new feature + FeatureGithubWebhooks = "feature_github_webhooks" +) +``` + +### Step 2: Add Feature Logic in ImportRepo + +In `pkg/github/github.go`, add your feature-gated code: + +```go +func ImportRepo(repoName string, cfg *Config) (*Repository, error) { + // ... existing code ... + + // ========================================================================= + // FEATURE: GitHub Webhooks + // ========================================================================= + var allWebhooks []*github.Hook + if cfg != nil && cfg.IsFeatureEnabled(FeatureGithubWebhooks) { + webhooks, _, err := v3client.Repositories.ListHooks( + context.Background(), + repoNameSplit[0], + repoNameSplit[1], + nil, + ) + if err != nil { + fmt.Printf("failed to get webhooks: %v\n", err) + } else { + allWebhooks = webhooks + + if err := dumpManager.WriteJSONFile("webhooks.json", webhooks); err != nil { + fmt.Printf("failed to write webhooks.json: %v\n", err) + } + } + } + + // ... rest of code ... + + return &Repository{ + // ... existing fields ... + Webhooks: resolveWebhooks(allWebhooks), // Your resolver function + }, nil +} +``` + +### Step 3: Add Data Structures + +In `pkg/github/repositories.go`, add the webhook field and structure: + +```go +type Repository struct { + // ... existing fields ... + Environments []Environment `yaml:"environments,omitempty"` + Webhooks []Webhook `yaml:"webhooks,omitempty"` // New field +} + +type Webhook struct { + URL string `yaml:"url"` + ContentType string `yaml:"content_type,omitempty"` + Events []string `yaml:"events,omitempty"` + Active bool `yaml:"active"` +} +``` + +### Step 4: Document in Config File + +In `gcss-config-repo/config/import-config.yaml`, document your feature: + +```yaml +# ============================================================================= +# FEATURE FLAGS +# ============================================================================= +# Control which features the importer should use when importing repositories. +# All features are disabled by default and must be explicitly enabled. +# +# Available feature flags: +# +# feature_github_environment: Import GitHub repository environments +# - When enabled, the importer fetches environment configurations from GitHub +# - This includes reviewers, deployment policies, and protection rules +# - Default: false (disabled - must explicitly enable) +# - Usage: Set to true to enable environment import +# +# feature_github_webhooks: Import GitHub repository webhooks (example - not implemented) +# - When enabled, the importer fetches and includes GitHub webhook configurations +# - Webhook secrets are NOT imported (GitHub API limitation) +# - Default: false (disabled - must explicitly enable) +# +# To enable environment import: +feature_github_environment: true + +# To disable (default behavior): +#feature_github_environment: false +``` + +### Step 5: That's It! + +**You're done!** The feature now works with: + +```bash +# Enable in config/import-config.yaml +feature_github_webhooks: true + +# Run importer (reads config automatically) +go run main.go import owner/repo + +# OR run `Bulk import` and verify plan for all repos with webhooks which will be imported in terraform state and yaml +``` + +## Key Patterns + +### Pattern 1: Feature Check +```go +if cfg != nil && cfg.IsFeatureEnabled(FeatureYourFeature) { + // Your feature code +} +``` + +### Pattern 2: Safe Defaults +- Always check `cfg != nil` before calling methods +- Features default to `false` if not in config +- Log when features are enabled for debugging + +### Pattern 3: Error Handling +- Don't fail the entire import if one feature fails +- Log errors with `fmt.Printf` for debugging +- Continue with other features + +### Pattern 4: Data Persistence +- Write API responses to JSON files via `dumpManager.WriteJSONFile()` +- This helps with debugging and auditing +- Files are stored in `dumps//` + +## Testing Your Feature + +### Test 1: Feature Disabled (Default) +```bash +# Without feature flag (should skip your feature) +go run main.go import owner/repo +# Check: dumps/owner-repo/ should NOT have your JSON file +``` + +### Test 2: Feature Enabled +```yaml +# In import-config.yaml +feature_your_feature: true +``` + +```bash +go run main.go import owner/repo +# Check: dumps/owner-repo/ should have your JSON file +# Check: YAML output should include your data +``` + +### Test 3: Bulk Import +```bash +go run main.go bulk-import +# Should respect feature flag for all repos +``` + +## Real-World Example: GitHub Environments + +See the `feature_github_environment` implementation as a complete reference: + +1. **Constant**: `pkg/github/constants.go:39` + ```go + FeatureGithubEnvironment = "feature_github_environment" + ``` + +2. **Logic**: `pkg/github/github.go:105-143` + ```go + if cfg != nil && cfg.IsFeatureEnabled(FeatureGithubEnvironment) { + // Fetch environments with pagination + // Write to JSON dump + // Store in allEnvironments + } + ``` + +3. **Data Structure**: `pkg/github/repositories.go:60-77` + ```go + type Environment struct { + Environment string + WaitTimer *int + // ... more fields + } + ``` + +4. **Resolution**: `pkg/github/github.go:667-720` + ```go + func resolveEnvironments(envs []*github.Environment) []Environment { + // Convert GitHub API response to YAML structure + } + ``` + +## Benefits of This Architecture + +✅ **No CLI Changes** - Features are purely config-driven +✅ **Single Binary** - One build works everywhere +✅ **CI/CD Friendly** - Change config without rebuilding +✅ **Backward Compatible** - Features default to disabled +✅ **Scalable** - Add unlimited features without refactoring +✅ **Type Safe** - Constants prevent typos +✅ **Self-Documenting** - Feature names clearly describe functionality + +## Common Pitfalls to Avoid + +❌ **Don't add function parameters** - Use `cfg.IsFeatureEnabled()` instead +❌ **Don't add CLI flags** - Keep it config-driven +❌ **Don't fail on missing data** - Handle errors gracefully +❌ **Don't forget nil checks** - Always check `cfg != nil` +❌ **Don't skip documentation** - Update import-config.yaml + +## Need Help? + +- Review existing features: `FeatureGithubEnvironment` +- Check the code comments in `pkg/github/github.go:105-113` +- Look at `pkg/github/constants.go:36-43` for examples +- Read `config/import-config.yaml` for feature documentation + +The architecture is intentionally simple - if you follow the 5 steps above, your feature will work perfectly with the existing system! diff --git a/feature/github-repo-importer/Justfile b/feature/github-repo-importer/Justfile index e007289..374efaa 100644 --- a/feature/github-repo-importer/Justfile +++ b/feature/github-repo-importer/Justfile @@ -17,4 +17,4 @@ compare dirA dirB: go run main.go compare {{dirA}} {{dirB}} generate-schema: - go run main.go schema \ No newline at end of file + go run main.go schema diff --git a/feature/github-repo-importer/LOCAL_DEVELOPMENT.md b/feature/github-repo-importer/LOCAL_DEVELOPMENT.md new file mode 100644 index 0000000..862ed06 --- /dev/null +++ b/feature/github-repo-importer/LOCAL_DEVELOPMENT.md @@ -0,0 +1,426 @@ +# Local Development Setup for GitHub Repo Importer + +This guide explains how to set up your local environment for developing and using the GitHub Repo Importer CLI tool. + +## Overview + +The GitHub Repo Importer is a Go-based CLI tool that imports GitHub repository configurations into YAML files for Terraform management. + +## Prerequisites + +1. **Go** (1.19+) + ```bash + # Check version + go version + + # Install if needed: https://golang.org/doc/install + ``` + +2. **Just** (command runner) + ```bash + # Install on Ubuntu/Debian + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin + + # Or with cargo + cargo install just + + # Or download from: https://github.com/casey/just/releases + ``` + +3. **GitHub Access** + - Personal Access Token or GitHub App credentials + - Required scopes: `repo`, `read:org`, `admin:org` + +## Directory Structure + +``` +github-repo-importer/ +├── cmd/ # Cobra CLI commands +│ ├── root.go # Root command setup +│ ├── import.go # Import single repository +│ ├── bulkImport.go # Import multiple repositories +│ └── compare.go # Compare configurations +├── pkg/ +│ ├── github/ # GitHub API interactions +│ │ ├── github.go # Main GitHub client +│ │ ├── repositories.go # Repository structures +│ │ └── constants.go # Feature flags and constants +│ ├── file/ # File operations +│ │ └── file.go # YAML file handling +│ └── compare/ # Configuration comparison +├── main.go # Entry point +├── Justfile # Task automation +├── go.mod # Go modules +└── import-config.yaml # Import configuration (local) +``` + +## Step 1: Environment Setup + +Create a `.env` file in the importer directory: + +```bash +cd github-terraformer/feature/github-repo-importer + +cat > .env << 'EOF' +# GitHub Authentication +export GITHUB_TOKEN="ghp_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" +export OWNER="your-org-name" + +# Feature Flags (via import-config.yaml) +# These are controlled in import-config.yaml, not env vars +EOF + +``` + +## Step 2: Import Configuration + +The importer uses `import-config.yaml` for configuration. Create or link it: + +### Option A: Link from gcss-config-repo (Recommended) + +```bash +# Create symlink to shared config +ln -sf ../../../gcss-config-repo/config/import-config.yaml import-config.yaml + +# Create symlink for app list (needed for GitHub App bypass actors) +ln -sf ../../../gcss-config-repo/config/app-list.yaml app-list.yaml + +# Optional: Create symlink for output directory (if you want imports to go directly to gcss-config-repo) +ln -sf ../../../gcss-config-repo/importer_tmp_dir importer_tmp_dir +``` +EOF +``` + +## Step 3: Build and Test + +```bash +# Load environment +source .env + +# Run tests +just test + +# Or with go directly +go test ./... + +# Build the binary +go build -o github-importer + +# Or run directly without building +go run main.go --help +``` + +## Step 4: Using the Importer + +### Import Single Repository + +```bash +# Using just +just import-repo your-org/repository-name + +# Using go run +go run main.go import your-org/repository-name + +# With custom output directory +go run main.go import your-org/repository-name -o ../custom-output-dir + +# With custom config +go run main.go import your-org/repository-name -c custom-config.yaml +``` + +### Bulk Import Repositories + +```bash +# Using just +just import-repos + +# Using go run +go run main.go bulk-import + +# With custom config +go run main.go bulk-import -c import-config.yaml + +# With custom output directory +go run main.go bulk-import -o ../output-dir +``` + +### Compare Configurations + +```bash +# Using just +just compare dirA dirB + +# Using go run +go run main.go compare dirA dirB + +# Compare with verbose output +go run main.go compare dirA dirB -v +``` + +## Step 5: Output Structure + +The importer generates YAML files in the output directory: + +```yaml +# output-dir/repository-name.yaml +description: "Repository description" +homepage_url: "https://example.com" +visibility: private +default_branch: main +has_issues: true +has_projects: true +has_wiki: false +has_downloads: true +allow_merge_commit: true +allow_rebase_merge: true +allow_squash_merge: true +delete_branch_on_merge: true +vulnerability_alerts_enabled: true + +# If feature_github_environment: true +environments: + - environment: production + wait_timer: 300 + can_admins_bypass: false + prevent_self_review: true + reviewers: + users: + - username1 + teams: + - team-slug + # IMPORTANT: deployment_ref_policy uses EITHER protected_branches_policy + # OR selected_branches_or_tags_policy, but NOT both + deployment_ref_policy: + # Option 1: Only protected branches can deploy + protected_branches_policy: true + + - environment: staging + deployment_ref_policy: + # Option 2: Custom branch/tag patterns (requires protected_branches_policy: false or omitted) + protected_branches_policy: false + selected_branches_or_tags_policy: + branch_patterns: + - "release/*" + - "hotfix/*" + tag_patterns: + - "v*" + +# Rulesets (if present) +rulesets: + - name: "Main Branch Protection" + target: "branch" + enforcement: "active" + # ... ruleset configuration +``` + +## Development Workflow + +### 1. Making Changes + +```bash +# Create a feature branch +git checkout -b feature/my-improvement + +# Make changes to code +vim pkg/github/github.go + +# Run tests +just test + +# Test specific functionality +go test ./pkg/github -v + +# Test import with your changes +go run main.go import your-org/test-repo +``` + +### 2. Testing Import Features + +```bash +# Test environment import +echo "feature_github_environment: true" >> import-config.yaml +go run main.go import your-org/repo-with-environments + +# Test ruleset import +go run main.go import your-org/repo-with-rulesets + +# Test bulk import with filters +cat > test-config.yaml << EOF +selected_repos: + - "your-org/test-repo1" + - "your-org/test-repo2" +feature_github_environment: true +EOF +go run main.go bulk-import -c test-config.yaml +``` + +### 3. Debugging + +```bash +# Enable debug output +export DEBUG=1 +go run main.go import your-org/repo + +# Use delve debugger +go get -u github.com/go-delve/delve/cmd/dlv +dlv debug main.go -- import your-org/repo + +# Check generated JSON dumps +ls -la jsondumps/ +``` + +## Common Commands Reference + +```bash +# Source environment +source .env + +# Run tests +just test +go test ./... +go test ./pkg/github -v -run TestSpecificFunction + +# Import operations +just import-repo owner/repo +just import-repos +just compare dir1 dir2 + +# Direct go commands +go run main.go import owner/repo +go run main.go bulk-import +go run main.go compare dir1 dir2 + +# Build +go build -o github-importer +./github-importer import owner/repo + +# Format code +go fmt ./... +gofmt -w . + +# Lint +golangci-lint run + +# Dependencies +go mod tidy +go mod download +``` + +## Troubleshooting + +### Issue: "GITHUB_TOKEN not set" + +**Solution**: Set and export the token: +```bash +export GITHUB_TOKEN="ghp_your_token_here" +# Or source .env file +source .env +``` + +### Issue: "401 Unauthorized" + +**Cause**: Invalid or expired GitHub token + +**Solution**: Generate a new token with required permissions: +- Go to GitHub Settings → Developer settings → Personal access tokens +- Required scopes: `repo`, `read:org`, `admin:org` + +### Issue: "403 rate limit exceeded" + +**Solution**: Wait for rate limit reset or use GitHub App authentication: +```bash +# Check rate limit +curl -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/rate_limit +``` + +### Issue: "repository not found" + +**Causes**: +1. Repository doesn't exist +2. No access to private repository +3. Wrong organization name + +**Solution**: Verify repository exists and token has access: +```bash +gh repo view your-org/repo-name +``` + +### Issue: Feature flags not working + +**Solution**: Ensure import-config.yaml is in the correct location: +```bash +# Default locations checked: +# 1. ./import-config.yaml +# 2. Specified with -c flag +ls -la import-config.yaml +cat import-config.yaml | grep feature_ +``` + +## Adding New Features + +1. **Add feature flag** in `pkg/github/constants.go`: + ```go + const FeatureMyNewFeature = "feature_my_new_feature" + ``` + +2. **Read feature flag** in `pkg/github/github.go`: + ```go + if cfg.Features[FeatureMyNewFeature] { + // Your feature code + } + ``` + +3. **Document in import-config.yaml**: + ```yaml + # feature_my_new_feature: Enable my new feature + feature_my_new_feature: false + ``` + +4. **Add tests**: + ```go + func TestMyNewFeature(t *testing.T) { + // Test implementation + } + ``` + +## Project Structure Best Practices + +- **cmd/**: CLI command definitions (Cobra) +- **pkg/github/**: GitHub API interactions +- **pkg/file/**: File I/O operations +- **pkg/compare/**: Comparison logic +- **internal/**: Private packages (if needed) + +## Testing + +```bash +# Unit tests +go test ./pkg/... + +# Integration tests +go test ./... -tags=integration + +# Coverage +go test ./... -cover +go test ./... -coverprofile=coverage.out +go tool cover -html=coverage.out + +# Benchmarks +go test -bench=. ./... +``` + +## CI/CD Integration + +The importer is used in GitHub Actions workflows: + +1. **Import workflow**: Imports single repository +2. **Bulk import workflow**: Imports multiple repositories +3. **Drift check**: Compares current vs desired state + +See `.github/workflows/` in github-terraformer for workflow definitions. + +## Support + +- Check `ADDING_FEATURES.md` for extending functionality +- Review existing imports in `gcss-config-repo/repos/` +- Open issues in the GitHub repository diff --git a/feature/github-repo-importer/QUICK_START.md b/feature/github-repo-importer/QUICK_START.md new file mode 100644 index 0000000..7efba25 --- /dev/null +++ b/feature/github-repo-importer/QUICK_START.md @@ -0,0 +1,41 @@ +# Quick Start - GitHub Repo Importer + +## Setup & Usage + +See the main documentation: [LOCAL_DEVELOPMENT_SETUP.md](../../docs/LOCAL_DEVELOPMENT_SETUP.md) + +## Quick Commands + +```bash +# Setup (one-time) +source .env + +# Import single repo +just import-repo owner/repo + +# Import all repos +just import-repos + +# Compare configs +just compare dir1 dir2 + +# Run tests +just test +``` + +## Common Patterns + +```bash +# Import with environments enabled +echo "feature_github_environment: true" >> import-config.yaml +just import-repo owner/repo + +# Custom output directory +go run main.go import owner/repo -o ../custom-dir + +# Debug mode +export DEBUG=1 +go run main.go import owner/repo +``` + +For full setup instructions and troubleshooting, see [LOCAL_DEVELOPMENT_SETUP.md](../../docs/LOCAL_DEVELOPMENT_SETUP.md) \ No newline at end of file diff --git a/feature/github-repo-importer/cmd/bulk-import.go b/feature/github-repo-importer/cmd/bulk-import.go index 1bc79a3..fb5594b 100644 --- a/feature/github-repo-importer/cmd/bulk-import.go +++ b/feature/github-repo-importer/cmd/bulk-import.go @@ -2,10 +2,8 @@ package cmd import ( "fmt" - "os" "github.com/spf13/cobra" - "gopkg.in/yaml.v3" "github.com/gr-oss-devops/github-repo-importer/pkg/github" ) @@ -50,28 +48,3 @@ func init() { rootCmd.AddCommand(bulkImportCmd) bulkImportCmd.Flags().StringVarP(&configFilePath, "config", "c", "./import-config.yaml", "Path to the yaml config file (defaults to ./import-config.yaml)") } - -func DecodeConfiguration(configFilePath string) (*github.Config, error) { - file, err := os.Open(configFilePath) - if err != nil { - return nil, fmt.Errorf("failed to open config file: %w", err) - } - defer func(file *os.File) { - err := file.Close() - if err != nil { - fmt.Printf("failed to close file: %v\n", err) - } - }(file) - - var cfg github.Config - if err := yaml.NewDecoder(file).Decode(&cfg); err != nil { - return nil, fmt.Errorf("failed to decode YAML: %w", err) - } - - if cfg.PageSize == nil { - ps := github.DefaultPageSize - cfg.PageSize = &ps - } - - return &cfg, nil -} diff --git a/feature/github-repo-importer/cmd/config.go b/feature/github-repo-importer/cmd/config.go new file mode 100644 index 0000000..44a6307 --- /dev/null +++ b/feature/github-repo-importer/cmd/config.go @@ -0,0 +1,36 @@ +package cmd + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" + + "github.com/gr-oss-devops/github-repo-importer/pkg/github" +) + +// DecodeConfiguration reads and decodes the import configuration file +func DecodeConfiguration(configFilePath string) (*github.Config, error) { + file, err := os.Open(configFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open config file: %w", err) + } + defer func(file *os.File) { + err := file.Close() + if err != nil { + fmt.Printf("failed to close file: %v\n", err) + } + }(file) + + var cfg github.Config + if err := yaml.NewDecoder(file).Decode(&cfg); err != nil { + return nil, fmt.Errorf("failed to decode YAML: %w", err) + } + + if cfg.PageSize == nil { + ps := github.DefaultPageSize + cfg.PageSize = &ps + } + + return &cfg, nil +} diff --git a/feature/github-repo-importer/cmd/import.go b/feature/github-repo-importer/cmd/import.go index c1844f9..e83ba72 100644 --- a/feature/github-repo-importer/cmd/import.go +++ b/feature/github-repo-importer/cmd/import.go @@ -8,29 +8,40 @@ import ( "github.com/gr-oss-devops/github-repo-importer/pkg/github" ) -var importCmd = &cobra.Command{ - Use: "import [owner/repo]", - Short: "Import command reads all repository details and creates a configuration yaml file", - Args: cobra.ExactArgs(1), - PreRun: func(cmd *cobra.Command, args []string) { - github.InitializeClients() - }, - RunE: func(cmd *cobra.Command, args []string) error { - repository := args[0] - - repo, err := github.ImportRepo(repository) - if err != nil { - return fmt.Errorf("failed to import repo: %w", err) - } - - if err := github.WriteRepositoryToYaml(repo); err != nil { - return fmt.Errorf("failed to handle repository: %w", err) - } - - return nil - }, -} +var ( + importConfigPath string + importCmd = &cobra.Command{ + Use: "import [owner/repo]", + Short: "Import command reads all repository details and creates a configuration yaml file", + Args: cobra.ExactArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + github.InitializeClients() + }, + RunE: func(cmd *cobra.Command, args []string) error { + repository := args[0] + + // Load configuration with all feature flags + cfg, err := DecodeConfiguration(importConfigPath) + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + // Pass the entire config to ImportRepo - it will check feature flags internally + repo, err := github.ImportRepo(repository, cfg) + if err != nil { + return fmt.Errorf("failed to import repo: %w", err) + } + + if err := github.WriteRepositoryToYaml(repo); err != nil { + return fmt.Errorf("failed to handle repository: %w", err) + } + + return nil + }, + } +) func init() { rootCmd.AddCommand(importCmd) + importCmd.Flags().StringVarP(&importConfigPath, "config", "c", "./import-config.yaml", "Path to the import config file (default: ./import-config.yaml)") } diff --git a/feature/github-repo-importer/pkg/github/config.go b/feature/github-repo-importer/pkg/github/config.go index fc838a6..67d4c13 100644 --- a/feature/github-repo-importer/pkg/github/config.go +++ b/feature/github-repo-importer/pkg/github/config.go @@ -5,10 +5,19 @@ import ( ) type Config struct { - IsPublic *bool `yaml:"is_public,omitempty"` - IgnoredRepos []string `yaml:"ignored_repos,omitempty"` - SelectedRepos []string `yaml:"selected_repos,omitempty"` - PageSize *int `yaml:"page_size,omitempty"` + IsPublic *bool `yaml:"is_public,omitempty"` + IgnoredRepos []string `yaml:"ignored_repos,omitempty"` + SelectedRepos []string `yaml:"selected_repos,omitempty"` + PageSize *int `yaml:"page_size,omitempty"` + Features map[string]bool `yaml:",inline"` +} + +// IsFeatureEnabled checks if a feature flag is enabled (default: false) +func (c *Config) IsFeatureEnabled(featureName string) bool { + if c.Features == nil { + return false + } + return c.Features[featureName] } func (c *Config) Validate() error { diff --git a/feature/github-repo-importer/pkg/github/constants.go b/feature/github-repo-importer/pkg/github/constants.go index a481b23..bff15a3 100644 --- a/feature/github-repo-importer/pkg/github/constants.go +++ b/feature/github-repo-importer/pkg/github/constants.go @@ -33,6 +33,15 @@ const ( DefaultPageSize = 100 + // Feature flags + // All feature flags follow the pattern: feature_ + // Add new features here and they'll automatically work with the config system + FeatureGithubEnvironment = "feature_github_environment" + // Example future features: + // FeatureGithubWebhooks = "feature_github_webhooks" + // FeatureGithubSecrets = "feature_github_secrets" + // FeatureGithubTopics = "feature_github_topics" + BypassActorType_RepositoryRole = "RepositoryRole" BypassActorType_OrganizationAdmin = "OrganizationAdmin" BypassActorType_Team = "Team" diff --git a/feature/github-repo-importer/pkg/github/github.go b/feature/github-repo-importer/pkg/github/github.go index 7c251bf..9cfc8f9 100644 --- a/feature/github-repo-importer/pkg/github/github.go +++ b/feature/github-repo-importer/pkg/github/github.go @@ -1,6 +1,7 @@ package github import ( + "bytes" "context" "encoding/json" "errors" @@ -51,9 +52,19 @@ func DecodeAppsList() (*AppsList, error) { return &appsList, nil } -func ImportRepo(repoName string) (*Repository, error) { +// ImportRepo imports a single repository with feature flags from the provided config +func ImportRepo(repoName string, cfg *Config) (*Repository, error) { fmt.Println("Importing repository: ", repoName) + // Log enabled features + if cfg != nil && cfg.Features != nil { + for featureName, enabled := range cfg.Features { + if enabled { + fmt.Printf("Feature enabled: %s\n", featureName) + } + } + } + if !isValidRepoFormat(repoName) { return nil, errors.New("invalid repository format. Use owner/repo") } @@ -92,6 +103,68 @@ func ImportRepo(repoName string) (*Repository, error) { fmt.Printf("failed to write pages.json: %v\n", err) } + // ========================================================================= + // FEATURE: GitHub Environments + // ========================================================================= + // Feature flag: feature_github_environment (default: DISABLED - opt-in feature) + // Set feature_github_environment: true in import-config.yaml to enable environment import. + // + // When enabled, environments are imported from GitHub and managed by Terraform. + var allEnvironments []*github.Environment + + // Check if feature is enabled (defaults to false - opt-in feature) + enableEnvironments := false + if cfg != nil && cfg.Features != nil { + if enabled, exists := cfg.Features[FeatureGithubEnvironment]; exists { + enableEnvironments = enabled + } + } + + if enableEnvironments { + envOpts := &github.EnvironmentListOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + for { + environments, res, err := v3client.Repositories.ListEnvironments(context.Background(), repoNameSplit[0], repoNameSplit[1], envOpts) + if err != nil { + if res != nil && res.StatusCode == http.StatusNotFound { + fmt.Printf("environments not found (this is normal for repositories without environments): %v\n", err) + } else { + fmt.Printf("failed to get environments: %v\n", err) + } + break + } + + if err := dumpManager.WriteJSONFile("environments.json", environments); err != nil { + fmt.Printf("failed to write environments.json: %v\n", err) + } + + // ListEnvironments returns basic info - we need to fetch full details for each environment + // to get reviewers, protection rules, and other detailed configuration + if environments != nil && environments.Environments != nil { + for _, env := range environments.Environments { + if env.Name != nil { + // Fetch full environment details including reviewers + fullEnv, _, err := v3client.Repositories.GetEnvironment(context.Background(), repoNameSplit[0], repoNameSplit[1], *env.Name) + if err != nil { + fmt.Printf("Warning: failed to get full details for environment %s: %v\n", *env.Name, err) + // Use basic info if detailed fetch fails + allEnvironments = append(allEnvironments, env) + } else { + // Use full environment data with all details + allEnvironments = append(allEnvironments, fullEnv) + } + } + } + } + + if res.NextPage == 0 { + break + } + envOpts.Page = res.NextPage + } + } + rulesets, r, err := v3client.Repositories.GetAllRulesets(context.Background(), repoNameSplit[0], repoNameSplit[1], false) if err != nil { if r.StatusCode == http.StatusForbidden { @@ -200,9 +273,11 @@ func ImportRepo(repoName string) (*Repository, error) { Rulesets: resolvedRulesets, VulnerabilityAlertsEnabled: &vulnerabilityAlertsEnabled, BranchProtectionsV4: resolveBranchProtectionsFromGraphQL(&branchProtectionRulesGraphQLQuery), + Environments: resolveEnvironments(allEnvironments, v3client, repoNameSplit[0], repoNameSplit[1]), }, nil } + func getRoleActors() map[int64]string { return map[int64]string{ BypassActorId_OrganizationAdminRole: BypassActorRoleName_OrganizationAdminRole, @@ -273,7 +348,7 @@ func ImportRepos(cfg Config) ([]*Repository, error) { var importedRepos []*Repository for _, repoToImport := range reposToImport { - repository, err := ImportRepo(repoToImport) + repository, err := ImportRepo(repoToImport, &cfg) if err != nil { return nil, fmt.Errorf("failed to import repository %s: %w", repository.Name, err) } @@ -679,6 +754,222 @@ func resolveRepositoryTemplate(githubRepository *github.Repository) *RepositoryT return nil } +// fetchDeploymentPolicies fetches deployment branch policies for a given environment +func fetchDeploymentPolicies(client *github.Client, owner, repo, envName string) ([]string, []string) { + + var branchPatterns []string + var tagPatterns []string + + // List deployment branch policies + // GitHub API endpoint: GET /repos/{owner}/{repo}/environments/{environment_name}/deployment-branch-policies + opts := &github.ListOptions{PerPage: 100} + for { + // Note: The go-github library may not have direct support for this endpoint yet + // We'll use the generic API call method + req, err := client.NewRequest("GET", fmt.Sprintf("repos/%s/%s/environments/%s/deployment-branch-policies", owner, repo, envName), nil) + if err != nil { + fmt.Printf("Warning: Failed to create request for deployment policies: %v\n", err) + break + } + + type DeploymentPolicy struct { + ID int64 `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + Type string `json:"type"` // "branch" or "tag" + } + + type DeploymentPoliciesResponse struct { + TotalCount int `json:"total_count"` + Policies []DeploymentPolicy `json:"branch_policies"` + } + + var result DeploymentPoliciesResponse + resp, err := client.Do(context.Background(), req, &result) + if err != nil { + // If 404, it might mean no custom policies are configured + if resp != nil && resp.StatusCode == 404 { + return nil, nil + } + fmt.Printf("Warning: Failed to fetch deployment policies for environment %s: %v\n", envName, err) + break + } + + for _, policy := range result.Policies { + if policy.Type == "branch" { + branchPatterns = append(branchPatterns, policy.Name) + } else if policy.Type == "tag" { + tagPatterns = append(tagPatterns, policy.Name) + } + } + + if resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + + return branchPatterns, tagPatterns +} + +func resolveEnvironments(envs []*github.Environment, client *github.Client, owner, repo string) []Environment { + if len(envs) == 0 { + return nil + } + + // Get organization info to obtain org ID (needed for team lookups) + org, _, err := client.Organizations.Get(context.Background(), owner) + if err != nil { + fmt.Printf("Warning: failed to get organization info: %v\n", err) + return nil + } + + var environments []Environment + for _, env := range envs { + environment := Environment{ + Environment: env.GetName(), + // WaitTimer will be extracted from ProtectionRules below + CanAdminsBypass: env.CanAdminsBypass, + } + + // Extract PreventSelfReview, WaitTimer, and Reviewers from ProtectionRules + // All are nested inside ProtectionRules array in the GitHub API + // Only set if actually found - don't assume defaults when importing + if env.ProtectionRules != nil && len(env.ProtectionRules) > 0 { + // We need to extract reviewers from ProtectionRules, not from env.Reviewers + protectionReviewers := &EnvironmentReviewers{} + + for _, rule := range env.ProtectionRules { + // Check for wait_timer rule type + if rule.Type != nil && *rule.Type == "wait_timer" { + if rule.WaitTimer != nil { + environment.WaitTimer = rule.WaitTimer + } + } + + if rule.PreventSelfReview != nil { + environment.PreventSelfReview = rule.PreventSelfReview + } + + // Check if this protection rule has reviewers + if rule.Reviewers != nil && len(rule.Reviewers) > 0 { + // Extract reviewers from this protection rule + // RequiredReviewer has Reviewer field (interface{}) that contains the actual user/team data + for _, reqReviewer := range rule.Reviewers { + if reqReviewer.Type == nil { + continue + } + + // The Reviewer field is an interface{} - try different type casts + if reqReviewer.Reviewer != nil { + switch *reqReviewer.Type { + case "Team": + // Try casting to *github.Team + if team, ok := reqReviewer.Reviewer.(*github.Team); ok { + if team.Slug != nil { + protectionReviewers.Teams = append(protectionReviewers.Teams, *team.Slug) + } else if team.Name != nil { + protectionReviewers.Teams = append(protectionReviewers.Teams, *team.Name) + } + } + case "User": + // Try casting to *github.User + if user, ok := reqReviewer.Reviewer.(*github.User); ok { + if user.Login != nil { + protectionReviewers.Users = append(protectionReviewers.Users, *user.Login) + } + } else { + // Fallback: try map[string]interface{} for older API versions + if reviewerData, ok := reqReviewer.Reviewer.(map[string]interface{}); ok { + if login, ok := reviewerData["login"].(string); ok { + protectionReviewers.Users = append(protectionReviewers.Users, login) + } + } + } + } + } + } + } + } + + // Set reviewers if we found any in ProtectionRules + if len(protectionReviewers.Teams) > 0 || len(protectionReviewers.Users) > 0 { + environment.Reviewers = protectionReviewers + } + } + + // Handle reviewers at top level (fallback if not in ProtectionRules) + // API may return array of EnvReviewers with Type and ID + // We resolve IDs to human-readable names (usernames and team slugs) + // Note: This is usually empty as reviewers are typically in ProtectionRules + if env.Reviewers != nil && len(env.Reviewers) > 0 && environment.Reviewers == nil { + reviewers := &EnvironmentReviewers{} + + // Separate reviewers by type and resolve IDs to names + for _, reviewer := range env.Reviewers { + if reviewer.Type != nil && reviewer.ID != nil { + switch *reviewer.Type { + case "Team": + // Resolve team ID to team slug + team, _, err := client.Teams.GetTeamByID(context.Background(), org.GetID(), *reviewer.ID) + if err != nil { + fmt.Printf("Warning: failed to resolve team ID %d: %v\n", *reviewer.ID, err) + continue + } + if team.Slug != nil { + reviewers.Teams = append(reviewers.Teams, *team.Slug) + } + case "User": + // Resolve user ID to username + user, _, err := client.Users.GetByID(context.Background(), *reviewer.ID) + if err != nil { + fmt.Printf("Warning: failed to resolve user ID %d: %v\n", *reviewer.ID, err) + continue + } + if user.Login != nil { + reviewers.Users = append(reviewers.Users, *user.Login) + } + } + } + } + + if len(reviewers.Teams) > 0 || len(reviewers.Users) > 0 { + environment.Reviewers = reviewers + } + } + + // Handle deployment ref policy + if env.DeploymentBranchPolicy != nil { + deploymentRefPolicy := &DeploymentRefPolicy{} + + // Set protected_branches_policy based on the protected_branches value + if env.DeploymentBranchPolicy.ProtectedBranches != nil { + protectedBranches := *env.DeploymentBranchPolicy.ProtectedBranches + deploymentRefPolicy.ProtectedBranchesPolicy = &protectedBranches + } + + // Check if custom_branch_policies is true + if env.DeploymentBranchPolicy.CustomBranchPolicies != nil && *env.DeploymentBranchPolicy.CustomBranchPolicies { + // Fetch deployment branch policies from GitHub API + branchPatterns, tagPatterns := fetchDeploymentPolicies(client, owner, repo, env.GetName()) + + if len(branchPatterns) > 0 || len(tagPatterns) > 0 { + deploymentRefPolicy.SelectedBranchesOrTagsPolicy = &SelectedBranchesOrTagsPolicy{ + BranchPatterns: branchPatterns, + TagPatterns: tagPatterns, + } + } + } + + environment.DeploymentRefPolicy = deploymentRefPolicy + } + + environments = append(environments, environment) + } + + return environments +} + func resolveVisibility(private bool) string { if private { return VisibilityPrivate @@ -823,17 +1114,21 @@ func CategorizeTeams(client *github.Client, owner, repo string, dumpManager *fil } func WriteRepositoryToYaml(repository *Repository) error { - data, err := yaml.Marshal(repository) - if err != nil { + // Use a buffer and encoder with 2-space indentation for consistent formatting + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) // Use 2-space indentation for consistency + if err := enc.Encode(repository); err != nil { return fmt.Errorf("failed to marshal repository to YAML: %w", err) } + enc.Close() configsBasePath := filepath.Join("./configs", repository.Owner) if err := os.MkdirAll(configsBasePath, os.ModePerm); err != nil { return fmt.Errorf("failed to create base directories: %w", err) } - if err := os.WriteFile(filepath.Join(configsBasePath, fmt.Sprintf("%s.yaml", repository.Name)), data, os.ModePerm); err != nil { + if err := os.WriteFile(filepath.Join(configsBasePath, fmt.Sprintf("%s.yaml", repository.Name)), buf.Bytes(), os.ModePerm); err != nil { return fmt.Errorf("failed to write repository to YAML: %w", err) } diff --git a/feature/github-repo-importer/pkg/github/github_test.go b/feature/github-repo-importer/pkg/github/github_test.go index fcb1ced..0474b71 100644 --- a/feature/github-repo-importer/pkg/github/github_test.go +++ b/feature/github-repo-importer/pkg/github/github_test.go @@ -237,7 +237,7 @@ func TestImportRepo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - repo, err := ImportRepo(tt.repoName) + repo, err := ImportRepo(tt.repoName, nil) if tt.wantError { assert.Error(t, err) diff --git a/feature/github-repo-importer/pkg/github/repositories.go b/feature/github-repo-importer/pkg/github/repositories.go index bc1c878..d6edd7c 100644 --- a/feature/github-repo-importer/pkg/github/repositories.go +++ b/feature/github-repo-importer/pkg/github/repositories.go @@ -45,6 +45,7 @@ type Repository struct { Rulesets []Ruleset `yaml:"rulesets,omitempty"` VulnerabilityAlertsEnabled *bool `yaml:"vulnerability_alerts_enabled,omitempty"` BranchProtectionsV4 []*BranchProtectionV4 `yaml:"branch_protections_v4,omitempty"` + Environments []Environment `yaml:"environments,omitempty"` } type RepositoryTemplate struct { @@ -58,3 +59,29 @@ type Pages struct { Path *string `yaml:"path,omitempty"` BuildType *string `yaml:"build_type,omitempty" jsonschema:"required,enum=workflow,enum=legacy"` } + +type Environment struct { + Environment string `yaml:"environment"` + WaitTimer *int `yaml:"wait_timer,omitempty"` + CanAdminsBypass *bool `yaml:"can_admins_bypass,omitempty"` + PreventSelfReview *bool `yaml:"prevent_self_review,omitempty"` // Extracted from ProtectionRules in API response + Reviewers *EnvironmentReviewers `yaml:"reviewers,omitempty"` + DeploymentRefPolicy *DeploymentRefPolicy `yaml:"deployment_ref_policy,omitempty"` +} + +type EnvironmentReviewers struct { + Teams []string `yaml:"teams,omitempty"` // Team slugs (e.g., "platform-team") + Users []string `yaml:"users,omitempty"` // GitHub usernames (e.g., "octocat") +} + +// DeploymentRefPolicy represents the new structure for deployment reference policies +type DeploymentRefPolicy struct { + ProtectedBranchesPolicy *bool `yaml:"protected_branches_policy,omitempty"` + SelectedBranchesOrTagsPolicy *SelectedBranchesOrTagsPolicy `yaml:"selected_branches_or_tags_policy,omitempty"` +} + +// SelectedBranchesOrTagsPolicy contains branch and tag patterns for custom deployment policies +type SelectedBranchesOrTagsPolicy struct { + BranchPatterns []string `yaml:"branch_patterns,omitempty"` // e.g., ["release/*", "main"] + TagPatterns []string `yaml:"tag_patterns,omitempty"` // e.g., ["v*", "release-*"] +} diff --git a/feature/github-repo-provisioning/Justfile b/feature/github-repo-provisioning/Justfile index 2df890a..3ec226a 100644 --- a/feature/github-repo-provisioning/Justfile +++ b/feature/github-repo-provisioning/Justfile @@ -2,4 +2,4 @@ clean: rm -rf .terraform .terraform.lock.hcl terraform.tfstate terraform.tfstate.backup init: - terraform init \ No newline at end of file + terraform init diff --git a/feature/github-repo-provisioning/QUICK_START.md b/feature/github-repo-provisioning/QUICK_START.md new file mode 100644 index 0000000..944aa24 --- /dev/null +++ b/feature/github-repo-provisioning/QUICK_START.md @@ -0,0 +1,58 @@ +# Quick Start - Local Terraform Development + +## One-Time Setup + +```bash +# 1. Create symlinks +ln -sfn ../../../gcss-config-repo gcss_config +ln -sf gcss_config/config/app-list.yaml app-list.yaml + +# 2. Create .env file (replace with your values) +cat > .env << 'EOF' +export GITHUB_TOKEN="ghp_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" +export GITHUB_OWNER="your-org-name" +export TF_VAR_app_private_key="$(cat ~/.secrets/github-app.pem)" +export TF_VAR_app_installation_id="12345678" +export TF_VAR_app_id="123456" +export TF_VAR_owner="your-org-name" +export TF_VAR_environment_directory="gcss_config" +EOF + +# 3. Use local backend +cp backend.tf.local backend.tf + +# 4. Initialize +source .env && terraform init +``` + +## Daily Usage + +```bash +# Always start with +source .env + +# Check changes +terraform plan + +# Apply changes +terraform apply +``` + +## File Locations + +- **New repos**: `gcss_config/repos/*.yaml` +- **Import existing**: `gcss_config/importer_tmp_dir/*.yaml` +- **App config**: `gcss_config/config/app-list.yaml` + +## Common Fixes + +```bash +# Repository doesn't exist (import error) +rm gcss_config/importer_tmp_dir/problematic-repo.yaml + +# Symlink broken +ln -sfn ../../../gcss-config-repo gcss_config + +# Variables not set +source .env +``` diff --git a/feature/github-repo-provisioning/main.tf b/feature/github-repo-provisioning/main.tf index 7ed838a..0990af3 100644 --- a/feature/github-repo-provisioning/main.tf +++ b/feature/github-repo-provisioning/main.tf @@ -1,9 +1,9 @@ provider "github" { owner = var.owner app_auth { - id = var.app_id + id = var.app_id installation_id = var.app_installation_id - pem_file = var.app_private_key + pem_file = var.app_private_key } } @@ -30,25 +30,26 @@ locals { ) all_repos = merge(local.generated_repos, local.new_repos) + } import { for_each = local.generated_repos - to = module.repository[each.key].github_repository.repository - id = each.key + to = module.repository[each.key].github_repository.repository + id = each.key } import { for_each = local.generated_repos - to = module.repository[each.key].github_branch_default.default[0] - id = each.key + to = module.repository[each.key].github_branch_default.default[0] + id = each.key } locals { flattened_generated_branch_protections_v4 = flatten([ for repo, config in local.generated_repos : [ for branch_protection in try(config.branch_protections_v4, []) : { - repository = repo + repository = repo branch_protection = branch_protection } ] @@ -76,51 +77,51 @@ locals { data "github_app" "app" { for_each = toset(local.app_actors) - slug = split("/", each.value)[1] + slug = split("/", each.value)[1] } locals { all_generated_collaborators = { for repo, config in local.generated_repos : repo => concat( - try([for i in config.pull_collaborators : { username: i, permission = "pull" }], []), - try([for i in config.push_collaborators : { username: i, permission = "push" }], []), - try([for i in config.admin_collaborators : { username: i, permission = "admin" }], []), - try([for i in config.maintain_collaborators : { username: i, permission = "maintain" }], []), - try([for i in config.triage_collaborators : { username: i, permission = "triage" }], []) - )} + try([for i in config.pull_collaborators : { username : i, permission = "pull" }], []), + try([for i in config.push_collaborators : { username : i, permission = "push" }], []), + try([for i in config.admin_collaborators : { username : i, permission = "admin" }], []), + try([for i in config.maintain_collaborators : { username : i, permission = "maintain" }], []), + try([for i in config.triage_collaborators : { username : i, permission = "triage" }], []) + ) } all_generated_teams = { for repo, config in local.generated_repos : repo => concat( - try([for i in config.pull_teams : { name: i, permission = "pull" }], []), - try([for i in config.push_teams : { name: i, permission = "push" }], []), - try([for i in config.admin_teams : { name: i, permission = "admin" }], []), - try([for i in config.maintain_teams : { name: i, permission = "maintain" }], []), - try([for i in config.triage_teams : { name: i, permission = "triage" }], []) - )} + try([for i in config.pull_teams : { name : i, permission = "pull" }], []), + try([for i in config.push_teams : { name : i, permission = "push" }], []), + try([for i in config.admin_teams : { name : i, permission = "admin" }], []), + try([for i in config.maintain_teams : { name : i, permission = "maintain" }], []), + try([for i in config.triage_teams : { name : i, permission = "triage" }], []) + ) } all_new_collaborators = { for repo, config in local.new_repos : repo => concat( - try([for i in config.pull_collaborators : { username: i, permission = "pull" }], []), - try([for i in config.push_collaborators : { username: i, permission = "push" }], []), - try([for i in config.admin_collaborators : { username: i, permission = "admin" }], []), - try([for i in config.maintain_collaborators : { username: i, permission = "maintain" }], []), - try([for i in config.triage_collaborators : { username: i, permission = "triage" }], []) - )} + try([for i in config.pull_collaborators : { username : i, permission = "pull" }], []), + try([for i in config.push_collaborators : { username : i, permission = "push" }], []), + try([for i in config.admin_collaborators : { username : i, permission = "admin" }], []), + try([for i in config.maintain_collaborators : { username : i, permission = "maintain" }], []), + try([for i in config.triage_collaborators : { username : i, permission = "triage" }], []) + ) } all_new_teams = { for repo, config in local.new_repos : repo => concat( - try([for i in config.pull_teams : { name: i, permission = "pull" }], []), - try([for i in config.push_teams : { name: i, permission = "push" }], []), - try([for i in config.admin_teams : { name: i, permission = "admin" }], []), - try([for i in config.maintain_teams : { name: i, permission = "maintain" }], []), - try([for i in config.triage_teams : { name: i, permission = "triage" }], []) - )} + try([for i in config.pull_teams : { name : i, permission = "pull" }], []), + try([for i in config.push_teams : { name : i, permission = "push" }], []), + try([for i in config.admin_teams : { name : i, permission = "admin" }], []), + try([for i in config.maintain_teams : { name : i, permission = "maintain" }], []), + try([for i in config.triage_teams : { name : i, permission = "triage" }], []) + ) } all_collaborators = merge(local.all_generated_collaborators, local.all_new_collaborators) - all_teams = merge(local.all_generated_teams, local.all_new_teams) + all_teams = merge(local.all_generated_teams, local.all_new_teams) } import { for_each = toset(flatten([for repo, collaborators in local.all_generated_collaborators : [ for collaborator in collaborators : { - repo = repo - username = collaborator.username + repo = repo + username = collaborator.username permission = collaborator.permission } ]])) @@ -141,9 +142,9 @@ data "github_team" "team" { import { for_each = toset(flatten([for repo, teams in local.all_generated_teams : [ for team in teams : { - repo = repo - name = team.name - team_id = data.github_team.team[team.name].id + repo = repo + name = team.name + team_id = data.github_team.team[team.name].id } ]])) @@ -153,45 +154,45 @@ import { module "repository" { - source = "./modules/terraform-github-repository" - for_each = local.all_repos + source = "./modules/terraform-github-repository" + for_each = local.all_repos # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Main resource configuration # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - name = each.key - allow_merge_commit = try(each.value.allow_merge_commit, true) - allow_rebase_merge = try(each.value.allow_rebase_merge, false) - allow_squash_merge = try(each.value.allow_squash_merge, false) - allow_auto_merge = try(each.value.allow_auto_merge, false) - allow_update_branch = try(each.value.allow_update_branch, null) - description = try(each.value.description, "") - delete_branch_on_merge = try(each.value.delete_branch_on_merge, true) - homepage_url = try(each.value.homepage_url, "") - visibility = try(each.value.visibility, "private") - has_issues = try(each.value.has_issues, false) - has_projects = try(each.value.has_projects, false) - has_wiki = try(each.value.has_wiki, false) - has_downloads = try(each.value.has_downloads, false) - has_discussions = try(each.value.has_discussions, null) - is_template = try(each.value.is_template, false) - default_branch = try(each.value.default_branch, "") - archived = try(each.value.archived, false) - topics = try(each.value.topics, []) - archive_on_destroy = try(each.value.archive_on_destroy, null) - pages = try(contains(keys(each.value), "pages") && try(each.value.pages != null, false) ? { - branch = try(each.value.pages.build_type, null) == "workflow" ? null : try(each.value.pages.branch, "gh-pages") - path = try(each.value.pages.build_type, null) == "workflow" ? null : try(each.value.pages.path, "/") - cname = try(each.value.pages.cname, null) - build_type = try(each.value.pages.build_type, null) - } : null) - vulnerability_alerts = try(each.value.vulnerability_alerts_enabled, null) - - squash_merge_commit_title = try(each.value.squash_merge_commit_title, null) + name = each.key + allow_merge_commit = try(each.value.allow_merge_commit, true) + allow_rebase_merge = try(each.value.allow_rebase_merge, false) + allow_squash_merge = try(each.value.allow_squash_merge, false) + allow_auto_merge = try(each.value.allow_auto_merge, false) + allow_update_branch = try(each.value.allow_update_branch, null) + description = try(each.value.description, "") + delete_branch_on_merge = try(each.value.delete_branch_on_merge, true) + homepage_url = try(each.value.homepage_url, "") + visibility = try(each.value.visibility, "private") + has_issues = try(each.value.has_issues, false) + has_projects = try(each.value.has_projects, false) + has_wiki = try(each.value.has_wiki, false) + has_downloads = try(each.value.has_downloads, false) + has_discussions = try(each.value.has_discussions, null) + is_template = try(each.value.is_template, false) + default_branch = try(each.value.default_branch, "") + archived = try(each.value.archived, false) + topics = try(each.value.topics, []) + archive_on_destroy = try(each.value.archive_on_destroy, null) + pages = try(contains(keys(each.value), "pages") && try(each.value.pages != null, false) ? { + branch = try(each.value.pages.branch, "gh-pages") + path = try(each.value.pages.path, "/") + cname = try(each.value.pages.cname, null) + build_type = try(each.value.pages.build_type, null) + } : null) + vulnerability_alerts = try(each.value.vulnerability_alerts_enabled, null) + + squash_merge_commit_title = try(each.value.squash_merge_commit_title, null) squash_merge_commit_message = try(each.value.squash_merge_commit_message, null) - merge_commit_title = try(each.value.merge_commit_title, null) - merge_commit_message = try(each.value.merge_commit_message, null) + merge_commit_title = try(each.value.merge_commit_title, null) + merge_commit_message = try(each.value.merge_commit_message, null) web_commit_signoff_required = try(each.value.web_commit_signoff_required, null) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -199,33 +200,33 @@ module "repository" { # Repository Creation Configuration # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - auto_init = try(each.value.auto_init, true) - gitignore_template = try(each.value.gitignore_template, "") - license_template = try(each.value.license_template, "") - template = try(contains(keys(each.value), "template") && try(each.value.template != null, false) ? { - owner = try(each.value.template.owner, "") - repository = try(each.value.template.repository, "") - } : null) + auto_init = try(each.value.auto_init, true) + gitignore_template = try(each.value.gitignore_template, "") + license_template = try(each.value.license_template, "") + template = try(contains(keys(each.value), "template") && try(each.value.template != null, false) ? { + owner = try(each.value.template.owner, "") + repository = try(each.value.template.repository, "") + } : null) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Teams Configuration # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - pull_teams = try([for i in each.value.pull_teams : data.github_team.team[i].id], []) - push_teams = try([for i in each.value.push_teams : data.github_team.team[i].id], []) - admin_teams = try([for i in each.value.admin_teams : data.github_team.team[i].id], []) - maintain_teams = try([for i in each.value.maintain_teams : data.github_team.team[i].id], []) - triage_teams = try([for i in each.value.triage_teams : data.github_team.team[i].id], []) + pull_teams = try([for i in each.value.pull_teams : data.github_team.team[i].id], []) + push_teams = try([for i in each.value.push_teams : data.github_team.team[i].id], []) + admin_teams = try([for i in each.value.admin_teams : data.github_team.team[i].id], []) + maintain_teams = try([for i in each.value.maintain_teams : data.github_team.team[i].id], []) + triage_teams = try([for i in each.value.triage_teams : data.github_team.team[i].id], []) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Collaborator Configuration # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - pull_collaborators = try(each.value.pull_collaborators, []) - push_collaborators = try(each.value.push_collaborators, []) - admin_collaborators = try(each.value.admin_collaborators, []) - maintain_collaborators = try(each.value.maintain_collaborators, []) - triage_collaborators = try(each.value.triage_collaborators, []) + pull_collaborators = try(each.value.pull_collaborators, []) + push_collaborators = try(each.value.push_collaborators, []) + admin_collaborators = try(each.value.admin_collaborators, []) + maintain_collaborators = try(each.value.maintain_collaborators, []) + triage_collaborators = try(each.value.triage_collaborators, []) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Branches Configuration @@ -244,16 +245,16 @@ module "repository" { # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ branch_protections_v4 = try([ for branch_protection in try(each.value.branch_protections_v4, []) : { - pattern = branch_protection.pattern - allows_deletions = try(branch_protection.allows_deletions, false) - allows_force_pushes = try(branch_protection.allows_force_pushes, false) - force_push_bypassers = try([for bypasser in branch_protection.force_push_bypassers : (!startswith(bypasser, "app/") ? bypasser : data.github_app.app[bypasser].node_id)], []) - enforce_admins = try(branch_protection.enforce_admins, true) - lock_branch = try(branch_protection.lock_branch, null) + pattern = branch_protection.pattern + allows_deletions = try(branch_protection.allows_deletions, false) + allows_force_pushes = try(branch_protection.allows_force_pushes, false) + force_push_bypassers = try([for bypasser in branch_protection.force_push_bypassers : (!startswith(bypasser, "app/") ? bypasser : data.github_app.app[bypasser].node_id)], []) + enforce_admins = try(branch_protection.enforce_admins, true) + lock_branch = try(branch_protection.lock_branch, null) - restricts_pushes = try(branch_protection.restricts_pushes, false) - blocks_creations = try(branch_protection.blocks_creations, false) - push_restrictions = try([for bypasser in branch_protection.push_restrictions : (!startswith(bypasser, "app/") ? bypasser : data.github_app.app[bypasser].node_id)], []) + restricts_pushes = try(branch_protection.restricts_pushes, false) + blocks_creations = try(branch_protection.blocks_creations, false) + push_restrictions = try([for bypasser in branch_protection.push_restrictions : (!startswith(bypasser, "app/") ? bypasser : data.github_app.app[bypasser].node_id)], []) require_conversation_resolution = try(branch_protection.require_conversation_resolution, false) require_signed_commits = try(branch_protection.require_signed_commits, false) @@ -302,15 +303,64 @@ module "repository" { # App Installations # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# app_installations = try(each.value.app_installations, []) + # app_installations = try(each.value.app_installations, []) + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Environments Configuration + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + environments = try(each.value.environments, []) +} + +# --------------------------------------------------------------------------------------------------------------------- +# Environment Management Strategy: +# +# 1. IMPORT (from importer_tmp_dir/): +# - These environments already exist in GitHub +# - Use import blocks to bring them into Terraform state +# - After import, move YAML to repos/ for ongoing management +# +# 2. CREATE (from repos/): +# - These are new environments or changes to existing repos +# - Terraform will create/update them without import blocks +# - No import needed - Terraform manages the full lifecycle +# +# --------------------------------------------------------------------------------------------------------------------- + +# Import environments from generated_repos (importer_tmp_dir/) only +# These should already exist in GitHub and need to be imported +locals { + flattened_generated_environments = flatten([ + for repo, config in local.generated_repos : [ + for environment in try(config.environments, []) : { + repository = repo + environment = environment + } + ] + ]) + + generated_environments_map = { + for item in local.flattened_generated_environments : + "${item.repository}:${item.environment.environment}" => item + } +} + +import { + for_each = local.generated_environments_map + + to = module.repository[each.value.repository].github_repository_environment.environment[each.value.environment.environment] + id = "${each.value.repository}:${each.value.environment.environment}" } +# Environments from new_repos (repos/ directory) are NOT imported +# They will be created/updated by Terraform as regular resources + locals { new_rulesets_flattened = flatten([ for repo, config in local.new_repos : [ for ruleset in try(config.rulesets, []) : { - repository = repo - ruleset = ruleset + repository = repo + ruleset = ruleset } ] ]) @@ -323,8 +373,8 @@ locals { generated_rulesets_flattened = flatten([ for repo, config in local.generated_repos : [ for ruleset in try(config.rulesets, []) : { - repository = repo - ruleset = ruleset + repository = repo + ruleset = ruleset } ] ]) @@ -339,8 +389,8 @@ locals { import { for_each = local.generated_rulesets_map - to = github_repository_ruleset.ruleset[each.key] - id = format("%s:%s", each.value.repository, each.value.ruleset.id) + to = github_repository_ruleset.ruleset[each.key] + id = format("%s:%s", each.value.repository, each.value.ruleset.id) } locals { @@ -380,7 +430,7 @@ locals { data "github_team" "ruleset_team" { for_each = toset(local.team_bypass_actors) - slug = each.value + slug = each.value } locals { @@ -393,8 +443,8 @@ locals { resource "github_repository_ruleset" "ruleset" { depends_on = [module.repository] - for_each = local.all_rulesets_map - name = each.value.ruleset.name + for_each = local.all_rulesets_map + name = each.value.ruleset.name enforcement = each.value.ruleset.enforcement target = each.value.ruleset.target repository = each.value.repository @@ -478,9 +528,9 @@ resource "github_repository_ruleset" "ruleset" { dynamic "required_status_checks" { for_each = ( - contains(keys(each.value.ruleset.rules), "required_status_checks") && - try(each.value.ruleset.rules.required_status_checks != null, false) && - length(try(each.value.ruleset.rules.required_status_checks.required_check, [])) > 0 + contains(keys(each.value.ruleset.rules), "required_status_checks") && + try(each.value.ruleset.rules.required_status_checks != null, false) && + length(try(each.value.ruleset.rules.required_status_checks.required_check, [])) > 0 ) ? [each.value.ruleset.rules.required_status_checks] : [] content { @@ -515,7 +565,7 @@ resource "github_repository_ruleset" "ruleset" { contains(keys(each.value.ruleset.rules), "required_code_scanning") && try(each.value.ruleset.rules.required_code_scanning != null, false) && length(try(each.value.ruleset.rules.required_code_scanning.required_code_scanning_tool, [])) > 0 - ? [each.value.ruleset.rules.required_code_scanning] # Only one block for `required_code_scanning` + ? [each.value.ruleset.rules.required_code_scanning] # Only one block for `required_code_scanning` : [] ) @@ -524,8 +574,8 @@ resource "github_repository_ruleset" "ruleset" { for_each = try(each.value.ruleset.rules.required_code_scanning.required_code_scanning_tool, []) content { - tool = required_code_scanning_tool.value.tool - alerts_threshold = required_code_scanning_tool.value.alerts_threshold + tool = required_code_scanning_tool.value.tool + alerts_threshold = required_code_scanning_tool.value.alerts_threshold security_alerts_threshold = required_code_scanning_tool.value.security_alerts_threshold } } @@ -537,13 +587,13 @@ resource "github_repository_ruleset" "ruleset" { for_each = try(each.value.ruleset.bypass_actors, []) content { - actor_id = startswith(bypass_actors.value.name, "team/") ? data.github_team.ruleset_team[replace(bypass_actors.value.name, "team/", "")].id : ( + actor_id = startswith(bypass_actors.value.name, "team/") ? data.github_team.ruleset_team[replace(bypass_actors.value.name, "team/", "")].id : ( startswith(bypass_actors.value.name, "app/") ? local.apps_map[bypass_actors.value.name].app_id : local.ruleset_actors[bypass_actors.value.name].actor_id ) - actor_type = startswith(bypass_actors.value.name, "team/") ? "Team" : ( + actor_type = startswith(bypass_actors.value.name, "team/") ? "Team" : ( startswith(bypass_actors.value.name, "app/") ? "Integration" : local.ruleset_actors[bypass_actors.value.name].actor_type ) - bypass_mode = try(bypass_actors.value.bypass_mode, "always") + bypass_mode = try(bypass_actors.value.bypass_mode, "always") } } } diff --git a/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf b/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf index ef54ccc..e19acf3 100644 --- a/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf +++ b/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf @@ -156,11 +156,11 @@ resource "github_repository" "repository" { ] } - squash_merge_commit_title = local.squash_merge_commit_title - squash_merge_commit_message = local.squash_merge_commit_message - merge_commit_title = local.merge_commit_title - merge_commit_message = local.merge_commit_message - web_commit_signoff_required = local.web_commit_signoff_required + squash_merge_commit_title = local.squash_merge_commit_title + squash_merge_commit_message = local.squash_merge_commit_message + merge_commit_title = local.merge_commit_title + merge_commit_message = local.merge_commit_message + web_commit_signoff_required = local.web_commit_signoff_required } # --------------------------------------------------------------------------------------------------------------------- @@ -249,15 +249,15 @@ resource "github_branch_protection" "branch_protection" { dynamic "restrict_pushes" { for_each = var.branch_protections_v4[each.value].restricts_pushes ? try([var.branch_protections_v4[each.value]], []) : [] content { - blocks_creations = try(var.branch_protections_v4[each.value].blocks_creations, true) - push_allowances = try(var.branch_protections_v4[each.value].push_restrictions, []) + blocks_creations = try(var.branch_protections_v4[each.value].blocks_creations, true) + push_allowances = try(var.branch_protections_v4[each.value].push_restrictions, []) } } - force_push_bypassers = try(var.branch_protections_v4[each.value].force_push_bypassers, []) - allows_force_pushes = try(var.branch_protections_v4[each.value].allows_force_pushes, null) - allows_deletions = try(var.branch_protections_v4[each.value].allows_deletions, null) - lock_branch = try(var.branch_protections_v4[each.value].lock_branch, null) + force_push_bypassers = try(var.branch_protections_v4[each.value].force_push_bypassers, []) + allows_force_pushes = try(var.branch_protections_v4[each.value].allows_force_pushes, null) + allows_deletions = try(var.branch_protections_v4[each.value].allows_deletions, null) + lock_branch = try(var.branch_protections_v4[each.value].lock_branch, null) } # --------------------------------------------------------------------------------------------------------------------- @@ -591,3 +591,128 @@ resource "github_app_installation_repository" "app_installation_repository" { repository = github_repository.repository.name installation_id = each.value } + +# --------------------------------------------------------------------------------------------------------------------- +# Repository Environments +# --------------------------------------------------------------------------------------------------------------------- + +locals { + # Create environments map + environments_map = { for e in var.environments : e.environment => e } + + # Flatten all usernames across all environments for lookup + all_reviewer_usernames = distinct(flatten([ + for env in var.environments : + try(env.reviewers.users, []) + ])) + + # Flatten all team slugs across all environments for lookup + all_reviewer_team_slugs = distinct(flatten([ + for env in var.environments : + try(env.reviewers.teams, []) + ])) +} + +# Data sources to resolve usernames to user IDs +data "github_user" "reviewer" { + for_each = toset(local.all_reviewer_usernames) + username = each.value +} + +# Data sources to resolve team slugs to team IDs +data "github_team" "reviewer" { + for_each = toset(local.all_reviewer_team_slugs) + slug = each.value +} + +resource "github_repository_environment" "environment" { + for_each = local.environments_map + + environment = each.key + repository = github_repository.repository.name + wait_timer = try(each.value.wait_timer, null) + can_admins_bypass = try(each.value.can_admins_bypass, true) + prevent_self_review = try(each.value.prevent_self_review, false) + + dynamic "reviewers" { + for_each = try(each.value.reviewers, null) != null ? [each.value.reviewers] : [] + + content { + # Convert team slugs to team IDs + teams = try(reviewers.value.teams, null) != null ? [ + for team_slug in reviewers.value.teams : data.github_team.reviewer[team_slug].id + ] : null + + # Convert usernames to user IDs + users = try(reviewers.value.users, null) != null ? [ + for username in reviewers.value.users : data.github_user.reviewer[username].id + ] : null + } + } + + dynamic "deployment_branch_policy" { + for_each = ( + try(each.value.deployment_ref_policy.protected_branches_policy, null) != null ? ( + # If protected_branches_policy is true, use protected branches + try(each.value.deployment_ref_policy.protected_branches_policy, false) == true ? [{ + protected_branches = true + custom_branch_policies = false + }] : + # If protected_branches_policy is false and we have custom policies, enable custom policies + try(each.value.deployment_ref_policy.selected_branches_or_tags_policy, null) != null ? [{ + protected_branches = false + custom_branch_policies = true + }] : [] + ) : [] + ) + + content { + protected_branches = deployment_branch_policy.value.protected_branches + custom_branch_policies = deployment_branch_policy.value.custom_branch_policies + } + } +} + +# Create deployment branch policies for custom branch patterns +resource "github_repository_environment_deployment_policy" "branch_policies" { + # Create a policy for each branch pattern in each environment that has custom policies + for_each = { + for item in flatten([ + for env_name, env in local.environments_map : [ + for branch_pattern in try(env.deployment_ref_policy.selected_branches_or_tags_policy.branch_patterns, []) : { + key = "${env_name}:branch:${branch_pattern}" + env = env_name + pattern = branch_pattern + } + ] + ]) : item.key => item + } + + repository = github_repository.repository.name + environment = github_repository_environment.environment[each.value.env].environment + branch_pattern = each.value.pattern + + depends_on = [github_repository_environment.environment] +} + +# Create deployment tag policies for custom tag patterns +resource "github_repository_environment_deployment_policy" "tag_policies" { + # Create a policy for each tag pattern in each environment that has custom policies + for_each = { + for item in flatten([ + for env_name, env in local.environments_map : [ + for tag_pattern in try(env.deployment_ref_policy.selected_branches_or_tags_policy.tag_patterns, []) : { + key = "${env_name}:tag:${tag_pattern}" + env = env_name + pattern = tag_pattern + } + ] + ]) : item.key => item + } + + repository = github_repository.repository.name + environment = github_repository_environment.environment[each.value.env].environment + tag_pattern = each.value.pattern + + depends_on = [github_repository_environment.environment] +} diff --git a/feature/github-repo-provisioning/modules/terraform-github-repository/outputs.tf b/feature/github-repo-provisioning/modules/terraform-github-repository/outputs.tf index d0c02a9..de0a87e 100644 --- a/feature/github-repo-provisioning/modules/terraform-github-repository/outputs.tf +++ b/feature/github-repo-provisioning/modules/terraform-github-repository/outputs.tf @@ -84,6 +84,21 @@ output "app_installations" { description = "A map of deploy app installations keyed by installation id." } +output "environments" { + value = github_repository_environment.environment + description = "A map of repository environments keyed by environment name." +} + +output "environment_deployment_branch_policies" { + value = github_repository_environment_deployment_policy.branch_policies + description = "A map of environment deployment branch policies." +} + +output "environment_deployment_tag_policies" { + value = github_repository_environment_deployment_policy.tag_policies + description = "A map of environment deployment tag policies." +} + # ---------------------------------------------------------------------------------------------------------------------- # OUTPUT MODULE CONFIGURATION # ---------------------------------------------------------------------------------------------------------------------- diff --git a/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf b/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf index f4a899b..4a8cba1 100644 --- a/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf +++ b/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf @@ -127,9 +127,9 @@ variable "merge_commit_title" { } variable "web_commit_signoff_required" { - description = "(Optional) Set to true to require commit signoff for all commits pushed to the repository. (Default: null)" - type = bool - default = null + description = "(Optional) Set to true to require commit signoff for all commits pushed to the repository. (Default: null)" + type = bool + default = null } variable "merge_commit_message" { @@ -598,6 +598,48 @@ variable "app_installations" { default = [] } +variable "environments" { + type = any + description = "(Optional) Configure repository environments with deployment protection rules and reviewers." + # type = list(object({ + # environment = string + # wait_timer = optional(number) + # can_admins_bypass = optional(bool) + # prevent_self_review = optional(bool) + # reviewers = optional(object({ + # teams = optional(list(string)) + # users = optional(list(string)) + # })) + # deployment_branch_policy = optional(object({ + # protected_branches = bool # Set to true to restrict to protected branches only + # # Set to false or omit to allow any branch to deploy + # })) + # })) + + default = [] + + # Example: + # environments = [ + # { + # environment = "production" + # wait_timer = 300 # seconds (5 minutes) + # can_admins_bypass = false + # prevent_self_review = true + # reviewers = { + # teams = ["platform-team"] + # users = ["octocat", "hubot"] + # } + # deployment_branch_policy = { + # protected_branches = true + # } + # }, + # { + # environment = "staging" + # # No deployment_branch_policy = any branch can deploy + # } + # ] +} + # ------------------------------------------------------------------------------ # MODULE CONFIGURATION PARAMETERS # These variables are used to configure the module. From 25f6f17d107deeefab84e217009fd75d89e1fac1 Mon Sep 17 00:00:00 2001 From: Ljubo Nikolic Date: Tue, 18 Nov 2025 20:17:16 +0100 Subject: [PATCH 02/15] new structure --- .gitignore | 11 +- docs/DEVELOPERS_GUIDE.md | 22 +- ...nment.md => FEATURE_GITHUB_ENVIRONMENT.md} | 57 ++- docs/GITHUB_ACTIONS_WORKFLOWS.md | 353 +++++++++++++++ docs/LOCAL_DEVELOPMENT_SETUP.md | 385 +++++++++++----- docs/{flow.md => ci_flows.md} | 92 ++-- docs/workflows.md | 44 -- .../github-repo-importer/ADDING_FEATURES.md | 326 ++++++-------- .../github-repo-importer/LOCAL_DEVELOPMENT.md | 426 ------------------ feature/github-repo-importer/QUICK_START.md | 41 -- .../github-repo-importer/pkg/github/github.go | 33 +- .../pkg/github/repositories.go | 27 +- .../github-repo-provisioning/QUICK_START.md | 58 --- .../terraform-github-repository/main.tf | 20 +- .../terraform-github-repository/variables.tf | 36 +- 15 files changed, 930 insertions(+), 1001 deletions(-) rename docs/{feature_github_environment.md => FEATURE_GITHUB_ENVIRONMENT.md} (67%) create mode 100644 docs/GITHUB_ACTIONS_WORKFLOWS.md rename docs/{flow.md => ci_flows.md} (74%) delete mode 100644 docs/workflows.md delete mode 100644 feature/github-repo-importer/LOCAL_DEVELOPMENT.md delete mode 100644 feature/github-repo-importer/QUICK_START.md delete mode 100644 feature/github-repo-provisioning/QUICK_START.md diff --git a/.gitignore b/.gitignore index d8150cf..fd871db 100644 --- a/.gitignore +++ b/.gitignore @@ -42,16 +42,7 @@ feature/github-repo-provisioning/.terraform feature/github-repo-provisioning/.terraform/* feature/github-repo-provisioning/app-list.yaml feature/github-repo-provisioning/import-config.yaml -feature/github-repo-provisioning/backend.tf -feature/github-repo-provisioning/gcss_config/* -!feature/github-repo-provisioning/gcss_config/.gitkeep - -feature/*/tfplan - -# Local development -feature/*/.env feature/github-repo-importer/dumps/* feature/github-repo-importer/configs/* -feature/github-repo-importer/import-config.yaml -feature/github-repo-importer/app-list.yaml +feature/github-repo-provisioning/backend.tf diff --git a/docs/DEVELOPERS_GUIDE.md b/docs/DEVELOPERS_GUIDE.md index 962ae05..f31dee6 100644 --- a/docs/DEVELOPERS_GUIDE.md +++ b/docs/DEVELOPERS_GUIDE.md @@ -125,14 +125,20 @@ Configure GitHub deployment environments with protection rules and reviewers. > - Manually grant access at: `https://github.com/{org}/{repo}/settings/access` > - Verify team access at: `https://github.com/orgs/{org}/teams/{team}/repositories` > - > **Without repository access, Terraform will apply successfully but teams won't be added as reviewers!** - -- **`deployment_ref_policy`**: *(optional, object)* Controls which branches/tags can deploy - - **EITHER** `protected_branches_policy: true` (protected branches only) - - **OR** `selected_branches_or_tags_policy` with `branch_patterns` and/or `tag_patterns` - - **Note**: These options are mutually exclusive - -**📖 For complete guide with examples, see [feature_github_environment.md](feature_github_environment.md)** + > **Without repository access, Terraform will apply successfully but teams won't be added as reviewers and next plan/apply will again be shown in expected changes** + +- **`deployment_policy`**: *(optional, object)* Controls which branches/tags can deploy to this environment + - **`policy_type`**: *(required, enum)* Must be one of: + - `"protected_branches"` - Only protected branches can deploy + - `"selected_branches_and_tags"` - Specific branch/tag patterns can deploy + - **`branch_patterns`**: *(optional, string[])* Branch patterns (e.g., `["main", "release/*"]`) + - Only used when `policy_type` is `"selected_branches_and_tags"` + - Set to `null` or omit when using `"protected_branches"` + - **`tag_patterns`**: *(optional, string[])* Tag patterns (e.g., `["v*"]`) + - Only used when `policy_type` is `"selected_branches_and_tags"` + - Set to `null` or omit when using `"protected_branches"` + +**📖 For complete guide with examples, see [FEATURE_GITHUB_ENVIRONMENT.md](FEATURE_GITHUB_ENVIRONMENT.md)** ## Template Configuration diff --git a/docs/feature_github_environment.md b/docs/FEATURE_GITHUB_ENVIRONMENT.md similarity index 67% rename from docs/feature_github_environment.md rename to docs/FEATURE_GITHUB_ENVIRONMENT.md index 1c479bf..f955e54 100644 --- a/docs/feature_github_environment.md +++ b/docs/FEATURE_GITHUB_ENVIRONMENT.md @@ -23,36 +23,35 @@ environments: reviewers: users: ["octocat"] teams: ["platform-team"] - deployment_ref_policy: - protected_branches_policy: true + deployment_policy: + policy_type: protected_branches # Option 2: Custom branch/tag patterns - environment: staging - deployment_ref_policy: - protected_branches_policy: false # MUST be false for custom patterns - selected_branches_or_tags_policy: - branch_patterns: - - "release/*" - - "main" - tag_patterns: - - "v*" + deployment_policy: + policy_type: selected_branches_and_tags + branch_patterns: + - "release/*" + - "main" + tag_patterns: + - "v*" # Option 3: Any branch can deploy (no restrictions) - environment: development - # No deployment_ref_policy = any branch can deploy + # No deployment_policy = any branch can deploy ``` -## ⚠️ Critical Rule: Deployment Policy Mutual Exclusivity +## ⚠️ Critical Rule: Deployment Policy Types **You MUST choose ONE of these options:** | Option | Configuration | Use Case | |--------|--------------|----------| -| **Protected Branches** | `protected_branches_policy: true` | Production - only protected branches | -| **Custom Patterns** | `protected_branches_policy: false` + patterns | Staging - specific branches/tags | -| **Any Branch** | Omit `deployment_ref_policy` entirely | Development - no restrictions | +| **Protected Branches** | `policy_type: protected_branches` | Production - only protected branches | +| **Custom Patterns** | `policy_type: selected_branches_and_tags` + patterns | Staging - specific branches/tags | +| **Any Branch** | Omit `deployment_policy` entirely | Development - no restrictions | -**NEVER set both `protected_branches_policy: true` AND `selected_branches_or_tags_policy` together.** +**The `policy_type` field determines which patterns are used.** ## Field Reference @@ -64,11 +63,10 @@ environments: | `prevent_self_review` | bool | Prevent self-approval | false | | `reviewers.users` | string[] | GitHub usernames (max 6 total with teams) | [] | | `reviewers.teams` | string[] | Team slugs (max 6 total with users) | [] | -| `deployment_ref_policy.*` | object | Deployment restrictions | - | -| ↳ `protected_branches_policy` | bool | Only protected branches | - | -| ↳ `selected_branches_or_tags_policy.*` | object | Custom patterns | - | -| ↳↳ `branch_patterns` | string[] | Branch patterns (e.g., `release/*`) | [] | -| ↳↳ `tag_patterns` | string[] | Tag patterns (e.g., `v*`) | [] | +| `deployment_policy.*` | object | Deployment restrictions | - | +| ↳ `policy_type` | string | `protected_branches` or `selected_branches_and_tags` | - | +| ↳ `branch_patterns` | string[] | Branch patterns (only for `selected_branches_and_tags`) | [] | +| ↳ `tag_patterns` | string[] | Tag patterns (only for `selected_branches_and_tags`) | [] | ## Pattern Matching @@ -121,16 +119,15 @@ environments: prevent_self_review: true reviewers: teams: ["platform-team"] - deployment_ref_policy: - protected_branches_policy: true + deployment_policy: + policy_type: protected_branches - environment: staging prevent_self_review: true - deployment_ref_policy: - protected_branches_policy: false - selected_branches_or_tags_policy: - branch_patterns: ["release/*", "main"] - tag_patterns: ["v*", "rc-*"] + deployment_policy: + policy_type: selected_branches_and_tags + branch_patterns: ["release/*", "main"] + tag_patterns: ["v*", "rc-*"] - environment: development # No restrictions - any branch can deploy @@ -141,7 +138,7 @@ environments: | Issue | Solution | |-------|----------| | "reviewers: must be 6 or fewer" | Combined users + teams must be ≤ 6 | -| Custom policies not working | Ensure `protected_branches_policy: false` | +| Custom policies not working | Ensure `policy_type: selected_branches_and_tags` | | Deployment policies not created | Check `custom_branch_policies = true` in Terraform | -For more configuration options, see [DEVELOPERS_GUIDE.md](DEVELOPERS_GUIDE.md) \ No newline at end of file +For more configuration options, see [DEVELOPERS_GUIDE.md](DEVELOPERS_GUIDE.md) diff --git a/docs/GITHUB_ACTIONS_WORKFLOWS.md b/docs/GITHUB_ACTIONS_WORKFLOWS.md new file mode 100644 index 0000000..abc1fa2 --- /dev/null +++ b/docs/GITHUB_ACTIONS_WORKFLOWS.md @@ -0,0 +1,353 @@ +# GitHub Actions Workflows Guide + +This guide documents the reusable GitHub Actions workflows provided by github-terraformer and how they're used by gcss-config-repo. + +## Architecture Overview + +```mermaid +graph TB + subgraph "gcss-config-repo (Client Triggers)" + A1[Manual: Import Single Repo] + A2[Manual: Bulk Import] + A3[Manual: Create Fork] + A4[PR Event: Bootstrap] + A5[Push to main: Apply] + A6[Manual: Promote Configs] + A7[Schedule/Manual: Drift Check] + end + + subgraph "github-terraformer (Reusable Workflows)" + B1[import.yaml] + B2[bulk-import.yaml] + B3[create-fork.yaml] + B4[tf-plan.yaml] + B5[tf-apply.yaml] + B6[promote-imported-configs.yaml] + B7[drift-check.yaml] + end + + subgraph "Internal Actions (Hidden from Client)" + C1[gcss-config-setup
Clones config repo] + C2[compare
Finds changes] + C3[pr-bot
Creates PRs] + C4[graformer
Runs Terraform] + end + + A1 -->|triggers| B1 + A2 -->|triggers| B2 + A3 -->|triggers| B3 + A4 -->|triggers| B4 + A5 -->|triggers| B5 + A6 -->|triggers| B6 + A7 -->|triggers| B7 + + B1 --> C1 + B1 --> C2 + B1 --> C3 + B2 --> C1 + B2 --> C2 + B2 --> C3 + B3 --> C1 + B3 --> C3 + B4 --> C1 + B4 --> C4 + B5 --> C1 + B5 --> C4 + B6 --> C1 + B6 --> C4 + B7 --> C1 + B7 --> C4 + + style C1 fill:#f9f,stroke:#333,stroke-width:2px,stroke-dasharray: 5 5 + style C2 fill:#f9f,stroke:#333,stroke-width:2px,stroke-dasharray: 5 5 + style C3 fill:#f9f,stroke:#333,stroke-width:2px,stroke-dasharray: 5 5 + style C4 fill:#f9f,stroke:#333,stroke-width:2px,stroke-dasharray: 5 5 +``` + +### Two-Repository System + +- **gcss-config-repo**: Triggers workflows (what users interact with) +- **github-terraformer**: Provides reusable workflows and internal actions (implementation details) + +## Reusable Workflows (github-terraformer) + +### 1. 📥 `import.yaml` - Single Repository Import + +**Purpose**: Import a single GitHub repository configuration + +**Inputs**: + +- `repo_name` (string): Repository to import (without org prefix) +- `gcss_ref` (string): Branch of github-terraformer to use + +**Process**: + +1. Generate GitHub App token +2. Clone gcss-config-repo via gcss-config-setup action +3. Run importer: `just import-repo {org}/{repo_name}` +4. Compare action identifies new/changed files +5. Create PR with changes in `importer_tmp_dir/` + +**Called by**: gcss-config-repo's import workflow + +--- + +### 2. 📦 `bulk-import.yaml` - Bulk Repository Import + +**Purpose**: Import multiple repositories based on import-config.yaml + +**Inputs**: + +- `gcss_ref` (string): Branch of github-terraformer to use + +**Process**: +1. Uses import-config.yaml to determine which repos to import +2. Runs `just import-repos` to import all configured repositories +3. Creates single PR with all changes + +**Called by**: gcss-config-repo's bulk-import workflow + +--- + +### 3. 🍴 `create-fork.yaml` - Fork and Import + +**Purpose**: Fork an external repository and import its configuration + +**Inputs**: + +- `upstream_repo` (string): Repository to fork (format: `owner/repo`) +- `new_repo_name` (string, optional): Name for the fork +- `gcss_ref` (string): Branch of github-terraformer to use + +**Process**: + +1. Fork the upstream repository +2. Import the forked repository configuration +3. Create PR with the new configuration + +**Called by**: gcss-config-repo's create-fork workflow + +--- + +### 4. 📋 `tf-plan.yaml` - Terraform Plan + +**Purpose**: Run Terraform plan on pull requests + +**Inputs**: + +- `commit_sha` (string): Git SHA to checkout +- `gcss_ref` (string): Branch of github-terraformer to use +- `tfc_org` (string): Terraform Cloud organization + +**Process**: + +1. Setup configuration from commit SHA +2. Initialize Terraform with HCP backend +3. Run `terraform plan` via graformer action +4. Post plan results to PR as comment + +**Called by**: gcss-config-repo on PR events + +--- + +### 5. ✅ `tf-apply.yaml` - Terraform Apply + +**Purpose**: Apply Terraform changes when PRs are merged to main + +**Inputs**: + +- `commit_sha` (string): Git SHA to apply +- `gcss_ref` (string): Branch of github-terraformer to use +- `tfc_org` (string): Terraform Cloud organization + +**Process**: + +1. Setup configuration from commit SHA +2. Run `terraform apply -auto-approve` +3. Changes are applied to GitHub repositories + +**Called by**: gcss-config-repo on push to main + +--- + +### 6. 🎯 `promote-imported-configs.yaml` - Promote Configurations + +**Purpose**: Move imported configurations from `importer_tmp_dir/` to `repos/` + +**Inputs**: + +- `commit_sha` (string): Commit with configs to promote +- `pr_number` (string): PR number for reference +- `tfc_org` (string): Terraform Cloud organization + +**Process**: + +1. Run Terraform apply to import resources +2. Sanitize YAML files (remove IDs via yq) +3. Move files from `importer_tmp_dir/` to `repos/` +4. Commit and push changes + +**Called by**: Manual trigger after import PR is merged + +--- + +### 7. 🔍 `drift-check.yaml` - Configuration Drift Detection + +**Purpose**: Detect drift between Terraform state and actual GitHub configuration + +**Inputs**: + +- `commit_sha` (string): Commit to check +- `gcss_ref` (string): Branch of github-terraformer to use +- `tfc_org` (string): Terraform Cloud organization + +**Process**: + +1. Run `terraform plan -detailed-exitcode` +2. Report any detected drift +3. Can be scheduled or manually triggered + +**Called by**: gcss-config-repo (scheduled or manual) + +## Custom Actions (github-terraformer) + +### `gcss-config-setup` + +Clones gcss-config-repo and copies configuration files to appropriate locations + +### `compare` + +Compares `importer_tmp_dir/` with `repos/` to identify new or changed files + +### `pr-bot` + +Creates pull requests with generated changes + +### `graformer` + +Handles Terraform operations with HCP Terraform backend + +## Workflow Triggers (gcss-config-repo) + +```yaml +# Example: gcss-config-repo/.github/workflows/import.yaml +name: Import single repository +on: + workflow_dispatch: + inputs: + repo_name: + description: 'Repository to import' + required: true + +jobs: + import-repo: + uses: org/github-terraformer/.github/workflows/import.yaml@main + with: + repo_name: ${{ github.event.inputs.repo_name }} + gcss_ref: main + secrets: + app_private_key: ${{ secrets.APP_PRIVATE_KEY }} +``` + +## File Flow + +```yaml +1. Import Stage: + importer → configs/{org}/*.yaml → copy → importer_tmp_dir/ + +2. Terraform Import: + importer_tmp_dir/*.yaml → terraform import → state + +3. Promotion: + importer_tmp_dir/*.yaml → sanitize → repos/*.yaml + +4. Management: + repos/*.yaml → terraform apply → GitHub +``` + +## Directory Structure Impact + +```yaml +gcss-config-repo/ +├── repos/ # Managed configurations (source of truth) +│ └── *.yaml # Repository configurations +├── importer_tmp_dir/ # Temporary import location +│ └── *.yaml # Pending imports +└── .github/workflows/ # Workflows calling github-terraformer +``` + +## Environment Variables & Secrets + +### Required in gcss-config-repo: + +- `APP_PRIVATE_KEY`: GitHub App private key +- `TFC_TOKEN`: Terraform Cloud API token +- `APP_ID`: GitHub App ID (stored as variable) + +### Set by Workflows: + +- `GITHUB_TOKEN`: Generated from App credentials +- `OWNER`: GitHub organization +- `TF_VAR_*`: Terraform variables + +## Common Patterns + +### PR-based Flow + +1. Workflow generates changes → Creates PR +2. PR triggers → Bootstrap → Terraform Plan +3. PR merged → Terraform Apply +4. Optional → Promote imported configs + +### Authentication Flow + +1. GitHub App private key → Generate installation token +2. Token used for → GitHub API calls & git operations +3. Terraform uses → App credentials for resource management + +## Debugging Workflows + +### Check Workflow Runs + +```bash +gh run list --workflow=import.yaml +gh run view +``` + +### View Logs + +```bash +gh run view --log +gh run view --log-failed +``` + +### Re-run Failed Jobs + +```bash +gh run rerun +gh run rerun --failed +``` + +## Best Practices + +1. **Always use reusable workflows** - Don't duplicate workflow logic +2. **Pass secrets explicitly** - Use `secrets:` in workflow calls +3. **Version with refs** - Use `@branch` or `@tag` for stability +4. **Test locally first** - Use LOCAL_DEVELOPMENT_SETUP.md guide +5. **Monitor drift** - Schedule drift-check workflow regularly + +## Troubleshooting + +| Issue | Solution | +|-------|----------| +| "Bad credentials" | Check APP_PRIVATE_KEY secret | +| "Resource not accessible" | Verify GitHub App permissions | +| "cannot find module" | Check gcss_ref points to correct branch | +| "Terraform lock timeout" | Check HCP Terraform for stuck runs | +| Files not promoting | Ensure promote workflow runs after apply | + +## See Also + +- [LOCAL_DEVELOPMENT_SETUP.md](LOCAL_DEVELOPMENT_SETUP.md) - Local development guide +- [DEVELOPERS_GUIDE.md](DEVELOPERS_GUIDE.md) - YAML configuration reference +- `.github/workflows/` - Actual workflow implementations diff --git a/docs/LOCAL_DEVELOPMENT_SETUP.md b/docs/LOCAL_DEVELOPMENT_SETUP.md index dfa2637..b8bd39a 100644 --- a/docs/LOCAL_DEVELOPMENT_SETUP.md +++ b/docs/LOCAL_DEVELOPMENT_SETUP.md @@ -1,62 +1,86 @@ # Local Development Setup -Quick setup guide for both GitHub Repo Importer and Terraform Provisioning. +This guide explains how to set up local development for the GitHub Terraformer system, which consists of two main repositories working together. + +## Repository Architecture + +### github-terraformer (This Repository) +Contains the tools and reusable workflows: +- **feature/github-repo-importer**: Go CLI tool for importing GitHub repositories to YAML +- **feature/github-repo-provisioning**: Terraform module for managing GitHub repositories +- **.github/workflows**: Reusable GitHub Actions workflows +- **.github/actions**: Custom GitHub Actions (gcss-config-setup, compare, pr-bot, graformer) + +### gcss-config-repo (Configuration Repository) +Contains the actual configuration: +- **repos/**: Repository YAML configuration files (source of truth) +- **importer_tmp_dir/**: Temporary location for imported repositories (before Terraform import) +- **config/**: Configuration files (import-config.yaml, app-list.yaml) +- **.github/workflows**: Workflows that call github-terraformer's reusable workflows ## Prerequisites -- **Go 1.21+** (for importer) +- **Go 1.21+** (for importer development) - **Terraform 1.0+** (for provisioning) -- **GitHub Token** or **GitHub App credentials** +- **Just** command runner (`brew install just` or from https://github.com/casey/just) +- **GitHub App** or **Personal Access Token** with appropriate permissions +- **yq** for YAML processing (`brew install yq`) ## Quick Setup -### 1. Clone Repositories +### 1. Clone Both Repositories ```bash +# Clone to adjacent directories git clone https://github.com/your-org/github-terraformer.git git clone https://github.com/your-org/gcss-config-repo.git + +# Your directory structure should be: +# workspace/ +# ├── github-terraformer/ +# └── gcss-config-repo/ ``` -### 2. Setup Symlinks +### 2. Setup Configuration Files -```bash -#!/bin/bash -# Run from workspace root +The workflows use file copying (not symlinks) to connect the repositories: -# For Terraform Provisioning +```bash +# Copy configuration files for local development cd github-terraformer/feature/github-repo-provisioning -ln -sfn ../../../gcss-config-repo gcss_config -ln -sf gcss_config/config/app-list.yaml app-list.yaml -# For GitHub Importer -cd ../github-repo-importer -ln -sf ../../../gcss-config-repo/config/import-config.yaml import-config.yaml -ln -sf ../../../gcss-config-repo/config/app-list.yaml app-list.yaml -ln -sf ../../../gcss-config-repo/importer_tmp_dir importer_tmp_dir +# Create gcss_config directory (mimics what gcss-config-setup action does) +mkdir -p gcss_config +cp -r ../../../gcss-config-repo/* gcss_config/ -echo "✅ Symlinks created" +# Copy required config files +cp gcss_config/config/app-list.yaml . +cp gcss_config/config/app-list.yaml ../github-repo-importer/ +cp gcss_config/config/import-config.yaml ../github-repo-importer/ ``` -### 3. Configure Environment +### 3. Configure Environment Variables -#### For Importer (Go Tool) +#### For GitHub Repo Importer (Go Tool) ```bash cd github-terraformer/feature/github-repo-importer +# Create .env file for importer cat > .env << 'EOF' -export GITHUB_TOKEN="ghp_your_token_here" +export GITHUB_TOKEN="ghp_your_personal_access_token" # For local testing export OWNER="your-org" EOF source .env ``` -#### For Terraform +#### For Terraform Provisioning ```bash cd github-terraformer/feature/github-repo-provisioning +# For local development with GitHub App cat > .env << 'EOF' # GitHub App credentials export TF_VAR_app_id="123456" @@ -71,192 +95,329 @@ EOF source .env ``` -## Common Operations +### 4. Terraform Backend Configuration + +For local development, you have two options: + +#### Option A: Local State (Development) +```bash +cd github-terraformer/feature/github-repo-provisioning + +# Create local backend configuration +cat > backend.tf << 'EOF' +terraform { + backend "local" { + path = "terraform.tfstate" + } +} +EOF + +terraform init -reconfigure +``` + +#### Option B: Terraform Cloud (Matches CI/CD) +```bash +cd github-terraformer/feature/github-repo-provisioning + +# Use the existing backend-hcp.tf (rename if needed) +cp backend-hcp.tf backend.tf + +# Set Terraform Cloud credentials +export TF_TOKEN_app_terraform_io="your-tfc-token" + +terraform init +``` + +## Development Workflows -### Import Repositories +### Workflow 1: Importing Existing Repositories +**Step 1: Import with CLI Tool** ```bash cd github-terraformer/feature/github-repo-importer source .env -# Single repository +# Import single repository just import-repo your-org/repo-name -# All repositories +# Or bulk import based on import-config.yaml just import-repos -# With custom output -go run main.go import your-org/repo -o ../custom-dir +# Files are created in: configs/your-org/*.yaml ``` -### Provision with Terraform +**Step 2: Copy to gcss_config** +```bash +# Copy imported files to provisioning directory +cp configs/$OWNER/*.yaml ../github-repo-provisioning/gcss_config/importer_tmp_dir/ +``` +**Step 3: Run Terraform Import** ```bash -cd github-terraformer/feature/github-repo-provisioning +cd ../github-repo-provisioning source .env -# Initialize -terraform init - -# Plan changes +# Review what will be imported terraform plan -# Apply changes +# Import the repositories terraform apply +``` -# Target specific repo -terraform apply -target='module.repository["repo-name"]' +**Step 4: Promote to Permanent Location** +```bash +# After successful Terraform import, move files from importer_tmp_dir to repos +cd gcss_config +mv importer_tmp_dir/*.yaml repos/ + +# Commit these changes +git add -A +git commit -m "Promote imported repositories" ``` -## Workflow +### Workflow 2: Creating New Repositories -### Creating New Repositories +**Step 1: Create YAML Configuration** +```bash +cd gcss-config-repo/repos -1. Create YAML in `gcss-config-repo/repos/new-repo.yaml`: -```yaml +# Create new repository configuration +cat > new-repo.yaml << 'EOF' description: "My new repository" visibility: private default_branch: main has_issues: true +has_projects: false +has_wiki: false +has_downloads: true vulnerability_alerts_enabled: true -# Optional environments +# Optional: Add environments environments: - environment: production wait_timer: 300 - deployment_ref_policy: - protected_branches_policy: true + deployment_policy: + policy_type: protected_branches + + - environment: staging + deployment_policy: + policy_type: selected_branches_and_tags + branch_patterns: ["release/*", "main"] + tag_patterns: ["v*"] +EOF ``` -2. Apply with Terraform: +**Step 2: Apply with Terraform** ```bash cd github-terraformer/feature/github-repo-provisioning -source .env && terraform apply +source .env + +# Update local copy +cp ../../../gcss-config-repo/repos/*.yaml gcss_config/repos/ + +# Plan and apply +terraform plan +terraform apply ``` -### Importing Existing Repositories +## Testing -1. Import with CLI tool: +### Test GitHub Repo Importer ```bash cd github-terraformer/feature/github-repo-importer -source .env && just import-repo owner/repo -``` -2. File appears in `gcss-config-repo/importer_tmp_dir/` +# Run unit tests +just test -3. Apply with Terraform to import: -```bash -cd ../github-repo-provisioning -source .env && terraform apply +# Test single import +GITHUB_TOKEN=$GITHUB_TOKEN go run main.go import your-org/test-repo + +# Test bulk import +GITHUB_TOKEN=$GITHUB_TOKEN go run main.go bulk-import -c import-config.yaml ``` -4. Move to permanent location: +### Test Terraform Configuration ```bash -mv gcss_config/importer_tmp_dir/repo.yaml gcss_config/repos/ +cd github-terraformer/feature/github-repo-provisioning + +# Validate configuration +terraform validate + +# Format check +terraform fmt -check + +# Plan without applying +terraform plan ``` -## Configuration Files +## CI/CD Workflow (GitHub Actions) -### import-config.yaml +The actual CI/CD uses a different flow with reusable workflows: -Controls import behavior: +### How It Works in CI/CD: +1. **gcss-config-repo** triggers workflows for: + - PR creation → Bootstrap → Terraform Plan + - Merge to main → Terraform Apply + - Manual import → Import workflow + +2. **Reusable Workflows** (in github-terraformer): + - Called by gcss-config-repo workflows + - Use GitHub App authentication + - Use Terraform Cloud backend + - Automatically handle file movements + +3. **Key Actions**: + - **gcss-config-setup**: Clones config repo and copies files + - **compare**: Compares importer_tmp_dir with repos to identify changes + - **pr-bot**: Creates pull requests with changes + - **graformer**: Handles Terraform operations with HCP Terraform + +### CI/CD Configuration Files + +**import-config.yaml** (controls import behavior): ```yaml -# Ignore specific repos +# Option 1: Ignore specific repositories ignored_repos: - - "your-org/gcss-config-repo" - - "your-org/github-terraformer" + - "your-org/gcss-config-repo" # Don't import the config repo itself + - "your-org/github-terraformer" # Don't import the tool repo + - "your-org/private-archived" # Skip archived repos -# Or select specific repos only +# Option 2: Only import specific repositories selected_repos: - - "your-org/repo1" - - "your-org/repo2" + - "your-org/important-repo" + - "your-org/another-repo" -# Enable environment import -feature_github_environment: true +# Features +feature_github_environment: true # Enable environment import ``` -### app-list.yaml - -GitHub App IDs for ruleset bypass actors: - +**app-list.yaml** (GitHub App IDs for bypass actors): ```yaml apps: - name: dependabot - id: 12345 + id: 29110 # GitHub's Dependabot App ID - name: renovate - id: 67890 -``` - -## Testing - -```bash -# Importer tests -cd github-terraformer/feature/github-repo-importer -just test - -# Terraform validation -cd ../github-repo-provisioning -terraform validate + id: 37453 # Renovate Bot App ID + - name: your-custom-app + id: 123456 ``` ## Troubleshooting -### Token/Authentication Issues +### Authentication Issues ```bash # Test GitHub token curl -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/user -# Check rate limit +# Test GitHub App authentication (if using App) +gh api user --header "Authorization: Bearer $(gh auth token)" + +# Check rate limits curl -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/rate_limit ``` -### Symlink Issues +### File Path Issues ```bash -# Verify targets exist -ls ../../../gcss-config-repo - -# Recreate if broken -rm -f symlink-name -ln -sf ../../../correct-path symlink-name +# Verify gcss_config structure +cd github-terraformer/feature/github-repo-provisioning +tree gcss_config -L 2 + +# Expected structure: +# gcss_config/ +# ├── config/ +# │ ├── app-list.yaml +# │ └── import-config.yaml +# ├── repos/ +# │ └── *.yaml (repository configs) +# └── importer_tmp_dir/ +# └── *.yaml (pending imports) ``` ### Terraform Import Failures -If "Cannot import non-existent remote object": -- Repository doesn't exist in GitHub -- Move YAML from `importer_tmp_dir/` to `repos/` to create it +Common issues and solutions: -### Environment Variables Not Set +1. **"Cannot import non-existent remote object"** + - Repository doesn't exist in GitHub yet + - Solution: Move YAML from `importer_tmp_dir/` to `repos/` to create it -Always run `source .env` before commands. +2. **"Error: Resource already exists"** + - Repository already managed by Terraform + - Solution: Check `terraform state list` and remove duplicate -## Local vs CI/CD +3. **"Unauthorized"** + - GitHub App permissions insufficient + - Solution: Check App installation permissions -### Local Development +### Importer Issues -Use local Terraform state: ```bash -cp backend.tf.local backend.tf -terraform init -reconfigure +# Debug import with verbose output +cd github-terraformer/feature/github-repo-importer +go run main.go import your-org/repo -v + +# Check imported file +cat configs/your-org/repo.yaml + +# Validate YAML syntax +yq eval . configs/your-org/repo.yaml ``` -### CI/CD (GitHub Actions) +## Local vs CI/CD Differences -Uses HCP Terraform backend - workflows handle this automatically. +| Aspect | Local Development | CI/CD (GitHub Actions) | +|--------|------------------|------------------------| +| **Authentication** | Personal token or App | GitHub App only | +| **File Management** | Manual copying | Automated via actions | +| **Terraform Backend** | Local or HCP | HCP Terraform only | +| **Config Repo** | Local directory | Checked out via action | +| **Promotion** | Manual move | Automated workflow | +| **PR Creation** | Manual | Automated via pr-bot | -## Security Notes +## Security Best Practices -Never commit: +### Never Commit: - `.env` files - `*.pem` keys - `terraform.tfvars` -- `*.tfstate` +- `*.tfstate` files +- Personal access tokens -Store secrets in `~/.secrets/` with proper permissions: +### Secure Storage: ```bash +# Create secure directory for secrets mkdir -p ~/.secrets && chmod 700 ~/.secrets -mv *.pem ~/.secrets/ && chmod 600 ~/.secrets/*.pem -``` \ No newline at end of file + +# Store keys securely +mv github-app.pem ~/.secrets/ && chmod 600 ~/.secrets/github-app.pem + +# Use environment variables +export GITHUB_TOKEN=$(cat ~/.secrets/github-token) +``` + +### Use .gitignore: +```gitignore +# Add to .gitignore +.env +*.pem +terraform.tfvars +*.tfstate +*.tfstate.backup +.terraform/ +``` + +## Next Steps + +1. **Set up GitHub App**: Create an App with repository management permissions +2. **Configure Terraform Cloud**: Set up workspace for state management +3. **Test Import**: Try importing a test repository +4. **Create Repository**: Test creating a new repository via YAML +5. **Set up CI/CD**: Configure workflows in your gcss-config-repo + +For more details on specific configurations, see: +- [DEVELOPERS_GUIDE.md](DEVELOPERS_GUIDE.md) - Complete YAML configuration reference +- [FEATURE_GITHUB_ENVIRONMENT.md](FEATURE_GITHUB_ENVIRONMENT.md) - Environment configuration guide +- Repository examples in `gcss-config-repo/repos/` diff --git a/docs/flow.md b/docs/ci_flows.md similarity index 74% rename from docs/flow.md rename to docs/ci_flows.md index fba129e..61444a7 100644 --- a/docs/flow.md +++ b/docs/ci_flows.md @@ -1,3 +1,4 @@ +# GitHub API -> YAML -> Terraform flow ```mermaid graph TD @@ -42,9 +43,9 @@ graph TD style MODULE fill:#fce4ec ``` -# 🔍 Detailed Process Breakdown +## 🔍 Detailed Process Breakdown -# 1️⃣ GitHub Import (github-repo-importer) +### 1️⃣ GitHub Import (github-repo-importer) Location: `feature/github-repo-importer/` @@ -54,10 +55,11 @@ go run main.go bulk-import -c import-config.yaml - Creates YAML files in: configs/{owner}/*.yaml ``` -# 2️⃣ YAML Storage Locations +### 2️⃣ YAML Storage Locations - Execution Context: Where each command runs from -``` +Execution Context: Where each command runs from + +```yaml | Stage | Full Path | Executed From Directory | |---------------------------|---------------------------------------------------------------------------------------------------|-----------------------------------| | 1. After Import | /home/.../github-terraformer/feature/github-repo-importer/configs/{owner}/*.yaml | feature/github-repo-importer/ | @@ -69,7 +71,7 @@ go run main.go bulk-import -c import-config.yaml Directory Structure: -```sh +```bash github-terraformer/ ├── feature/ │ ├── github-repo-importer/ # 🔧 Import tool runs here @@ -93,20 +95,18 @@ Copies from: `configs/{owner}/*.yaml` Copies to: `../github-repo-provisioning/gcss_config/{repos or importer_tmp_dir}/` -## Important Note: +#### Important Note: __The gcss_config/ directory is actually a checkout of the gcss-config-repo (done by GitHub Actions), not a permanent part of github-terraformer!__ - The gcss_config/ directory is actually a checkout of the gcss-config-repo (done by GitHub Actions), not a permanent part of github-terraformer! - -# 3️⃣ YAML → Terraform Transformation +### 3️⃣ YAML → Terraform Transformation ```hcl # In main.tf - YAML becomes Terraform data locals { -# Read YAML files and decode them - generated_repos = { - for file_path in fileset(..., "importer_tmp_dir/*.yaml") : - basename(file_path) => yamldecode(file(file_path)) # ← YAML → HCL - } + generated_repos = { + # Read YAML files and decode them + for file_path in fileset(path.module, "gcss_config/importer_tmp_dir/*.yaml") : + basename(file_path) => yamldecode(file(file_path)) # YAML → HCL + } } ``` @@ -114,43 +114,43 @@ YAML structure becomes module variables ```hcl module "repository" { - for_each = local.all_repos + for_each = local.all_repos - # YAML fields map to module inputs - name = each.key # From filename - description = try(each.value.description, "") # From YAML content - visibility = try(each.value.visibility, "") # From YAML content - environments = try(each.value.environments, []) # From YAML content + # YAML fields map to module inputs + name = each.key # From filename + description = try(each.value.description, "") # From YAML content + visibility = try(each.value.visibility, "") # From YAML content + environments = try(each.value.environments, []) # From YAML content } ``` -# 4️⃣ Example YAML → Resource Flow +### 4️⃣ Example YAML → Resource Flow ```yaml -YAML File (123123123.yaml): +YAML File (demo1.yaml): description: "My repo" visibility: public environments: -- environment: tesdt12 - wait_timer: 44 +- environment: development + wait_timer: 30 reviewers: users: - - ljubon + - octocat ``` -Becomes Terraform Resources: +#### Becomes Terraform Resources If in `importer_tmp_dir/` → Import block generated -```hcl +```hcl import { - to = module.repository["123123123"].github_repository.repository - id = "123123123" + to = module.repository["demo1"].github_repository.repository + id = "demo1" } import { - to = module.repository["123123123"].github_repository_environment.environment["tesdt12"] - id = "123123123:tesdt12" + to = module.repository["demo1"].github_repository_environment.environment["development"] + id = "demo1:development" } ``` @@ -158,24 +158,24 @@ Module creates actual resources ```hcl module "repository" { - # YAML filename → module key - for_each = { "123123123" = } - - # YAML fields → module variables - name = "123123123" - description = "My repo" - visibility = "public" - environments = [{ - environment = "tesdt12" - wait_timer = 44 - reviewers = { users = ["ljubon"] } - }] + # YAML filename → module key + for_each = { "demo1" = } + + # YAML fields → module variables + name = "demo1" + description = "My repo" + visibility = "public" + environments = [{ + environment = "development" + wait_timer = 44 + reviewers = { users = ["octocat"] } + }] } ``` -# 5️⃣ Decision Tree +### 5️⃣ Decision Tree -```bash +```yaml Is repo already in repos/? ├─ YES → Update existing file in repos/ │ └─ Terraform updates resource diff --git a/docs/workflows.md b/docs/workflows.md deleted file mode 100644 index 4f3beae..0000000 --- a/docs/workflows.md +++ /dev/null @@ -1,44 +0,0 @@ -> [!IMPORTANT] -> This is a work in progress document and may change in the future - -## 🚀 GitHub Actions Workflows - -### 🔄 `Import` Workflow - -- **Trigger**: Manually via GitHub Actions -- **Inputs**: - - `branch`: Target environment (`dev` or `prod`) - - `repo_name`: Name of the GitHub repository to import - - `owner`: Name of the Github organization that owns the repository -- **Behavior**: - 1. Fetches repo metadata via GitHub API: - - General repository settings - - Branch protection rules - - Default branch - - Teams and collaborators - - Repository rulesets - 2. Generates a YAML configuration - 3. Places the YAML into: - ``` - feature/github-repo-provisioning/importer_tmp_dir/{organization}/{repository}.yaml - ``` - 4. Creates an automated pull request targeting the selected branch - 5. Upon PR merge, Terraform Cloud plans and applies the configuration - 6. Configuration file is then sanitized (ids removed) and moved to the appropriate directory `feature/github-repo-provisioning/repo_configs/{branch}/{organization}` - -## 📥 Importing Existing Repositories - -To import an **existing GitHub repository** into Terraform: - -1. Navigate to **Actions** > **Import** workflow in GitHub -2. Select: - - `prod` (or `dev`) as the target branch - - The name of the repository to import - - The owner of the repository (e.g., `G-Research` or `armadaproject`) -3. The workflow will: - - Generate a YAML config - - Place it under `feature/github-repo-provisioning/importer_tmp_dir/{organization}/` - - The name of the YAML file will be the same as the repository name - - Create a PR against the `prod` branch -4. Review, approve, and merge the PR -5. Terraform Cloud will detect and apply the changes \ No newline at end of file diff --git a/feature/github-repo-importer/ADDING_FEATURES.md b/feature/github-repo-importer/ADDING_FEATURES.md index 1f7f815..c48f565 100644 --- a/feature/github-repo-importer/ADDING_FEATURES.md +++ b/feature/github-repo-importer/ADDING_FEATURES.md @@ -1,244 +1,210 @@ # Adding New Features to the Importer -This guide shows how to add new feature-gated functionality to the importer. The architecture is designed to be extremely simple and scalable. +Quick guide for adding feature-gated functionality to the importer. -## Architecture Overview +## Architecture -The importer uses a **config-driven feature flag system**: -- Features are controlled by `config/import-config.yaml` -- Same compiled binary works for all features -- Adding features requires **zero changes to CLI or command structure** -- Perfect for CI/CD workflows +- Features controlled by `config/import-config.yaml` +- No CLI changes needed - purely config-driven +- Single binary works for all features -## Adding a New Feature: Step-by-Step Example +## Adding a Feature: 5 Steps -Let's add a hypothetical `feature_github_webhooks` feature that imports GitHub repository webhooks. +### Example: Adding `feature_github_webhooks` -### Step 1: Add Feature Constant +#### 1. Add Constant -In `pkg/github/constants.go`, add your feature constant: +`pkg/github/constants.go`: ```go const ( - // Existing features FeatureGithubEnvironment = "feature_github_environment" - - // Your new feature - FeatureGithubWebhooks = "feature_github_webhooks" + FeatureGithubWebhooks = "feature_github_webhooks" // NEW ) ``` -### Step 2: Add Feature Logic in ImportRepo +#### 2. Add Logic -In `pkg/github/github.go`, add your feature-gated code: +`pkg/github/github.go` in `ImportRepo()`: ```go -func ImportRepo(repoName string, cfg *Config) (*Repository, error) { - // ... existing code ... - - // ========================================================================= - // FEATURE: GitHub Webhooks - // ========================================================================= - var allWebhooks []*github.Hook - if cfg != nil && cfg.IsFeatureEnabled(FeatureGithubWebhooks) { - webhooks, _, err := v3client.Repositories.ListHooks( - context.Background(), - repoNameSplit[0], - repoNameSplit[1], - nil, - ) - if err != nil { - fmt.Printf("failed to get webhooks: %v\n", err) - } else { - allWebhooks = webhooks - - if err := dumpManager.WriteJSONFile("webhooks.json", webhooks); err != nil { - fmt.Printf("failed to write webhooks.json: %v\n", err) - } - } +// ========================================================================= +// FEATURE: GitHub Webhooks +// ========================================================================= +var allWebhooks []*github.Hook +if cfg != nil && cfg.IsFeatureEnabled(FeatureGithubWebhooks) { + webhooks, _, err := v3client.Repositories.ListHooks( + context.Background(), owner, repo, nil) + if err != nil { + fmt.Printf("failed to get webhooks: %v\n", err) + } else { + allWebhooks = webhooks + dumpManager.WriteJSONFile("webhooks.json", webhooks) } - - // ... rest of code ... - - return &Repository{ - // ... existing fields ... - Webhooks: resolveWebhooks(allWebhooks), // Your resolver function - }, nil } ``` -### Step 3: Add Data Structures +#### 3. Add Data Structure -In `pkg/github/repositories.go`, add the webhook field and structure: +`pkg/github/repositories.go`: ```go type Repository struct { // ... existing fields ... - Environments []Environment `yaml:"environments,omitempty"` - Webhooks []Webhook `yaml:"webhooks,omitempty"` // New field + Webhooks []Webhook `yaml:"webhooks,omitempty"` // NEW } type Webhook struct { - URL string `yaml:"url"` - ContentType string `yaml:"content_type,omitempty"` - Events []string `yaml:"events,omitempty"` - Active bool `yaml:"active"` + URL string `yaml:"url"` + ContentType string `yaml:"content_type,omitempty"` + Events []string `yaml:"events,omitempty"` + Active bool `yaml:"active"` } ``` -### Step 4: Document in Config File +#### 4. Add Resolver Function -In `gcss-config-repo/config/import-config.yaml`, document your feature: +`pkg/github/github.go`: -```yaml -# ============================================================================= -# FEATURE FLAGS -# ============================================================================= -# Control which features the importer should use when importing repositories. -# All features are disabled by default and must be explicitly enabled. -# -# Available feature flags: -# -# feature_github_environment: Import GitHub repository environments -# - When enabled, the importer fetches environment configurations from GitHub -# - This includes reviewers, deployment policies, and protection rules -# - Default: false (disabled - must explicitly enable) -# - Usage: Set to true to enable environment import -# -# feature_github_webhooks: Import GitHub repository webhooks (example - not implemented) -# - When enabled, the importer fetches and includes GitHub webhook configurations -# - Webhook secrets are NOT imported (GitHub API limitation) -# - Default: false (disabled - must explicitly enable) -# -# To enable environment import: -feature_github_environment: true - -# To disable (default behavior): -#feature_github_environment: false +```go +func resolveWebhooks(hooks []*github.Hook) []Webhook { + // Convert GitHub API response to YAML structure + var webhooks []Webhook + for _, hook := range hooks { + webhooks = append(webhooks, Webhook{ + URL: hook.GetURL(), + ContentType: hook.Config["content_type"].(string), + Events: hook.Events, + Active: hook.GetActive(), + }) + } + return webhooks +} ``` -### Step 5: That's It! +#### 5. Document in Config -**You're done!** The feature now works with: +`gcss-config-repo/config/import-config.yaml`: -```bash -# Enable in config/import-config.yaml -feature_github_webhooks: true +```yaml +# feature_github_webhooks: Import repository webhooks +# - Webhook secrets are NOT imported (API limitation) +# - Default: false +feature_github_webhooks: true # Enable the feature +``` -# Run importer (reads config automatically) -go run main.go import owner/repo +## Usage -# OR run `Bulk import` and verify plan for all repos with webhooks which will be imported in terraform state and yaml +```bash +# Enable in import-config.yaml, then: +go run main.go import owner/repo +# or +go run main.go bulk-import ``` ## Key Patterns -### Pattern 1: Feature Check +### Always Check Config + ```go if cfg != nil && cfg.IsFeatureEnabled(FeatureYourFeature) { - // Your feature code + // Your code } ``` -### Pattern 2: Safe Defaults -- Always check `cfg != nil` before calling methods -- Features default to `false` if not in config -- Log when features are enabled for debugging +### Error Handling -### Pattern 3: Error Handling -- Don't fail the entire import if one feature fails -- Log errors with `fmt.Printf` for debugging +- Don't fail entire import on feature errors +- Log with `fmt.Printf` - Continue with other features -### Pattern 4: Data Persistence -- Write API responses to JSON files via `dumpManager.WriteJSONFile()` -- This helps with debugging and auditing -- Files are stored in `dumps//` - -## Testing Your Feature +### Data Dumps -### Test 1: Feature Disabled (Default) -```bash -# Without feature flag (should skip your feature) -go run main.go import owner/repo -# Check: dumps/owner-repo/ should NOT have your JSON file +```go +dumpManager.WriteJSONFile("feature_data.json", data) +// Saves to: dumps//feature_data.json ``` -### Test 2: Feature Enabled -```yaml -# In import-config.yaml -feature_your_feature: true +## Real Example: GitHub Environments + +Current implementation shows the new `deployment_policy` structure: + +```go +// Data structure (pkg/github/repositories.go) +type Environment struct { + Environment string `yaml:"environment"` + WaitTimer *int `yaml:"wait_timer,omitempty"` + DeploymentPolicy *DeploymentPolicy `yaml:"deployment_policy,omitempty"` +} + +type DeploymentPolicy struct { + PolicyType string `yaml:"policy_type"` // "protected_branches" or "selected_branches_and_tags" + BranchPatterns []string `yaml:"branch_patterns,omitempty"` + TagPatterns []string `yaml:"tag_patterns,omitempty"` +} + +// Logic (pkg/github/github.go ~line 940) +if env.DeploymentBranchPolicy != nil { + deploymentPolicy := &DeploymentPolicy{} + + if env.DeploymentBranchPolicy.ProtectedBranches != nil && + *env.DeploymentBranchPolicy.ProtectedBranches { + deploymentPolicy.PolicyType = "protected_branches" + } else if env.DeploymentBranchPolicy.CustomBranchPolicies != nil && + *env.DeploymentBranchPolicy.CustomBranchPolicies { + deploymentPolicy.PolicyType = "selected_branches_and_tags" + // Fetch patterns from API + branchPatterns, tagPatterns := fetchDeploymentPolicies(...) + deploymentPolicy.BranchPatterns = branchPatterns + deploymentPolicy.TagPatterns = tagPatterns + } + + if deploymentPolicy.PolicyType != "" { + environment.DeploymentPolicy = deploymentPolicy + } +} ``` +## Testing + ```bash +# 1. Feature disabled (default) go run main.go import owner/repo -# Check: dumps/owner-repo/ should have your JSON file -# Check: YAML output should include your data -``` +# Should NOT create dumps/owner-repo/webhooks.json -### Test 3: Bulk Import -```bash +# 2. Feature enabled +echo "feature_github_webhooks: true" >> import-config.yaml +go run main.go import owner/repo +# Should create dumps/owner-repo/webhooks.json + +# 3. Bulk import go run main.go bulk-import -# Should respect feature flag for all repos +# Respects feature flag for all repos ``` -## Real-World Example: GitHub Environments - -See the `feature_github_environment` implementation as a complete reference: - -1. **Constant**: `pkg/github/constants.go:39` - ```go - FeatureGithubEnvironment = "feature_github_environment" - ``` - -2. **Logic**: `pkg/github/github.go:105-143` - ```go - if cfg != nil && cfg.IsFeatureEnabled(FeatureGithubEnvironment) { - // Fetch environments with pagination - // Write to JSON dump - // Store in allEnvironments - } - ``` - -3. **Data Structure**: `pkg/github/repositories.go:60-77` - ```go - type Environment struct { - Environment string - WaitTimer *int - // ... more fields - } - ``` - -4. **Resolution**: `pkg/github/github.go:667-720` - ```go - func resolveEnvironments(envs []*github.Environment) []Environment { - // Convert GitHub API response to YAML structure - } - ``` - -## Benefits of This Architecture - -✅ **No CLI Changes** - Features are purely config-driven -✅ **Single Binary** - One build works everywhere -✅ **CI/CD Friendly** - Change config without rebuilding -✅ **Backward Compatible** - Features default to disabled -✅ **Scalable** - Add unlimited features without refactoring -✅ **Type Safe** - Constants prevent typos -✅ **Self-Documenting** - Feature names clearly describe functionality - -## Common Pitfalls to Avoid - -❌ **Don't add function parameters** - Use `cfg.IsFeatureEnabled()` instead -❌ **Don't add CLI flags** - Keep it config-driven -❌ **Don't fail on missing data** - Handle errors gracefully -❌ **Don't forget nil checks** - Always check `cfg != nil` -❌ **Don't skip documentation** - Update import-config.yaml - -## Need Help? - -- Review existing features: `FeatureGithubEnvironment` -- Check the code comments in `pkg/github/github.go:105-113` -- Look at `pkg/github/constants.go:36-43` for examples -- Read `config/import-config.yaml` for feature documentation - -The architecture is intentionally simple - if you follow the 5 steps above, your feature will work perfectly with the existing system! +## Do's and Don'ts + +✅ **DO** + +- Use `cfg.IsFeatureEnabled()` +- Handle errors gracefully +- Write dumps for debugging +- Document in import-config.yaml + +❌ **DON'T** + +- Add CLI flags or parameters +- Fail on missing data +- Skip nil checks +- Forget documentation + +## Quick Reference + +| File | Purpose | +|------|---------| +| `pkg/github/constants.go` | Define feature constant | +| `pkg/github/github.go` | Add feature logic in ImportRepo() | +| `pkg/github/repositories.go` | Define data structures | +| `config/import-config.yaml` | Document & enable feature | + +That's it! Follow these 5 steps and your feature will integrate seamlessly. diff --git a/feature/github-repo-importer/LOCAL_DEVELOPMENT.md b/feature/github-repo-importer/LOCAL_DEVELOPMENT.md deleted file mode 100644 index 862ed06..0000000 --- a/feature/github-repo-importer/LOCAL_DEVELOPMENT.md +++ /dev/null @@ -1,426 +0,0 @@ -# Local Development Setup for GitHub Repo Importer - -This guide explains how to set up your local environment for developing and using the GitHub Repo Importer CLI tool. - -## Overview - -The GitHub Repo Importer is a Go-based CLI tool that imports GitHub repository configurations into YAML files for Terraform management. - -## Prerequisites - -1. **Go** (1.19+) - ```bash - # Check version - go version - - # Install if needed: https://golang.org/doc/install - ``` - -2. **Just** (command runner) - ```bash - # Install on Ubuntu/Debian - curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin - - # Or with cargo - cargo install just - - # Or download from: https://github.com/casey/just/releases - ``` - -3. **GitHub Access** - - Personal Access Token or GitHub App credentials - - Required scopes: `repo`, `read:org`, `admin:org` - -## Directory Structure - -``` -github-repo-importer/ -├── cmd/ # Cobra CLI commands -│ ├── root.go # Root command setup -│ ├── import.go # Import single repository -│ ├── bulkImport.go # Import multiple repositories -│ └── compare.go # Compare configurations -├── pkg/ -│ ├── github/ # GitHub API interactions -│ │ ├── github.go # Main GitHub client -│ │ ├── repositories.go # Repository structures -│ │ └── constants.go # Feature flags and constants -│ ├── file/ # File operations -│ │ └── file.go # YAML file handling -│ └── compare/ # Configuration comparison -├── main.go # Entry point -├── Justfile # Task automation -├── go.mod # Go modules -└── import-config.yaml # Import configuration (local) -``` - -## Step 1: Environment Setup - -Create a `.env` file in the importer directory: - -```bash -cd github-terraformer/feature/github-repo-importer - -cat > .env << 'EOF' -# GitHub Authentication -export GITHUB_TOKEN="ghp_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" -export OWNER="your-org-name" - -# Feature Flags (via import-config.yaml) -# These are controlled in import-config.yaml, not env vars -EOF - -``` - -## Step 2: Import Configuration - -The importer uses `import-config.yaml` for configuration. Create or link it: - -### Option A: Link from gcss-config-repo (Recommended) - -```bash -# Create symlink to shared config -ln -sf ../../../gcss-config-repo/config/import-config.yaml import-config.yaml - -# Create symlink for app list (needed for GitHub App bypass actors) -ln -sf ../../../gcss-config-repo/config/app-list.yaml app-list.yaml - -# Optional: Create symlink for output directory (if you want imports to go directly to gcss-config-repo) -ln -sf ../../../gcss-config-repo/importer_tmp_dir importer_tmp_dir -``` -EOF -``` - -## Step 3: Build and Test - -```bash -# Load environment -source .env - -# Run tests -just test - -# Or with go directly -go test ./... - -# Build the binary -go build -o github-importer - -# Or run directly without building -go run main.go --help -``` - -## Step 4: Using the Importer - -### Import Single Repository - -```bash -# Using just -just import-repo your-org/repository-name - -# Using go run -go run main.go import your-org/repository-name - -# With custom output directory -go run main.go import your-org/repository-name -o ../custom-output-dir - -# With custom config -go run main.go import your-org/repository-name -c custom-config.yaml -``` - -### Bulk Import Repositories - -```bash -# Using just -just import-repos - -# Using go run -go run main.go bulk-import - -# With custom config -go run main.go bulk-import -c import-config.yaml - -# With custom output directory -go run main.go bulk-import -o ../output-dir -``` - -### Compare Configurations - -```bash -# Using just -just compare dirA dirB - -# Using go run -go run main.go compare dirA dirB - -# Compare with verbose output -go run main.go compare dirA dirB -v -``` - -## Step 5: Output Structure - -The importer generates YAML files in the output directory: - -```yaml -# output-dir/repository-name.yaml -description: "Repository description" -homepage_url: "https://example.com" -visibility: private -default_branch: main -has_issues: true -has_projects: true -has_wiki: false -has_downloads: true -allow_merge_commit: true -allow_rebase_merge: true -allow_squash_merge: true -delete_branch_on_merge: true -vulnerability_alerts_enabled: true - -# If feature_github_environment: true -environments: - - environment: production - wait_timer: 300 - can_admins_bypass: false - prevent_self_review: true - reviewers: - users: - - username1 - teams: - - team-slug - # IMPORTANT: deployment_ref_policy uses EITHER protected_branches_policy - # OR selected_branches_or_tags_policy, but NOT both - deployment_ref_policy: - # Option 1: Only protected branches can deploy - protected_branches_policy: true - - - environment: staging - deployment_ref_policy: - # Option 2: Custom branch/tag patterns (requires protected_branches_policy: false or omitted) - protected_branches_policy: false - selected_branches_or_tags_policy: - branch_patterns: - - "release/*" - - "hotfix/*" - tag_patterns: - - "v*" - -# Rulesets (if present) -rulesets: - - name: "Main Branch Protection" - target: "branch" - enforcement: "active" - # ... ruleset configuration -``` - -## Development Workflow - -### 1. Making Changes - -```bash -# Create a feature branch -git checkout -b feature/my-improvement - -# Make changes to code -vim pkg/github/github.go - -# Run tests -just test - -# Test specific functionality -go test ./pkg/github -v - -# Test import with your changes -go run main.go import your-org/test-repo -``` - -### 2. Testing Import Features - -```bash -# Test environment import -echo "feature_github_environment: true" >> import-config.yaml -go run main.go import your-org/repo-with-environments - -# Test ruleset import -go run main.go import your-org/repo-with-rulesets - -# Test bulk import with filters -cat > test-config.yaml << EOF -selected_repos: - - "your-org/test-repo1" - - "your-org/test-repo2" -feature_github_environment: true -EOF -go run main.go bulk-import -c test-config.yaml -``` - -### 3. Debugging - -```bash -# Enable debug output -export DEBUG=1 -go run main.go import your-org/repo - -# Use delve debugger -go get -u github.com/go-delve/delve/cmd/dlv -dlv debug main.go -- import your-org/repo - -# Check generated JSON dumps -ls -la jsondumps/ -``` - -## Common Commands Reference - -```bash -# Source environment -source .env - -# Run tests -just test -go test ./... -go test ./pkg/github -v -run TestSpecificFunction - -# Import operations -just import-repo owner/repo -just import-repos -just compare dir1 dir2 - -# Direct go commands -go run main.go import owner/repo -go run main.go bulk-import -go run main.go compare dir1 dir2 - -# Build -go build -o github-importer -./github-importer import owner/repo - -# Format code -go fmt ./... -gofmt -w . - -# Lint -golangci-lint run - -# Dependencies -go mod tidy -go mod download -``` - -## Troubleshooting - -### Issue: "GITHUB_TOKEN not set" - -**Solution**: Set and export the token: -```bash -export GITHUB_TOKEN="ghp_your_token_here" -# Or source .env file -source .env -``` - -### Issue: "401 Unauthorized" - -**Cause**: Invalid or expired GitHub token - -**Solution**: Generate a new token with required permissions: -- Go to GitHub Settings → Developer settings → Personal access tokens -- Required scopes: `repo`, `read:org`, `admin:org` - -### Issue: "403 rate limit exceeded" - -**Solution**: Wait for rate limit reset or use GitHub App authentication: -```bash -# Check rate limit -curl -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/rate_limit -``` - -### Issue: "repository not found" - -**Causes**: -1. Repository doesn't exist -2. No access to private repository -3. Wrong organization name - -**Solution**: Verify repository exists and token has access: -```bash -gh repo view your-org/repo-name -``` - -### Issue: Feature flags not working - -**Solution**: Ensure import-config.yaml is in the correct location: -```bash -# Default locations checked: -# 1. ./import-config.yaml -# 2. Specified with -c flag -ls -la import-config.yaml -cat import-config.yaml | grep feature_ -``` - -## Adding New Features - -1. **Add feature flag** in `pkg/github/constants.go`: - ```go - const FeatureMyNewFeature = "feature_my_new_feature" - ``` - -2. **Read feature flag** in `pkg/github/github.go`: - ```go - if cfg.Features[FeatureMyNewFeature] { - // Your feature code - } - ``` - -3. **Document in import-config.yaml**: - ```yaml - # feature_my_new_feature: Enable my new feature - feature_my_new_feature: false - ``` - -4. **Add tests**: - ```go - func TestMyNewFeature(t *testing.T) { - // Test implementation - } - ``` - -## Project Structure Best Practices - -- **cmd/**: CLI command definitions (Cobra) -- **pkg/github/**: GitHub API interactions -- **pkg/file/**: File I/O operations -- **pkg/compare/**: Comparison logic -- **internal/**: Private packages (if needed) - -## Testing - -```bash -# Unit tests -go test ./pkg/... - -# Integration tests -go test ./... -tags=integration - -# Coverage -go test ./... -cover -go test ./... -coverprofile=coverage.out -go tool cover -html=coverage.out - -# Benchmarks -go test -bench=. ./... -``` - -## CI/CD Integration - -The importer is used in GitHub Actions workflows: - -1. **Import workflow**: Imports single repository -2. **Bulk import workflow**: Imports multiple repositories -3. **Drift check**: Compares current vs desired state - -See `.github/workflows/` in github-terraformer for workflow definitions. - -## Support - -- Check `ADDING_FEATURES.md` for extending functionality -- Review existing imports in `gcss-config-repo/repos/` -- Open issues in the GitHub repository diff --git a/feature/github-repo-importer/QUICK_START.md b/feature/github-repo-importer/QUICK_START.md deleted file mode 100644 index 7efba25..0000000 --- a/feature/github-repo-importer/QUICK_START.md +++ /dev/null @@ -1,41 +0,0 @@ -# Quick Start - GitHub Repo Importer - -## Setup & Usage - -See the main documentation: [LOCAL_DEVELOPMENT_SETUP.md](../../docs/LOCAL_DEVELOPMENT_SETUP.md) - -## Quick Commands - -```bash -# Setup (one-time) -source .env - -# Import single repo -just import-repo owner/repo - -# Import all repos -just import-repos - -# Compare configs -just compare dir1 dir2 - -# Run tests -just test -``` - -## Common Patterns - -```bash -# Import with environments enabled -echo "feature_github_environment: true" >> import-config.yaml -just import-repo owner/repo - -# Custom output directory -go run main.go import owner/repo -o ../custom-dir - -# Debug mode -export DEBUG=1 -go run main.go import owner/repo -``` - -For full setup instructions and troubleshooting, see [LOCAL_DEVELOPMENT_SETUP.md](../../docs/LOCAL_DEVELOPMENT_SETUP.md) \ No newline at end of file diff --git a/feature/github-repo-importer/pkg/github/github.go b/feature/github-repo-importer/pkg/github/github.go index 9cfc8f9..d1c5ba0 100644 --- a/feature/github-repo-importer/pkg/github/github.go +++ b/feature/github-repo-importer/pkg/github/github.go @@ -938,30 +938,33 @@ func resolveEnvironments(envs []*github.Environment, client *github.Client, owne } } - // Handle deployment ref policy + // Handle deployment policy if env.DeploymentBranchPolicy != nil { - deploymentRefPolicy := &DeploymentRefPolicy{} + deploymentPolicy := &DeploymentPolicy{} - // Set protected_branches_policy based on the protected_branches value - if env.DeploymentBranchPolicy.ProtectedBranches != nil { - protectedBranches := *env.DeploymentBranchPolicy.ProtectedBranches - deploymentRefPolicy.ProtectedBranchesPolicy = &protectedBranches - } + // Determine policy type based on protected_branches and custom_branch_policies + if env.DeploymentBranchPolicy.ProtectedBranches != nil && *env.DeploymentBranchPolicy.ProtectedBranches { + // Protected branches only + deploymentPolicy.PolicyType = "protected_branches" + } else if env.DeploymentBranchPolicy.CustomBranchPolicies != nil && *env.DeploymentBranchPolicy.CustomBranchPolicies { + // Custom branch/tag patterns + deploymentPolicy.PolicyType = "selected_branches_and_tags" - // Check if custom_branch_policies is true - if env.DeploymentBranchPolicy.CustomBranchPolicies != nil && *env.DeploymentBranchPolicy.CustomBranchPolicies { // Fetch deployment branch policies from GitHub API branchPatterns, tagPatterns := fetchDeploymentPolicies(client, owner, repo, env.GetName()) - if len(branchPatterns) > 0 || len(tagPatterns) > 0 { - deploymentRefPolicy.SelectedBranchesOrTagsPolicy = &SelectedBranchesOrTagsPolicy{ - BranchPatterns: branchPatterns, - TagPatterns: tagPatterns, - } + if len(branchPatterns) > 0 { + deploymentPolicy.BranchPatterns = branchPatterns + } + if len(tagPatterns) > 0 { + deploymentPolicy.TagPatterns = tagPatterns } } - environment.DeploymentRefPolicy = deploymentRefPolicy + // Only set deployment policy if we have a valid policy type + if deploymentPolicy.PolicyType != "" { + environment.DeploymentPolicy = deploymentPolicy + } } environments = append(environments, environment) diff --git a/feature/github-repo-importer/pkg/github/repositories.go b/feature/github-repo-importer/pkg/github/repositories.go index d6edd7c..641b640 100644 --- a/feature/github-repo-importer/pkg/github/repositories.go +++ b/feature/github-repo-importer/pkg/github/repositories.go @@ -61,12 +61,12 @@ type Pages struct { } type Environment struct { - Environment string `yaml:"environment"` - WaitTimer *int `yaml:"wait_timer,omitempty"` - CanAdminsBypass *bool `yaml:"can_admins_bypass,omitempty"` - PreventSelfReview *bool `yaml:"prevent_self_review,omitempty"` // Extracted from ProtectionRules in API response - Reviewers *EnvironmentReviewers `yaml:"reviewers,omitempty"` - DeploymentRefPolicy *DeploymentRefPolicy `yaml:"deployment_ref_policy,omitempty"` + Environment string `yaml:"environment"` + WaitTimer *int `yaml:"wait_timer,omitempty"` + CanAdminsBypass *bool `yaml:"can_admins_bypass,omitempty"` + PreventSelfReview *bool `yaml:"prevent_self_review,omitempty"` // Extracted from ProtectionRules in API response + Reviewers *EnvironmentReviewers `yaml:"reviewers,omitempty"` + DeploymentPolicy *DeploymentPolicy `yaml:"deployment_policy,omitempty"` } type EnvironmentReviewers struct { @@ -74,14 +74,9 @@ type EnvironmentReviewers struct { Users []string `yaml:"users,omitempty"` // GitHub usernames (e.g., "octocat") } -// DeploymentRefPolicy represents the new structure for deployment reference policies -type DeploymentRefPolicy struct { - ProtectedBranchesPolicy *bool `yaml:"protected_branches_policy,omitempty"` - SelectedBranchesOrTagsPolicy *SelectedBranchesOrTagsPolicy `yaml:"selected_branches_or_tags_policy,omitempty"` -} - -// SelectedBranchesOrTagsPolicy contains branch and tag patterns for custom deployment policies -type SelectedBranchesOrTagsPolicy struct { - BranchPatterns []string `yaml:"branch_patterns,omitempty"` // e.g., ["release/*", "main"] - TagPatterns []string `yaml:"tag_patterns,omitempty"` // e.g., ["v*", "release-*"] +// DeploymentPolicy represents the cleaner structure for deployment policies +type DeploymentPolicy struct { + PolicyType string `yaml:"policy_type"` // "protected_branches" or "selected_branches_and_tags" + BranchPatterns []string `yaml:"branch_patterns,omitempty"` // e.g., ["release/*", "main"] - only for selected_branches_and_tags + TagPatterns []string `yaml:"tag_patterns,omitempty"` // e.g., ["v*"] - only for selected_branches_and_tags } diff --git a/feature/github-repo-provisioning/QUICK_START.md b/feature/github-repo-provisioning/QUICK_START.md deleted file mode 100644 index 944aa24..0000000 --- a/feature/github-repo-provisioning/QUICK_START.md +++ /dev/null @@ -1,58 +0,0 @@ -# Quick Start - Local Terraform Development - -## One-Time Setup - -```bash -# 1. Create symlinks -ln -sfn ../../../gcss-config-repo gcss_config -ln -sf gcss_config/config/app-list.yaml app-list.yaml - -# 2. Create .env file (replace with your values) -cat > .env << 'EOF' -export GITHUB_TOKEN="ghp_XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" -export GITHUB_OWNER="your-org-name" -export TF_VAR_app_private_key="$(cat ~/.secrets/github-app.pem)" -export TF_VAR_app_installation_id="12345678" -export TF_VAR_app_id="123456" -export TF_VAR_owner="your-org-name" -export TF_VAR_environment_directory="gcss_config" -EOF - -# 3. Use local backend -cp backend.tf.local backend.tf - -# 4. Initialize -source .env && terraform init -``` - -## Daily Usage - -```bash -# Always start with -source .env - -# Check changes -terraform plan - -# Apply changes -terraform apply -``` - -## File Locations - -- **New repos**: `gcss_config/repos/*.yaml` -- **Import existing**: `gcss_config/importer_tmp_dir/*.yaml` -- **App config**: `gcss_config/config/app-list.yaml` - -## Common Fixes - -```bash -# Repository doesn't exist (import error) -rm gcss_config/importer_tmp_dir/problematic-repo.yaml - -# Symlink broken -ln -sfn ../../../gcss-config-repo gcss_config - -# Variables not set -source .env -``` diff --git a/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf b/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf index e19acf3..1a092c1 100644 --- a/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf +++ b/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf @@ -652,14 +652,14 @@ resource "github_repository_environment" "environment" { dynamic "deployment_branch_policy" { for_each = ( - try(each.value.deployment_ref_policy.protected_branches_policy, null) != null ? ( - # If protected_branches_policy is true, use protected branches - try(each.value.deployment_ref_policy.protected_branches_policy, false) == true ? [{ + try(each.value.deployment_policy.policy_type, null) != null ? ( + # If policy_type is "protected_branches", use protected branches only + try(each.value.deployment_policy.policy_type, "") == "protected_branches" ? [{ protected_branches = true custom_branch_policies = false }] : - # If protected_branches_policy is false and we have custom policies, enable custom policies - try(each.value.deployment_ref_policy.selected_branches_or_tags_policy, null) != null ? [{ + # If policy_type is "selected_branches_and_tags", use custom policies + try(each.value.deployment_policy.policy_type, "") == "selected_branches_and_tags" ? [{ protected_branches = false custom_branch_policies = true }] : [] @@ -679,7 +679,10 @@ resource "github_repository_environment_deployment_policy" "branch_policies" { for_each = { for item in flatten([ for env_name, env in local.environments_map : [ - for branch_pattern in try(env.deployment_ref_policy.selected_branches_or_tags_policy.branch_patterns, []) : { + for branch_pattern in ( + try(env.deployment_policy.policy_type, "") == "selected_branches_and_tags" ? + try(env.deployment_policy.branch_patterns, []) : [] + ) : { key = "${env_name}:branch:${branch_pattern}" env = env_name pattern = branch_pattern @@ -701,7 +704,10 @@ resource "github_repository_environment_deployment_policy" "tag_policies" { for_each = { for item in flatten([ for env_name, env in local.environments_map : [ - for tag_pattern in try(env.deployment_ref_policy.selected_branches_or_tags_policy.tag_patterns, []) : { + for tag_pattern in ( + try(env.deployment_policy.policy_type, "") == "selected_branches_and_tags" ? + try(env.deployment_policy.tag_patterns, []) : [] + ) : { key = "${env_name}:tag:${tag_pattern}" env = env_name pattern = tag_pattern diff --git a/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf b/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf index 4a8cba1..8d9a9f6 100644 --- a/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf +++ b/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf @@ -610,16 +610,18 @@ variable "environments" { # teams = optional(list(string)) # users = optional(list(string)) # })) - # deployment_branch_policy = optional(object({ - # protected_branches = bool # Set to true to restrict to protected branches only - # # Set to false or omit to allow any branch to deploy + # deployment_policy = optional(object({ + # policy_type = string # "protected_branches" or "selected_branches_and_tags" + # branch_patterns = optional(list(string)) # Only for selected_branches_and_tags + # tag_patterns = optional(list(string)) # Only for selected_branches_and_tags # })) # })) default = [] - # Example: + # Examples: # environments = [ + # # Example 1: Protected branches only # { # environment = "production" # wait_timer = 300 # seconds (5 minutes) @@ -629,13 +631,31 @@ variable "environments" { # teams = ["platform-team"] # users = ["octocat", "hubot"] # } - # deployment_branch_policy = { - # protected_branches = true + # deployment_policy = { + # policy_type = "protected_branches" + # } + # }, + # + # # Example 2: Selected branches and tags + # { + # environment = "staging" + # wait_timer = 60 + # can_admins_bypass = true + # prevent_self_review = false + # reviewers = { + # users = ["developer1"] + # } + # deployment_policy = { + # policy_type = "selected_branches_and_tags" + # branch_patterns = ["main", "release/*", "hotfix/*"] + # tag_patterns = ["v*", "release-*"] # } # }, + # + # # Example 3: Any branch can deploy (no restrictions) # { - # environment = "staging" - # # No deployment_branch_policy = any branch can deploy + # environment = "development" + # # No deployment_policy = any branch can deploy # } # ] } From 7b24c3e4a3db4b68b53b03813f4bd63b4bc552d0 Mon Sep 17 00:00:00 2001 From: Ljubo Nikolic Date: Tue, 18 Nov 2025 21:19:42 +0100 Subject: [PATCH 03/15] revert ljubon-org changes --- .github/workflows/bulk-import.yaml | 6 +++--- .github/workflows/drift-check.yaml | 4 ++-- .github/workflows/import.yaml | 6 +++--- .github/workflows/promote-imported-configs.yaml | 4 ++-- .github/workflows/tf-apply.yaml | 4 ++-- .github/workflows/tf-plan.yaml | 4 ++-- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/bulk-import.yaml b/.github/workflows/bulk-import.yaml index e31749a..a85ed23 100644 --- a/.github/workflows/bulk-import.yaml +++ b/.github/workflows/bulk-import.yaml @@ -32,7 +32,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: ljubon-org/github-terraformer + repository: G-Research/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -50,7 +50,7 @@ jobs: - name: Setup vars id: setup-vars run: | - date_suffix=$(date +%Y%m%d%H%M) + date_suffix=$(date +%Y%m%d) echo "automation_branch_name=import/${{ github.ref_name }}/${{ github.repository_owner }}/bulk-import/$date_suffix" >> $GITHUB_OUTPUT - name: Bulk import repos @@ -73,4 +73,4 @@ jobs: branch-name: ${{ steps.setup-vars.outputs.automation_branch_name }} pr-title: "Bulk import repos" commit-message: "Auto-generated changes. Bulk import repos" - github-token: ${{ steps.generate-token.outputs.token }} + github-token: ${{ steps.generate-token.outputs.token }} \ No newline at end of file diff --git a/.github/workflows/drift-check.yaml b/.github/workflows/drift-check.yaml index 9e47ea5..43ced96 100644 --- a/.github/workflows/drift-check.yaml +++ b/.github/workflows/drift-check.yaml @@ -26,7 +26,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: ljubon-org/github-terraformer + repository: G-Research/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -52,4 +52,4 @@ jobs: exit 1 else echo "No drift detected." - fi + fi \ No newline at end of file diff --git a/.github/workflows/import.yaml b/.github/workflows/import.yaml index 6a6ea5f..d7f6b61 100644 --- a/.github/workflows/import.yaml +++ b/.github/workflows/import.yaml @@ -37,7 +37,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: ljubon-org/github-terraformer + repository: G-Research/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -55,7 +55,7 @@ jobs: - name: Setup vars id: setup-vars run: | - date_suffix=$(date +%Y%m%d%H%M) + date_suffix=$(date +%Y%m%d) echo "full_repo_name=${{ github.repository_owner }}/${{ inputs.repo_name }}" >> $GITHUB_OUTPUT echo "automation_branch_name=import/${{ github.repository_owner }}/${{ inputs.repo_name }}/$date_suffix" >> $GITHUB_OUTPUT @@ -78,4 +78,4 @@ jobs: branch-name: ${{ steps.setup-vars.outputs.automation_branch_name }} pr-title: "Import ${{ steps.setup-vars.outputs.full_repo_name }}" commit-message: "Auto-generated changes. Import ${{ steps.setup-vars.outputs.full_repo_name }} repository" - github-token: ${{ steps.generate-token.outputs.token }} + github-token: ${{ steps.generate-token.outputs.token }} \ No newline at end of file diff --git a/.github/workflows/promote-imported-configs.yaml b/.github/workflows/promote-imported-configs.yaml index 3484d9c..675e323 100644 --- a/.github/workflows/promote-imported-configs.yaml +++ b/.github/workflows/promote-imported-configs.yaml @@ -44,7 +44,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: ljubon-org/github-terraformer + repository: G-Research/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -108,4 +108,4 @@ jobs: date_suffix=$(date +%Y-%m-%d) git commit -m "Promote configs from PR #${{ inputs.pr_number }} on $date_suffix" git push origin "${{ github.ref_name }}" - } + } \ No newline at end of file diff --git a/.github/workflows/tf-apply.yaml b/.github/workflows/tf-apply.yaml index 042f99c..449bce2 100644 --- a/.github/workflows/tf-apply.yaml +++ b/.github/workflows/tf-apply.yaml @@ -33,7 +33,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: ljubon-org/github-terraformer + repository: G-Research/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -58,4 +58,4 @@ jobs: env: TF_CLOUD_ORGANIZATION: ${{ inputs.tfc_org }} TF_WORKSPACE: ${{ vars.WORKSPACE }} - run: terraform apply -no-color -input=false -auto-approve + run: terraform apply -no-color -input=false -auto-approve \ No newline at end of file diff --git a/.github/workflows/tf-plan.yaml b/.github/workflows/tf-plan.yaml index f294cc3..83fb82a 100644 --- a/.github/workflows/tf-plan.yaml +++ b/.github/workflows/tf-plan.yaml @@ -65,7 +65,7 @@ jobs: - name: Checkout GCSS uses: actions/checkout@v4 with: - repository: ljubon-org/github-terraformer + repository: G-Research/github-terraformer ref: ${{ inputs.gcss_ref }} persist-credentials: false @@ -234,4 +234,4 @@ jobs: summary: summary, text: text } - }); + }); \ No newline at end of file From ffd521b222be2a73c9256e81426fcb3a482de3ce Mon Sep 17 00:00:00 2001 From: ljubon Date: Tue, 18 Nov 2025 21:40:44 +0100 Subject: [PATCH 04/15] update mermaid --- docs/ci_flows.md | 89 ++++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/docs/ci_flows.md b/docs/ci_flows.md index 61444a7..1535fae 100644 --- a/docs/ci_flows.md +++ b/docs/ci_flows.md @@ -1,48 +1,48 @@ # GitHub API -> YAML -> Terraform flow ```mermaid +--- +config: + theme: forest +--- graph TD - %% Step 1: GitHub Import GH[("🌐 GitHub API
(Existing Repos)")] - GH -->|"go run main.go bulk-import"| IMP["🔧 github-repo-importer
(Go CLI Tool)"] - - %% Step 2: YAML Storage + IMPORT_WF["📋 import.yaml workflow
Single repo import"] -.-> IMP + BULK_WF["📋 bulk-import.yaml workflow
Multiple repos"] -.-> IMP IMP -->|"Creates YAML files"| CONFIGS["📁 configs/{owner}/*.yaml
(Temporary storage)"] - - %% Step 3: Justfile Processing CONFIGS -->|"just import-repos"| JUST["📜 Justfile
(Bash script)"] - JUST -->|"Check if exists"| DECISION{{"File exists in
repos/ ?"}} - - %% Step 4: File Distribution - DECISION -->|"NO: New import"| TMP["📁 importer_tmp_dir/
├── newRepo1232.yaml
└── 123123123.yaml"] - DECISION -->|"YES: Update"| REPOS["📁 repos/
├── test-env-111.yaml
└── existing-repo.yaml"] - - %% Step 5: Terraform Processing + DECISION -->|"NO: New import"| TMP["📁 importer_tmp_dir/
├── demo1.yaml
└── demo2.yaml"] + DECISION -->|"YES: Update"| REPOS["📁 repos/
├── repo1.yaml
└── repo2.yaml"] TMP -->|"local.generated_repos"| TF_IMPORT["🔄 Terraform Import Blocks
import { for_each = local.generated_repos }"] REPOS -->|"local.new_repos"| TF_CREATE["🏗️ Terraform Resources
(Create/Update)"] - + PLAN_WF["📋 tf-plan.yaml workflow
On PR"] -.-> TF_IMPORT + APPLY_WF["📋 tf-apply.yaml workflow
On merge to main"] -.-> MODULE TF_IMPORT -->|"merge()"| MODULE["📦 module.repository
for_each = local.all_repos"] TF_CREATE -->|"merge()"| MODULE - - %% Step 6: GitHub Operations MODULE -->|"terraform apply"| GH_FINAL[("🌐 GitHub
(Creates/Updates/Imports)")] - - %% Step 7: Promotion GH_FINAL -->|"After successful apply"| PROMOTE["🚀 Promote Workflow
(Move files)"] - PROMOTE -->|"mv importer_tmp_dir/* repos/"| REPOS_FINAL["📁 repos/
(Single source of truth)"] - + PROMOTE_WF["📋 promote-imported-configs.yaml
Manual trigger"] -.-> PROMOTE + PROMOTE -->|"mv importer_tmp_dir/* repos/"| REPOS_FINAL["⭐ 📁 repos/
ALL FILES MERGED HERE
(SINGLE SOURCE OF TRUTH)"] + REPOS -->|"Already managed repos"| REPOS_FINAL style GH fill:#e1f5fe style GH_FINAL fill:#e1f5fe style IMP fill:#fff3e0 style JUST fill:#f3e5f5 style TMP fill:#ffebee style REPOS fill:#e8f5e9 - style REPOS_FINAL fill:#c8e6c9 + style REPOS_FINAL fill:#4caf50,stroke:#2e7d32,stroke-width:3px,color:#fff style MODULE fill:#fce4ec + style IMPORT_WF fill:#f0f0f0,stroke-dasharray: 5 5 + style BULK_WF fill:#f0f0f0,stroke-dasharray: 5 5 + style PLAN_WF fill:#f0f0f0,stroke-dasharray: 5 5 + style APPLY_WF fill:#f0f0f0,stroke-dasharray: 5 5 + style PROMOTE_WF fill:#f0f0f0,stroke-dasharray: 5 5 ``` +[MermaidLive editor](https://www.mermaidchart.com/play?utm_source=mermaid_live_editor&utm_medium=share#pako:eNqVVVGL20YQ_iuD8uILkX25i2lj2is-n2wrlS0jy4QjOowsrWT1ZK1YSXGMz5BC3lpIoIZCCfStfep7f8_9gfYndLW7kmWfAz1ssOSZ-b6Zb3Zm15KDXSS1JFmWrcjBkRf4LSsCSOdogVrgYYKS1IqY2Sd2PAfzKrcD9Ppvapb07-8_f4JekPazGbRH6jcz0rioKe-CJA0iHwwU4-TEkk5uihiQ5Ys7S_IxkCyChR1Edfo8y8JbOVjEmKSWdAfqYPQmh97-CX6QzrOZTCiQcECEk_QwdDQVTIxDyiAIaKRumNPXXRb_y0_AY-orexHCEpNbL8RLFj-m-YUIcmAomG9ArssXOQhHu5xo31ewKlkeARxkYRrEAjJ5CEYfRPEdguwUJXDdHmjgBSFK8qI7-rCr9sac7EfgvUgaa7yMENk0njJKXrqJ8iRssoIkpb8-2gkgUATTD1mSiupkkdYdvJqMTc7yGV5RhzwDjntpJ3NIHBLE6Q4xdy8SnyPnFgIPUN5gBnaldNSxqg_Xa0vqUiBhgiBikIy0Ad9Z0mbD4YoAATnUWzBES9g13xTNpxoUDZ-mi3jqBqTBMO-3n--37-kXXLTAz3e63G-3FcMZM5RlHPBeK-MWTGKXNiInNZSRXkrPkz7gyv88ypUbDrjMstUhduyw7iPaQsrkTssmmN0pP6v8oH8AExFi03FbgMqqhksae5swMq4DrPNxnCLbmcO3cBQZNmUOrKK9LCK03OPvGErbVBj_x1__-ftjJQUDJTgjDuL0NX5gG1yu3ckYae1hZTxST45DOzoyGnoEI6OciLJ0jtIejbTrfRg7jsPVcZwFIj6CFLPVUUIO9KuJpgjxC3hRPIuoneRFczdO9AcssJuFqM4kCegcrRjHA4ntMCxkuykZuHZfYuBu_Fn4pKW2rLjct9efdtVhWztYo1XJE6F50uCHYn-X8nBB0PYoBSSZQ7uWeFm44xkZ-kAXjf7tPYwIXuAUweuqsrUBfov4Lqr0lwdWehPz2GITu7LYUru5GNhRZoeQksD3ESkbJKD2cAv13j6c86diBsvRFEpZ0v1fn-BwSNuaBl1VU8YwUIyecgV9xVB4WWN12KM9GOsTo6OA3gXTmJj9k6NT0g6p5u6KnqyIrlQXylmpZMCjknRFNx29zKhgYesJeu41PbRvEr35kkN-GXCb53nn6LRqYwtXGM9R02tWjWYlEM3QHigvRlB-7TXRywfWvbReOLbXPH2WpATfotaTM_SVe34mXuVl4Kbz1nn87pmDQ0xYolU4cbpFLg56gZyDAvlFXHic5p8C3KUXjU0HYtWCJuzVJ67cR0aJTfTIqGLzPJasHIv_FSht_gNVFiuQ) + ## 🔍 Detailed Process Breakdown ### 1️⃣ GitHub Import (github-repo-importer) @@ -128,14 +128,22 @@ module "repository" { ```yaml YAML File (demo1.yaml): -description: "My repo" +description: "Demo repository for testing" visibility: public environments: -- environment: development - wait_timer: 30 + - environment: production + wait_timer: 300 # 5 minutes reviewers: - users: + users: - octocat + - maintainer1 + teams: + - platform-team + - environment: staging + wait_timer: 60 # 1 minute + reviewers: + users: + - developer1 ``` #### Becomes Terraform Resources @@ -149,8 +157,13 @@ import { } import { - to = module.repository["demo1"].github_repository_environment.environment["development"] - id = "demo1:development" + to = module.repository["demo1"].github_repository_environment.environment["production"] + id = "demo1:production" +} + +import { + to = module.repository["demo1"].github_repository_environment.environment["staging"] + id = "demo1:staging" } ``` @@ -163,13 +176,23 @@ module "repository" { # YAML fields → module variables name = "demo1" - description = "My repo" + description = "Demo repository for testing" visibility = "public" - environments = [{ - environment = "development" - wait_timer = 44 - reviewers = { users = ["octocat"] } - }] + environments = [ + { + environment = "production" + wait_timer = 300 + reviewers = { + users = ["octocat", "maintainer1"] + teams = ["platform-team"] + } + }, + { + environment = "staging" + wait_timer = 60 + reviewers = { users = ["developer1"] } + } + ] } ``` From e57c35c687b09d9a4cda17bd0ba7e6d7b3046682 Mon Sep 17 00:00:00 2001 From: ljuboops257 Date: Mon, 24 Nov 2025 17:53:19 +0100 Subject: [PATCH 05/15] Update docs/DEVELOPERS_GUIDE.md Co-authored-by: Ivan Pavlovic --- docs/DEVELOPERS_GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/DEVELOPERS_GUIDE.md b/docs/DEVELOPERS_GUIDE.md index f31dee6..15421fe 100644 --- a/docs/DEVELOPERS_GUIDE.md +++ b/docs/DEVELOPERS_GUIDE.md @@ -125,7 +125,7 @@ Configure GitHub deployment environments with protection rules and reviewers. > - Manually grant access at: `https://github.com/{org}/{repo}/settings/access` > - Verify team access at: `https://github.com/orgs/{org}/teams/{team}/repositories` > - > **Without repository access, Terraform will apply successfully but teams won't be added as reviewers and next plan/apply will again be shown in expected changes** + > **Without repository access, Terraform will apply successfully but teams won't be added as reviewers and next plan/apply will show them as proposed changes** - **`deployment_policy`**: *(optional, object)* Controls which branches/tags can deploy to this environment - **`policy_type`**: *(required, enum)* Must be one of: From f12635a37e4e2a6f44fc08c335be4f1b09643bfa Mon Sep 17 00:00:00 2001 From: ljuboops257 Date: Mon, 24 Nov 2025 17:53:46 +0100 Subject: [PATCH 06/15] Update docs/DEVELOPERS_GUIDE.md Co-authored-by: Ivan Pavlovic --- docs/DEVELOPERS_GUIDE.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/DEVELOPERS_GUIDE.md b/docs/DEVELOPERS_GUIDE.md index 15421fe..5aba08d 100644 --- a/docs/DEVELOPERS_GUIDE.md +++ b/docs/DEVELOPERS_GUIDE.md @@ -131,9 +131,7 @@ Configure GitHub deployment environments with protection rules and reviewers. - **`policy_type`**: *(required, enum)* Must be one of: - `"protected_branches"` - Only protected branches can deploy - `"selected_branches_and_tags"` - Specific branch/tag patterns can deploy - - **`branch_patterns`**: *(optional, string[])* Branch patterns (e.g., `["main", "release/*"]`) - - Only used when `policy_type` is `"selected_branches_and_tags"` - - Set to `null` or omit when using `"protected_branches"` + - **`branch_patterns`**: *(optional, string[])* Branch patterns (e.g., `["main", "release/*"]`). Only used when `policy_type` is `"selected_branches_and_tags"`. Set to `null` or omit when using `"protected_branches"` - **`tag_patterns`**: *(optional, string[])* Tag patterns (e.g., `["v*"]`) - Only used when `policy_type` is `"selected_branches_and_tags"` - Set to `null` or omit when using `"protected_branches"` From 0c9ed54be0104efeada8e59f4d4ac9e4b7af80a3 Mon Sep 17 00:00:00 2001 From: ljuboops257 Date: Mon, 24 Nov 2025 17:54:21 +0100 Subject: [PATCH 07/15] Update docs/DEVELOPERS_GUIDE.md Co-authored-by: Ivan Pavlovic --- docs/DEVELOPERS_GUIDE.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/DEVELOPERS_GUIDE.md b/docs/DEVELOPERS_GUIDE.md index 5aba08d..3fd245b 100644 --- a/docs/DEVELOPERS_GUIDE.md +++ b/docs/DEVELOPERS_GUIDE.md @@ -132,9 +132,7 @@ Configure GitHub deployment environments with protection rules and reviewers. - `"protected_branches"` - Only protected branches can deploy - `"selected_branches_and_tags"` - Specific branch/tag patterns can deploy - **`branch_patterns`**: *(optional, string[])* Branch patterns (e.g., `["main", "release/*"]`). Only used when `policy_type` is `"selected_branches_and_tags"`. Set to `null` or omit when using `"protected_branches"` - - **`tag_patterns`**: *(optional, string[])* Tag patterns (e.g., `["v*"]`) - - Only used when `policy_type` is `"selected_branches_and_tags"` - - Set to `null` or omit when using `"protected_branches"` + - **`tag_patterns`**: *(optional, string[])* Tag patterns (e.g., `["v*"]`). Only used when `policy_type` is `"selected_branches_and_tags"`. Set to `null` or omit when using `"protected_branches"` **📖 For complete guide with examples, see [FEATURE_GITHUB_ENVIRONMENT.md](FEATURE_GITHUB_ENVIRONMENT.md)** From cf370bd6bfc455ec1e6ea2831b98d877c4aebbed Mon Sep 17 00:00:00 2001 From: ljuboops257 Date: Mon, 24 Nov 2025 17:54:38 +0100 Subject: [PATCH 08/15] Update docs/DEVELOPERS_GUIDE.md Co-authored-by: Ivan Pavlovic --- docs/DEVELOPERS_GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/DEVELOPERS_GUIDE.md b/docs/DEVELOPERS_GUIDE.md index 3fd245b..1b95df9 100644 --- a/docs/DEVELOPERS_GUIDE.md +++ b/docs/DEVELOPERS_GUIDE.md @@ -128,7 +128,7 @@ Configure GitHub deployment environments with protection rules and reviewers. > **Without repository access, Terraform will apply successfully but teams won't be added as reviewers and next plan/apply will show them as proposed changes** - **`deployment_policy`**: *(optional, object)* Controls which branches/tags can deploy to this environment - - **`policy_type`**: *(required, enum)* Must be one of: + - **`policy_type`**: *(required, string)* Must be one of: - `"protected_branches"` - Only protected branches can deploy - `"selected_branches_and_tags"` - Specific branch/tag patterns can deploy - **`branch_patterns`**: *(optional, string[])* Branch patterns (e.g., `["main", "release/*"]`). Only used when `policy_type` is `"selected_branches_and_tags"`. Set to `null` or omit when using `"protected_branches"` From cb8cf2a0016052881b3bbf0b6a48d90af2be3028 Mon Sep 17 00:00:00 2001 From: ljuboops257 Date: Mon, 24 Nov 2025 17:55:15 +0100 Subject: [PATCH 09/15] Update docs/DEVELOPERS_GUIDE.md Co-authored-by: Ivan Pavlovic --- docs/DEVELOPERS_GUIDE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/DEVELOPERS_GUIDE.md b/docs/DEVELOPERS_GUIDE.md index 1b95df9..762e4f3 100644 --- a/docs/DEVELOPERS_GUIDE.md +++ b/docs/DEVELOPERS_GUIDE.md @@ -121,6 +121,7 @@ Configure GitHub deployment environments with protection rules and reviewers. - **`teams`**: *(string[])* Team slugs (max 6 total) > ⚠️ **IMPORTANT: Team Access Requirement** + > > Teams specified as reviewers MUST have repository access first! > - Manually grant access at: `https://github.com/{org}/{repo}/settings/access` > - Verify team access at: `https://github.com/orgs/{org}/teams/{team}/repositories` From 38953f78d6e05c6fe43e42df73f26a775467c2b0 Mon Sep 17 00:00:00 2001 From: Ljubo Nikolic Date: Thu, 25 Dec 2025 17:16:54 +0100 Subject: [PATCH 10/15] move out import control from DEVELOPERS_GUIDE --- docs/DEVELOPERS_GUIDE.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/DEVELOPERS_GUIDE.md b/docs/DEVELOPERS_GUIDE.md index 762e4f3..ceb13ad 100644 --- a/docs/DEVELOPERS_GUIDE.md +++ b/docs/DEVELOPERS_GUIDE.md @@ -106,10 +106,6 @@ These are the primary configuration options for each repository. ## Environment Configuration -Configure GitHub deployment environments with protection rules and reviewers. - -**Import Control**: Set `feature_github_environment: true` in `import-config.yaml` to import environments. - ### Environment Fields - **`environment`**: *(required, string)* Environment name From 33c828820f531d804fd0871f069c9fb7d1a59aaa Mon Sep 17 00:00:00 2001 From: Ljubo Nikolic Date: Thu, 25 Dec 2025 17:18:20 +0100 Subject: [PATCH 11/15] fix wait_timer description - minutes instead of seconds --- docs/DEVELOPERS_GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/DEVELOPERS_GUIDE.md b/docs/DEVELOPERS_GUIDE.md index ceb13ad..d34b339 100644 --- a/docs/DEVELOPERS_GUIDE.md +++ b/docs/DEVELOPERS_GUIDE.md @@ -109,7 +109,7 @@ These are the primary configuration options for each repository. ### Environment Fields - **`environment`**: *(required, string)* Environment name -- **`wait_timer`**: *(optional, int)* Delay in seconds (max 43200) +- **`wait_timer`**: *(optional, int)* Delay in minutes (max 43200 or 30 days) - **`can_admins_bypass`**: *(optional, bool)* Admin bypass allowed (default: true) - **`prevent_self_review`**: *(optional, bool)* Prevent self-approval (default: false) - **`reviewers`**: *(optional, object)* From 32986afe483cd1ace333a67ec8be780ff140ad12 Mon Sep 17 00:00:00 2001 From: Ljubo Nikolic Date: Thu, 25 Dec 2025 17:19:53 +0100 Subject: [PATCH 12/15] out of scope of PR --- docs/GITHUB_ACTIONS_WORKFLOWS.md | 353 ------------------------------- 1 file changed, 353 deletions(-) delete mode 100644 docs/GITHUB_ACTIONS_WORKFLOWS.md diff --git a/docs/GITHUB_ACTIONS_WORKFLOWS.md b/docs/GITHUB_ACTIONS_WORKFLOWS.md deleted file mode 100644 index abc1fa2..0000000 --- a/docs/GITHUB_ACTIONS_WORKFLOWS.md +++ /dev/null @@ -1,353 +0,0 @@ -# GitHub Actions Workflows Guide - -This guide documents the reusable GitHub Actions workflows provided by github-terraformer and how they're used by gcss-config-repo. - -## Architecture Overview - -```mermaid -graph TB - subgraph "gcss-config-repo (Client Triggers)" - A1[Manual: Import Single Repo] - A2[Manual: Bulk Import] - A3[Manual: Create Fork] - A4[PR Event: Bootstrap] - A5[Push to main: Apply] - A6[Manual: Promote Configs] - A7[Schedule/Manual: Drift Check] - end - - subgraph "github-terraformer (Reusable Workflows)" - B1[import.yaml] - B2[bulk-import.yaml] - B3[create-fork.yaml] - B4[tf-plan.yaml] - B5[tf-apply.yaml] - B6[promote-imported-configs.yaml] - B7[drift-check.yaml] - end - - subgraph "Internal Actions (Hidden from Client)" - C1[gcss-config-setup
Clones config repo] - C2[compare
Finds changes] - C3[pr-bot
Creates PRs] - C4[graformer
Runs Terraform] - end - - A1 -->|triggers| B1 - A2 -->|triggers| B2 - A3 -->|triggers| B3 - A4 -->|triggers| B4 - A5 -->|triggers| B5 - A6 -->|triggers| B6 - A7 -->|triggers| B7 - - B1 --> C1 - B1 --> C2 - B1 --> C3 - B2 --> C1 - B2 --> C2 - B2 --> C3 - B3 --> C1 - B3 --> C3 - B4 --> C1 - B4 --> C4 - B5 --> C1 - B5 --> C4 - B6 --> C1 - B6 --> C4 - B7 --> C1 - B7 --> C4 - - style C1 fill:#f9f,stroke:#333,stroke-width:2px,stroke-dasharray: 5 5 - style C2 fill:#f9f,stroke:#333,stroke-width:2px,stroke-dasharray: 5 5 - style C3 fill:#f9f,stroke:#333,stroke-width:2px,stroke-dasharray: 5 5 - style C4 fill:#f9f,stroke:#333,stroke-width:2px,stroke-dasharray: 5 5 -``` - -### Two-Repository System - -- **gcss-config-repo**: Triggers workflows (what users interact with) -- **github-terraformer**: Provides reusable workflows and internal actions (implementation details) - -## Reusable Workflows (github-terraformer) - -### 1. 📥 `import.yaml` - Single Repository Import - -**Purpose**: Import a single GitHub repository configuration - -**Inputs**: - -- `repo_name` (string): Repository to import (without org prefix) -- `gcss_ref` (string): Branch of github-terraformer to use - -**Process**: - -1. Generate GitHub App token -2. Clone gcss-config-repo via gcss-config-setup action -3. Run importer: `just import-repo {org}/{repo_name}` -4. Compare action identifies new/changed files -5. Create PR with changes in `importer_tmp_dir/` - -**Called by**: gcss-config-repo's import workflow - ---- - -### 2. 📦 `bulk-import.yaml` - Bulk Repository Import - -**Purpose**: Import multiple repositories based on import-config.yaml - -**Inputs**: - -- `gcss_ref` (string): Branch of github-terraformer to use - -**Process**: -1. Uses import-config.yaml to determine which repos to import -2. Runs `just import-repos` to import all configured repositories -3. Creates single PR with all changes - -**Called by**: gcss-config-repo's bulk-import workflow - ---- - -### 3. 🍴 `create-fork.yaml` - Fork and Import - -**Purpose**: Fork an external repository and import its configuration - -**Inputs**: - -- `upstream_repo` (string): Repository to fork (format: `owner/repo`) -- `new_repo_name` (string, optional): Name for the fork -- `gcss_ref` (string): Branch of github-terraformer to use - -**Process**: - -1. Fork the upstream repository -2. Import the forked repository configuration -3. Create PR with the new configuration - -**Called by**: gcss-config-repo's create-fork workflow - ---- - -### 4. 📋 `tf-plan.yaml` - Terraform Plan - -**Purpose**: Run Terraform plan on pull requests - -**Inputs**: - -- `commit_sha` (string): Git SHA to checkout -- `gcss_ref` (string): Branch of github-terraformer to use -- `tfc_org` (string): Terraform Cloud organization - -**Process**: - -1. Setup configuration from commit SHA -2. Initialize Terraform with HCP backend -3. Run `terraform plan` via graformer action -4. Post plan results to PR as comment - -**Called by**: gcss-config-repo on PR events - ---- - -### 5. ✅ `tf-apply.yaml` - Terraform Apply - -**Purpose**: Apply Terraform changes when PRs are merged to main - -**Inputs**: - -- `commit_sha` (string): Git SHA to apply -- `gcss_ref` (string): Branch of github-terraformer to use -- `tfc_org` (string): Terraform Cloud organization - -**Process**: - -1. Setup configuration from commit SHA -2. Run `terraform apply -auto-approve` -3. Changes are applied to GitHub repositories - -**Called by**: gcss-config-repo on push to main - ---- - -### 6. 🎯 `promote-imported-configs.yaml` - Promote Configurations - -**Purpose**: Move imported configurations from `importer_tmp_dir/` to `repos/` - -**Inputs**: - -- `commit_sha` (string): Commit with configs to promote -- `pr_number` (string): PR number for reference -- `tfc_org` (string): Terraform Cloud organization - -**Process**: - -1. Run Terraform apply to import resources -2. Sanitize YAML files (remove IDs via yq) -3. Move files from `importer_tmp_dir/` to `repos/` -4. Commit and push changes - -**Called by**: Manual trigger after import PR is merged - ---- - -### 7. 🔍 `drift-check.yaml` - Configuration Drift Detection - -**Purpose**: Detect drift between Terraform state and actual GitHub configuration - -**Inputs**: - -- `commit_sha` (string): Commit to check -- `gcss_ref` (string): Branch of github-terraformer to use -- `tfc_org` (string): Terraform Cloud organization - -**Process**: - -1. Run `terraform plan -detailed-exitcode` -2. Report any detected drift -3. Can be scheduled or manually triggered - -**Called by**: gcss-config-repo (scheduled or manual) - -## Custom Actions (github-terraformer) - -### `gcss-config-setup` - -Clones gcss-config-repo and copies configuration files to appropriate locations - -### `compare` - -Compares `importer_tmp_dir/` with `repos/` to identify new or changed files - -### `pr-bot` - -Creates pull requests with generated changes - -### `graformer` - -Handles Terraform operations with HCP Terraform backend - -## Workflow Triggers (gcss-config-repo) - -```yaml -# Example: gcss-config-repo/.github/workflows/import.yaml -name: Import single repository -on: - workflow_dispatch: - inputs: - repo_name: - description: 'Repository to import' - required: true - -jobs: - import-repo: - uses: org/github-terraformer/.github/workflows/import.yaml@main - with: - repo_name: ${{ github.event.inputs.repo_name }} - gcss_ref: main - secrets: - app_private_key: ${{ secrets.APP_PRIVATE_KEY }} -``` - -## File Flow - -```yaml -1. Import Stage: - importer → configs/{org}/*.yaml → copy → importer_tmp_dir/ - -2. Terraform Import: - importer_tmp_dir/*.yaml → terraform import → state - -3. Promotion: - importer_tmp_dir/*.yaml → sanitize → repos/*.yaml - -4. Management: - repos/*.yaml → terraform apply → GitHub -``` - -## Directory Structure Impact - -```yaml -gcss-config-repo/ -├── repos/ # Managed configurations (source of truth) -│ └── *.yaml # Repository configurations -├── importer_tmp_dir/ # Temporary import location -│ └── *.yaml # Pending imports -└── .github/workflows/ # Workflows calling github-terraformer -``` - -## Environment Variables & Secrets - -### Required in gcss-config-repo: - -- `APP_PRIVATE_KEY`: GitHub App private key -- `TFC_TOKEN`: Terraform Cloud API token -- `APP_ID`: GitHub App ID (stored as variable) - -### Set by Workflows: - -- `GITHUB_TOKEN`: Generated from App credentials -- `OWNER`: GitHub organization -- `TF_VAR_*`: Terraform variables - -## Common Patterns - -### PR-based Flow - -1. Workflow generates changes → Creates PR -2. PR triggers → Bootstrap → Terraform Plan -3. PR merged → Terraform Apply -4. Optional → Promote imported configs - -### Authentication Flow - -1. GitHub App private key → Generate installation token -2. Token used for → GitHub API calls & git operations -3. Terraform uses → App credentials for resource management - -## Debugging Workflows - -### Check Workflow Runs - -```bash -gh run list --workflow=import.yaml -gh run view -``` - -### View Logs - -```bash -gh run view --log -gh run view --log-failed -``` - -### Re-run Failed Jobs - -```bash -gh run rerun -gh run rerun --failed -``` - -## Best Practices - -1. **Always use reusable workflows** - Don't duplicate workflow logic -2. **Pass secrets explicitly** - Use `secrets:` in workflow calls -3. **Version with refs** - Use `@branch` or `@tag` for stability -4. **Test locally first** - Use LOCAL_DEVELOPMENT_SETUP.md guide -5. **Monitor drift** - Schedule drift-check workflow regularly - -## Troubleshooting - -| Issue | Solution | -|-------|----------| -| "Bad credentials" | Check APP_PRIVATE_KEY secret | -| "Resource not accessible" | Verify GitHub App permissions | -| "cannot find module" | Check gcss_ref points to correct branch | -| "Terraform lock timeout" | Check HCP Terraform for stuck runs | -| Files not promoting | Ensure promote workflow runs after apply | - -## See Also - -- [LOCAL_DEVELOPMENT_SETUP.md](LOCAL_DEVELOPMENT_SETUP.md) - Local development guide -- [DEVELOPERS_GUIDE.md](DEVELOPERS_GUIDE.md) - YAML configuration reference -- `.github/workflows/` - Actual workflow implementations From f4bda553eabdde5cc338adeaa5019b062f0ab341 Mon Sep 17 00:00:00 2001 From: Ljubo Nikolic Date: Thu, 25 Dec 2025 17:20:25 +0100 Subject: [PATCH 13/15] out of scope of PR - move to docs pr --- docs/LOCAL_DEVELOPMENT_SETUP.md | 423 -------------------------------- 1 file changed, 423 deletions(-) delete mode 100644 docs/LOCAL_DEVELOPMENT_SETUP.md diff --git a/docs/LOCAL_DEVELOPMENT_SETUP.md b/docs/LOCAL_DEVELOPMENT_SETUP.md deleted file mode 100644 index b8bd39a..0000000 --- a/docs/LOCAL_DEVELOPMENT_SETUP.md +++ /dev/null @@ -1,423 +0,0 @@ -# Local Development Setup - -This guide explains how to set up local development for the GitHub Terraformer system, which consists of two main repositories working together. - -## Repository Architecture - -### github-terraformer (This Repository) -Contains the tools and reusable workflows: -- **feature/github-repo-importer**: Go CLI tool for importing GitHub repositories to YAML -- **feature/github-repo-provisioning**: Terraform module for managing GitHub repositories -- **.github/workflows**: Reusable GitHub Actions workflows -- **.github/actions**: Custom GitHub Actions (gcss-config-setup, compare, pr-bot, graformer) - -### gcss-config-repo (Configuration Repository) -Contains the actual configuration: -- **repos/**: Repository YAML configuration files (source of truth) -- **importer_tmp_dir/**: Temporary location for imported repositories (before Terraform import) -- **config/**: Configuration files (import-config.yaml, app-list.yaml) -- **.github/workflows**: Workflows that call github-terraformer's reusable workflows - -## Prerequisites - -- **Go 1.21+** (for importer development) -- **Terraform 1.0+** (for provisioning) -- **Just** command runner (`brew install just` or from https://github.com/casey/just) -- **GitHub App** or **Personal Access Token** with appropriate permissions -- **yq** for YAML processing (`brew install yq`) - -## Quick Setup - -### 1. Clone Both Repositories - -```bash -# Clone to adjacent directories -git clone https://github.com/your-org/github-terraformer.git -git clone https://github.com/your-org/gcss-config-repo.git - -# Your directory structure should be: -# workspace/ -# ├── github-terraformer/ -# └── gcss-config-repo/ -``` - -### 2. Setup Configuration Files - -The workflows use file copying (not symlinks) to connect the repositories: - -```bash -# Copy configuration files for local development -cd github-terraformer/feature/github-repo-provisioning - -# Create gcss_config directory (mimics what gcss-config-setup action does) -mkdir -p gcss_config -cp -r ../../../gcss-config-repo/* gcss_config/ - -# Copy required config files -cp gcss_config/config/app-list.yaml . -cp gcss_config/config/app-list.yaml ../github-repo-importer/ -cp gcss_config/config/import-config.yaml ../github-repo-importer/ -``` - -### 3. Configure Environment Variables - -#### For GitHub Repo Importer (Go Tool) - -```bash -cd github-terraformer/feature/github-repo-importer - -# Create .env file for importer -cat > .env << 'EOF' -export GITHUB_TOKEN="ghp_your_personal_access_token" # For local testing -export OWNER="your-org" -EOF - -source .env -``` - -#### For Terraform Provisioning - -```bash -cd github-terraformer/feature/github-repo-provisioning - -# For local development with GitHub App -cat > .env << 'EOF' -# GitHub App credentials -export TF_VAR_app_id="123456" -export TF_VAR_app_installation_id="12345678" -export TF_VAR_app_private_key="$(cat ~/.secrets/github-app.pem)" - -# Required variables -export TF_VAR_owner="your-org" -export TF_VAR_environment_directory="gcss_config" -EOF - -source .env -``` - -### 4. Terraform Backend Configuration - -For local development, you have two options: - -#### Option A: Local State (Development) -```bash -cd github-terraformer/feature/github-repo-provisioning - -# Create local backend configuration -cat > backend.tf << 'EOF' -terraform { - backend "local" { - path = "terraform.tfstate" - } -} -EOF - -terraform init -reconfigure -``` - -#### Option B: Terraform Cloud (Matches CI/CD) -```bash -cd github-terraformer/feature/github-repo-provisioning - -# Use the existing backend-hcp.tf (rename if needed) -cp backend-hcp.tf backend.tf - -# Set Terraform Cloud credentials -export TF_TOKEN_app_terraform_io="your-tfc-token" - -terraform init -``` - -## Development Workflows - -### Workflow 1: Importing Existing Repositories - -**Step 1: Import with CLI Tool** -```bash -cd github-terraformer/feature/github-repo-importer -source .env - -# Import single repository -just import-repo your-org/repo-name - -# Or bulk import based on import-config.yaml -just import-repos - -# Files are created in: configs/your-org/*.yaml -``` - -**Step 2: Copy to gcss_config** -```bash -# Copy imported files to provisioning directory -cp configs/$OWNER/*.yaml ../github-repo-provisioning/gcss_config/importer_tmp_dir/ -``` - -**Step 3: Run Terraform Import** -```bash -cd ../github-repo-provisioning -source .env - -# Review what will be imported -terraform plan - -# Import the repositories -terraform apply -``` - -**Step 4: Promote to Permanent Location** -```bash -# After successful Terraform import, move files from importer_tmp_dir to repos -cd gcss_config -mv importer_tmp_dir/*.yaml repos/ - -# Commit these changes -git add -A -git commit -m "Promote imported repositories" -``` - -### Workflow 2: Creating New Repositories - -**Step 1: Create YAML Configuration** -```bash -cd gcss-config-repo/repos - -# Create new repository configuration -cat > new-repo.yaml << 'EOF' -description: "My new repository" -visibility: private -default_branch: main -has_issues: true -has_projects: false -has_wiki: false -has_downloads: true -vulnerability_alerts_enabled: true - -# Optional: Add environments -environments: - - environment: production - wait_timer: 300 - deployment_policy: - policy_type: protected_branches - - - environment: staging - deployment_policy: - policy_type: selected_branches_and_tags - branch_patterns: ["release/*", "main"] - tag_patterns: ["v*"] -EOF -``` - -**Step 2: Apply with Terraform** -```bash -cd github-terraformer/feature/github-repo-provisioning -source .env - -# Update local copy -cp ../../../gcss-config-repo/repos/*.yaml gcss_config/repos/ - -# Plan and apply -terraform plan -terraform apply -``` - -## Testing - -### Test GitHub Repo Importer -```bash -cd github-terraformer/feature/github-repo-importer - -# Run unit tests -just test - -# Test single import -GITHUB_TOKEN=$GITHUB_TOKEN go run main.go import your-org/test-repo - -# Test bulk import -GITHUB_TOKEN=$GITHUB_TOKEN go run main.go bulk-import -c import-config.yaml -``` - -### Test Terraform Configuration -```bash -cd github-terraformer/feature/github-repo-provisioning - -# Validate configuration -terraform validate - -# Format check -terraform fmt -check - -# Plan without applying -terraform plan -``` - -## CI/CD Workflow (GitHub Actions) - -The actual CI/CD uses a different flow with reusable workflows: - -### How It Works in CI/CD: - -1. **gcss-config-repo** triggers workflows for: - - PR creation → Bootstrap → Terraform Plan - - Merge to main → Terraform Apply - - Manual import → Import workflow - -2. **Reusable Workflows** (in github-terraformer): - - Called by gcss-config-repo workflows - - Use GitHub App authentication - - Use Terraform Cloud backend - - Automatically handle file movements - -3. **Key Actions**: - - **gcss-config-setup**: Clones config repo and copies files - - **compare**: Compares importer_tmp_dir with repos to identify changes - - **pr-bot**: Creates pull requests with changes - - **graformer**: Handles Terraform operations with HCP Terraform - -### CI/CD Configuration Files - -**import-config.yaml** (controls import behavior): -```yaml -# Option 1: Ignore specific repositories -ignored_repos: - - "your-org/gcss-config-repo" # Don't import the config repo itself - - "your-org/github-terraformer" # Don't import the tool repo - - "your-org/private-archived" # Skip archived repos - -# Option 2: Only import specific repositories -selected_repos: - - "your-org/important-repo" - - "your-org/another-repo" - -# Features -feature_github_environment: true # Enable environment import -``` - -**app-list.yaml** (GitHub App IDs for bypass actors): -```yaml -apps: - - name: dependabot - id: 29110 # GitHub's Dependabot App ID - - name: renovate - id: 37453 # Renovate Bot App ID - - name: your-custom-app - id: 123456 -``` - -## Troubleshooting - -### Authentication Issues - -```bash -# Test GitHub token -curl -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/user - -# Test GitHub App authentication (if using App) -gh api user --header "Authorization: Bearer $(gh auth token)" - -# Check rate limits -curl -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/rate_limit -``` - -### File Path Issues - -```bash -# Verify gcss_config structure -cd github-terraformer/feature/github-repo-provisioning -tree gcss_config -L 2 - -# Expected structure: -# gcss_config/ -# ├── config/ -# │ ├── app-list.yaml -# │ └── import-config.yaml -# ├── repos/ -# │ └── *.yaml (repository configs) -# └── importer_tmp_dir/ -# └── *.yaml (pending imports) -``` - -### Terraform Import Failures - -Common issues and solutions: - -1. **"Cannot import non-existent remote object"** - - Repository doesn't exist in GitHub yet - - Solution: Move YAML from `importer_tmp_dir/` to `repos/` to create it - -2. **"Error: Resource already exists"** - - Repository already managed by Terraform - - Solution: Check `terraform state list` and remove duplicate - -3. **"Unauthorized"** - - GitHub App permissions insufficient - - Solution: Check App installation permissions - -### Importer Issues - -```bash -# Debug import with verbose output -cd github-terraformer/feature/github-repo-importer -go run main.go import your-org/repo -v - -# Check imported file -cat configs/your-org/repo.yaml - -# Validate YAML syntax -yq eval . configs/your-org/repo.yaml -``` - -## Local vs CI/CD Differences - -| Aspect | Local Development | CI/CD (GitHub Actions) | -|--------|------------------|------------------------| -| **Authentication** | Personal token or App | GitHub App only | -| **File Management** | Manual copying | Automated via actions | -| **Terraform Backend** | Local or HCP | HCP Terraform only | -| **Config Repo** | Local directory | Checked out via action | -| **Promotion** | Manual move | Automated workflow | -| **PR Creation** | Manual | Automated via pr-bot | - -## Security Best Practices - -### Never Commit: -- `.env` files -- `*.pem` keys -- `terraform.tfvars` -- `*.tfstate` files -- Personal access tokens - -### Secure Storage: -```bash -# Create secure directory for secrets -mkdir -p ~/.secrets && chmod 700 ~/.secrets - -# Store keys securely -mv github-app.pem ~/.secrets/ && chmod 600 ~/.secrets/github-app.pem - -# Use environment variables -export GITHUB_TOKEN=$(cat ~/.secrets/github-token) -``` - -### Use .gitignore: -```gitignore -# Add to .gitignore -.env -*.pem -terraform.tfvars -*.tfstate -*.tfstate.backup -.terraform/ -``` - -## Next Steps - -1. **Set up GitHub App**: Create an App with repository management permissions -2. **Configure Terraform Cloud**: Set up workspace for state management -3. **Test Import**: Try importing a test repository -4. **Create Repository**: Test creating a new repository via YAML -5. **Set up CI/CD**: Configure workflows in your gcss-config-repo - -For more details on specific configurations, see: -- [DEVELOPERS_GUIDE.md](DEVELOPERS_GUIDE.md) - Complete YAML configuration reference -- [FEATURE_GITHUB_ENVIRONMENT.md](FEATURE_GITHUB_ENVIRONMENT.md) - Environment configuration guide -- Repository examples in `gcss-config-repo/repos/` From d17e72e8358dbb547438529cfda09e42fb0dad63 Mon Sep 17 00:00:00 2001 From: Ljubo Nikolic Date: Thu, 25 Dec 2025 17:21:35 +0100 Subject: [PATCH 14/15] out of scope of PR --- docs/ci_flows.md | 208 ----------------------------------------------- 1 file changed, 208 deletions(-) delete mode 100644 docs/ci_flows.md diff --git a/docs/ci_flows.md b/docs/ci_flows.md deleted file mode 100644 index 1535fae..0000000 --- a/docs/ci_flows.md +++ /dev/null @@ -1,208 +0,0 @@ -# GitHub API -> YAML -> Terraform flow - -```mermaid ---- -config: - theme: forest ---- -graph TD - GH[("🌐 GitHub API
(Existing Repos)")] - GH -->|"go run main.go bulk-import"| IMP["🔧 github-repo-importer
(Go CLI Tool)"] - IMPORT_WF["📋 import.yaml workflow
Single repo import"] -.-> IMP - BULK_WF["📋 bulk-import.yaml workflow
Multiple repos"] -.-> IMP - IMP -->|"Creates YAML files"| CONFIGS["📁 configs/{owner}/*.yaml
(Temporary storage)"] - CONFIGS -->|"just import-repos"| JUST["📜 Justfile
(Bash script)"] - JUST -->|"Check if exists"| DECISION{{"File exists in
repos/ ?"}} - DECISION -->|"NO: New import"| TMP["📁 importer_tmp_dir/
├── demo1.yaml
└── demo2.yaml"] - DECISION -->|"YES: Update"| REPOS["📁 repos/
├── repo1.yaml
└── repo2.yaml"] - TMP -->|"local.generated_repos"| TF_IMPORT["🔄 Terraform Import Blocks
import { for_each = local.generated_repos }"] - REPOS -->|"local.new_repos"| TF_CREATE["🏗️ Terraform Resources
(Create/Update)"] - PLAN_WF["📋 tf-plan.yaml workflow
On PR"] -.-> TF_IMPORT - APPLY_WF["📋 tf-apply.yaml workflow
On merge to main"] -.-> MODULE - TF_IMPORT -->|"merge()"| MODULE["📦 module.repository
for_each = local.all_repos"] - TF_CREATE -->|"merge()"| MODULE - MODULE -->|"terraform apply"| GH_FINAL[("🌐 GitHub
(Creates/Updates/Imports)")] - GH_FINAL -->|"After successful apply"| PROMOTE["🚀 Promote Workflow
(Move files)"] - PROMOTE_WF["📋 promote-imported-configs.yaml
Manual trigger"] -.-> PROMOTE - PROMOTE -->|"mv importer_tmp_dir/* repos/"| REPOS_FINAL["⭐ 📁 repos/
ALL FILES MERGED HERE
(SINGLE SOURCE OF TRUTH)"] - REPOS -->|"Already managed repos"| REPOS_FINAL - style GH fill:#e1f5fe - style GH_FINAL fill:#e1f5fe - style IMP fill:#fff3e0 - style JUST fill:#f3e5f5 - style TMP fill:#ffebee - style REPOS fill:#e8f5e9 - style REPOS_FINAL fill:#4caf50,stroke:#2e7d32,stroke-width:3px,color:#fff - style MODULE fill:#fce4ec - style IMPORT_WF fill:#f0f0f0,stroke-dasharray: 5 5 - style BULK_WF fill:#f0f0f0,stroke-dasharray: 5 5 - style PLAN_WF fill:#f0f0f0,stroke-dasharray: 5 5 - style APPLY_WF fill:#f0f0f0,stroke-dasharray: 5 5 - style PROMOTE_WF fill:#f0f0f0,stroke-dasharray: 5 5 -``` - -[MermaidLive editor](https://www.mermaidchart.com/play?utm_source=mermaid_live_editor&utm_medium=share#pako:eNqVVVGL20YQ_iuD8uILkX25i2lj2is-n2wrlS0jy4QjOowsrWT1ZK1YSXGMz5BC3lpIoIZCCfStfep7f8_9gfYndLW7kmWfAz1ssOSZ-b6Zb3Zm15KDXSS1JFmWrcjBkRf4LSsCSOdogVrgYYKS1IqY2Sd2PAfzKrcD9Ppvapb07-8_f4JekPazGbRH6jcz0rioKe-CJA0iHwwU4-TEkk5uihiQ5Ys7S_IxkCyChR1Edfo8y8JbOVjEmKSWdAfqYPQmh97-CX6QzrOZTCiQcECEk_QwdDQVTIxDyiAIaKRumNPXXRb_y0_AY-orexHCEpNbL8RLFj-m-YUIcmAomG9ArssXOQhHu5xo31ewKlkeARxkYRrEAjJ5CEYfRPEdguwUJXDdHmjgBSFK8qI7-rCr9sac7EfgvUgaa7yMENk0njJKXrqJ8iRssoIkpb8-2gkgUATTD1mSiupkkdYdvJqMTc7yGV5RhzwDjntpJ3NIHBLE6Q4xdy8SnyPnFgIPUN5gBnaldNSxqg_Xa0vqUiBhgiBikIy0Ad9Z0mbD4YoAATnUWzBES9g13xTNpxoUDZ-mi3jqBqTBMO-3n--37-kXXLTAz3e63G-3FcMZM5RlHPBeK-MWTGKXNiInNZSRXkrPkz7gyv88ypUbDrjMstUhduyw7iPaQsrkTssmmN0pP6v8oH8AExFi03FbgMqqhksae5swMq4DrPNxnCLbmcO3cBQZNmUOrKK9LCK03OPvGErbVBj_x1__-ftjJQUDJTgjDuL0NX5gG1yu3ckYae1hZTxST45DOzoyGnoEI6OciLJ0jtIejbTrfRg7jsPVcZwFIj6CFLPVUUIO9KuJpgjxC3hRPIuoneRFczdO9AcssJuFqM4kCegcrRjHA4ntMCxkuykZuHZfYuBu_Fn4pKW2rLjct9efdtVhWztYo1XJE6F50uCHYn-X8nBB0PYoBSSZQ7uWeFm44xkZ-kAXjf7tPYwIXuAUweuqsrUBfov4Lqr0lwdWehPz2GITu7LYUru5GNhRZoeQksD3ESkbJKD2cAv13j6c86diBsvRFEpZ0v1fn-BwSNuaBl1VU8YwUIyecgV9xVB4WWN12KM9GOsTo6OA3gXTmJj9k6NT0g6p5u6KnqyIrlQXylmpZMCjknRFNx29zKhgYesJeu41PbRvEr35kkN-GXCb53nn6LRqYwtXGM9R02tWjWYlEM3QHigvRlB-7TXRywfWvbReOLbXPH2WpATfotaTM_SVe34mXuVl4Kbz1nn87pmDQ0xYolU4cbpFLg56gZyDAvlFXHic5p8C3KUXjU0HYtWCJuzVJ67cR0aJTfTIqGLzPJasHIv_FSht_gNVFiuQ) - -## 🔍 Detailed Process Breakdown - -### 1️⃣ GitHub Import (github-repo-importer) - -Location: `feature/github-repo-importer/` - -```go -go run main.go bulk-import -c import-config.yaml -- Fetches repo data from GitHub API -- Creates YAML files in: configs/{owner}/*.yaml -``` - -### 2️⃣ YAML Storage Locations - -Execution Context: Where each command runs from - -```yaml - | Stage | Full Path | Executed From Directory | - |---------------------------|---------------------------------------------------------------------------------------------------|-----------------------------------| - | 1. After Import | /home/.../github-terraformer/feature/github-repo-importer/configs/{owner}/*.yaml | feature/github-repo-importer/ | - | 2. Justfile Copy - New | /home/.../github-terraformer/feature/github-repo-provisioning/gcss_config/importer_tmp_dir/*.yaml | feature/github-repo-importer/ | - | 3. Justfile Copy - Update | /home/.../github-terraformer/feature/github-repo-provisioning/gcss_config/repos/*.yaml | feature/github-repo-importer/ | - | 4. Terraform Reads | gcss_config/importer_tmp_dir/*.yamlgcss_config/repos/*.yaml | feature/github-repo-provisioning/ | - | 5. Final (Promoted) | /home/.../gcss-config-repo/repos/*.yaml | N/A (separate repo) | -``` - -Directory Structure: - -```bash - github-terraformer/ - ├── feature/ - │ ├── github-repo-importer/ # 🔧 Import tool runs here - │ │ ├── Justfile - │ │ ├── main.go - │ │ └── configs/ # Step 1: Import creates YAMLs here - │ │ └── {owner}/ - │ │ ├── repo1.yaml - │ │ └── repo2.yaml - │ │ - │ └── github-repo-provisioning/ # 📦 Terraform runs here - │ ├── main.tf - │ └── gcss_config/ # This is actually gcss-config-repo checkout - │ ├── repos/ # Step 3: Existing repos updated here - │ │ └── existing.yaml - │ └── importer_tmp_dir/ # Step 2: New imports placed here - │ └── newRepo.yaml -``` - -Copies from: `configs/{owner}/*.yaml` - -Copies to: `../github-repo-provisioning/gcss_config/{repos or importer_tmp_dir}/` - -#### Important Note: __The gcss_config/ directory is actually a checkout of the gcss-config-repo (done by GitHub Actions), not a permanent part of github-terraformer!__ - -### 3️⃣ YAML → Terraform Transformation - -```hcl -# In main.tf - YAML becomes Terraform data -locals { - generated_repos = { - # Read YAML files and decode them - for file_path in fileset(path.module, "gcss_config/importer_tmp_dir/*.yaml") : - basename(file_path) => yamldecode(file(file_path)) # YAML → HCL - } -} -``` - -YAML structure becomes module variables - -```hcl -module "repository" { - for_each = local.all_repos - - # YAML fields map to module inputs - name = each.key # From filename - description = try(each.value.description, "") # From YAML content - visibility = try(each.value.visibility, "") # From YAML content - environments = try(each.value.environments, []) # From YAML content -} -``` - -### 4️⃣ Example YAML → Resource Flow - -```yaml -YAML File (demo1.yaml): -description: "Demo repository for testing" -visibility: public -environments: - - environment: production - wait_timer: 300 # 5 minutes - reviewers: - users: - - octocat - - maintainer1 - teams: - - platform-team - - environment: staging - wait_timer: 60 # 1 minute - reviewers: - users: - - developer1 -``` - -#### Becomes Terraform Resources - -If in `importer_tmp_dir/` → Import block generated - -```hcl -import { - to = module.repository["demo1"].github_repository.repository - id = "demo1" -} - -import { - to = module.repository["demo1"].github_repository_environment.environment["production"] - id = "demo1:production" -} - -import { - to = module.repository["demo1"].github_repository_environment.environment["staging"] - id = "demo1:staging" -} -``` - -Module creates actual resources - -```hcl -module "repository" { - # YAML filename → module key - for_each = { "demo1" = } - - # YAML fields → module variables - name = "demo1" - description = "Demo repository for testing" - visibility = "public" - environments = [ - { - environment = "production" - wait_timer = 300 - reviewers = { - users = ["octocat", "maintainer1"] - teams = ["platform-team"] - } - }, - { - environment = "staging" - wait_timer = 60 - reviewers = { users = ["developer1"] } - } - ] -} -``` - -### 5️⃣ Decision Tree - -```yaml - Is repo already in repos/? - ├─ YES → Update existing file in repos/ - │ └─ Terraform updates resource - └─ NO → Place in importer_tmp_dir/ - ├─ Terraform imports from GitHub - └─ After success → Move to repos/ -``` From e6b4622ab998363fecb424be08428297bd8c4923 Mon Sep 17 00:00:00 2001 From: Ljubo Nikolic Date: Thu, 25 Dec 2025 18:04:34 +0100 Subject: [PATCH 15/15] move out envs from repositories.go --- .../pkg/github/environments.go | 21 +++++++++++++++++++ .../pkg/github/repositories.go | 20 ------------------ 2 files changed, 21 insertions(+), 20 deletions(-) create mode 100644 feature/github-repo-importer/pkg/github/environments.go diff --git a/feature/github-repo-importer/pkg/github/environments.go b/feature/github-repo-importer/pkg/github/environments.go new file mode 100644 index 0000000..04d6355 --- /dev/null +++ b/feature/github-repo-importer/pkg/github/environments.go @@ -0,0 +1,21 @@ +package github + +type Environment struct { + Environment string `yaml:"environment"` + WaitTimer *int `yaml:"wait_timer,omitempty"` + CanAdminsBypass *bool `yaml:"can_admins_bypass,omitempty"` + PreventSelfReview *bool `yaml:"prevent_self_review,omitempty"` + Reviewers *EnvironmentReviewers `yaml:"reviewers,omitempty"` + DeploymentPolicy *DeploymentPolicy `yaml:"deployment_policy,omitempty"` +} + +type EnvironmentReviewers struct { + Teams []string `yaml:"teams,omitempty"` // Team slugs (e.g., "platform-team") + Users []string `yaml:"users,omitempty"` // GitHub usernames (e.g., "octocat") +} + +type DeploymentPolicy struct { + PolicyType string `yaml:"policy_type"` // "protected_branches" or "selected_branches_and_tags" + BranchPatterns []string `yaml:"branch_patterns,omitempty"` // e.g., ["release/*", "main"] - only for selected_branches_and_tags + TagPatterns []string `yaml:"tag_patterns,omitempty"` // e.g., ["v*"] - only for selected_branches_and_tags +} diff --git a/feature/github-repo-importer/pkg/github/repositories.go b/feature/github-repo-importer/pkg/github/repositories.go index 641b640..a1b7aa4 100644 --- a/feature/github-repo-importer/pkg/github/repositories.go +++ b/feature/github-repo-importer/pkg/github/repositories.go @@ -60,23 +60,3 @@ type Pages struct { BuildType *string `yaml:"build_type,omitempty" jsonschema:"required,enum=workflow,enum=legacy"` } -type Environment struct { - Environment string `yaml:"environment"` - WaitTimer *int `yaml:"wait_timer,omitempty"` - CanAdminsBypass *bool `yaml:"can_admins_bypass,omitempty"` - PreventSelfReview *bool `yaml:"prevent_self_review,omitempty"` // Extracted from ProtectionRules in API response - Reviewers *EnvironmentReviewers `yaml:"reviewers,omitempty"` - DeploymentPolicy *DeploymentPolicy `yaml:"deployment_policy,omitempty"` -} - -type EnvironmentReviewers struct { - Teams []string `yaml:"teams,omitempty"` // Team slugs (e.g., "platform-team") - Users []string `yaml:"users,omitempty"` // GitHub usernames (e.g., "octocat") -} - -// DeploymentPolicy represents the cleaner structure for deployment policies -type DeploymentPolicy struct { - PolicyType string `yaml:"policy_type"` // "protected_branches" or "selected_branches_and_tags" - BranchPatterns []string `yaml:"branch_patterns,omitempty"` // e.g., ["release/*", "main"] - only for selected_branches_and_tags - TagPatterns []string `yaml:"tag_patterns,omitempty"` // e.g., ["v*"] - only for selected_branches_and_tags -}