diff --git a/.github/workflows/tf-apply.yaml b/.github/workflows/tf-apply.yaml index fab3cfd..449bce2 100644 --- a/.github/workflows/tf-apply.yaml +++ b/.github/workflows/tf-apply.yaml @@ -58,8 +58,4 @@ jobs: env: TF_CLOUD_ORGANIZATION: ${{ inputs.tfc_org }} TF_WORKSPACE: ${{ vars.WORKSPACE }} - run: terraform apply -no-color -input=false ${{ steps.graformer.outputs.plan-file }} - - - name: Clean up plan file - if: always() && steps.graformer.outputs.plan-file != '' - run: rm -f "${{ steps.graformer.outputs.plan-file }}" \ No newline at end of file + run: terraform apply -no-color -input=false -auto-approve \ No newline at end of file diff --git a/README.md b/README.md index eb79c32..5e45aca 100644 --- a/README.md +++ b/README.md @@ -43,4 +43,4 @@ To import a **forked** repository into the organization: > 📝 We are working on improving this so that the user has the same experience as when creating a new repo > [!IMPORTANT] -> All important attributes are documented in the [Developer's Guide](DEVELOPERS_GUIDE.md). \ No newline at end of file +> All important attributes are documented in the [Developer's Guide](docs/DEVELOPERS_GUIDE.md). diff --git a/DEVELOPERS_GUIDE.md b/docs/DEVELOPERS_GUIDE.md similarity index 88% rename from DEVELOPERS_GUIDE.md rename to docs/DEVELOPERS_GUIDE.md index 0b687cc..d34b339 100644 --- a/DEVELOPERS_GUIDE.md +++ b/docs/DEVELOPERS_GUIDE.md @@ -100,29 +100,38 @@ These are the primary configuration options for each repository. - **`vulnerability_alerts_enabled`**: *(optional, boolean)* If `true`, vulnerability alerts are enabled. -- **`branch_protections_v4`**: *(optional, object[] [BranchProtectionV4](#branch-protection-configuration-v4))* Configuration for branch protection rules. - -- **`high_integrity`**: *(optional, object [HighIntegrity](#high-integrity-configuration))* Expansion directives for high-integrity repositories. This field is consumed by the `expand` command and is **not** passed to Terraform — it is removed from the output after expansion. +- **`environments`**: *(optional, object[] [Environment](#environment-configuration))* Configuration for repository environments. Requires `feature_github_environment: true` in import config. When imported, environments are automatically managed by Terraform. -## High Integrity Configuration - -Options for enabling high-integrity mode on a repository. This block is a pre-processing directive consumed by the `expand` command — it is **not** forwarded to Terraform. +- **`branch_protections_v4`**: *(optional, object[] [BranchProtectionV4](#branch-protection-configuration-v4))* Configuration for branch protection rules. -When `enabled` is `true`, the `expand` command automatically appends two rulesets to the repository's `rulesets` list: +## Environment Configuration -- **Protect main branch** — an active branch ruleset targeting `~DEFAULT_BRANCH` that enforces deletion protection, no fast-forward pushes, linear history, and a pull request review policy (1 approver, stale review dismissal on push, last-push approval required). -- **Make tags immutable** — an active tag ruleset targeting `~ALL` that prevents deletion, non-fast-forward updates, and tag updates. +### Environment Fields -The `high_integrity` block is then removed from the expanded output. +- **`environment`**: *(required, string)* Environment name +- **`wait_timer`**: *(optional, int)* Delay in minutes (max 43200 or 30 days) +- **`can_admins_bypass`**: *(optional, bool)* Admin bypass allowed (default: true) +- **`prevent_self_review`**: *(optional, bool)* Prevent self-approval (default: false) +- **`reviewers`**: *(optional, object)* + - **`users`**: *(string[])* GitHub usernames (max 6 total) + - **`teams`**: *(string[])* Team slugs (max 6 total) -- **`enabled`**: *(required, boolean)* If `true`, the two high-integrity rulesets are injected during expansion. + > ⚠️ **IMPORTANT: Team Access Requirement** + > + > Teams specified as reviewers MUST have repository access first! + > - Manually grant access at: `https://github.com/{org}/{repo}/settings/access` + > - Verify team access at: `https://github.com/orgs/{org}/teams/{team}/repositories` + > + > **Without repository access, Terraform will apply successfully but teams won't be added as reviewers and next plan/apply will show them as proposed changes** -Example: +- **`deployment_policy`**: *(optional, object)* Controls which branches/tags can deploy to this environment + - **`policy_type`**: *(required, string)* Must be one of: + - `"protected_branches"` - Only protected branches can deploy + - `"selected_branches_and_tags"` - Specific branch/tag patterns can deploy + - **`branch_patterns`**: *(optional, string[])* Branch patterns (e.g., `["main", "release/*"]`). Only used when `policy_type` is `"selected_branches_and_tags"`. Set to `null` or omit when using `"protected_branches"` + - **`tag_patterns`**: *(optional, string[])* Tag patterns (e.g., `["v*"]`). Only used when `policy_type` is `"selected_branches_and_tags"`. Set to `null` or omit when using `"protected_branches"` -```yaml -high_integrity: - enabled: true -``` +**📖 For complete guide with examples, see [FEATURE_GITHUB_ENVIRONMENT.md](FEATURE_GITHUB_ENVIRONMENT.md)** ## Template Configuration @@ -366,4 +375,4 @@ Options for configuring required status checks in V4. - **`strict`**: *(optional, boolean)* If `true`, strict status checks are enforced. -- **`contexts`**: *(optional, string[])* A list of required status check contexts. \ No newline at end of file +- **`contexts`**: *(optional, string[])* A list of required status check contexts. diff --git a/docs/FEATURE_GITHUB_ENVIRONMENT.md b/docs/FEATURE_GITHUB_ENVIRONMENT.md new file mode 100644 index 0000000..f955e54 --- /dev/null +++ b/docs/FEATURE_GITHUB_ENVIRONMENT.md @@ -0,0 +1,144 @@ +# GitHub Environments Configuration Guide + +This guide explains how to configure GitHub repository environments using the YAML → Terraform workflow. + +## Quick Start + +### Enable Environment Import +```yaml +# gcss-config-repo/config/import-config.yaml +feature_github_environment: true # Required to import environments +``` + +### Environment Configuration + +```yaml +# repos/my-app.yaml +environments: + # Option 1: Protected branches only + - environment: production + wait_timer: 300 # 5 minutes wait before deployment + can_admins_bypass: false # Admins cannot bypass + prevent_self_review: true # Cannot approve own deployments + reviewers: + users: ["octocat"] + teams: ["platform-team"] + deployment_policy: + policy_type: protected_branches + + # Option 2: Custom branch/tag patterns + - environment: staging + deployment_policy: + policy_type: selected_branches_and_tags + branch_patterns: + - "release/*" + - "main" + tag_patterns: + - "v*" + + # Option 3: Any branch can deploy (no restrictions) + - environment: development + # No deployment_policy = any branch can deploy +``` + +## ⚠️ Critical Rule: Deployment Policy Types + +**You MUST choose ONE of these options:** + +| Option | Configuration | Use Case | +|--------|--------------|----------| +| **Protected Branches** | `policy_type: protected_branches` | Production - only protected branches | +| **Custom Patterns** | `policy_type: selected_branches_and_tags` + patterns | Staging - specific branches/tags | +| **Any Branch** | Omit `deployment_policy` entirely | Development - no restrictions | + +**The `policy_type` field determines which patterns are used.** + +## Field Reference + +| Field | Type | Description | Default | +|-------|------|-------------|---------| +| `environment` | string | **Required** - Environment name | - | +| `wait_timer` | int | Wait time in seconds (max 43200) | 0 | +| `can_admins_bypass` | bool | Admins can bypass protections | true | +| `prevent_self_review` | bool | Prevent self-approval | false | +| `reviewers.users` | string[] | GitHub usernames (max 6 total with teams) | [] | +| `reviewers.teams` | string[] | Team slugs (max 6 total with users) | [] | +| `deployment_policy.*` | object | Deployment restrictions | - | +| ↳ `policy_type` | string | `protected_branches` or `selected_branches_and_tags` | - | +| ↳ `branch_patterns` | string[] | Branch patterns (only for `selected_branches_and_tags`) | [] | +| ↳ `tag_patterns` | string[] | Tag patterns (only for `selected_branches_and_tags`) | [] | + +## Pattern Matching + +Patterns support wildcards: +- `main` - Exact match +- `release/*` - Matches `release/1.0`, `release/2.0` +- `v*` - Matches `v1.0.0`, `v2.0.0` +- `*-final` - Matches `1.0-final`, `2.0-final` + +## Generated Terraform Resources + +The YAML configuration generates: + +1. **Environment Resource** +```hcl +resource "github_repository_environment" "environment" { + environment = "production" + repository = "my-app" + # ... other settings + + deployment_branch_policy { + protected_branches = true/false + custom_branch_policies = true/false + } +} +``` + +2. **Deployment Policies** (for custom patterns) +```hcl +resource "github_repository_environment_deployment_policy" "branch_policies" { + repository = "my-app" + environment = "staging" + branch_pattern = "release/*" +} + +resource "github_repository_environment_deployment_policy" "tag_policies" { + repository = "my-app" + environment = "staging" + tag_pattern = "v*" +} +``` + +## Complete Example + +```yaml +environments: + - environment: production + wait_timer: 300 + can_admins_bypass: false + prevent_self_review: true + reviewers: + teams: ["platform-team"] + deployment_policy: + policy_type: protected_branches + + - environment: staging + prevent_self_review: true + deployment_policy: + policy_type: selected_branches_and_tags + branch_patterns: ["release/*", "main"] + tag_patterns: ["v*", "rc-*"] + + - environment: development + # No restrictions - any branch can deploy +``` + +## Troubleshooting + +| Issue | Solution | +|-------|----------| +| "reviewers: must be 6 or fewer" | Combined users + teams must be ≤ 6 | +| Custom policies not working | Ensure `policy_type: selected_branches_and_tags` | +| Deployment policies not created | Check `custom_branch_policies = true` in Terraform | + +For more configuration options, see [DEVELOPERS_GUIDE.md](DEVELOPERS_GUIDE.md) diff --git a/docs/workflows.md b/docs/workflows.md deleted file mode 100644 index 4f3beae..0000000 --- a/docs/workflows.md +++ /dev/null @@ -1,44 +0,0 @@ -> [!IMPORTANT] -> This is a work in progress document and may change in the future - -## 🚀 GitHub Actions Workflows - -### 🔄 `Import` Workflow - -- **Trigger**: Manually via GitHub Actions -- **Inputs**: - - `branch`: Target environment (`dev` or `prod`) - - `repo_name`: Name of the GitHub repository to import - - `owner`: Name of the Github organization that owns the repository -- **Behavior**: - 1. Fetches repo metadata via GitHub API: - - General repository settings - - Branch protection rules - - Default branch - - Teams and collaborators - - Repository rulesets - 2. Generates a YAML configuration - 3. Places the YAML into: - ``` - feature/github-repo-provisioning/importer_tmp_dir/{organization}/{repository}.yaml - ``` - 4. Creates an automated pull request targeting the selected branch - 5. Upon PR merge, Terraform Cloud plans and applies the configuration - 6. Configuration file is then sanitized (ids removed) and moved to the appropriate directory `feature/github-repo-provisioning/repo_configs/{branch}/{organization}` - -## 📥 Importing Existing Repositories - -To import an **existing GitHub repository** into Terraform: - -1. Navigate to **Actions** > **Import** workflow in GitHub -2. Select: - - `prod` (or `dev`) as the target branch - - The name of the repository to import - - The owner of the repository (e.g., `G-Research` or `armadaproject`) -3. The workflow will: - - Generate a YAML config - - Place it under `feature/github-repo-provisioning/importer_tmp_dir/{organization}/` - - The name of the YAML file will be the same as the repository name - - Create a PR against the `prod` branch -4. Review, approve, and merge the PR -5. Terraform Cloud will detect and apply the changes \ No newline at end of file diff --git a/feature/github-repo-importer/ADDING_FEATURES.md b/feature/github-repo-importer/ADDING_FEATURES.md new file mode 100644 index 0000000..c48f565 --- /dev/null +++ b/feature/github-repo-importer/ADDING_FEATURES.md @@ -0,0 +1,210 @@ +# Adding New Features to the Importer + +Quick guide for adding feature-gated functionality to the importer. + +## Architecture + +- Features controlled by `config/import-config.yaml` +- No CLI changes needed - purely config-driven +- Single binary works for all features + +## Adding a Feature: 5 Steps + +### Example: Adding `feature_github_webhooks` + +#### 1. Add Constant + +`pkg/github/constants.go`: + +```go +const ( + FeatureGithubEnvironment = "feature_github_environment" + FeatureGithubWebhooks = "feature_github_webhooks" // NEW +) +``` + +#### 2. Add Logic + +`pkg/github/github.go` in `ImportRepo()`: + +```go +// ========================================================================= +// FEATURE: GitHub Webhooks +// ========================================================================= +var allWebhooks []*github.Hook +if cfg != nil && cfg.IsFeatureEnabled(FeatureGithubWebhooks) { + webhooks, _, err := v3client.Repositories.ListHooks( + context.Background(), owner, repo, nil) + if err != nil { + fmt.Printf("failed to get webhooks: %v\n", err) + } else { + allWebhooks = webhooks + dumpManager.WriteJSONFile("webhooks.json", webhooks) + } +} +``` + +#### 3. Add Data Structure + +`pkg/github/repositories.go`: + +```go +type Repository struct { + // ... existing fields ... + Webhooks []Webhook `yaml:"webhooks,omitempty"` // NEW +} + +type Webhook struct { + URL string `yaml:"url"` + ContentType string `yaml:"content_type,omitempty"` + Events []string `yaml:"events,omitempty"` + Active bool `yaml:"active"` +} +``` + +#### 4. Add Resolver Function + +`pkg/github/github.go`: + +```go +func resolveWebhooks(hooks []*github.Hook) []Webhook { + // Convert GitHub API response to YAML structure + var webhooks []Webhook + for _, hook := range hooks { + webhooks = append(webhooks, Webhook{ + URL: hook.GetURL(), + ContentType: hook.Config["content_type"].(string), + Events: hook.Events, + Active: hook.GetActive(), + }) + } + return webhooks +} +``` + +#### 5. Document in Config + +`gcss-config-repo/config/import-config.yaml`: + +```yaml +# feature_github_webhooks: Import repository webhooks +# - Webhook secrets are NOT imported (API limitation) +# - Default: false +feature_github_webhooks: true # Enable the feature +``` + +## Usage + +```bash +# Enable in import-config.yaml, then: +go run main.go import owner/repo +# or +go run main.go bulk-import +``` + +## Key Patterns + +### Always Check Config + +```go +if cfg != nil && cfg.IsFeatureEnabled(FeatureYourFeature) { + // Your code +} +``` + +### Error Handling + +- Don't fail entire import on feature errors +- Log with `fmt.Printf` +- Continue with other features + +### Data Dumps + +```go +dumpManager.WriteJSONFile("feature_data.json", data) +// Saves to: dumps//feature_data.json +``` + +## Real Example: GitHub Environments + +Current implementation shows the new `deployment_policy` structure: + +```go +// Data structure (pkg/github/repositories.go) +type Environment struct { + Environment string `yaml:"environment"` + WaitTimer *int `yaml:"wait_timer,omitempty"` + DeploymentPolicy *DeploymentPolicy `yaml:"deployment_policy,omitempty"` +} + +type DeploymentPolicy struct { + PolicyType string `yaml:"policy_type"` // "protected_branches" or "selected_branches_and_tags" + BranchPatterns []string `yaml:"branch_patterns,omitempty"` + TagPatterns []string `yaml:"tag_patterns,omitempty"` +} + +// Logic (pkg/github/github.go ~line 940) +if env.DeploymentBranchPolicy != nil { + deploymentPolicy := &DeploymentPolicy{} + + if env.DeploymentBranchPolicy.ProtectedBranches != nil && + *env.DeploymentBranchPolicy.ProtectedBranches { + deploymentPolicy.PolicyType = "protected_branches" + } else if env.DeploymentBranchPolicy.CustomBranchPolicies != nil && + *env.DeploymentBranchPolicy.CustomBranchPolicies { + deploymentPolicy.PolicyType = "selected_branches_and_tags" + // Fetch patterns from API + branchPatterns, tagPatterns := fetchDeploymentPolicies(...) + deploymentPolicy.BranchPatterns = branchPatterns + deploymentPolicy.TagPatterns = tagPatterns + } + + if deploymentPolicy.PolicyType != "" { + environment.DeploymentPolicy = deploymentPolicy + } +} +``` + +## Testing + +```bash +# 1. Feature disabled (default) +go run main.go import owner/repo +# Should NOT create dumps/owner-repo/webhooks.json + +# 2. Feature enabled +echo "feature_github_webhooks: true" >> import-config.yaml +go run main.go import owner/repo +# Should create dumps/owner-repo/webhooks.json + +# 3. Bulk import +go run main.go bulk-import +# Respects feature flag for all repos +``` + +## Do's and Don'ts + +✅ **DO** + +- Use `cfg.IsFeatureEnabled()` +- Handle errors gracefully +- Write dumps for debugging +- Document in import-config.yaml + +❌ **DON'T** + +- Add CLI flags or parameters +- Fail on missing data +- Skip nil checks +- Forget documentation + +## Quick Reference + +| File | Purpose | +|------|---------| +| `pkg/github/constants.go` | Define feature constant | +| `pkg/github/github.go` | Add feature logic in ImportRepo() | +| `pkg/github/repositories.go` | Define data structures | +| `config/import-config.yaml` | Document & enable feature | + +That's it! Follow these 5 steps and your feature will integrate seamlessly. diff --git a/feature/github-repo-importer/Justfile b/feature/github-repo-importer/Justfile index e007289..374efaa 100644 --- a/feature/github-repo-importer/Justfile +++ b/feature/github-repo-importer/Justfile @@ -17,4 +17,4 @@ compare dirA dirB: go run main.go compare {{dirA}} {{dirB}} generate-schema: - go run main.go schema \ No newline at end of file + go run main.go schema diff --git a/feature/github-repo-importer/cmd/bulk-import.go b/feature/github-repo-importer/cmd/bulk-import.go index 1bc79a3..fb5594b 100644 --- a/feature/github-repo-importer/cmd/bulk-import.go +++ b/feature/github-repo-importer/cmd/bulk-import.go @@ -2,10 +2,8 @@ package cmd import ( "fmt" - "os" "github.com/spf13/cobra" - "gopkg.in/yaml.v3" "github.com/gr-oss-devops/github-repo-importer/pkg/github" ) @@ -50,28 +48,3 @@ func init() { rootCmd.AddCommand(bulkImportCmd) bulkImportCmd.Flags().StringVarP(&configFilePath, "config", "c", "./import-config.yaml", "Path to the yaml config file (defaults to ./import-config.yaml)") } - -func DecodeConfiguration(configFilePath string) (*github.Config, error) { - file, err := os.Open(configFilePath) - if err != nil { - return nil, fmt.Errorf("failed to open config file: %w", err) - } - defer func(file *os.File) { - err := file.Close() - if err != nil { - fmt.Printf("failed to close file: %v\n", err) - } - }(file) - - var cfg github.Config - if err := yaml.NewDecoder(file).Decode(&cfg); err != nil { - return nil, fmt.Errorf("failed to decode YAML: %w", err) - } - - if cfg.PageSize == nil { - ps := github.DefaultPageSize - cfg.PageSize = &ps - } - - return &cfg, nil -} diff --git a/feature/github-repo-importer/cmd/config.go b/feature/github-repo-importer/cmd/config.go new file mode 100644 index 0000000..44a6307 --- /dev/null +++ b/feature/github-repo-importer/cmd/config.go @@ -0,0 +1,36 @@ +package cmd + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" + + "github.com/gr-oss-devops/github-repo-importer/pkg/github" +) + +// DecodeConfiguration reads and decodes the import configuration file +func DecodeConfiguration(configFilePath string) (*github.Config, error) { + file, err := os.Open(configFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open config file: %w", err) + } + defer func(file *os.File) { + err := file.Close() + if err != nil { + fmt.Printf("failed to close file: %v\n", err) + } + }(file) + + var cfg github.Config + if err := yaml.NewDecoder(file).Decode(&cfg); err != nil { + return nil, fmt.Errorf("failed to decode YAML: %w", err) + } + + if cfg.PageSize == nil { + ps := github.DefaultPageSize + cfg.PageSize = &ps + } + + return &cfg, nil +} diff --git a/feature/github-repo-importer/cmd/import.go b/feature/github-repo-importer/cmd/import.go index c1844f9..e83ba72 100644 --- a/feature/github-repo-importer/cmd/import.go +++ b/feature/github-repo-importer/cmd/import.go @@ -8,29 +8,40 @@ import ( "github.com/gr-oss-devops/github-repo-importer/pkg/github" ) -var importCmd = &cobra.Command{ - Use: "import [owner/repo]", - Short: "Import command reads all repository details and creates a configuration yaml file", - Args: cobra.ExactArgs(1), - PreRun: func(cmd *cobra.Command, args []string) { - github.InitializeClients() - }, - RunE: func(cmd *cobra.Command, args []string) error { - repository := args[0] - - repo, err := github.ImportRepo(repository) - if err != nil { - return fmt.Errorf("failed to import repo: %w", err) - } - - if err := github.WriteRepositoryToYaml(repo); err != nil { - return fmt.Errorf("failed to handle repository: %w", err) - } - - return nil - }, -} +var ( + importConfigPath string + importCmd = &cobra.Command{ + Use: "import [owner/repo]", + Short: "Import command reads all repository details and creates a configuration yaml file", + Args: cobra.ExactArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + github.InitializeClients() + }, + RunE: func(cmd *cobra.Command, args []string) error { + repository := args[0] + + // Load configuration with all feature flags + cfg, err := DecodeConfiguration(importConfigPath) + if err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + // Pass the entire config to ImportRepo - it will check feature flags internally + repo, err := github.ImportRepo(repository, cfg) + if err != nil { + return fmt.Errorf("failed to import repo: %w", err) + } + + if err := github.WriteRepositoryToYaml(repo); err != nil { + return fmt.Errorf("failed to handle repository: %w", err) + } + + return nil + }, + } +) func init() { rootCmd.AddCommand(importCmd) + importCmd.Flags().StringVarP(&importConfigPath, "config", "c", "./import-config.yaml", "Path to the import config file (default: ./import-config.yaml)") } diff --git a/feature/github-repo-importer/pkg/github/config.go b/feature/github-repo-importer/pkg/github/config.go index fc838a6..67d4c13 100644 --- a/feature/github-repo-importer/pkg/github/config.go +++ b/feature/github-repo-importer/pkg/github/config.go @@ -5,10 +5,19 @@ import ( ) type Config struct { - IsPublic *bool `yaml:"is_public,omitempty"` - IgnoredRepos []string `yaml:"ignored_repos,omitempty"` - SelectedRepos []string `yaml:"selected_repos,omitempty"` - PageSize *int `yaml:"page_size,omitempty"` + IsPublic *bool `yaml:"is_public,omitempty"` + IgnoredRepos []string `yaml:"ignored_repos,omitempty"` + SelectedRepos []string `yaml:"selected_repos,omitempty"` + PageSize *int `yaml:"page_size,omitempty"` + Features map[string]bool `yaml:",inline"` +} + +// IsFeatureEnabled checks if a feature flag is enabled (default: false) +func (c *Config) IsFeatureEnabled(featureName string) bool { + if c.Features == nil { + return false + } + return c.Features[featureName] } func (c *Config) Validate() error { diff --git a/feature/github-repo-importer/pkg/github/constants.go b/feature/github-repo-importer/pkg/github/constants.go index a481b23..bff15a3 100644 --- a/feature/github-repo-importer/pkg/github/constants.go +++ b/feature/github-repo-importer/pkg/github/constants.go @@ -33,6 +33,15 @@ const ( DefaultPageSize = 100 + // Feature flags + // All feature flags follow the pattern: feature_ + // Add new features here and they'll automatically work with the config system + FeatureGithubEnvironment = "feature_github_environment" + // Example future features: + // FeatureGithubWebhooks = "feature_github_webhooks" + // FeatureGithubSecrets = "feature_github_secrets" + // FeatureGithubTopics = "feature_github_topics" + BypassActorType_RepositoryRole = "RepositoryRole" BypassActorType_OrganizationAdmin = "OrganizationAdmin" BypassActorType_Team = "Team" diff --git a/feature/github-repo-importer/pkg/github/environments.go b/feature/github-repo-importer/pkg/github/environments.go new file mode 100644 index 0000000..04d6355 --- /dev/null +++ b/feature/github-repo-importer/pkg/github/environments.go @@ -0,0 +1,21 @@ +package github + +type Environment struct { + Environment string `yaml:"environment"` + WaitTimer *int `yaml:"wait_timer,omitempty"` + CanAdminsBypass *bool `yaml:"can_admins_bypass,omitempty"` + PreventSelfReview *bool `yaml:"prevent_self_review,omitempty"` + Reviewers *EnvironmentReviewers `yaml:"reviewers,omitempty"` + DeploymentPolicy *DeploymentPolicy `yaml:"deployment_policy,omitempty"` +} + +type EnvironmentReviewers struct { + Teams []string `yaml:"teams,omitempty"` // Team slugs (e.g., "platform-team") + Users []string `yaml:"users,omitempty"` // GitHub usernames (e.g., "octocat") +} + +type DeploymentPolicy struct { + PolicyType string `yaml:"policy_type"` // "protected_branches" or "selected_branches_and_tags" + BranchPatterns []string `yaml:"branch_patterns,omitempty"` // e.g., ["release/*", "main"] - only for selected_branches_and_tags + TagPatterns []string `yaml:"tag_patterns,omitempty"` // e.g., ["v*"] - only for selected_branches_and_tags +} diff --git a/feature/github-repo-importer/pkg/github/github.go b/feature/github-repo-importer/pkg/github/github.go index 7c251bf..d1c5ba0 100644 --- a/feature/github-repo-importer/pkg/github/github.go +++ b/feature/github-repo-importer/pkg/github/github.go @@ -1,6 +1,7 @@ package github import ( + "bytes" "context" "encoding/json" "errors" @@ -51,9 +52,19 @@ func DecodeAppsList() (*AppsList, error) { return &appsList, nil } -func ImportRepo(repoName string) (*Repository, error) { +// ImportRepo imports a single repository with feature flags from the provided config +func ImportRepo(repoName string, cfg *Config) (*Repository, error) { fmt.Println("Importing repository: ", repoName) + // Log enabled features + if cfg != nil && cfg.Features != nil { + for featureName, enabled := range cfg.Features { + if enabled { + fmt.Printf("Feature enabled: %s\n", featureName) + } + } + } + if !isValidRepoFormat(repoName) { return nil, errors.New("invalid repository format. Use owner/repo") } @@ -92,6 +103,68 @@ func ImportRepo(repoName string) (*Repository, error) { fmt.Printf("failed to write pages.json: %v\n", err) } + // ========================================================================= + // FEATURE: GitHub Environments + // ========================================================================= + // Feature flag: feature_github_environment (default: DISABLED - opt-in feature) + // Set feature_github_environment: true in import-config.yaml to enable environment import. + // + // When enabled, environments are imported from GitHub and managed by Terraform. + var allEnvironments []*github.Environment + + // Check if feature is enabled (defaults to false - opt-in feature) + enableEnvironments := false + if cfg != nil && cfg.Features != nil { + if enabled, exists := cfg.Features[FeatureGithubEnvironment]; exists { + enableEnvironments = enabled + } + } + + if enableEnvironments { + envOpts := &github.EnvironmentListOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + for { + environments, res, err := v3client.Repositories.ListEnvironments(context.Background(), repoNameSplit[0], repoNameSplit[1], envOpts) + if err != nil { + if res != nil && res.StatusCode == http.StatusNotFound { + fmt.Printf("environments not found (this is normal for repositories without environments): %v\n", err) + } else { + fmt.Printf("failed to get environments: %v\n", err) + } + break + } + + if err := dumpManager.WriteJSONFile("environments.json", environments); err != nil { + fmt.Printf("failed to write environments.json: %v\n", err) + } + + // ListEnvironments returns basic info - we need to fetch full details for each environment + // to get reviewers, protection rules, and other detailed configuration + if environments != nil && environments.Environments != nil { + for _, env := range environments.Environments { + if env.Name != nil { + // Fetch full environment details including reviewers + fullEnv, _, err := v3client.Repositories.GetEnvironment(context.Background(), repoNameSplit[0], repoNameSplit[1], *env.Name) + if err != nil { + fmt.Printf("Warning: failed to get full details for environment %s: %v\n", *env.Name, err) + // Use basic info if detailed fetch fails + allEnvironments = append(allEnvironments, env) + } else { + // Use full environment data with all details + allEnvironments = append(allEnvironments, fullEnv) + } + } + } + } + + if res.NextPage == 0 { + break + } + envOpts.Page = res.NextPage + } + } + rulesets, r, err := v3client.Repositories.GetAllRulesets(context.Background(), repoNameSplit[0], repoNameSplit[1], false) if err != nil { if r.StatusCode == http.StatusForbidden { @@ -200,9 +273,11 @@ func ImportRepo(repoName string) (*Repository, error) { Rulesets: resolvedRulesets, VulnerabilityAlertsEnabled: &vulnerabilityAlertsEnabled, BranchProtectionsV4: resolveBranchProtectionsFromGraphQL(&branchProtectionRulesGraphQLQuery), + Environments: resolveEnvironments(allEnvironments, v3client, repoNameSplit[0], repoNameSplit[1]), }, nil } + func getRoleActors() map[int64]string { return map[int64]string{ BypassActorId_OrganizationAdminRole: BypassActorRoleName_OrganizationAdminRole, @@ -273,7 +348,7 @@ func ImportRepos(cfg Config) ([]*Repository, error) { var importedRepos []*Repository for _, repoToImport := range reposToImport { - repository, err := ImportRepo(repoToImport) + repository, err := ImportRepo(repoToImport, &cfg) if err != nil { return nil, fmt.Errorf("failed to import repository %s: %w", repository.Name, err) } @@ -679,6 +754,225 @@ func resolveRepositoryTemplate(githubRepository *github.Repository) *RepositoryT return nil } +// fetchDeploymentPolicies fetches deployment branch policies for a given environment +func fetchDeploymentPolicies(client *github.Client, owner, repo, envName string) ([]string, []string) { + + var branchPatterns []string + var tagPatterns []string + + // List deployment branch policies + // GitHub API endpoint: GET /repos/{owner}/{repo}/environments/{environment_name}/deployment-branch-policies + opts := &github.ListOptions{PerPage: 100} + for { + // Note: The go-github library may not have direct support for this endpoint yet + // We'll use the generic API call method + req, err := client.NewRequest("GET", fmt.Sprintf("repos/%s/%s/environments/%s/deployment-branch-policies", owner, repo, envName), nil) + if err != nil { + fmt.Printf("Warning: Failed to create request for deployment policies: %v\n", err) + break + } + + type DeploymentPolicy struct { + ID int64 `json:"id"` + NodeID string `json:"node_id"` + Name string `json:"name"` + Type string `json:"type"` // "branch" or "tag" + } + + type DeploymentPoliciesResponse struct { + TotalCount int `json:"total_count"` + Policies []DeploymentPolicy `json:"branch_policies"` + } + + var result DeploymentPoliciesResponse + resp, err := client.Do(context.Background(), req, &result) + if err != nil { + // If 404, it might mean no custom policies are configured + if resp != nil && resp.StatusCode == 404 { + return nil, nil + } + fmt.Printf("Warning: Failed to fetch deployment policies for environment %s: %v\n", envName, err) + break + } + + for _, policy := range result.Policies { + if policy.Type == "branch" { + branchPatterns = append(branchPatterns, policy.Name) + } else if policy.Type == "tag" { + tagPatterns = append(tagPatterns, policy.Name) + } + } + + if resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + + return branchPatterns, tagPatterns +} + +func resolveEnvironments(envs []*github.Environment, client *github.Client, owner, repo string) []Environment { + if len(envs) == 0 { + return nil + } + + // Get organization info to obtain org ID (needed for team lookups) + org, _, err := client.Organizations.Get(context.Background(), owner) + if err != nil { + fmt.Printf("Warning: failed to get organization info: %v\n", err) + return nil + } + + var environments []Environment + for _, env := range envs { + environment := Environment{ + Environment: env.GetName(), + // WaitTimer will be extracted from ProtectionRules below + CanAdminsBypass: env.CanAdminsBypass, + } + + // Extract PreventSelfReview, WaitTimer, and Reviewers from ProtectionRules + // All are nested inside ProtectionRules array in the GitHub API + // Only set if actually found - don't assume defaults when importing + if env.ProtectionRules != nil && len(env.ProtectionRules) > 0 { + // We need to extract reviewers from ProtectionRules, not from env.Reviewers + protectionReviewers := &EnvironmentReviewers{} + + for _, rule := range env.ProtectionRules { + // Check for wait_timer rule type + if rule.Type != nil && *rule.Type == "wait_timer" { + if rule.WaitTimer != nil { + environment.WaitTimer = rule.WaitTimer + } + } + + if rule.PreventSelfReview != nil { + environment.PreventSelfReview = rule.PreventSelfReview + } + + // Check if this protection rule has reviewers + if rule.Reviewers != nil && len(rule.Reviewers) > 0 { + // Extract reviewers from this protection rule + // RequiredReviewer has Reviewer field (interface{}) that contains the actual user/team data + for _, reqReviewer := range rule.Reviewers { + if reqReviewer.Type == nil { + continue + } + + // The Reviewer field is an interface{} - try different type casts + if reqReviewer.Reviewer != nil { + switch *reqReviewer.Type { + case "Team": + // Try casting to *github.Team + if team, ok := reqReviewer.Reviewer.(*github.Team); ok { + if team.Slug != nil { + protectionReviewers.Teams = append(protectionReviewers.Teams, *team.Slug) + } else if team.Name != nil { + protectionReviewers.Teams = append(protectionReviewers.Teams, *team.Name) + } + } + case "User": + // Try casting to *github.User + if user, ok := reqReviewer.Reviewer.(*github.User); ok { + if user.Login != nil { + protectionReviewers.Users = append(protectionReviewers.Users, *user.Login) + } + } else { + // Fallback: try map[string]interface{} for older API versions + if reviewerData, ok := reqReviewer.Reviewer.(map[string]interface{}); ok { + if login, ok := reviewerData["login"].(string); ok { + protectionReviewers.Users = append(protectionReviewers.Users, login) + } + } + } + } + } + } + } + } + + // Set reviewers if we found any in ProtectionRules + if len(protectionReviewers.Teams) > 0 || len(protectionReviewers.Users) > 0 { + environment.Reviewers = protectionReviewers + } + } + + // Handle reviewers at top level (fallback if not in ProtectionRules) + // API may return array of EnvReviewers with Type and ID + // We resolve IDs to human-readable names (usernames and team slugs) + // Note: This is usually empty as reviewers are typically in ProtectionRules + if env.Reviewers != nil && len(env.Reviewers) > 0 && environment.Reviewers == nil { + reviewers := &EnvironmentReviewers{} + + // Separate reviewers by type and resolve IDs to names + for _, reviewer := range env.Reviewers { + if reviewer.Type != nil && reviewer.ID != nil { + switch *reviewer.Type { + case "Team": + // Resolve team ID to team slug + team, _, err := client.Teams.GetTeamByID(context.Background(), org.GetID(), *reviewer.ID) + if err != nil { + fmt.Printf("Warning: failed to resolve team ID %d: %v\n", *reviewer.ID, err) + continue + } + if team.Slug != nil { + reviewers.Teams = append(reviewers.Teams, *team.Slug) + } + case "User": + // Resolve user ID to username + user, _, err := client.Users.GetByID(context.Background(), *reviewer.ID) + if err != nil { + fmt.Printf("Warning: failed to resolve user ID %d: %v\n", *reviewer.ID, err) + continue + } + if user.Login != nil { + reviewers.Users = append(reviewers.Users, *user.Login) + } + } + } + } + + if len(reviewers.Teams) > 0 || len(reviewers.Users) > 0 { + environment.Reviewers = reviewers + } + } + + // Handle deployment policy + if env.DeploymentBranchPolicy != nil { + deploymentPolicy := &DeploymentPolicy{} + + // Determine policy type based on protected_branches and custom_branch_policies + if env.DeploymentBranchPolicy.ProtectedBranches != nil && *env.DeploymentBranchPolicy.ProtectedBranches { + // Protected branches only + deploymentPolicy.PolicyType = "protected_branches" + } else if env.DeploymentBranchPolicy.CustomBranchPolicies != nil && *env.DeploymentBranchPolicy.CustomBranchPolicies { + // Custom branch/tag patterns + deploymentPolicy.PolicyType = "selected_branches_and_tags" + + // Fetch deployment branch policies from GitHub API + branchPatterns, tagPatterns := fetchDeploymentPolicies(client, owner, repo, env.GetName()) + + if len(branchPatterns) > 0 { + deploymentPolicy.BranchPatterns = branchPatterns + } + if len(tagPatterns) > 0 { + deploymentPolicy.TagPatterns = tagPatterns + } + } + + // Only set deployment policy if we have a valid policy type + if deploymentPolicy.PolicyType != "" { + environment.DeploymentPolicy = deploymentPolicy + } + } + + environments = append(environments, environment) + } + + return environments +} + func resolveVisibility(private bool) string { if private { return VisibilityPrivate @@ -823,17 +1117,21 @@ func CategorizeTeams(client *github.Client, owner, repo string, dumpManager *fil } func WriteRepositoryToYaml(repository *Repository) error { - data, err := yaml.Marshal(repository) - if err != nil { + // Use a buffer and encoder with 2-space indentation for consistent formatting + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) // Use 2-space indentation for consistency + if err := enc.Encode(repository); err != nil { return fmt.Errorf("failed to marshal repository to YAML: %w", err) } + enc.Close() configsBasePath := filepath.Join("./configs", repository.Owner) if err := os.MkdirAll(configsBasePath, os.ModePerm); err != nil { return fmt.Errorf("failed to create base directories: %w", err) } - if err := os.WriteFile(filepath.Join(configsBasePath, fmt.Sprintf("%s.yaml", repository.Name)), data, os.ModePerm); err != nil { + if err := os.WriteFile(filepath.Join(configsBasePath, fmt.Sprintf("%s.yaml", repository.Name)), buf.Bytes(), os.ModePerm); err != nil { return fmt.Errorf("failed to write repository to YAML: %w", err) } diff --git a/feature/github-repo-importer/pkg/github/github_test.go b/feature/github-repo-importer/pkg/github/github_test.go index fcb1ced..0474b71 100644 --- a/feature/github-repo-importer/pkg/github/github_test.go +++ b/feature/github-repo-importer/pkg/github/github_test.go @@ -237,7 +237,7 @@ func TestImportRepo(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - repo, err := ImportRepo(tt.repoName) + repo, err := ImportRepo(tt.repoName, nil) if tt.wantError { assert.Error(t, err) diff --git a/feature/github-repo-importer/pkg/github/repositories.go b/feature/github-repo-importer/pkg/github/repositories.go index bc1c878..a1b7aa4 100644 --- a/feature/github-repo-importer/pkg/github/repositories.go +++ b/feature/github-repo-importer/pkg/github/repositories.go @@ -45,6 +45,7 @@ type Repository struct { Rulesets []Ruleset `yaml:"rulesets,omitempty"` VulnerabilityAlertsEnabled *bool `yaml:"vulnerability_alerts_enabled,omitempty"` BranchProtectionsV4 []*BranchProtectionV4 `yaml:"branch_protections_v4,omitempty"` + Environments []Environment `yaml:"environments,omitempty"` } type RepositoryTemplate struct { @@ -58,3 +59,4 @@ type Pages struct { Path *string `yaml:"path,omitempty"` BuildType *string `yaml:"build_type,omitempty" jsonschema:"required,enum=workflow,enum=legacy"` } + diff --git a/feature/github-repo-provisioning/Justfile b/feature/github-repo-provisioning/Justfile index 2df890a..3ec226a 100644 --- a/feature/github-repo-provisioning/Justfile +++ b/feature/github-repo-provisioning/Justfile @@ -2,4 +2,4 @@ clean: rm -rf .terraform .terraform.lock.hcl terraform.tfstate terraform.tfstate.backup init: - terraform init \ No newline at end of file + terraform init diff --git a/feature/github-repo-provisioning/main.tf b/feature/github-repo-provisioning/main.tf index 7ed838a..0990af3 100644 --- a/feature/github-repo-provisioning/main.tf +++ b/feature/github-repo-provisioning/main.tf @@ -1,9 +1,9 @@ provider "github" { owner = var.owner app_auth { - id = var.app_id + id = var.app_id installation_id = var.app_installation_id - pem_file = var.app_private_key + pem_file = var.app_private_key } } @@ -30,25 +30,26 @@ locals { ) all_repos = merge(local.generated_repos, local.new_repos) + } import { for_each = local.generated_repos - to = module.repository[each.key].github_repository.repository - id = each.key + to = module.repository[each.key].github_repository.repository + id = each.key } import { for_each = local.generated_repos - to = module.repository[each.key].github_branch_default.default[0] - id = each.key + to = module.repository[each.key].github_branch_default.default[0] + id = each.key } locals { flattened_generated_branch_protections_v4 = flatten([ for repo, config in local.generated_repos : [ for branch_protection in try(config.branch_protections_v4, []) : { - repository = repo + repository = repo branch_protection = branch_protection } ] @@ -76,51 +77,51 @@ locals { data "github_app" "app" { for_each = toset(local.app_actors) - slug = split("/", each.value)[1] + slug = split("/", each.value)[1] } locals { all_generated_collaborators = { for repo, config in local.generated_repos : repo => concat( - try([for i in config.pull_collaborators : { username: i, permission = "pull" }], []), - try([for i in config.push_collaborators : { username: i, permission = "push" }], []), - try([for i in config.admin_collaborators : { username: i, permission = "admin" }], []), - try([for i in config.maintain_collaborators : { username: i, permission = "maintain" }], []), - try([for i in config.triage_collaborators : { username: i, permission = "triage" }], []) - )} + try([for i in config.pull_collaborators : { username : i, permission = "pull" }], []), + try([for i in config.push_collaborators : { username : i, permission = "push" }], []), + try([for i in config.admin_collaborators : { username : i, permission = "admin" }], []), + try([for i in config.maintain_collaborators : { username : i, permission = "maintain" }], []), + try([for i in config.triage_collaborators : { username : i, permission = "triage" }], []) + ) } all_generated_teams = { for repo, config in local.generated_repos : repo => concat( - try([for i in config.pull_teams : { name: i, permission = "pull" }], []), - try([for i in config.push_teams : { name: i, permission = "push" }], []), - try([for i in config.admin_teams : { name: i, permission = "admin" }], []), - try([for i in config.maintain_teams : { name: i, permission = "maintain" }], []), - try([for i in config.triage_teams : { name: i, permission = "triage" }], []) - )} + try([for i in config.pull_teams : { name : i, permission = "pull" }], []), + try([for i in config.push_teams : { name : i, permission = "push" }], []), + try([for i in config.admin_teams : { name : i, permission = "admin" }], []), + try([for i in config.maintain_teams : { name : i, permission = "maintain" }], []), + try([for i in config.triage_teams : { name : i, permission = "triage" }], []) + ) } all_new_collaborators = { for repo, config in local.new_repos : repo => concat( - try([for i in config.pull_collaborators : { username: i, permission = "pull" }], []), - try([for i in config.push_collaborators : { username: i, permission = "push" }], []), - try([for i in config.admin_collaborators : { username: i, permission = "admin" }], []), - try([for i in config.maintain_collaborators : { username: i, permission = "maintain" }], []), - try([for i in config.triage_collaborators : { username: i, permission = "triage" }], []) - )} + try([for i in config.pull_collaborators : { username : i, permission = "pull" }], []), + try([for i in config.push_collaborators : { username : i, permission = "push" }], []), + try([for i in config.admin_collaborators : { username : i, permission = "admin" }], []), + try([for i in config.maintain_collaborators : { username : i, permission = "maintain" }], []), + try([for i in config.triage_collaborators : { username : i, permission = "triage" }], []) + ) } all_new_teams = { for repo, config in local.new_repos : repo => concat( - try([for i in config.pull_teams : { name: i, permission = "pull" }], []), - try([for i in config.push_teams : { name: i, permission = "push" }], []), - try([for i in config.admin_teams : { name: i, permission = "admin" }], []), - try([for i in config.maintain_teams : { name: i, permission = "maintain" }], []), - try([for i in config.triage_teams : { name: i, permission = "triage" }], []) - )} + try([for i in config.pull_teams : { name : i, permission = "pull" }], []), + try([for i in config.push_teams : { name : i, permission = "push" }], []), + try([for i in config.admin_teams : { name : i, permission = "admin" }], []), + try([for i in config.maintain_teams : { name : i, permission = "maintain" }], []), + try([for i in config.triage_teams : { name : i, permission = "triage" }], []) + ) } all_collaborators = merge(local.all_generated_collaborators, local.all_new_collaborators) - all_teams = merge(local.all_generated_teams, local.all_new_teams) + all_teams = merge(local.all_generated_teams, local.all_new_teams) } import { for_each = toset(flatten([for repo, collaborators in local.all_generated_collaborators : [ for collaborator in collaborators : { - repo = repo - username = collaborator.username + repo = repo + username = collaborator.username permission = collaborator.permission } ]])) @@ -141,9 +142,9 @@ data "github_team" "team" { import { for_each = toset(flatten([for repo, teams in local.all_generated_teams : [ for team in teams : { - repo = repo - name = team.name - team_id = data.github_team.team[team.name].id + repo = repo + name = team.name + team_id = data.github_team.team[team.name].id } ]])) @@ -153,45 +154,45 @@ import { module "repository" { - source = "./modules/terraform-github-repository" - for_each = local.all_repos + source = "./modules/terraform-github-repository" + for_each = local.all_repos # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Main resource configuration # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - name = each.key - allow_merge_commit = try(each.value.allow_merge_commit, true) - allow_rebase_merge = try(each.value.allow_rebase_merge, false) - allow_squash_merge = try(each.value.allow_squash_merge, false) - allow_auto_merge = try(each.value.allow_auto_merge, false) - allow_update_branch = try(each.value.allow_update_branch, null) - description = try(each.value.description, "") - delete_branch_on_merge = try(each.value.delete_branch_on_merge, true) - homepage_url = try(each.value.homepage_url, "") - visibility = try(each.value.visibility, "private") - has_issues = try(each.value.has_issues, false) - has_projects = try(each.value.has_projects, false) - has_wiki = try(each.value.has_wiki, false) - has_downloads = try(each.value.has_downloads, false) - has_discussions = try(each.value.has_discussions, null) - is_template = try(each.value.is_template, false) - default_branch = try(each.value.default_branch, "") - archived = try(each.value.archived, false) - topics = try(each.value.topics, []) - archive_on_destroy = try(each.value.archive_on_destroy, null) - pages = try(contains(keys(each.value), "pages") && try(each.value.pages != null, false) ? { - branch = try(each.value.pages.build_type, null) == "workflow" ? null : try(each.value.pages.branch, "gh-pages") - path = try(each.value.pages.build_type, null) == "workflow" ? null : try(each.value.pages.path, "/") - cname = try(each.value.pages.cname, null) - build_type = try(each.value.pages.build_type, null) - } : null) - vulnerability_alerts = try(each.value.vulnerability_alerts_enabled, null) - - squash_merge_commit_title = try(each.value.squash_merge_commit_title, null) + name = each.key + allow_merge_commit = try(each.value.allow_merge_commit, true) + allow_rebase_merge = try(each.value.allow_rebase_merge, false) + allow_squash_merge = try(each.value.allow_squash_merge, false) + allow_auto_merge = try(each.value.allow_auto_merge, false) + allow_update_branch = try(each.value.allow_update_branch, null) + description = try(each.value.description, "") + delete_branch_on_merge = try(each.value.delete_branch_on_merge, true) + homepage_url = try(each.value.homepage_url, "") + visibility = try(each.value.visibility, "private") + has_issues = try(each.value.has_issues, false) + has_projects = try(each.value.has_projects, false) + has_wiki = try(each.value.has_wiki, false) + has_downloads = try(each.value.has_downloads, false) + has_discussions = try(each.value.has_discussions, null) + is_template = try(each.value.is_template, false) + default_branch = try(each.value.default_branch, "") + archived = try(each.value.archived, false) + topics = try(each.value.topics, []) + archive_on_destroy = try(each.value.archive_on_destroy, null) + pages = try(contains(keys(each.value), "pages") && try(each.value.pages != null, false) ? { + branch = try(each.value.pages.branch, "gh-pages") + path = try(each.value.pages.path, "/") + cname = try(each.value.pages.cname, null) + build_type = try(each.value.pages.build_type, null) + } : null) + vulnerability_alerts = try(each.value.vulnerability_alerts_enabled, null) + + squash_merge_commit_title = try(each.value.squash_merge_commit_title, null) squash_merge_commit_message = try(each.value.squash_merge_commit_message, null) - merge_commit_title = try(each.value.merge_commit_title, null) - merge_commit_message = try(each.value.merge_commit_message, null) + merge_commit_title = try(each.value.merge_commit_title, null) + merge_commit_message = try(each.value.merge_commit_message, null) web_commit_signoff_required = try(each.value.web_commit_signoff_required, null) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -199,33 +200,33 @@ module "repository" { # Repository Creation Configuration # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - auto_init = try(each.value.auto_init, true) - gitignore_template = try(each.value.gitignore_template, "") - license_template = try(each.value.license_template, "") - template = try(contains(keys(each.value), "template") && try(each.value.template != null, false) ? { - owner = try(each.value.template.owner, "") - repository = try(each.value.template.repository, "") - } : null) + auto_init = try(each.value.auto_init, true) + gitignore_template = try(each.value.gitignore_template, "") + license_template = try(each.value.license_template, "") + template = try(contains(keys(each.value), "template") && try(each.value.template != null, false) ? { + owner = try(each.value.template.owner, "") + repository = try(each.value.template.repository, "") + } : null) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Teams Configuration # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - pull_teams = try([for i in each.value.pull_teams : data.github_team.team[i].id], []) - push_teams = try([for i in each.value.push_teams : data.github_team.team[i].id], []) - admin_teams = try([for i in each.value.admin_teams : data.github_team.team[i].id], []) - maintain_teams = try([for i in each.value.maintain_teams : data.github_team.team[i].id], []) - triage_teams = try([for i in each.value.triage_teams : data.github_team.team[i].id], []) + pull_teams = try([for i in each.value.pull_teams : data.github_team.team[i].id], []) + push_teams = try([for i in each.value.push_teams : data.github_team.team[i].id], []) + admin_teams = try([for i in each.value.admin_teams : data.github_team.team[i].id], []) + maintain_teams = try([for i in each.value.maintain_teams : data.github_team.team[i].id], []) + triage_teams = try([for i in each.value.triage_teams : data.github_team.team[i].id], []) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Collaborator Configuration # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - pull_collaborators = try(each.value.pull_collaborators, []) - push_collaborators = try(each.value.push_collaborators, []) - admin_collaborators = try(each.value.admin_collaborators, []) - maintain_collaborators = try(each.value.maintain_collaborators, []) - triage_collaborators = try(each.value.triage_collaborators, []) + pull_collaborators = try(each.value.pull_collaborators, []) + push_collaborators = try(each.value.push_collaborators, []) + admin_collaborators = try(each.value.admin_collaborators, []) + maintain_collaborators = try(each.value.maintain_collaborators, []) + triage_collaborators = try(each.value.triage_collaborators, []) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # Branches Configuration @@ -244,16 +245,16 @@ module "repository" { # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ branch_protections_v4 = try([ for branch_protection in try(each.value.branch_protections_v4, []) : { - pattern = branch_protection.pattern - allows_deletions = try(branch_protection.allows_deletions, false) - allows_force_pushes = try(branch_protection.allows_force_pushes, false) - force_push_bypassers = try([for bypasser in branch_protection.force_push_bypassers : (!startswith(bypasser, "app/") ? bypasser : data.github_app.app[bypasser].node_id)], []) - enforce_admins = try(branch_protection.enforce_admins, true) - lock_branch = try(branch_protection.lock_branch, null) + pattern = branch_protection.pattern + allows_deletions = try(branch_protection.allows_deletions, false) + allows_force_pushes = try(branch_protection.allows_force_pushes, false) + force_push_bypassers = try([for bypasser in branch_protection.force_push_bypassers : (!startswith(bypasser, "app/") ? bypasser : data.github_app.app[bypasser].node_id)], []) + enforce_admins = try(branch_protection.enforce_admins, true) + lock_branch = try(branch_protection.lock_branch, null) - restricts_pushes = try(branch_protection.restricts_pushes, false) - blocks_creations = try(branch_protection.blocks_creations, false) - push_restrictions = try([for bypasser in branch_protection.push_restrictions : (!startswith(bypasser, "app/") ? bypasser : data.github_app.app[bypasser].node_id)], []) + restricts_pushes = try(branch_protection.restricts_pushes, false) + blocks_creations = try(branch_protection.blocks_creations, false) + push_restrictions = try([for bypasser in branch_protection.push_restrictions : (!startswith(bypasser, "app/") ? bypasser : data.github_app.app[bypasser].node_id)], []) require_conversation_resolution = try(branch_protection.require_conversation_resolution, false) require_signed_commits = try(branch_protection.require_signed_commits, false) @@ -302,15 +303,64 @@ module "repository" { # App Installations # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# app_installations = try(each.value.app_installations, []) + # app_installations = try(each.value.app_installations, []) + + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + # Environments Configuration + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + environments = try(each.value.environments, []) +} + +# --------------------------------------------------------------------------------------------------------------------- +# Environment Management Strategy: +# +# 1. IMPORT (from importer_tmp_dir/): +# - These environments already exist in GitHub +# - Use import blocks to bring them into Terraform state +# - After import, move YAML to repos/ for ongoing management +# +# 2. CREATE (from repos/): +# - These are new environments or changes to existing repos +# - Terraform will create/update them without import blocks +# - No import needed - Terraform manages the full lifecycle +# +# --------------------------------------------------------------------------------------------------------------------- + +# Import environments from generated_repos (importer_tmp_dir/) only +# These should already exist in GitHub and need to be imported +locals { + flattened_generated_environments = flatten([ + for repo, config in local.generated_repos : [ + for environment in try(config.environments, []) : { + repository = repo + environment = environment + } + ] + ]) + + generated_environments_map = { + for item in local.flattened_generated_environments : + "${item.repository}:${item.environment.environment}" => item + } +} + +import { + for_each = local.generated_environments_map + + to = module.repository[each.value.repository].github_repository_environment.environment[each.value.environment.environment] + id = "${each.value.repository}:${each.value.environment.environment}" } +# Environments from new_repos (repos/ directory) are NOT imported +# They will be created/updated by Terraform as regular resources + locals { new_rulesets_flattened = flatten([ for repo, config in local.new_repos : [ for ruleset in try(config.rulesets, []) : { - repository = repo - ruleset = ruleset + repository = repo + ruleset = ruleset } ] ]) @@ -323,8 +373,8 @@ locals { generated_rulesets_flattened = flatten([ for repo, config in local.generated_repos : [ for ruleset in try(config.rulesets, []) : { - repository = repo - ruleset = ruleset + repository = repo + ruleset = ruleset } ] ]) @@ -339,8 +389,8 @@ locals { import { for_each = local.generated_rulesets_map - to = github_repository_ruleset.ruleset[each.key] - id = format("%s:%s", each.value.repository, each.value.ruleset.id) + to = github_repository_ruleset.ruleset[each.key] + id = format("%s:%s", each.value.repository, each.value.ruleset.id) } locals { @@ -380,7 +430,7 @@ locals { data "github_team" "ruleset_team" { for_each = toset(local.team_bypass_actors) - slug = each.value + slug = each.value } locals { @@ -393,8 +443,8 @@ locals { resource "github_repository_ruleset" "ruleset" { depends_on = [module.repository] - for_each = local.all_rulesets_map - name = each.value.ruleset.name + for_each = local.all_rulesets_map + name = each.value.ruleset.name enforcement = each.value.ruleset.enforcement target = each.value.ruleset.target repository = each.value.repository @@ -478,9 +528,9 @@ resource "github_repository_ruleset" "ruleset" { dynamic "required_status_checks" { for_each = ( - contains(keys(each.value.ruleset.rules), "required_status_checks") && - try(each.value.ruleset.rules.required_status_checks != null, false) && - length(try(each.value.ruleset.rules.required_status_checks.required_check, [])) > 0 + contains(keys(each.value.ruleset.rules), "required_status_checks") && + try(each.value.ruleset.rules.required_status_checks != null, false) && + length(try(each.value.ruleset.rules.required_status_checks.required_check, [])) > 0 ) ? [each.value.ruleset.rules.required_status_checks] : [] content { @@ -515,7 +565,7 @@ resource "github_repository_ruleset" "ruleset" { contains(keys(each.value.ruleset.rules), "required_code_scanning") && try(each.value.ruleset.rules.required_code_scanning != null, false) && length(try(each.value.ruleset.rules.required_code_scanning.required_code_scanning_tool, [])) > 0 - ? [each.value.ruleset.rules.required_code_scanning] # Only one block for `required_code_scanning` + ? [each.value.ruleset.rules.required_code_scanning] # Only one block for `required_code_scanning` : [] ) @@ -524,8 +574,8 @@ resource "github_repository_ruleset" "ruleset" { for_each = try(each.value.ruleset.rules.required_code_scanning.required_code_scanning_tool, []) content { - tool = required_code_scanning_tool.value.tool - alerts_threshold = required_code_scanning_tool.value.alerts_threshold + tool = required_code_scanning_tool.value.tool + alerts_threshold = required_code_scanning_tool.value.alerts_threshold security_alerts_threshold = required_code_scanning_tool.value.security_alerts_threshold } } @@ -537,13 +587,13 @@ resource "github_repository_ruleset" "ruleset" { for_each = try(each.value.ruleset.bypass_actors, []) content { - actor_id = startswith(bypass_actors.value.name, "team/") ? data.github_team.ruleset_team[replace(bypass_actors.value.name, "team/", "")].id : ( + actor_id = startswith(bypass_actors.value.name, "team/") ? data.github_team.ruleset_team[replace(bypass_actors.value.name, "team/", "")].id : ( startswith(bypass_actors.value.name, "app/") ? local.apps_map[bypass_actors.value.name].app_id : local.ruleset_actors[bypass_actors.value.name].actor_id ) - actor_type = startswith(bypass_actors.value.name, "team/") ? "Team" : ( + actor_type = startswith(bypass_actors.value.name, "team/") ? "Team" : ( startswith(bypass_actors.value.name, "app/") ? "Integration" : local.ruleset_actors[bypass_actors.value.name].actor_type ) - bypass_mode = try(bypass_actors.value.bypass_mode, "always") + bypass_mode = try(bypass_actors.value.bypass_mode, "always") } } } diff --git a/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf b/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf index ef54ccc..1a092c1 100644 --- a/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf +++ b/feature/github-repo-provisioning/modules/terraform-github-repository/main.tf @@ -156,11 +156,11 @@ resource "github_repository" "repository" { ] } - squash_merge_commit_title = local.squash_merge_commit_title - squash_merge_commit_message = local.squash_merge_commit_message - merge_commit_title = local.merge_commit_title - merge_commit_message = local.merge_commit_message - web_commit_signoff_required = local.web_commit_signoff_required + squash_merge_commit_title = local.squash_merge_commit_title + squash_merge_commit_message = local.squash_merge_commit_message + merge_commit_title = local.merge_commit_title + merge_commit_message = local.merge_commit_message + web_commit_signoff_required = local.web_commit_signoff_required } # --------------------------------------------------------------------------------------------------------------------- @@ -249,15 +249,15 @@ resource "github_branch_protection" "branch_protection" { dynamic "restrict_pushes" { for_each = var.branch_protections_v4[each.value].restricts_pushes ? try([var.branch_protections_v4[each.value]], []) : [] content { - blocks_creations = try(var.branch_protections_v4[each.value].blocks_creations, true) - push_allowances = try(var.branch_protections_v4[each.value].push_restrictions, []) + blocks_creations = try(var.branch_protections_v4[each.value].blocks_creations, true) + push_allowances = try(var.branch_protections_v4[each.value].push_restrictions, []) } } - force_push_bypassers = try(var.branch_protections_v4[each.value].force_push_bypassers, []) - allows_force_pushes = try(var.branch_protections_v4[each.value].allows_force_pushes, null) - allows_deletions = try(var.branch_protections_v4[each.value].allows_deletions, null) - lock_branch = try(var.branch_protections_v4[each.value].lock_branch, null) + force_push_bypassers = try(var.branch_protections_v4[each.value].force_push_bypassers, []) + allows_force_pushes = try(var.branch_protections_v4[each.value].allows_force_pushes, null) + allows_deletions = try(var.branch_protections_v4[each.value].allows_deletions, null) + lock_branch = try(var.branch_protections_v4[each.value].lock_branch, null) } # --------------------------------------------------------------------------------------------------------------------- @@ -591,3 +591,134 @@ resource "github_app_installation_repository" "app_installation_repository" { repository = github_repository.repository.name installation_id = each.value } + +# --------------------------------------------------------------------------------------------------------------------- +# Repository Environments +# --------------------------------------------------------------------------------------------------------------------- + +locals { + # Create environments map + environments_map = { for e in var.environments : e.environment => e } + + # Flatten all usernames across all environments for lookup + all_reviewer_usernames = distinct(flatten([ + for env in var.environments : + try(env.reviewers.users, []) + ])) + + # Flatten all team slugs across all environments for lookup + all_reviewer_team_slugs = distinct(flatten([ + for env in var.environments : + try(env.reviewers.teams, []) + ])) +} + +# Data sources to resolve usernames to user IDs +data "github_user" "reviewer" { + for_each = toset(local.all_reviewer_usernames) + username = each.value +} + +# Data sources to resolve team slugs to team IDs +data "github_team" "reviewer" { + for_each = toset(local.all_reviewer_team_slugs) + slug = each.value +} + +resource "github_repository_environment" "environment" { + for_each = local.environments_map + + environment = each.key + repository = github_repository.repository.name + wait_timer = try(each.value.wait_timer, null) + can_admins_bypass = try(each.value.can_admins_bypass, true) + prevent_self_review = try(each.value.prevent_self_review, false) + + dynamic "reviewers" { + for_each = try(each.value.reviewers, null) != null ? [each.value.reviewers] : [] + + content { + # Convert team slugs to team IDs + teams = try(reviewers.value.teams, null) != null ? [ + for team_slug in reviewers.value.teams : data.github_team.reviewer[team_slug].id + ] : null + + # Convert usernames to user IDs + users = try(reviewers.value.users, null) != null ? [ + for username in reviewers.value.users : data.github_user.reviewer[username].id + ] : null + } + } + + dynamic "deployment_branch_policy" { + for_each = ( + try(each.value.deployment_policy.policy_type, null) != null ? ( + # If policy_type is "protected_branches", use protected branches only + try(each.value.deployment_policy.policy_type, "") == "protected_branches" ? [{ + protected_branches = true + custom_branch_policies = false + }] : + # If policy_type is "selected_branches_and_tags", use custom policies + try(each.value.deployment_policy.policy_type, "") == "selected_branches_and_tags" ? [{ + protected_branches = false + custom_branch_policies = true + }] : [] + ) : [] + ) + + content { + protected_branches = deployment_branch_policy.value.protected_branches + custom_branch_policies = deployment_branch_policy.value.custom_branch_policies + } + } +} + +# Create deployment branch policies for custom branch patterns +resource "github_repository_environment_deployment_policy" "branch_policies" { + # Create a policy for each branch pattern in each environment that has custom policies + for_each = { + for item in flatten([ + for env_name, env in local.environments_map : [ + for branch_pattern in ( + try(env.deployment_policy.policy_type, "") == "selected_branches_and_tags" ? + try(env.deployment_policy.branch_patterns, []) : [] + ) : { + key = "${env_name}:branch:${branch_pattern}" + env = env_name + pattern = branch_pattern + } + ] + ]) : item.key => item + } + + repository = github_repository.repository.name + environment = github_repository_environment.environment[each.value.env].environment + branch_pattern = each.value.pattern + + depends_on = [github_repository_environment.environment] +} + +# Create deployment tag policies for custom tag patterns +resource "github_repository_environment_deployment_policy" "tag_policies" { + # Create a policy for each tag pattern in each environment that has custom policies + for_each = { + for item in flatten([ + for env_name, env in local.environments_map : [ + for tag_pattern in ( + try(env.deployment_policy.policy_type, "") == "selected_branches_and_tags" ? + try(env.deployment_policy.tag_patterns, []) : [] + ) : { + key = "${env_name}:tag:${tag_pattern}" + env = env_name + pattern = tag_pattern + } + ] + ]) : item.key => item + } + + repository = github_repository.repository.name + environment = github_repository_environment.environment[each.value.env].environment + tag_pattern = each.value.pattern + + depends_on = [github_repository_environment.environment] +} diff --git a/feature/github-repo-provisioning/modules/terraform-github-repository/outputs.tf b/feature/github-repo-provisioning/modules/terraform-github-repository/outputs.tf index d0c02a9..de0a87e 100644 --- a/feature/github-repo-provisioning/modules/terraform-github-repository/outputs.tf +++ b/feature/github-repo-provisioning/modules/terraform-github-repository/outputs.tf @@ -84,6 +84,21 @@ output "app_installations" { description = "A map of deploy app installations keyed by installation id." } +output "environments" { + value = github_repository_environment.environment + description = "A map of repository environments keyed by environment name." +} + +output "environment_deployment_branch_policies" { + value = github_repository_environment_deployment_policy.branch_policies + description = "A map of environment deployment branch policies." +} + +output "environment_deployment_tag_policies" { + value = github_repository_environment_deployment_policy.tag_policies + description = "A map of environment deployment tag policies." +} + # ---------------------------------------------------------------------------------------------------------------------- # OUTPUT MODULE CONFIGURATION # ---------------------------------------------------------------------------------------------------------------------- diff --git a/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf b/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf index f4a899b..8d9a9f6 100644 --- a/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf +++ b/feature/github-repo-provisioning/modules/terraform-github-repository/variables.tf @@ -127,9 +127,9 @@ variable "merge_commit_title" { } variable "web_commit_signoff_required" { - description = "(Optional) Set to true to require commit signoff for all commits pushed to the repository. (Default: null)" - type = bool - default = null + description = "(Optional) Set to true to require commit signoff for all commits pushed to the repository. (Default: null)" + type = bool + default = null } variable "merge_commit_message" { @@ -598,6 +598,68 @@ variable "app_installations" { default = [] } +variable "environments" { + type = any + description = "(Optional) Configure repository environments with deployment protection rules and reviewers." + # type = list(object({ + # environment = string + # wait_timer = optional(number) + # can_admins_bypass = optional(bool) + # prevent_self_review = optional(bool) + # reviewers = optional(object({ + # teams = optional(list(string)) + # users = optional(list(string)) + # })) + # deployment_policy = optional(object({ + # policy_type = string # "protected_branches" or "selected_branches_and_tags" + # branch_patterns = optional(list(string)) # Only for selected_branches_and_tags + # tag_patterns = optional(list(string)) # Only for selected_branches_and_tags + # })) + # })) + + default = [] + + # Examples: + # environments = [ + # # Example 1: Protected branches only + # { + # environment = "production" + # wait_timer = 300 # seconds (5 minutes) + # can_admins_bypass = false + # prevent_self_review = true + # reviewers = { + # teams = ["platform-team"] + # users = ["octocat", "hubot"] + # } + # deployment_policy = { + # policy_type = "protected_branches" + # } + # }, + # + # # Example 2: Selected branches and tags + # { + # environment = "staging" + # wait_timer = 60 + # can_admins_bypass = true + # prevent_self_review = false + # reviewers = { + # users = ["developer1"] + # } + # deployment_policy = { + # policy_type = "selected_branches_and_tags" + # branch_patterns = ["main", "release/*", "hotfix/*"] + # tag_patterns = ["v*", "release-*"] + # } + # }, + # + # # Example 3: Any branch can deploy (no restrictions) + # { + # environment = "development" + # # No deployment_policy = any branch can deploy + # } + # ] +} + # ------------------------------------------------------------------------------ # MODULE CONFIGURATION PARAMETERS # These variables are used to configure the module.