diff --git a/README.md b/README.md index b0d5fcce..2ea82b87 100644 --- a/README.md +++ b/README.md @@ -200,6 +200,7 @@ aws-secret-key | aws access key bucket | aws bucket | | BUCKET | s3-endpoint | Custom S3 endpoint. | | S3_ENDPOINT | s3-region | region of the s3 bucket | eu-west-1 | S3_REGION | +s3-credentials-type | S3 credential mode (`legacy` or `default-sdk-credential-chain`) | legacy | S3_CREDENTIALS_TYPE | s3-no-multipart | disables s3 multipart upload | false | S3_NO_MULTIPART | s3-path-style | Forces path style URLs, required for Minio. | false | S3_PATH_STYLE | storj-access | Access for the project | | STORJ_ACCESS | @@ -339,6 +340,12 @@ For the usage with a AWS S3 Bucket, you just need to specify the following optio - bucket _(either via flag or environment variable `BUCKET`)_ - s3-region _(either via flag or environment variable `S3_REGION`)_ +The `legacy` credential type is the default and requires both static credential options. +To use the AWS SDK default credential chain, explicitly set +`--s3-credentials-type default-sdk-credential-chain` (or +`S3_CREDENTIALS_TYPE=default-sdk-credential-chain`). The SDK chain supports environment credentials, shared AWS +configuration files, ECS task roles, EC2 instance profiles, and EKS IRSA. + If you specify the s3-region, you don't need to set the endpoint URL since the correct endpoint will used automatically.
diff --git a/cmd/cmd.go b/cmd/cmd.go index 76a2e886..d56bf922 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -137,6 +137,12 @@ var globalFlags = []cli.Flag{ Value: "eu-west-1", EnvVars: []string{"S3_REGION"}, }, + &cli.StringFlag{ + Name: "s3-credentials-type", + Usage: "legacy|default-sdk-credential-chain", + Value: storage.S3CredentialsTypeLegacy, + EnvVars: []string{"S3_CREDENTIALS_TYPE"}, + }, &cli.StringFlag{ Name: "aws-access-key", Usage: "", @@ -484,13 +490,9 @@ func New() *Cmd { switch provider := c.String("provider"); provider { case "s3": - if accessKey := c.String("aws-access-key"); accessKey == "" { - return errors.New("access-key not set") - } else if secretKey := c.String("aws-secret-key"); secretKey == "" { - return errors.New("secret-key not set") - } else if bucket := c.String("bucket"); bucket == "" { + if bucket := c.String("bucket"); bucket == "" { return errors.New("bucket not set") - } else if store, err := storage.NewS3Storage(c.Context, accessKey, secretKey, bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil { + } else if store, err := storage.NewS3Storage(c.Context, c.String("s3-credentials-type"), c.String("aws-access-key"), c.String("aws-secret-key"), bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil { return err } else { options = append(options, server.UseStorage(store)) diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go new file mode 100644 index 00000000..d3568288 --- /dev/null +++ b/cmd/cmd_test.go @@ -0,0 +1,23 @@ +package cmd + +import ( + "testing" + + "github.com/dutchcoders/transfer.sh/server/storage" + "github.com/urfave/cli/v2" +) + +func TestS3CredentialsTypeDefaultsToLegacy(t *testing.T) { + for _, flag := range globalFlags { + stringFlag, ok := flag.(*cli.StringFlag) + if !ok || stringFlag.Name != "s3-credentials-type" { + continue + } + if stringFlag.Value != storage.S3CredentialsTypeLegacy { + t.Fatalf("s3-credentials-type default = %q, want %q", stringFlag.Value, storage.S3CredentialsTypeLegacy) + } + return + } + + t.Fatal("s3-credentials-type flag not found") +} diff --git a/k8s/transfer.sh/README.md b/k8s/transfer.sh/README.md index eb4ad547..5002cce5 100644 --- a/k8s/transfer.sh/README.md +++ b/k8s/transfer.sh/README.md @@ -46,6 +46,12 @@ persistence: Compatible with AWS S3 and any S3-compatible storage (MinIO, Ceph, etc.). +The `legacy` credential type is the default and requires static credentials. Set +`transfersh.s3.credentialsType: default-sdk-credential-chain` to use the AWS SDK +default credential chain with EC2 instance profiles, ECS task roles, or EKS IAM +roles for service accounts (IRSA). For IRSA, annotate the chart's service account +with the role ARN and set `serviceAccount.automount: true`. + **Using a Kubernetes Secret (recommended):** ```bash @@ -315,6 +321,7 @@ gatewayApi: | `transfersh.randomTokenLength` | `6` | Length of the random token in file URLs | | `transfersh.local.basedir` | `/data` | Base directory for local storage | | `transfersh.s3.bucket` | `""` | S3 bucket name | +| `transfersh.s3.credentialsType` | `legacy` | Credential mode: `legacy` or `default-sdk-credential-chain` | | `transfersh.s3.region` | `eu-west-1` | S3 region | | `transfersh.s3.endpoint` | `""` | Custom S3 endpoint (MinIO, etc.) | | `transfersh.s3.pathStyle` | `false` | Force path-style URLs (required for MinIO) | diff --git a/k8s/transfer.sh/templates/configmap.yaml b/k8s/transfer.sh/templates/configmap.yaml index e5292813..617f6542 100644 --- a/k8s/transfer.sh/templates/configmap.yaml +++ b/k8s/transfer.sh/templates/configmap.yaml @@ -27,6 +27,7 @@ data: {{- with .Values.transfersh.s3 }} BUCKET: {{ .bucket | quote }} S3_REGION: {{ .region | quote }} + S3_CREDENTIALS_TYPE: {{ .credentialsType | quote }} {{- if .endpoint }} S3_ENDPOINT: {{ .endpoint | quote }} {{- end }} diff --git a/k8s/transfer.sh/templates/deployment.yaml b/k8s/transfer.sh/templates/deployment.yaml index 8f960c1b..09d63777 100644 --- a/k8s/transfer.sh/templates/deployment.yaml +++ b/k8s/transfer.sh/templates/deployment.yaml @@ -53,7 +53,7 @@ spec: {{- $s3 := .Values.transfersh.s3 }} {{- $storj := .Values.transfersh.storj }} {{- $httpAuth := .Values.transfersh.httpAuth }} - {{- $s3Creds := and (eq .Values.transfersh.provider "s3") (or $s3.existingSecret $s3.accessKey $s3.secretKey) }} + {{- $s3Creds := and (eq .Values.transfersh.provider "s3") (eq $s3.credentialsType "legacy") (or $s3.existingSecret $s3.accessKey $s3.secretKey) }} {{- $storjCreds := and (eq .Values.transfersh.provider "storj") (or $storj.existingSecret $storj.access) }} {{- $httpAuthCreds := and $httpAuth.enabled (not $httpAuth.htpasswd) }} {{- if or $s3Creds $storjCreds $httpAuthCreds }} diff --git a/k8s/transfer.sh/templates/secret.yaml b/k8s/transfer.sh/templates/secret.yaml index 09bd1eba..c4f04a31 100644 --- a/k8s/transfer.sh/templates/secret.yaml +++ b/k8s/transfer.sh/templates/secret.yaml @@ -1,7 +1,7 @@ {{- $data := dict }} {{- if eq .Values.transfersh.provider "s3" }} {{- with .Values.transfersh.s3 }} -{{- if and (not .existingSecret) (or .accessKey .secretKey) }} +{{- if and (eq .credentialsType "legacy") (not .existingSecret) (or .accessKey .secretKey) }} {{- $_ := set $data "AWS_ACCESS_KEY" .accessKey }} {{- $_ := set $data "AWS_SECRET_KEY" .secretKey }} {{- end }} diff --git a/k8s/transfer.sh/values.yaml b/k8s/transfer.sh/values.yaml index 4a234c98..65a299c5 100644 --- a/k8s/transfer.sh/values.yaml +++ b/k8s/transfer.sh/values.yaml @@ -37,6 +37,8 @@ transfersh: # -- S3 storage settings (provider: s3) s3: bucket: "" + # Credential mode: legacy or default-sdk-credential-chain + credentialsType: "legacy" # Choose what region your provider is region: "eu-west-1" # Custom endpoint for S3-compatible storage (MinIO, etc.) diff --git a/server/storage/s3.go b/server/storage/s3.go index 6cc6e0af..fc4087ca 100644 --- a/server/storage/s3.go +++ b/server/storage/s3.go @@ -16,6 +16,11 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3/types" ) +const ( + S3CredentialsTypeLegacy = "legacy" + S3CredentialsTypeDefault = "default-sdk-credential-chain" +) + // S3Storage is a storage backed by AWS S3 type S3Storage struct { Storage @@ -27,8 +32,8 @@ type S3Storage struct { } // NewS3Storage is the factory for S3Storage -func NewS3Storage(ctx context.Context, accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) { - cfg, err := getAwsConfig(ctx, accessKey, secretKey) +func NewS3Storage(ctx context.Context, credentialsType, accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) { + cfg, err := getAwsConfig(ctx, credentialsType, accessKey, secretKey) if err != nil { return nil, err } @@ -179,16 +184,31 @@ func (s *S3Storage) Put(ctx context.Context, token string, filename string, read func (s *S3Storage) IsRangeSupported() bool { return true } -func getAwsConfig(ctx context.Context, accessKey, secretKey string) (aws.Config, error) { - return config.LoadDefaultConfig(ctx, - config.WithCredentialsProvider(credentials.StaticCredentialsProvider{ +func getAwsConfig(ctx context.Context, credentialsType, accessKey, secretKey string) (aws.Config, error) { + options := []func(*config.LoadOptions) error{ + config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired), + config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired), + } + + switch credentialsType { + case S3CredentialsTypeLegacy: + if accessKey == "" { + return aws.Config{}, errors.New("access-key not set") + } + if secretKey == "" { + return aws.Config{}, errors.New("secret-key not set") + } + options = append(options, config.WithCredentialsProvider(credentials.StaticCredentialsProvider{ Value: aws.Credentials{ AccessKeyID: accessKey, SecretAccessKey: secretKey, SessionToken: "", }, - }), - config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired), - config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired), - ) + })) + case S3CredentialsTypeDefault: + default: + return aws.Config{}, fmt.Errorf("unsupported S3 credentials type %q", credentialsType) + } + + return config.LoadDefaultConfig(ctx, options...) } diff --git a/server/storage/s3_test.go b/server/storage/s3_test.go new file mode 100644 index 00000000..c6fb0c60 --- /dev/null +++ b/server/storage/s3_test.go @@ -0,0 +1,69 @@ +package storage + +import ( + "context" + "testing" +) + +func TestGetAwsConfigUsesStaticCredentialsWhenProvided(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "environment-access-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "environment-secret-key") + t.Setenv("AWS_SESSION_TOKEN", "environment-session-token") + + cfg, err := getAwsConfig(context.Background(), S3CredentialsTypeLegacy, "configured-access-key", "configured-secret-key") + if err != nil { + t.Fatalf("getAwsConfig returned an error: %v", err) + } + + credentials, err := cfg.Credentials.Retrieve(context.Background()) + if err != nil { + t.Fatalf("retrieve credentials: %v", err) + } + if credentials.AccessKeyID != "configured-access-key" || credentials.SecretAccessKey != "configured-secret-key" || credentials.SessionToken != "" { + t.Fatalf("got credentials %q/%q, want configured static credentials and empty session token", credentials.AccessKeyID, credentials.SecretAccessKey) + } +} + +func TestGetAwsConfigUsesDefaultCredentialChain(t *testing.T) { + t.Setenv("AWS_ACCESS_KEY_ID", "environment-access-key") + t.Setenv("AWS_SECRET_ACCESS_KEY", "environment-secret-key") + t.Setenv("AWS_SESSION_TOKEN", "environment-session-token") + + cfg, err := getAwsConfig(context.Background(), S3CredentialsTypeDefault, "ignored-access-key", "ignored-secret-key") + if err != nil { + t.Fatalf("getAwsConfig returned an error: %v", err) + } + + credentials, err := cfg.Credentials.Retrieve(context.Background()) + if err != nil { + t.Fatalf("retrieve credentials: %v", err) + } + if credentials.AccessKeyID != "environment-access-key" || credentials.SecretAccessKey != "environment-secret-key" || credentials.SessionToken != "environment-session-token" { + t.Fatalf("got credentials %q/%q, want credentials from the default chain and session token", credentials.AccessKeyID, credentials.SecretAccessKey) + } +} + +func TestGetAwsConfigRejectsPartialStaticCredentials(t *testing.T) { + testCases := []struct { + name string + accessKey string + secretKey string + }{ + {name: "missing secret key", accessKey: "access-key"}, + {name: "missing access key", secretKey: "secret-key"}, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + if _, err := getAwsConfig(context.Background(), S3CredentialsTypeLegacy, testCase.accessKey, testCase.secretKey); err == nil { + t.Fatal("getAwsConfig returned no error for partial static credentials") + } + }) + } +} + +func TestGetAwsConfigRejectsUnsupportedCredentialsType(t *testing.T) { + if _, err := getAwsConfig(context.Background(), "unsupported", "", ""); err == nil { + t.Fatal("getAwsConfig returned no error for unsupported credentials type") + } +}