Support AWS default credential chain for S3

README.md

@@ -200,6 +200,7 @@ aws-secret-key | aws access key
 bucket | aws bucket                                                                                 |                               | BUCKET                        |
 s3-endpoint | Custom S3 endpoint.                                                                   |                               | S3_ENDPOINT                   |
 s3-region | region of the s3 bucket                                                                 | eu-west-1                     | S3_REGION                     |
+s3-credentials-type | S3 credential mode (`legacy` or `default-sdk-credential-chain`)                      | legacy                        | S3_CREDENTIALS_TYPE           |
 s3-no-multipart | disables s3 multipart upload                                                      | false                         | S3_NO_MULTIPART               |
 s3-path-style | Forces path style URLs, required for Minio.                                         | false                         | S3_PATH_STYLE                 |
 storj-access | Access for the project                                                               |                               | STORJ_ACCESS                  |
@@ -339,6 +340,12 @@ For the usage with a AWS S3 Bucket, you just need to specify the following optio
 - bucket _(either via flag or environment variable `BUCKET`)_
 - s3-region _(either via flag or environment variable `S3_REGION`)_
 
+The `legacy` credential type is the default and requires both static credential options.
+To use the AWS SDK default credential chain, explicitly set
+`--s3-credentials-type default-sdk-credential-chain` (or
+`S3_CREDENTIALS_TYPE=default-sdk-credential-chain`). The SDK chain supports environment credentials, shared AWS
+configuration files, ECS task roles, EC2 instance profiles, and EKS IRSA.
+
 If you specify the s3-region, you don't need to set the endpoint URL since the correct endpoint will used automatically.
 
 <br />

cmd/cmd.go

@@ -137,6 +137,12 @@ var globalFlags = []cli.Flag{
 		Value:   "eu-west-1",
 		EnvVars: []string{"S3_REGION"},
 	},
+	&cli.StringFlag{
+		Name:    "s3-credentials-type",
+		Usage:   "legacy|default-sdk-credential-chain",
+		Value:   storage.S3CredentialsTypeLegacy,
+		EnvVars: []string{"S3_CREDENTIALS_TYPE"},
+	},
 	&cli.StringFlag{
 		Name:    "aws-access-key",
 		Usage:   "",
@@ -484,13 +490,9 @@ func New() *Cmd {
 
 		switch provider := c.String("provider"); provider {
 		case "s3":
-			if accessKey := c.String("aws-access-key"); accessKey == "" {
-				return errors.New("access-key not set")
-			} else if secretKey := c.String("aws-secret-key"); secretKey == "" {
-				return errors.New("secret-key not set")
-			} else if bucket := c.String("bucket"); bucket == "" {
+			if bucket := c.String("bucket"); bucket == "" {
 				return errors.New("bucket not set")
-			} else if store, err := storage.NewS3Storage(c.Context, accessKey, secretKey, bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil {
+			} else if store, err := storage.NewS3Storage(c.Context, c.String("s3-credentials-type"), c.String("aws-access-key"), c.String("aws-secret-key"), bucket, purgeDays, c.String("s3-region"), c.String("s3-endpoint"), c.Bool("s3-no-multipart"), c.Bool("s3-path-style"), logger); err != nil {
 				return err
 			} else {
 				options = append(options, server.UseStorage(store))

cmd/cmd_test.go

@@ -0,0 +1,23 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/dutchcoders/transfer.sh/server/storage"
+	"github.com/urfave/cli/v2"
+)
+
+func TestS3CredentialsTypeDefaultsToLegacy(t *testing.T) {
+	for _, flag := range globalFlags {
+		stringFlag, ok := flag.(*cli.StringFlag)
+		if !ok || stringFlag.Name != "s3-credentials-type" {
+			continue
+		}
+		if stringFlag.Value != storage.S3CredentialsTypeLegacy {
+			t.Fatalf("s3-credentials-type default = %q, want %q", stringFlag.Value, storage.S3CredentialsTypeLegacy)
+		}
+		return
+	}
+
+	t.Fatal("s3-credentials-type flag not found")
+}

k8s/transfer.sh/README.md

@@ -46,6 +46,12 @@ persistence:
 
 Compatible with AWS S3 and any S3-compatible storage (MinIO, Ceph, etc.).
 
+The `legacy` credential type is the default and requires static credentials. Set
+`transfersh.s3.credentialsType: default-sdk-credential-chain` to use the AWS SDK
+default credential chain with EC2 instance profiles, ECS task roles, or EKS IAM
+roles for service accounts (IRSA). For IRSA, annotate the chart's service account
+with the role ARN and set `serviceAccount.automount: true`.
+
 **Using a Kubernetes Secret (recommended):**
 
 ```bash
@@ -315,6 +321,7 @@ gatewayApi:
 | `transfersh.randomTokenLength` | `6` | Length of the random token in file URLs |
 | `transfersh.local.basedir` | `/data` | Base directory for local storage |
 | `transfersh.s3.bucket` | `""` | S3 bucket name |
+| `transfersh.s3.credentialsType` | `legacy` | Credential mode: `legacy` or `default-sdk-credential-chain` |
 | `transfersh.s3.region` | `eu-west-1` | S3 region |
 | `transfersh.s3.endpoint` | `""` | Custom S3 endpoint (MinIO, etc.) |
 | `transfersh.s3.pathStyle` | `false` | Force path-style URLs (required for MinIO) |

k8s/transfer.sh/templates/configmap.yaml

@@ -27,6 +27,7 @@ data:
   {{- with .Values.transfersh.s3 }}
   BUCKET: {{ .bucket | quote }}
   S3_REGION: {{ .region | quote }}
+  S3_CREDENTIALS_TYPE: {{ .credentialsType | quote }}
   {{- if .endpoint }}
   S3_ENDPOINT: {{ .endpoint | quote }}
   {{- end }}

k8s/transfer.sh/templates/deployment.yaml

@@ -53,7 +53,7 @@ spec:
           {{- $s3 := .Values.transfersh.s3 }}
           {{- $storj := .Values.transfersh.storj }}
           {{- $httpAuth := .Values.transfersh.httpAuth }}
-          {{- $s3Creds := and (eq .Values.transfersh.provider "s3") (or $s3.existingSecret $s3.accessKey $s3.secretKey) }}
+          {{- $s3Creds := and (eq .Values.transfersh.provider "s3") (eq $s3.credentialsType "legacy") (or $s3.existingSecret $s3.accessKey $s3.secretKey) }}
           {{- $storjCreds := and (eq .Values.transfersh.provider "storj") (or $storj.existingSecret $storj.access) }}
           {{- $httpAuthCreds := and $httpAuth.enabled (not $httpAuth.htpasswd) }}
           {{- if or $s3Creds $storjCreds $httpAuthCreds }}

k8s/transfer.sh/templates/secret.yaml

@@ -1,7 +1,7 @@
 {{- $data := dict }}
 {{- if eq .Values.transfersh.provider "s3" }}
 {{- with .Values.transfersh.s3 }}
-{{- if and (not .existingSecret) (or .accessKey .secretKey) }}
+{{- if and (eq .credentialsType "legacy") (not .existingSecret) (or .accessKey .secretKey) }}
 {{- $_ := set $data "AWS_ACCESS_KEY" .accessKey }}
 {{- $_ := set $data "AWS_SECRET_KEY" .secretKey }}
 {{- end }}

k8s/transfer.sh/values.yaml

@@ -37,6 +37,8 @@ transfersh:
   # -- S3 storage settings (provider: s3)
   s3:
     bucket: ""
+    # Credential mode: legacy or default-sdk-credential-chain
+    credentialsType: "legacy"
     # Choose what region your provider is
     region: "eu-west-1"
     # Custom endpoint for S3-compatible storage (MinIO, etc.)

server/storage/s3.go

@@ -16,6 +16,11 @@ import (
 	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
+const (
+	S3CredentialsTypeLegacy  = "legacy"
+	S3CredentialsTypeDefault = "default-sdk-credential-chain"
+)
+
 // S3Storage is a storage backed by AWS S3
 type S3Storage struct {
 	Storage
@@ -27,8 +32,8 @@ type S3Storage struct {
 }
 
 // NewS3Storage is the factory for S3Storage
-func NewS3Storage(ctx context.Context, accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) {
-	cfg, err := getAwsConfig(ctx, accessKey, secretKey)
+func NewS3Storage(ctx context.Context, credentialsType, accessKey, secretKey, bucketName string, purgeDays int, region, endpoint string, disableMultipart bool, forcePathStyle bool, logger *log.Logger) (*S3Storage, error) {
+	cfg, err := getAwsConfig(ctx, credentialsType, accessKey, secretKey)
 	if err != nil {
 		return nil, err
 	}
@@ -179,16 +184,31 @@ func (s *S3Storage) Put(ctx context.Context, token string, filename string, read
 
 func (s *S3Storage) IsRangeSupported() bool { return true }
 
-func getAwsConfig(ctx context.Context, accessKey, secretKey string) (aws.Config, error) {
-	return config.LoadDefaultConfig(ctx,
-		config.WithCredentialsProvider(credentials.StaticCredentialsProvider{
+func getAwsConfig(ctx context.Context, credentialsType, accessKey, secretKey string) (aws.Config, error) {
+	options := []func(*config.LoadOptions) error{
+		config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired),
+		config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired),
+	}
+
+	switch credentialsType {
+	case S3CredentialsTypeLegacy:
+		if accessKey == "" {
+			return aws.Config{}, errors.New("access-key not set")
+		}
+		if secretKey == "" {
+			return aws.Config{}, errors.New("secret-key not set")
+		}
+		options = append(options, config.WithCredentialsProvider(credentials.StaticCredentialsProvider{
 			Value: aws.Credentials{
 				AccessKeyID:     accessKey,
 				SecretAccessKey: secretKey,
 				SessionToken:    "",
 			},
-		}),
-		config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired),
-		config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired),
-	)
+		}))
+	case S3CredentialsTypeDefault:
+	default:
+		return aws.Config{}, fmt.Errorf("unsupported S3 credentials type %q", credentialsType)
+	}
+
+	return config.LoadDefaultConfig(ctx, options...)
 }

server/storage/s3_test.go

@@ -0,0 +1,69 @@
+package storage
+
+import (
+	"context"
+	"testing"
+)
+
+func TestGetAwsConfigUsesStaticCredentialsWhenProvided(t *testing.T) {
+	t.Setenv("AWS_ACCESS_KEY_ID", "environment-access-key")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "environment-secret-key")
+	t.Setenv("AWS_SESSION_TOKEN", "environment-session-token")
+
+	cfg, err := getAwsConfig(context.Background(), S3CredentialsTypeLegacy, "configured-access-key", "configured-secret-key")
+	if err != nil {
+		t.Fatalf("getAwsConfig returned an error: %v", err)
+	}
+
+	credentials, err := cfg.Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("retrieve credentials: %v", err)
+	}
+	if credentials.AccessKeyID != "configured-access-key" || credentials.SecretAccessKey != "configured-secret-key" || credentials.SessionToken != "" {
+		t.Fatalf("got credentials %q/%q, want configured static credentials and empty session token", credentials.AccessKeyID, credentials.SecretAccessKey)
+	}
+}
+
+func TestGetAwsConfigUsesDefaultCredentialChain(t *testing.T) {
+	t.Setenv("AWS_ACCESS_KEY_ID", "environment-access-key")
+	t.Setenv("AWS_SECRET_ACCESS_KEY", "environment-secret-key")
+	t.Setenv("AWS_SESSION_TOKEN", "environment-session-token")
+
+	cfg, err := getAwsConfig(context.Background(), S3CredentialsTypeDefault, "ignored-access-key", "ignored-secret-key")
+	if err != nil {
+		t.Fatalf("getAwsConfig returned an error: %v", err)
+	}
+
+	credentials, err := cfg.Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatalf("retrieve credentials: %v", err)
+	}
+	if credentials.AccessKeyID != "environment-access-key" || credentials.SecretAccessKey != "environment-secret-key" || credentials.SessionToken != "environment-session-token" {
+		t.Fatalf("got credentials %q/%q, want credentials from the default chain and session token", credentials.AccessKeyID, credentials.SecretAccessKey)
+	}
+}
+
+func TestGetAwsConfigRejectsPartialStaticCredentials(t *testing.T) {
+	testCases := []struct {
+		name      string
+		accessKey string
+		secretKey string
+	}{
+		{name: "missing secret key", accessKey: "access-key"},
+		{name: "missing access key", secretKey: "secret-key"},
+	}
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			if _, err := getAwsConfig(context.Background(), S3CredentialsTypeLegacy, testCase.accessKey, testCase.secretKey); err == nil {
+				t.Fatal("getAwsConfig returned no error for partial static credentials")
+			}
+		})
+	}
+}
+
+func TestGetAwsConfigRejectsUnsupportedCredentialsType(t *testing.T) {
+	if _, err := getAwsConfig(context.Background(), "unsupported", "", ""); err == nil {
+		t.Fatal("getAwsConfig returned no error for unsupported credentials type")
+	}
+}