From 5977ec889be5fbcea23f77af7560adad4a8e734f Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 26 Mar 2026 14:16:47 +0000 Subject: [PATCH 1/6] RM-251: user & guild exports Signed-off-by: Ben --- cmd/main.go | 66 ++++- go.mod | 38 +-- go.sum | 108 +++---- i18n/messages.go | 4 + internal/callback/callback.go | 89 +++++- internal/config/config.go | 24 +- internal/export/zip.go | 57 ++++ internal/gdprrelay/gdprrelay.go | 2 + internal/processor/processor.go | 493 +++++++++++++++++++++++++++++++- internal/utils/utils.go | 4 + 10 files changed, 775 insertions(+), 110 deletions(-) create mode 100644 internal/export/zip.go diff --git a/cmd/main.go b/cmd/main.go index d8a299f..760611e 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "os" "os/signal" "syscall" @@ -16,7 +17,11 @@ import ( "github.com/TicketsBot-cloud/gdpr-worker/internal/heartbeat" "github.com/TicketsBot-cloud/gdpr-worker/internal/processor" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" + "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" "github.com/go-redis/redis/v8" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -61,14 +66,57 @@ func main() { return } - logger.Info("Initializing archiver client") + logger.Info("Initialising archiver client") archiver.Initialize( logger.With(), config.Conf.Archiver.Url, config.Conf.Archiver.AesKey, ) - proc := processor.New(logger.With()) + // Initialise S3 client for export operations + var s3Client *s3client.S3Client + if config.Conf.S3.Endpoint != "" { + logger.Info("Initialising S3 client") + minioClient, err := minio.New(config.Conf.S3.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(config.Conf.S3.AccessKey, config.Conf.S3.SecretKey, ""), + Secure: config.Conf.S3.Secure, + }) + if err != nil { + logger.Fatal("Failed to initialise S3 client", zap.Error(err)) + return + } + s3Client = s3client.NewS3Client(minioClient, config.Conf.S3.Bucket) + logger.Info("S3 client initialised") + } else { + logger.Warn("S3 not configured, export features will be unavailable") + } + + // Initialise cache database connection for user export operations + var cachePool *pgxpool.Pool + if config.Conf.CacheDatabase.Host != "" { + logger.Info("Connecting to cache database") + cacheUri := fmt.Sprintf("postgres://%s:%s@%s/%s?pool_max_conns=%d", + config.Conf.CacheDatabase.Username, + config.Conf.CacheDatabase.Password, + config.Conf.CacheDatabase.Host, + config.Conf.CacheDatabase.Database, + config.Conf.CacheDatabase.Threads, + ) + + var err error + cachePool, err = pgxpool.Connect(context.Background(), cacheUri) + if err != nil { + logger.Fatal("Failed to connect to cache database", zap.Error(err)) + return + } + logger.Info("Connected to cache database") + } else { + logger.Warn("Cache database not configured, user export cache data will be unavailable") + } + + aesKey := []byte(config.Conf.Archiver.AesKey) + + proc := processor.New(logger.With(), s3Client, aesKey, cachePool) callbackHandler := callback.New( logger.With(), @@ -142,12 +190,14 @@ func main() { } callbackData := callback.ResultData{ - TranscriptsDeleted: result.TranscriptsDeleted, - MessagesDeleted: result.MessagesDeleted, - Error: result.Error, - RequestType: req.Request.Type, - GuildIds: req.Request.GuildIds, - TicketIds: req.Request.TicketIds, + TranscriptsDeleted: result.TranscriptsDeleted, + MessagesDeleted: result.MessagesDeleted, + Error: result.Error, + RequestType: req.Request.Type, + GuildIds: req.Request.GuildIds, + TicketIds: req.Request.TicketIds, + ExportData: result.ExportData, + ExportFileName: result.ExportFileName, } callbackCtx, callbackCancel := context.WithTimeout(context.Background(), 30*time.Second) diff --git a/go.mod b/go.mod index 6dfd03e..634fd13 100644 --- a/go.mod +++ b/go.mod @@ -1,37 +1,39 @@ module github.com/TicketsBot-cloud/gdpr-worker -go 1.24.0 +go 1.25.0 //replace github.com/TicketsBot-cloud/database => ../database -//replace github.com/TicketsBot-cloud/gdl => ../gdl +replace github.com/TicketsBot-cloud/gdl => ../gdl //replace github.com/TicketsBot-cloud/archiverclient => ./archiverclient -//replace github.com/TicketsBot-cloud/logarchiver => ../logarchiver +replace github.com/TicketsBot-cloud/logarchiver => ../logarchiver require ( github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704 - github.com/TicketsBot-cloud/database v0.0.0-20251018202538-7f9567e1aeab - github.com/TicketsBot-cloud/gdl v0.0.0-20251007163257-7e59b92d02dd + github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e + github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b + github.com/TicketsBot-cloud/gdl v0.0.0-20260306134952-cccb0116fef6 github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf github.com/caarlos0/env/v10 v10.0.0 github.com/go-redis/redis/v8 v8.11.5 github.com/jackc/pgx/v4 v4.18.3 github.com/joho/godotenv v1.5.1 - go.uber.org/zap v1.27.0 + github.com/minio/minio-go/v7 v7.0.99 + go.uber.org/zap v1.27.1 + golang.org/x/sync v0.20.0 ) require ( - github.com/TicketsBot-cloud/common v0.0.0-20250509064208-a2d357175463 // indirect github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 // indirect github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 // indirect + github.com/boltdb/bolt v1.3.1 // indirect github.com/caarlos0/env v3.5.0+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.14.3 // indirect @@ -41,16 +43,16 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgtype v1.14.4 // indirect github.com/jackc/pgx v3.6.2+incompatible // indirect - github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle v1.3.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/ratelimit v1.0.1 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.5 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.95 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pasztorpisti/qs v0.0.0-20171216220353-8d6c33ee906c // indirect @@ -59,12 +61,12 @@ require ( github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect - github.com/tinylib/msgp v1.4.0 // indirect - go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.43.0 // indirect + github.com/tinylib/msgp v1.6.3 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect - golang.org/x/net v0.46.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/text v0.30.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect ) diff --git a/go.sum b/go.sum index f76ed9d..eaacc9f 100644 --- a/go.sum +++ b/go.sum @@ -1,32 +1,18 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/ReneKroon/ttlcache v1.6.0/go.mod h1:DG6nbhXKUQhrExfwwLuZUdH7UnRDDRA1IW+nBuCssvs= -github.com/TicketsBot-cloud/archiverclient v0.0.0-20250514201416-cf23f65eb3fc h1:2EFgb6gQMedsO7YkqUPH1zY8+L6i3sGuV/YAv0gPqhQ= -github.com/TicketsBot-cloud/archiverclient v0.0.0-20250514201416-cf23f65eb3fc/go.mod h1:Mux1bEPpOHwRw1wo6Fa6qJLJH9Erk9qv1yAIfLi1Wmw= github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704 h1:liLfvCrzoJ89DXFHzsd1iK3cyP8s4i0CnZPRFEj53zg= github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704/go.mod h1:Mux1bEPpOHwRw1wo6Fa6qJLJH9Erk9qv1yAIfLi1Wmw= -github.com/TicketsBot-cloud/common v0.0.0-20250208132851-d5083bb04d98 h1:HwtXrqSv6y5E8mTnkOtISTaQr6dsK/lfUC8tUA6SzSU= -github.com/TicketsBot-cloud/common v0.0.0-20250208132851-d5083bb04d98/go.mod h1:iiZhl7w5DeTEGzAeq/hzani8+ILQz60g4JFkOy2vkuM= -github.com/TicketsBot-cloud/common v0.0.0-20250509064208-a2d357175463 h1:ZO2kw9lpsy4umoWFmAt9WKVFb3WvU5oSif2Yw4K7Lj4= -github.com/TicketsBot-cloud/common v0.0.0-20250509064208-a2d357175463/go.mod h1:PL5j/omFvU0NeyTKCESmOF+3GscaEuM0aqmI4yFcCFY= -github.com/TicketsBot-cloud/database v0.0.0-20250918212912-4cc263bc1b41 h1:X+V7IZtmDbE9h5cGSKnhCuisBFZ14Suzsdjnk8UUN3A= -github.com/TicketsBot-cloud/database v0.0.0-20250918212912-4cc263bc1b41/go.mod h1:LPDEn9e5wccH7rq/pUlVcL3UhyLnnwdM2dhj0tp/ljo= -github.com/TicketsBot-cloud/database v0.0.0-20251018202538-7f9567e1aeab h1:J8loePxYfe0oSqwaW0Ub51ku2+/fPiekQxHBYwX5S5A= -github.com/TicketsBot-cloud/database v0.0.0-20251018202538-7f9567e1aeab/go.mod h1:LPDEn9e5wccH7rq/pUlVcL3UhyLnnwdM2dhj0tp/ljo= -github.com/TicketsBot-cloud/gdl v0.0.0-20250917180424-569348f7a55b h1:k4kIIhpD3m0tx8Sz8gefZH8DBbuAwbeJJdV0iRr/x5Y= -github.com/TicketsBot-cloud/gdl v0.0.0-20250917180424-569348f7a55b/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= -github.com/TicketsBot-cloud/gdl v0.0.0-20251007163257-7e59b92d02dd h1:C4GzdEYarK2V81oAarvRW2RA2wzRGRydCAzBKtb/lGU= -github.com/TicketsBot-cloud/gdl v0.0.0-20251007163257-7e59b92d02dd/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250514201320-d5141071a6eb h1:BapforRlvTfWP8MX8DTsxVM40oDgQorJVo/cnNGTaKU= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250514201320-d5141071a6eb/go.mod h1:pZqkzPNNTqnwKZvCT8kCaTHxrG7HJbxZV83S0p7mmzM= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf h1:InP2ht90XOkLJKLjVrsCYPJunxlLkdsorikgNRk35bQ= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf/go.mod h1:pZqkzPNNTqnwKZvCT8kCaTHxrG7HJbxZV83S0p7mmzM= -github.com/TicketsBot/common v0.0.0-20241104184641-e39c64bdcf3e h1:cYfBjPX/FhD/MCViBI2Wz2YlC2esLiTbDE65Qku2WVg= -github.com/TicketsBot/common v0.0.0-20241104184641-e39c64bdcf3e/go.mod h1:N7zwetwx8B3RK/ZajWwMroJSyv2ZJ+bIOZWv/z8DhaM= +github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e h1:nFKV7yEm8MWbCP7dtsJ88+agcxDUD0YKIotVHMVvytw= +github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e/go.mod h1:tGrTHFz09OM3eDWF+62hIi9ELpT4igCFi868FKSvKBg= +github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b h1:bHkfJWo8T/9TiHuYHxaOz8GAILIiKPugC1k3CzdOq/A= +github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b/go.mod h1:HQXAgmNSm7/FmBYwcsa6qpZqMrDhbLoEl+AyqFQ+RwY= github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 h1:FqC1KGOsmB+ikvbmDkyNQU6bGUWyfYq8Ip9r4KxTveY= github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1/go.mod h1:N7zwetwx8B3RK/ZajWwMroJSyv2ZJ+bIOZWv/z8DhaM= github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 h1:NHD5GB6cjlkpZFjC76Yli2S63/J2nhr8MuE6KlYJpQM= github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261/go.mod h1:2zPxDAN2TAPpxUPjxszjs3QFKreKrQh5al/R3cMXmYk= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs= github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y= github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA= @@ -54,17 +40,12 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -85,7 +66,6 @@ github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5W github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -96,7 +76,6 @@ github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= @@ -104,7 +83,6 @@ github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01C github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgtype v1.14.4 h1:fKuNiCumbKTAIxQwXfB/nsrnkEI6bPJrrSiMKgbJ2j8= github.com/jackc/pgtype v1.14.4/go.mod h1:aKeozOde08iifGosdJpz9MBZonJOUJxqNpPBcMJTlVA= @@ -117,17 +95,13 @@ github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgS github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= -github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -137,15 +111,13 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -170,10 +142,8 @@ github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo= -github.com/minio/minio-go/v7 v7.0.73/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= -github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= -github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -198,8 +168,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= @@ -225,10 +193,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tinylib/msgp v1.4.0 h1:SYOeDRiydzOw9kSiwdYp9UcBgPFtLU2WDHaJXyHruf8= -github.com/tinylib/msgp v1.4.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= +github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -241,14 +209,16 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -261,10 +231,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= -golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -281,17 +249,13 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= -golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -310,10 +274,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -329,10 +291,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= diff --git a/i18n/messages.go b/i18n/messages.go index 7a047cc..008ab8e 100644 --- a/i18n/messages.go +++ b/i18n/messages.go @@ -14,4 +14,8 @@ var ( GdprFollowupError MessageId = "gdpr.followup.error" GdprFollowupNoData MessageId = "gdpr.followup.no_data" GdprFollowupSuccess MessageId = "gdpr.followup.success" + GdprCompletedExportGuild MessageId = "gdpr.completed.export_guild" + GdprCompletedExportGuildMulti MessageId = "gdpr.completed.export_guild_multi" + GdprCompletedExportUser MessageId = "gdpr.completed.export_user" + GdprExportDmMessage MessageId = "gdpr.export.dm_message" ) diff --git a/internal/callback/callback.go b/internal/callback/callback.go index 2daab54..be4b84f 100644 --- a/internal/callback/callback.go +++ b/internal/callback/callback.go @@ -1,6 +1,7 @@ package callback import ( + "bytes" "context" "fmt" "strings" @@ -9,10 +10,11 @@ import ( "github.com/TicketsBot-cloud/gdl/objects/interaction/component" "github.com/TicketsBot-cloud/gdl/rest" "github.com/TicketsBot-cloud/gdl/rest/ratelimit" + "github.com/TicketsBot-cloud/gdl/rest/request" + "github.com/TicketsBot-cloud/gdpr-worker/i18n" "github.com/TicketsBot-cloud/gdpr-worker/internal/config" "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" - "github.com/TicketsBot-cloud/gdpr-worker/i18n" "go.uber.org/zap" ) @@ -24,6 +26,8 @@ type ResultData struct { RequestType gdprrelay.RequestType // Type of GDPR request that was processed GuildIds []uint64 // Guild IDs affected by this request TicketIds []int // Ticket IDs affected by this request + ExportData []byte // ZIP file bytes for export requests + ExportFileName string // Suggested filename for the export archive } type Callback struct { @@ -69,6 +73,19 @@ func (c *Callback) SendCompletion(ctx context.Context, request gdprrelay.GDPRReq return err } + // For export types, send the ZIP file via DM + isExportType := request.Type == gdprrelay.RequestTypeExportGuild || request.Type == gdprrelay.RequestTypeExportUser + if isExportType && result.Error == nil && len(result.ExportData) > 0 { + if err := c.sendExportViaDM(ctx, request, locale, result); err != nil { + c.logger.Error("Failed to send export via DM", + zap.Error(err), + zap.String("scrambled_user_id", scrambledUserId), + ) + return err + } + return nil + } + if err := c.sendEphemeralFollowup(ctx, request, locale, result); err != nil { if c.isTokenExpired(err) { return nil @@ -139,6 +156,21 @@ func (c *Callback) buildResultMessage(locale *i18n.Locale, result ResultData, gu } else { content = i18n.GetMessage(locale, i18n.GdprCompletedSpecificMessages, "Unknown", result.MessagesDeleted) } + + case gdprrelay.RequestTypeExportGuild: + if len(result.GuildIds) == 1 { + guildDisplay := utils.FormatGuildDisplay(result.GuildIds[0], guildNames) + content = i18n.GetMessage(locale, i18n.GdprCompletedExportGuild, guildDisplay) + } else { + guildDisplays := make([]string, len(result.GuildIds)) + for idx, guildId := range result.GuildIds { + guildDisplays[idx] = utils.FormatGuildDisplay(guildId, guildNames) + } + content = i18n.GetMessage(locale, i18n.GdprCompletedExportGuildMulti, strings.Join(guildDisplays, "\n* ")) + } + + case gdprrelay.RequestTypeExportUser: + content = i18n.GetMessage(locale, i18n.GdprCompletedExportUser) } if result.Error != nil { @@ -233,3 +265,58 @@ func (c *Callback) sendCompletionViaDM(ctx context.Context, request gdprrelay.GD return nil } + +// sendExportViaDM sends the data export ZIP file to the user via a direct message. +func (c *Callback) sendExportViaDM(ctx context.Context, req gdprrelay.GDPRRequest, locale *i18n.Locale, result ResultData) error { + scrambledUserId := utils.ScrambleUserId(req.UserId) + + if config.Conf.Discord.Token == "" { + c.logger.Error("Discord token not configured, cannot send export DM", + zap.String("scrambled_user_id", scrambledUserId), + ) + return fmt.Errorf("discord token not configured") + } + + dmChannel, err := rest.CreateDM(ctx, config.Conf.Discord.Token, c.rateLimiter, req.UserId) + if err != nil { + c.logger.Error("Failed to create DM channel for export", + zap.Error(err), + zap.String("scrambled_user_id", scrambledUserId), + ) + return fmt.Errorf("failed to create DM channel: %w", err) + } + + content := i18n.GetMessage(locale, i18n.GdprExportDmMessage) + + data := rest.CreateMessageData{ + Content: content, + Attachments: []request.Attachment{ + { + Id: 0, + FileName: result.ExportFileName, + File: request.File{ + ContentType: "application/zip", + Reader: bytes.NewReader(result.ExportData), + }, + }, + }, + } + + _, err = rest.CreateMessage(ctx, config.Conf.Discord.Token, c.rateLimiter, dmChannel.Id, data) + if err != nil { + c.logger.Error("Failed to send export DM with attachment", + zap.Error(err), + zap.String("scrambled_user_id", scrambledUserId), + zap.Uint64("channel_id", dmChannel.Id), + zap.String("export_file", result.ExportFileName), + ) + return fmt.Errorf("failed to send export DM: %w", err) + } + + c.logger.Info("Export sent via DM", + zap.String("scrambled_user_id", scrambledUserId), + zap.String("export_file", result.ExportFileName), + ) + + return nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 06ff6c1..da1d74f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,10 +6,10 @@ import ( ) type Config struct { - JsonLogs bool `env:"JSON_LOGS" envDefault:"false"` - LogLevel zapcore.Level `env:"LOG_LEVEL" envDefault:"info"` - MaxConcurrency int `env:"MAX_CONCURRENCY" envDefault:"1"` - MaxRetries int `env:"MAX_RETRIES" envDefault:"3"` + JsonLogs bool `env:"JSON_LOGS" envDefault:"false"` + LogLevel zapcore.Level `env:"LOG_LEVEL" envDefault:"info"` + MaxConcurrency int `env:"MAX_CONCURRENCY" envDefault:"1"` + MaxRetries int `env:"MAX_RETRIES" envDefault:"3"` Database struct { Host string `env:"HOST"` @@ -34,6 +34,22 @@ type Config struct { ProxyUrl string `env:"PROXY_URL"` Token string `env:"TOKEN"` } `envPrefix:"DISCORD_"` + + S3 struct { + Endpoint string `env:"ENDPOINT"` + AccessKey string `env:"ACCESS_KEY"` + SecretKey string `env:"SECRET_KEY"` + Bucket string `env:"BUCKET"` + Secure bool `env:"SECURE" envDefault:"true"` + } `envPrefix:"S3_"` + + CacheDatabase struct { + Host string `env:"HOST"` + Database string `env:"NAME"` + Username string `env:"USER"` + Password string `env:"PASSWORD"` + Threads int `env:"THREADS"` + } `envPrefix:"CACHE_"` } var Conf Config diff --git a/internal/export/zip.go b/internal/export/zip.go new file mode 100644 index 0000000..10f584b --- /dev/null +++ b/internal/export/zip.go @@ -0,0 +1,57 @@ +package export + +import ( + "archive/zip" + "bytes" + "fmt" +) + +// ZipBuilder provides an in-memory ZIP archive builder for constructing +// data export archives without writing to disk. +type ZipBuilder struct { + buf *bytes.Buffer + writer *zip.Writer + closed bool +} + +// NewZipBuilder creates a new ZipBuilder that writes to an in-memory buffer. +func NewZipBuilder() *ZipBuilder { + buf := new(bytes.Buffer) + return &ZipBuilder{ + buf: buf, + writer: zip.NewWriter(buf), + } +} + +// AddFile adds a file with the given name and data to the ZIP archive. +func (z *ZipBuilder) AddFile(name string, data []byte) error { + if z.closed { + return fmt.Errorf("zip builder is already closed") + } + + f, err := z.writer.Create(name) + if err != nil { + return fmt.Errorf("failed to create zip entry %q: %w", name, err) + } + + if _, err := f.Write(data); err != nil { + return fmt.Errorf("failed to write zip entry %q: %w", name, err) + } + + return nil +} + +// Close finalises the ZIP archive and returns the resulting bytes. +func (z *ZipBuilder) Close() ([]byte, error) { + if z.closed { + return nil, fmt.Errorf("zip builder is already closed") + } + + z.closed = true + + if err := z.writer.Close(); err != nil { + return nil, fmt.Errorf("failed to close zip writer: %w", err) + } + + return z.buf.Bytes(), nil +} diff --git a/internal/gdprrelay/gdprrelay.go b/internal/gdprrelay/gdprrelay.go index a29be0f..89809f8 100644 --- a/internal/gdprrelay/gdprrelay.go +++ b/internal/gdprrelay/gdprrelay.go @@ -20,6 +20,8 @@ const ( RequestTypeSpecificTranscripts // Delete specific transcript archives by ticket IDs RequestTypeAllMessages // Delete all ticket messages for specified guilds RequestTypeSpecificMessages // Delete specific ticket messages by ticket IDs + RequestTypeExportGuild // Export all transcript data for specified guilds + RequestTypeExportUser // Export all personal data for the requesting user ) // GDPRRequest represents a user's request to delete their data under GDPR regulations diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 8b7447e..62798b5 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -3,40 +3,61 @@ package processor import ( "context" "encoding/json" + "errors" "fmt" + "strconv" "strings" + "sync" "github.com/TicketsBot-cloud/archiverclient" + "github.com/TicketsBot-cloud/common/encryption" + "github.com/TicketsBot-cloud/gdl/cache" + "github.com/TicketsBot-cloud/gdl/objects/channel/message" "github.com/TicketsBot-cloud/gdl/rest" "github.com/TicketsBot-cloud/gdl/rest/ratelimit" "github.com/TicketsBot-cloud/gdpr-worker/internal/archiver" "github.com/TicketsBot-cloud/gdpr-worker/internal/config" "github.com/TicketsBot-cloud/gdpr-worker/internal/database" + "github.com/TicketsBot-cloud/gdpr-worker/internal/export" "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" + "github.com/TicketsBot-cloud/logarchiver/pkg/model" + v1 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v1" v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" + "github.com/TicketsBot-cloud/logarchiver/pkg/export/user" + "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" + "github.com/jackc/pgx/v4/pgxpool" "go.uber.org/zap" + "golang.org/x/sync/errgroup" ) -// Processor handles the execution of GDPR data deletion requests +// Processor handles the execution of GDPR data deletion and export requests type Processor struct { logger *zap.Logger rateLimiter *ratelimit.Ratelimiter + s3Client *s3client.S3Client + aesKey []byte + cachePool *pgxpool.Pool } -func New(logger *zap.Logger) *Processor { +func New(logger *zap.Logger, s3Client *s3client.S3Client, aesKey []byte, cachePool *pgxpool.Pool) *Processor { store := ratelimit.NewMemoryStore() return &Processor{ logger: logger, rateLimiter: ratelimit.NewRateLimiter(store, 0), + s3Client: s3Client, + aesKey: aesKey, + cachePool: cachePool, } } // ProcessResult contains the outcome of processing a GDPR request type ProcessResult struct { - TranscriptsDeleted int // Number of transcript archives deleted from archiver - MessagesDeleted int // Number of ticket messages deleted from database - Error error // Error if the processing failed, nil on success + TranscriptsDeleted int // Number of transcript archives deleted from archiver + MessagesDeleted int // Number of ticket messages deleted from database + Error error // Error if the processing failed, nil on success + ExportData []byte // ZIP file bytes for export requests + ExportFileName string // Suggested filename for the export archive } func (p *Processor) Process(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { @@ -49,6 +70,10 @@ func (p *Processor) Process(ctx context.Context, request gdprrelay.GDPRRequest) return p.processAllMessages(ctx, request) case gdprrelay.RequestTypeSpecificMessages: return p.processSpecificMessages(ctx, request) + case gdprrelay.RequestTypeExportGuild: + return p.processExportGuild(ctx, request) + case gdprrelay.RequestTypeExportUser: + return p.processExportUser(ctx, request) default: return ProcessResult{Error: fmt.Errorf("unknown GDPR request type: %d", request.Type)} } @@ -559,3 +584,461 @@ func (p *Processor) storeTranscript(ctx context.Context, guildId uint64, ticketI return archiver.Client.ImportTranscript(ctx, guildId, ticketId, data) } + +// processExportGuild exports all transcript data for the specified guilds into a ZIP archive. +func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { + if len(request.GuildIds) == 0 { + return ProcessResult{Error: fmt.Errorf("no server ID provided")} + } + + if p.s3Client == nil { + return ProcessResult{Error: fmt.Errorf("S3 client not configured, cannot process export")} + } + + scrambledUserId := utils.ScrambleUserId(request.UserId) + + if err := p.verifyAllGuildsOwnership(ctx, request.GuildIds, request.UserId); err != nil { + p.logger.Error("Guild ownership verification failed for export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: err} + } + + zipBuilder := export.NewZipBuilder() + + for _, guildId := range request.GuildIds { + keys, err := p.s3Client.GetAllKeysForGuild(ctx, guildId) + if err != nil { + p.logger.Error("Failed to list S3 keys for guild export", + zap.Uint64("guild_id", guildId), + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + continue + } + + if len(keys) == 0 { + p.logger.Info("No transcripts found for guild export", + zap.Uint64("guild_id", guildId), + zap.String("scrambled_user_id", scrambledUserId), + ) + continue + } + + // Use errgroup with bounded concurrency for parallel downloads + type transcriptEntry struct { + ticketId int + data []byte + } + + var mu sync.Mutex + var entries []transcriptEntry + + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(15) + + for _, key := range keys { + key := key + g.Go(func() error { + // Extract ticket ID from key format: {guildId}/{ticketId} + parts := strings.SplitN(key, "/", 2) + if len(parts) != 2 { + p.logger.Warn("Skipping S3 key with unexpected format", + zap.String("key", key), + zap.Uint64("guild_id", guildId), + ) + return nil + } + + ticketId, err := strconv.Atoi(parts[1]) + if err != nil { + p.logger.Warn("Skipping S3 key with non-numeric ticket ID", + zap.String("key", key), + zap.Uint64("guild_id", guildId), + zap.Error(err), + ) + return nil + } + + rawData, err := p.s3Client.GetTicket(gCtx, guildId, ticketId) + if err != nil { + if errors.Is(err, s3client.ErrTicketNotFound) { + p.logger.Debug("Transcript not found in S3, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + ) + return nil + } + p.logger.Warn("Failed to download transcript from S3", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + return nil + } + + decrypted, err := encryption.Decrypt(p.aesKey, rawData) + if err != nil { + p.logger.Warn("Failed to decrypt transcript, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + return nil + } + + var transcript v2.Transcript + version := model.GetVersion(decrypted) + switch version { + case model.V1: + var messages []message.Message + if err := json.Unmarshal(decrypted, &messages); err != nil { + p.logger.Warn("Failed to unmarshal V1 transcript, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + return nil + } + transcript = v1.ConvertToV2(messages) + case model.V2: + if err := json.Unmarshal(decrypted, &transcript); err != nil { + p.logger.Warn("Failed to unmarshal V2 transcript, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + return nil + } + default: + p.logger.Warn("Unknown transcript version, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Int("version", version.Int()), + ) + return nil + } + + prettyData, err := json.MarshalIndent(transcript, "", " ") + if err != nil { + p.logger.Warn("Failed to marshal transcript for export", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + return nil + } + + mu.Lock() + entries = append(entries, transcriptEntry{ticketId: ticketId, data: prettyData}) + mu.Unlock() + + return nil + }) + } + + if err := g.Wait(); err != nil { + p.logger.Error("Error during parallel transcript download", + zap.Uint64("guild_id", guildId), + zap.Error(err), + ) + } + + // Add all collected entries to the ZIP + for _, entry := range entries { + var fileName string + if len(request.GuildIds) > 1 { + fileName = fmt.Sprintf("%d/%d.json", guildId, entry.ticketId) + } else { + fileName = fmt.Sprintf("%d.json", entry.ticketId) + } + + if err := zipBuilder.AddFile(fileName, entry.data); err != nil { + p.logger.Error("Failed to add transcript to ZIP", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", entry.ticketId), + zap.Error(err), + ) + } + } + } + + zipData, err := zipBuilder.Close() + if err != nil { + return ProcessResult{Error: fmt.Errorf("failed to finalise export ZIP: %w", err)} + } + + var exportFileName string + if len(request.GuildIds) == 1 { + exportFileName = fmt.Sprintf("guild_%d.zip", request.GuildIds[0]) + } else { + exportFileName = "guild_export.zip" + } + + p.logger.Info("Guild export completed", + zap.String("scrambled_user_id", scrambledUserId), + zap.String("export_file", exportFileName), + zap.Int("zip_size_bytes", len(zipData)), + ) + + return ProcessResult{ + ExportData: zipData, + ExportFileName: exportFileName, + } +} + +// processExportUser exports all personal data for the requesting user into a ZIP archive. +func (p *Processor) processExportUser(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { + scrambledUserId := utils.ScrambleUserId(request.UserId) + + zipBuilder := export.NewZipBuilder() + + // Export database data + userData, err := user.GetUserData(database.Client, request.UserId) + if err != nil { + p.logger.Error("Failed to get user database data for export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to retrieve database data: %w", err)} + } + + dbJson, err := json.MarshalIndent(userData, "", " ") + if err != nil { + return ProcessResult{Error: fmt.Errorf("failed to serialise database data: %w", err)} + } + + if err := zipBuilder.AddFile("database.json", dbJson); err != nil { + return ProcessResult{Error: fmt.Errorf("failed to add database.json to export: %w", err)} + } + + // Export cache data if cache pool is available + if p.cachePool != nil { + pgCache := cache.NewPgCache(p.cachePool, cache.CacheOptions{ + Users: true, + Members: true, + }) + + cacheData, err := user.GetCacheData(&pgCache, request.UserId) + if err != nil { + p.logger.Warn("Failed to get user cache data for export, continuing without it", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + } else { + cacheJson, err := json.MarshalIndent(cacheData, "", " ") + if err != nil { + p.logger.Warn("Failed to serialise cache data", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + } else { + if err := zipBuilder.AddFile("cache.json", cacheJson); err != nil { + p.logger.Warn("Failed to add cache.json to export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + } + } + } + } + + // Collect all ticket IDs where the user participated or is the creator + transcriptIds := make(map[uint64][]int) + + // Query tickets where user is a participant + { + query := `SELECT participant.guild_id, participant.ticket_id FROM participant INNER JOIN tickets ON participant.guild_id = tickets.guild_id AND tickets.id = participant.ticket_id WHERE participant.user_id = $1 AND tickets.has_transcript='t' AND tickets.open='f';` + rows, err := database.Client.Participants.Query(ctx, query, request.UserId) + if err != nil { + p.logger.Error("Failed to query participant tickets for user export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to query participant tickets: %w", err)} + } + + for rows.Next() { + var guildId uint64 + var ticketId int + if err := rows.Scan(&guildId, &ticketId); err != nil { + p.logger.Warn("Failed to scan participant ticket row", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + continue + } + transcriptIds[guildId] = append(transcriptIds[guildId], ticketId) + } + rows.Close() + } + + // Query tickets where user is the creator + { + query := `SELECT guild_id, id FROM tickets WHERE user_id = $1 AND has_transcript='t' AND open='f';` + rows, err := database.Client.Tickets.Query(ctx, query, request.UserId) + if err != nil { + p.logger.Error("Failed to query user-created tickets for export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to query user tickets: %w", err)} + } + + for rows.Next() { + var guildId uint64 + var ticketId int + if err := rows.Scan(&guildId, &ticketId); err != nil { + p.logger.Warn("Failed to scan user ticket row", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + continue + } + transcriptIds[guildId] = append(transcriptIds[guildId], ticketId) + } + rows.Close() + } + + // Deduplicate ticket IDs per guild + for guildId, ticketIds := range transcriptIds { + seen := make(map[int]struct{}) + deduped := make([]int, 0, len(ticketIds)) + for _, id := range ticketIds { + if _, exists := seen[id]; !exists { + seen[id] = struct{}{} + deduped = append(deduped, id) + } + } + transcriptIds[guildId] = deduped + } + + // Download, decrypt, filter and add transcripts to ZIP + if p.s3Client != nil { + for guildId, ticketIds := range transcriptIds { + for _, ticketId := range ticketIds { + rawData, err := p.s3Client.GetTicket(ctx, guildId, ticketId) + if err != nil { + if errors.Is(err, s3client.ErrTicketNotFound) { + p.logger.Debug("Transcript not found for user export, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + ) + continue + } + p.logger.Warn("Failed to download transcript for user export", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } + + decrypted, err := encryption.Decrypt(p.aesKey, rawData) + if err != nil { + p.logger.Warn("Failed to decrypt transcript for user export, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } + + var transcript v2.Transcript + version := model.GetVersion(decrypted) + switch version { + case model.V1: + var messages []message.Message + if err := json.Unmarshal(decrypted, &messages); err != nil { + p.logger.Warn("Failed to unmarshal V1 transcript for user export, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } + transcript = v1.ConvertToV2(messages) + case model.V2: + if err := json.Unmarshal(decrypted, &transcript); err != nil { + p.logger.Warn("Failed to unmarshal V2 transcript for user export, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } + default: + p.logger.Warn("Unknown transcript version for user export, skipping", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Int("version", version.Int()), + ) + continue + } + + // Filter to only the user's data + transcript.Entities.Channels = nil + transcript.Entities.Roles = nil + + userEntity, ok := transcript.Entities.Users[request.UserId] + if !ok { + transcript.Entities.Users = nil + } else { + transcript.Entities.Users = map[uint64]v2.User{ + userEntity.Id: userEntity, + } + } + + var filteredMessages []v2.Message + for _, msg := range transcript.Messages { + if msg.AuthorId == request.UserId { + filteredMessages = append(filteredMessages, msg) + } + } + transcript.Messages = filteredMessages + + prettyData, err := json.MarshalIndent(transcript, "", " ") + if err != nil { + p.logger.Warn("Failed to marshal filtered transcript for user export", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } + + fileName := fmt.Sprintf("transcripts/%d-%d.json", guildId, ticketId) + if err := zipBuilder.AddFile(fileName, prettyData); err != nil { + p.logger.Error("Failed to add transcript to user export ZIP", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + } + } + } + } else { + p.logger.Warn("S3 client not configured, skipping transcript export for user", + zap.String("scrambled_user_id", scrambledUserId), + ) + } + + zipData, err := zipBuilder.Close() + if err != nil { + return ProcessResult{Error: fmt.Errorf("failed to finalise user export ZIP: %w", err)} + } + + exportFileName := fmt.Sprintf("user_%d.zip", request.UserId) + + p.logger.Info("User export completed", + zap.String("scrambled_user_id", scrambledUserId), + zap.String("export_file", exportFileName), + zap.Int("zip_size_bytes", len(zipData)), + ) + + return ProcessResult{ + ExportData: zipData, + ExportFileName: exportFileName, + } +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 9b54513..39c90b4 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -73,6 +73,10 @@ func GetRequestTypeName(requestType int) string { return "AllMessages" case 3: return "SpecificMessages" + case 4: + return "ExportGuild" + case 5: + return "ExportUser" default: return fmt.Sprintf("Unknown(%d)", requestType) } From e9aee44a5063922378c671a8036ae85a4aa1b9ea Mon Sep 17 00:00:00 2001 From: biast12 Date: Thu, 26 Mar 2026 22:07:43 +0100 Subject: [PATCH 2/6] Use archiver client for transcripts (remove S3/minio) Replace direct S3/MinIO access and local decryption with the archiver service client. Removed S3 client initialization and MinIO deps from cmd/main.go and dropped s3client, AES key and decryption/model-version handling from the processor. Processor now queries the tickets DB for transcript IDs and calls archiver.Client.Get to fetch transcripts (handling archiverclient.ErrNotFound), and the Processor constructor signature was simplified to only accept cachePool. Imports and error logs were adjusted accordingly. --- cmd/main.go | 25 +--- internal/processor/processor.go | 252 +++++++++----------------------- 2 files changed, 72 insertions(+), 205 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 760611e..2d987f7 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,11 +17,8 @@ import ( "github.com/TicketsBot-cloud/gdpr-worker/internal/heartbeat" "github.com/TicketsBot-cloud/gdpr-worker/internal/processor" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" - "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" "github.com/go-redis/redis/v8" "github.com/jackc/pgx/v4/pgxpool" - "github.com/minio/minio-go/v7" - "github.com/minio/minio-go/v7/pkg/credentials" "go.uber.org/zap" "go.uber.org/zap/zapcore" @@ -73,24 +70,6 @@ func main() { config.Conf.Archiver.AesKey, ) - // Initialise S3 client for export operations - var s3Client *s3client.S3Client - if config.Conf.S3.Endpoint != "" { - logger.Info("Initialising S3 client") - minioClient, err := minio.New(config.Conf.S3.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(config.Conf.S3.AccessKey, config.Conf.S3.SecretKey, ""), - Secure: config.Conf.S3.Secure, - }) - if err != nil { - logger.Fatal("Failed to initialise S3 client", zap.Error(err)) - return - } - s3Client = s3client.NewS3Client(minioClient, config.Conf.S3.Bucket) - logger.Info("S3 client initialised") - } else { - logger.Warn("S3 not configured, export features will be unavailable") - } - // Initialise cache database connection for user export operations var cachePool *pgxpool.Pool if config.Conf.CacheDatabase.Host != "" { @@ -114,9 +93,7 @@ func main() { logger.Warn("Cache database not configured, user export cache data will be unavailable") } - aesKey := []byte(config.Conf.Archiver.AesKey) - - proc := processor.New(logger.With(), s3Client, aesKey, cachePool) + proc := processor.New(logger.With(), cachePool) callbackHandler := callback.New( logger.With(), diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 62798b5..5e1ac54 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -3,16 +3,12 @@ package processor import ( "context" "encoding/json" - "errors" "fmt" - "strconv" "strings" "sync" "github.com/TicketsBot-cloud/archiverclient" - "github.com/TicketsBot-cloud/common/encryption" "github.com/TicketsBot-cloud/gdl/cache" - "github.com/TicketsBot-cloud/gdl/objects/channel/message" "github.com/TicketsBot-cloud/gdl/rest" "github.com/TicketsBot-cloud/gdl/rest/ratelimit" "github.com/TicketsBot-cloud/gdpr-worker/internal/archiver" @@ -21,11 +17,8 @@ import ( "github.com/TicketsBot-cloud/gdpr-worker/internal/export" "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" - "github.com/TicketsBot-cloud/logarchiver/pkg/model" - v1 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v1" v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" "github.com/TicketsBot-cloud/logarchiver/pkg/export/user" - "github.com/TicketsBot-cloud/logarchiver/pkg/s3client" "github.com/jackc/pgx/v4/pgxpool" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -35,18 +28,14 @@ import ( type Processor struct { logger *zap.Logger rateLimiter *ratelimit.Ratelimiter - s3Client *s3client.S3Client - aesKey []byte cachePool *pgxpool.Pool } -func New(logger *zap.Logger, s3Client *s3client.S3Client, aesKey []byte, cachePool *pgxpool.Pool) *Processor { +func New(logger *zap.Logger, cachePool *pgxpool.Pool) *Processor { store := ratelimit.NewMemoryStore() return &Processor{ logger: logger, rateLimiter: ratelimit.NewRateLimiter(store, 0), - s3Client: s3Client, - aesKey: aesKey, cachePool: cachePool, } } @@ -591,10 +580,6 @@ func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GD return ProcessResult{Error: fmt.Errorf("no server ID provided")} } - if p.s3Client == nil { - return ProcessResult{Error: fmt.Errorf("S3 client not configured, cannot process export")} - } - scrambledUserId := utils.ScrambleUserId(request.UserId) if err := p.verifyAllGuildsOwnership(ctx, request.GuildIds, request.UserId); err != nil { @@ -608,9 +593,10 @@ func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GD zipBuilder := export.NewZipBuilder() for _, guildId := range request.GuildIds { - keys, err := p.s3Client.GetAllKeysForGuild(ctx, guildId) + query := `SELECT id FROM tickets WHERE guild_id = $1 AND has_transcript = 't' AND open = 'f' ORDER BY id;` + rows, err := database.Client.Tickets.Query(ctx, query, guildId) if err != nil { - p.logger.Error("Failed to list S3 keys for guild export", + p.logger.Error("Failed to query tickets for guild export", zap.Uint64("guild_id", guildId), zap.String("scrambled_user_id", scrambledUserId), zap.Error(err), @@ -618,7 +604,21 @@ func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GD continue } - if len(keys) == 0 { + var ticketIds []int + for rows.Next() { + var id int + if err := rows.Scan(&id); err != nil { + p.logger.Warn("Failed to scan ticket ID for guild export", + zap.Uint64("guild_id", guildId), + zap.Error(err), + ) + continue + } + ticketIds = append(ticketIds, id) + } + rows.Close() + + if len(ticketIds) == 0 { p.logger.Info("No transcripts found for guild export", zap.Uint64("guild_id", guildId), zap.String("scrambled_user_id", scrambledUserId), @@ -638,49 +638,19 @@ func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GD g, gCtx := errgroup.WithContext(ctx) g.SetLimit(15) - for _, key := range keys { - key := key + for _, ticketId := range ticketIds { + ticketId := ticketId g.Go(func() error { - // Extract ticket ID from key format: {guildId}/{ticketId} - parts := strings.SplitN(key, "/", 2) - if len(parts) != 2 { - p.logger.Warn("Skipping S3 key with unexpected format", - zap.String("key", key), - zap.Uint64("guild_id", guildId), - ) - return nil - } - - ticketId, err := strconv.Atoi(parts[1]) - if err != nil { - p.logger.Warn("Skipping S3 key with non-numeric ticket ID", - zap.String("key", key), - zap.Uint64("guild_id", guildId), - zap.Error(err), - ) - return nil - } - - rawData, err := p.s3Client.GetTicket(gCtx, guildId, ticketId) + transcript, err := archiver.Client.Get(gCtx, guildId, ticketId) if err != nil { - if errors.Is(err, s3client.ErrTicketNotFound) { - p.logger.Debug("Transcript not found in S3, skipping", + if err == archiverclient.ErrNotFound { + p.logger.Debug("Transcript not found in archiver, skipping", zap.Uint64("guild_id", guildId), zap.Int("ticket_id", ticketId), ) return nil } - p.logger.Warn("Failed to download transcript from S3", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - return nil - } - - decrypted, err := encryption.Decrypt(p.aesKey, rawData) - if err != nil { - p.logger.Warn("Failed to decrypt transcript, skipping", + p.logger.Warn("Failed to download transcript from archiver", zap.Uint64("guild_id", guildId), zap.Int("ticket_id", ticketId), zap.Error(err), @@ -688,38 +658,6 @@ func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GD return nil } - var transcript v2.Transcript - version := model.GetVersion(decrypted) - switch version { - case model.V1: - var messages []message.Message - if err := json.Unmarshal(decrypted, &messages); err != nil { - p.logger.Warn("Failed to unmarshal V1 transcript, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - return nil - } - transcript = v1.ConvertToV2(messages) - case model.V2: - if err := json.Unmarshal(decrypted, &transcript); err != nil { - p.logger.Warn("Failed to unmarshal V2 transcript, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - return nil - } - default: - p.logger.Warn("Unknown transcript version, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Int("version", version.Int()), - ) - return nil - } - prettyData, err := json.MarshalIndent(transcript, "", " ") if err != nil { p.logger.Warn("Failed to marshal transcript for export", @@ -914,114 +852,66 @@ func (p *Processor) processExportUser(ctx context.Context, request gdprrelay.GDP transcriptIds[guildId] = deduped } - // Download, decrypt, filter and add transcripts to ZIP - if p.s3Client != nil { - for guildId, ticketIds := range transcriptIds { - for _, ticketId := range ticketIds { - rawData, err := p.s3Client.GetTicket(ctx, guildId, ticketId) - if err != nil { - if errors.Is(err, s3client.ErrTicketNotFound) { - p.logger.Debug("Transcript not found for user export, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - ) - continue - } - p.logger.Warn("Failed to download transcript for user export", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } - - decrypted, err := encryption.Decrypt(p.aesKey, rawData) - if err != nil { - p.logger.Warn("Failed to decrypt transcript for user export, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } - - var transcript v2.Transcript - version := model.GetVersion(decrypted) - switch version { - case model.V1: - var messages []message.Message - if err := json.Unmarshal(decrypted, &messages); err != nil { - p.logger.Warn("Failed to unmarshal V1 transcript for user export, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } - transcript = v1.ConvertToV2(messages) - case model.V2: - if err := json.Unmarshal(decrypted, &transcript); err != nil { - p.logger.Warn("Failed to unmarshal V2 transcript for user export, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } - default: - p.logger.Warn("Unknown transcript version for user export, skipping", + // Download, filter and add transcripts to ZIP + for guildId, ticketIds := range transcriptIds { + for _, ticketId := range ticketIds { + transcript, err := archiver.Client.Get(ctx, guildId, ticketId) + if err != nil { + if err == archiverclient.ErrNotFound { + p.logger.Debug("Transcript not found for user export, skipping", zap.Uint64("guild_id", guildId), zap.Int("ticket_id", ticketId), - zap.Int("version", version.Int()), ) continue } + p.logger.Warn("Failed to download transcript for user export", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } - // Filter to only the user's data - transcript.Entities.Channels = nil - transcript.Entities.Roles = nil + // Filter to only the user's data + transcript.Entities.Channels = nil + transcript.Entities.Roles = nil - userEntity, ok := transcript.Entities.Users[request.UserId] - if !ok { - transcript.Entities.Users = nil - } else { - transcript.Entities.Users = map[uint64]v2.User{ - userEntity.Id: userEntity, - } + userEntity, ok := transcript.Entities.Users[request.UserId] + if !ok { + transcript.Entities.Users = nil + } else { + transcript.Entities.Users = map[uint64]v2.User{ + userEntity.Id: userEntity, } + } - var filteredMessages []v2.Message - for _, msg := range transcript.Messages { - if msg.AuthorId == request.UserId { - filteredMessages = append(filteredMessages, msg) - } + var filteredMessages []v2.Message + for _, msg := range transcript.Messages { + if msg.AuthorId == request.UserId { + filteredMessages = append(filteredMessages, msg) } - transcript.Messages = filteredMessages + } + transcript.Messages = filteredMessages - prettyData, err := json.MarshalIndent(transcript, "", " ") - if err != nil { - p.logger.Warn("Failed to marshal filtered transcript for user export", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } + prettyData, err := json.MarshalIndent(transcript, "", " ") + if err != nil { + p.logger.Warn("Failed to marshal filtered transcript for user export", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) + continue + } - fileName := fmt.Sprintf("transcripts/%d-%d.json", guildId, ticketId) - if err := zipBuilder.AddFile(fileName, prettyData); err != nil { - p.logger.Error("Failed to add transcript to user export ZIP", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - } + fileName := fmt.Sprintf("transcripts/%d-%d.json", guildId, ticketId) + if err := zipBuilder.AddFile(fileName, prettyData); err != nil { + p.logger.Error("Failed to add transcript to user export ZIP", + zap.Uint64("guild_id", guildId), + zap.Int("ticket_id", ticketId), + zap.Error(err), + ) } } - } else { - p.logger.Warn("S3 client not configured, skipping transcript export for user", - zap.String("scrambled_user_id", scrambledUserId), - ) } zipData, err := zipBuilder.Close() From 70bc82d28894929f132407a7b19a61028748a154 Mon Sep 17 00:00:00 2001 From: biast12 <53872542+biast12@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:23:11 +0200 Subject: [PATCH 3/6] Bump Go builder to 1.25 Update the Docker build image from Go 1.24 to Go 1.25 so the project builds with the newer Go toolchain. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e980cba..f9d8d52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build container -FROM golang:1.24 AS builder +FROM golang:1.25 AS builder RUN go version From e08686e1ca08eb1e97df7e61bd15b79cbeb58e96 Mon Sep 17 00:00:00 2001 From: biast12 <53872542+biast12@users.noreply.github.com> Date: Thu, 10 Sep 2026 20:23:42 +0200 Subject: [PATCH 4/6] Bump gdl and logarchiver Refresh the Go module versions to newer releases of gdl and logarchiver and align the dependency metadata with the current build setup. This also disables the local path replace directives so the project resolves from the module proxy instead of the developer worktree, preventing accidental local override usage. --- go.mod | 12 ++++++------ go.sum | 4 ++++ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 634fd13..7cabcde 100644 --- a/go.mod +++ b/go.mod @@ -4,28 +4,27 @@ go 1.25.0 //replace github.com/TicketsBot-cloud/database => ../database -replace github.com/TicketsBot-cloud/gdl => ../gdl +//replace github.com/TicketsBot-cloud/gdl => ../gdl //replace github.com/TicketsBot-cloud/archiverclient => ./archiverclient -replace github.com/TicketsBot-cloud/logarchiver => ../logarchiver +//replace github.com/TicketsBot-cloud/logarchiver => ../logarchiver require ( github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704 - github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b - github.com/TicketsBot-cloud/gdl v0.0.0-20260306134952-cccb0116fef6 - github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf + github.com/TicketsBot-cloud/gdl v0.0.0-20260612070331-a3947b410d3e + github.com/TicketsBot-cloud/logarchiver v0.0.0-20260909063657-e32e33f1289b github.com/caarlos0/env/v10 v10.0.0 github.com/go-redis/redis/v8 v8.11.5 github.com/jackc/pgx/v4 v4.18.3 github.com/joho/godotenv v1.5.1 - github.com/minio/minio-go/v7 v7.0.99 go.uber.org/zap v1.27.1 golang.org/x/sync v0.20.0 ) require ( + github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e // indirect github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 // indirect github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 // indirect github.com/boltdb/bolt v1.3.1 // indirect @@ -53,6 +52,7 @@ require ( github.com/klauspost/crc32 v1.3.0 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/minio-go/v7 v7.0.99 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pasztorpisti/qs v0.0.0-20171216220353-8d6c33ee906c // indirect diff --git a/go.sum b/go.sum index eaacc9f..1c90c26 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,10 @@ github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e h1:nFKV7yE github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e/go.mod h1:tGrTHFz09OM3eDWF+62hIi9ELpT4igCFi868FKSvKBg= github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b h1:bHkfJWo8T/9TiHuYHxaOz8GAILIiKPugC1k3CzdOq/A= github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b/go.mod h1:HQXAgmNSm7/FmBYwcsa6qpZqMrDhbLoEl+AyqFQ+RwY= +github.com/TicketsBot-cloud/gdl v0.0.0-20260612070331-a3947b410d3e h1:qCibZQmO2rrBFrvn6+oAchPEQ7k/0Izr3ruL5P9aVZU= +github.com/TicketsBot-cloud/gdl v0.0.0-20260612070331-a3947b410d3e/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= +github.com/TicketsBot-cloud/logarchiver v0.0.0-20260909063657-e32e33f1289b h1:XbZOnvTk3xGrWA+gHIGU97Eabpm7MV7PRqrypIG6SWI= +github.com/TicketsBot-cloud/logarchiver v0.0.0-20260909063657-e32e33f1289b/go.mod h1:1pv+CJbf7yr1cBJVDILqLgE7iJJsoRC4zK3381a/8mM= github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 h1:FqC1KGOsmB+ikvbmDkyNQU6bGUWyfYq8Ip9r4KxTveY= github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1/go.mod h1:N7zwetwx8B3RK/ZajWwMroJSyv2ZJ+bIOZWv/z8DhaM= github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 h1:NHD5GB6cjlkpZFjC76Yli2S63/J2nhr8MuE6KlYJpQM= From 2705ceef981eeca5558c7676625b07029993ed81 Mon Sep 17 00:00:00 2001 From: biast12 <53872542+biast12@users.noreply.github.com> Date: Sat, 12 Sep 2026 01:07:46 +0200 Subject: [PATCH 5/6] Refactor GDPR exports into multipart delivery Reworks export processing to build bounded-size multi-part ZIP archives with manifest metadata, withheld-file tracking, and expanded user data sources. Export delivery now sends parts via DM in batches with retry/backoff and reports partial/failed delivery in callback messaging. Also adds export/cache config knobs, tighter required env parsing, DB pool creation improvements, proxy request hook support, and per-request processing timeouts. --- .env.example | 18 + cmd/main.go | 55 ++- i18n/messages.go | 2 + internal/callback/callback.go | 340 ++++++++++++------ internal/config/config.go | 43 ++- internal/database/database.go | 27 +- internal/export/manifest.go | 35 ++ internal/export/userdata.go | 170 +++++++++ internal/export/zip.go | 162 +++++++-- internal/processor/export_requests.go | 491 ++++++++++++++++++++++++++ internal/processor/processor.go | 373 +------------------ 11 files changed, 1170 insertions(+), 546 deletions(-) create mode 100644 internal/export/manifest.go create mode 100644 internal/export/userdata.go create mode 100644 internal/processor/export_requests.go diff --git a/.env.example b/.env.example index bf5d645..ce7a751 100644 --- a/.env.example +++ b/.env.example @@ -25,3 +25,21 @@ ARCHIVER_AES_KEY= # Discord Configuration DISCORD_PROXY_URL= DISCORD_TOKEN= + +# Cache Database Configuration (optional) +# When unset, exports omit cache.json and record the omission in the manifest. +CACHE_HOST= +CACHE_NAME= +CACHE_USER= +CACHE_PASSWORD= +CACHE_THREADS= + +# Data Export Configuration +# Discord caps a request at 25 MiB, an attachment at 20 MiB, and a message at 10 attachments. +# 12 MiB parts put two per message, using the request budget almost exactly. +EXPORT_MAX_PART_BYTES= +EXPORT_MAX_MESSAGE_BYTES= +EXPORT_MAX_ATTACHMENT_BYTES= +EXPORT_DM_DELAY= +EXPORT_TIMEOUT= +EXPORT_CONCURRENCY= diff --git a/cmd/main.go b/cmd/main.go index 2d987f7..2e5ee10 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,12 +2,14 @@ package main import ( "context" - "fmt" + "net/http" "os" "os/signal" + "strings" "syscall" "time" + "github.com/TicketsBot-cloud/gdl/rest/request" "github.com/TicketsBot-cloud/gdpr-worker/i18n" "github.com/TicketsBot-cloud/gdpr-worker/internal/archiver" "github.com/TicketsBot-cloud/gdpr-worker/internal/callback" @@ -25,6 +27,15 @@ import ( _ "github.com/joho/godotenv/autoload" ) +// The proxy expects the request addressed to its own host rather than the usual CONNECT protocol. +// Application endpoints stay on discord.com, matching how the bot worker registers the same hook. +func proxyHook(token string, req *http.Request) { + if !strings.HasPrefix(req.URL.Path, "/api/v9/applications/") { + req.URL.Scheme = "http" + req.URL.Host = config.Conf.Discord.ProxyUrl + } +} + func main() { config.Parse() @@ -37,10 +48,16 @@ func main() { return } + if config.Conf.Discord.ProxyUrl != "" { + logger.Info("Configuring REST proxy", zap.String("url", config.Conf.Discord.ProxyUrl)) + request.RegisterPreRequestHook(proxyHook) + } + logger.Info("Connecting to Redis") redisClient := redis.NewClient(&redis.Options{ Addr: config.Conf.Redis.Address, Password: config.Conf.Redis.Password, + PoolSize: config.Conf.Redis.Threads, DB: 0, }) @@ -70,20 +87,19 @@ func main() { config.Conf.Archiver.AesKey, ) - // Initialise cache database connection for user export operations var cachePool *pgxpool.Pool if config.Conf.CacheDatabase.Host != "" { logger.Info("Connecting to cache database") - cacheUri := fmt.Sprintf("postgres://%s:%s@%s/%s?pool_max_conns=%d", - config.Conf.CacheDatabase.Username, - config.Conf.CacheDatabase.Password, + + var err error + cachePool, err = database.NewPool( + context.Background(), config.Conf.CacheDatabase.Host, config.Conf.CacheDatabase.Database, + config.Conf.CacheDatabase.Username, + config.Conf.CacheDatabase.Password, config.Conf.CacheDatabase.Threads, ) - - var err error - cachePool, err = pgxpool.Connect(context.Background(), cacheUri) if err != nil { logger.Fatal("Failed to connect to cache database", zap.Error(err)) return @@ -120,7 +136,8 @@ func main() { <-semaphore }() - processCtx := context.Background() + processCtx, processCancel := context.WithTimeout(context.Background(), config.Conf.Export.Timeout) + defer processCancel() scrambledId := utils.ScrambleUserId(req.Request.UserId) requestTypeName := utils.GetRequestTypeName(int(req.Request.Type)) @@ -173,19 +190,21 @@ func main() { RequestType: req.Request.Type, GuildIds: req.Request.GuildIds, TicketIds: req.Request.TicketIds, - ExportData: result.ExportData, - ExportFileName: result.ExportFileName, + ExportParts: result.ExportParts, + ExportedFiles: result.ExportedFiles, } - callbackCtx, callbackCancel := context.WithTimeout(context.Background(), 30*time.Second) + callbackCtx, callbackCancel := context.WithTimeout(context.Background(), 10*time.Minute) defer callbackCancel() - if updateErr := database.Client.GdprLogs.UpdateLogStatus(req.RequestID, "Completed"); updateErr != nil { - logger.Error("Failed to update GDPR log status to Completed", - zap.Uint64("request_id", uint64(req.RequestID)), - zap.String("scrambled_user_id", scrambledId), - zap.Error(updateErr), - ) + if result.Error == nil { + if updateErr := database.Client.GdprLogs.UpdateLogStatus(req.RequestID, "Completed"); updateErr != nil { + logger.Error("Failed to update GDPR log status to Completed", + zap.Uint64("request_id", uint64(req.RequestID)), + zap.String("scrambled_user_id", scrambledId), + zap.Error(updateErr), + ) + } } if err := callbackHandler.SendCompletion(callbackCtx, req.Request, callbackData); err != nil { diff --git a/i18n/messages.go b/i18n/messages.go index 008ab8e..312dd69 100644 --- a/i18n/messages.go +++ b/i18n/messages.go @@ -18,4 +18,6 @@ var ( GdprCompletedExportGuildMulti MessageId = "gdpr.completed.export_guild_multi" GdprCompletedExportUser MessageId = "gdpr.completed.export_user" GdprExportDmMessage MessageId = "gdpr.export.dm_message" + GdprExportDmMessagePart MessageId = "gdpr.export.dm_message_part" + GdprErrorExportDmFailed MessageId = "gdpr.error.export_dm_failed" ) diff --git a/internal/callback/callback.go b/internal/callback/callback.go index be4b84f..efa367f 100644 --- a/internal/callback/callback.go +++ b/internal/callback/callback.go @@ -3,8 +3,10 @@ package callback import ( "bytes" "context" + "errors" "fmt" "strings" + "time" "github.com/TicketsBot-cloud/gdl/objects/channel/message" "github.com/TicketsBot-cloud/gdl/objects/interaction/component" @@ -13,11 +15,20 @@ import ( "github.com/TicketsBot-cloud/gdl/rest/request" "github.com/TicketsBot-cloud/gdpr-worker/i18n" "github.com/TicketsBot-cloud/gdpr-worker/internal/config" + "github.com/TicketsBot-cloud/gdpr-worker/internal/export" "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" "go.uber.org/zap" ) +const maxPartAttempts = 3 + +// Discord accepts at most ten attachments on a message. +const maxAttachmentsPerMessage = 10 + +// Fallback when the message budget is unset; Discord's request cap is 25 MiB. +const defaultMaxMessageBytes = 24 * 1024 * 1024 + // ResultData contains the result of a GDPR request to be sent back to the user type ResultData struct { TranscriptsDeleted int // Number of transcript archives deleted @@ -26,8 +37,12 @@ type ResultData struct { RequestType gdprrelay.RequestType // Type of GDPR request that was processed GuildIds []uint64 // Guild IDs affected by this request TicketIds []int // Ticket IDs affected by this request - ExportData []byte // ZIP file bytes for export requests - ExportFileName string // Suggested filename for the export archive + ExportParts []export.Part // Deliverable archives for export requests + ExportedFiles int // Number of files across those archives +} + +func (r ResultData) isExport() bool { + return r.RequestType == gdprrelay.RequestTypeExportGuild || r.RequestType == gdprrelay.RequestTypeExportUser } type Callback struct { @@ -44,26 +59,44 @@ func New(logger *zap.Logger, proxyUrl string) *Callback { } } -func (c *Callback) SendCompletion(ctx context.Context, request gdprrelay.GDPRRequest, result ResultData) error { - if request.InteractionToken == "" { +// The export is delivered before the interaction message is updated: that token expires after +// fifteen minutes, which is exactly the case a long export hits. +func (c *Callback) SendCompletion(ctx context.Context, req gdprrelay.GDPRRequest, result ResultData) error { + scrambledUserId := utils.ScrambleUserId(req.UserId) + locale := i18n.GetLocale(req.Language) + + var deliveryErr error + delivered := 0 + + if result.isExport() && result.Error == nil && len(result.ExportParts) > 0 { + delivered, deliveryErr = c.sendExportViaDM(ctx, req, locale, result) + if deliveryErr != nil { + c.logger.Error("Failed to deliver export via DM", + zap.Error(deliveryErr), + zap.String("scrambled_user_id", scrambledUserId), + zap.Int("delivered_parts", delivered), + zap.Int("total_parts", len(result.ExportParts)), + ) + } + } + + if req.InteractionToken == "" { c.logger.Debug("No interaction token, skipping callback") - return nil + return deliveryErr } - scrambledUserId := utils.ScrambleUserId(request.UserId) - locale := i18n.GetLocale(request.Language) - components := c.buildResultComponents(locale, result, request.GuildNames) + components := c.buildResultComponents(locale, result, req.GuildNames, deliveryErr, delivered) - if err := c.editOriginalMessage(ctx, request, components); err != nil { + if err := c.editOriginalMessage(ctx, req, components); err != nil { if c.isTokenExpired(err) { - if dmErr := c.sendCompletionViaDM(ctx, request, locale, result); dmErr != nil { + if dmErr := c.sendCompletionViaDM(ctx, req, components); dmErr != nil { c.logger.Error("Failed to send completion via DM", zap.Error(dmErr), zap.String("scrambled_user_id", scrambledUserId), ) return dmErr } - return nil + return deliveryErr } c.logger.Error("Failed to edit original message", @@ -73,31 +106,16 @@ func (c *Callback) SendCompletion(ctx context.Context, request gdprrelay.GDPRReq return err } - // For export types, send the ZIP file via DM - isExportType := request.Type == gdprrelay.RequestTypeExportGuild || request.Type == gdprrelay.RequestTypeExportUser - if isExportType && result.Error == nil && len(result.ExportData) > 0 { - if err := c.sendExportViaDM(ctx, request, locale, result); err != nil { - c.logger.Error("Failed to send export via DM", + if err := c.sendEphemeralFollowup(ctx, req, locale, result); err != nil { + if !c.isTokenExpired(err) { + c.logger.Error("Failed to send ephemeral follow-up", zap.Error(err), zap.String("scrambled_user_id", scrambledUserId), ) - return err } - return nil } - if err := c.sendEphemeralFollowup(ctx, request, locale, result); err != nil { - if c.isTokenExpired(err) { - return nil - } - - c.logger.Error("Failed to send ephemeral follow-up", - zap.Error(err), - zap.String("scrambled_user_id", scrambledUserId), - ) - } - - return nil + return deliveryErr } func (c *Callback) isTokenExpired(err error) bool { @@ -122,11 +140,7 @@ func (c *Callback) buildResultMessage(locale *i18n.Locale, result ResultData, gu guildDisplay := utils.FormatGuildDisplay(result.GuildIds[0], guildNames) content = i18n.GetMessage(locale, i18n.GdprCompletedAllTranscripts, guildDisplay, result.TranscriptsDeleted) } else { - guildDisplays := make([]string, len(result.GuildIds)) - for i, guildId := range result.GuildIds { - guildDisplays[i] = utils.FormatGuildDisplay(guildId, guildNames) - } - content = i18n.GetMessage(locale, i18n.GdprCompletedAllTranscriptsMulti, strings.Join(guildDisplays, "\n* "), result.TranscriptsDeleted) + content = i18n.GetMessage(locale, i18n.GdprCompletedAllTranscriptsMulti, joinGuildDisplays(result.GuildIds, guildNames), result.TranscriptsDeleted) } case gdprrelay.RequestTypeSpecificTranscripts: @@ -142,11 +156,7 @@ func (c *Callback) buildResultMessage(locale *i18n.Locale, result ResultData, gu guildDisplay := utils.FormatGuildDisplay(result.GuildIds[0], guildNames) content = i18n.GetMessage(locale, i18n.GdprCompletedAllMessages, guildDisplay, result.MessagesDeleted) } else { - guildDisplays := make([]string, len(result.GuildIds)) - for i, guildId := range result.GuildIds { - guildDisplays[i] = utils.FormatGuildDisplay(guildId, guildNames) - } - content = i18n.GetMessage(locale, i18n.GdprCompletedAllMessagesMulti, strings.Join(guildDisplays, "\n* "), result.MessagesDeleted) + content = i18n.GetMessage(locale, i18n.GdprCompletedAllMessagesMulti, joinGuildDisplays(result.GuildIds, guildNames), result.MessagesDeleted) } case gdprrelay.RequestTypeSpecificMessages: @@ -160,17 +170,13 @@ func (c *Callback) buildResultMessage(locale *i18n.Locale, result ResultData, gu case gdprrelay.RequestTypeExportGuild: if len(result.GuildIds) == 1 { guildDisplay := utils.FormatGuildDisplay(result.GuildIds[0], guildNames) - content = i18n.GetMessage(locale, i18n.GdprCompletedExportGuild, guildDisplay) + content = i18n.GetMessage(locale, i18n.GdprCompletedExportGuild, guildDisplay, result.ExportedFiles) } else { - guildDisplays := make([]string, len(result.GuildIds)) - for idx, guildId := range result.GuildIds { - guildDisplays[idx] = utils.FormatGuildDisplay(guildId, guildNames) - } - content = i18n.GetMessage(locale, i18n.GdprCompletedExportGuildMulti, strings.Join(guildDisplays, "\n* ")) + content = i18n.GetMessage(locale, i18n.GdprCompletedExportGuildMulti, joinGuildDisplays(result.GuildIds, guildNames), result.ExportedFiles) } case gdprrelay.RequestTypeExportUser: - content = i18n.GetMessage(locale, i18n.GdprCompletedExportUser) + content = i18n.GetMessage(locale, i18n.GdprCompletedExportUser, result.ExportedFiles) } if result.Error != nil { @@ -180,15 +186,29 @@ func (c *Callback) buildResultMessage(locale *i18n.Locale, result ResultData, gu return content } -func (c *Callback) buildResultComponents(locale *i18n.Locale, result ResultData, guildNames map[uint64]string) []component.Component { +func joinGuildDisplays(guildIds []uint64, guildNames map[uint64]string) string { + displays := make([]string, len(guildIds)) + for i, guildId := range guildIds { + displays[i] = utils.FormatGuildDisplay(guildId, guildNames) + } + + return strings.Join(displays, "\n* ") +} + +func (c *Callback) buildResultComponents(locale *i18n.Locale, result ResultData, guildNames map[uint64]string, deliveryErr error, delivered int) []component.Component { colour := utils.Green - if result.Error != nil { + content := c.buildResultMessage(locale, result, guildNames) + + if deliveryErr != nil { + colour = utils.Red + content = i18n.GetMessage(locale, i18n.GdprErrorExportDmFailed, delivered, len(result.ExportParts)) + } else if result.Error != nil { colour = utils.Red } innerComponents := []component.Component{ component.BuildTextDisplay(component.TextDisplay{ - Content: c.buildResultMessage(locale, result, guildNames), + Content: content, }), } @@ -210,9 +230,14 @@ func (c *Callback) editOriginalMessage(ctx context.Context, request gdprrelay.GD func (c *Callback) sendEphemeralFollowup(ctx context.Context, request gdprrelay.GDPRRequest, locale *i18n.Locale, result ResultData) error { var content string + nothingFound := result.TranscriptsDeleted == 0 && result.MessagesDeleted == 0 + if result.isExport() { + nothingFound = result.ExportedFiles == 0 + } + if result.Error != nil { content = i18n.GetMessage(locale, i18n.GdprFollowupError, result.Error.Error()) - } else if result.TranscriptsDeleted == 0 && result.MessagesDeleted == 0 { + } else if nothingFound { content = i18n.GetMessage(locale, i18n.GdprFollowupNoData) } else { content = i18n.GetMessage(locale, i18n.GdprFollowupSuccess) @@ -227,96 +252,195 @@ func (c *Callback) sendEphemeralFollowup(ctx context.Context, request gdprrelay. return err } -func (c *Callback) sendCompletionViaDM(ctx context.Context, request gdprrelay.GDPRRequest, locale *i18n.Locale, result ResultData) error { - scrambledUserId := utils.ScrambleUserId(request.UserId) - +func (c *Callback) openDM(ctx context.Context, userId uint64) (uint64, error) { if config.Conf.Discord.Token == "" { - c.logger.Error("Discord token not configured, cannot send DM", - zap.String("scrambled_user_id", scrambledUserId), - ) - return fmt.Errorf("discord token not configured") + return 0, fmt.Errorf("discord token not configured") } - dmChannel, err := rest.CreateDM(ctx, config.Conf.Discord.Token, c.rateLimiter, request.UserId) + channel, err := rest.CreateDM(ctx, config.Conf.Discord.Token, c.rateLimiter, userId) if err != nil { - c.logger.Error("Failed to create DM channel", - zap.Error(err), - zap.String("scrambled_user_id", scrambledUserId), - ) - return fmt.Errorf("failed to create DM channel: %w", err) + return 0, fmt.Errorf("failed to create DM channel: %w", err) } - components := c.buildResultComponents(locale, result, request.GuildNames) + return channel.Id, nil +} + +func (c *Callback) sendCompletionViaDM(ctx context.Context, request gdprrelay.GDPRRequest, components []component.Component) error { + channelId, err := c.openDM(ctx, request.UserId) + if err != nil { + return err + } data := rest.CreateMessageData{ Components: components, Flags: uint(message.FlagComponentsV2), } - _, err = rest.CreateMessage(ctx, config.Conf.Discord.Token, c.rateLimiter, dmChannel.Id, data) - if err != nil { - c.logger.Error("Failed to send DM message", - zap.Error(err), - zap.String("scrambled_user_id", scrambledUserId), - zap.Uint64("channel_id", dmChannel.Id), - ) + if _, err := rest.CreateMessage(ctx, config.Conf.Discord.Token, c.rateLimiter, channelId, data); err != nil { return fmt.Errorf("failed to send DM message: %w", err) } return nil } -// sendExportViaDM sends the data export ZIP file to the user via a direct message. -func (c *Callback) sendExportViaDM(ctx context.Context, req gdprrelay.GDPRRequest, locale *i18n.Locale, result ResultData) error { - scrambledUserId := utils.ScrambleUserId(req.UserId) +// The byte budget binds before the attachment count does whenever a part approaches the message +// limit, so a message carries several attachments only when the parts are small. +func batchParts(parts []export.Part, maxBytes int) [][]export.Part { + if maxBytes <= 0 { + maxBytes = defaultMaxMessageBytes + } - if config.Conf.Discord.Token == "" { - c.logger.Error("Discord token not configured, cannot send export DM", - zap.String("scrambled_user_id", scrambledUserId), - ) - return fmt.Errorf("discord token not configured") + var batches [][]export.Part + var current []export.Part + size := 0 + + for _, part := range parts { + tooMany := len(current) >= maxAttachmentsPerMessage + tooBig := size+len(part.Data) > maxBytes + + if len(current) > 0 && (tooMany || tooBig) { + batches = append(batches, current) + current, size = nil, 0 + } + + current = append(current, part) + size += len(part.Data) } - dmChannel, err := rest.CreateDM(ctx, config.Conf.Discord.Token, c.rateLimiter, req.UserId) + if len(current) > 0 { + batches = append(batches, current) + } + + return batches +} + +// Spaced out and retried per message so a rate limit delays delivery rather than dropping it. +// Returns how many archives reached the user. +func (c *Callback) sendExportViaDM(ctx context.Context, req gdprrelay.GDPRRequest, locale *i18n.Locale, result ResultData) (int, error) { + scrambledUserId := utils.ScrambleUserId(req.UserId) + + channelId, err := c.openDM(ctx, req.UserId) if err != nil { - c.logger.Error("Failed to create DM channel for export", - zap.Error(err), + return 0, err + } + + batches := batchParts(result.ExportParts, config.Conf.Export.MaxMessageBytes) + delivered := 0 + + for i, batch := range batches { + if i > 0 { + if err := sleep(ctx, config.Conf.Export.DmDelay); err != nil { + return delivered, err + } + } + + content := i18n.GetMessage(locale, i18n.GdprExportDmMessage) + if len(batches) > 1 { + content = i18n.GetMessage(locale, i18n.GdprExportDmMessagePart, i+1, len(batches)) + } + + if err := c.sendBatch(ctx, channelId, content, batch); err != nil { + return delivered, fmt.Errorf("failed to send export message %d of %d: %w", i+1, len(batches), err) + } + + delivered += len(batch) + + c.logger.Debug("Export message delivered", zap.String("scrambled_user_id", scrambledUserId), + zap.Int("message", i+1), + zap.Int("messages", len(batches)), + zap.Int("attachments", len(batch)), ) - return fmt.Errorf("failed to create DM channel: %w", err) } - content := i18n.GetMessage(locale, i18n.GdprExportDmMessage) + c.logger.Info("Export delivered via DM", + zap.String("scrambled_user_id", scrambledUserId), + zap.Int("parts", delivered), + zap.Int("messages", len(batches)), + zap.Int("files", result.ExportedFiles), + ) + + return delivered, nil +} - data := rest.CreateMessageData{ - Content: content, - Attachments: []request.Attachment{ - { - Id: 0, - FileName: result.ExportFileName, +func (c *Callback) sendBatch(ctx context.Context, channelId uint64, content string, batch []export.Part) error { + var lastErr error + + for attempt := 1; attempt <= maxPartAttempts; attempt++ { + if attempt > 1 { + backoff := time.Duration(attempt-1) * 2 * time.Second + if retryAfter, ok := retryAfterOf(lastErr); ok { + backoff = retryAfter + } + + if err := sleep(ctx, backoff); err != nil { + return err + } + } + + // Id must match the attachment's position: the encoder names the form field files[i]. + attachments := make([]request.Attachment, len(batch)) + for i, part := range batch { + attachments[i] = request.Attachment{ + Id: i, + FileName: part.Name, File: request.File{ ContentType: "application/zip", - Reader: bytes.NewReader(result.ExportData), + // Fresh reader per attempt, or a retry uploads nothing. + Reader: bytes.NewReader(part.Data), }, - }, - }, + } + } + + data := rest.CreateMessageData{ + Content: content, + Attachments: attachments, + } + + _, err := rest.CreateMessage(ctx, config.Conf.Discord.Token, c.rateLimiter, channelId, data) + if err == nil { + return nil + } + + lastErr = err + + if ctx.Err() != nil { + return ctx.Err() + } } - _, err = rest.CreateMessage(ctx, config.Conf.Discord.Token, c.rateLimiter, dmChannel.Id, data) - if err != nil { - c.logger.Error("Failed to send export DM with attachment", - zap.Error(err), - zap.String("scrambled_user_id", scrambledUserId), - zap.Uint64("channel_id", dmChannel.Id), - zap.String("export_file", result.ExportFileName), - ) - return fmt.Errorf("failed to send export DM: %w", err) + return lastErr +} + +func retryAfterOf(err error) (time.Duration, bool) { + if err == nil { + return 0, false } - c.logger.Info("Export sent via DM", - zap.String("scrambled_user_id", scrambledUserId), - zap.String("export_file", result.ExportFileName), - ) + var restErr request.RestError + if errors.As(err, &restErr) && restErr.StatusCode == 429 { + return 5 * time.Second, true + } - return nil + if strings.Contains(strings.ToLower(err.Error()), "rate limit") { + return 5 * time.Second, true + } + + return 0, false +} + +func sleep(ctx context.Context, d time.Duration) error { + if d <= 0 { + return nil + } + + timer := time.NewTimer(d) + defer timer.Stop() + + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } } diff --git a/internal/config/config.go b/internal/config/config.go index da1d74f..72e5dfc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,8 @@ package config import ( + "time" + "github.com/caarlos0/env/v10" "go.uber.org/zap/zapcore" ) @@ -12,44 +14,45 @@ type Config struct { MaxRetries int `env:"MAX_RETRIES" envDefault:"3"` Database struct { - Host string `env:"HOST"` - Database string `env:"NAME"` - Username string `env:"USER"` + Host string `env:"HOST,required,notEmpty"` + Database string `env:"NAME,required,notEmpty"` + Username string `env:"USER,required,notEmpty"` Password string `env:"PASSWORD"` - Threads int `env:"THREADS"` + Threads int `env:"THREADS" envDefault:"5"` } `envPrefix:"DATABASE_"` Redis struct { - Address string `env:"ADDR"` + Address string `env:"ADDR,required,notEmpty"` Password string `env:"PASSWD"` - Threads int `env:"THREADS"` + Threads int `env:"THREADS" envDefault:"5"` } `envPrefix:"REDIS_"` Archiver struct { - Url string `env:"URL"` - AesKey string `env:"AES_KEY"` + Url string `env:"URL,required,notEmpty"` + AesKey string `env:"AES_KEY,required,notEmpty"` } `envPrefix:"ARCHIVER_"` Discord struct { ProxyUrl string `env:"PROXY_URL"` - Token string `env:"TOKEN"` + Token string `env:"TOKEN,required,notEmpty"` } `envPrefix:"DISCORD_"` - S3 struct { - Endpoint string `env:"ENDPOINT"` - AccessKey string `env:"ACCESS_KEY"` - SecretKey string `env:"SECRET_KEY"` - Bucket string `env:"BUCKET"` - Secure bool `env:"SECURE" envDefault:"true"` - } `envPrefix:"S3_"` - CacheDatabase struct { Host string `env:"HOST"` Database string `env:"NAME"` Username string `env:"USER"` Password string `env:"PASSWORD"` - Threads int `env:"THREADS"` + Threads int `env:"THREADS" envDefault:"5"` } `envPrefix:"CACHE_"` + + Export struct { + MaxPartBytes int `env:"MAX_PART_BYTES" envDefault:"12582912"` + MaxMessageBytes int `env:"MAX_MESSAGE_BYTES" envDefault:"25165824"` + MaxAttachmentBytes int `env:"MAX_ATTACHMENT_BYTES" envDefault:"20971520"` + DmDelay time.Duration `env:"DM_DELAY" envDefault:"3s"` + Timeout time.Duration `env:"TIMEOUT" envDefault:"30m"` + Concurrency int `env:"CONCURRENCY" envDefault:"15"` + } `envPrefix:"EXPORT_"` } var Conf Config @@ -59,7 +62,3 @@ func Parse() { panic(err) } } - -func init() { - Parse() -} diff --git a/internal/database/database.go b/internal/database/database.go index 4487191..751b0b2 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -9,19 +9,38 @@ import ( "go.uber.org/zap" ) -var Client *database.Database +var ( + Client *database.Database + Pool *pgxpool.Pool +) func Connect(logger *zap.Logger, host, dbName, username, password string, threads int) error { - uri := fmt.Sprintf("postgres://%s:%s@%s/%s?pool_max_conns=%d", username, password, host, dbName, threads) - - pool, err := pgxpool.Connect(context.Background(), uri) + pool, err := NewPool(context.Background(), host, dbName, username, password, threads) if err != nil { return fmt.Errorf("failed to connect to database: %w", err) } logger.Info("Connected to database") + Pool = pool Client = database.NewDatabase(pool) return nil } + +// The pool size is applied to the parsed config rather than interpolated into the URI: an unset +// thread count would render as "pool_max_conns=0", which pgx rejects, taking the worker down. +func NewPool(ctx context.Context, host, dbName, username, password string, threads int) (*pgxpool.Pool, error) { + uri := fmt.Sprintf("postgres://%s:%s@%s/%s", username, password, host, dbName) + + cfg, err := pgxpool.ParseConfig(uri) + if err != nil { + return nil, fmt.Errorf("failed to parse connection string: %w", err) + } + + if threads > 0 { + cfg.MaxConns = int32(threads) + } + + return pgxpool.ConnectConfig(ctx, cfg) +} diff --git a/internal/export/manifest.go b/internal/export/manifest.go new file mode 100644 index 0000000..2f105d7 --- /dev/null +++ b/internal/export/manifest.go @@ -0,0 +1,35 @@ +package export + +import "time" + +const ManifestVersion = 1 + +// Manifest lists every source considered, including ones deliberately withheld, so a +// data-subject request can be answered with evidence. +type Manifest struct { + Version int `json:"version"` + Kind string `json:"kind"` + GeneratedAt time.Time `json:"generated_at"` + Sources []Source `json:"sources"` + Files ManifestFiles `json:"files"` +} + +type ManifestFiles struct { + Written int `json:"written"` + Withheld []string `json:"withheld_too_large,omitempty"` +} + +type Source struct { + Name string `json:"name"` + Included bool `json:"included"` + Records int `json:"records"` + Reason string `json:"reason,omitempty"` +} + +func (m *Manifest) Add(name string, records int) { + m.Sources = append(m.Sources, Source{Name: name, Included: true, Records: records}) +} + +func (m *Manifest) Withhold(name, reason string) { + m.Sources = append(m.Sources, Source{Name: name, Included: false, Reason: reason}) +} diff --git a/internal/export/userdata.go b/internal/export/userdata.go new file mode 100644 index 0000000..c940f6a --- /dev/null +++ b/internal/export/userdata.go @@ -0,0 +1,170 @@ +package export + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/jackc/pgx/v4" +) + +const UserDataVersion = 1 + +type Querier interface { + Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error) +} + +type UserData struct { + Version int `json:"version"` + UserId string `json:"user_id"` + GeneratedAt time.Time `json:"generated_at"` + Tables map[string][]map[string]interface{} `json:"tables"` +} + +// Every user-keyed table in the database module is either listed here or withheld in GetUserData, +// so the manifest can account for all of them. +type source struct { + name string + query string +} + +var userSources = []source{ + {"blacklist", `SELECT "guild_id" FROM blacklist WHERE "user_id" = $1;`}, + {"close_request", `SELECT "guild_id", "ticket_id", "close_at", "close_reason" FROM close_request WHERE "user_id" = $1;`}, + {"first_response_time", `SELECT "guild_id", "ticket_id", "response_time" FROM first_response_time WHERE "user_id" = $1;`}, + {"participant", `SELECT "guild_id", "ticket_id" FROM participant WHERE "user_id" = $1;`}, + {"permissions", `SELECT "guild_id", "support", "admin" FROM permissions WHERE "user_id" = $1;`}, + {"support_team_members", `SELECT "team_id" FROM support_team_members WHERE "user_id" = $1;`}, + {"ticket_claims", `SELECT "guild_id", "ticket_id" FROM ticket_claims WHERE "user_id" = $1;`}, + {"ticket_members", `SELECT "guild_id", "ticket_id" FROM ticket_members WHERE "user_id" = $1;`}, + {"tickets", `SELECT "id", "guild_id", "channel_id", "open", "open_time", "welcome_message_id", "panel_id", "has_transcript" FROM tickets WHERE "user_id" = $1;`}, + {"used_keys", `SELECT "guild_id", "key", "activated_by" FROM used_keys WHERE "activated_by" = $1;`}, + {"user_guilds", `SELECT "guild_id", "name", "owner", "permissions", "icon" FROM user_guilds WHERE "user_id" = $1;`}, + {"whitelabel_users", `SELECT "expiry" FROM whitelabel_users WHERE "user_id" = $1;`}, + + // Not read by the previous export helper at all. + {"dashboard_users", `SELECT "last_seen" FROM dashboard_users WHERE "user_id" = $1;`}, + {"votes", `SELECT "vote_time" FROM votes WHERE "user_id" = $1;`}, + {"audit_logs", `SELECT "id", "guild_id", "action_type", "resource_type", "resource_id", "old_data", "new_data", "metadata", "created_at" FROM audit_logs WHERE "user_id" = $1;`}, + {"entitlements", `SELECT "id", "guild_id", "sku_id", "source", "expires_at" FROM entitlements WHERE "user_id" = $1;`}, + {"legacy_premium_entitlements", `SELECT "tier", "sku_label", "sku_id", "is_legacy", "expires_at" FROM legacy_premium_entitlements WHERE "user_id" = $1;`}, + {"legacy_premium_entitlement_guilds", `SELECT "guild_id", "entitlement_id" FROM legacy_premium_entitlement_guilds WHERE "user_id" = $1;`}, + {"patreon_entitlements", `SELECT "entitlement_id" FROM patreon_entitlements WHERE "user_id" = $1;`}, + {"discord_entitlements", `SELECT "entitlement_id" FROM discord_entitlements WHERE "discord_id" = $1;`}, + { + "exit_survey_responses", + `SELECT e."guild_id", e."ticket_id", e."form_id", e."question_id", e."response" + FROM exit_survey_responses e + INNER JOIN tickets t ON t."guild_id" = e."guild_id" AND t."id" = e."ticket_id" + WHERE t."user_id" = $1;`, + }, +} + +// Unlike the upstream helper this replaces, it takes a context, so a slow export is bounded by the +// caller's deadline instead of running unbounded on context.Background(). +func GetUserData(ctx context.Context, q Querier, userId uint64, manifest *Manifest) (*UserData, error) { + data := &UserData{ + Version: UserDataVersion, + UserId: fmt.Sprintf("%d", userId), + GeneratedAt: time.Now().UTC(), + Tables: make(map[string][]map[string]interface{}), + } + + for _, src := range userSources { + rows, err := q.Query(ctx, src.query, userId) + if err != nil { + return nil, fmt.Errorf("failed to query %s: %w", src.name, err) + } + + records, err := rowsToMaps(rows) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", src.name, err) + } + + data.Tables[src.name] = records + manifest.Add(src.name, len(records)) + } + + if err := addWhitelabel(ctx, q, userId, data, manifest); err != nil { + return nil, err + } + + return data, nil +} + +// The bot token is a live credential. Shipping it in an archive delivered over Discord would hand +// anyone who sees that message full control of the user's bot, so it is withheld. +func addWhitelabel(ctx context.Context, q Querier, userId uint64, data *UserData, manifest *Manifest) error { + rows, err := q.Query(ctx, `SELECT "bot_id", "public_key" FROM whitelabel WHERE "user_id" = $1;`, userId) + if err != nil { + return fmt.Errorf("failed to query whitelabel: %w", err) + } + + records, err := rowsToMaps(rows) + if err != nil { + return fmt.Errorf("failed to read whitelabel: %w", err) + } + + data.Tables["whitelabel"] = records + manifest.Add("whitelabel", len(records)) + + if len(records) > 0 { + manifest.Withhold("whitelabel.token", "live bot credential withheld; rotate it from the dashboard if you need a copy") + } + + return nil +} + +func rowsToMaps(rows pgx.Rows) ([]map[string]interface{}, error) { + defer rows.Close() + + fields := rows.FieldDescriptions() + out := make([]map[string]interface{}, 0) + + for rows.Next() { + values, err := rows.Values() + if err != nil { + return nil, err + } + + record := make(map[string]interface{}, len(fields)) + for i, field := range fields { + if i < len(values) { + record[string(field.Name)] = normalise(values[i]) + } + } + + out = append(out, record) + } + + return out, rows.Err() +} + +// Discord snowflakes exceed the range a JSON number survives in most parsers, so anything that +// large is emitted as a string. +func normalise(v interface{}) interface{} { + switch t := v.(type) { + case nil: + return nil + case int64: + if t > 1<<53 || t < -(1<<53) { + return fmt.Sprintf("%d", t) + } + return t + case uint64: + if t > 1<<53 { + return fmt.Sprintf("%d", t) + } + return t + case [16]byte: + return fmt.Sprintf("%x-%x-%x-%x-%x", t[0:4], t[4:6], t[6:8], t[8:10], t[10:16]) + case []byte: + if json.Valid(t) { + return json.RawMessage(t) + } + return string(t) + default: + return v + } +} diff --git a/internal/export/zip.go b/internal/export/zip.go index 10f584b..33ff94b 100644 --- a/internal/export/zip.go +++ b/internal/export/zip.go @@ -3,55 +3,165 @@ package export import ( "archive/zip" "bytes" + "compress/flate" "fmt" + "hash/crc32" + "io" + "time" ) -// ZipBuilder provides an in-memory ZIP archive builder for constructing -// data export archives without writing to disk. -type ZipBuilder struct { - buf *bytes.Buffer - writer *zip.Writer - closed bool +// Two per message: Discord caps a request at 25 MiB and an attachment at 20 MiB, so 12 MiB uses +// the request budget almost exactly where a single 20 MiB part would waste a fifth of it. +const DefaultMaxPartBytes = 12 * 1024 * 1024 + +// Local header (30) + central directory record (46) + slack for extra fields the writer may emit. +const perEntryOverhead = 30 + 46 + 64 + +const trailerOverhead = 128 + +type Part struct { + Name string + Data []byte +} + +// Each part is independently valid, so a recipient can open any one without the others. +type Builder struct { + maxBytes int + + buf *bytes.Buffer + writer *zip.Writer + entries int + // Central directory bytes the current part still has to write. + reserved int + + done [][]byte + oversized []string + total int } -// NewZipBuilder creates a new ZipBuilder that writes to an in-memory buffer. -func NewZipBuilder() *ZipBuilder { - buf := new(bytes.Buffer) - return &ZipBuilder{ - buf: buf, - writer: zip.NewWriter(buf), +func NewBuilder(maxBytes int) *Builder { + if maxBytes <= 0 { + maxBytes = DefaultMaxPartBytes } + + return &Builder{maxBytes: maxBytes} +} + +func (b *Builder) start() { + b.buf = new(bytes.Buffer) + b.writer = zip.NewWriter(b.buf) + b.entries = 0 + b.reserved = trailerOverhead } -// AddFile adds a file with the given name and data to the ZIP archive. -func (z *ZipBuilder) AddFile(name string, data []byte) error { - if z.closed { - return fmt.Errorf("zip builder is already closed") +// An entry too large for even an empty part is recorded by Oversized and skipped, rather than +// emitting an archive Discord will reject. +func (b *Builder) Add(name string, data []byte) error { + compressed, crc, err := deflate(data) + if err != nil { + return fmt.Errorf("failed to compress zip entry %q: %w", name, err) + } + + cost := len(compressed) + perEntryOverhead + 2*len(name) + + if cost+trailerOverhead > b.maxBytes { + b.oversized = append(b.oversized, name) + return nil + } + + if b.writer == nil { + b.start() + } + + if err := b.writer.Flush(); err != nil { + return fmt.Errorf("failed to flush zip part: %w", err) + } + + if b.entries > 0 && b.buf.Len()+b.reserved+cost > b.maxBytes { + if err := b.closePart(); err != nil { + return err + } + b.start() } - f, err := z.writer.Create(name) + header := &zip.FileHeader{ + Name: name, + Method: zip.Deflate, + Modified: time.Now().UTC(), + CRC32: crc, + CompressedSize64: uint64(len(compressed)), + UncompressedSize64: uint64(len(data)), + } + + w, err := b.writer.CreateRaw(header) if err != nil { return fmt.Errorf("failed to create zip entry %q: %w", name, err) } - if _, err := f.Write(data); err != nil { + if _, err := w.Write(compressed); err != nil { return fmt.Errorf("failed to write zip entry %q: %w", name, err) } + b.entries++ + b.total++ + b.reserved += 46 + len(name) + 64 + + return nil +} + +func (b *Builder) closePart() error { + if err := b.writer.Close(); err != nil { + return fmt.Errorf("failed to close zip part: %w", err) + } + + b.done = append(b.done, b.buf.Bytes()) + b.writer = nil + b.buf = nil + return nil } -// Close finalises the ZIP archive and returns the resulting bytes. -func (z *ZipBuilder) Close() ([]byte, error) { - if z.closed { - return nil, fmt.Errorf("zip builder is already closed") +func (b *Builder) Count() int { return b.total } + +func (b *Builder) Oversized() []string { return b.oversized } + +func (b *Builder) Finish(baseName string) ([]Part, error) { + if b.writer != nil { + if b.entries == 0 { + b.writer = nil + b.buf = nil + } else if err := b.closePart(); err != nil { + return nil, err + } + } + + parts := make([]Part, len(b.done)) + for i, data := range b.done { + name := fmt.Sprintf("%s.zip", baseName) + if len(b.done) > 1 { + name = fmt.Sprintf("%s_%d-%d.zip", baseName, i+1, len(b.done)) + } + parts[i] = Part{Name: name, Data: data} + } + + return parts, nil +} + +func deflate(data []byte) ([]byte, uint32, error) { + var out bytes.Buffer + + w, err := flate.NewWriter(&out, flate.DefaultCompression) + if err != nil { + return nil, 0, err } - z.closed = true + if _, err := io.Copy(w, bytes.NewReader(data)); err != nil { + return nil, 0, err + } - if err := z.writer.Close(); err != nil { - return nil, fmt.Errorf("failed to close zip writer: %w", err) + if err := w.Close(); err != nil { + return nil, 0, err } - return z.buf.Bytes(), nil + return out.Bytes(), crc32.ChecksumIEEE(data), nil } diff --git a/internal/processor/export_requests.go b/internal/processor/export_requests.go new file mode 100644 index 0000000..9a6af72 --- /dev/null +++ b/internal/processor/export_requests.go @@ -0,0 +1,491 @@ +package processor + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync/atomic" + "time" + + "github.com/TicketsBot-cloud/archiverclient" + "github.com/TicketsBot-cloud/gdpr-worker/internal/archiver" + "github.com/TicketsBot-cloud/gdpr-worker/internal/config" + "github.com/TicketsBot-cloud/gdpr-worker/internal/database" + "github.com/TicketsBot-cloud/gdpr-worker/internal/export" + "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" + "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" + v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type transcriptRef struct { + guildId uint64 + ticketId int +} + +type transcriptEntry struct { + ref transcriptRef + name string + data []byte +} + +type archiveFile struct { + name string + data []byte +} + +func (p *Processor) exportConcurrency() int { + if config.Conf.Export.Concurrency > 0 { + return config.Conf.Export.Concurrency + } + return 15 +} + +func transcriptPath(ref transcriptRef) string { + return fmt.Sprintf("transcripts/%d/%d.json", ref.guildId, ref.ticketId) +} + +// A part has to fit both Discord ceilings on its own, or it could never be delivered. +func maxPartBytes() int { + size := config.Conf.Export.MaxPartBytes + for _, limit := range []int{config.Conf.Export.MaxAttachmentBytes, config.Conf.Export.MaxMessageBytes} { + if limit > 0 && limit < size { + size = limit + } + } + + return size +} + +func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { + if len(request.GuildIds) == 0 { + return ProcessResult{Error: fmt.Errorf("no server ID provided")} + } + + scrambledUserId := utils.ScrambleUserId(request.UserId) + + if err := p.verifyAllGuildsOwnership(ctx, request.GuildIds, request.UserId); err != nil { + p.logger.Error("Guild ownership verification failed for export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: err} + } + + manifest := &export.Manifest{ + Version: export.ManifestVersion, + Kind: "guild", + GeneratedAt: time.Now().UTC(), + } + + var refs []transcriptRef + for _, guildId := range request.GuildIds { + found, err := p.transcriptRefsForGuild(ctx, guildId) + if err != nil { + p.logger.Error("Failed to query tickets for guild export", + zap.Uint64("guild_id", guildId), + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to list transcripts for server %d: %w", guildId, err)} + } + + manifest.Add(fmt.Sprintf("transcripts/%d", guildId), len(found)) + refs = append(refs, found...) + } + + baseName := "guild_export" + if len(request.GuildIds) == 1 { + baseName = fmt.Sprintf("guild_export_%d", request.GuildIds[0]) + } + + return p.buildTranscriptExport(ctx, exportJob{ + refs: refs, + baseName: baseName, + manifest: manifest, + scrambledUserId: scrambledUserId, + logLabel: "guild export", + }) +} + +func (p *Processor) processExportUser(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { + scrambledUserId := utils.ScrambleUserId(request.UserId) + + manifest := &export.Manifest{ + Version: export.ManifestVersion, + Kind: "user", + GeneratedAt: time.Now().UTC(), + } + + userData, err := export.GetUserData(ctx, database.Pool, request.UserId, manifest) + if err != nil { + p.logger.Error("Failed to collect user database records for export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to retrieve database data: %w", err)} + } + + dbJson, err := json.Marshal(userData) + if err != nil { + return ProcessResult{Error: fmt.Errorf("failed to serialise database data: %w", err)} + } + + extras := []archiveFile{{name: "database.json", data: dbJson}} + + if cacheJson := p.userCacheData(ctx, request.UserId, manifest); cacheJson != nil { + extras = append(extras, archiveFile{name: "cache.json", data: cacheJson}) + } + + refs, err := p.transcriptRefsForUser(ctx, request.UserId) + if err != nil { + p.logger.Error("Failed to query tickets for user export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to query user tickets: %w", err)} + } + + manifest.Add("transcripts", len(refs)) + + return p.buildTranscriptExport(ctx, exportJob{ + refs: refs, + baseName: "user_export", + manifest: manifest, + scrambledUserId: scrambledUserId, + logLabel: "user export", + extras: extras, + filterFor: request.UserId, + }) +} + +type exportJob struct { + refs []transcriptRef + baseName string + manifest *export.Manifest + scrambledUserId string + logLabel string + extras []archiveFile + // When non-zero, each transcript is reduced to that user's own messages. + filterFor uint64 +} + +// Streamed into the archive as they arrive, so peak memory tracks the concurrency limit rather +// than the size of the whole export. +func (p *Processor) buildTranscriptExport(ctx context.Context, job exportJob) ProcessResult { + builder := export.NewBuilder(maxPartBytes()) + + for _, extra := range job.extras { + if err := builder.Add(extra.name, extra.data); err != nil { + return ProcessResult{Error: err} + } + } + + results := make(chan transcriptEntry, p.exportConcurrency()) + done := make(chan struct{}) + + var downloadErr error + var missing atomic.Int64 + + go func() { + defer close(done) + + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(p.exportConcurrency()) + + for _, ref := range job.refs { + g.Go(func() error { + entry, absent, err := p.fetchTranscript(gCtx, ref, job.filterFor) + if err != nil { + return err + } + + if absent { + missing.Add(1) + } + + if entry == nil { + return nil + } + + select { + case results <- *entry: + return nil + case <-gCtx.Done(): + return gCtx.Err() + } + }) + } + + downloadErr = g.Wait() + close(results) + }() + + var addErr error + for entry := range results { + if addErr != nil { + continue + } + if err := builder.Add(entry.name, entry.data); err != nil { + addErr = err + } + } + + <-done + + if downloadErr != nil { + p.logger.Error("Transcript download failed during export", + zap.String("scrambled_user_id", job.scrambledUserId), + zap.String("export", job.logLabel), + zap.Error(downloadErr), + ) + return ProcessResult{Error: fmt.Errorf("failed to download transcripts: %w", downloadErr)} + } + + if addErr != nil { + return ProcessResult{Error: addErr} + } + + if absent := int(missing.Load()); absent > 0 { + job.manifest.Withhold("transcripts.unavailable", fmt.Sprintf("%d recorded transcripts are no longer held by the archive store", absent)) + p.logger.Warn("Transcripts recorded in the database were absent from the archive store", + zap.String("scrambled_user_id", job.scrambledUserId), + zap.String("export", job.logLabel), + zap.Int("count", absent), + ) + } + + if oversized := builder.Oversized(); len(oversized) > 0 { + job.manifest.Files.Withheld = oversized + p.logger.Warn("Entries too large to deliver were withheld from export", + zap.String("scrambled_user_id", job.scrambledUserId), + zap.Int("count", len(oversized)), + ) + } + + // Reported as no-data rather than delivering an empty archive. + if builder.Count() == 0 { + p.logger.Info("Export produced no files", + zap.String("scrambled_user_id", job.scrambledUserId), + zap.String("export", job.logLabel), + ) + return ProcessResult{} + } + + job.manifest.Files.Written = builder.Count() + + if manifestJson, err := json.Marshal(job.manifest); err == nil { + if err := builder.Add("manifest.json", manifestJson); err != nil { + return ProcessResult{Error: err} + } + } + + parts, err := builder.Finish(job.baseName) + if err != nil { + return ProcessResult{Error: fmt.Errorf("failed to finalise export archive: %w", err)} + } + + total := 0 + for _, part := range parts { + total += len(part.Data) + } + + p.logger.Info("Export completed", + zap.String("scrambled_user_id", job.scrambledUserId), + zap.String("export", job.logLabel), + zap.Int("files", builder.Count()), + zap.Int("parts", len(parts)), + zap.Int("bytes", total), + ) + + return ProcessResult{ + ExportParts: parts, + ExportedFiles: builder.Count(), + } +} + +// Returns nil for a transcript that is absent or, when filtering, holds nothing of the +// requester's. The bool marks the absent case, which is ordinary rather than a failure: the object +// can be purged independently of the database row. +func (p *Processor) fetchTranscript(ctx context.Context, ref transcriptRef, filterFor uint64) (*transcriptEntry, bool, error) { + transcript, err := archiver.Client.Get(ctx, ref.guildId, ref.ticketId) + if err != nil { + if isTranscriptMissing(err) { + p.logger.Debug("Transcript absent from archive store, skipping", + zap.Uint64("guild_id", ref.guildId), + zap.Int("ticket_id", ref.ticketId), + zap.Error(err), + ) + return nil, true, nil + } + return nil, false, err + } + + if filterFor != 0 { + if !filterTranscriptToUser(&transcript, filterFor) { + return nil, false, nil + } + } + + data, err := json.Marshal(transcript) + if err != nil { + return nil, false, fmt.Errorf("failed to serialise transcript %d/%d: %w", ref.guildId, ref.ticketId, err) + } + + return &transcriptEntry{ref: ref, name: transcriptPath(ref), data: data}, false, nil +} + +// The archiver proxy only maps a 404 to ErrNotFound; an object missing from the bucket surfaces as +// the storage backend's own wording behind another status, so the text has to be matched too. +func isTranscriptMissing(err error) bool { + if errors.Is(err, archiverclient.ErrNotFound) { + return true + } + + msg := strings.ToLower(err.Error()) + for _, want := range []string{"does not exist", "no such key", "not found"} { + if strings.Contains(msg, want) { + return true + } + } + + return false +} + +// Reports whether anything of the requester's remains. +func filterTranscriptToUser(transcript *v2.Transcript, userId uint64) bool { + var messages []v2.Message + for _, msg := range transcript.Messages { + if msg.AuthorId == userId { + messages = append(messages, msg) + } + } + + if len(messages) == 0 { + return false + } + + transcript.Messages = messages + transcript.Entities.Channels = nil + transcript.Entities.Roles = nil + + // Key by the requested id, not by the entity's own Id: once the anonymization flow has run the + // stored entity carries Id 0, and keying by that would leave the messages unresolvable. + entity := transcript.Entities.Users[userId] + if entity.Id == 0 { + entity.Id = userId + } + + transcript.Entities.Users = map[uint64]v2.User{userId: entity} + + return true +} + +func (p *Processor) transcriptRefsForGuild(ctx context.Context, guildId uint64) ([]transcriptRef, error) { + // Open tickets are excluded: the transcript only reaches the archive store on close. + query := `SELECT "id" FROM tickets WHERE "guild_id" = $1 AND "has_transcript" = true AND "open" = false ORDER BY "id";` + + rows, err := database.Client.Tickets.Query(ctx, query, guildId) + if err != nil { + return nil, err + } + defer rows.Close() + + var refs []transcriptRef + for rows.Next() { + var ticketId int + if err := rows.Scan(&ticketId); err != nil { + return nil, err + } + refs = append(refs, transcriptRef{guildId: guildId, ticketId: ticketId}) + } + + return refs, rows.Err() +} + +// The anonymization flow reads ticket_members and the previous export read participant; taking +// either alone silently drops transcripts the user appears in. +func (p *Processor) transcriptRefsForUser(ctx context.Context, userId uint64) ([]transcriptRef, error) { + query := ` + SELECT s.guild_id, s.ticket_id + FROM ( + SELECT "guild_id", "id" AS ticket_id FROM tickets WHERE "user_id" = $1 + UNION + SELECT "guild_id", "ticket_id" FROM participant WHERE "user_id" = $1 + UNION + SELECT "guild_id", "ticket_id" FROM ticket_members WHERE "user_id" = $1 + ) s + INNER JOIN tickets t ON t."guild_id" = s.guild_id AND t."id" = s.ticket_id + WHERE t."has_transcript" = true AND t."open" = false + ORDER BY s.guild_id, s.ticket_id;` + + rows, err := database.Client.Tickets.Query(ctx, query, userId) + if err != nil { + return nil, err + } + defer rows.Close() + + var refs []transcriptRef + for rows.Next() { + var ref transcriptRef + if err := rows.Scan(&ref.guildId, &ref.ticketId); err != nil { + return nil, err + } + refs = append(refs, ref) + } + + return refs, rows.Err() +} + +// Returns nil when the cache database is unconfigured or unreadable; the manifest records which. +func (p *Processor) userCacheData(ctx context.Context, userId uint64, manifest *export.Manifest) []byte { + if p.cachePool == nil { + manifest.Withhold("cache", "cache database not configured for this deployment") + return nil + } + + data := make(map[string]interface{}) + + var user json.RawMessage + if err := p.cachePool.QueryRow(ctx, `SELECT "data" FROM users WHERE "user_id" = $1;`, userId).Scan(&user); err == nil { + data["user"] = user + } + + members := make(map[string]json.RawMessage) + + rows, err := p.cachePool.Query(ctx, `SELECT "guild_id", "data" FROM members WHERE "user_id" = $1;`, userId) + if err != nil { + manifest.Withhold("cache", "cache database could not be read") + p.logger.Warn("Failed to read cache member data for export", + zap.String("scrambled_user_id", utils.ScrambleUserId(userId)), + zap.Error(err), + ) + return nil + } + + defer rows.Close() + + for rows.Next() { + var guildId uint64 + var raw json.RawMessage + if err := rows.Scan(&guildId, &raw); err != nil { + continue + } + members[fmt.Sprintf("%d", guildId)] = raw + } + + data["member_data"] = members + + encoded, err := json.Marshal(data) + if err != nil { + manifest.Withhold("cache", "cache data could not be serialised") + return nil + } + + manifest.Add("cache", len(members)) + + return encoded +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 5e1ac54..32cf505 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -5,10 +5,8 @@ import ( "encoding/json" "fmt" "strings" - "sync" "github.com/TicketsBot-cloud/archiverclient" - "github.com/TicketsBot-cloud/gdl/cache" "github.com/TicketsBot-cloud/gdl/rest" "github.com/TicketsBot-cloud/gdl/rest/ratelimit" "github.com/TicketsBot-cloud/gdpr-worker/internal/archiver" @@ -18,10 +16,8 @@ import ( "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" - "github.com/TicketsBot-cloud/logarchiver/pkg/export/user" "github.com/jackc/pgx/v4/pgxpool" "go.uber.org/zap" - "golang.org/x/sync/errgroup" ) // Processor handles the execution of GDPR data deletion and export requests @@ -42,11 +38,11 @@ func New(logger *zap.Logger, cachePool *pgxpool.Pool) *Processor { // ProcessResult contains the outcome of processing a GDPR request type ProcessResult struct { - TranscriptsDeleted int // Number of transcript archives deleted from archiver - MessagesDeleted int // Number of ticket messages deleted from database - Error error // Error if the processing failed, nil on success - ExportData []byte // ZIP file bytes for export requests - ExportFileName string // Suggested filename for the export archive + TranscriptsDeleted int // Number of transcript archives deleted from archiver + MessagesDeleted int // Number of ticket messages deleted from database + Error error // Error if the processing failed, nil on success + ExportParts []export.Part // Deliverable ZIP archives for export requests + ExportedFiles int // Number of files written across those archives } func (p *Processor) Process(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { @@ -573,362 +569,3 @@ func (p *Processor) storeTranscript(ctx context.Context, guildId uint64, ticketI return archiver.Client.ImportTranscript(ctx, guildId, ticketId, data) } - -// processExportGuild exports all transcript data for the specified guilds into a ZIP archive. -func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { - if len(request.GuildIds) == 0 { - return ProcessResult{Error: fmt.Errorf("no server ID provided")} - } - - scrambledUserId := utils.ScrambleUserId(request.UserId) - - if err := p.verifyAllGuildsOwnership(ctx, request.GuildIds, request.UserId); err != nil { - p.logger.Error("Guild ownership verification failed for export", - zap.String("scrambled_user_id", scrambledUserId), - zap.Error(err), - ) - return ProcessResult{Error: err} - } - - zipBuilder := export.NewZipBuilder() - - for _, guildId := range request.GuildIds { - query := `SELECT id FROM tickets WHERE guild_id = $1 AND has_transcript = 't' AND open = 'f' ORDER BY id;` - rows, err := database.Client.Tickets.Query(ctx, query, guildId) - if err != nil { - p.logger.Error("Failed to query tickets for guild export", - zap.Uint64("guild_id", guildId), - zap.String("scrambled_user_id", scrambledUserId), - zap.Error(err), - ) - continue - } - - var ticketIds []int - for rows.Next() { - var id int - if err := rows.Scan(&id); err != nil { - p.logger.Warn("Failed to scan ticket ID for guild export", - zap.Uint64("guild_id", guildId), - zap.Error(err), - ) - continue - } - ticketIds = append(ticketIds, id) - } - rows.Close() - - if len(ticketIds) == 0 { - p.logger.Info("No transcripts found for guild export", - zap.Uint64("guild_id", guildId), - zap.String("scrambled_user_id", scrambledUserId), - ) - continue - } - - // Use errgroup with bounded concurrency for parallel downloads - type transcriptEntry struct { - ticketId int - data []byte - } - - var mu sync.Mutex - var entries []transcriptEntry - - g, gCtx := errgroup.WithContext(ctx) - g.SetLimit(15) - - for _, ticketId := range ticketIds { - ticketId := ticketId - g.Go(func() error { - transcript, err := archiver.Client.Get(gCtx, guildId, ticketId) - if err != nil { - if err == archiverclient.ErrNotFound { - p.logger.Debug("Transcript not found in archiver, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - ) - return nil - } - p.logger.Warn("Failed to download transcript from archiver", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - return nil - } - - prettyData, err := json.MarshalIndent(transcript, "", " ") - if err != nil { - p.logger.Warn("Failed to marshal transcript for export", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - return nil - } - - mu.Lock() - entries = append(entries, transcriptEntry{ticketId: ticketId, data: prettyData}) - mu.Unlock() - - return nil - }) - } - - if err := g.Wait(); err != nil { - p.logger.Error("Error during parallel transcript download", - zap.Uint64("guild_id", guildId), - zap.Error(err), - ) - } - - // Add all collected entries to the ZIP - for _, entry := range entries { - var fileName string - if len(request.GuildIds) > 1 { - fileName = fmt.Sprintf("%d/%d.json", guildId, entry.ticketId) - } else { - fileName = fmt.Sprintf("%d.json", entry.ticketId) - } - - if err := zipBuilder.AddFile(fileName, entry.data); err != nil { - p.logger.Error("Failed to add transcript to ZIP", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", entry.ticketId), - zap.Error(err), - ) - } - } - } - - zipData, err := zipBuilder.Close() - if err != nil { - return ProcessResult{Error: fmt.Errorf("failed to finalise export ZIP: %w", err)} - } - - var exportFileName string - if len(request.GuildIds) == 1 { - exportFileName = fmt.Sprintf("guild_%d.zip", request.GuildIds[0]) - } else { - exportFileName = "guild_export.zip" - } - - p.logger.Info("Guild export completed", - zap.String("scrambled_user_id", scrambledUserId), - zap.String("export_file", exportFileName), - zap.Int("zip_size_bytes", len(zipData)), - ) - - return ProcessResult{ - ExportData: zipData, - ExportFileName: exportFileName, - } -} - -// processExportUser exports all personal data for the requesting user into a ZIP archive. -func (p *Processor) processExportUser(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { - scrambledUserId := utils.ScrambleUserId(request.UserId) - - zipBuilder := export.NewZipBuilder() - - // Export database data - userData, err := user.GetUserData(database.Client, request.UserId) - if err != nil { - p.logger.Error("Failed to get user database data for export", - zap.String("scrambled_user_id", scrambledUserId), - zap.Error(err), - ) - return ProcessResult{Error: fmt.Errorf("failed to retrieve database data: %w", err)} - } - - dbJson, err := json.MarshalIndent(userData, "", " ") - if err != nil { - return ProcessResult{Error: fmt.Errorf("failed to serialise database data: %w", err)} - } - - if err := zipBuilder.AddFile("database.json", dbJson); err != nil { - return ProcessResult{Error: fmt.Errorf("failed to add database.json to export: %w", err)} - } - - // Export cache data if cache pool is available - if p.cachePool != nil { - pgCache := cache.NewPgCache(p.cachePool, cache.CacheOptions{ - Users: true, - Members: true, - }) - - cacheData, err := user.GetCacheData(&pgCache, request.UserId) - if err != nil { - p.logger.Warn("Failed to get user cache data for export, continuing without it", - zap.String("scrambled_user_id", scrambledUserId), - zap.Error(err), - ) - } else { - cacheJson, err := json.MarshalIndent(cacheData, "", " ") - if err != nil { - p.logger.Warn("Failed to serialise cache data", - zap.String("scrambled_user_id", scrambledUserId), - zap.Error(err), - ) - } else { - if err := zipBuilder.AddFile("cache.json", cacheJson); err != nil { - p.logger.Warn("Failed to add cache.json to export", - zap.String("scrambled_user_id", scrambledUserId), - zap.Error(err), - ) - } - } - } - } - - // Collect all ticket IDs where the user participated or is the creator - transcriptIds := make(map[uint64][]int) - - // Query tickets where user is a participant - { - query := `SELECT participant.guild_id, participant.ticket_id FROM participant INNER JOIN tickets ON participant.guild_id = tickets.guild_id AND tickets.id = participant.ticket_id WHERE participant.user_id = $1 AND tickets.has_transcript='t' AND tickets.open='f';` - rows, err := database.Client.Participants.Query(ctx, query, request.UserId) - if err != nil { - p.logger.Error("Failed to query participant tickets for user export", - zap.String("scrambled_user_id", scrambledUserId), - zap.Error(err), - ) - return ProcessResult{Error: fmt.Errorf("failed to query participant tickets: %w", err)} - } - - for rows.Next() { - var guildId uint64 - var ticketId int - if err := rows.Scan(&guildId, &ticketId); err != nil { - p.logger.Warn("Failed to scan participant ticket row", - zap.String("scrambled_user_id", scrambledUserId), - zap.Error(err), - ) - continue - } - transcriptIds[guildId] = append(transcriptIds[guildId], ticketId) - } - rows.Close() - } - - // Query tickets where user is the creator - { - query := `SELECT guild_id, id FROM tickets WHERE user_id = $1 AND has_transcript='t' AND open='f';` - rows, err := database.Client.Tickets.Query(ctx, query, request.UserId) - if err != nil { - p.logger.Error("Failed to query user-created tickets for export", - zap.String("scrambled_user_id", scrambledUserId), - zap.Error(err), - ) - return ProcessResult{Error: fmt.Errorf("failed to query user tickets: %w", err)} - } - - for rows.Next() { - var guildId uint64 - var ticketId int - if err := rows.Scan(&guildId, &ticketId); err != nil { - p.logger.Warn("Failed to scan user ticket row", - zap.String("scrambled_user_id", scrambledUserId), - zap.Error(err), - ) - continue - } - transcriptIds[guildId] = append(transcriptIds[guildId], ticketId) - } - rows.Close() - } - - // Deduplicate ticket IDs per guild - for guildId, ticketIds := range transcriptIds { - seen := make(map[int]struct{}) - deduped := make([]int, 0, len(ticketIds)) - for _, id := range ticketIds { - if _, exists := seen[id]; !exists { - seen[id] = struct{}{} - deduped = append(deduped, id) - } - } - transcriptIds[guildId] = deduped - } - - // Download, filter and add transcripts to ZIP - for guildId, ticketIds := range transcriptIds { - for _, ticketId := range ticketIds { - transcript, err := archiver.Client.Get(ctx, guildId, ticketId) - if err != nil { - if err == archiverclient.ErrNotFound { - p.logger.Debug("Transcript not found for user export, skipping", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - ) - continue - } - p.logger.Warn("Failed to download transcript for user export", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } - - // Filter to only the user's data - transcript.Entities.Channels = nil - transcript.Entities.Roles = nil - - userEntity, ok := transcript.Entities.Users[request.UserId] - if !ok { - transcript.Entities.Users = nil - } else { - transcript.Entities.Users = map[uint64]v2.User{ - userEntity.Id: userEntity, - } - } - - var filteredMessages []v2.Message - for _, msg := range transcript.Messages { - if msg.AuthorId == request.UserId { - filteredMessages = append(filteredMessages, msg) - } - } - transcript.Messages = filteredMessages - - prettyData, err := json.MarshalIndent(transcript, "", " ") - if err != nil { - p.logger.Warn("Failed to marshal filtered transcript for user export", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - continue - } - - fileName := fmt.Sprintf("transcripts/%d-%d.json", guildId, ticketId) - if err := zipBuilder.AddFile(fileName, prettyData); err != nil { - p.logger.Error("Failed to add transcript to user export ZIP", - zap.Uint64("guild_id", guildId), - zap.Int("ticket_id", ticketId), - zap.Error(err), - ) - } - } - } - - zipData, err := zipBuilder.Close() - if err != nil { - return ProcessResult{Error: fmt.Errorf("failed to finalise user export ZIP: %w", err)} - } - - exportFileName := fmt.Sprintf("user_%d.zip", request.UserId) - - p.logger.Info("User export completed", - zap.String("scrambled_user_id", scrambledUserId), - zap.String("export_file", exportFileName), - zap.Int("zip_size_bytes", len(zipData)), - ) - - return ProcessResult{ - ExportData: zipData, - ExportFileName: exportFileName, - } -} From 472900fcff7a7886d1e5ef39e02ec4beb622e992 Mon Sep 17 00:00:00 2001 From: biast12 <53872542+biast12@users.noreply.github.com> Date: Sat, 12 Sep 2026 11:10:04 +0200 Subject: [PATCH 6/6] Clarify guild ownership check failure message Updates the ownership verification error when guild info cannot be fetched to explicitly state the bot is not in the server and instructs adding it back before retrying. This gives users a clear, actionable reason instead of a generic fetch failure. --- internal/processor/processor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 32cf505..f68d1e8 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -82,7 +82,7 @@ func (p *Processor) verifyGuildOwnership(ctx context.Context, guildId, userId ui zap.Uint64("guild_id", guildId), zap.Error(err), ) - return fmt.Errorf("failed to verify guild ownership: unable to fetch guild information") + return fmt.Errorf("the bot is not in server %d, so ownership could not be verified - add it back and try again", guildId) } if guild.OwnerId != userId {