diff --git a/.env.example b/.env.example index bf5d645..ce7a751 100644 --- a/.env.example +++ b/.env.example @@ -25,3 +25,21 @@ ARCHIVER_AES_KEY= # Discord Configuration DISCORD_PROXY_URL= DISCORD_TOKEN= + +# Cache Database Configuration (optional) +# When unset, exports omit cache.json and record the omission in the manifest. +CACHE_HOST= +CACHE_NAME= +CACHE_USER= +CACHE_PASSWORD= +CACHE_THREADS= + +# Data Export Configuration +# Discord caps a request at 25 MiB, an attachment at 20 MiB, and a message at 10 attachments. +# 12 MiB parts put two per message, using the request budget almost exactly. +EXPORT_MAX_PART_BYTES= +EXPORT_MAX_MESSAGE_BYTES= +EXPORT_MAX_ATTACHMENT_BYTES= +EXPORT_DM_DELAY= +EXPORT_TIMEOUT= +EXPORT_CONCURRENCY= diff --git a/Dockerfile b/Dockerfile index e980cba..f9d8d52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build container -FROM golang:1.24 AS builder +FROM golang:1.25 AS builder RUN go version diff --git a/cmd/main.go b/cmd/main.go index d8a299f..2e5ee10 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,11 +2,14 @@ package main import ( "context" + "net/http" "os" "os/signal" + "strings" "syscall" "time" + "github.com/TicketsBot-cloud/gdl/rest/request" "github.com/TicketsBot-cloud/gdpr-worker/i18n" "github.com/TicketsBot-cloud/gdpr-worker/internal/archiver" "github.com/TicketsBot-cloud/gdpr-worker/internal/callback" @@ -17,12 +20,22 @@ import ( "github.com/TicketsBot-cloud/gdpr-worker/internal/processor" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" "github.com/go-redis/redis/v8" + "github.com/jackc/pgx/v4/pgxpool" "go.uber.org/zap" "go.uber.org/zap/zapcore" _ "github.com/joho/godotenv/autoload" ) +// The proxy expects the request addressed to its own host rather than the usual CONNECT protocol. +// Application endpoints stay on discord.com, matching how the bot worker registers the same hook. +func proxyHook(token string, req *http.Request) { + if !strings.HasPrefix(req.URL.Path, "/api/v9/applications/") { + req.URL.Scheme = "http" + req.URL.Host = config.Conf.Discord.ProxyUrl + } +} + func main() { config.Parse() @@ -35,10 +48,16 @@ func main() { return } + if config.Conf.Discord.ProxyUrl != "" { + logger.Info("Configuring REST proxy", zap.String("url", config.Conf.Discord.ProxyUrl)) + request.RegisterPreRequestHook(proxyHook) + } + logger.Info("Connecting to Redis") redisClient := redis.NewClient(&redis.Options{ Addr: config.Conf.Redis.Address, Password: config.Conf.Redis.Password, + PoolSize: config.Conf.Redis.Threads, DB: 0, }) @@ -61,14 +80,36 @@ func main() { return } - logger.Info("Initializing archiver client") + logger.Info("Initialising archiver client") archiver.Initialize( logger.With(), config.Conf.Archiver.Url, config.Conf.Archiver.AesKey, ) - proc := processor.New(logger.With()) + var cachePool *pgxpool.Pool + if config.Conf.CacheDatabase.Host != "" { + logger.Info("Connecting to cache database") + + var err error + cachePool, err = database.NewPool( + context.Background(), + config.Conf.CacheDatabase.Host, + config.Conf.CacheDatabase.Database, + config.Conf.CacheDatabase.Username, + config.Conf.CacheDatabase.Password, + config.Conf.CacheDatabase.Threads, + ) + if err != nil { + logger.Fatal("Failed to connect to cache database", zap.Error(err)) + return + } + logger.Info("Connected to cache database") + } else { + logger.Warn("Cache database not configured, user export cache data will be unavailable") + } + + proc := processor.New(logger.With(), cachePool) callbackHandler := callback.New( logger.With(), @@ -95,7 +136,8 @@ func main() { <-semaphore }() - processCtx := context.Background() + processCtx, processCancel := context.WithTimeout(context.Background(), config.Conf.Export.Timeout) + defer processCancel() scrambledId := utils.ScrambleUserId(req.Request.UserId) requestTypeName := utils.GetRequestTypeName(int(req.Request.Type)) @@ -142,23 +184,27 @@ func main() { } callbackData := callback.ResultData{ - TranscriptsDeleted: result.TranscriptsDeleted, - MessagesDeleted: result.MessagesDeleted, - Error: result.Error, - RequestType: req.Request.Type, - GuildIds: req.Request.GuildIds, - TicketIds: req.Request.TicketIds, + TranscriptsDeleted: result.TranscriptsDeleted, + MessagesDeleted: result.MessagesDeleted, + Error: result.Error, + RequestType: req.Request.Type, + GuildIds: req.Request.GuildIds, + TicketIds: req.Request.TicketIds, + ExportParts: result.ExportParts, + ExportedFiles: result.ExportedFiles, } - callbackCtx, callbackCancel := context.WithTimeout(context.Background(), 30*time.Second) + callbackCtx, callbackCancel := context.WithTimeout(context.Background(), 10*time.Minute) defer callbackCancel() - if updateErr := database.Client.GdprLogs.UpdateLogStatus(req.RequestID, "Completed"); updateErr != nil { - logger.Error("Failed to update GDPR log status to Completed", - zap.Uint64("request_id", uint64(req.RequestID)), - zap.String("scrambled_user_id", scrambledId), - zap.Error(updateErr), - ) + if result.Error == nil { + if updateErr := database.Client.GdprLogs.UpdateLogStatus(req.RequestID, "Completed"); updateErr != nil { + logger.Error("Failed to update GDPR log status to Completed", + zap.Uint64("request_id", uint64(req.RequestID)), + zap.String("scrambled_user_id", scrambledId), + zap.Error(updateErr), + ) + } } if err := callbackHandler.SendCompletion(callbackCtx, req.Request, callbackData); err != nil { diff --git a/go.mod b/go.mod index 6dfd03e..7cabcde 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/TicketsBot-cloud/gdpr-worker -go 1.24.0 +go 1.25.0 //replace github.com/TicketsBot-cloud/database => ../database @@ -12,26 +12,27 @@ go 1.24.0 require ( github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704 - github.com/TicketsBot-cloud/database v0.0.0-20251018202538-7f9567e1aeab - github.com/TicketsBot-cloud/gdl v0.0.0-20251007163257-7e59b92d02dd - github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf + github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b + github.com/TicketsBot-cloud/gdl v0.0.0-20260612070331-a3947b410d3e + github.com/TicketsBot-cloud/logarchiver v0.0.0-20260909063657-e32e33f1289b github.com/caarlos0/env/v10 v10.0.0 github.com/go-redis/redis/v8 v8.11.5 github.com/jackc/pgx/v4 v4.18.3 github.com/joho/godotenv v1.5.1 - go.uber.org/zap v1.27.0 + go.uber.org/zap v1.27.1 + golang.org/x/sync v0.20.0 ) require ( - github.com/TicketsBot-cloud/common v0.0.0-20250509064208-a2d357175463 // indirect + github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e // indirect github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 // indirect github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 // indirect + github.com/boltdb/bolt v1.3.1 // indirect github.com/caarlos0/env v3.5.0+incompatible // indirect github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.14.3 // indirect @@ -41,16 +42,17 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgtype v1.14.4 // indirect github.com/jackc/pgx v3.6.2+incompatible // indirect - github.com/jackc/pgx/v5 v5.7.6 // indirect + github.com/jackc/pgx/v5 v5.9.1 // indirect github.com/jackc/puddle v1.3.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/juju/ratelimit v1.0.1 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.5 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect - github.com/minio/minio-go/v7 v7.0.95 // indirect + github.com/minio/minio-go/v7 v7.0.99 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pasztorpisti/qs v0.0.0-20171216220353-8d6c33ee906c // indirect @@ -59,12 +61,12 @@ require ( github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect - github.com/tinylib/msgp v1.4.0 // indirect - go.uber.org/multierr v1.10.0 // indirect - golang.org/x/crypto v0.43.0 // indirect + github.com/tinylib/msgp v1.6.3 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect - golang.org/x/net v0.46.0 // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/text v0.30.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect ) diff --git a/go.sum b/go.sum index f76ed9d..1c90c26 100644 --- a/go.sum +++ b/go.sum @@ -1,32 +1,22 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/ReneKroon/ttlcache v1.6.0/go.mod h1:DG6nbhXKUQhrExfwwLuZUdH7UnRDDRA1IW+nBuCssvs= -github.com/TicketsBot-cloud/archiverclient v0.0.0-20250514201416-cf23f65eb3fc h1:2EFgb6gQMedsO7YkqUPH1zY8+L6i3sGuV/YAv0gPqhQ= -github.com/TicketsBot-cloud/archiverclient v0.0.0-20250514201416-cf23f65eb3fc/go.mod h1:Mux1bEPpOHwRw1wo6Fa6qJLJH9Erk9qv1yAIfLi1Wmw= github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704 h1:liLfvCrzoJ89DXFHzsd1iK3cyP8s4i0CnZPRFEj53zg= github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704/go.mod h1:Mux1bEPpOHwRw1wo6Fa6qJLJH9Erk9qv1yAIfLi1Wmw= -github.com/TicketsBot-cloud/common v0.0.0-20250208132851-d5083bb04d98 h1:HwtXrqSv6y5E8mTnkOtISTaQr6dsK/lfUC8tUA6SzSU= -github.com/TicketsBot-cloud/common v0.0.0-20250208132851-d5083bb04d98/go.mod h1:iiZhl7w5DeTEGzAeq/hzani8+ILQz60g4JFkOy2vkuM= -github.com/TicketsBot-cloud/common v0.0.0-20250509064208-a2d357175463 h1:ZO2kw9lpsy4umoWFmAt9WKVFb3WvU5oSif2Yw4K7Lj4= -github.com/TicketsBot-cloud/common v0.0.0-20250509064208-a2d357175463/go.mod h1:PL5j/omFvU0NeyTKCESmOF+3GscaEuM0aqmI4yFcCFY= -github.com/TicketsBot-cloud/database v0.0.0-20250918212912-4cc263bc1b41 h1:X+V7IZtmDbE9h5cGSKnhCuisBFZ14Suzsdjnk8UUN3A= -github.com/TicketsBot-cloud/database v0.0.0-20250918212912-4cc263bc1b41/go.mod h1:LPDEn9e5wccH7rq/pUlVcL3UhyLnnwdM2dhj0tp/ljo= -github.com/TicketsBot-cloud/database v0.0.0-20251018202538-7f9567e1aeab h1:J8loePxYfe0oSqwaW0Ub51ku2+/fPiekQxHBYwX5S5A= -github.com/TicketsBot-cloud/database v0.0.0-20251018202538-7f9567e1aeab/go.mod h1:LPDEn9e5wccH7rq/pUlVcL3UhyLnnwdM2dhj0tp/ljo= -github.com/TicketsBot-cloud/gdl v0.0.0-20250917180424-569348f7a55b h1:k4kIIhpD3m0tx8Sz8gefZH8DBbuAwbeJJdV0iRr/x5Y= -github.com/TicketsBot-cloud/gdl v0.0.0-20250917180424-569348f7a55b/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= -github.com/TicketsBot-cloud/gdl v0.0.0-20251007163257-7e59b92d02dd h1:C4GzdEYarK2V81oAarvRW2RA2wzRGRydCAzBKtb/lGU= -github.com/TicketsBot-cloud/gdl v0.0.0-20251007163257-7e59b92d02dd/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250514201320-d5141071a6eb h1:BapforRlvTfWP8MX8DTsxVM40oDgQorJVo/cnNGTaKU= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250514201320-d5141071a6eb/go.mod h1:pZqkzPNNTqnwKZvCT8kCaTHxrG7HJbxZV83S0p7mmzM= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf h1:InP2ht90XOkLJKLjVrsCYPJunxlLkdsorikgNRk35bQ= -github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf/go.mod h1:pZqkzPNNTqnwKZvCT8kCaTHxrG7HJbxZV83S0p7mmzM= -github.com/TicketsBot/common v0.0.0-20241104184641-e39c64bdcf3e h1:cYfBjPX/FhD/MCViBI2Wz2YlC2esLiTbDE65Qku2WVg= -github.com/TicketsBot/common v0.0.0-20241104184641-e39c64bdcf3e/go.mod h1:N7zwetwx8B3RK/ZajWwMroJSyv2ZJ+bIOZWv/z8DhaM= +github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e h1:nFKV7yEm8MWbCP7dtsJ88+agcxDUD0YKIotVHMVvytw= +github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e/go.mod h1:tGrTHFz09OM3eDWF+62hIi9ELpT4igCFi868FKSvKBg= +github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b h1:bHkfJWo8T/9TiHuYHxaOz8GAILIiKPugC1k3CzdOq/A= +github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b/go.mod h1:HQXAgmNSm7/FmBYwcsa6qpZqMrDhbLoEl+AyqFQ+RwY= +github.com/TicketsBot-cloud/gdl v0.0.0-20260612070331-a3947b410d3e h1:qCibZQmO2rrBFrvn6+oAchPEQ7k/0Izr3ruL5P9aVZU= +github.com/TicketsBot-cloud/gdl v0.0.0-20260612070331-a3947b410d3e/go.mod h1:CdwBR2egPtxUXjD2CgC9ZwfuB8dz9HPePM8nuG6dt7Y= +github.com/TicketsBot-cloud/logarchiver v0.0.0-20260909063657-e32e33f1289b h1:XbZOnvTk3xGrWA+gHIGU97Eabpm7MV7PRqrypIG6SWI= +github.com/TicketsBot-cloud/logarchiver v0.0.0-20260909063657-e32e33f1289b/go.mod h1:1pv+CJbf7yr1cBJVDILqLgE7iJJsoRC4zK3381a/8mM= github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 h1:FqC1KGOsmB+ikvbmDkyNQU6bGUWyfYq8Ip9r4KxTveY= github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1/go.mod h1:N7zwetwx8B3RK/ZajWwMroJSyv2ZJ+bIOZWv/z8DhaM= github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 h1:NHD5GB6cjlkpZFjC76Yli2S63/J2nhr8MuE6KlYJpQM= github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261/go.mod h1:2zPxDAN2TAPpxUPjxszjs3QFKreKrQh5al/R3cMXmYk= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs= github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y= github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA= @@ -54,17 +44,12 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= -github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -85,7 +70,6 @@ github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5W github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgproto3 v1.1.0 h1:FYYE4yRw+AgI8wXIinMlNjBbp/UitDJwfj5LqqewP1A= github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA= github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190609003834-432c2951c711/go.mod h1:uH0AWtUmuShn0bcesswc4aBTWGvw0cAxIJp+6OB//Wg= @@ -96,7 +80,6 @@ github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= -github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= @@ -104,7 +87,6 @@ github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01C github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgtype v1.14.4 h1:fKuNiCumbKTAIxQwXfB/nsrnkEI6bPJrrSiMKgbJ2j8= github.com/jackc/pgtype v1.14.4/go.mod h1:aKeozOde08iifGosdJpz9MBZonJOUJxqNpPBcMJTlVA= @@ -117,17 +99,13 @@ github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgS github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/pgx/v4 v4.18.3 h1:dE2/TrEsGX3RBprb3qryqSV9Y60iZN1C6i8IrmW9/BA= github.com/jackc/pgx/v4 v4.18.3/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= -github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= -github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= -github.com/jackc/pgx/v5 v5.7.6 h1:rWQc5FwZSPX58r1OQmkuaNicxdmExaEz5A2DO2hUuTk= -github.com/jackc/pgx/v5 v5.7.6/go.mod h1:aruU7o91Tc2q2cFp5h4uP3f6ztExVpyVv88Xl/8Vl8M= +github.com/jackc/pgx/v5 v5.9.1 h1:uwrxJXBnx76nyISkhr33kQLlUqjv7et7b9FjCen/tdc= +github.com/jackc/pgx/v5 v5.9.1/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= -github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= @@ -137,15 +115,13 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -170,10 +146,8 @@ github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo= -github.com/minio/minio-go/v7 v7.0.73/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= -github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU= -github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -198,8 +172,6 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= -github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= @@ -225,10 +197,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tinylib/msgp v1.4.0 h1:SYOeDRiydzOw9kSiwdYp9UcBgPFtLU2WDHaJXyHruf8= -github.com/tinylib/msgp v1.4.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= +github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -241,14 +213,16 @@ go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/multierr v1.10.0 h1:S0h4aNzvfcFsC3dRF1jLoaov7oRaKqRGC/pUEJ2yvPQ= -go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -261,10 +235,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= -golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= -golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= -golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= -golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f h1:XdNn9LlyWAhLVp6P/i8QYBW+hlyhrhei9uErw2B5GJo= golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f/go.mod h1:D5SMRVC3C2/4+F/DB1wZsLRnSNimn2Sp/NPsCrsv8ak= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -281,17 +253,13 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= -golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= -golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= -golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= -golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= -golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -310,10 +278,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= -golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -329,10 +295,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= -golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= -golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425163242-31fd60d6bfdc/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= diff --git a/i18n/messages.go b/i18n/messages.go index 7a047cc..312dd69 100644 --- a/i18n/messages.go +++ b/i18n/messages.go @@ -14,4 +14,10 @@ var ( GdprFollowupError MessageId = "gdpr.followup.error" GdprFollowupNoData MessageId = "gdpr.followup.no_data" GdprFollowupSuccess MessageId = "gdpr.followup.success" + GdprCompletedExportGuild MessageId = "gdpr.completed.export_guild" + GdprCompletedExportGuildMulti MessageId = "gdpr.completed.export_guild_multi" + GdprCompletedExportUser MessageId = "gdpr.completed.export_user" + GdprExportDmMessage MessageId = "gdpr.export.dm_message" + GdprExportDmMessagePart MessageId = "gdpr.export.dm_message_part" + GdprErrorExportDmFailed MessageId = "gdpr.error.export_dm_failed" ) diff --git a/internal/callback/callback.go b/internal/callback/callback.go index 2daab54..efa367f 100644 --- a/internal/callback/callback.go +++ b/internal/callback/callback.go @@ -1,21 +1,34 @@ package callback import ( + "bytes" "context" + "errors" "fmt" "strings" + "time" "github.com/TicketsBot-cloud/gdl/objects/channel/message" "github.com/TicketsBot-cloud/gdl/objects/interaction/component" "github.com/TicketsBot-cloud/gdl/rest" "github.com/TicketsBot-cloud/gdl/rest/ratelimit" + "github.com/TicketsBot-cloud/gdl/rest/request" + "github.com/TicketsBot-cloud/gdpr-worker/i18n" "github.com/TicketsBot-cloud/gdpr-worker/internal/config" + "github.com/TicketsBot-cloud/gdpr-worker/internal/export" "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" - "github.com/TicketsBot-cloud/gdpr-worker/i18n" "go.uber.org/zap" ) +const maxPartAttempts = 3 + +// Discord accepts at most ten attachments on a message. +const maxAttachmentsPerMessage = 10 + +// Fallback when the message budget is unset; Discord's request cap is 25 MiB. +const defaultMaxMessageBytes = 24 * 1024 * 1024 + // ResultData contains the result of a GDPR request to be sent back to the user type ResultData struct { TranscriptsDeleted int // Number of transcript archives deleted @@ -24,6 +37,12 @@ type ResultData struct { RequestType gdprrelay.RequestType // Type of GDPR request that was processed GuildIds []uint64 // Guild IDs affected by this request TicketIds []int // Ticket IDs affected by this request + ExportParts []export.Part // Deliverable archives for export requests + ExportedFiles int // Number of files across those archives +} + +func (r ResultData) isExport() bool { + return r.RequestType == gdprrelay.RequestTypeExportGuild || r.RequestType == gdprrelay.RequestTypeExportUser } type Callback struct { @@ -40,26 +59,44 @@ func New(logger *zap.Logger, proxyUrl string) *Callback { } } -func (c *Callback) SendCompletion(ctx context.Context, request gdprrelay.GDPRRequest, result ResultData) error { - if request.InteractionToken == "" { +// The export is delivered before the interaction message is updated: that token expires after +// fifteen minutes, which is exactly the case a long export hits. +func (c *Callback) SendCompletion(ctx context.Context, req gdprrelay.GDPRRequest, result ResultData) error { + scrambledUserId := utils.ScrambleUserId(req.UserId) + locale := i18n.GetLocale(req.Language) + + var deliveryErr error + delivered := 0 + + if result.isExport() && result.Error == nil && len(result.ExportParts) > 0 { + delivered, deliveryErr = c.sendExportViaDM(ctx, req, locale, result) + if deliveryErr != nil { + c.logger.Error("Failed to deliver export via DM", + zap.Error(deliveryErr), + zap.String("scrambled_user_id", scrambledUserId), + zap.Int("delivered_parts", delivered), + zap.Int("total_parts", len(result.ExportParts)), + ) + } + } + + if req.InteractionToken == "" { c.logger.Debug("No interaction token, skipping callback") - return nil + return deliveryErr } - scrambledUserId := utils.ScrambleUserId(request.UserId) - locale := i18n.GetLocale(request.Language) - components := c.buildResultComponents(locale, result, request.GuildNames) + components := c.buildResultComponents(locale, result, req.GuildNames, deliveryErr, delivered) - if err := c.editOriginalMessage(ctx, request, components); err != nil { + if err := c.editOriginalMessage(ctx, req, components); err != nil { if c.isTokenExpired(err) { - if dmErr := c.sendCompletionViaDM(ctx, request, locale, result); dmErr != nil { + if dmErr := c.sendCompletionViaDM(ctx, req, components); dmErr != nil { c.logger.Error("Failed to send completion via DM", zap.Error(dmErr), zap.String("scrambled_user_id", scrambledUserId), ) return dmErr } - return nil + return deliveryErr } c.logger.Error("Failed to edit original message", @@ -69,18 +106,16 @@ func (c *Callback) SendCompletion(ctx context.Context, request gdprrelay.GDPRReq return err } - if err := c.sendEphemeralFollowup(ctx, request, locale, result); err != nil { - if c.isTokenExpired(err) { - return nil + if err := c.sendEphemeralFollowup(ctx, req, locale, result); err != nil { + if !c.isTokenExpired(err) { + c.logger.Error("Failed to send ephemeral follow-up", + zap.Error(err), + zap.String("scrambled_user_id", scrambledUserId), + ) } - - c.logger.Error("Failed to send ephemeral follow-up", - zap.Error(err), - zap.String("scrambled_user_id", scrambledUserId), - ) } - return nil + return deliveryErr } func (c *Callback) isTokenExpired(err error) bool { @@ -105,11 +140,7 @@ func (c *Callback) buildResultMessage(locale *i18n.Locale, result ResultData, gu guildDisplay := utils.FormatGuildDisplay(result.GuildIds[0], guildNames) content = i18n.GetMessage(locale, i18n.GdprCompletedAllTranscripts, guildDisplay, result.TranscriptsDeleted) } else { - guildDisplays := make([]string, len(result.GuildIds)) - for i, guildId := range result.GuildIds { - guildDisplays[i] = utils.FormatGuildDisplay(guildId, guildNames) - } - content = i18n.GetMessage(locale, i18n.GdprCompletedAllTranscriptsMulti, strings.Join(guildDisplays, "\n* "), result.TranscriptsDeleted) + content = i18n.GetMessage(locale, i18n.GdprCompletedAllTranscriptsMulti, joinGuildDisplays(result.GuildIds, guildNames), result.TranscriptsDeleted) } case gdprrelay.RequestTypeSpecificTranscripts: @@ -125,11 +156,7 @@ func (c *Callback) buildResultMessage(locale *i18n.Locale, result ResultData, gu guildDisplay := utils.FormatGuildDisplay(result.GuildIds[0], guildNames) content = i18n.GetMessage(locale, i18n.GdprCompletedAllMessages, guildDisplay, result.MessagesDeleted) } else { - guildDisplays := make([]string, len(result.GuildIds)) - for i, guildId := range result.GuildIds { - guildDisplays[i] = utils.FormatGuildDisplay(guildId, guildNames) - } - content = i18n.GetMessage(locale, i18n.GdprCompletedAllMessagesMulti, strings.Join(guildDisplays, "\n* "), result.MessagesDeleted) + content = i18n.GetMessage(locale, i18n.GdprCompletedAllMessagesMulti, joinGuildDisplays(result.GuildIds, guildNames), result.MessagesDeleted) } case gdprrelay.RequestTypeSpecificMessages: @@ -139,6 +166,17 @@ func (c *Callback) buildResultMessage(locale *i18n.Locale, result ResultData, gu } else { content = i18n.GetMessage(locale, i18n.GdprCompletedSpecificMessages, "Unknown", result.MessagesDeleted) } + + case gdprrelay.RequestTypeExportGuild: + if len(result.GuildIds) == 1 { + guildDisplay := utils.FormatGuildDisplay(result.GuildIds[0], guildNames) + content = i18n.GetMessage(locale, i18n.GdprCompletedExportGuild, guildDisplay, result.ExportedFiles) + } else { + content = i18n.GetMessage(locale, i18n.GdprCompletedExportGuildMulti, joinGuildDisplays(result.GuildIds, guildNames), result.ExportedFiles) + } + + case gdprrelay.RequestTypeExportUser: + content = i18n.GetMessage(locale, i18n.GdprCompletedExportUser, result.ExportedFiles) } if result.Error != nil { @@ -148,15 +186,29 @@ func (c *Callback) buildResultMessage(locale *i18n.Locale, result ResultData, gu return content } -func (c *Callback) buildResultComponents(locale *i18n.Locale, result ResultData, guildNames map[uint64]string) []component.Component { +func joinGuildDisplays(guildIds []uint64, guildNames map[uint64]string) string { + displays := make([]string, len(guildIds)) + for i, guildId := range guildIds { + displays[i] = utils.FormatGuildDisplay(guildId, guildNames) + } + + return strings.Join(displays, "\n* ") +} + +func (c *Callback) buildResultComponents(locale *i18n.Locale, result ResultData, guildNames map[uint64]string, deliveryErr error, delivered int) []component.Component { colour := utils.Green - if result.Error != nil { + content := c.buildResultMessage(locale, result, guildNames) + + if deliveryErr != nil { + colour = utils.Red + content = i18n.GetMessage(locale, i18n.GdprErrorExportDmFailed, delivered, len(result.ExportParts)) + } else if result.Error != nil { colour = utils.Red } innerComponents := []component.Component{ component.BuildTextDisplay(component.TextDisplay{ - Content: c.buildResultMessage(locale, result, guildNames), + Content: content, }), } @@ -178,9 +230,14 @@ func (c *Callback) editOriginalMessage(ctx context.Context, request gdprrelay.GD func (c *Callback) sendEphemeralFollowup(ctx context.Context, request gdprrelay.GDPRRequest, locale *i18n.Locale, result ResultData) error { var content string + nothingFound := result.TranscriptsDeleted == 0 && result.MessagesDeleted == 0 + if result.isExport() { + nothingFound = result.ExportedFiles == 0 + } + if result.Error != nil { content = i18n.GetMessage(locale, i18n.GdprFollowupError, result.Error.Error()) - } else if result.TranscriptsDeleted == 0 && result.MessagesDeleted == 0 { + } else if nothingFound { content = i18n.GetMessage(locale, i18n.GdprFollowupNoData) } else { content = i18n.GetMessage(locale, i18n.GdprFollowupSuccess) @@ -195,41 +252,195 @@ func (c *Callback) sendEphemeralFollowup(ctx context.Context, request gdprrelay. return err } -func (c *Callback) sendCompletionViaDM(ctx context.Context, request gdprrelay.GDPRRequest, locale *i18n.Locale, result ResultData) error { - scrambledUserId := utils.ScrambleUserId(request.UserId) - +func (c *Callback) openDM(ctx context.Context, userId uint64) (uint64, error) { if config.Conf.Discord.Token == "" { - c.logger.Error("Discord token not configured, cannot send DM", - zap.String("scrambled_user_id", scrambledUserId), - ) - return fmt.Errorf("discord token not configured") + return 0, fmt.Errorf("discord token not configured") } - dmChannel, err := rest.CreateDM(ctx, config.Conf.Discord.Token, c.rateLimiter, request.UserId) + channel, err := rest.CreateDM(ctx, config.Conf.Discord.Token, c.rateLimiter, userId) if err != nil { - c.logger.Error("Failed to create DM channel", - zap.Error(err), - zap.String("scrambled_user_id", scrambledUserId), - ) - return fmt.Errorf("failed to create DM channel: %w", err) + return 0, fmt.Errorf("failed to create DM channel: %w", err) } - components := c.buildResultComponents(locale, result, request.GuildNames) + return channel.Id, nil +} + +func (c *Callback) sendCompletionViaDM(ctx context.Context, request gdprrelay.GDPRRequest, components []component.Component) error { + channelId, err := c.openDM(ctx, request.UserId) + if err != nil { + return err + } data := rest.CreateMessageData{ Components: components, Flags: uint(message.FlagComponentsV2), } - _, err = rest.CreateMessage(ctx, config.Conf.Discord.Token, c.rateLimiter, dmChannel.Id, data) + if _, err := rest.CreateMessage(ctx, config.Conf.Discord.Token, c.rateLimiter, channelId, data); err != nil { + return fmt.Errorf("failed to send DM message: %w", err) + } + + return nil +} + +// The byte budget binds before the attachment count does whenever a part approaches the message +// limit, so a message carries several attachments only when the parts are small. +func batchParts(parts []export.Part, maxBytes int) [][]export.Part { + if maxBytes <= 0 { + maxBytes = defaultMaxMessageBytes + } + + var batches [][]export.Part + var current []export.Part + size := 0 + + for _, part := range parts { + tooMany := len(current) >= maxAttachmentsPerMessage + tooBig := size+len(part.Data) > maxBytes + + if len(current) > 0 && (tooMany || tooBig) { + batches = append(batches, current) + current, size = nil, 0 + } + + current = append(current, part) + size += len(part.Data) + } + + if len(current) > 0 { + batches = append(batches, current) + } + + return batches +} + +// Spaced out and retried per message so a rate limit delays delivery rather than dropping it. +// Returns how many archives reached the user. +func (c *Callback) sendExportViaDM(ctx context.Context, req gdprrelay.GDPRRequest, locale *i18n.Locale, result ResultData) (int, error) { + scrambledUserId := utils.ScrambleUserId(req.UserId) + + channelId, err := c.openDM(ctx, req.UserId) if err != nil { - c.logger.Error("Failed to send DM message", - zap.Error(err), + return 0, err + } + + batches := batchParts(result.ExportParts, config.Conf.Export.MaxMessageBytes) + delivered := 0 + + for i, batch := range batches { + if i > 0 { + if err := sleep(ctx, config.Conf.Export.DmDelay); err != nil { + return delivered, err + } + } + + content := i18n.GetMessage(locale, i18n.GdprExportDmMessage) + if len(batches) > 1 { + content = i18n.GetMessage(locale, i18n.GdprExportDmMessagePart, i+1, len(batches)) + } + + if err := c.sendBatch(ctx, channelId, content, batch); err != nil { + return delivered, fmt.Errorf("failed to send export message %d of %d: %w", i+1, len(batches), err) + } + + delivered += len(batch) + + c.logger.Debug("Export message delivered", zap.String("scrambled_user_id", scrambledUserId), - zap.Uint64("channel_id", dmChannel.Id), + zap.Int("message", i+1), + zap.Int("messages", len(batches)), + zap.Int("attachments", len(batch)), ) - return fmt.Errorf("failed to send DM message: %w", err) } - return nil + c.logger.Info("Export delivered via DM", + zap.String("scrambled_user_id", scrambledUserId), + zap.Int("parts", delivered), + zap.Int("messages", len(batches)), + zap.Int("files", result.ExportedFiles), + ) + + return delivered, nil +} + +func (c *Callback) sendBatch(ctx context.Context, channelId uint64, content string, batch []export.Part) error { + var lastErr error + + for attempt := 1; attempt <= maxPartAttempts; attempt++ { + if attempt > 1 { + backoff := time.Duration(attempt-1) * 2 * time.Second + if retryAfter, ok := retryAfterOf(lastErr); ok { + backoff = retryAfter + } + + if err := sleep(ctx, backoff); err != nil { + return err + } + } + + // Id must match the attachment's position: the encoder names the form field files[i]. + attachments := make([]request.Attachment, len(batch)) + for i, part := range batch { + attachments[i] = request.Attachment{ + Id: i, + FileName: part.Name, + File: request.File{ + ContentType: "application/zip", + // Fresh reader per attempt, or a retry uploads nothing. + Reader: bytes.NewReader(part.Data), + }, + } + } + + data := rest.CreateMessageData{ + Content: content, + Attachments: attachments, + } + + _, err := rest.CreateMessage(ctx, config.Conf.Discord.Token, c.rateLimiter, channelId, data) + if err == nil { + return nil + } + + lastErr = err + + if ctx.Err() != nil { + return ctx.Err() + } + } + + return lastErr +} + +func retryAfterOf(err error) (time.Duration, bool) { + if err == nil { + return 0, false + } + + var restErr request.RestError + if errors.As(err, &restErr) && restErr.StatusCode == 429 { + return 5 * time.Second, true + } + + if strings.Contains(strings.ToLower(err.Error()), "rate limit") { + return 5 * time.Second, true + } + + return 0, false +} + +func sleep(ctx context.Context, d time.Duration) error { + if d <= 0 { + return nil + } + + timer := time.NewTimer(d) + defer timer.Stop() + + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } } diff --git a/internal/config/config.go b/internal/config/config.go index 06ff6c1..72e5dfc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,39 +1,58 @@ package config import ( + "time" + "github.com/caarlos0/env/v10" "go.uber.org/zap/zapcore" ) type Config struct { - JsonLogs bool `env:"JSON_LOGS" envDefault:"false"` - LogLevel zapcore.Level `env:"LOG_LEVEL" envDefault:"info"` - MaxConcurrency int `env:"MAX_CONCURRENCY" envDefault:"1"` - MaxRetries int `env:"MAX_RETRIES" envDefault:"3"` + JsonLogs bool `env:"JSON_LOGS" envDefault:"false"` + LogLevel zapcore.Level `env:"LOG_LEVEL" envDefault:"info"` + MaxConcurrency int `env:"MAX_CONCURRENCY" envDefault:"1"` + MaxRetries int `env:"MAX_RETRIES" envDefault:"3"` Database struct { - Host string `env:"HOST"` - Database string `env:"NAME"` - Username string `env:"USER"` + Host string `env:"HOST,required,notEmpty"` + Database string `env:"NAME,required,notEmpty"` + Username string `env:"USER,required,notEmpty"` Password string `env:"PASSWORD"` - Threads int `env:"THREADS"` + Threads int `env:"THREADS" envDefault:"5"` } `envPrefix:"DATABASE_"` Redis struct { - Address string `env:"ADDR"` + Address string `env:"ADDR,required,notEmpty"` Password string `env:"PASSWD"` - Threads int `env:"THREADS"` + Threads int `env:"THREADS" envDefault:"5"` } `envPrefix:"REDIS_"` Archiver struct { - Url string `env:"URL"` - AesKey string `env:"AES_KEY"` + Url string `env:"URL,required,notEmpty"` + AesKey string `env:"AES_KEY,required,notEmpty"` } `envPrefix:"ARCHIVER_"` Discord struct { ProxyUrl string `env:"PROXY_URL"` - Token string `env:"TOKEN"` + Token string `env:"TOKEN,required,notEmpty"` } `envPrefix:"DISCORD_"` + + CacheDatabase struct { + Host string `env:"HOST"` + Database string `env:"NAME"` + Username string `env:"USER"` + Password string `env:"PASSWORD"` + Threads int `env:"THREADS" envDefault:"5"` + } `envPrefix:"CACHE_"` + + Export struct { + MaxPartBytes int `env:"MAX_PART_BYTES" envDefault:"12582912"` + MaxMessageBytes int `env:"MAX_MESSAGE_BYTES" envDefault:"25165824"` + MaxAttachmentBytes int `env:"MAX_ATTACHMENT_BYTES" envDefault:"20971520"` + DmDelay time.Duration `env:"DM_DELAY" envDefault:"3s"` + Timeout time.Duration `env:"TIMEOUT" envDefault:"30m"` + Concurrency int `env:"CONCURRENCY" envDefault:"15"` + } `envPrefix:"EXPORT_"` } var Conf Config @@ -43,7 +62,3 @@ func Parse() { panic(err) } } - -func init() { - Parse() -} diff --git a/internal/database/database.go b/internal/database/database.go index 4487191..751b0b2 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -9,19 +9,38 @@ import ( "go.uber.org/zap" ) -var Client *database.Database +var ( + Client *database.Database + Pool *pgxpool.Pool +) func Connect(logger *zap.Logger, host, dbName, username, password string, threads int) error { - uri := fmt.Sprintf("postgres://%s:%s@%s/%s?pool_max_conns=%d", username, password, host, dbName, threads) - - pool, err := pgxpool.Connect(context.Background(), uri) + pool, err := NewPool(context.Background(), host, dbName, username, password, threads) if err != nil { return fmt.Errorf("failed to connect to database: %w", err) } logger.Info("Connected to database") + Pool = pool Client = database.NewDatabase(pool) return nil } + +// The pool size is applied to the parsed config rather than interpolated into the URI: an unset +// thread count would render as "pool_max_conns=0", which pgx rejects, taking the worker down. +func NewPool(ctx context.Context, host, dbName, username, password string, threads int) (*pgxpool.Pool, error) { + uri := fmt.Sprintf("postgres://%s:%s@%s/%s", username, password, host, dbName) + + cfg, err := pgxpool.ParseConfig(uri) + if err != nil { + return nil, fmt.Errorf("failed to parse connection string: %w", err) + } + + if threads > 0 { + cfg.MaxConns = int32(threads) + } + + return pgxpool.ConnectConfig(ctx, cfg) +} diff --git a/internal/export/manifest.go b/internal/export/manifest.go new file mode 100644 index 0000000..2f105d7 --- /dev/null +++ b/internal/export/manifest.go @@ -0,0 +1,35 @@ +package export + +import "time" + +const ManifestVersion = 1 + +// Manifest lists every source considered, including ones deliberately withheld, so a +// data-subject request can be answered with evidence. +type Manifest struct { + Version int `json:"version"` + Kind string `json:"kind"` + GeneratedAt time.Time `json:"generated_at"` + Sources []Source `json:"sources"` + Files ManifestFiles `json:"files"` +} + +type ManifestFiles struct { + Written int `json:"written"` + Withheld []string `json:"withheld_too_large,omitempty"` +} + +type Source struct { + Name string `json:"name"` + Included bool `json:"included"` + Records int `json:"records"` + Reason string `json:"reason,omitempty"` +} + +func (m *Manifest) Add(name string, records int) { + m.Sources = append(m.Sources, Source{Name: name, Included: true, Records: records}) +} + +func (m *Manifest) Withhold(name, reason string) { + m.Sources = append(m.Sources, Source{Name: name, Included: false, Reason: reason}) +} diff --git a/internal/export/userdata.go b/internal/export/userdata.go new file mode 100644 index 0000000..c940f6a --- /dev/null +++ b/internal/export/userdata.go @@ -0,0 +1,170 @@ +package export + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/jackc/pgx/v4" +) + +const UserDataVersion = 1 + +type Querier interface { + Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error) +} + +type UserData struct { + Version int `json:"version"` + UserId string `json:"user_id"` + GeneratedAt time.Time `json:"generated_at"` + Tables map[string][]map[string]interface{} `json:"tables"` +} + +// Every user-keyed table in the database module is either listed here or withheld in GetUserData, +// so the manifest can account for all of them. +type source struct { + name string + query string +} + +var userSources = []source{ + {"blacklist", `SELECT "guild_id" FROM blacklist WHERE "user_id" = $1;`}, + {"close_request", `SELECT "guild_id", "ticket_id", "close_at", "close_reason" FROM close_request WHERE "user_id" = $1;`}, + {"first_response_time", `SELECT "guild_id", "ticket_id", "response_time" FROM first_response_time WHERE "user_id" = $1;`}, + {"participant", `SELECT "guild_id", "ticket_id" FROM participant WHERE "user_id" = $1;`}, + {"permissions", `SELECT "guild_id", "support", "admin" FROM permissions WHERE "user_id" = $1;`}, + {"support_team_members", `SELECT "team_id" FROM support_team_members WHERE "user_id" = $1;`}, + {"ticket_claims", `SELECT "guild_id", "ticket_id" FROM ticket_claims WHERE "user_id" = $1;`}, + {"ticket_members", `SELECT "guild_id", "ticket_id" FROM ticket_members WHERE "user_id" = $1;`}, + {"tickets", `SELECT "id", "guild_id", "channel_id", "open", "open_time", "welcome_message_id", "panel_id", "has_transcript" FROM tickets WHERE "user_id" = $1;`}, + {"used_keys", `SELECT "guild_id", "key", "activated_by" FROM used_keys WHERE "activated_by" = $1;`}, + {"user_guilds", `SELECT "guild_id", "name", "owner", "permissions", "icon" FROM user_guilds WHERE "user_id" = $1;`}, + {"whitelabel_users", `SELECT "expiry" FROM whitelabel_users WHERE "user_id" = $1;`}, + + // Not read by the previous export helper at all. + {"dashboard_users", `SELECT "last_seen" FROM dashboard_users WHERE "user_id" = $1;`}, + {"votes", `SELECT "vote_time" FROM votes WHERE "user_id" = $1;`}, + {"audit_logs", `SELECT "id", "guild_id", "action_type", "resource_type", "resource_id", "old_data", "new_data", "metadata", "created_at" FROM audit_logs WHERE "user_id" = $1;`}, + {"entitlements", `SELECT "id", "guild_id", "sku_id", "source", "expires_at" FROM entitlements WHERE "user_id" = $1;`}, + {"legacy_premium_entitlements", `SELECT "tier", "sku_label", "sku_id", "is_legacy", "expires_at" FROM legacy_premium_entitlements WHERE "user_id" = $1;`}, + {"legacy_premium_entitlement_guilds", `SELECT "guild_id", "entitlement_id" FROM legacy_premium_entitlement_guilds WHERE "user_id" = $1;`}, + {"patreon_entitlements", `SELECT "entitlement_id" FROM patreon_entitlements WHERE "user_id" = $1;`}, + {"discord_entitlements", `SELECT "entitlement_id" FROM discord_entitlements WHERE "discord_id" = $1;`}, + { + "exit_survey_responses", + `SELECT e."guild_id", e."ticket_id", e."form_id", e."question_id", e."response" + FROM exit_survey_responses e + INNER JOIN tickets t ON t."guild_id" = e."guild_id" AND t."id" = e."ticket_id" + WHERE t."user_id" = $1;`, + }, +} + +// Unlike the upstream helper this replaces, it takes a context, so a slow export is bounded by the +// caller's deadline instead of running unbounded on context.Background(). +func GetUserData(ctx context.Context, q Querier, userId uint64, manifest *Manifest) (*UserData, error) { + data := &UserData{ + Version: UserDataVersion, + UserId: fmt.Sprintf("%d", userId), + GeneratedAt: time.Now().UTC(), + Tables: make(map[string][]map[string]interface{}), + } + + for _, src := range userSources { + rows, err := q.Query(ctx, src.query, userId) + if err != nil { + return nil, fmt.Errorf("failed to query %s: %w", src.name, err) + } + + records, err := rowsToMaps(rows) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", src.name, err) + } + + data.Tables[src.name] = records + manifest.Add(src.name, len(records)) + } + + if err := addWhitelabel(ctx, q, userId, data, manifest); err != nil { + return nil, err + } + + return data, nil +} + +// The bot token is a live credential. Shipping it in an archive delivered over Discord would hand +// anyone who sees that message full control of the user's bot, so it is withheld. +func addWhitelabel(ctx context.Context, q Querier, userId uint64, data *UserData, manifest *Manifest) error { + rows, err := q.Query(ctx, `SELECT "bot_id", "public_key" FROM whitelabel WHERE "user_id" = $1;`, userId) + if err != nil { + return fmt.Errorf("failed to query whitelabel: %w", err) + } + + records, err := rowsToMaps(rows) + if err != nil { + return fmt.Errorf("failed to read whitelabel: %w", err) + } + + data.Tables["whitelabel"] = records + manifest.Add("whitelabel", len(records)) + + if len(records) > 0 { + manifest.Withhold("whitelabel.token", "live bot credential withheld; rotate it from the dashboard if you need a copy") + } + + return nil +} + +func rowsToMaps(rows pgx.Rows) ([]map[string]interface{}, error) { + defer rows.Close() + + fields := rows.FieldDescriptions() + out := make([]map[string]interface{}, 0) + + for rows.Next() { + values, err := rows.Values() + if err != nil { + return nil, err + } + + record := make(map[string]interface{}, len(fields)) + for i, field := range fields { + if i < len(values) { + record[string(field.Name)] = normalise(values[i]) + } + } + + out = append(out, record) + } + + return out, rows.Err() +} + +// Discord snowflakes exceed the range a JSON number survives in most parsers, so anything that +// large is emitted as a string. +func normalise(v interface{}) interface{} { + switch t := v.(type) { + case nil: + return nil + case int64: + if t > 1<<53 || t < -(1<<53) { + return fmt.Sprintf("%d", t) + } + return t + case uint64: + if t > 1<<53 { + return fmt.Sprintf("%d", t) + } + return t + case [16]byte: + return fmt.Sprintf("%x-%x-%x-%x-%x", t[0:4], t[4:6], t[6:8], t[8:10], t[10:16]) + case []byte: + if json.Valid(t) { + return json.RawMessage(t) + } + return string(t) + default: + return v + } +} diff --git a/internal/export/zip.go b/internal/export/zip.go new file mode 100644 index 0000000..33ff94b --- /dev/null +++ b/internal/export/zip.go @@ -0,0 +1,167 @@ +package export + +import ( + "archive/zip" + "bytes" + "compress/flate" + "fmt" + "hash/crc32" + "io" + "time" +) + +// Two per message: Discord caps a request at 25 MiB and an attachment at 20 MiB, so 12 MiB uses +// the request budget almost exactly where a single 20 MiB part would waste a fifth of it. +const DefaultMaxPartBytes = 12 * 1024 * 1024 + +// Local header (30) + central directory record (46) + slack for extra fields the writer may emit. +const perEntryOverhead = 30 + 46 + 64 + +const trailerOverhead = 128 + +type Part struct { + Name string + Data []byte +} + +// Each part is independently valid, so a recipient can open any one without the others. +type Builder struct { + maxBytes int + + buf *bytes.Buffer + writer *zip.Writer + entries int + // Central directory bytes the current part still has to write. + reserved int + + done [][]byte + oversized []string + total int +} + +func NewBuilder(maxBytes int) *Builder { + if maxBytes <= 0 { + maxBytes = DefaultMaxPartBytes + } + + return &Builder{maxBytes: maxBytes} +} + +func (b *Builder) start() { + b.buf = new(bytes.Buffer) + b.writer = zip.NewWriter(b.buf) + b.entries = 0 + b.reserved = trailerOverhead +} + +// An entry too large for even an empty part is recorded by Oversized and skipped, rather than +// emitting an archive Discord will reject. +func (b *Builder) Add(name string, data []byte) error { + compressed, crc, err := deflate(data) + if err != nil { + return fmt.Errorf("failed to compress zip entry %q: %w", name, err) + } + + cost := len(compressed) + perEntryOverhead + 2*len(name) + + if cost+trailerOverhead > b.maxBytes { + b.oversized = append(b.oversized, name) + return nil + } + + if b.writer == nil { + b.start() + } + + if err := b.writer.Flush(); err != nil { + return fmt.Errorf("failed to flush zip part: %w", err) + } + + if b.entries > 0 && b.buf.Len()+b.reserved+cost > b.maxBytes { + if err := b.closePart(); err != nil { + return err + } + b.start() + } + + header := &zip.FileHeader{ + Name: name, + Method: zip.Deflate, + Modified: time.Now().UTC(), + CRC32: crc, + CompressedSize64: uint64(len(compressed)), + UncompressedSize64: uint64(len(data)), + } + + w, err := b.writer.CreateRaw(header) + if err != nil { + return fmt.Errorf("failed to create zip entry %q: %w", name, err) + } + + if _, err := w.Write(compressed); err != nil { + return fmt.Errorf("failed to write zip entry %q: %w", name, err) + } + + b.entries++ + b.total++ + b.reserved += 46 + len(name) + 64 + + return nil +} + +func (b *Builder) closePart() error { + if err := b.writer.Close(); err != nil { + return fmt.Errorf("failed to close zip part: %w", err) + } + + b.done = append(b.done, b.buf.Bytes()) + b.writer = nil + b.buf = nil + + return nil +} + +func (b *Builder) Count() int { return b.total } + +func (b *Builder) Oversized() []string { return b.oversized } + +func (b *Builder) Finish(baseName string) ([]Part, error) { + if b.writer != nil { + if b.entries == 0 { + b.writer = nil + b.buf = nil + } else if err := b.closePart(); err != nil { + return nil, err + } + } + + parts := make([]Part, len(b.done)) + for i, data := range b.done { + name := fmt.Sprintf("%s.zip", baseName) + if len(b.done) > 1 { + name = fmt.Sprintf("%s_%d-%d.zip", baseName, i+1, len(b.done)) + } + parts[i] = Part{Name: name, Data: data} + } + + return parts, nil +} + +func deflate(data []byte) ([]byte, uint32, error) { + var out bytes.Buffer + + w, err := flate.NewWriter(&out, flate.DefaultCompression) + if err != nil { + return nil, 0, err + } + + if _, err := io.Copy(w, bytes.NewReader(data)); err != nil { + return nil, 0, err + } + + if err := w.Close(); err != nil { + return nil, 0, err + } + + return out.Bytes(), crc32.ChecksumIEEE(data), nil +} diff --git a/internal/gdprrelay/gdprrelay.go b/internal/gdprrelay/gdprrelay.go index a29be0f..89809f8 100644 --- a/internal/gdprrelay/gdprrelay.go +++ b/internal/gdprrelay/gdprrelay.go @@ -20,6 +20,8 @@ const ( RequestTypeSpecificTranscripts // Delete specific transcript archives by ticket IDs RequestTypeAllMessages // Delete all ticket messages for specified guilds RequestTypeSpecificMessages // Delete specific ticket messages by ticket IDs + RequestTypeExportGuild // Export all transcript data for specified guilds + RequestTypeExportUser // Export all personal data for the requesting user ) // GDPRRequest represents a user's request to delete their data under GDPR regulations diff --git a/internal/processor/export_requests.go b/internal/processor/export_requests.go new file mode 100644 index 0000000..9a6af72 --- /dev/null +++ b/internal/processor/export_requests.go @@ -0,0 +1,491 @@ +package processor + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "sync/atomic" + "time" + + "github.com/TicketsBot-cloud/archiverclient" + "github.com/TicketsBot-cloud/gdpr-worker/internal/archiver" + "github.com/TicketsBot-cloud/gdpr-worker/internal/config" + "github.com/TicketsBot-cloud/gdpr-worker/internal/database" + "github.com/TicketsBot-cloud/gdpr-worker/internal/export" + "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" + "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" + v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +type transcriptRef struct { + guildId uint64 + ticketId int +} + +type transcriptEntry struct { + ref transcriptRef + name string + data []byte +} + +type archiveFile struct { + name string + data []byte +} + +func (p *Processor) exportConcurrency() int { + if config.Conf.Export.Concurrency > 0 { + return config.Conf.Export.Concurrency + } + return 15 +} + +func transcriptPath(ref transcriptRef) string { + return fmt.Sprintf("transcripts/%d/%d.json", ref.guildId, ref.ticketId) +} + +// A part has to fit both Discord ceilings on its own, or it could never be delivered. +func maxPartBytes() int { + size := config.Conf.Export.MaxPartBytes + for _, limit := range []int{config.Conf.Export.MaxAttachmentBytes, config.Conf.Export.MaxMessageBytes} { + if limit > 0 && limit < size { + size = limit + } + } + + return size +} + +func (p *Processor) processExportGuild(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { + if len(request.GuildIds) == 0 { + return ProcessResult{Error: fmt.Errorf("no server ID provided")} + } + + scrambledUserId := utils.ScrambleUserId(request.UserId) + + if err := p.verifyAllGuildsOwnership(ctx, request.GuildIds, request.UserId); err != nil { + p.logger.Error("Guild ownership verification failed for export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: err} + } + + manifest := &export.Manifest{ + Version: export.ManifestVersion, + Kind: "guild", + GeneratedAt: time.Now().UTC(), + } + + var refs []transcriptRef + for _, guildId := range request.GuildIds { + found, err := p.transcriptRefsForGuild(ctx, guildId) + if err != nil { + p.logger.Error("Failed to query tickets for guild export", + zap.Uint64("guild_id", guildId), + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to list transcripts for server %d: %w", guildId, err)} + } + + manifest.Add(fmt.Sprintf("transcripts/%d", guildId), len(found)) + refs = append(refs, found...) + } + + baseName := "guild_export" + if len(request.GuildIds) == 1 { + baseName = fmt.Sprintf("guild_export_%d", request.GuildIds[0]) + } + + return p.buildTranscriptExport(ctx, exportJob{ + refs: refs, + baseName: baseName, + manifest: manifest, + scrambledUserId: scrambledUserId, + logLabel: "guild export", + }) +} + +func (p *Processor) processExportUser(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { + scrambledUserId := utils.ScrambleUserId(request.UserId) + + manifest := &export.Manifest{ + Version: export.ManifestVersion, + Kind: "user", + GeneratedAt: time.Now().UTC(), + } + + userData, err := export.GetUserData(ctx, database.Pool, request.UserId, manifest) + if err != nil { + p.logger.Error("Failed to collect user database records for export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to retrieve database data: %w", err)} + } + + dbJson, err := json.Marshal(userData) + if err != nil { + return ProcessResult{Error: fmt.Errorf("failed to serialise database data: %w", err)} + } + + extras := []archiveFile{{name: "database.json", data: dbJson}} + + if cacheJson := p.userCacheData(ctx, request.UserId, manifest); cacheJson != nil { + extras = append(extras, archiveFile{name: "cache.json", data: cacheJson}) + } + + refs, err := p.transcriptRefsForUser(ctx, request.UserId) + if err != nil { + p.logger.Error("Failed to query tickets for user export", + zap.String("scrambled_user_id", scrambledUserId), + zap.Error(err), + ) + return ProcessResult{Error: fmt.Errorf("failed to query user tickets: %w", err)} + } + + manifest.Add("transcripts", len(refs)) + + return p.buildTranscriptExport(ctx, exportJob{ + refs: refs, + baseName: "user_export", + manifest: manifest, + scrambledUserId: scrambledUserId, + logLabel: "user export", + extras: extras, + filterFor: request.UserId, + }) +} + +type exportJob struct { + refs []transcriptRef + baseName string + manifest *export.Manifest + scrambledUserId string + logLabel string + extras []archiveFile + // When non-zero, each transcript is reduced to that user's own messages. + filterFor uint64 +} + +// Streamed into the archive as they arrive, so peak memory tracks the concurrency limit rather +// than the size of the whole export. +func (p *Processor) buildTranscriptExport(ctx context.Context, job exportJob) ProcessResult { + builder := export.NewBuilder(maxPartBytes()) + + for _, extra := range job.extras { + if err := builder.Add(extra.name, extra.data); err != nil { + return ProcessResult{Error: err} + } + } + + results := make(chan transcriptEntry, p.exportConcurrency()) + done := make(chan struct{}) + + var downloadErr error + var missing atomic.Int64 + + go func() { + defer close(done) + + g, gCtx := errgroup.WithContext(ctx) + g.SetLimit(p.exportConcurrency()) + + for _, ref := range job.refs { + g.Go(func() error { + entry, absent, err := p.fetchTranscript(gCtx, ref, job.filterFor) + if err != nil { + return err + } + + if absent { + missing.Add(1) + } + + if entry == nil { + return nil + } + + select { + case results <- *entry: + return nil + case <-gCtx.Done(): + return gCtx.Err() + } + }) + } + + downloadErr = g.Wait() + close(results) + }() + + var addErr error + for entry := range results { + if addErr != nil { + continue + } + if err := builder.Add(entry.name, entry.data); err != nil { + addErr = err + } + } + + <-done + + if downloadErr != nil { + p.logger.Error("Transcript download failed during export", + zap.String("scrambled_user_id", job.scrambledUserId), + zap.String("export", job.logLabel), + zap.Error(downloadErr), + ) + return ProcessResult{Error: fmt.Errorf("failed to download transcripts: %w", downloadErr)} + } + + if addErr != nil { + return ProcessResult{Error: addErr} + } + + if absent := int(missing.Load()); absent > 0 { + job.manifest.Withhold("transcripts.unavailable", fmt.Sprintf("%d recorded transcripts are no longer held by the archive store", absent)) + p.logger.Warn("Transcripts recorded in the database were absent from the archive store", + zap.String("scrambled_user_id", job.scrambledUserId), + zap.String("export", job.logLabel), + zap.Int("count", absent), + ) + } + + if oversized := builder.Oversized(); len(oversized) > 0 { + job.manifest.Files.Withheld = oversized + p.logger.Warn("Entries too large to deliver were withheld from export", + zap.String("scrambled_user_id", job.scrambledUserId), + zap.Int("count", len(oversized)), + ) + } + + // Reported as no-data rather than delivering an empty archive. + if builder.Count() == 0 { + p.logger.Info("Export produced no files", + zap.String("scrambled_user_id", job.scrambledUserId), + zap.String("export", job.logLabel), + ) + return ProcessResult{} + } + + job.manifest.Files.Written = builder.Count() + + if manifestJson, err := json.Marshal(job.manifest); err == nil { + if err := builder.Add("manifest.json", manifestJson); err != nil { + return ProcessResult{Error: err} + } + } + + parts, err := builder.Finish(job.baseName) + if err != nil { + return ProcessResult{Error: fmt.Errorf("failed to finalise export archive: %w", err)} + } + + total := 0 + for _, part := range parts { + total += len(part.Data) + } + + p.logger.Info("Export completed", + zap.String("scrambled_user_id", job.scrambledUserId), + zap.String("export", job.logLabel), + zap.Int("files", builder.Count()), + zap.Int("parts", len(parts)), + zap.Int("bytes", total), + ) + + return ProcessResult{ + ExportParts: parts, + ExportedFiles: builder.Count(), + } +} + +// Returns nil for a transcript that is absent or, when filtering, holds nothing of the +// requester's. The bool marks the absent case, which is ordinary rather than a failure: the object +// can be purged independently of the database row. +func (p *Processor) fetchTranscript(ctx context.Context, ref transcriptRef, filterFor uint64) (*transcriptEntry, bool, error) { + transcript, err := archiver.Client.Get(ctx, ref.guildId, ref.ticketId) + if err != nil { + if isTranscriptMissing(err) { + p.logger.Debug("Transcript absent from archive store, skipping", + zap.Uint64("guild_id", ref.guildId), + zap.Int("ticket_id", ref.ticketId), + zap.Error(err), + ) + return nil, true, nil + } + return nil, false, err + } + + if filterFor != 0 { + if !filterTranscriptToUser(&transcript, filterFor) { + return nil, false, nil + } + } + + data, err := json.Marshal(transcript) + if err != nil { + return nil, false, fmt.Errorf("failed to serialise transcript %d/%d: %w", ref.guildId, ref.ticketId, err) + } + + return &transcriptEntry{ref: ref, name: transcriptPath(ref), data: data}, false, nil +} + +// The archiver proxy only maps a 404 to ErrNotFound; an object missing from the bucket surfaces as +// the storage backend's own wording behind another status, so the text has to be matched too. +func isTranscriptMissing(err error) bool { + if errors.Is(err, archiverclient.ErrNotFound) { + return true + } + + msg := strings.ToLower(err.Error()) + for _, want := range []string{"does not exist", "no such key", "not found"} { + if strings.Contains(msg, want) { + return true + } + } + + return false +} + +// Reports whether anything of the requester's remains. +func filterTranscriptToUser(transcript *v2.Transcript, userId uint64) bool { + var messages []v2.Message + for _, msg := range transcript.Messages { + if msg.AuthorId == userId { + messages = append(messages, msg) + } + } + + if len(messages) == 0 { + return false + } + + transcript.Messages = messages + transcript.Entities.Channels = nil + transcript.Entities.Roles = nil + + // Key by the requested id, not by the entity's own Id: once the anonymization flow has run the + // stored entity carries Id 0, and keying by that would leave the messages unresolvable. + entity := transcript.Entities.Users[userId] + if entity.Id == 0 { + entity.Id = userId + } + + transcript.Entities.Users = map[uint64]v2.User{userId: entity} + + return true +} + +func (p *Processor) transcriptRefsForGuild(ctx context.Context, guildId uint64) ([]transcriptRef, error) { + // Open tickets are excluded: the transcript only reaches the archive store on close. + query := `SELECT "id" FROM tickets WHERE "guild_id" = $1 AND "has_transcript" = true AND "open" = false ORDER BY "id";` + + rows, err := database.Client.Tickets.Query(ctx, query, guildId) + if err != nil { + return nil, err + } + defer rows.Close() + + var refs []transcriptRef + for rows.Next() { + var ticketId int + if err := rows.Scan(&ticketId); err != nil { + return nil, err + } + refs = append(refs, transcriptRef{guildId: guildId, ticketId: ticketId}) + } + + return refs, rows.Err() +} + +// The anonymization flow reads ticket_members and the previous export read participant; taking +// either alone silently drops transcripts the user appears in. +func (p *Processor) transcriptRefsForUser(ctx context.Context, userId uint64) ([]transcriptRef, error) { + query := ` + SELECT s.guild_id, s.ticket_id + FROM ( + SELECT "guild_id", "id" AS ticket_id FROM tickets WHERE "user_id" = $1 + UNION + SELECT "guild_id", "ticket_id" FROM participant WHERE "user_id" = $1 + UNION + SELECT "guild_id", "ticket_id" FROM ticket_members WHERE "user_id" = $1 + ) s + INNER JOIN tickets t ON t."guild_id" = s.guild_id AND t."id" = s.ticket_id + WHERE t."has_transcript" = true AND t."open" = false + ORDER BY s.guild_id, s.ticket_id;` + + rows, err := database.Client.Tickets.Query(ctx, query, userId) + if err != nil { + return nil, err + } + defer rows.Close() + + var refs []transcriptRef + for rows.Next() { + var ref transcriptRef + if err := rows.Scan(&ref.guildId, &ref.ticketId); err != nil { + return nil, err + } + refs = append(refs, ref) + } + + return refs, rows.Err() +} + +// Returns nil when the cache database is unconfigured or unreadable; the manifest records which. +func (p *Processor) userCacheData(ctx context.Context, userId uint64, manifest *export.Manifest) []byte { + if p.cachePool == nil { + manifest.Withhold("cache", "cache database not configured for this deployment") + return nil + } + + data := make(map[string]interface{}) + + var user json.RawMessage + if err := p.cachePool.QueryRow(ctx, `SELECT "data" FROM users WHERE "user_id" = $1;`, userId).Scan(&user); err == nil { + data["user"] = user + } + + members := make(map[string]json.RawMessage) + + rows, err := p.cachePool.Query(ctx, `SELECT "guild_id", "data" FROM members WHERE "user_id" = $1;`, userId) + if err != nil { + manifest.Withhold("cache", "cache database could not be read") + p.logger.Warn("Failed to read cache member data for export", + zap.String("scrambled_user_id", utils.ScrambleUserId(userId)), + zap.Error(err), + ) + return nil + } + + defer rows.Close() + + for rows.Next() { + var guildId uint64 + var raw json.RawMessage + if err := rows.Scan(&guildId, &raw); err != nil { + continue + } + members[fmt.Sprintf("%d", guildId)] = raw + } + + data["member_data"] = members + + encoded, err := json.Marshal(data) + if err != nil { + manifest.Withhold("cache", "cache data could not be serialised") + return nil + } + + manifest.Add("cache", len(members)) + + return encoded +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 8b7447e..f68d1e8 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -12,31 +12,37 @@ import ( "github.com/TicketsBot-cloud/gdpr-worker/internal/archiver" "github.com/TicketsBot-cloud/gdpr-worker/internal/config" "github.com/TicketsBot-cloud/gdpr-worker/internal/database" + "github.com/TicketsBot-cloud/gdpr-worker/internal/export" "github.com/TicketsBot-cloud/gdpr-worker/internal/gdprrelay" "github.com/TicketsBot-cloud/gdpr-worker/internal/utils" v2 "github.com/TicketsBot-cloud/logarchiver/pkg/model/v2" + "github.com/jackc/pgx/v4/pgxpool" "go.uber.org/zap" ) -// Processor handles the execution of GDPR data deletion requests +// Processor handles the execution of GDPR data deletion and export requests type Processor struct { logger *zap.Logger rateLimiter *ratelimit.Ratelimiter + cachePool *pgxpool.Pool } -func New(logger *zap.Logger) *Processor { +func New(logger *zap.Logger, cachePool *pgxpool.Pool) *Processor { store := ratelimit.NewMemoryStore() return &Processor{ logger: logger, rateLimiter: ratelimit.NewRateLimiter(store, 0), + cachePool: cachePool, } } // ProcessResult contains the outcome of processing a GDPR request type ProcessResult struct { - TranscriptsDeleted int // Number of transcript archives deleted from archiver - MessagesDeleted int // Number of ticket messages deleted from database - Error error // Error if the processing failed, nil on success + TranscriptsDeleted int // Number of transcript archives deleted from archiver + MessagesDeleted int // Number of ticket messages deleted from database + Error error // Error if the processing failed, nil on success + ExportParts []export.Part // Deliverable ZIP archives for export requests + ExportedFiles int // Number of files written across those archives } func (p *Processor) Process(ctx context.Context, request gdprrelay.GDPRRequest) ProcessResult { @@ -49,6 +55,10 @@ func (p *Processor) Process(ctx context.Context, request gdprrelay.GDPRRequest) return p.processAllMessages(ctx, request) case gdprrelay.RequestTypeSpecificMessages: return p.processSpecificMessages(ctx, request) + case gdprrelay.RequestTypeExportGuild: + return p.processExportGuild(ctx, request) + case gdprrelay.RequestTypeExportUser: + return p.processExportUser(ctx, request) default: return ProcessResult{Error: fmt.Errorf("unknown GDPR request type: %d", request.Type)} } @@ -72,7 +82,7 @@ func (p *Processor) verifyGuildOwnership(ctx context.Context, guildId, userId ui zap.Uint64("guild_id", guildId), zap.Error(err), ) - return fmt.Errorf("failed to verify guild ownership: unable to fetch guild information") + return fmt.Errorf("the bot is not in server %d, so ownership could not be verified - add it back and try again", guildId) } if guild.OwnerId != userId { diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 9b54513..39c90b4 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -73,6 +73,10 @@ func GetRequestTypeName(requestType int) string { return "AllMessages" case 3: return "SpecificMessages" + case 4: + return "ExportGuild" + case 5: + return "ExportUser" default: return fmt.Sprintf("Unknown(%d)", requestType) }