RM-251: user & guild exports

.env.example

@@ -25,3 +25,21 @@ ARCHIVER_AES_KEY=
 # Discord Configuration
 DISCORD_PROXY_URL=
 DISCORD_TOKEN=
+
+# Cache Database Configuration (optional)
+# When unset, exports omit cache.json and record the omission in the manifest.
+CACHE_HOST=
+CACHE_NAME=
+CACHE_USER=
+CACHE_PASSWORD=
+CACHE_THREADS=
+
+# Data Export Configuration
+# Discord caps a request at 25 MiB, an attachment at 20 MiB, and a message at 10 attachments.
+# 12 MiB parts put two per message, using the request budget almost exactly.
+EXPORT_MAX_PART_BYTES=
+EXPORT_MAX_MESSAGE_BYTES=
+EXPORT_MAX_ATTACHMENT_BYTES=
+EXPORT_DM_DELAY=
+EXPORT_TIMEOUT=
+EXPORT_CONCURRENCY=

Dockerfile

@@ -1,5 +1,5 @@
 # Build container
-FROM golang:1.24 AS builder
+FROM golang:1.25 AS builder
 
 RUN go version
 

cmd/main.go

@@ -2,11 +2,14 @@ package main
 
 import (
 	"context"
+	"net/http"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
+	"github.com/TicketsBot-cloud/gdl/rest/request"
 	"github.com/TicketsBot-cloud/gdpr-worker/i18n"
 	"github.com/TicketsBot-cloud/gdpr-worker/internal/archiver"
 	"github.com/TicketsBot-cloud/gdpr-worker/internal/callback"
@@ -17,12 +20,22 @@ import (
 	"github.com/TicketsBot-cloud/gdpr-worker/internal/processor"
 	"github.com/TicketsBot-cloud/gdpr-worker/internal/utils"
 	"github.com/go-redis/redis/v8"
+	"github.com/jackc/pgx/v4/pgxpool"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 
 	_ "github.com/joho/godotenv/autoload"
 )
 
+// The proxy expects the request addressed to its own host rather than the usual CONNECT protocol.
+// Application endpoints stay on discord.com, matching how the bot worker registers the same hook.
+func proxyHook(token string, req *http.Request) {
+	if !strings.HasPrefix(req.URL.Path, "/api/v9/applications/") {
+		req.URL.Scheme = "http"
+		req.URL.Host = config.Conf.Discord.ProxyUrl
+	}
+}
+
 func main() {
 	config.Parse()
 
@@ -35,10 +48,16 @@ func main() {
 		return
 	}
 
+	if config.Conf.Discord.ProxyUrl != "" {
+		logger.Info("Configuring REST proxy", zap.String("url", config.Conf.Discord.ProxyUrl))
+		request.RegisterPreRequestHook(proxyHook)
+	}
+
 	logger.Info("Connecting to Redis")
 	redisClient := redis.NewClient(&redis.Options{
 		Addr:     config.Conf.Redis.Address,
 		Password: config.Conf.Redis.Password,
+		PoolSize: config.Conf.Redis.Threads,
 		DB:       0,
 	})
 
@@ -61,14 +80,36 @@ func main() {
 		return
 	}
 
-	logger.Info("Initializing archiver client")
+	logger.Info("Initialising archiver client")
 	archiver.Initialize(
 		logger.With(),
 		config.Conf.Archiver.Url,
 		config.Conf.Archiver.AesKey,
 	)
 
-	proc := processor.New(logger.With())
+	var cachePool *pgxpool.Pool
+	if config.Conf.CacheDatabase.Host != "" {
+		logger.Info("Connecting to cache database")
+
+		var err error
+		cachePool, err = database.NewPool(
+			context.Background(),
+			config.Conf.CacheDatabase.Host,
+			config.Conf.CacheDatabase.Database,
+			config.Conf.CacheDatabase.Username,
+			config.Conf.CacheDatabase.Password,
+			config.Conf.CacheDatabase.Threads,
+		)
+		if err != nil {
+			logger.Fatal("Failed to connect to cache database", zap.Error(err))
+			return
+		}
+		logger.Info("Connected to cache database")
+	} else {
+		logger.Warn("Cache database not configured, user export cache data will be unavailable")
+	}
+
+	proc := processor.New(logger.With(), cachePool)
 
 	callbackHandler := callback.New(
 		logger.With(),
@@ -95,7 +136,8 @@ func main() {
 					<-semaphore
 				}()
 
-				processCtx := context.Background()
+				processCtx, processCancel := context.WithTimeout(context.Background(), config.Conf.Export.Timeout)
+				defer processCancel()
 
 				scrambledId := utils.ScrambleUserId(req.Request.UserId)
 				requestTypeName := utils.GetRequestTypeName(int(req.Request.Type))
@@ -142,23 +184,27 @@ func main() {
 				}
 
 				callbackData := callback.ResultData{
-					TranscriptsDeleted:    result.TranscriptsDeleted,
-					MessagesDeleted: result.MessagesDeleted,
-					Error:           result.Error,
-					RequestType:     req.Request.Type,
-					GuildIds:        req.Request.GuildIds,
-					TicketIds:       req.Request.TicketIds,
+					TranscriptsDeleted: result.TranscriptsDeleted,
+					MessagesDeleted:    result.MessagesDeleted,
+					Error:              result.Error,
+					RequestType:        req.Request.Type,
+					GuildIds:           req.Request.GuildIds,
+					TicketIds:          req.Request.TicketIds,
+					ExportParts:        result.ExportParts,
+					ExportedFiles:      result.ExportedFiles,
 				}
 
-				callbackCtx, callbackCancel := context.WithTimeout(context.Background(), 30*time.Second)
+				callbackCtx, callbackCancel := context.WithTimeout(context.Background(), 10*time.Minute)
 				defer callbackCancel()
 
-				if updateErr := database.Client.GdprLogs.UpdateLogStatus(req.RequestID, "Completed"); updateErr != nil {
-					logger.Error("Failed to update GDPR log status to Completed",
-						zap.Uint64("request_id", uint64(req.RequestID)),
-						zap.String("scrambled_user_id", scrambledId),
-						zap.Error(updateErr),
-					)
+				if result.Error == nil {
+					if updateErr := database.Client.GdprLogs.UpdateLogStatus(req.RequestID, "Completed"); updateErr != nil {
+						logger.Error("Failed to update GDPR log status to Completed",
+							zap.Uint64("request_id", uint64(req.RequestID)),
+							zap.String("scrambled_user_id", scrambledId),
+							zap.Error(updateErr),
+						)
+					}
 				}
 
 				if err := callbackHandler.SendCompletion(callbackCtx, req.Request, callbackData); err != nil {

go.mod

@@ -1,6 +1,6 @@
 module github.com/TicketsBot-cloud/gdpr-worker
 
-go 1.24.0
+go 1.25.0
 
 //replace github.com/TicketsBot-cloud/database => ../database
 
@@ -12,26 +12,27 @@ go 1.24.0
 
 require (
 	github.com/TicketsBot-cloud/archiverclient v0.0.0-20251015181023-f0b66a074704
-	github.com/TicketsBot-cloud/database v0.0.0-20251018202538-7f9567e1aeab
-	github.com/TicketsBot-cloud/gdl v0.0.0-20251007163257-7e59b92d02dd
-	github.com/TicketsBot-cloud/logarchiver v0.0.0-20250809082842-70aa389bcbdf
+	github.com/TicketsBot-cloud/database v0.0.0-20260308193919-30a698fefa8b
+	github.com/TicketsBot-cloud/gdl v0.0.0-20260612070331-a3947b410d3e
+	github.com/TicketsBot-cloud/logarchiver v0.0.0-20260909063657-e32e33f1289b
 	github.com/caarlos0/env/v10 v10.0.0
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/jackc/pgx/v4 v4.18.3
 	github.com/joho/godotenv v1.5.1
-	go.uber.org/zap v1.27.0
+	go.uber.org/zap v1.27.1
+	golang.org/x/sync v0.20.0
 )
 
 require (
-	github.com/TicketsBot-cloud/common v0.0.0-20250509064208-a2d357175463 // indirect
+	github.com/TicketsBot-cloud/common v0.0.0-20260210203202-54154661338e // indirect
 	github.com/TicketsBot/common v0.0.0-20241117150316-ff54c97b45c1 // indirect
 	github.com/TicketsBot/ttlcache v1.6.1-0.20200405150101-acc18e37b261 // indirect
+	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/caarlos0/env v3.5.0+incompatible // indirect
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
 	github.com/jackc/pgconn v1.14.3 // indirect
@@ -41,16 +42,17 @@ require (
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/pgtype v1.14.4 // indirect
 	github.com/jackc/pgx v3.6.2+incompatible // indirect
-	github.com/jackc/pgx/v5 v5.7.6 // indirect
+	github.com/jackc/pgx/v5 v5.9.1 // indirect
 	github.com/jackc/puddle v1.3.0 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/juju/ratelimit v1.0.1 // indirect
-	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/klauspost/compress v1.18.5 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/klauspost/crc32 v1.3.0 // indirect
 	github.com/minio/crc64nvme v1.1.1 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/minio-go/v7 v7.0.95 // indirect
+	github.com/minio/minio-go/v7 v7.0.99 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pasztorpisti/qs v0.0.0-20171216220353-8d6c33ee906c // indirect
@@ -59,12 +61,12 @@ require (
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
-	github.com/tinylib/msgp v1.4.0 // indirect
-	go.uber.org/multierr v1.10.0 // indirect
-	golang.org/x/crypto v0.43.0 // indirect
+	github.com/tinylib/msgp v1.6.3 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/crypto v0.49.0 // indirect
 	golang.org/x/exp v0.0.0-20241108190413-2d47ceb2692f // indirect
-	golang.org/x/net v0.46.0 // indirect
-	golang.org/x/sync v0.17.0 // indirect
-	golang.org/x/sys v0.37.0 // indirect
-	golang.org/x/text v0.30.0 // indirect
+	golang.org/x/net v0.52.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	golang.org/x/text v0.35.0 // indirect
 )