diff --git a/src/main/java/com/kw/readwith/apiPayload/code/status/ErrorStatus.java b/src/main/java/com/kw/readwith/apiPayload/code/status/ErrorStatus.java index 24cf2e9..ae69abc 100644 --- a/src/main/java/com/kw/readwith/apiPayload/code/status/ErrorStatus.java +++ b/src/main/java/com/kw/readwith/apiPayload/code/status/ErrorStatus.java @@ -65,6 +65,8 @@ public enum ErrorStatus implements BaseErrorCode { BOOK_DELETE_JOB_IN_PROGRESS(HttpStatus.CONFLICT, "ADMIN4027", "진행 중인 처리 job이 있어 책을 삭제할 수 없습니다."), IMAGE_FANOUT_JOB_ACTIVE(HttpStatus.CONFLICT, "ADMIN4028", "해당 책의 캐릭터 이미지 fan-out job이 이미 진행 중입니다."), IMAGE_FANOUT_JOB_NOT_FOUND(HttpStatus.NOT_FOUND, "ADMIN4029", "캐릭터 이미지 fan-out job을 찾을 수 없습니다."), + IMAGE_FANOUT_RETRY_NOT_ALLOWED(HttpStatus.BAD_REQUEST, "ADMIN4030", "결과 게시를 재시도할 수 없는 캐릭터 이미지 fan-out job 상태입니다."), + IMAGE_FANOUT_RESULT_NOT_AVAILABLE(HttpStatus.BAD_REQUEST, "ADMIN4031", "재적용할 OpenAI Batch 결과 파일이 없습니다."), IMAGE_GENERATION_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, "ADMIN5020", "캐릭터 이미지 생성에 실패했습니다."), // Bookmark diff --git a/src/main/java/com/kw/readwith/domain/CharacterImageAsset.java b/src/main/java/com/kw/readwith/domain/CharacterImageAsset.java index 8fc7185..5a3f82f 100644 --- a/src/main/java/com/kw/readwith/domain/CharacterImageAsset.java +++ b/src/main/java/com/kw/readwith/domain/CharacterImageAsset.java @@ -136,6 +136,11 @@ public void fail(String failureCode) { this.status = CharacterImageAssetStatus.FAILED; } + public void retryResultApplication() { + this.failureCode = null; + this.status = CharacterImageAssetStatus.GENERATING; + } + public void markStaleReference() { this.status = CharacterImageAssetStatus.STALE_REFERENCE; } diff --git a/src/main/java/com/kw/readwith/service/CharacterImageFanoutJobService.java b/src/main/java/com/kw/readwith/service/CharacterImageFanoutJobService.java index 6709727..989cfa8 100644 --- a/src/main/java/com/kw/readwith/service/CharacterImageFanoutJobService.java +++ b/src/main/java/com/kw/readwith/service/CharacterImageFanoutJobService.java @@ -165,6 +165,7 @@ public List findSubmittedJobIds() { EnumSet.of(ProcessingJobStatus.PROCESSING) ).stream() .filter(job -> job.getExternalJobId() != null && !job.getExternalJobId().isBlank()) + .filter(job -> !isApplyingResults(job.getCurrentStep())) .map(ProcessingJob::getId) .toList(); } @@ -244,6 +245,51 @@ public void refresh(Long jobId) { } } + public void retryResultApplication(Long jobId) { + ResultFileContext context = prepareResultApplicationRetry(jobId); + try { + batchClient.streamResults(context.outputFileId(), result -> processResultSafely(jobId, result)); + batchClient.streamResults(context.errorFileId(), result -> processResultSafely(jobId, result)); + finalizeResults(jobId, "completed"); + } catch (Exception e) { + completeFailure(jobId, "IMAGE_BATCH_RESULT_APPLY_FAILED", messageOf(e)); + } + } + + private ResultFileContext prepareResultApplicationRetry(Long jobId) { + return writableTransaction().execute(status -> { + ProcessingJob job = findImageJobForUpdate(jobId); + if (job.getStatus() != ProcessingJobStatus.FAILED) { + throw new GeneralException(ErrorStatus.IMAGE_FANOUT_RETRY_NOT_ALLOWED); + } + if (isBlank(job.getOutputFileId()) && isBlank(job.getErrorFileId())) { + throw new GeneralException(ErrorStatus.IMAGE_FANOUT_RESULT_NOT_AVAILABLE); + } + + List assets = assetRepository.findByProcessingJobOrderByIdAsc(job); + if (assets.isEmpty()) { + throw new GeneralException(ErrorStatus.IMAGE_FANOUT_RESULT_NOT_AVAILABLE); + } + for (CharacterImageAsset asset : assets) { + if (asset.getStatus() != CharacterImageAssetStatus.PUBLISHED) { + asset.retryResultApplication(); + characterRepository.updateImageGenerationStatus( + asset.getCharacter().getId(), + ImageGenerationStatus.GENERATING + ); + } + } + + job.markProcessing("reapplying_batch_results"); + writeLog(job, ProcessingJobLogLevel.INFO, "reapplying_batch_results", + "Reapplying stored OpenAI Batch image results without submitting a new Batch.", Map.of( + "outputFileId", Optional.ofNullable(job.getOutputFileId()).orElse(""), + "errorFileId", Optional.ofNullable(job.getErrorFileId()).orElse("") + )); + return new ResultFileContext(job.getOutputFileId(), job.getErrorFileId()); + }); + } + private SubmissionContext transitionToSubmitting(Long jobId) { return writableTransaction().execute(status -> { ProcessingJob job = findImageJobForUpdate(jobId); @@ -414,10 +460,10 @@ private void processResultSafely(Long jobId, OpenAiBatchImageResult result) { return; } String s3Url = characterImageService.uploadGeneratedImage( - target.character(), result.imageData(), characterImageService.buildPublishedS3KeyName( - target.character(), + target.bookId(), + target.characterId(), target.referenceVersion(), target.attemptNo() ) @@ -449,7 +495,8 @@ private TargetUploadContext loadTargetForUpload(Long jobId, Long assetId) { return null; } return new TargetUploadContext( - asset.getCharacter(), + asset.getBook().getId(), + asset.getCharacter().getId(), asset.getReferenceVersion(), asset.getAttemptNo() ); @@ -714,6 +761,15 @@ private String messageOf(Exception exception) { return message == null || message.isBlank() ? exception.getClass().getSimpleName() : message; } + private boolean isApplyingResults(String currentStep) { + return "applying_batch_results".equals(currentStep) + || "reapplying_batch_results".equals(currentStep); + } + + private boolean isBlank(String value) { + return value == null || value.isBlank(); + } + private void writeLog(ProcessingJob job, ProcessingJobLogLevel level, String step, @@ -764,6 +820,14 @@ private record SubmissionContext( private record PollContext(String externalJobId) { } - private record TargetUploadContext(Character character, int referenceVersion, int attemptNo) { + private record TargetUploadContext( + Long bookId, + Long characterId, + int referenceVersion, + int attemptNo + ) { + } + + private record ResultFileContext(String outputFileId, String errorFileId) { } } diff --git a/src/main/java/com/kw/readwith/service/CharacterImageService.java b/src/main/java/com/kw/readwith/service/CharacterImageService.java index ec431e0..1e85bd7 100644 --- a/src/main/java/com/kw/readwith/service/CharacterImageService.java +++ b/src/main/java/com/kw/readwith/service/CharacterImageService.java @@ -220,6 +220,10 @@ public GeneratedCharacterImage generateTextImage(Character character) throws IOE } public String uploadGeneratedImage(Character character, byte[] imageData, String s3KeyName) { + return uploadGeneratedImage(imageData, s3KeyName); + } + + public String uploadGeneratedImage(byte[] imageData, String s3KeyName) { s3Manager.uploadBytes(s3KeyName, imageData, "image/png"); return s3Manager.getObjectUrl(s3KeyName); } @@ -403,11 +407,23 @@ public String buildPublishedS3KeyName(Character character) { } public String buildPublishedS3KeyName(Character character, int referenceVersion, int attemptNo) { - return String.format("%s/%d/%d/reference-v%d/attempt-%d.png", - imageProperties.getS3Path(), + return buildPublishedS3KeyName( character.getBook().getId(), character.getId(), referenceVersion, + attemptNo + ); + } + + public String buildPublishedS3KeyName(Long bookId, + Long characterId, + int referenceVersion, + int attemptNo) { + return String.format("%s/%d/%d/reference-v%d/attempt-%d.png", + imageProperties.getS3Path(), + bookId, + characterId, + referenceVersion, attemptNo); } diff --git a/src/main/java/com/kw/readwith/web/controller/AdminImageGenerationController.java b/src/main/java/com/kw/readwith/web/controller/AdminImageGenerationController.java index 056ecb7..bb7c3fb 100644 --- a/src/main/java/com/kw/readwith/web/controller/AdminImageGenerationController.java +++ b/src/main/java/com/kw/readwith/web/controller/AdminImageGenerationController.java @@ -196,6 +196,38 @@ public ApiResponse> getFanoutJobLogs( return ApiResponse.onSuccess(fanoutJobService.getFanoutJobLogs(jobId)); } + @Operation( + summary = "실패한 fan-out job의 Batch 결과 게시 재시도", + description = """ + OpenAI Batch는 완료됐지만 S3 또는 DB 게시 단계에서 실패한 job의 기존 output/error 파일을 다시 적용합니다. + 새로운 OpenAI Batch나 이미지 생성 요청을 제출하지 않으므로 이미지 생성 비용이 다시 발생하지 않습니다. + status=FAILED이고 outputFileId 또는 errorFileId가 저장된 job만 재시도할 수 있습니다. + 응답의 status=READY여야 모든 대상 이미지 게시가 완료된 상태입니다. + """ + ) + @ApiResponses(value = { + @io.swagger.v3.oas.annotations.responses.ApiResponse( + responseCode = "200", + description = "기존 Batch 결과 게시 재시도 완료", + content = @Content(schema = @Schema(implementation = ProcessingJobResponseDTO.class)) + ), + @io.swagger.v3.oas.annotations.responses.ApiResponse( + responseCode = "400", + description = "ADMIN4030: 재시도할 수 없는 job 상태입니다. ADMIN4031: 저장된 Batch 결과 파일이 없습니다." + ), + @io.swagger.v3.oas.annotations.responses.ApiResponse( + responseCode = "404", + description = "ADMIN4029: 캐릭터 이미지 fan-out job을 찾을 수 없습니다." + ) + }) + @PostMapping("/fanout-jobs/{jobId}/retry-publish") + public ApiResponse retryFanoutResultPublish( + @Parameter(description = "결과 게시를 다시 시도할 실패한 fan-out job ID", required = true, example = "123") + @PathVariable Long jobId) { + fanoutJobService.retryResultApplication(jobId); + return ApiResponse.onSuccess(fanoutJobService.getFanoutJob(jobId)); + } + @Operation( summary = "개별 캐릭터 이미지 재생성", description = """ diff --git a/src/test/java/com/kw/readwith/service/CharacterImageFanoutJobServiceTest.java b/src/test/java/com/kw/readwith/service/CharacterImageFanoutJobServiceTest.java index 827f609..5c0cafb 100644 --- a/src/test/java/com/kw/readwith/service/CharacterImageFanoutJobServiceTest.java +++ b/src/test/java/com/kw/readwith/service/CharacterImageFanoutJobServiceTest.java @@ -48,6 +48,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; @ExtendWith(MockitoExtension.class) @DisplayName("CharacterImageFanoutJobService") @@ -252,10 +253,9 @@ void refresh_publishesCompletedBatchOutput() { )); return null; }).when(batchClient).streamResults(eq("file-output"), any()); - given(characterImageService.buildPublishedS3KeyName(targetCharacter, 1, 1)) + given(characterImageService.buildPublishedS3KeyName(20L, 101L, 1, 1)) .willReturn("character-images/20/101/reference-v1/attempt-1.png"); given(characterImageService.uploadGeneratedImage( - eq(targetCharacter), any(byte[].class), eq("character-images/20/101/reference-v1/attempt-1.png") )).willReturn("https://cdn.readwith.store/character-images/20/101/reference-v1/attempt-1.png"); @@ -278,6 +278,56 @@ void refresh_publishesCompletedBatchOutput() { ); } + @Test + @DisplayName("reapplies a stored Batch output without submitting or charging for another Batch") + void retryResultApplication_reusesStoredBatchOutput() { + ProcessingJob job = ProcessingJob.builder() + .id(630L) + .book(book) + .pipelineType(ProcessingPipelineType.IMAGE_GENERATION) + .runId("character-image-fanout-run-3") + .artifactPath("character-images/20/fanout-jobs/run-3") + .externalJobId("batch-630") + .inputFileId("file-input") + .outputFileId("file-output") + .status(ProcessingJobStatus.FAILED) + .currentStep("completed_with_failures") + .failureCode("IMAGE_BATCH_PARTIAL_FAILURE") + .build(); + CharacterImageAsset targetAsset = buildTargetAsset(730L, job); + targetAsset.fail("BATCH_IMAGE_PUBLISH_FAILED"); + given(transactionManager.getTransaction(any())).willAnswer(invocation -> new SimpleTransactionStatus()); + given(processingJobRepository.findByIdForUpdate(630L)).willReturn(Optional.of(job)); + given(assetRepository.findByProcessingJobOrderByIdAsc(job)).willReturn(List.of(targetAsset)); + given(assetRepository.findById(730L)).willReturn(Optional.of(targetAsset)); + doAnswer(invocation -> { + @SuppressWarnings("unchecked") + Consumer consumer = invocation.getArgument(1); + consumer.accept(new OpenAiImageBatchClient.OpenAiBatchImageResult( + "character-image-asset-730", 200, "req-730", new byte[]{4, 5, 6}, null, null + )); + return null; + }).when(batchClient).streamResults(eq("file-output"), any()); + given(characterImageService.buildPublishedS3KeyName(20L, 101L, 1, 1)) + .willReturn("character-images/20/101/reference-v1/attempt-1.png"); + given(characterImageService.uploadGeneratedImage( + any(byte[].class), + eq("character-images/20/101/reference-v1/attempt-1.png") + )).willReturn("https://cdn.readwith.store/character-images/20/101/reference-v1/attempt-1.png"); + + service.retryResultApplication(630L); + + assertThat(job.getStatus()).isEqualTo(ProcessingJobStatus.READY); + assertThat(targetAsset.getStatus()).isEqualTo(CharacterImageAssetStatus.PUBLISHED); + verify(batchClient, never()).submit(any(), any(), any(), any()); + verify(batchClient).streamResults(eq("file-output"), any()); + verify(characterRepository).updateProfileImageAndStatus( + 101L, + "https://cdn.readwith.store/character-images/20/101/reference-v1/attempt-1.png", + ImageGenerationStatus.COMPLETED + ); + } + private CharacterImageAsset buildTargetAsset(Long assetId, ProcessingJob job) { return CharacterImageAsset.builder() .id(assetId)