Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,8 @@ public enum ErrorStatus implements BaseErrorCode {
BOOK_DELETE_JOB_IN_PROGRESS(HttpStatus.CONFLICT, "ADMIN4027", "진행 중인 처리 job이 있어 책을 삭제할 수 없습니다."),
IMAGE_FANOUT_JOB_ACTIVE(HttpStatus.CONFLICT, "ADMIN4028", "해당 책의 캐릭터 이미지 fan-out job이 이미 진행 중입니다."),
IMAGE_FANOUT_JOB_NOT_FOUND(HttpStatus.NOT_FOUND, "ADMIN4029", "캐릭터 이미지 fan-out job을 찾을 수 없습니다."),
IMAGE_FANOUT_RETRY_NOT_ALLOWED(HttpStatus.BAD_REQUEST, "ADMIN4030", "결과 게시를 재시도할 수 없는 캐릭터 이미지 fan-out job 상태입니다."),
IMAGE_FANOUT_RESULT_NOT_AVAILABLE(HttpStatus.BAD_REQUEST, "ADMIN4031", "재적용할 OpenAI Batch 결과 파일이 없습니다."),
IMAGE_GENERATION_FAILED(HttpStatus.INTERNAL_SERVER_ERROR, "ADMIN5020", "캐릭터 이미지 생성에 실패했습니다."),

// Bookmark
Expand Down
5 changes: 5 additions & 0 deletions src/main/java/com/kw/readwith/domain/CharacterImageAsset.java
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,11 @@ public void fail(String failureCode) {
this.status = CharacterImageAssetStatus.FAILED;
}

public void retryResultApplication() {
this.failureCode = null;
this.status = CharacterImageAssetStatus.GENERATING;
}

public void markStaleReference() {
this.status = CharacterImageAssetStatus.STALE_REFERENCE;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ public List<Long> findSubmittedJobIds() {
EnumSet.of(ProcessingJobStatus.PROCESSING)
).stream()
.filter(job -> job.getExternalJobId() != null && !job.getExternalJobId().isBlank())
.filter(job -> !isApplyingResults(job.getCurrentStep()))
.map(ProcessingJob::getId)
.toList();
}
Expand Down Expand Up @@ -244,6 +245,51 @@ public void refresh(Long jobId) {
}
}

public void retryResultApplication(Long jobId) {
ResultFileContext context = prepareResultApplicationRetry(jobId);
try {
batchClient.streamResults(context.outputFileId(), result -> processResultSafely(jobId, result));
batchClient.streamResults(context.errorFileId(), result -> processResultSafely(jobId, result));
finalizeResults(jobId, "completed");
} catch (Exception e) {
completeFailure(jobId, "IMAGE_BATCH_RESULT_APPLY_FAILED", messageOf(e));
}
}

private ResultFileContext prepareResultApplicationRetry(Long jobId) {
return writableTransaction().execute(status -> {
ProcessingJob job = findImageJobForUpdate(jobId);
if (job.getStatus() != ProcessingJobStatus.FAILED) {
throw new GeneralException(ErrorStatus.IMAGE_FANOUT_RETRY_NOT_ALLOWED);
}
if (isBlank(job.getOutputFileId()) && isBlank(job.getErrorFileId())) {
throw new GeneralException(ErrorStatus.IMAGE_FANOUT_RESULT_NOT_AVAILABLE);
}

List<CharacterImageAsset> assets = assetRepository.findByProcessingJobOrderByIdAsc(job);
if (assets.isEmpty()) {
throw new GeneralException(ErrorStatus.IMAGE_FANOUT_RESULT_NOT_AVAILABLE);
}
for (CharacterImageAsset asset : assets) {
if (asset.getStatus() != CharacterImageAssetStatus.PUBLISHED) {
asset.retryResultApplication();
characterRepository.updateImageGenerationStatus(
asset.getCharacter().getId(),
ImageGenerationStatus.GENERATING
);
}
}

job.markProcessing("reapplying_batch_results");
writeLog(job, ProcessingJobLogLevel.INFO, "reapplying_batch_results",
"Reapplying stored OpenAI Batch image results without submitting a new Batch.", Map.of(
"outputFileId", Optional.ofNullable(job.getOutputFileId()).orElse(""),
"errorFileId", Optional.ofNullable(job.getErrorFileId()).orElse("")
));
return new ResultFileContext(job.getOutputFileId(), job.getErrorFileId());
});
}

private SubmissionContext transitionToSubmitting(Long jobId) {
return writableTransaction().execute(status -> {
ProcessingJob job = findImageJobForUpdate(jobId);
Expand Down Expand Up @@ -414,10 +460,10 @@ private void processResultSafely(Long jobId, OpenAiBatchImageResult result) {
return;
}
String s3Url = characterImageService.uploadGeneratedImage(
target.character(),
result.imageData(),
characterImageService.buildPublishedS3KeyName(
target.character(),
target.bookId(),
target.characterId(),
target.referenceVersion(),
target.attemptNo()
)
Expand Down Expand Up @@ -449,7 +495,8 @@ private TargetUploadContext loadTargetForUpload(Long jobId, Long assetId) {
return null;
}
return new TargetUploadContext(
asset.getCharacter(),
asset.getBook().getId(),
asset.getCharacter().getId(),
asset.getReferenceVersion(),
asset.getAttemptNo()
);
Expand Down Expand Up @@ -714,6 +761,15 @@ private String messageOf(Exception exception) {
return message == null || message.isBlank() ? exception.getClass().getSimpleName() : message;
}

private boolean isApplyingResults(String currentStep) {
return "applying_batch_results".equals(currentStep)
|| "reapplying_batch_results".equals(currentStep);
}

private boolean isBlank(String value) {
return value == null || value.isBlank();
}

private void writeLog(ProcessingJob job,
ProcessingJobLogLevel level,
String step,
Expand Down Expand Up @@ -764,6 +820,14 @@ private record SubmissionContext(
private record PollContext(String externalJobId) {
}

private record TargetUploadContext(Character character, int referenceVersion, int attemptNo) {
private record TargetUploadContext(
Long bookId,
Long characterId,
int referenceVersion,
int attemptNo
) {
}

private record ResultFileContext(String outputFileId, String errorFileId) {
}
}
20 changes: 18 additions & 2 deletions src/main/java/com/kw/readwith/service/CharacterImageService.java
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,10 @@ public GeneratedCharacterImage generateTextImage(Character character) throws IOE
}

public String uploadGeneratedImage(Character character, byte[] imageData, String s3KeyName) {
return uploadGeneratedImage(imageData, s3KeyName);
}

public String uploadGeneratedImage(byte[] imageData, String s3KeyName) {
s3Manager.uploadBytes(s3KeyName, imageData, "image/png");
return s3Manager.getObjectUrl(s3KeyName);
}
Expand Down Expand Up @@ -403,11 +407,23 @@ public String buildPublishedS3KeyName(Character character) {
}

public String buildPublishedS3KeyName(Character character, int referenceVersion, int attemptNo) {
return String.format("%s/%d/%d/reference-v%d/attempt-%d.png",
imageProperties.getS3Path(),
return buildPublishedS3KeyName(
character.getBook().getId(),
character.getId(),
referenceVersion,
attemptNo
);
}

public String buildPublishedS3KeyName(Long bookId,
Long characterId,
int referenceVersion,
int attemptNo) {
return String.format("%s/%d/%d/reference-v%d/attempt-%d.png",
imageProperties.getS3Path(),
bookId,
characterId,
referenceVersion,
attemptNo);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,38 @@ public ApiResponse<List<ProcessingJobLogResponseDTO>> getFanoutJobLogs(
return ApiResponse.onSuccess(fanoutJobService.getFanoutJobLogs(jobId));
}

@Operation(
summary = "실패한 fan-out job의 Batch 결과 게시 재시도",
description = """
OpenAI Batch는 완료됐지만 S3 또는 DB 게시 단계에서 실패한 job의 기존 output/error 파일을 다시 적용합니다.
새로운 OpenAI Batch나 이미지 생성 요청을 제출하지 않으므로 이미지 생성 비용이 다시 발생하지 않습니다.
status=FAILED이고 outputFileId 또는 errorFileId가 저장된 job만 재시도할 수 있습니다.
응답의 status=READY여야 모든 대상 이미지 게시가 완료된 상태입니다.
"""
)
@ApiResponses(value = {
@io.swagger.v3.oas.annotations.responses.ApiResponse(
responseCode = "200",
description = "기존 Batch 결과 게시 재시도 완료",
content = @Content(schema = @Schema(implementation = ProcessingJobResponseDTO.class))
),
@io.swagger.v3.oas.annotations.responses.ApiResponse(
responseCode = "400",
description = "ADMIN4030: 재시도할 수 없는 job 상태입니다. ADMIN4031: 저장된 Batch 결과 파일이 없습니다."
),
@io.swagger.v3.oas.annotations.responses.ApiResponse(
responseCode = "404",
description = "ADMIN4029: 캐릭터 이미지 fan-out job을 찾을 수 없습니다."
)
})
@PostMapping("/fanout-jobs/{jobId}/retry-publish")
public ApiResponse<ProcessingJobResponseDTO> retryFanoutResultPublish(
@Parameter(description = "결과 게시를 다시 시도할 실패한 fan-out job ID", required = true, example = "123")
@PathVariable Long jobId) {
fanoutJobService.retryResultApplication(jobId);
return ApiResponse.onSuccess(fanoutJobService.getFanoutJob(jobId));
}

@Operation(
summary = "개별 캐릭터 이미지 재생성",
description = """
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.never;

@ExtendWith(MockitoExtension.class)
@DisplayName("CharacterImageFanoutJobService")
Expand Down Expand Up @@ -252,10 +253,9 @@ void refresh_publishesCompletedBatchOutput() {
));
return null;
}).when(batchClient).streamResults(eq("file-output"), any());
given(characterImageService.buildPublishedS3KeyName(targetCharacter, 1, 1))
given(characterImageService.buildPublishedS3KeyName(20L, 101L, 1, 1))
.willReturn("character-images/20/101/reference-v1/attempt-1.png");
given(characterImageService.uploadGeneratedImage(
eq(targetCharacter),
any(byte[].class),
eq("character-images/20/101/reference-v1/attempt-1.png")
)).willReturn("https://cdn.readwith.store/character-images/20/101/reference-v1/attempt-1.png");
Expand All @@ -278,6 +278,56 @@ void refresh_publishesCompletedBatchOutput() {
);
}

@Test
@DisplayName("reapplies a stored Batch output without submitting or charging for another Batch")
void retryResultApplication_reusesStoredBatchOutput() {
ProcessingJob job = ProcessingJob.builder()
.id(630L)
.book(book)
.pipelineType(ProcessingPipelineType.IMAGE_GENERATION)
.runId("character-image-fanout-run-3")
.artifactPath("character-images/20/fanout-jobs/run-3")
.externalJobId("batch-630")
.inputFileId("file-input")
.outputFileId("file-output")
.status(ProcessingJobStatus.FAILED)
.currentStep("completed_with_failures")
.failureCode("IMAGE_BATCH_PARTIAL_FAILURE")
.build();
CharacterImageAsset targetAsset = buildTargetAsset(730L, job);
targetAsset.fail("BATCH_IMAGE_PUBLISH_FAILED");
given(transactionManager.getTransaction(any())).willAnswer(invocation -> new SimpleTransactionStatus());
given(processingJobRepository.findByIdForUpdate(630L)).willReturn(Optional.of(job));
given(assetRepository.findByProcessingJobOrderByIdAsc(job)).willReturn(List.of(targetAsset));
given(assetRepository.findById(730L)).willReturn(Optional.of(targetAsset));
doAnswer(invocation -> {
@SuppressWarnings("unchecked")
Consumer<OpenAiImageBatchClient.OpenAiBatchImageResult> consumer = invocation.getArgument(1);
consumer.accept(new OpenAiImageBatchClient.OpenAiBatchImageResult(
"character-image-asset-730", 200, "req-730", new byte[]{4, 5, 6}, null, null
));
return null;
}).when(batchClient).streamResults(eq("file-output"), any());
given(characterImageService.buildPublishedS3KeyName(20L, 101L, 1, 1))
.willReturn("character-images/20/101/reference-v1/attempt-1.png");
given(characterImageService.uploadGeneratedImage(
any(byte[].class),
eq("character-images/20/101/reference-v1/attempt-1.png")
)).willReturn("https://cdn.readwith.store/character-images/20/101/reference-v1/attempt-1.png");

service.retryResultApplication(630L);

assertThat(job.getStatus()).isEqualTo(ProcessingJobStatus.READY);
assertThat(targetAsset.getStatus()).isEqualTo(CharacterImageAssetStatus.PUBLISHED);
verify(batchClient, never()).submit(any(), any(), any(), any());
verify(batchClient).streamResults(eq("file-output"), any());
verify(characterRepository).updateProfileImageAndStatus(
101L,
"https://cdn.readwith.store/character-images/20/101/reference-v1/attempt-1.png",
ImageGenerationStatus.COMPLETED
);
}

private CharacterImageAsset buildTargetAsset(Long assetId, ProcessingJob job) {
return CharacterImageAsset.builder()
.id(assetId)
Expand Down
Loading