diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationAiClient.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationAiClient.java index 117d39f..4bca620 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationAiClient.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationAiClient.java @@ -45,6 +45,12 @@ class NlgEvaluationAiClient { 2. JobDri 서비스 정책상 missing keyword 제외 대상이 아니다. 3. answer에 동일하거나 의미상 충족되는 내용이 없다. 4. 현재 missingKeywords에 동일하거나 충분히 유사한 키워드가 없다. + - MISSED_MISSING_KEYWORD를 부여한다면 missedMissingKeywordEvaluations에도 같은 개수 이상의 근거를 작성한다. + - missedMissingKeywordEvaluations의 각 항목은 keyword, source, relatedRequirement, reason을 모두 포함한다. + - relatedRequirement는 mainTasks 또는 qualifications에 실제 존재하는 JD 원문 일부를 그대로 사용한다. + - source는 MAIN_TASK 또는 QUALIFICATION만 사용한다. + - "JD의 핵심 경험 요구사항", "채용 관련 경험", "업무 경험 부족", "핵심 경험"처럼 추상적인 표현을 keyword나 relatedRequirement로 사용하지 않는다. + - JD에 없는 요구사항을 생성하지 않는다. - missing keyword 평가 대상에서 제외한다: 자격증, 면허, 학력, 경력 연차, 나이, 법적/정형 보유 조건, 선택형 자격요건의 다른 선택지. - 제외 예시: 사회복지사, 청소년상담사, 직업상담사, 운전면허, 대졸 이상, 경력 3년 이상. - 제외 대상이 아닌 예시: Spring Boot 실무 경험, 포토샵 활용 능력, 엑셀 고급 활용, 더존 사용 능력, 4대보험 신고 경험. diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationBatchService.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationBatchService.java index 556ea2b..85af0d9 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationBatchService.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationBatchService.java @@ -6,6 +6,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.exc.InvalidFormatException; import com.jobdri.jobdri_api.domain.analysis.dto.llm.AnalysisLlmResponse; +import com.jobdri.jobdri_api.domain.analysis.dto.response.MissingKeywordSource; +import com.jobdri.jobdri_api.domain.analysis.service.sanitization.AnalysisSanitizationRules; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.core.NestedExceptionUtils; @@ -181,9 +183,17 @@ private NlgEvaluationResult validateAndBuildResult( List evaluations = validQuestionEvaluations(input.questionAnalyses(), response.questionAnalysisEvaluations()); + List validMissingKeywordMissEvaluations = + validMissingKeywordMissEvaluations(input, response.missedMissingKeywordEvaluations()); boolean missedValidatedMissingKeywords = input.validatedMissingKeywordCandidateCount() > 0 && input.actualMissingKeywordCount() == 0; - List errorCodes = mergeErrorCodes(response, evaluations, missedValidatedMissingKeywords); + List errorCodes = mergeErrorCodes( + input, + response, + evaluations, + missedValidatedMissingKeywords, + !validMissingKeywordMissEvaluations.isEmpty() + ); boolean hasFatalError = hasFatalError(errorCodes); Integer missingKeywordsCoverage = validCaseScore(response.missingKeywordsCoverage()); if (missedValidatedMissingKeywords) { @@ -245,6 +255,7 @@ private NlgEvaluationResponse normalizeQuestionAnalysisEvaluations( response.missingKeywordsPrecision(), response.missingKeywordsCoverage(), response.overallUsefulness(), + response.missedMissingKeywordEvaluations(), response.caseErrorCodes(), response.shortRationale() ); @@ -323,15 +334,28 @@ && validScore(evaluation.nonMeta()) } private List mergeErrorCodes( + NlgEvaluationAiClient.NlgJudgeInput input, NlgEvaluationResponse response, List evaluations, - boolean missedValidatedMissingKeywords + boolean missedValidatedMissingKeywords, + boolean hasValidMissingKeywordMissEvidence ) { boolean hasLowScore = hasLowCaseScore(response) || missedValidatedMissingKeywords; Set codes = new HashSet<>(sanitizeErrorCodes( response.caseErrorCodes(), hasLowScore )); + if (codes.remove(NlgEvaluationErrorCode.MISSED_MISSING_KEYWORD)) { + if (hasValidMissingKeywordMissEvidence || missedValidatedMissingKeywords) { + codes.add(NlgEvaluationErrorCode.MISSED_MISSING_KEYWORD); + } else { + log.warn( + "Removed NLG Judge missing keyword error without valid evidence. reason=judge_missing_keyword_without_valid_evidence, caseId={}, sourceResultFile={}", + input.caseId(), + input.sourceResultFile() + ); + } + } if (missedValidatedMissingKeywords) { codes.add(NlgEvaluationErrorCode.MISSED_MISSING_KEYWORD); codes.remove(NlgEvaluationErrorCode.NONE); @@ -358,6 +382,106 @@ private List mergeErrorCodes( .toList(); } + private List validMissingKeywordMissEvaluations( + NlgEvaluationAiClient.NlgJudgeInput input, + List evaluations + ) { + if (evaluations == null || evaluations.isEmpty()) { + return List.of(); + } + List actualMissingKeywords = + readActualMissingKeywords(input.missingKeywordsJson(), input.caseId()); + List valid = new ArrayList<>(); + for (NlgEvaluationResponse.MissingKeywordMissEvaluation evaluation : evaluations) { + Optional invalidReason = + missingKeywordMissInvalidReason(input, actualMissingKeywords, evaluation); + if (invalidReason.isPresent()) { + log.warn( + "Removed NLG Judge missing keyword evidence. caseId={}, keyword={}, relatedRequirement={}, reason={}, invalidReason={}", + input.caseId(), + evaluation == null ? null : evaluation.keyword(), + evaluation == null ? null : evaluation.relatedRequirement(), + evaluation == null ? null : evaluation.reason(), + invalidReason.get() + ); + continue; + } + valid.add(evaluation); + } + return valid; + } + + private Optional missingKeywordMissInvalidReason( + NlgEvaluationAiClient.NlgJudgeInput input, + List actualMissingKeywords, + NlgEvaluationResponse.MissingKeywordMissEvaluation evaluation + ) { + if (evaluation == null) { + return Optional.of(MissingKeywordMissInvalidReason.NULL_EVIDENCE); + } + if (!StringUtils.hasText(evaluation.keyword())) { + return Optional.of(MissingKeywordMissInvalidReason.BLANK_KEYWORD); + } + if (!StringUtils.hasText(evaluation.relatedRequirement())) { + return Optional.of(MissingKeywordMissInvalidReason.BLANK_RELATED_REQUIREMENT); + } + if (!StringUtils.hasText(evaluation.reason())) { + return Optional.of(MissingKeywordMissInvalidReason.BLANK_REASON); + } + Optional source = parseJudgeMissingKeywordSource(evaluation.source()); + if (source.isEmpty() || source.get() == MissingKeywordSource.PREFERENCE) { + return Optional.of(MissingKeywordMissInvalidReason.INVALID_SOURCE); + } + if (AnalysisSanitizationRules.isStructuredQualificationKeyword(evaluation.keyword()) + || AnalysisSanitizationRules.isStructuredQualificationKeyword(evaluation.relatedRequirement())) { + return Optional.of(MissingKeywordMissInvalidReason.STRUCTURED_QUALIFICATION); + } + String sourceText = source.get() == MissingKeywordSource.MAIN_TASK + ? input.mainTasks() + : input.qualifications(); + if (!containsNormalized(sourceText, evaluation.relatedRequirement())) { + return Optional.of(MissingKeywordMissInvalidReason.RELATED_REQUIREMENT_NOT_IN_JD); + } + if (!AnalysisSanitizationRules.isValidMissingKeyword( + evaluation.keyword(), + source.get(), + input.mainTasks(), + input.qualifications() + )) { + return Optional.of(MissingKeywordMissInvalidReason.INVALID_KEYWORD); + } + if (actualMissingKeywords.stream().anyMatch(keyword -> sameMissingKeyword(keyword, evaluation, source.get()))) { + return Optional.of(MissingKeywordMissInvalidReason.ALREADY_PRESENT_IN_FINAL_MISSING_KEYWORDS); + } + return Optional.empty(); + } + + private Optional parseJudgeMissingKeywordSource(String source) { + if (!StringUtils.hasText(source)) { + return Optional.empty(); + } + String normalized = source.trim(); + if ("MAIN_TASK".equalsIgnoreCase(normalized) || "MAIN_TASKS".equalsIgnoreCase(normalized)) { + return Optional.of(MissingKeywordSource.MAIN_TASK); + } + if ("QUALIFICATION".equalsIgnoreCase(normalized) || "QUALIFICATIONS".equalsIgnoreCase(normalized)) { + return Optional.of(MissingKeywordSource.QUALIFICATION); + } + return MissingKeywordSource.from(normalized); + } + + private boolean sameMissingKeyword( + AnalysisLlmResponse.MissingKeywordItem actual, + NlgEvaluationResponse.MissingKeywordMissEvaluation evaluation, + MissingKeywordSource source + ) { + return actual != null + && normalize(actual.keyword()).equals(normalize(evaluation.keyword())) + && parseJudgeMissingKeywordSource(actual.source()) + .map(actualSource -> actualSource == source) + .orElse(false); + } + private List sanitizeErrorCodes( List errorCodes, boolean hasLowScore @@ -477,6 +601,22 @@ private int readJsonArraySize(String json, String fieldName, String caseId) { } } + private List readActualMissingKeywords(String json, String caseId) { + if (!StringUtils.hasText(json)) { + return List.of(); + } + try { + List values = objectMapper.readValue( + json, + new TypeReference<>() { + } + ); + return values == null ? List.of() : values; + } catch (JsonProcessingException e) { + throw new IllegalArgumentException("aiMissingKeywordsJson must be a JSON array. caseId=" + caseId, e); + } + } + private int readMissingKeywordCandidateCount(String sanitizedCandidateResponseJson) { if (!StringUtils.hasText(sanitizedCandidateResponseJson)) { return 0; @@ -741,6 +881,16 @@ private Double parseDouble(String value) { } } + private boolean containsNormalized(String sourceText, String fragment) { + return StringUtils.hasText(sourceText) + && StringUtils.hasText(fragment) + && normalize(sourceText).contains(normalize(fragment)); + } + + private String normalize(String value) { + return AnalysisSanitizationRules.normalizeText(value); + } + private String truncateShortRationale(String rationale) { if (!StringUtils.hasText(rationale)) { return ""; @@ -957,6 +1107,18 @@ private enum HeaderLocation { MISSING } + private enum MissingKeywordMissInvalidReason { + NULL_EVIDENCE, + BLANK_KEYWORD, + BLANK_RELATED_REQUIREMENT, + BLANK_REASON, + INVALID_SOURCE, + STRUCTURED_QUALIFICATION, + RELATED_REQUIREMENT_NOT_IN_JD, + INVALID_KEYWORD, + ALREADY_PRESENT_IN_FINAL_MISSING_KEYWORDS + } + private record ResolvedHeader(HeaderLocation location, String headerName) { static ResolvedHeader input(String headerName) { return new ResolvedHeader(HeaderLocation.INPUT, headerName); diff --git a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationResponse.java b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationResponse.java index c402989..8d7cd57 100644 --- a/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationResponse.java +++ b/src/main/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationResponse.java @@ -11,9 +11,37 @@ public record NlgEvaluationResponse( Integer missingKeywordsPrecision, Integer missingKeywordsCoverage, Integer overallUsefulness, + List missedMissingKeywordEvaluations, List caseErrorCodes, String shortRationale ) { + public NlgEvaluationResponse( + String caseId, + List questionAnalysisEvaluations, + Integer noAnalysisAppropriateness, + Integer strengthsPrecision, + Integer strengthsCoverage, + Integer missingKeywordsPrecision, + Integer missingKeywordsCoverage, + Integer overallUsefulness, + List caseErrorCodes, + String shortRationale + ) { + this( + caseId, + questionAnalysisEvaluations, + noAnalysisAppropriateness, + strengthsPrecision, + strengthsCoverage, + missingKeywordsPrecision, + missingKeywordsCoverage, + overallUsefulness, + List.of(), + caseErrorCodes, + shortRationale + ); + } + public record QuestionAnalysisEvaluation( Integer analysisIndex, String sentence, @@ -30,4 +58,12 @@ public record QuestionAnalysisEvaluation( List errorCodes ) { } + + public record MissingKeywordMissEvaluation( + String keyword, + String source, + String relatedRequirement, + String reason + ) { + } } diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationAiClientTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationAiClientTest.java index 7b79fdc..304b71c 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationAiClientTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationAiClientTest.java @@ -163,6 +163,9 @@ void buildPromptAlignsMissedMissingKeywordWithServicePolicy() { .contains("[MISSED_MISSING_KEYWORD 판정 기준]") .contains("필요한 비정형 업무·역량 누락 키워드가 존재하면 actual=[]라도") .contains("정형 자격요건은 이후 [MISSED_MISSING_KEYWORD 판정 기준]의 제외 정책이 우선") + .contains("missedMissingKeywordEvaluations에도 같은 개수 이상의 근거를 작성한다") + .contains("relatedRequirement는 mainTasks 또는 qualifications에 실제 존재하는 JD 원문 일부를 그대로 사용한다") + .contains("\"JD의 핵심 경험 요구사항\", \"채용 관련 경험\", \"업무 경험 부족\", \"핵심 경험\"") .contains("실제 JD 원문의 mainTasks 또는 qualifications에 요구사항이 존재한다") .contains("JobDri 서비스 정책상 missing keyword 제외 대상이 아니다") .contains("answer에 동일하거나 의미상 충족되는 내용이 없다") diff --git a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationBatchServiceTest.java b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationBatchServiceTest.java index 1a1762b..dbcbe46 100644 --- a/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationBatchServiceTest.java +++ b/src/test/java/com/jobdri/jobdri_api/domain/analysis/evaluation/NlgEvaluationBatchServiceTest.java @@ -375,6 +375,12 @@ void writesCoverageScores() throws Exception { 5, 2, 3, + List.of(new NlgEvaluationResponse.MissingKeywordMissEvaluation( + "장애 대응 경험", + "QUALIFICATION", + "장애 대응 경험", + "답변에서 해당 경험을 확인할 수 없습니다." + )), List.of(NlgEvaluationErrorCode.MISSED_STRENGTH, NlgEvaluationErrorCode.MISSED_MISSING_KEYWORD), "강점과 누락 키워드 coverage가 낮습니다." ), @@ -396,6 +402,116 @@ void writesCoverageScores() throws Exception { .contains("MISSED_MISSING_KEYWORD"); } + @Test + @DisplayName("MISSED_MISSING_KEYWORD는 JD requirement 근거가 유효하면 유지한다") + void keepsMissedMissingKeywordWhenEvidenceIsGroundedInJd() throws Exception { + Map row = runJudgeWithMissedMissingKeywordEvidence( + new NlgEvaluationResponse.MissingKeywordMissEvaluation( + "4대보험 신고", + "MAIN_TASK", + "4대보험 신고 및 지원금 신청", + "답변에서 4대보험 신고 경험을 확인할 수 없습니다." + ), + "4대보험 신고 및 지원금 신청", + "엑셀 고급 활용", + "[]" + ); + + assertThat(row.get("errorCodes")).contains("MISSED_MISSING_KEYWORD"); + } + + @Test + @DisplayName("MISSED_MISSING_KEYWORD는 Spring Boot 실무 경험처럼 비정형 역량 근거가 유효하면 유지한다") + void keepsTechnicalMissingKeywordEvidence() throws Exception { + Map row = runJudgeWithMissedMissingKeywordEvidence( + new NlgEvaluationResponse.MissingKeywordMissEvaluation( + "Spring Boot 실무 경험", + "QUALIFICATION", + "Spring Boot 실무 경험", + "답변에서 Spring Boot 사용 경험을 확인할 수 없습니다." + ), + "REST API 개발", + "Spring Boot 실무 경험", + "[]" + ); + + assertThat(row.get("errorCodes")).contains("MISSED_MISSING_KEYWORD"); + } + + @Test + @DisplayName("MISSED_MISSING_KEYWORD는 JD에 없는 추상 relatedRequirement면 제거한다") + void removesMissedMissingKeywordWhenRelatedRequirementIsNotInJd() throws Exception { + for (String relatedRequirement : List.of("채용 관련 경험", "핵심 경험")) { + Map row = runJudgeWithMissedMissingKeywordEvidence( + new NlgEvaluationResponse.MissingKeywordMissEvaluation( + relatedRequirement, + "MAIN_TASK", + relatedRequirement, + "답변에서 해당 경험을 확인할 수 없습니다." + ), + "엑셀 고급 활용, 4대보험 신고, 더존 사용", + "인사 회계 경험", + "[]" + ); + + assertThat(row.get("errorCodes")).doesNotContain("MISSED_MISSING_KEYWORD"); + } + } + + @Test + @DisplayName("MISSED_MISSING_KEYWORD는 정형 자격요건 근거면 제거한다") + void removesStructuredQualificationMissedMissingKeywordEvidence() throws Exception { + for (String keyword : List.of("사회복지사", "청소년상담사", "운전면허", "대졸", "경력 3년")) { + Map row = runJudgeWithMissedMissingKeywordEvidence( + new NlgEvaluationResponse.MissingKeywordMissEvaluation( + keyword, + "QUALIFICATION", + keyword, + "답변에서 해당 조건을 확인할 수 없습니다." + ), + "상담 지원", + keyword, + "[]" + ); + + assertThat(row.get("errorCodes")).doesNotContain("MISSED_MISSING_KEYWORD"); + } + } + + @Test + @DisplayName("MISSED_MISSING_KEYWORD는 이미 final missingKeywords에 있으면 제거한다") + void removesMissedMissingKeywordAlreadyCoveredByFinalMissingKeywords() throws Exception { + Map row = runJudgeWithMissedMissingKeywordEvidence( + new NlgEvaluationResponse.MissingKeywordMissEvaluation( + "4대보험 신고", + "MAIN_TASK", + "4대보험 신고 및 지원금 신청", + "답변에서 4대보험 신고 경험을 확인할 수 없습니다." + ), + "4대보험 신고 및 지원금 신청", + "엑셀 고급 활용", + objectMapper.writeValueAsString(List.of(new AnalysisLlmResponse.MissingKeywordItem( + "4대보험 신고", + "mainTask" + ))) + ); + + assertThat(row.get("errorCodes")).doesNotContain("MISSED_MISSING_KEYWORD"); + } + + @Test + @DisplayName("MISSED_MISSING_KEYWORD는 evidence 없이 errorCode만 있으면 제거한다") + void removesMissedMissingKeywordWithoutEvidence() throws Exception { + Map row = runJudgeWithMissedMissingKeywordEvidence( + null, + "4대보험 신고 및 지원금 신청", + "엑셀 고급 활용", + "[]" + ); + + assertThat(row.get("errorCodes")).doesNotContain("MISSED_MISSING_KEYWORD"); + } + @Test @DisplayName("검증된 missing keyword 후보가 있는데 actual이 빈 배열이면 coverage와 errorCode를 서버에서 보정한다") void correctsEmptyActualMissingKeywordsWhenValidatedCandidatesExist() throws Exception { @@ -882,6 +998,80 @@ private Path writeJudgeInputWithMissingKeywordState( return input; } + private Map runJudgeWithMissedMissingKeywordEvidence( + NlgEvaluationResponse.MissingKeywordMissEvaluation evidence, + String mainTasks, + String qualifications, + String missingKeywordsJson + ) throws Exception { + NlgEvaluationAiClient aiClient = mock(NlgEvaluationAiClient.class); + when(aiClient.evaluate(any())).thenReturn(new NlgEvaluationAiClient.JudgeCallResult( + new NlgEvaluationResponse( + "EV-20", + List.of(), + 2, + 5, + 5, + 2, + 2, + 3, + evidence == null ? List.of() : List.of(evidence), + List.of(NlgEvaluationErrorCode.MISSED_MISSING_KEYWORD), + "누락 키워드 근거를 평가했습니다." + ), + 100L, + null, + null + )); + Path input = writeJudgeInputWithContext("EV-20", mainTasks, qualifications, missingKeywordsJson, "[]"); + Path output = tempDir.resolve("judge_missed_keyword_" + System.nanoTime() + ".csv"); + + new NlgEvaluationBatchService(aiClient, objectMapper).run(input, output); + + return EvaluationCsvSupport.read(output).getFirst(); + } + + private Path writeJudgeInputWithContext( + String caseId, + String mainTasks, + String qualifications, + String missingKeywordsJson, + String analysesJson + ) throws Exception { + Path input = tempDir.resolve(caseId + "_context_" + System.nanoTime() + ".csv"); + Files.writeString( + input, + String.join(",", List.of( + "caseId", + "mainTasks", + "qualifications", + "preferences", + "question", + "answer", + "aiQuestionAnalysesJson", + "aiMissingKeywordsJson", + "rawLlmResponseJson", + "rawCandidateResponseJson", + "sanitizedCandidateResponseJson", + "candidateReviewResponseJson" + )) + "\n" + + csv(caseId) + "," + + csv(mainTasks) + "," + + csv(qualifications) + "," + + csv("") + "," + + csv("지원 동기") + "," + + csv("답변") + "," + + csv(analysesJson) + "," + + csv(missingKeywordsJson) + "," + + csv(rawLlmResponseJson()) + "," + + csv("") + "," + + csv("{\"missingKeywordCandidates\":[]}") + "," + + csv("") + "\n", + StandardCharsets.UTF_8 + ); + return input; + } + private void stubJudge(NlgEvaluationAiClient aiClient, String caseId) { when(aiClient.evaluate(any())).thenReturn(new NlgEvaluationAiClient.JudgeCallResult( responseWithoutQuestionEvaluations(caseId),