diff --git a/tools/dictionary-parser/ParseWikidata b/tools/dictionary-parser/ParseWikidata index f19c1c44a..6dfb9e833 100755 --- a/tools/dictionary-parser/ParseWikidata +++ b/tools/dictionary-parser/ParseWikidata @@ -23,7 +23,7 @@ if [[ $# -eq 2 && $1 == "--all" ]]; then if [[ "$LAST_LINE" =~ $COMMAND_OPTS_PREFIX ]]; then OPTIONS=`echo $LAST_LINE | sed "s/$COMMAND_OPTS_PREFIX//"` echo Generating $LANGUAGE from $LEXEME_FILE - $RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst $LEXEME_FILE + $RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst --wikidata $LEXEME_FILE else echo skipping $LANGUAGE fi diff --git a/tools/dictionary-parser/README.md b/tools/dictionary-parser/README.md index 736bda902..7e76f58f9 100644 --- a/tools/dictionary-parser/README.md +++ b/tools/dictionary-parser/README.md @@ -20,5 +20,29 @@ These tools generate files that describes the grammatical properties of words fr ./ParseWikidata \ --inflections ../../resources/org/unicode/inflection/dictionary/inflectional_XX.xml \ --dictionary ../../resources/org/unicode/inflection/dictionary/dictionary_XX.lst \ - + --wikidata ``` + +## Usage for OASIS DMLex and Multi-Source Inputs + +`ParseWikidata` also supports [OASIS DMLex v1.0](https://docs.oasis-open.org/lexidma/dmlex/v1.0/dmlex-v1.0.html) JSON files via `--dmlex`, either standalone (bypassing Wikidata completely) or combined with `--wikidata`: + +- **Standalone DMLex (single or multiple files)**: +``` + ./ParseWikidata --language \ + --inflections inflectional_.xml \ + --dictionary dictionary_.lst \ + --dmlex [ ...] +``` +- **Multi-Source (Wikidata + DMLex overlay)**: +``` + ./ParseWikidata \ + --inflections inflectional_.xml \ + --dictionary dictionary_.lst \ + --wikidata \ + --dmlex [ ...] +``` +- **Conflict Resolution (`--merge-conflicts`)**: + - By default (`--merge-conflicts=false`), files are processed in CLI order and a conflicting surface form in a later source file overrides all previous grammemes and inflection patterns for that surface form. + - Pass `--merge-conflicts=true` (or `--merge-conflicts`) to merge grammemes and inflection patterns across files, placing the later source's conflicting pattern first in priority. + diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java new file mode 100644 index 000000000..0f89a2ae4 --- /dev/null +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java @@ -0,0 +1,49 @@ +/* + * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved. + */ +package org.unicode.wikidata; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import java.util.ArrayList; +import java.util.List; + +/** + * A lexical entry in OASIS DMLex v1.0 JSON format. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DMLexEntry { + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Lexicon { + public String langCode; + public List entries; + } + + public String id; + public String headword; + public String langCode; + + @JsonAlias({"pos"}) + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public List partOfSpeech; + + public List labels = new ArrayList<>(); + + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public List pronunciations; + + @JsonAlias({"forms"}) + public List inflectedForms; + + @JsonSetter("labels") + @JsonAlias({"grammaticalFeatures", "features", "tag", "tags"}) + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public void addLabels(List incoming) { + if (incoming != null) { + this.labels.addAll(incoming); + } + } +} diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java new file mode 100644 index 000000000..f7bd2affb --- /dev/null +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java @@ -0,0 +1,37 @@ +/* + * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved. + */ +package org.unicode.wikidata; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import java.util.ArrayList; +import java.util.List; + +/** + * An inflected form of a lexical entry in OASIS DMLex v1.0 JSON format. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DMLexInflectedForm { + public String id; + + @JsonAlias({"text", "inflection"}) + public String value; + + public List labels = new ArrayList<>(); + + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public List pronunciations; + + @JsonSetter("labels") + @JsonAlias({"grammaticalFeatures", "features", "tag", "tags"}) + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public void addLabels(List incoming) { + if (incoming != null) { + this.labels.addAll(incoming); + } + } +} diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java new file mode 100644 index 000000000..bf8342c0d --- /dev/null +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java @@ -0,0 +1,67 @@ +/* + * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved. + */ +package org.unicode.wikidata; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.exc.MismatchedInputException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Deserializes OASIS DMLex string or tag/label/pronunciation lists, accepting a single string, + * an array of strings, or an object/array of objects containing {@code "transcription"}, + * {@code "tag"}, {@code "value"}, {@code "label"}, or {@code "text"}. + */ +public class DMLexStringListDeserializer extends JsonDeserializer> { + private static final String[] OBJECT_VALUE_KEYS = { + "transcription", "tag", "value", "label", "text" + }; + + @Override + public List deserialize(JsonParser jsonParser, DeserializationContext ctxt) throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + List result = new ArrayList<>(); + extractValues(node, result, jsonParser); + return result; + } + + private static void extractValues(JsonNode node, List result, JsonParser jsonParser) throws IOException { + if (node == null || node.isNull()) { + return; + } + if (node.isTextual()) { + String text = node.textValue(); + if (!text.isBlank()) { + result.add(text); + } + } else if (node.isArray()) { + for (JsonNode elem : node) { + extractValues(elem, result, jsonParser); + } + } else if (node.isObject()) { + JsonNode transcriptions = node.get("transcriptions"); + if (transcriptions != null) { + extractValues(transcriptions, result, jsonParser); + } else { + for (String key : OBJECT_VALUE_KEYS) { + JsonNode field = node.get(key); + if (field != null && field.isTextual() && !field.textValue().isBlank()) { + result.add(field.textValue()); + break; + } + } + } + } else { + throw MismatchedInputException.from( + jsonParser, + List.class, + "Expected string, object, or array for DMLex list field, got " + node.getNodeType()); + } + } +} diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java index 20144fa4b..8f5f583fb 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java @@ -4,12 +4,14 @@ */ package org.unicode.wikidata; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; import javax.annotation.Nonnull; /** @@ -25,20 +27,27 @@ public class DictionaryEntry implements Comparable { private final TreeSet> grammemes; private final List inflectionalPatterns; private final List rareInflectionalPatterns = new ArrayList<>(); + private int sourceIndex; + private Map patternSourceIndices = null; public static final Comparator INFLECTION_PATTERN_COMPARATOR = Comparator .comparing(InflectionPattern::getCount) .reversed(); public DictionaryEntry(String phrase, boolean isLemmaRare, TreeSet> grammemes, InflectionPattern inflectionalPattern) { - this(phrase, phrase, isLemmaRare, grammemes, inflectionalPattern); + this(phrase, phrase, isLemmaRare, grammemes, inflectionalPattern, 0); } public DictionaryEntry(String phrase, String lemmas, boolean isLemmaRare, TreeSet> grammemes, InflectionPattern inflectionalPattern) { + this(phrase, lemmas, isLemmaRare, grammemes, inflectionalPattern, 0); + } + + public DictionaryEntry(String phrase, String lemmas, boolean isLemmaRare, TreeSet> grammemes, InflectionPattern inflectionalPattern, int sourceIndex) { this.phrase = phrase; this.lemmas = new TreeSet<>(Collections.singleton(lemmas)); this.grammemes = new TreeSet<>(grammemes.comparator()); this.grammemes.addAll(grammemes); + this.sourceIndex = sourceIndex; if (inflectionalPattern != null) { this.inflectionalPatterns = new ArrayList<>(2); this.inflectionalPatterns.add(inflectionalPattern); @@ -51,7 +60,53 @@ public DictionaryEntry(String phrase, String lemmas, boolean isLemmaRare, TreeSe } } + private int getPatternSourceIndex(InflectionPattern pattern) { + if (patternSourceIndices != null) { + return patternSourceIndices.getOrDefault(pattern, sourceIndex); + } + return sourceIndex; + } + public void merge(DictionaryEntry entry) { + merge(entry, false); + } + + public void merge(DictionaryEntry entry, boolean mergeConflicts) { + if (!mergeConflicts) { + if (entry.sourceIndex > this.sourceIndex) { + this.sourceIndex = entry.sourceIndex; + this.grammemes.clear(); + this.grammemes.addAll(entry.grammemes); + this.inflectionalPatterns.clear(); + this.inflectionalPatterns.addAll(entry.inflectionalPatterns); + this.rareInflectionalPatterns.clear(); + this.rareInflectionalPatterns.addAll(entry.rareInflectionalPatterns); + this.lemmas.clear(); + this.lemmas.addAll(entry.lemmas); + this.patternSourceIndices = null; + return; + } + if (entry.sourceIndex < this.sourceIndex) { + return; + } + } else if (entry.sourceIndex != this.sourceIndex || this.patternSourceIndices != null || entry.patternSourceIndices != null) { + if (this.patternSourceIndices == null) { + this.patternSourceIndices = new IdentityHashMap<>(); + for (InflectionPattern pattern : this.inflectionalPatterns) { + this.patternSourceIndices.put(pattern, this.sourceIndex); + } + } + for (InflectionPattern pattern : entry.inflectionalPatterns) { + int existingSource = this.patternSourceIndices.getOrDefault(pattern, this.sourceIndex); + int incomingSource = entry.getPatternSourceIndex(pattern); + if (incomingSource > existingSource && !entry.rareInflectionalPatterns.contains(pattern)) { + this.rareInflectionalPatterns.remove(pattern); + } + this.patternSourceIndices.put(pattern, Math.max(existingSource, incomingSource)); + } + this.sourceIndex = Math.max(this.sourceIndex, entry.sourceIndex); + } + this.grammemes.addAll(entry.grammemes); for (InflectionPattern inflectionalPattern : entry.inflectionalPatterns) { if (!this.inflectionalPatterns.contains(inflectionalPattern)) { @@ -70,6 +125,10 @@ public Set> getGrammemes() { return grammemes; } + List getInflectionalPatterns() { + return inflectionalPatterns; + } + @Override public String toString() { return this.toString(true); @@ -82,10 +141,15 @@ public String toString(boolean isInflectional) { sb.append(' ').append(enumVal); } if (isInflectional) { + Comparator patternComparator = (patternSourceIndices == null) + ? INFLECTION_PATTERN_COMPARATOR + : Comparator.comparing( + p -> patternSourceIndices.getOrDefault(p, sourceIndex), Comparator.reverseOrder()) + .thenComparing(INFLECTION_PATTERN_COMPARATOR); if (inflectionalPatterns.size() > 1) { - inflectionalPatterns.sort(INFLECTION_PATTERN_COMPARATOR); + inflectionalPatterns.sort(patternComparator); if (rareInflectionalPatterns.size() > 1) { - rareInflectionalPatterns.sort(INFLECTION_PATTERN_COMPARATOR); + rareInflectionalPatterns.sort(patternComparator); } } for (InflectionPattern inflectionPattern : inflectionalPatterns) { diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java index 3078fe619..be3304cc3 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java @@ -13,9 +13,11 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.IdentityHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.regex.Pattern; @@ -48,17 +50,31 @@ private void sortInflectionPatterns(ArrayList inflectionPatte } public void addDictionaryEntry(DictionaryEntry dictionaryEntry) { + addDictionaryEntry(dictionaryEntry, false); + } + + public void addDictionaryEntry(DictionaryEntry dictionaryEntry, boolean mergeConflicts) { String phrase = dictionaryEntry.phrase; DictionaryEntry existingDictionaryEntry = dictionary.get(phrase); if (existingDictionaryEntry == null) { dictionary.put(phrase, dictionaryEntry); } else { mergedCount++; - existingDictionaryEntry.merge(dictionaryEntry); + existingDictionaryEntry.merge(dictionaryEntry, mergeConflicts); } } void printDocument(ParserOptions parserOptions, long startTime) throws FileNotFoundException { + if (!inflectionPatterns.isEmpty()) { + Set referencedPatterns = Collections.newSetFromMap(new IdentityHashMap<>()); + for (DictionaryEntry dictionaryEntry : dictionary.values()) { + if (!dictionaryEntry.getGrammemes().isEmpty()) { + referencedPatterns.addAll(dictionaryEntry.getInflectionalPatterns()); + } + } + inflectionPatterns.removeIf(pattern -> !referencedPatterns.contains(pattern)); + } + TreeMap, Integer> grammemeCounts = new TreeMap<>(EnumComparator.ENUM_COMPARATOR); int unclassifiedTerms = 0; if (isInflectional()) { diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java index a8ee68787..4f72f4375 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java @@ -1175,9 +1175,18 @@ public String toString() { TYPEMAP.put("Q1909485", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // quantifier, but it might be used frequently as an indefinite noun TYPEMAP.put("Q694268", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // collective TYPEMAP.put("Q1450795", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // singulative + + for (var entry : DEFAULTMAP.entrySet()) { + @SuppressWarnings({"unchecked", "rawtypes"}) + Set> enumSet = EnumSet.of((Enum) entry.getValue()); + TYPEMAP.putIfAbsent(entry.getKey(), enumSet); + } } static Set> getMappedGrammemes(String grammeme) { + if (grammeme == null) { + throw new IllegalArgumentException("Grammeme cannot be null"); + } return TYPEMAP.get(grammeme); } } diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java index 33c038bab..3356f358a 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java @@ -18,6 +18,7 @@ import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.MismatchedInputException; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.lang3.StringUtils; @@ -126,13 +127,78 @@ private Lemma() { static final String VARIANT_SEPARATOR = "-x-"; - private void analyzeLexeme(int lineNumber, Lexeme lexeme) { + /** + * Finalizes a single surface-form inflection with lemma grammeme propagation, ignorable filtering, + * rarity extraction, and {@code --expand-grammemes} expansion. Shared by both Wikidata and DMLex paths. + * + * @return {@code false} if the entire lemma must be aborted due to {@code IGNORABLE_LEMMA}; {@code true} otherwise. + */ + private boolean finalizeAndAddInflection( + Lemma lemma, + @Nullable TreeSet> genderlessLemmaGrammemes, + Inflection currentInflection) { + if (genderlessLemmaGrammemes == null || countGrammemeType(currentInflection.grammemeSet, Gender.class) == 0) { + currentInflection.grammemeSet.addAll(lemma.grammemes); + } else { + currentInflection.grammemeSet.addAll(genderlessLemmaGrammemes); + } + if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_LEMMA)) { + documentState.unusableLemmaCount++; + return false; + } + if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_INFLECTION)) { + documentState.unusableSurfaceFormCount++; + return true; + } + currentInflection.rareUsage = currentInflection.grammemeSet.contains(Grammar.Usage.RARE); + if (currentInflection.rareUsage) { + currentInflection.grammemeSet.remove(Grammar.Usage.RARE); + } + currentInflection.grammemeSet.remove(Ignorable.IGNORABLE_PROPERTY); + var grammemeExpansion = parserOptions.expandGramemes != null + ? parserOptions.expandGramemes.get(currentInflection.grammemeSet) + : null; + if (grammemeExpansion == null) { + lemma.inflections.add(currentInflection); + } else { + for (var grammemeSet : grammemeExpansion) { + var expandedInflection = new Inflection(currentInflection.inflection, + currentInflection.rareUsage); + expandedInflection.grammemeSet.addAll(currentInflection.grammemeSet); + expandedInflection.grammemeSet.addAll(grammemeSet); + lemma.inflections.add(expandedInflection); + } + } + return true; + } + + /** + * Performs post-form lemma cleanup shared by Wikidata and DMLex: counts incoming surface forms, + * extracts lemma-level {@link Grammar.Usage#RARE}, rejects lemmas with no usable surface forms, + * and strips {@link Gender} from {@code lemma.grammemes}. + */ + private boolean finalizeLemmaAfterForms(Lemma lemma) { + documentState.incomingSurfaceForm += lemma.inflections.size(); + lemma.isRare = lemma.grammemes.contains(Grammar.Usage.RARE); + if (lemma.isRare) { + lemma.grammemes.remove(Grammar.Usage.RARE); + } + if (lemma.inflections.isEmpty()) { + documentState.unusableLemmaCount++; + return false; + } + removeGrammemeType(lemma.grammemes, Gender.class); + return true; + } + + private void analyzeLexeme(int lineNumber, Lexeme lexeme, int sourceIndex) { if (omitLemmas.contains(lexeme.id)) { // We really don't want this junk. return; } Lemma lemma = new Lemma(); Set> partOfSpeechSet = null; + boolean addedToDeferredInThisLexeme = false; for (var lemmaEntry : lexeme.lemmas.entrySet()) { var currentLemmaLanguage = lemmaEntry.getKey(); lemma.reset(); @@ -204,77 +270,186 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme) { } } convertGrammemes(form, currentInflection, lexeme.id, lemma.value); - if (genderlessLemmaGrammemes == null || countGrammemeType(currentInflection.grammemeSet, Gender.class) == 0) { - currentInflection.grammemeSet.addAll(lemma.grammemes); - } - else { - currentInflection.grammemeSet.addAll(genderlessLemmaGrammemes); - } - if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_LEMMA)) { - documentState.unusableLemmaCount++; + if (!finalizeAndAddInflection(lemma, genderlessLemmaGrammemes, currentInflection)) { return; } - if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_INFLECTION)) { - documentState.unusableSurfaceFormCount++; - continue; - } - currentInflection.rareUsage = currentInflection.grammemeSet.contains(Grammar.Usage.RARE); - if (currentInflection.rareUsage) { - currentInflection.grammemeSet.remove(Grammar.Usage.RARE); - } - currentInflection.grammemeSet.remove(Ignorable.IGNORABLE_PROPERTY); - var grammemeExpansion = parserOptions.expandGramemes != null - ? parserOptions.expandGramemes.get(currentInflection.grammemeSet) - : null; - if (grammemeExpansion == null) { - lemma.inflections.add(currentInflection); + } + if (!finalizeLemmaAfterForms(lemma)) { + return; + } + if (deferredLexemes.contains(lexeme.id)) { + if (addedToDeferredInThisLexeme) { + Lemma existing = deferredLemmaMap.get(lexeme.id); + existing.inflections.addAll(lemma.inflections); + existing.isRare |= lemma.isRare; + } else if (deferredLemmaMap.put(lexeme.id, lemma) != null) { + throw new IllegalStateException("Duplicate lexeme " + lexeme.id); } else { - for (var grammemeSet : grammemeExpansion) { - var expandedInflection = new Inflection(currentInflection.inflection, - currentInflection.rareUsage); - expandedInflection.grammemeSet.addAll(currentInflection.grammemeSet); - expandedInflection.grammemeSet.addAll(grammemeSet); - lemma.inflections.add(expandedInflection); - } + addedToDeferredInThisLexeme = true; + lemma = new Lemma(); } + } else { + analyzeLemma(lemma, sourceIndex); } - documentState.incomingSurfaceForm += lemma.inflections.size(); - lemma.isRare = lemma.grammemes.contains(Grammar.Usage.RARE); - if (lemma.isRare) { - lemma.grammemes.remove(Grammar.Usage.RARE); + } + } + + /** + * Converts a single OASIS DMLex {@link DMLexEntry} into the internal {@link Lemma} representation + * and routes it through the shared normalization and analysis pipeline. + */ + private void analyzeDMLexEntry(Lemma lemma, DMLexEntry entry, String lexiconLangCode, int sourceIndex) { + if (entry == null) { + throw new IllegalArgumentException("Null DMLex entry"); + } + String entryLang = (entry.langCode != null && !entry.langCode.isBlank()) ? entry.langCode : lexiconLangCode; + if (!LexemesJsonDeserializer.isContained(entryLang)) { + return; + } + String entryId = (entry.id != null && !entry.id.isBlank()) ? entry.id : ""; + if (omitLemmas.contains(entryId)) { + return; + } + lemma.reset(); + documentState.lemmaCount++; + if (entry.headword == null || entry.headword.isBlank()) { + throw new IllegalArgumentException("Missing headword for DMLex entry " + entryId); + } + + lemma.value = entry.headword; + + int qVariantIdx = entryLang.indexOf(VARIANT_SEPARATOR); + if (qVariantIdx >= 0) { + var additionalCategory = entryLang.substring(qVariantIdx + VARIANT_SEPARATOR.length()); + var variant = Grammar.getMappedGrammemes(additionalCategory); + if (variant == null) { + if (parserOptions.debug) { + System.err.println(additionalCategory + + " is not a known grammeme for the language variant " + entryId + "(" + lemma.value + ")"); + } + return; } - if (lemma.inflections.isEmpty()) { - documentState.unusableLemmaCount++; + if (variant.contains(Ignorable.IGNORABLE_INFLECTION)) { return; } - removeGrammemeType(lemma.grammemes, Gender.class); - if (deferredLexemes.contains(lexeme.id)) { - if (deferredLemmaMap.put(lexeme.id, lemma) != null) { - throw new IllegalStateException("Duplicate lexeme " + lexeme.id); + lemma.grammemes.addAll(variant); + } + + if (entry.partOfSpeech != null) { + for (String posStr : entry.partOfSpeech) { + var mappedPos = Grammar.getMappedGrammemes(posStr); + if (mappedPos == null) { + throw new IllegalArgumentException(posStr + + " is not a known part of speech grammeme for " + entryId + "(" + lemma.value + ")"); + } + lemma.grammemes.addAll(mappedPos); + } + } + if (rareLemmas.contains(entryId)) { + lemma.grammemes.add(Grammar.Usage.RARE); + } + if (entry.labels != null) { + for (String labelStr : entry.labels) { + var mappedLabel = Grammar.getMappedGrammemes(labelStr); + if (mappedLabel == null) { + throw new IllegalArgumentException(labelStr + + " is not a known grammeme for " + entryId + "(" + lemma.value + ")"); + } + lemma.grammemes.addAll(mappedLabel); + } + } + if (parserOptions.addSound && entry.pronunciations != null && !entry.pronunciations.isEmpty()) { + addSound("P898", entry.pronunciations, lemma.grammemes, entryId, lemma.value); + } + + if (lemma.grammemes.contains(Ignorable.IGNORABLE_LEMMA) + || lemma.grammemes.contains(Ignorable.IGNORABLE_INFLECTION)) { + documentState.unusableLemmaCount++; + return; + } + lemma.grammemes.remove(Ignorable.IGNORABLE_PROPERTY); + + TreeSet> genderlessLemmaGrammemes = null; + if (countGrammemeType(lemma.grammemes, Gender.class) > 0) { + genderlessLemmaGrammemes = new TreeSet<>(lemma.grammemes); + removeGrammemeType(genderlessLemmaGrammemes, Gender.class); + } + + List forms = entry.inflectedForms; + if (forms == null || forms.isEmpty()) { + DMLexInflectedForm defaultForm = new DMLexInflectedForm(); + defaultForm.value = entry.headword; + forms = List.of(defaultForm); + } + + for (DMLexInflectedForm form : forms) { + if (form == null || form.value == null || form.value.isBlank()) { + throw new IllegalArgumentException("Missing inflected form value for " + entryId + "(" + lemma.value + ")"); + } + Inflection currentInflection = new Inflection(form.value); + if (form.labels != null) { + for (String labelStr : form.labels) { + Set> values = Grammar.getMappedGrammemes(labelStr); + if (values == null) { + throw new IllegalArgumentException(labelStr + + " is not a known grammeme for " + entryId + "(" + lemma.value + ")"); + } + currentInflection.grammemeSet.addAll(values); } } - else { - analyzeLemma(lemma); + if (parserOptions.addSound && form.pronunciations != null && !form.pronunciations.isEmpty()) { + addSound("P898", form.pronunciations, currentInflection.grammemeSet, entryId, lemma.value); + } + if (!finalizeAndAddInflection(lemma, genderlessLemmaGrammemes, currentInflection)) { + return; } } + + if (!finalizeLemmaAfterForms(lemma)) { + return; + } + analyzeLemma(lemma, sourceIndex); } - // Method to process and merge lexemes - private void processAndMergeLexemes() { - for (Map.Entry> entry : mergeMap.entrySet()) { - Lemma lemma = deferredLemmaMap.computeIfAbsent(entry.getKey(), key -> { - throw new IllegalArgumentException(key + ": id not found"); - }); - for (var value : entry.getValue()) { - Lemma lemmaToMerge = deferredLemmaMap.computeIfAbsent(value, key -> { + private void processAndMergeLexemes(int sourceIndex) { + if (sourceIndex == 0) { + for (Map.Entry> entry : mergeMap.entrySet()) { + Lemma lemma = deferredLemmaMap.computeIfAbsent(entry.getKey(), key -> { throw new IllegalArgumentException(key + ": id not found"); }); - lemma.inflections.addAll(lemmaToMerge.inflections); - lemma.isRare |= lemmaToMerge.isRare; - // The lemma grammemes should already have the relevant POS data. + for (var value : entry.getValue()) { + Lemma lemmaToMerge = deferredLemmaMap.computeIfAbsent(value, key -> { + throw new IllegalArgumentException(key + ": id not found"); + }); + lemma.inflections.addAll(lemmaToMerge.inflections); + lemma.isRare |= lemmaToMerge.isRare; + // The lemma grammemes should already have the relevant POS data. + } + analyzeLemma(lemma, sourceIndex); + } + } else { + for (Map.Entry> entry : mergeMap.entrySet()) { + Lemma lemma = deferredLemmaMap.remove(entry.getKey()); + for (var value : entry.getValue()) { + Lemma lemmaToMerge = deferredLemmaMap.remove(value); + if (lemmaToMerge != null) { + if (lemma == null) { + lemma = lemmaToMerge; + } else { + lemma.inflections.addAll(lemmaToMerge.inflections); + lemma.isRare |= lemmaToMerge.isRare; + } + } + } + if (lemma != null) { + analyzeLemma(lemma, sourceIndex); + } + } + for (Lemma remaining : deferredLemmaMap.values()) { + analyzeLemma(remaining, sourceIndex); } - analyzeLemma(lemma); } + deferredLemmaMap.clear(); } private void removeGrammemeType(TreeSet> grammemes, Class grammemeType) { @@ -480,7 +655,7 @@ private InflectionPattern getInflectionPattern(Lemma lemma, String lemmaSuffix, return inflectionPattern; } - private void analyzeInflections(Lemma lemma, List inputInflections) { + private void analyzeInflections(Lemma lemma, List inputInflections, int sourceIndex) { List inflections = new ArrayList<>(); for (Inflection inflection : inputInflections) { inflections.addAll(enumerateInflectionsForGrammemeCombinations(inflection)); @@ -527,8 +702,10 @@ private void analyzeInflections(Lemma lemma, List inputInflections) String phrase = inflection.getInflection(); InflectionPattern inflectionPatternForDict = nonEmptyInflectionIndices.contains(i) ? inflectionPattern : null; - documentState.addDictionaryEntry(new DictionaryEntry(phrase, phrase, lemma.isRare, - inflection.getGrammemeSet(), inflectionPatternForDict)); + documentState.addDictionaryEntry( + new DictionaryEntry(phrase, lemma.value, lemma.isRare, + inflection.getGrammemeSet(), inflectionPatternForDict, sourceIndex), + parserOptions.mergeConflicts); } } @@ -598,12 +775,12 @@ private void mergeAdditionalGrammemes() { } } - private void analyzeLemma(Lemma lemma) { - analyzeInflections(lemma, lemma.inflections); + private void analyzeLemma(Lemma lemma, int sourceIndex) { + analyzeInflections(lemma, lemma.inflections, sourceIndex); if (parserOptions.includeLemmasWithoutWords) { documentState.dictionary.computeIfAbsent(lemma.value, - d -> new DictionaryEntry(lemma.value, lemma.isRare, lemma.grammemes, null)); + d -> new DictionaryEntry(lemma.value, lemma.value, lemma.isRare, lemma.grammemes, null, sourceIndex)); } } @@ -611,6 +788,48 @@ public void writeInflectionPatterns(long startTime) throws FileNotFoundException documentState.printDocument(parserOptions, startTime); } + private void parseWikidataStream(ObjectMapper objectMapper, JsonParser parser, int sourceIndex) throws IOException { + JsonToken currToken; + while ((currToken = parser.nextToken()) != JsonToken.START_OBJECT && currToken != null) { + // Find the first object in the array. + } + if (currToken == JsonToken.START_OBJECT) { + do { + Lexeme lexeme = objectMapper.readValue(parser, Lexeme.class); + try { + analyzeLexeme(parser.currentLocation().getLineNr(), lexeme, sourceIndex); + } catch (IllegalArgumentException e) { + documentState.unusableLemmaCount++; + System.err.println( + "Line " + parser.currentLocation().getLineNr() + ": " + e.getMessage()); + } + } while (parser.nextToken() != JsonToken.END_ARRAY); + processAndMergeLexemes(sourceIndex); + } + } + + private void parseDMLexStream( + ObjectMapper objectMapper, + JsonParser parser, + ParserOptions.SourceInput sourceInput) throws IOException { + DMLexEntry.Lexicon lexicon = objectMapper.readValue(parser, DMLexEntry.Lexicon.class); + if (lexicon == null || lexicon.langCode == null || lexicon.langCode.isBlank() || lexicon.entries == null) { + throw MismatchedInputException.from( + parser, + DMLexEntry.Lexicon.class, + "Invalid DMLex file (missing required 'langCode' or 'entries'): " + sourceInput.filename()); + } + Lemma lemma = new Lemma(); + for (DMLexEntry entry : lexicon.entries) { + try { + analyzeDMLexEntry(lemma, entry, lexicon.langCode, sourceInput.sourceIndex()); + } catch (IllegalArgumentException e) { + documentState.unusableLemmaCount++; + System.err.println(e.getMessage()); + } + } + } + public static void main(String[] args) throws Exception { ParserOptions parserOptions = new ParserOptions(args); long startTime = System.currentTimeMillis(); @@ -625,7 +844,8 @@ public static void main(String[] args) throws Exception { // being interpreted as malformed UTF-8. // We use a large buffer because we're reading a large file, and we're // frequently reading file data. - for (String sourceFilename : parserOptions.sourceFilenames) { + for (ParserOptions.SourceInput sourceInput : parserOptions.sourceInputs) { + String sourceFilename = sourceInput.filename(); try (InputStream fileInputStream = new FileInputStream(sourceFilename)) { InputStream inputStream = fileInputStream; if (sourceFilename.endsWith(".bz2")) { @@ -633,22 +853,10 @@ public static void main(String[] args) throws Exception { inputStream = new BZip2CompressorInputStream(new BufferedInputStream(inputStream, 32768)); } try (JsonParser parser = objectMapper.createParser(inputStream)) { - JsonToken currToken; - while ((currToken = parser.nextToken()) != JsonToken.START_OBJECT && currToken != null) { - // Find the first object in the array. - } - if (currToken == JsonToken.START_OBJECT) { - do { - Lexeme lexeme = objectMapper.readValue(parser, Lexeme.class); - try { - lexParser.analyzeLexeme(parser.currentLocation().getLineNr(), lexeme); - } catch (IllegalArgumentException e) { - lexParser.documentState.unusableLemmaCount++; - System.err.println( - "Line " + parser.currentLocation().getLineNr() + ": " + e.getMessage()); - } - } while (parser.nextToken() != JsonToken.END_ARRAY); - lexParser.processAndMergeLexemes(); + if (sourceInput.format() == ParserOptions.SourceFormat.DMLEX) { + lexParser.parseDMLexStream(objectMapper, parser, sourceInput); + } else { + lexParser.parseWikidataStream(objectMapper, parser, sourceInput.sourceIndex()); } } } diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java index 98768b47b..65eee98c6 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java @@ -31,7 +31,17 @@ * The options to extract the data from the data source. */ final class ParserOptions { + enum SourceFormat { + WIKIDATA, + DMLEX + } + + record SourceInput(String filename, SourceFormat format, int sourceIndex) {} + private static final char COLON_SEPARATOR = ':'; + static final String WIKIDATA_FILE = "--wikidata"; + static final String DMLEX_FILE = "--dmlex"; + static final String MERGE_CONFLICTS = "--merge-conflicts"; static final String INFLECTIONS_FILE = "--inflections"; static final String DICTIONARY_FILE = "--dictionary"; static final String MAP_GRAMMEME = "--map-grammeme"; @@ -47,6 +57,7 @@ final class ParserOptions { static final String ADD_SOUND = "--add-sound"; boolean includeLemmasWithoutWords = false; + boolean mergeConflicts = false; boolean debug = false; final boolean addSound; @@ -56,6 +67,7 @@ final class ParserOptions { TreeMap> claimsToSound; ArrayList sourceFilenames; + ArrayList sourceInputs; String inflectionalFilename = ParserDefaults.DEFAULT_INFLECTION_FILE_NAME; String lexicalDictionaryFilename = ParserDefaults.DEFAULT_DICTIONARY_FILE_NAME; ArrayList locales = new ArrayList<>(List.of(Locale.ENGLISH.getLanguage())); @@ -63,8 +75,11 @@ final class ParserOptions { List optionsUsedToInvoke = new ArrayList<>(); private static void printUsage() { - System.err.println("Usage: ParseLexicon [OPTIONS] [ ...]"); + System.err.println("Usage: ParseLexicon [OPTIONS] (--wikidata [ ...] | --dmlex [ ...]) ..."); System.err.println("\nOPTIONS"); + System.err.println(WIKIDATA_FILE + " [ ...]\tOne or more Wikidata lexeme JSON/bz2 files (space- or comma-separated)"); + System.err.println(DMLEX_FILE + " [ ...]\tOne or more OASIS DMLex JSON files (space- or comma-separated)"); + System.err.println(MERGE_CONFLICTS + "[={true|false}]\tWhen true, merge conflicting cross-file surface forms (with the later source's pattern first) instead of overwriting, default: false"); System.err.println(INFLECTIONS_FILE + " \tthe file for the inflectional patterns to be generated, default: inflectional.xml"); System.err.println(DICTIONARY_FILE + " \tthe file for the lexical dictionary to be generated, default: dictionary.lst"); System.err.println(ADD_EXTRA_GRAMMEMES + " \tFile containing words with the extra grammemes to be added, provide path relative to tools/dictionary-parser/src/main/resources/org/unicode/wikidata/ (only to be used for a temporary grammeme addition)"); @@ -84,11 +99,34 @@ private static void printUsage() { posToBeInflected = EnumSet.of(Grammar.PartOfSpeech.NOUN); additionalGrammemesDict = new TreeMap<>(); sourceFilenames = new ArrayList<>(); + sourceInputs = new ArrayList<>(); claimsToSound = new TreeMap<>(); for (int i = 0; i < args.length; i++) { String arg = args[i]; - if (ParserOptions.INFLECTIONS_FILE.equals(arg)) { + if (arg == null) { + printUsage(); + throw new IllegalArgumentException("Null argument at index " + i); + } + if (ParserOptions.WIKIDATA_FILE.equals(arg) || arg.startsWith(ParserOptions.WIKIDATA_FILE + "=")) { + i = consumeSourceFiles(args, i, ParserOptions.WIKIDATA_FILE, SourceFormat.WIKIDATA); + } else if (ParserOptions.DMLEX_FILE.equals(arg) || arg.startsWith(ParserOptions.DMLEX_FILE + "=")) { + i = consumeSourceFiles(args, i, ParserOptions.DMLEX_FILE, SourceFormat.DMLEX); + } else if (ParserOptions.MERGE_CONFLICTS.equals(arg) || arg.startsWith(ParserOptions.MERGE_CONFLICTS + "=")) { + String valueStr = "true"; + if (arg.startsWith(ParserOptions.MERGE_CONFLICTS + "=")) { + valueStr = arg.substring(ParserOptions.MERGE_CONFLICTS.length() + 1).trim(); + } else if (i + 1 < args.length && ("true".equalsIgnoreCase(args[i + 1]) || "false".equalsIgnoreCase(args[i + 1]))) { + valueStr = args[++i].trim(); + } + if ("true".equalsIgnoreCase(valueStr) || "false".equalsIgnoreCase(valueStr)) { + mergeConflicts = Boolean.parseBoolean(valueStr); + } else { + printUsage(); + throw new IllegalArgumentException("Invalid value for " + ParserOptions.MERGE_CONFLICTS + ": " + valueStr); + } + optionsUsedToInvoke.add(ParserOptions.MERGE_CONFLICTS + "=" + mergeConflicts); + } else if (ParserOptions.INFLECTIONS_FILE.equals(arg)) { inflectionalFilename = args[++i]; } else if (ParserOptions.DICTIONARY_FILE.equals(arg)) { lexicalDictionaryFilename = args[++i]; @@ -141,7 +179,7 @@ private static void printUsage() { posToBeInflected.clear(); for (String pos : inflectionTypes.split(",")) { - posToBeInflected.add(Grammar.PartOfSpeech.valueOf(pos.toUpperCase().replace('-', '_'))); + posToBeInflected.add(Grammar.PartOfSpeech.valueOf(pos.toUpperCase(Locale.ROOT).replace('-', '_'))); } optionsUsedToInvoke.add(ParserOptions.INFLECTION_TYPES); @@ -202,28 +240,53 @@ private static void printUsage() { optionsUsedToInvoke.add(ParserOptions.ADD_SOUND); optionsUsedToInvoke.add(soundGrammemeTypes); } else { - sourceFilenames.add(arg); + printUsage(); + throw new IllegalArgumentException("Unknown option: " + arg); } } addSound = !claimsToSound.isEmpty(); - if (sourceFilenames.isEmpty()) { + if (sourceInputs.isEmpty()) { + printUsage(); + throw new IllegalArgumentException("At least one input file must be specified via --wikidata or --dmlex"); + } + } + + private int consumeSourceFiles(String[] args, int i, String flagPrefix, SourceFormat format) { + int startCount = sourceInputs.size(); + if (args[i].startsWith(flagPrefix + "=")) { + addSourceFiles(args[i].substring(flagPrefix.length() + 1), format); + } + while (i + 1 < args.length && args[i + 1] != null && !args[i + 1].startsWith("--")) { + addSourceFiles(args[++i], format); + } + if (sourceInputs.size() == startCount) { printUsage(); - throw new IllegalArgumentException(); + throw new IllegalArgumentException(flagPrefix + " requires at least one file argument"); + } + return i; + } + + private void addSourceFiles(String rawArg, SourceFormat format) { + for (String part : rawArg.split(",")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + sourceInputs.add(new SourceInput(trimmed, format, sourceInputs.size())); + sourceFilenames.add(trimmed); + } } } void setIgnoreProperty(String[] grammemes, Grammar.Ignorable ignorable) { var ignorableSet = EnumSet.of(ignorable); for (String grammeme : grammemes) { - if (grammeme.matches("Q\\d*")) { - TYPEMAP.put(grammeme, ignorableSet); - } else { + boolean matched = false; + if (!grammeme.matches("Q\\d*")) { for (Map.Entry>> entry : TYPEMAP.entrySet()) { for (var grammemeEnum : entry.getValue()) { - String name = grammemeEnum.name(); - if (name.equalsIgnoreCase(grammeme)) { + if (grammemeEnum.toString().equalsIgnoreCase(grammeme)) { + matched = true; if (entry.getValue().size() == 1) { entry.setValue(ignorableSet); } else { @@ -237,6 +300,9 @@ void setIgnoreProperty(String[] grammemes, Grammar.Ignorable ignorable) { } } } + if (!matched) { + TYPEMAP.put(grammeme, ignorableSet); + } } } diff --git a/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java b/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java index cf057c899..158ba53d7 100644 --- a/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java +++ b/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java @@ -7,8 +7,19 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.exc.MismatchedInputException; import org.junit.jupiter.api.Assertions; @@ -18,13 +29,19 @@ class ParseWikidataTest { private static final String lexiconSource = Objects .requireNonNull(MethodHandles.lookup().lookupClass().getClassLoader().getResource("sourceLexicon.json")).getFile(); private static final String baseDir = lexiconSource.substring(0, lexiconSource.lastIndexOf("/")); - private static final String inflectionalFile = baseDir+"/LexiconInflectOut.xml"; - private static final String dictionaryFile = baseDir+"/LexiconInflectOut.lst"; - private static final String expectedOutputFile = baseDir+ "/lexiconCorrectOut.txt"; - private static final String fatalErrorFile = baseDir+ "/lexiconFatalError.json"; - private static final String missingGrammemeErrorFile = baseDir+ "/lexiconMissingGrammemeError.json"; - private static final String caseInsensitiveGrammemeFile = baseDir+ "/lexiconcaseInsenstiveGrammeme.json"; + private static final String inflectionalFile = baseDir + "/LexiconInflectOut.xml"; + private static final String dictionaryFile = baseDir + "/LexiconInflectOut.lst"; + private static final String expectedOutputFile = baseDir + "/lexiconCorrectOut.txt"; + private static final String fatalErrorFile = baseDir + "/lexiconFatalError.json"; + private static final String missingGrammemeErrorFile = baseDir + "/lexiconMissingGrammemeError.json"; + private static final String caseInsensitiveGrammemeFile = baseDir + "/lexiconcaseInsenstiveGrammeme.json"; + private static final String dmlexLexiconFile = baseDir + "/dmlexLexicon.json"; + private static final String dmlexOverlayFile = baseDir + "/dmlexOverlay.json"; + private static final String dmlexPatchFile = baseDir + "/dmlexPatch.json"; + private static final String dmlexInvalidSchemaFile = baseDir + "/dmlexInvalidSchema.json"; private static final String LINE_END = "=============================================="; + private static final Pattern XML_PATTERN_NAME = Pattern.compile(" parseDictionaryEntries(String output) { + return Arrays.stream(output.split("\n")) + .takeWhile(line -> !line.equals(LINE_END)) + .filter(line -> line.contains(":")) + .collect(Collectors.toMap( + line -> line.substring(0, line.indexOf(':')), + line -> line.substring(line.indexOf(':') + 1).trim())); + } + + private Set assertAllXmlPatternsReferencedInDictionary(String dictionaryOutput) throws Exception { + String xml = Files.readString(Paths.get(inflectionalFile), StandardCharsets.UTF_8); + Set xmlPatternIds = new TreeSet<>(); + Matcher xmlMatcher = XML_PATTERN_NAME.matcher(xml); + while (xmlMatcher.find()) { + Assertions.assertTrue(xmlPatternIds.add(xmlMatcher.group(1)), + "Duplicate pattern name in inflectional XML: " + xmlMatcher.group(1)); + } + Set dictPatternIds = new TreeSet<>(); + for (String line : dictionaryOutput.split("\n")) { + if (line.equals(LINE_END)) { + break; + } + Matcher dictMatcher = DICT_INFLECTION_ID.matcher(line); + while (dictMatcher.find()) { + dictPatternIds.add(dictMatcher.group(1)); + } + } + Assertions.assertEquals(dictPatternIds, xmlPatternIds, + "Every in inflectional.xml must be referenced as inflection=X in dictionary.lst"); + return xmlPatternIds; + } + @Test public void missingInputFile() { - Assertions.assertThrows(Exception.class,() -> ParseWikidata.main(new String[1])); + Assertions.assertThrows(Exception.class, () -> ParseWikidata.main(new String[1])); } @Test public void enumTest() { String fieldValue = Grammar.Alternate.SPELLING.toString(); Assertions.assertEquals("spelling", fieldValue); + Assertions.assertThrows(IllegalArgumentException.class, () -> Grammar.getMappedGrammemes(null)); } @Test public void fatalErrorHandlerTest() { // Malformed JSON syntax cannot be parsed by the Wikidata reader. - String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, fatalErrorFile}; - Assertions.assertThrows(JsonParseException.class,() -> ParseWikidata.main(args)); + String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", fatalErrorFile}; + Assertions.assertThrows(JsonParseException.class, () -> ParseWikidata.main(args)); } @Test public void missingGrammemeTest() { // The grammatical features are not in the expected array form, so the reader rejects the input. - String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, missingGrammemeErrorFile}; - Assertions.assertThrows(MismatchedInputException.class,() -> ParseWikidata.main(args)); + String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", missingGrammemeErrorFile}; + Assertions.assertThrows(MismatchedInputException.class, () -> ParseWikidata.main(args)); } @Test public void caseInsensitiveGrammemeTest() { - String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, caseInsensitiveGrammemeFile}; + String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", caseInsensitiveGrammemeFile}; Assertions.assertDoesNotThrow(() -> ParseWikidata.main(args)); } @Test public void lexiconParserTest() throws Exception { String[] args = {"--inflection-types", "noun,adjective,proper-noun", "--add-sound", "vowel-start", - "--inflections", inflectionalFile, "--dictionary", dictionaryFile, lexiconSource}; + "--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", lexiconSource}; String actual = getParserOutput(args); String expected = Files.readString(Paths.get(expectedOutputFile), StandardCharsets.UTF_8); compareOutputs(actual, expected); } + + @Test + public void standaloneDMLexParserTest() throws Exception { + String[] args = { + "--inflection-types", "noun,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexLexiconFile + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // cat and bat share pattern 1 (count=2, most frequent) + Assertions.assertEquals("singular masculine noun inflection=1", entries.get("cat")); + Assertions.assertEquals("plural masculine noun inflection=1", entries.get("cats")); + Assertions.assertEquals("singular masculine noun inflection=1", entries.get("bat")); + Assertions.assertEquals("plural masculine noun inflection=1", entries.get("bats")); + + // apple has IPA pronunciation matching vowel-start + Assertions.assertEquals("singular vowel-start noun inflection=2", entries.get("apple")); + Assertions.assertEquals("plural vowel-start noun inflection=2", entries.get("apples")); + + // London uses proper-noun + Assertions.assertEquals("singular proper-noun inflection=3", entries.get("London")); + + // wow has no inflectedForms array, synthesized default form + Assertions.assertEquals("interjection", entries.get("wow")); + + // French entry "bonjour" must be filtered out because default --language is en + Assertions.assertFalse(entries.containsKey("bonjour")); + + // Verify inflectional XML was generated + String xml = Files.readString(Paths.get(inflectionalFile), StandardCharsets.UTF_8); + Assertions.assertTrue(xml.contains("")); + Assertions.assertTrue(xml.contains("")); + } + + @Test + public void multiSourceWikidataAndDMLexOverwriteTest() throws Exception { + // Default --merge-conflicts=false (Option 4 + Option 3): normal processing happens for earlier + // files, and when a surface form is encountered in a later file, the new pattern and grammemes + // override all previous data for that surface form. + String[] args = { + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", lexiconSource, + "--dmlex", dmlexOverlayFile + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // 1. New lemma "book" from DMLex is added + Assertions.assertTrue(entries.containsKey("book")); + Assertions.assertTrue(entries.get("book").startsWith("singular noun inflection=")); + Assertions.assertTrue(entries.containsKey("book's")); + Assertions.assertTrue(entries.containsKey("books")); + Assertions.assertTrue(entries.containsKey("books'")); + + // 2. "hey" was interjection in Wikidata and singular noun in DMLex -> DMLex overrides "hey" + String heyEntry = entries.get("hey"); + Assertions.assertNotNull(heyEntry); + Assertions.assertTrue(heyEntry.startsWith("singular noun inflection="), "Expected singular noun override in: " + heyEntry); + Assertions.assertFalse(heyEntry.contains("interjection"), "Did not expect interjection after override in: " + heyEntry); + Assertions.assertTrue(entries.containsKey("heys")); + + // 3. "ciel" and "cieux" were masculine noun in Wikidata, overridden by feminine noun in DMLex + String cielEntry = entries.get("ciel"); + Assertions.assertNotNull(cielEntry); + Assertions.assertTrue(cielEntry.contains("feminine"), "Expected feminine in: " + cielEntry); + Assertions.assertFalse(cielEntry.contains("masculine"), "Did not expect masculine in: " + cielEntry); + Assertions.assertEquals(1, cielEntry.split("inflection=").length - 1, "Expected 1 override pattern in: " + cielEntry); + + String cieuxEntry = entries.get("cieux"); + Assertions.assertNotNull(cieuxEntry); + Assertions.assertTrue(cieuxEntry.contains("feminine"), "Expected feminine in: " + cieuxEntry); + Assertions.assertFalse(cieuxEntry.contains("masculine"), "Did not expect masculine in: " + cieuxEntry); + Assertions.assertEquals(1, cieuxEntry.split("inflection=").length - 1, "Expected 1 override pattern in: " + cieuxEntry); + + // "ciels" was processed in Wikidata and not overridden in DMLex (Option 4) + Assertions.assertTrue(entries.containsKey("ciels")); + Assertions.assertTrue(entries.get("ciels").startsWith("plural masculine noun inflection=")); + assertAllXmlPatternsReferencedInDictionary(actual); + } + + @Test + public void multiSourceWikidataAndDMLexMergeConflictsTrueTest() throws Exception { + // --merge-conflicts=true (Option 4 + Option 2): conflicting entries across files mix in + // their grammemes and patterns, and the later source's conflicting pattern comes first. + String[] args = { + "--merge-conflicts=true", + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", lexiconSource, + "--dmlex", dmlexOverlayFile + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // "hey" merges Wikidata (interjection) and DMLex (singular noun) + String heyEntry = entries.get("hey"); + Assertions.assertNotNull(heyEntry); + Assertions.assertTrue(heyEntry.contains("noun"), "Expected noun in: " + heyEntry); + Assertions.assertTrue(heyEntry.contains("interjection"), "Expected interjection in: " + heyEntry); + + // "ciel" merges Wikidata (masculine) and DMLex (feminine), keeping both patterns, + // with the DMLex conflict pattern placed FIRST before the higher-count Wikidata pattern! + String cielEntry = entries.get("ciel"); + Assertions.assertNotNull(cielEntry); + Assertions.assertTrue(cielEntry.contains("masculine"), "Expected masculine in: " + cielEntry); + Assertions.assertTrue(cielEntry.contains("feminine"), "Expected feminine in: " + cielEntry); + String[] cielParts = cielEntry.split("inflection="); + Assertions.assertEquals(3, cielParts.length, "Expected 2 inflection patterns in: " + cielEntry); + String firstPatternId = cielParts[1].trim(); + String secondPatternId = cielParts[2].trim(); + + // "ciels" only exists in Wikidata, so its pattern ID is the Wikidata pattern ID. + // Verify that on "ciel", the Wikidata pattern ID is second and the DMLex override pattern ID is first! + String cielsEntry = entries.get("ciels"); + Assertions.assertNotNull(cielsEntry); + String wikidataPatternId = cielsEntry.split("inflection=")[1].trim(); + Assertions.assertEquals(wikidataPatternId, secondPatternId, + "Expected Wikidata pattern to be second after DMLex override pattern in: " + cielEntry); + Assertions.assertNotEquals(firstPatternId, secondPatternId); + assertAllXmlPatternsReferencedInDictionary(actual); + } + + @Test + public void multiSourceSameFileHomographAndMergePriorityTest() throws Exception { + // Tests: + // 1. When a later DMLex file defines multiple entries for the same surface form ("Élie" as both + // proper-noun and verb in the same file), under --merge-conflicts=false (Option 4 + Option 3) + // the first entry overrides Wikidata's "Élie" and the second entry from the same DMLex file + // merges with it. + // 2. Under --merge-conflicts=true (Option 4 + Option 2), both Wikidata and DMLex grammemes/patterns + // are mixed in. + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-homograph-", ".json"); + try { + String json = """ + { + "langCode": "en", + "entries": [ + { + "id": "elie-proper-noun", + "headword": "Élie", + "partOfSpeech": ["proper-noun"], + "tag": "feminine", + "pronunciations": [ + { + "transcriptions": [ + { "text": "e.li", "scheme": "IPA" } + ] + } + ], + "inflectedForms": [ + { + "value": "Élie", + "tag": "singular" + } + ] + }, + { + "id": "elie-verb", + "headword": "Élie", + "partOfSpeech": ["verb"], + "inflectedForms": [ + { + "value": "Élie" + } + ] + } + ] + } + """; + Files.writeString(tempDmlex, json, StandardCharsets.UTF_8); + + String[] overwriteArgs = { + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", lexiconSource, + "--dmlex=" + tempDmlex + }; + String overwriteOutput = getParserOutput(overwriteArgs); + Map overwriteEntries = parseDictionaryEntries(overwriteOutput); + String elieOverwrite = overwriteEntries.get("Élie"); + Assertions.assertNotNull(elieOverwrite); + // Wikidata's "plural masculine noun proper-noun" was overridden by tempDmlex's first entry + // ("singular feminine vowel-start proper-noun"), and tempDmlex's second entry ("verb") merged with it. + Assertions.assertTrue(elieOverwrite.startsWith("singular feminine vowel-start proper-noun verb inflection="), + "Unexpected overwrite entry: " + elieOverwrite); + Assertions.assertEquals(1, elieOverwrite.split("inflection=").length - 1); + // Because Wikidata's "Élie" was the only surface form referencing its inflection pattern, + // overriding "Élie" must prune that superseded pattern from inflectional.xml so all XML patterns + // are referenced in dictionary.lst (7 patterns total instead of 8). + Set overwriteXmlPatternIds = assertAllXmlPatternsReferencedInDictionary(overwriteOutput); + Assertions.assertEquals(7, overwriteXmlPatternIds.size()); + + String[] mergeArgs = { + "--merge-conflicts=true", + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", lexiconSource, + "--dmlex=" + tempDmlex + }; + String mergeOutput = getParserOutput(mergeArgs); + Map mergeEntries = parseDictionaryEntries(mergeOutput); + String elieMerge = mergeEntries.get("Élie"); + Assertions.assertNotNull(elieMerge); + Assertions.assertTrue( + elieMerge.startsWith("singular plural masculine feminine vowel-start noun proper-noun verb inflection="), + "Unexpected merge entry: " + elieMerge); + Assertions.assertEquals(2, elieMerge.split("inflection=").length - 1); + // Under --merge-conflicts=true, "Élie" retains both the DMLex pattern and the Wikidata pattern, + // so all 8 patterns remain referenced in dictionary.lst and present in inflectional.xml. + Set mergeXmlPatternIds = assertAllXmlPatternsReferencedInDictionary(mergeOutput); + Assertions.assertEquals(8, mergeXmlPatternIds.size()); + + // Verify that if a pattern was rare in an earlier source (sourceIndex=0) alongside a non-rare + // pattern, and a later source (sourceIndex=1) uses that same pattern non-rarely, it is promoted + // to non-rare and sorted first under --merge-conflicts=true. + TreeSet> grammemes = new TreeSet<>(EnumComparator.ENUM_COMPARATOR); + grammemes.add(Grammar.PartOfSpeech.NOUN); + Inflection suffixS = new Inflection("s", false); + suffixS.addGrammemes(List.of(Grammar.PartOfSpeech.NOUN, Grammar.Number.PLURAL)); + Inflection suffixEs = new Inflection("es", false); + suffixEs.addGrammemes(List.of(Grammar.PartOfSpeech.NOUN, Grammar.Number.PLURAL)); + InflectionPattern pattern1 = new InflectionPattern(1, "", grammemes, List.of(suffixS)); + InflectionPattern pattern2 = new InflectionPattern(2, "", grammemes, List.of(suffixEs)); + // Give pattern2 a higher count than pattern1 + pattern2.merge(pattern2); + + DictionaryEntry baseRareEntry = new DictionaryEntry("word", "word", true, grammemes, pattern1, 0); + DictionaryEntry baseNonRareEntry = new DictionaryEntry("word", "word", false, grammemes, pattern2, 0); + baseRareEntry.merge(baseNonRareEntry, true); + // Before override, pattern1 is rare so pattern2 (non-rare) comes first: + Assertions.assertEquals("word: noun inflection=2 inflection=1", baseRareEntry.toString(true)); + + // Now merge a later source (sourceIndex=1) that uses pattern1 as non-rare: + DictionaryEntry overrideNonRareEntry = new DictionaryEntry("word", "word", false, grammemes, pattern1, 1); + baseRareEntry.merge(overrideNonRareEntry, true); + Assertions.assertEquals("word: noun inflection=1 inflection=2", baseRareEntry.toString(true)); + } finally { + Files.deleteIfExists(tempDmlex); + } + } + + @Test + public void multiFileDMLexSpaceAndCommaSeparatedTest() throws Exception { + String[] spaceArgs = { + "--inflection-types", "noun,proper-noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexLexiconFile, dmlexPatchFile + }; + String spaceOutput = getParserOutput(spaceArgs); + + String[] commaArgs = { + "--inflection-types", "noun,proper-noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexLexiconFile + "," + dmlexPatchFile + }; + String commaOutput = getParserOutput(commaArgs); + + compareOutputs(spaceOutput, commaOutput); + + Map entries = parseDictionaryEntries(spaceOutput); + // "cat" surface form was overridden by dmlexPatch.json: feminine, plural "kitties" added, + // while "cats" from dmlexLexicon.json remains from normal processing (Option 4 + Option 3). + Assertions.assertTrue(entries.get("cat").contains("singular feminine noun")); + Assertions.assertFalse(entries.get("cat").contains("masculine")); + Assertions.assertTrue(entries.containsKey("kitties")); + Assertions.assertTrue(entries.containsKey("cats")); + + // "bats" surface form from lemma "bat" (masculine noun) was overridden by lemma "batte" (feminine noun) + Assertions.assertTrue(entries.get("bats").contains("plural feminine noun")); + Assertions.assertFalse(entries.get("bats").contains("masculine")); + + // "wow" surface form was overridden by dmlexPatch.json (verb) + Assertions.assertEquals("verb", entries.get("wow")); + Assertions.assertEquals("verb", entries.get("wowed")); + assertAllXmlPatternsReferencedInDictionary(spaceOutput); + } + + @Test + public void invalidDMLexSchemaAndUnknownOptionTest() { + // 1. JSON object missing required "langCode" / "entries" + String[] invalidObjArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexInvalidSchemaFile + }; + Assertions.assertThrows(MismatchedInputException.class, () -> ParseWikidata.main(invalidObjArgs)); + + // 2. Passing Wikidata JSON array to --dmlex + String[] wikidataAsDmlexArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", lexiconSource + }; + Assertions.assertThrows(MismatchedInputException.class, () -> ParseWikidata.main(wikidataAsDmlexArgs)); + + // 3. Bare positional file without --wikidata or --dmlex is rejected + String[] bareFileArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + lexiconSource + }; + Assertions.assertThrows(IllegalArgumentException.class, () -> ParseWikidata.main(bareFileArgs)); + + // 4. --dmlex with no file argument is rejected + String[] emptyDmlexArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex" + }; + Assertions.assertThrows(IllegalArgumentException.class, () -> ParseWikidata.main(emptyDmlexArgs)); + } + + @Test + public void dmlexCliOptionsCompatibilityTest() throws Exception { + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-opts-", ".json"); + Map>> savedTypeMap = new HashMap<>(Grammar.TYPEMAP.size()); + for (var entry : Grammar.TYPEMAP.entrySet()) { + savedTypeMap.put(entry.getKey(), new HashSet<>(entry.getValue())); + } + try { + String json = """ + { + "langCode": "ro", + "entries": [ + { + "id": "ro-1", + "headword": "casă", + "partOfSpeech": ["noun"], + "labels": ["feminine", "countable", "dialectal"], + "inflectedForms": [ + { "value": "casă", "labels": ["singular", "customNomAcc"] }, + { "value": "case", "labels": ["plural", "customNomAcc"] }, + { "value": "casă-short", "labels": ["singular", "short-form"] }, + { "value": "casă-archaic", "labels": ["singular", "archaic"] } + ] + }, + { + "id": "ro-md-1", + "headword": "moldovanWord", + "langCode": "ro-md", + "partOfSpeech": ["noun"], + "inflectedForms": [ + { "value": "moldovanWord", "labels": ["singular"] } + ] + } + ] + } + """; + Files.writeString(tempDmlex, json, StandardCharsets.UTF_8); + + String[] args = { + "--language", "ro", + "--exclude-language", "ro-md", + "--inflection-types", "noun,proper-noun", + "--map-grammeme", "customNomAcc,nominative,accusative", + "--expand-grammemes", "singular,nominative,accusative,feminine,noun:indefinite", + "--ignore-property", "countable,dialectal", + "--ignore-entries-with-grammemes", "short-form,archaic", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", tempDmlex.toString() + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // ro-md entry excluded via --exclude-language + Assertions.assertFalse(entries.containsKey("moldovanWord")); + // short-form and unmapped "archaic" surface forms ignored via --ignore-entries-with-grammemes + Assertions.assertFalse(entries.containsKey("casă-short")); + Assertions.assertFalse(entries.containsKey("casă-archaic")); + // countable and unmapped "dialectal" ignored via --ignore-property; + // customNomAcc mapped to nominative,accusative; singular expanded with indefinite via --expand-grammemes + Assertions.assertEquals("singular accusative nominative indefinite feminine noun", entries.get("casă")); + Assertions.assertEquals("plural accusative nominative feminine noun", entries.get("case")); + } finally { + Grammar.TYPEMAP.clear(); + Grammar.TYPEMAP.putAll(savedTypeMap); + Files.deleteIfExists(tempDmlex); + } + } + + @Test + public void dmlexEntriesBeforeLangCodeMultiFieldObjectsAndReverseOrderTest() throws Exception { + // Tests: + // 1. "entries" appearing before "langCode" in the root DMLex JSON object. + // 2. Single-object multi-field labels ({"tag": "singular", "label": "Singular number"}) and + // pronunciations ({"label": "US", "transcription": "ˈæp.əl"}). + // 3. Reverse source order (--dmlex --wikidata ), where Wikidata overrides DMLex. + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-reverse-", ".json"); + try { + String json = """ + { + "entries": [ + { + "id": "dmlex-apple", + "headword": "apple", + "partOfSpeech": [ + { "label": "Common noun", "tag": "noun" } + ], + "labels": [ + { "tag": "nonhuman", "label": "Non-human animacy" } + ], + "pronunciations": [ + { "label": "US", "transcription": "ˈæp.əl" } + ], + "inflectedForms": [ + { + "value": "apple", + "labels": [ + { "tag": "singular", "label": "Singular number" } + ] + }, + { + "value": "apples", + "labels": [ + { "label": "Plural number", "tag": "plural" } + ] + } + ] + }, + { + "id": "dmlex-ciel-early", + "headword": "ciel", + "partOfSpeech": ["noun"], + "labels": ["feminine"], + "pronunciations": [ + { + "label": "en-US", + "transcriptions": [ + { "text": "sjɛl", "scheme": "IPA" } + ] + } + ], + "inflectedForms": [ + { "value": "ciel", "labels": ["singular"] }, + { "value": "cieux", "labels": ["plural"] }, + { "value": "ciel-dmlex-only", "labels": ["plural"] } + ] + }, + { + "id": "dmlex-posless-early", + "headword": "uh-oh", + "inflectedForms": [ + { "value": "uh-oh", "labels": ["singular"] } + ] + } + ], + "langCode": "en" + } + """; + Files.writeString(tempDmlex, json, StandardCharsets.UTF_8); + + Path tempDmlex2 = Files.createTempFile(Paths.get(baseDir), "dmlex-posless-late-", ".json"); + try { + String json2 = """ + { + "langCode": "en", + "entries": [ + { + "id": "dmlex-posless-late", + "headword": "uh-oh", + "inflectedForms": [ + { "value": "uh-oh", "labels": ["plural"] } + ] + } + ] + } + """; + Files.writeString(tempDmlex2, json2, StandardCharsets.UTF_8); + + String[] args = { + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", tempDmlex.toString(), + "--wikidata", lexiconSource, + "--dmlex", tempDmlex2.toString() + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // "apple" from DMLex is parsed with multi-field objects and "entries" before "langCode" + Assertions.assertTrue(entries.get("apple").startsWith("singular nonhuman vowel-start noun inflection=")); + Assertions.assertTrue(entries.get("apples").startsWith("plural nonhuman vowel-start noun inflection=")); + + // Because --wikidata came AFTER --dmlex, Wikidata's masculine "ciel" overrides DMLex's feminine "ciel" + Assertions.assertTrue(entries.get("ciel").contains("masculine")); + Assertions.assertFalse(entries.get("ciel").contains("feminine")); + Assertions.assertTrue(entries.containsKey("ciels")); + Assertions.assertTrue(entries.get("cieux").contains("masculine")); + Assertions.assertFalse(entries.get("cieux").contains("feminine")); + // "ciel-dmlex-only" was not in Wikidata, and its pronunciation container's "label": "en-US" + // must not be mistaken for a vowel-start transcription when "transcriptions" is present. + Assertions.assertTrue(entries.get("ciel-dmlex-only").startsWith("plural feminine noun inflection=")); + Assertions.assertFalse(entries.get("ciel-dmlex-only").contains("vowel-start")); + + // POS-less surface form "uh-oh" from tempDmlex was overridden by tempDmlex2 + Assertions.assertEquals("plural", entries.get("uh-oh")); + assertAllXmlPatternsReferencedInDictionary(actual); + } finally { + Files.deleteIfExists(tempDmlex2); + } + } finally { + Files.deleteIfExists(tempDmlex); + } + } + + @Test + public void deferredLexemeMultiVariantAndSupplementalMergeTest() throws Exception { + // filter_fr.properties defines L27261=L1566399 (organisateur = organisatrice). + // Tests: + // 1. Primary Wikidata file (sourceIndex == 0) with multi-variant L27261 ("fr" and "fr-x-Q56352306") + // and L1566399 merges both variants and both partner lexemes without throwing "Duplicate lexeme". + // 2. Supplemental Wikidata file (sourceIndex == 1) containing only L27261 (without L1566399) + // does not throw "L1566399: id not found" and overrides the "organisateur" surface form. + // 3. Duplicate top-level Lexeme objects with the same deferred ID in one file still throw IllegalStateException. + Path primaryWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-primary-", ".json"); + Path supplementalWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-supp-", ".json"); + Path duplicateWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-dup-", ".json"); + try { + String primaryJson = """ + [ + { + "type": "lexeme", + "id": "L27261", + "lemmas": { + "fr": { "language": "fr", "value": "organisateur" }, + "fr-x-Q56352306": { "language": "fr-x-Q56352306", "value": "organisateur" } + }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { + "fr": { "language": "fr", "value": "organisateur" } + }, + "grammaticalFeatures": ["Q110786", "Q499327"], + "claims": {} + }, + { + "id": "L27261-F2", + "representations": { + "fr": { "language": "fr", "value": "organisateurs-old" } + }, + "grammaticalFeatures": ["Q146786", "Q499327"], + "claims": {} + } + ], + "senses": [] + }, + { + "type": "lexeme", + "id": "L1566399", + "lemmas": { + "fr": { "language": "fr", "value": "organisatrice" } + }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L1566399-F1", + "representations": { + "fr": { "language": "fr", "value": "organisatrice" } + }, + "grammaticalFeatures": ["Q110786", "Q1775415"], + "claims": {} + } + ], + "senses": [] + } + ] + """; + String supplementalJson = """ + [ + { + "type": "lexeme", + "id": "L27261", + "lemmas": { + "fr": { "language": "fr", "value": "organisateur" } + }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { + "fr": { "language": "fr", "value": "organisateur" } + }, + "grammaticalFeatures": ["Q110786", "Q499327"], + "claims": {} + }, + { + "id": "L27261-F2", + "representations": { + "fr": { "language": "fr", "value": "organisateurs" } + }, + "grammaticalFeatures": ["Q146786", "Q499327"], + "claims": {} + } + ], + "senses": [] + } + ] + """; + String duplicateJson = """ + [ + { + "type": "lexeme", + "id": "L27261", + "lemmas": { "fr": { "language": "fr", "value": "organisateur" } }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { "fr": { "language": "fr", "value": "organisateur" } }, + "grammaticalFeatures": ["Q110786"], + "claims": {} + } + ], + "senses": [] + }, + { + "type": "lexeme", + "id": "L27261", + "lemmas": { "fr": { "language": "fr", "value": "organisateur" } }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { "fr": { "language": "fr", "value": "organisateur" } }, + "grammaticalFeatures": ["Q110786"], + "claims": {} + } + ], + "senses": [] + } + ] + """; + Files.writeString(primaryWikidata, primaryJson, StandardCharsets.UTF_8); + Files.writeString(supplementalWikidata, supplementalJson, StandardCharsets.UTF_8); + Files.writeString(duplicateWikidata, duplicateJson, StandardCharsets.UTF_8); + + String[] args = { + "--language", "fr", + "--inflection-types", "noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", primaryWikidata.toString(), supplementalWikidata.toString() + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + Assertions.assertTrue(entries.containsKey("organisateur")); + Assertions.assertTrue(entries.containsKey("organisateurs")); + Assertions.assertTrue(entries.containsKey("organisatrice")); + // "organisateur" was overridden by supplementalWikidata, so it shares the same pattern as "organisateurs" + Assertions.assertEquals( + entries.get("organisateurs").split("inflection=")[1], + entries.get("organisateur").split("inflection=")[1]); + assertAllXmlPatternsReferencedInDictionary(actual); + + String[] dupArgs = { + "--language", "fr", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", duplicateWikidata.toString() + }; + Assertions.assertThrows(IllegalStateException.class, () -> ParseWikidata.main(dupArgs)); + } finally { + Files.deleteIfExists(primaryWikidata); + Files.deleteIfExists(supplementalWikidata); + Files.deleteIfExists(duplicateWikidata); + } + } } diff --git a/tools/dictionary-parser/src/test/resources/dmlexInvalidSchema.json b/tools/dictionary-parser/src/test/resources/dmlexInvalidSchema.json new file mode 100644 index 000000000..ec1f4c8f9 --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexInvalidSchema.json @@ -0,0 +1,3 @@ +{ + "unexpectedField": "not a valid DMLex document" +} diff --git a/tools/dictionary-parser/src/test/resources/dmlexLexicon.json b/tools/dictionary-parser/src/test/resources/dmlexLexicon.json new file mode 100644 index 000000000..3bb27cb19 --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexLexicon.json @@ -0,0 +1,92 @@ +{ + "uri": "http://example.org/lexicon/en", + "langCode": "en", + "entries": [ + { + "id": "dmlex-1", + "headword": "cat", + "partOfSpeech": ["noun"], + "labels": ["masculine"], + "inflectedForms": [ + { + "id": "dmlex-1-f1", + "value": "cat", + "labels": ["singular"] + }, + { + "id": "dmlex-1-f2", + "value": "cats", + "labels": ["plural"] + } + ] + }, + { + "id": "dmlex-2", + "headword": "bat", + "partOfSpeech": ["noun"], + "labels": ["masculine"], + "inflectedForms": [ + { + "id": "dmlex-2-f1", + "value": "bat", + "labels": ["singular"] + }, + { + "id": "dmlex-2-f2", + "value": "bats", + "labels": ["plural"] + } + ] + }, + { + "id": "dmlex-3", + "headword": "apple", + "partOfSpeech": "noun", + "pronunciations": [ + { + "transcription": "ˈæp.əl", + "scheme": "IPA" + } + ], + "inflectedForms": [ + { + "id": "dmlex-3-f1", + "value": "apple", + "labels": [ + { "tag": "singular" } + ] + }, + { + "id": "dmlex-3-f2", + "value": "apples", + "labels": [ + { "tag": "plural" } + ] + } + ] + }, + { + "id": "dmlex-4", + "headword": "wow", + "partOfSpeech": ["interjection"] + }, + { + "id": "dmlex-5", + "headword": "London", + "partOfSpeech": ["proper-noun"], + "inflectedForms": [ + { + "id": "dmlex-5-f1", + "value": "London", + "labels": ["singular"] + } + ] + }, + { + "id": "dmlex-fr-ignored", + "headword": "bonjour", + "langCode": "fr", + "partOfSpeech": ["interjection"] + } + ] +} diff --git a/tools/dictionary-parser/src/test/resources/dmlexOverlay.json b/tools/dictionary-parser/src/test/resources/dmlexOverlay.json new file mode 100644 index 000000000..c77b9093c --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexOverlay.json @@ -0,0 +1,60 @@ +{ + "uri": "http://example.org/lexicon/en-overlay", + "langCode": "en", + "entries": [ + { + "id": "overlay-new", + "headword": "book", + "partOfSpeech": ["noun"], + "inflectedForms": [ + { + "value": "book", + "labels": ["singular"] + }, + { + "value": "book's", + "labels": ["singular", "genitive"] + }, + { + "value": "books", + "labels": ["plural"] + }, + { + "value": "books'", + "labels": ["plural", "genitive"] + } + ] + }, + { + "id": "overlay-diff-pos", + "headword": "hey", + "partOfSpeech": ["noun"], + "inflectedForms": [ + { + "value": "hey", + "labels": ["singular"] + }, + { + "value": "heys", + "labels": ["plural"] + } + ] + }, + { + "id": "overlay-same-pos-conflict", + "headword": "ciel", + "partOfSpeech": ["noun"], + "labels": ["feminine"], + "inflectedForms": [ + { + "value": "ciel", + "labels": ["singular"] + }, + { + "value": "cieux", + "labels": ["plural"] + } + ] + } + ] +} diff --git a/tools/dictionary-parser/src/test/resources/dmlexPatch.json b/tools/dictionary-parser/src/test/resources/dmlexPatch.json new file mode 100644 index 000000000..1769f0afd --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexPatch.json @@ -0,0 +1,53 @@ +{ + "uri": "http://example.org/lexicon/en-patch", + "langCode": "en", + "entries": [ + { + "id": "patch-1", + "headword": "cat", + "partOfSpeech": ["noun"], + "labels": ["feminine"], + "inflectedForms": [ + { + "value": "cat", + "labels": ["singular"] + }, + { + "value": "kitties", + "labels": ["plural"] + } + ] + }, + { + "id": "patch-2", + "headword": "wow", + "partOfSpeech": ["verb"], + "inflectedForms": [ + { + "value": "wow", + "labels": [] + }, + { + "value": "wowed", + "labels": [] + } + ] + }, + { + "id": "patch-3", + "headword": "batte", + "partOfSpeech": ["noun"], + "labels": ["feminine"], + "inflectedForms": [ + { + "value": "batte", + "labels": ["singular"] + }, + { + "value": "bats", + "labels": ["plural"] + } + ] + } + ] +}