From 98b9bca33edd81363eb9749051384d259237a0a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Neboj=C5=A1a=20=C4=86iri=C4=87?= Date: Fri, 9 Oct 2026 18:38:08 +0000 Subject: [PATCH 1/4] Inflection-208 Add multisource and DMLex dictionary parsing support --- tools/dictionary-parser/ParseWikidata | 2 +- tools/dictionary-parser/README.md | 26 +- .../java/org/unicode/wikidata/DMLexEntry.java | 50 ++ .../unicode/wikidata/DMLexInflectedForm.java | 37 + .../wikidata/DMLexStringListDeserializer.java | 161 +++++ .../org/unicode/wikidata/DictionaryEntry.java | 224 +++++- .../org/unicode/wikidata/DocumentState.java | 6 +- .../java/org/unicode/wikidata/Grammar.java | 109 ++- .../org/unicode/wikidata/ParseWikidata.java | 614 ++++++++++++++--- .../org/unicode/wikidata/ParserOptions.java | 194 +++++- .../unicode/wikidata/ParseWikidataTest.java | 639 +++++++++++++++++- .../test/resources/dmlexInvalidSchema.json | 3 + .../src/test/resources/dmlexLexicon.json | 92 +++ .../src/test/resources/dmlexOverlay.json | 60 ++ .../src/test/resources/dmlexPatch.json | 53 ++ 15 files changed, 2103 insertions(+), 167 deletions(-) create mode 100644 tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java create mode 100644 tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java create mode 100644 tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java create mode 100644 tools/dictionary-parser/src/test/resources/dmlexInvalidSchema.json create mode 100644 tools/dictionary-parser/src/test/resources/dmlexLexicon.json create mode 100644 tools/dictionary-parser/src/test/resources/dmlexOverlay.json create mode 100644 tools/dictionary-parser/src/test/resources/dmlexPatch.json diff --git a/tools/dictionary-parser/ParseWikidata b/tools/dictionary-parser/ParseWikidata index f19c1c44a..6dfb9e833 100755 --- a/tools/dictionary-parser/ParseWikidata +++ b/tools/dictionary-parser/ParseWikidata @@ -23,7 +23,7 @@ if [[ $# -eq 2 && $1 == "--all" ]]; then if [[ "$LAST_LINE" =~ $COMMAND_OPTS_PREFIX ]]; then OPTIONS=`echo $LAST_LINE | sed "s/$COMMAND_OPTS_PREFIX//"` echo Generating $LANGUAGE from $LEXEME_FILE - $RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst $LEXEME_FILE + $RUN_COMMAND $OPTIONS --inflections $LEXICAL_DIR/inflectional_${LANGUAGE}.xml --dictionary $LEXICAL_DIR/dictionary_${LANGUAGE}.lst --wikidata $LEXEME_FILE else echo skipping $LANGUAGE fi diff --git a/tools/dictionary-parser/README.md b/tools/dictionary-parser/README.md index 736bda902..be1369a66 100644 --- a/tools/dictionary-parser/README.md +++ b/tools/dictionary-parser/README.md @@ -20,5 +20,29 @@ These tools generate files that describes the grammatical properties of words fr ./ParseWikidata \ --inflections ../../resources/org/unicode/inflection/dictionary/inflectional_XX.xml \ --dictionary ../../resources/org/unicode/inflection/dictionary/dictionary_XX.lst \ - + --wikidata ``` + +## Usage for OASIS DMLex and Multi-Source Inputs + +`ParseWikidata` also supports [OASIS DMLex v1.0](https://docs.oasis-open.org/lexidma/dmlex/v1.0/dmlex-v1.0.html) JSON files via `--dmlex`, either standalone (bypassing Wikidata completely) or combined with `--wikidata`: + +- **Standalone DMLex (single or multiple files)**: +``` + ./ParseWikidata --language \ + --inflections inflectional_.xml \ + --dictionary dictionary_.lst \ + --dmlex [ ...] +``` +- **Multi-Source (Wikidata + DMLex overlay)**: +``` + ./ParseWikidata \ + --inflections inflectional_.xml \ + --dictionary dictionary_.lst \ + --wikidata \ + --dmlex [ ...] +``` +- **Conflict Resolution (`--merge-conflicts`)**: + - By default (`--merge-conflicts=false`), later source files take higher precedence per `(surfaceForm, PartOfSpeech)` and `(lemma, PartOfSpeech)` (overwriting conflicting entries of the same part of speech from earlier files while preserving distinct parts of speech). + - Pass `--merge-conflicts=true` (or `--merge-conflicts`) to merge conflicting entries across files instead of overwriting. + diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java new file mode 100644 index 000000000..4241d5318 --- /dev/null +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java @@ -0,0 +1,50 @@ +/* + * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved. + */ +package org.unicode.wikidata; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import java.util.ArrayList; +import java.util.List; + +/** + * A lexical entry in OASIS DMLex v1.0 JSON format. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DMLexEntry { + public String id; + public String headword; + public String langCode; + + public List partOfSpeech = new ArrayList<>(); + + public List labels = new ArrayList<>(); + + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public List pronunciations; + + @JsonAlias({"forms"}) + public List inflectedForms; + + @JsonSetter("partOfSpeech") + @JsonAlias({"pos"}) + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public void addPartOfSpeech(List incoming) { + if (incoming != null) { + this.partOfSpeech.addAll(incoming); + } + } + + @JsonSetter("labels") + @JsonAlias({"grammaticalFeatures", "features", "tag", "tags"}) + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public void addLabels(List incoming) { + if (incoming != null) { + this.labels.addAll(incoming); + } + } +} diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java new file mode 100644 index 000000000..f7bd2affb --- /dev/null +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexInflectedForm.java @@ -0,0 +1,37 @@ +/* + * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved. + */ +package org.unicode.wikidata; + +import com.fasterxml.jackson.annotation.JsonAlias; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + +import java.util.ArrayList; +import java.util.List; + +/** + * An inflected form of a lexical entry in OASIS DMLex v1.0 JSON format. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DMLexInflectedForm { + public String id; + + @JsonAlias({"text", "inflection"}) + public String value; + + public List labels = new ArrayList<>(); + + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public List pronunciations; + + @JsonSetter("labels") + @JsonAlias({"grammaticalFeatures", "features", "tag", "tags"}) + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public void addLabels(List incoming) { + if (incoming != null) { + this.labels.addAll(incoming); + } + } +} diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java new file mode 100644 index 000000000..26b32ee9c --- /dev/null +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java @@ -0,0 +1,161 @@ +/* + * Copyright 2025-2026 Unicode Incorporated and others. All rights reserved. + */ +package org.unicode.wikidata; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.exc.MismatchedInputException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Deserializes OASIS DMLex string or tag/label/pronunciation lists, accepting: + *
    + *
  • a single string (e.g. {@code "noun"})
    • + *
  • an array of strings (e.g. {@code ["singular", "nominative"]})
    • + *
  • an object or array of objects containing {@code "tag"}, {@code "value"}, + * {@code "transcription"}, {@code "text"}, or {@code "label"}
    • + *
+ */ +public class DMLexStringListDeserializer extends JsonDeserializer> { + @Override + public List deserialize(JsonParser jsonParser, DeserializationContext ctxt) throws IOException { + List result = new ArrayList<>(); + JsonToken currentToken = jsonParser.currentToken(); + if (currentToken == JsonToken.VALUE_STRING) { + String text = jsonParser.getText(); + if (text != null && !text.isBlank()) { + result.add(text); + } + return result; + } + if (currentToken == JsonToken.START_OBJECT) { + readValuesFromObject(jsonParser, result); + return result; + } + if (currentToken != JsonToken.START_ARRAY) { + throw MismatchedInputException.from( + jsonParser, + List.class, + "Expected string, object, or array for DMLex list field, got " + currentToken); + } + + while (jsonParser.nextToken() != JsonToken.END_ARRAY) { + JsonToken elemToken = jsonParser.currentToken(); + if (elemToken == JsonToken.VALUE_STRING) { + String text = jsonParser.getText(); + if (text != null && !text.isBlank()) { + result.add(text); + } + } else if (elemToken == JsonToken.START_OBJECT) { + readValuesFromObject(jsonParser, result); + } else { + throw MismatchedInputException.from( + jsonParser, + String.class, + "Expected string or object element in DMLex array, got " + elemToken); + } + } + return result; + } + + /** + * Extracts value(s) from a single JSON object. When an object carries multiple scalar string + * fields (for example {@code {"tag": "singular", "label": "Singular number"}} or + * {@code {"label": "US", "transcription": "ˈæp.əl"}}), only the highest-priority non-blank + * scalar field ({@code "transcription"} → {@code "tag"} → {@code "value"} → + * {@code "label"} → {@code "text"}) is selected, while also supporting nested + * {@code "transcriptions"} arrays/objects. + */ + private static void readValuesFromObject(JsonParser jsonParser, List result) throws IOException { + String transcription = null; + String tag = null; + String value = null; + String label = null; + String text = null; + List nestedTranscriptions = null; + + while (jsonParser.nextToken() != JsonToken.END_OBJECT) { + String fieldName = jsonParser.currentName(); + JsonToken valueToken = jsonParser.nextToken(); + if (valueToken == JsonToken.VALUE_STRING) { + String str = jsonParser.getText(); + if (str != null && !str.isBlank()) { + switch (fieldName) { + case "transcription" -> { + if (transcription == null) { + transcription = str; + } + } + case "tag" -> { + if (tag == null) { + tag = str; + } + } + case "value" -> { + if (value == null) { + value = str; + } + } + case "label" -> { + if (label == null) { + label = str; + } + } + case "text" -> { + if (text == null) { + text = str; + } + } + default -> {} + } + } + } else if ("transcriptions".equals(fieldName) || "transcription".equals(fieldName)) { + if (nestedTranscriptions == null) { + nestedTranscriptions = new ArrayList<>(); + } + if (valueToken == JsonToken.START_OBJECT) { + readValuesFromObject(jsonParser, nestedTranscriptions); + } else if (valueToken == JsonToken.START_ARRAY) { + while (jsonParser.nextToken() != JsonToken.END_ARRAY) { + JsonToken elemToken = jsonParser.currentToken(); + if (elemToken == JsonToken.VALUE_STRING) { + String str = jsonParser.getText(); + if (str != null && !str.isBlank()) { + nestedTranscriptions.add(str); + } + } else if (elemToken == JsonToken.START_OBJECT) { + readValuesFromObject(jsonParser, nestedTranscriptions); + } else if (elemToken == JsonToken.START_ARRAY) { + jsonParser.skipChildren(); + } + } + } + } else if (valueToken == JsonToken.START_OBJECT || valueToken == JsonToken.START_ARRAY) { + jsonParser.skipChildren(); + } + } + + if (transcription != null) { + result.add(transcription); + if (nestedTranscriptions != null) { + result.addAll(nestedTranscriptions); + } + } else if (nestedTranscriptions != null && !nestedTranscriptions.isEmpty()) { + result.addAll(nestedTranscriptions); + } else if (tag != null) { + result.add(tag); + } else if (value != null) { + result.add(value); + } else if (label != null) { + result.add(label); + } else if (text != null) { + result.add(text); + } + } +} diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java index 20144fa4b..5260ddb5c 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java @@ -4,70 +4,197 @@ */ package org.unicode.wikidata; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.EnumSet; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import javax.annotation.Nonnull; /** - * A dictionary entry has a lemma and a set of inflections. The lemma may have additional part of speech information - * that should be included in the inflections. Rare inflections are sorted last so that they're accessible, - * but not directly referencable by default, which is important when there may be more than one inflection with the same + * A dictionary entry represents a single surface form ({@code phrase}) in the generated dictionary, + * along with its combined grammatical properties and references to inflection patterns. + * + *

To support multi-source inputs with fine-grained conflict resolution, a {@code DictionaryEntry} + * partitions its grammemes and inflection patterns into per-{@link Grammar.PartOfSpeech} slices + * ({@link #posSlices}) rather than storing a single flat set. This allows a higher-priority source + * file (higher {@code sourceIndex}) to override a word's reading for one part of speech (e.g., {@code noun}) + * without clobbering readings for other parts of speech (e.g., {@code verb} or {@code adjective}) + * defined in earlier source files. + * + *

Rare inflections are sorted last when serialized so that they remain accessible, + * but are not directly referenceable by default when multiple inflections share the same * set of grammemes. */ public class DictionaryEntry implements Comparable { + /** + * Holds the grammatical features and inflection patterns contributed for a single + * {@link Grammar.PartOfSpeech} (or for a POS-less entry) from a specific {@link #sourceIndex}. + */ + private static final class PosSlice { + final int sourceIndex; + final TreeSet> grammemes; + final List inflectionalPatterns = new ArrayList<>(2); + final List rareInflectionalPatterns = new ArrayList<>(); + + PosSlice(int sourceIndex, Comparator> comparator) { + this.sourceIndex = sourceIndex; + this.grammemes = new TreeSet<>(comparator); + } + + PosSlice copy() { + PosSlice copy = new PosSlice(this.sourceIndex, this.grammemes.comparator()); + copy.grammemes.addAll(this.grammemes); + copy.inflectionalPatterns.addAll(this.inflectionalPatterns); + copy.rareInflectionalPatterns.addAll(this.rareInflectionalPatterns); + return copy; + } + } + final String phrase; - private final TreeSet lemmas; - private final TreeSet> grammemes; - private final List inflectionalPatterns; - private final List rareInflectionalPatterns = new ArrayList<>(); + private final Comparator> comparator; + // LinkedHashMap preserves PartOfSpeech insertion order so tie-breaking between + // equal-frequency InflectionPatterns remains deterministic. + private final Map posSlices = new LinkedHashMap<>(); + // Holds grammemes and patterns for entries that have no PartOfSpeech tag. + private PosSlice poslessSlice = null; + // Holds POS-less supplemental grammemes injected via --add-extra-grammemes, + // which are always unioned regardless of source precedence. + private final TreeSet> extraGrammemes; public static final Comparator INFLECTION_PATTERN_COMPARATOR = Comparator .comparing(InflectionPattern::getCount) .reversed(); public DictionaryEntry(String phrase, boolean isLemmaRare, TreeSet> grammemes, InflectionPattern inflectionalPattern) { - this(phrase, phrase, isLemmaRare, grammemes, inflectionalPattern); + this(phrase, phrase, isLemmaRare, grammemes, inflectionalPattern, 0); } public DictionaryEntry(String phrase, String lemmas, boolean isLemmaRare, TreeSet> grammemes, InflectionPattern inflectionalPattern) { + this(phrase, lemmas, isLemmaRare, grammemes, inflectionalPattern, 0); + } + + public DictionaryEntry(String phrase, String lemmas, boolean isLemmaRare, TreeSet> grammemes, InflectionPattern inflectionalPattern, int sourceIndex) { this.phrase = phrase; - this.lemmas = new TreeSet<>(Collections.singleton(lemmas)); - this.grammemes = new TreeSet<>(grammemes.comparator()); - this.grammemes.addAll(grammemes); - if (inflectionalPattern != null) { - this.inflectionalPatterns = new ArrayList<>(2); - this.inflectionalPatterns.add(inflectionalPattern); - if (isLemmaRare) { - rareInflectionalPatterns.add(inflectionalPattern); + this.comparator = Inflection.ENUM_COMPARATOR; + this.extraGrammemes = new TreeSet<>(this.comparator); + + // Separate PartOfSpeech values from all other grammatical categories (gender, number, case, etc.) + // so that each PosSlice only stores its own PartOfSpeech plus the shared non-POS grammemes. + // If a multi-POS entry (e.g. noun + proper-noun) later has only one POS overridden by a + // higher-priority source, the remaining POS slice will not retain a stale POS tag. + EnumSet posSet = EnumSet.noneOf(Grammar.PartOfSpeech.class); + TreeSet> nonPosGrammemes = new TreeSet<>(this.comparator); + for (Enum g : grammemes) { + if (g instanceof Grammar.PartOfSpeech pos) { + posSet.add(pos); + } else { + nonPosGrammemes.add(g); } } - else { - this.inflectionalPatterns = new ArrayList<>(); + + if (posSet.isEmpty()) { + this.poslessSlice = new PosSlice(sourceIndex, this.comparator); + this.poslessSlice.grammemes.addAll(nonPosGrammemes); + if (inflectionalPattern != null) { + this.poslessSlice.inflectionalPatterns.add(inflectionalPattern); + if (isLemmaRare) { + this.poslessSlice.rareInflectionalPatterns.add(inflectionalPattern); + } + } + } else { + for (Grammar.PartOfSpeech pos : posSet) { + PosSlice slice = new PosSlice(sourceIndex, this.comparator); + slice.grammemes.addAll(nonPosGrammemes); + slice.grammemes.add(pos); + if (inflectionalPattern != null) { + slice.inflectionalPatterns.add(inflectionalPattern); + if (isLemmaRare) { + slice.rareInflectionalPatterns.add(inflectionalPattern); + } + } + this.posSlices.put(pos, slice); + } } } + /** + * Unions the grammemes and deduplicates inflection patterns from {@code incoming} into {@code target}. + * Used when two entries for the same surface form and {@link Grammar.PartOfSpeech} originate from + * the same source file (or when {@code --merge-conflicts=true} is enabled). + */ + private static void mergeSlice(PosSlice target, PosSlice incoming) { + target.grammemes.addAll(incoming.grammemes); + for (InflectionPattern inflectionalPattern : incoming.inflectionalPatterns) { + if (!target.inflectionalPatterns.contains(inflectionalPattern)) { + target.inflectionalPatterns.add(inflectionalPattern); + } + } + for (InflectionPattern inflectionalPattern : incoming.rareInflectionalPatterns) { + if (!target.rareInflectionalPatterns.contains(inflectionalPattern)) { + target.rareInflectionalPatterns.add(inflectionalPattern); + } + } + } + + /** + * Merges another {@code DictionaryEntry} for the same surface form into this entry. + * + *

Conflict resolution is performed independently for each {@link Grammar.PartOfSpeech} slice: + *

    + *
  1. New POS or higher-priority source ({@code incomingSlice.sourceIndex > existingSlice.sourceIndex}): + * Replaces the existing slice for that POS with the incoming slice, leaving other POS slices intact.
    2. + *
  2. Same-priority source ({@code incomingSlice.sourceIndex == existingSlice.sourceIndex}, + * or when {@code --merge-conflicts=true} normalizes all indices to {@code 0}): + * Unions grammemes and inflection patterns via {@link #mergeSlice(PosSlice, PosSlice)}, preserving + * intra-file homograph and syncretism merging.
    3. + *
  3. Lower-priority source ({@code incomingSlice.sourceIndex < existingSlice.sourceIndex}): + * Ignored for that POS.
    4. + *
+ */ public void merge(DictionaryEntry entry) { - this.grammemes.addAll(entry.grammemes); - for (InflectionPattern inflectionalPattern : entry.inflectionalPatterns) { - if (!this.inflectionalPatterns.contains(inflectionalPattern)) { - this.inflectionalPatterns.add(inflectionalPattern); + for (Map.Entry incomingEntry : entry.posSlices.entrySet()) { + Grammar.PartOfSpeech pos = incomingEntry.getKey(); + PosSlice incomingSlice = incomingEntry.getValue(); + PosSlice existingSlice = this.posSlices.get(pos); + if (existingSlice == null || incomingSlice.sourceIndex > existingSlice.sourceIndex) { + this.posSlices.put(pos, incomingSlice.copy()); + } else if (incomingSlice.sourceIndex == existingSlice.sourceIndex) { + mergeSlice(existingSlice, incomingSlice); } } - for (InflectionPattern inflectionalPattern : entry.rareInflectionalPatterns) { - if (!this.rareInflectionalPatterns.contains(inflectionalPattern)) { - this.rareInflectionalPatterns.add(inflectionalPattern); + if (entry.poslessSlice != null) { + if (this.poslessSlice == null || entry.poslessSlice.sourceIndex > this.poslessSlice.sourceIndex) { + this.poslessSlice = entry.poslessSlice.copy(); + } else if (entry.poslessSlice.sourceIndex == this.poslessSlice.sourceIndex) { + mergeSlice(this.poslessSlice, entry.poslessSlice); } } - this.lemmas.addAll(entry.lemmas); + this.extraGrammemes.addAll(entry.extraGrammemes); + } + + public void addExtraGrammemes(Set> grammemes) { + this.extraGrammemes.addAll(grammemes); } + /** + * Recombines grammemes across all active {@link PosSlice}s, {@link #poslessSlice}, and + * {@link #extraGrammemes} into a single sorted set ordered by {@link Inflection#ENUM_COMPARATOR}. + */ public Set> getGrammemes() { - return grammemes; + TreeSet> combined = new TreeSet<>(this.comparator); + for (PosSlice slice : posSlices.values()) { + combined.addAll(slice.grammemes); + } + if (poslessSlice != null) { + combined.addAll(poslessSlice.grammemes); + } + combined.addAll(extraGrammemes); + return combined; } @Override @@ -75,13 +202,48 @@ public String toString() { return this.toString(true); } + /** + * Formats this entry as a single line for the output {@code dictionary_*.lst} file: + * {@code : [inflection= ...]}. + * + *

When {@code isInflectional} is {@code true}, inflection patterns from all active slices + * are deduplicated in slice insertion order, sorted by descending usage frequency + * ({@link #INFLECTION_PATTERN_COMPARATOR}), and emitted with non-rare patterns preceding rare patterns. + */ public String toString(boolean isInflectional) { StringBuilder sb = new StringBuilder(256); sb.append(phrase).append(':'); - for (Enum enumVal : grammemes) { + for (Enum enumVal : getGrammemes()) { sb.append(' ').append(enumVal); } if (isInflectional) { + List inflectionalPatterns = new ArrayList<>(); + List rareInflectionalPatterns = new ArrayList<>(); + for (PosSlice slice : posSlices.values()) { + for (InflectionPattern pattern : slice.inflectionalPatterns) { + if (!inflectionalPatterns.contains(pattern)) { + inflectionalPatterns.add(pattern); + } + } + for (InflectionPattern pattern : slice.rareInflectionalPatterns) { + if (!rareInflectionalPatterns.contains(pattern)) { + rareInflectionalPatterns.add(pattern); + } + } + } + if (poslessSlice != null) { + for (InflectionPattern pattern : poslessSlice.inflectionalPatterns) { + if (!inflectionalPatterns.contains(pattern)) { + inflectionalPatterns.add(pattern); + } + } + for (InflectionPattern pattern : poslessSlice.rareInflectionalPatterns) { + if (!rareInflectionalPatterns.contains(pattern)) { + rareInflectionalPatterns.add(pattern); + } + } + } + if (inflectionalPatterns.size() > 1) { inflectionalPatterns.sort(INFLECTION_PATTERN_COMPARATOR); if (rareInflectionalPatterns.size() > 1) { diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java index 3078fe619..b43586cac 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java @@ -16,6 +16,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.regex.Pattern; @@ -78,14 +79,15 @@ void printDocument(ParserOptions parserOptions, long startTime) throws FileNotFo new FileOutputStream(parserOptions.lexicalDictionaryFilename), StandardCharsets.UTF_8))) { for (Map.Entry entry : dictionary.entrySet()) { DictionaryEntry dictionaryEntry = entry.getValue(); - if (dictionaryEntry.getGrammemes().isEmpty()) { + Set> grammemes = dictionaryEntry.getGrammemes(); + if (grammemes.isEmpty()) { // We don't care about only known words. We need grammeme data unclassifiedTerms++; continue; } // Print the dictionary entry to the .lst file. lexicalDictionaryStream.println(dictionaryEntry.toString(isInflectional())); - for (Enum grammeme : dictionaryEntry.getGrammemes()) { + for (Enum grammeme : grammemes) { grammemeCounts.merge(grammeme, 1, Integer::sum); } } diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java index a8ee68787..4e55dc6ce 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java @@ -8,6 +8,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Set; @@ -1175,9 +1176,115 @@ public String toString() { TYPEMAP.put("Q1909485", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // quantifier, but it might be used frequently as an indefinite noun TYPEMAP.put("Q694268", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // collective TYPEMAP.put("Q1450795", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // singulative + + // Seed TYPEMAP with all canonical grammeme names from DEFAULTMAP (e.g. "noun", "singular", + // "proper-noun") so both Wikidata Q-IDs and DMLex string labels can be resolved through + // a single read-only lookup table. + DEFAULTMAP.forEach((key, value) -> TYPEMAP.putIfAbsent(key, Set.of(value))); + + // Standard OASIS DMLex and Universal Dependencies / morphological shorthand tag aliases. + // Part-of-speech abbreviations: + TYPEMAP.putIfAbsent("propn", EnumSet.of(PartOfSpeech.PROPER_NOUN)); + TYPEMAP.putIfAbsent("adj", EnumSet.of(PartOfSpeech.ADJECTIVE)); + TYPEMAP.putIfAbsent("adv", EnumSet.of(PartOfSpeech.ADVERB)); + TYPEMAP.putIfAbsent("adp", EnumSet.of(PartOfSpeech.ADPOSITION)); + TYPEMAP.putIfAbsent("prep", EnumSet.of(PartOfSpeech.PREPOSITION)); + TYPEMAP.putIfAbsent("conj", EnumSet.of(PartOfSpeech.CONJUNCTION)); + TYPEMAP.putIfAbsent("cconj", EnumSet.of(PartOfSpeech.CONJUNCTION)); + TYPEMAP.putIfAbsent("sconj", EnumSet.of(PartOfSpeech.CONJUNCTION)); + TYPEMAP.putIfAbsent("det", EnumSet.of(PartOfSpeech.DETERMINER)); + TYPEMAP.putIfAbsent("intj", EnumSet.of(PartOfSpeech.INTERJECTION)); + TYPEMAP.putIfAbsent("num", EnumSet.of(PartOfSpeech.NUMERAL)); + TYPEMAP.putIfAbsent("part", EnumSet.of(PartOfSpeech.PARTICLE)); + TYPEMAP.putIfAbsent("pron", EnumSet.of(PartOfSpeech.PRONOUN)); + + // Person aliases: + TYPEMAP.putIfAbsent("first-person", EnumSet.of(Person.FIRST)); + TYPEMAP.putIfAbsent("1st-person", EnumSet.of(Person.FIRST)); + TYPEMAP.putIfAbsent("second-person", EnumSet.of(Person.SECOND)); + TYPEMAP.putIfAbsent("2nd-person", EnumSet.of(Person.SECOND)); + TYPEMAP.putIfAbsent("third-person", EnumSet.of(Person.THIRD)); + TYPEMAP.putIfAbsent("3rd-person", EnumSet.of(Person.THIRD)); + + // Number abbreviations: + TYPEMAP.putIfAbsent("sg", EnumSet.of(Number.SINGULAR)); + TYPEMAP.putIfAbsent("sing", EnumSet.of(Number.SINGULAR)); + TYPEMAP.putIfAbsent("pl", EnumSet.of(Number.PLURAL)); + TYPEMAP.putIfAbsent("plur", EnumSet.of(Number.PLURAL)); + + // Gender abbreviations: + TYPEMAP.putIfAbsent("masc", EnumSet.of(Gender.MASCULINE)); + TYPEMAP.putIfAbsent("fem", EnumSet.of(Gender.FEMININE)); + TYPEMAP.putIfAbsent("neut", EnumSet.of(Gender.NEUTER)); + TYPEMAP.putIfAbsent("common-gender", EnumSet.of(Gender.COMMON)); + + // Case abbreviations: + TYPEMAP.putIfAbsent("nom", EnumSet.of(Case.NOMINATIVE)); + TYPEMAP.putIfAbsent("acc", EnumSet.of(Case.ACCUSATIVE)); + TYPEMAP.putIfAbsent("gen", EnumSet.of(Case.GENITIVE)); + TYPEMAP.putIfAbsent("dat", EnumSet.of(Case.DATIVE)); + TYPEMAP.putIfAbsent("ins", EnumSet.of(Case.INSTRUMENTAL)); + TYPEMAP.putIfAbsent("inst", EnumSet.of(Case.INSTRUMENTAL)); + TYPEMAP.putIfAbsent("loc", EnumSet.of(Case.LOCATIVE)); + TYPEMAP.putIfAbsent("voc", EnumSet.of(Case.VOCATIVE)); + TYPEMAP.putIfAbsent("abl", EnumSet.of(Case.ABLATIVE)); + + // Definiteness abbreviations: + TYPEMAP.putIfAbsent("def", EnumSet.of(Definiteness.DEFINITE)); + TYPEMAP.putIfAbsent("indef", EnumSet.of(Definiteness.INDEFINITE)); + TYPEMAP.putIfAbsent("construct-state", EnumSet.of(Definiteness.CONSTRUCT)); + + // Hyphenated aliases for compound enums declared without underscores: + TYPEMAP.putIfAbsent("non-finite", EnumSet.of(VerbType.NONFINITE)); + TYPEMAP.putIfAbsent("non-human", EnumSet.of(Animacy.NONHUMAN)); + } + + /** + * Normalizes a grammeme label into canonical lowercase hyphenated form (e.g., converting + * camelCase {@code "properNoun"}, snake_case {@code "proper_noun"}, or uppercase + * {@code "PROPER_NOUN"} to {@code "proper-noun"}). + */ + static String normalizeGrammemeKey(String grammeme) { + if (grammeme == null) { + return ""; + } + String trimmed = grammeme.trim(); + StringBuilder sb = new StringBuilder(trimmed.length() + 4); + for (int i = 0; i < trimmed.length(); i++) { + char c = trimmed.charAt(i); + if (c == '_' || Character.isWhitespace(c)) { + if (sb.length() > 0 && sb.charAt(sb.length() - 1) != '-') { + sb.append('-'); + } + } else if (Character.isUpperCase(c)) { + if (i > 0) { + char prev = trimmed.charAt(i - 1); + if (Character.isLowerCase(prev) || Character.isDigit(prev)) { + sb.append('-'); + } + } + sb.append(Character.toLowerCase(c)); + } else { + sb.append(Character.toLowerCase(c)); + } + } + return sb.toString().toLowerCase(Locale.ROOT); } + /** + * Looks up the default mapped grammeme enums for a Wikidata Q-ID or DMLex grammatical label + * in the read-only {@link #TYPEMAP} table. Checks for an exact match first (fast path for + * Q-IDs like {@code "Q1084"} and canonical strings like {@code "proper-noun"}), then falls + * back to a normalized key lookup via {@link #normalizeGrammemeKey(String)}. + */ static Set> getMappedGrammemes(String grammeme) { - return TYPEMAP.get(grammeme); + if (grammeme == null) { + return null; + } + Set> mapped = TYPEMAP.get(grammeme); + if (mapped != null) { + return mapped; + } + return TYPEMAP.get(normalizeGrammemeKey(grammeme)); } } diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java index 33c038bab..d7e82375f 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java @@ -18,22 +18,24 @@ import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.exc.MismatchedInputException; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream; import org.apache.commons.lang3.StringUtils; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.EnumMap; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; -import java.util.HashMap; import java.util.TreeSet; -import java.util.Arrays; import static org.unicode.wikidata.Grammar.Gender; import static org.unicode.wikidata.Grammar.Ignorable; import static org.unicode.wikidata.Grammar.PartOfSpeech; @@ -64,17 +66,12 @@ static class Lemma { final TreeSet> grammemes = new TreeSet<>(EnumComparator.ENUM_COMPARATOR); final List inflections = new ArrayList<>(64); - void reset() { - value = null; - isRare = false; - grammemes.clear(); - inflections.clear(); - } - private Lemma() { } } + private record StagedLemma(Lemma lemma, int sourceIndex) {} + private final ParserOptions parserOptions; private final DocumentState documentState = new DocumentState(); private final TreeSet rareLemmas = new TreeSet<>(); @@ -82,6 +79,14 @@ private Lemma() { private final Map> mergeMap = new HashMap<>(); private final TreeSet deferredLexemes = new TreeSet<>(); private final Map deferredLemmaMap = new HashMap<>(); + // Holds normalized lemmas in encounter order when multi-source overwrite mode is active + // (--merge-conflicts=false with >1 source file). + private final List stagedLemmas = new ArrayList<>(); + // Tracks the highest sourceIndex that defines a given (lemma.value, PartOfSpeech) pair + // so earlier paradigms for the same lemma and POS can be preempted before pattern generation. + private final Map> highestSourceForLemmaPos = new HashMap<>(); + // Tracks the highest sourceIndex that defines a POS-less lemma for a given headword. + private final Map highestSourceForPoslessLemma = new HashMap<>(); ParseWikidata(ParserOptions parserOptions) { this.parserOptions = parserOptions; @@ -126,16 +131,164 @@ private Lemma() { static final String VARIANT_SEPARATOR = "-x-"; - private void analyzeLexeme(int lineNumber, Lexeme lexeme) { + /** + * Returns {@code true} when multiple input files are provided and {@code --merge-conflicts} is {@code false} + * (the default). In this mode, later files on the CLI override conflicting entries of the same + * {@link PartOfSpeech} from earlier files via a two-tier strategy: + *

    + *
  • Tier 1 (Lemma/Paradigm level): {@link #recordOrAnalyzeLemma} stages lemmas and + * {@link #processStagedLemmas} strips any {@code (lemma.value, PartOfSpeech)} superseded by a later + * source before {@link #analyzeInflections} runs. This prevents removed inflected surface forms + * (e.g., an obsolete plural dropped in a patch file) from lingering as orphans in {@code dictionary.lst} + * and prevents superseded paradigms from inflating {@link InflectionPattern} counts in {@code inflectional.xml}.
    • + *
  • Tier 2 (Surface-form level): {@link DictionaryEntry#merge} resolves remaining surface-form + * collisions per {@link PartOfSpeech} using {@code sourceIndex}.
    • + *
+ */ + private boolean isMultiSourceOverwriteMode() { + return !parserOptions.mergeConflicts && parserOptions.sourceInputs.size() > 1; + } + + /** + * Either analyzes a normalized lemma immediately (single-source or {@code --merge-conflicts=true}, using + * {@code effectiveSourceIndex = 0} so all sources merge identically) or stages it for Tier 1 + * cross-source lemma preemption when {@link #isMultiSourceOverwriteMode()} is active. + */ + private void recordOrAnalyzeLemma(Lemma lemma, int sourceIndex) { + if (!isMultiSourceOverwriteMode()) { + analyzeLemma(lemma, 0); + return; + } + stagedLemmas.add(new StagedLemma(lemma, sourceIndex)); + boolean hasPos = false; + for (Enum g : lemma.grammemes) { + if (g instanceof PartOfSpeech pos) { + hasPos = true; + highestSourceForLemmaPos + .computeIfAbsent(lemma.value, k -> new EnumMap<>(PartOfSpeech.class)) + .merge(pos, sourceIndex, Math::max); + } + } + if (!hasPos) { + highestSourceForPoslessLemma.merge(lemma.value, sourceIndex, Math::max); + } + } + + /** + * Executes Tier 1 lemma preemption across all staged lemmas after all source files have been read. + * For each staged lemma, any {@link PartOfSpeech} (or POS-less reading) that appears in a later + * source file for the same headword ({@code highestSource > sourceIndex}) is stripped from the + * lemma and its inflections; if all of the lemma's parts of speech were superseded, the earlier + * lemma is skipped entirely. + */ + private void processStagedLemmas() { + for (StagedLemma staged : stagedLemmas) { + Lemma lemma = staged.lemma(); + int sourceIndex = staged.sourceIndex(); + EnumSet originalPos = EnumSet.noneOf(PartOfSpeech.class); + for (Enum g : lemma.grammemes) { + if (g instanceof PartOfSpeech pos) { + originalPos.add(pos); + } + } + if (originalPos.isEmpty()) { + Integer highestPoslessSource = highestSourceForPoslessLemma.get(lemma.value); + if (highestPoslessSource != null && highestPoslessSource > sourceIndex) { + // This POS-less lemma was superseded by a later POS-less entry for the same headword. + continue; + } + } else { + EnumMap posMap = highestSourceForLemmaPos.get(lemma.value); + if (posMap != null) { + EnumSet supersededPos = EnumSet.noneOf(PartOfSpeech.class); + for (PartOfSpeech pos : originalPos) { + Integer highestSource = posMap.get(pos); + if (highestSource != null && highestSource > sourceIndex) { + supersededPos.add(pos); + } + } + if (!supersededPos.isEmpty()) { + if (supersededPos.size() == originalPos.size()) { + // Every PartOfSpeech of this lemma was superseded by a later source. + continue; + } + lemma.grammemes.removeAll(supersededPos); + for (Inflection inflection : lemma.inflections) { + inflection.getGrammemeSet().removeAll(supersededPos); + } + } + } + } + analyzeLemma(lemma, sourceIndex); + } + } + + /** + * Finalizes a single surface-form inflection with lemma grammeme propagation, ignorable filtering, + * and rarity extraction. Shared by both Wikidata and DMLex paths. {@code --expand-grammemes} + * expansion is deferred to {@link #expandAndCountInflections(Lemma)} at the start of {@link #analyzeLemma}. + * + * @return {@code false} if the entire lemma must be aborted due to {@code IGNORABLE_LEMMA}; {@code true} otherwise. + */ + private boolean finalizeAndAddInflection( + Lemma lemma, + @Nullable TreeSet> genderlessLemmaGrammemes, + Inflection currentInflection) { + // 1. Propagate lemma-level grammemes to the surface form. If the surface form specifies its + // own Gender, inherit genderlessLemmaGrammemes so form-level Gender overrides lemma Gender. + if (genderlessLemmaGrammemes == null || countGrammemeType(currentInflection.grammemeSet, Gender.class) == 0) { + currentInflection.grammemeSet.addAll(lemma.grammemes); + } else { + currentInflection.grammemeSet.addAll(genderlessLemmaGrammemes); + } + // 2. Check for grammemes configured to discard the entire lemma or this specific surface form. + if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_LEMMA)) { + documentState.unusableLemmaCount++; + return false; + } + if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_INFLECTION)) { + documentState.unusableSurfaceFormCount++; + return true; + } + // 3. Extract Usage.RARE into the boolean flag on Inflection and strip IGNORABLE_PROPERTY. + currentInflection.rareUsage = currentInflection.grammemeSet.contains(Grammar.Usage.RARE); + if (currentInflection.rareUsage) { + currentInflection.grammemeSet.remove(Grammar.Usage.RARE); + } + currentInflection.grammemeSet.remove(Ignorable.IGNORABLE_PROPERTY); + lemma.inflections.add(currentInflection); + return true; + } + + /** + * Performs post-form lemma cleanup shared by Wikidata and DMLex: extracts lemma-level + * {@link Grammar.Usage#RARE}, rejects lemmas with no usable surface forms, and strips {@link Gender} + * from {@code lemma.grammemes} (since forms have already inherited gender and {@code } + * elements in {@code inflectional.xml} do not include gender). + */ + private boolean finalizeLemmaAfterForms(Lemma lemma) { + lemma.isRare = lemma.grammemes.contains(Grammar.Usage.RARE); + if (lemma.isRare) { + lemma.grammemes.remove(Grammar.Usage.RARE); + } + if (lemma.inflections.isEmpty()) { + documentState.unusableLemmaCount++; + return false; + } + removeGrammemeType(lemma.grammemes, Gender.class); + return true; + } + + private void analyzeLexeme(int lineNumber, Lexeme lexeme, int sourceIndex) { if (omitLemmas.contains(lexeme.id)) { // We really don't want this junk. return; } - Lemma lemma = new Lemma(); Set> partOfSpeechSet = null; + boolean addedToDeferredInThisLexeme = false; for (var lemmaEntry : lexeme.lemmas.entrySet()) { var currentLemmaLanguage = lemmaEntry.getKey(); - lemma.reset(); + Lemma lemma = new Lemma(); documentState.lemmaCount++; LexemeRepresentation lemmaRepresentation = lemmaEntry.getValue(); lemma.value = lemmaRepresentation.value; @@ -144,7 +297,7 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme) { // The languages can have wierd Q entry after the desired language. // A spelling variant is informative. Most of the rest are irrelevant. var additionalCategory = currentLemmaLanguage.substring(qVariantIdx + VARIANT_SEPARATOR.length()); - var variant = Grammar.getMappedGrammemes(additionalCategory); + var variant = parserOptions.getMappedGrammemes(additionalCategory); if (variant == null) { if (parserOptions.debug) { System.err.println("Line " + lineNumber + ": " + additionalCategory @@ -160,7 +313,7 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme) { lemma.grammemes.addAll(variant); } if (partOfSpeechSet == null) { - partOfSpeechSet = Grammar.getMappedGrammemes(lexeme.lexicalCategory); + partOfSpeechSet = parserOptions.getMappedGrammemes(lexeme.lexicalCategory); if (partOfSpeechSet == null) { throw new IllegalArgumentException(lexeme.lexicalCategory + " is not a known part of speech grammeme for " + lexeme.id + "(" + lemma.value + ")"); @@ -204,77 +357,196 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme) { } } convertGrammemes(form, currentInflection, lexeme.id, lemma.value); - if (genderlessLemmaGrammemes == null || countGrammemeType(currentInflection.grammemeSet, Gender.class) == 0) { - currentInflection.grammemeSet.addAll(lemma.grammemes); - } - else { - currentInflection.grammemeSet.addAll(genderlessLemmaGrammemes); - } - if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_LEMMA)) { - documentState.unusableLemmaCount++; + if (!finalizeAndAddInflection(lemma, genderlessLemmaGrammemes, currentInflection)) { return; } - if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_INFLECTION)) { - documentState.unusableSurfaceFormCount++; - continue; - } - currentInflection.rareUsage = currentInflection.grammemeSet.contains(Grammar.Usage.RARE); - if (currentInflection.rareUsage) { - currentInflection.grammemeSet.remove(Grammar.Usage.RARE); - } - currentInflection.grammemeSet.remove(Ignorable.IGNORABLE_PROPERTY); - var grammemeExpansion = parserOptions.expandGramemes != null - ? parserOptions.expandGramemes.get(currentInflection.grammemeSet) - : null; - if (grammemeExpansion == null) { - lemma.inflections.add(currentInflection); + } + if (!finalizeLemmaAfterForms(lemma)) { + continue; + } + if (deferredLexemes.contains(lexeme.id)) { + if (addedToDeferredInThisLexeme) { + Lemma existing = deferredLemmaMap.get(lexeme.id); + existing.inflections.addAll(lemma.inflections); + existing.isRare |= lemma.isRare; + } else if (deferredLemmaMap.put(lexeme.id, lemma) != null) { + throw new IllegalStateException("Duplicate lexeme " + lexeme.id); } else { - for (var grammemeSet : grammemeExpansion) { - var expandedInflection = new Inflection(currentInflection.inflection, - currentInflection.rareUsage); - expandedInflection.grammemeSet.addAll(currentInflection.grammemeSet); - expandedInflection.grammemeSet.addAll(grammemeSet); - lemma.inflections.add(expandedInflection); - } + addedToDeferredInThisLexeme = true; } + } else { + recordOrAnalyzeLemma(lemma, sourceIndex); } - documentState.incomingSurfaceForm += lemma.inflections.size(); - lemma.isRare = lemma.grammemes.contains(Grammar.Usage.RARE); - if (lemma.isRare) { - lemma.grammemes.remove(Grammar.Usage.RARE); + } + } + + /** + * Converts a single OASIS DMLex {@link DMLexEntry} into the internal {@link Lemma} representation + * and routes it through the shared normalization and conflict-resolution pipeline. + */ + private void analyzeDMLexEntry(int lineNumber, DMLexEntry entry, String lexiconLangCode, int sourceIndex) { + // 1. Filter by language and --exclude-language options (allowing entry-level langCode override). + String entryLang = (entry.langCode != null && !entry.langCode.isBlank()) ? entry.langCode : lexiconLangCode; + if (!LexemesJsonDeserializer.isContained(entryLang)) { + return; + } + String entryId = (entry.id != null && !entry.id.isBlank()) ? entry.id : ""; + if (omitLemmas.contains(entryId)) { + return; + } + documentState.lemmaCount++; + if (entry.headword == null || entry.headword.isBlank()) { + throw new IllegalArgumentException("Missing headword for DMLex entry " + entryId); + } + + Lemma lemma = new Lemma(); + lemma.value = entry.headword; + + // 2. Extract any "-x-" subtag from the language code (matching Wikidata behavior). + int qVariantIdx = entryLang.indexOf(VARIANT_SEPARATOR); + if (qVariantIdx >= 0) { + var additionalCategory = entryLang.substring(qVariantIdx + VARIANT_SEPARATOR.length()); + var variant = parserOptions.getMappedGrammemes(additionalCategory); + if (variant == null) { + if (parserOptions.debug) { + System.err.println("Line " + lineNumber + ": " + additionalCategory + + " is not a known grammeme for the language variant " + entryId + "(" + lemma.value + ")"); + } + return; } - if (lemma.inflections.isEmpty()) { - documentState.unusableLemmaCount++; + if (variant.contains(Ignorable.IGNORABLE_INFLECTION)) { return; } - removeGrammemeType(lemma.grammemes, Gender.class); - if (deferredLexemes.contains(lexeme.id)) { - if (deferredLemmaMap.put(lexeme.id, lemma) != null) { - throw new IllegalStateException("Duplicate lexeme " + lexeme.id); + lemma.grammemes.addAll(variant); + } + + // 3. Map entry-level partOfSpeech, rarity filter, grammatical labels, and IPA pronunciations. + if (entry.partOfSpeech != null) { + for (String posStr : entry.partOfSpeech) { + var mappedPos = parserOptions.getMappedGrammemes(posStr); + if (mappedPos == null) { + throw new IllegalArgumentException(posStr + + " is not a known part of speech grammeme for " + entryId + "(" + lemma.value + ")"); } + lemma.grammemes.addAll(mappedPos); } - else { - analyzeLemma(lemma); + } + if (rareLemmas.contains(entryId)) { + lemma.grammemes.add(Grammar.Usage.RARE); + } + if (entry.labels != null) { + for (String labelStr : entry.labels) { + var mappedLabel = parserOptions.getMappedGrammemes(labelStr); + if (mappedLabel == null) { + throw new IllegalArgumentException(labelStr + + " is not a known grammeme for " + entryId + "(" + lemma.value + ")"); + } + lemma.grammemes.addAll(mappedLabel); } } + if (parserOptions.addSound && entry.pronunciations != null && !entry.pronunciations.isEmpty()) { + addSound("P898", entry.pronunciations, lemma.grammemes, entryId, lemma.value); + } + + if (lemma.grammemes.contains(Ignorable.IGNORABLE_LEMMA) + || lemma.grammemes.contains(Ignorable.IGNORABLE_INFLECTION)) { + documentState.unusableLemmaCount++; + return; + } + lemma.grammemes.remove(Ignorable.IGNORABLE_PROPERTY); + + TreeSet> genderlessLemmaGrammemes = null; + if (countGrammemeType(lemma.grammemes, Gender.class) > 0) { + genderlessLemmaGrammemes = new TreeSet<>(lemma.grammemes); + removeGrammemeType(genderlessLemmaGrammemes, Gender.class); + } + + // 4. If a DMLex entry has no explicit inflectedForms (e.g. an uninflected particle or noun + // entry listing only its headword), synthesize a single default form from the headword. + List forms = entry.inflectedForms; + if (forms == null || forms.isEmpty()) { + DMLexInflectedForm defaultForm = new DMLexInflectedForm(); + defaultForm.value = entry.headword; + forms = List.of(defaultForm); + } + + // 5. Convert each DMLexInflectedForm and run shared form normalization. + for (DMLexInflectedForm form : forms) { + if (form == null || form.value == null || form.value.isBlank()) { + throw new IllegalArgumentException("Missing inflected form value for " + entryId + "(" + lemma.value + ")"); + } + Inflection currentInflection = new Inflection(form.value); + if (form.labels != null) { + for (String labelStr : form.labels) { + Set> values = parserOptions.getMappedGrammemes(labelStr); + if (values == null) { + throw new IllegalArgumentException(labelStr + + " is not a known grammeme for " + entryId + "(" + lemma.value + ")"); + } + currentInflection.grammemeSet.addAll(values); + } + } + if (parserOptions.addSound && form.pronunciations != null && !form.pronunciations.isEmpty()) { + addSound("P898", form.pronunciations, currentInflection.grammemeSet, entryId, lemma.value); + } + if (!finalizeAndAddInflection(lemma, genderlessLemmaGrammemes, currentInflection)) { + return; + } + } + + if (!finalizeLemmaAfterForms(lemma)) { + return; + } + recordOrAnalyzeLemma(lemma, sourceIndex); } - // Method to process and merge lexemes - private void processAndMergeLexemes() { - for (Map.Entry> entry : mergeMap.entrySet()) { - Lemma lemma = deferredLemmaMap.computeIfAbsent(entry.getKey(), key -> { - throw new IllegalArgumentException(key + ": id not found"); - }); - for (var value : entry.getValue()) { - Lemma lemmaToMerge = deferredLemmaMap.computeIfAbsent(value, key -> { + /** + * Merges deferred Wikidata lexemes configured in {@code filter_.properties} at the end + * of a Wikidata file stream. For the primary Wikidata dump ({@code sourceIndex == 0}), missing + * partner IDs throw {@link IllegalArgumentException}. For supplemental Wikidata files + * ({@code sourceIndex > 0}), whichever partner IDs are present in that file are merged and any + * remaining deferred lemmas are still analyzed. Clears {@link #deferredLemmaMap} afterward so + * deferred state does not bleed into subsequent files. + */ + private void processAndMergeLexemes(int sourceIndex) { + if (sourceIndex == 0) { + for (Map.Entry> entry : mergeMap.entrySet()) { + Lemma lemma = deferredLemmaMap.computeIfAbsent(entry.getKey(), key -> { throw new IllegalArgumentException(key + ": id not found"); }); - lemma.inflections.addAll(lemmaToMerge.inflections); - lemma.isRare |= lemmaToMerge.isRare; - // The lemma grammemes should already have the relevant POS data. + for (var value : entry.getValue()) { + Lemma lemmaToMerge = deferredLemmaMap.computeIfAbsent(value, key -> { + throw new IllegalArgumentException(key + ": id not found"); + }); + lemma.inflections.addAll(lemmaToMerge.inflections); + lemma.isRare |= lemmaToMerge.isRare; + // The lemma grammemes should already have the relevant POS data. + } + recordOrAnalyzeLemma(lemma, sourceIndex); + } + } else { + for (Map.Entry> entry : mergeMap.entrySet()) { + Lemma lemma = deferredLemmaMap.remove(entry.getKey()); + for (var value : entry.getValue()) { + Lemma lemmaToMerge = deferredLemmaMap.remove(value); + if (lemmaToMerge != null) { + if (lemma == null) { + lemma = lemmaToMerge; + } else { + lemma.inflections.addAll(lemmaToMerge.inflections); + lemma.isRare |= lemmaToMerge.isRare; + } + } + } + if (lemma != null) { + recordOrAnalyzeLemma(lemma, sourceIndex); + } + } + for (Lemma remaining : deferredLemmaMap.values()) { + recordOrAnalyzeLemma(remaining, sourceIndex); } - analyzeLemma(lemma); } + deferredLemmaMap.clear(); } private void removeGrammemeType(TreeSet> grammemes, Class grammemeType) { @@ -314,7 +586,7 @@ private void removeConflicts(TreeSet> grammemes, Class grammemeType) private void convertGrammemes(LexemeForm form, Inflection currentInflection, String id, String lemma) { for (var feature : form.grammaticalFeatures) { - Set> values = Grammar.getMappedGrammemes(feature); + Set> values = parserOptions.getMappedGrammemes(feature); if (values == null) { throw new IllegalArgumentException(feature + " is not a known grammeme for " + id + "(" + lemma + ")"); } @@ -342,7 +614,7 @@ private boolean extractImportantProperties(Map> claims, Tre var claim = claimEntry.getValue(); conflicts = conflicts || claim.size() > 1; for (var grammemeStr : claim) { - var grammemeEnum = Grammar.getMappedGrammemes(grammemeStr); + var grammemeEnum = parserOptions.getMappedGrammemes(grammemeStr); if (grammemeEnum != null) { grammemes.addAll(grammemeEnum); } else if (parserOptions.debug) { @@ -480,7 +752,7 @@ private InflectionPattern getInflectionPattern(Lemma lemma, String lemmaSuffix, return inflectionPattern; } - private void analyzeInflections(Lemma lemma, List inputInflections) { + private void analyzeInflections(Lemma lemma, List inputInflections, int sourceIndex) { List inflections = new ArrayList<>(); for (Inflection inflection : inputInflections) { inflections.addAll(enumerateInflectionsForGrammemeCombinations(inflection)); @@ -527,8 +799,8 @@ private void analyzeInflections(Lemma lemma, List inputInflections) String phrase = inflection.getInflection(); InflectionPattern inflectionPatternForDict = nonEmptyInflectionIndices.contains(i) ? inflectionPattern : null; - documentState.addDictionaryEntry(new DictionaryEntry(phrase, phrase, lemma.isRare, - inflection.getGrammemeSet(), inflectionPatternForDict)); + documentState.addDictionaryEntry(new DictionaryEntry(phrase, lemma.value, lemma.isRare, + inflection.getGrammemeSet(), inflectionPatternForDict, sourceIndex)); } } @@ -593,17 +865,55 @@ private void mergeAdditionalGrammemes() { DictionaryEntry existingEntry = documentState.dictionary.putIfAbsent(entry.getKey(), newDictionaryEntry); if (existingEntry != null) { // Add instead of replace. - existingEntry.getGrammemes().addAll(grammemes); + existingEntry.addExtraGrammemes(grammemes); } } } - private void analyzeLemma(Lemma lemma) { - analyzeInflections(lemma, lemma.inflections); + /** + * Applies any {@code --expand-grammemes} rules to {@code lemma.inflections} and records the + * resulting surface-form count in {@link DocumentState#incomingSurfaceForm}. + * + *

Running this at the start of {@link #analyzeLemma} (after Tier 1 preemption in + * {@link #processStagedLemmas}) ensures that: + *

    + *
  1. lemmas completely superseded by a later source file do not inflate + * {@code incomingSurfaceForm}, and
    2. + *
  2. {@code --expand-grammemes} keys are matched against each inflection's surviving + * {@link PartOfSpeech} set after any superseded POS has been stripped.
    3. + *
+ */ + private void expandAndCountInflections(Lemma lemma) { + if (parserOptions.expandGramemes != null) { + List expanded = new ArrayList<>(lemma.inflections.size()); + for (Inflection currentInflection : lemma.inflections) { + var grammemeExpansion = parserOptions.expandGramemes.get(currentInflection.grammemeSet); + if (grammemeExpansion == null) { + expanded.add(currentInflection); + } else { + for (var grammemeSet : grammemeExpansion) { + var expandedInflection = new Inflection( + currentInflection.inflection, + currentInflection.rareUsage); + expandedInflection.grammemeSet.addAll(currentInflection.grammemeSet); + expandedInflection.grammemeSet.addAll(grammemeSet); + expanded.add(expandedInflection); + } + } + } + lemma.inflections.clear(); + lemma.inflections.addAll(expanded); + } + documentState.incomingSurfaceForm += lemma.inflections.size(); + } + + private void analyzeLemma(Lemma lemma, int sourceIndex) { + expandAndCountInflections(lemma); + analyzeInflections(lemma, lemma.inflections, sourceIndex); if (parserOptions.includeLemmasWithoutWords) { documentState.dictionary.computeIfAbsent(lemma.value, - d -> new DictionaryEntry(lemma.value, lemma.isRare, lemma.grammemes, null)); + d -> new DictionaryEntry(lemma.value, lemma.value, lemma.isRare, lemma.grammemes, null, sourceIndex)); } } @@ -611,6 +921,128 @@ public void writeInflectionPatterns(long startTime) throws FileNotFoundException documentState.printDocument(parserOptions, startTime); } + /** + * Streams a Wikidata lexeme JSON array (`[ { "id": "L...", ... }, ... ]`), analyzing each + * {@link Lexeme} one by one and running per-file deferred lexeme merges at the end of the array. + */ + private void parseWikidataStream(ObjectMapper objectMapper, JsonParser parser, int sourceIndex) throws IOException { + JsonToken currToken; + while ((currToken = parser.nextToken()) != JsonToken.START_OBJECT && currToken != null) { + // Find the first object in the array. + } + if (currToken == JsonToken.START_OBJECT) { + do { + Lexeme lexeme = objectMapper.readValue(parser, Lexeme.class); + try { + analyzeLexeme(parser.currentLocation().getLineNr(), lexeme, sourceIndex); + } catch (IllegalArgumentException e) { + documentState.unusableLemmaCount++; + System.err.println( + "Line " + parser.currentLocation().getLineNr() + ": " + e.getMessage()); + } + } while (parser.nextToken() != JsonToken.END_ARRAY); + processAndMergeLexemes(sourceIndex); + } + } + + /** + * Streams an OASIS DMLex v1.0 JSON lexicon object (`{ "langCode": "...", "entries": [ ... ] }`), + * validating that the root is a JSON object containing both required top-level {@code "langCode"} + * and {@code "entries"} fields. + * + *

Entries are deserialized and analyzed in streaming fashion as they are encountered. In the + * unusual case where {@code "entries"} appears before {@code "langCode"} in the JSON key order, + * entries are buffered in {@code entriesBeforeLangCode} until {@code "langCode"} is read. + */ + private void parseDMLexStream( + ObjectMapper objectMapper, + JsonParser parser, + ParserOptions.SourceInput sourceInput) throws IOException { + JsonToken firstToken = parser.nextToken(); + if (firstToken != JsonToken.START_OBJECT) { + throw MismatchedInputException.from( + parser, + DMLexEntry.class, + "Expected JSON object at root of DMLex file: " + sourceInput.filename()); + } + + String lexiconLangCode = null; + boolean sawEntries = false; + record DeferredDMLexEntry(int lineNumber, DMLexEntry entry) {} + List entriesBeforeLangCode = new ArrayList<>(); + + while (parser.nextToken() != JsonToken.END_OBJECT) { + if (parser.currentToken() == null) { + throw MismatchedInputException.from( + parser, + DMLexEntry.class, + "Unexpected end of DMLex file: " + sourceInput.filename()); + } + String fieldName = parser.currentName(); + JsonToken valueToken = parser.nextToken(); + if ("langCode".equals(fieldName)) { + if (valueToken != JsonToken.VALUE_STRING) { + throw MismatchedInputException.from( + parser, + String.class, + "Expected string for DMLex 'langCode' in " + sourceInput.filename()); + } + String langVal = parser.getText(); + if (langVal != null && !langVal.isBlank()) { + lexiconLangCode = langVal; + } + } else if ("entries".equals(fieldName)) { + if (valueToken != JsonToken.START_ARRAY) { + throw MismatchedInputException.from( + parser, + List.class, + "Expected array for DMLex 'entries' in " + sourceInput.filename()); + } + sawEntries = true; + while (parser.nextToken() != JsonToken.END_ARRAY) { + if (parser.currentToken() != JsonToken.START_OBJECT) { + throw MismatchedInputException.from( + parser, + DMLexEntry.class, + "Expected object in DMLex 'entries' array in " + sourceInput.filename()); + } + int lineNr = parser.currentLocation().getLineNr(); + DMLexEntry entry = objectMapper.readValue(parser, DMLexEntry.class); + if (lexiconLangCode != null) { + try { + analyzeDMLexEntry(lineNr, entry, lexiconLangCode, sourceInput.sourceIndex()); + } catch (IllegalArgumentException e) { + documentState.unusableLemmaCount++; + System.err.println("Line " + lineNr + ": " + e.getMessage()); + } + } else { + // "entries" appeared before "langCode" in the JSON object; defer until "langCode" is read. + entriesBeforeLangCode.add(new DeferredDMLexEntry(lineNr, entry)); + } + } + } else if (valueToken == JsonToken.START_OBJECT || valueToken == JsonToken.START_ARRAY) { + // Skip any other top-level metadata arrays/objects in the DMLex lexicon. + parser.skipChildren(); + } + } + + if (lexiconLangCode == null || !sawEntries) { + throw MismatchedInputException.from( + parser, + DMLexEntry.class, + "Invalid DMLex file (missing required 'langCode' or 'entries'): " + sourceInput.filename()); + } + + for (DeferredDMLexEntry deferred : entriesBeforeLangCode) { + try { + analyzeDMLexEntry(deferred.lineNumber(), deferred.entry(), lexiconLangCode, sourceInput.sourceIndex()); + } catch (IllegalArgumentException e) { + documentState.unusableLemmaCount++; + System.err.println("Line " + deferred.lineNumber() + ": " + e.getMessage()); + } + } + } + public static void main(String[] args) throws Exception { ParserOptions parserOptions = new ParserOptions(args); long startTime = System.currentTimeMillis(); @@ -625,7 +1057,9 @@ public static void main(String[] args) throws Exception { // being interpreted as malformed UTF-8. // We use a large buffer because we're reading a large file, and we're // frequently reading file data. - for (String sourceFilename : parserOptions.sourceFilenames) { + // Iterate through all configured source files (--wikidata and --dmlex) in CLI order. + for (ParserOptions.SourceInput sourceInput : parserOptions.sourceInputs) { + String sourceFilename = sourceInput.filename(); try (InputStream fileInputStream = new FileInputStream(sourceFilename)) { InputStream inputStream = fileInputStream; if (sourceFilename.endsWith(".bz2")) { @@ -633,27 +1067,21 @@ public static void main(String[] args) throws Exception { inputStream = new BZip2CompressorInputStream(new BufferedInputStream(inputStream, 32768)); } try (JsonParser parser = objectMapper.createParser(inputStream)) { - JsonToken currToken; - while ((currToken = parser.nextToken()) != JsonToken.START_OBJECT && currToken != null) { - // Find the first object in the array. - } - if (currToken == JsonToken.START_OBJECT) { - do { - Lexeme lexeme = objectMapper.readValue(parser, Lexeme.class); - try { - lexParser.analyzeLexeme(parser.currentLocation().getLineNr(), lexeme); - } catch (IllegalArgumentException e) { - lexParser.documentState.unusableLemmaCount++; - System.err.println( - "Line " + parser.currentLocation().getLineNr() + ": " + e.getMessage()); - } - } while (parser.nextToken() != JsonToken.END_ARRAY); - lexParser.processAndMergeLexemes(); + if (sourceInput.format() == ParserOptions.SourceFormat.DMLEX) { + lexParser.parseDMLexStream(objectMapper, parser, sourceInput); + } else { + lexParser.parseWikidataStream(objectMapper, parser, sourceInput.sourceIndex()); } } } } + // In multi-source overwrite mode (--merge-conflicts=false with >1 source file), + // preempt superseded (lemma, PartOfSpeech) paradigms and analyze the surviving lemmas now. + if (lexParser.isMultiSourceOverwriteMode()) { + lexParser.processStagedLemmas(); + } + if (!lexParser.parserOptions.additionalGrammemesDict.isEmpty()) { lexParser.mergeAdditionalGrammemes(); } diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java index 98768b47b..17893cff6 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -25,13 +26,22 @@ import java.util.regex.Pattern; import static org.unicode.wikidata.Grammar.DEFAULTMAP; -import static org.unicode.wikidata.Grammar.TYPEMAP; /** * The options to extract the data from the data source. */ final class ParserOptions { + enum SourceFormat { + WIKIDATA, + DMLEX + } + + record SourceInput(String filename, SourceFormat format, int sourceIndex) {} + private static final char COLON_SEPARATOR = ':'; + static final String WIKIDATA_FILE = "--wikidata"; + static final String DMLEX_FILE = "--dmlex"; + static final String MERGE_CONFLICTS = "--merge-conflicts"; static final String INFLECTIONS_FILE = "--inflections"; static final String DICTIONARY_FILE = "--dictionary"; static final String MAP_GRAMMEME = "--map-grammeme"; @@ -47,6 +57,7 @@ final class ParserOptions { static final String ADD_SOUND = "--add-sound"; boolean includeLemmasWithoutWords = false; + boolean mergeConflicts = false; boolean debug = false; final boolean addSound; @@ -54,8 +65,17 @@ final class ParserOptions { TreeMap>, List>>> expandGramemes; TreeMap> additionalGrammemesDict; TreeMap> claimsToSound; + // Per-run string-to-grammeme overrides configured via --map-grammeme or Q-ID arguments to + // --ignore-property / --ignore-entries-with-grammemes. Stored per ParserOptions instance so + // static Grammar.TYPEMAP is never mutated in place. + private final Map>> customTypeMap = new HashMap<>(); + // Per-run enum overrides configured via --ignore-property or --ignore-entries-with-grammemes + // when specified by grammeme name (e.g. "countable", "shortForm"). Any resolved grammeme enum + // present in this map is replaced with its configured Grammar.Ignorable token during lookup. + private final Map, Grammar.Ignorable> ignoredGrammemes = new HashMap<>(); ArrayList sourceFilenames; + ArrayList sourceInputs; String inflectionalFilename = ParserDefaults.DEFAULT_INFLECTION_FILE_NAME; String lexicalDictionaryFilename = ParserDefaults.DEFAULT_DICTIONARY_FILE_NAME; ArrayList locales = new ArrayList<>(List.of(Locale.ENGLISH.getLanguage())); @@ -63,8 +83,11 @@ final class ParserOptions { List optionsUsedToInvoke = new ArrayList<>(); private static void printUsage() { - System.err.println("Usage: ParseLexicon [OPTIONS] [ ...]"); + System.err.println("Usage: ParseLexicon [OPTIONS] (--wikidata [ ...] | --dmlex [ ...]) ..."); System.err.println("\nOPTIONS"); + System.err.println(WIKIDATA_FILE + " [ ...]\tOne or more Wikidata lexeme JSON/bz2 files (space- or comma-separated)"); + System.err.println(DMLEX_FILE + " [ ...]\tOne or more OASIS DMLex JSON files (space- or comma-separated)"); + System.err.println(MERGE_CONFLICTS + "[={true|false}]\tWhen true, merge conflicting cross-file entries of the same part of speech instead of overwriting, default: false"); System.err.println(INFLECTIONS_FILE + " \tthe file for the inflectional patterns to be generated, default: inflectional.xml"); System.err.println(DICTIONARY_FILE + " \tthe file for the lexical dictionary to be generated, default: dictionary.lst"); System.err.println(ADD_EXTRA_GRAMMEMES + " \tFile containing words with the extra grammemes to be added, provide path relative to tools/dictionary-parser/src/main/resources/org/unicode/wikidata/ (only to be used for a temporary grammeme addition)"); @@ -84,11 +107,39 @@ private static void printUsage() { posToBeInflected = EnumSet.of(Grammar.PartOfSpeech.NOUN); additionalGrammemesDict = new TreeMap<>(); sourceFilenames = new ArrayList<>(); + sourceInputs = new ArrayList<>(); claimsToSound = new TreeMap<>(); for (int i = 0; i < args.length; i++) { String arg = args[i]; - if (ParserOptions.INFLECTIONS_FILE.equals(arg)) { + if (arg == null) { + printUsage(); + throw new IllegalArgumentException("Null argument at index " + i); + } + if (ParserOptions.WIKIDATA_FILE.equals(arg) || arg.startsWith(ParserOptions.WIKIDATA_FILE + "=")) { + i = consumeSourceFiles(args, i, ParserOptions.WIKIDATA_FILE, SourceFormat.WIKIDATA); + } else if (ParserOptions.DMLEX_FILE.equals(arg) || arg.startsWith(ParserOptions.DMLEX_FILE + "=")) { + i = consumeSourceFiles(args, i, ParserOptions.DMLEX_FILE, SourceFormat.DMLEX); + } else if (ParserOptions.MERGE_CONFLICTS.equals(arg) || arg.startsWith(ParserOptions.MERGE_CONFLICTS + "=")) { + String valueStr; + if (arg.startsWith(ParserOptions.MERGE_CONFLICTS + "=")) { + valueStr = arg.substring((ParserOptions.MERGE_CONFLICTS + "=").length()).trim(); + } else if (i + 1 < args.length && args[i + 1] != null + && ("true".equalsIgnoreCase(args[i + 1]) || "false".equalsIgnoreCase(args[i + 1]))) { + valueStr = args[++i].trim(); + } else { + valueStr = "true"; + } + if ("true".equalsIgnoreCase(valueStr)) { + mergeConflicts = true; + } else if ("false".equalsIgnoreCase(valueStr)) { + mergeConflicts = false; + } else { + printUsage(); + throw new IllegalArgumentException("Invalid value for " + ParserOptions.MERGE_CONFLICTS + ": " + valueStr); + } + optionsUsedToInvoke.add(ParserOptions.MERGE_CONFLICTS + "=" + mergeConflicts); + } else if (ParserOptions.INFLECTIONS_FILE.equals(arg)) { inflectionalFilename = args[++i]; } else if (ParserOptions.DICTIONARY_FILE.equals(arg)) { lexicalDictionaryFilename = args[++i]; @@ -113,7 +164,9 @@ private static void printUsage() { } else if (ParserOptions.MAP_GRAMMEME.equals(arg)) { String mapGrammeme = args[++i]; String[] split = mapGrammeme.split(",", 2); - TYPEMAP.put(split[0], toEnumSet(split[1])); + TreeSet> mappedSet = toEnumSet(split[1]); + customTypeMap.put(split[0], mappedSet); + customTypeMap.put(Grammar.normalizeGrammemeKey(split[0]), mappedSet); optionsUsedToInvoke.add(ParserOptions.MAP_GRAMMEME); optionsUsedToInvoke.add(mapGrammeme); @@ -141,7 +194,8 @@ private static void printUsage() { posToBeInflected.clear(); for (String pos : inflectionTypes.split(",")) { - posToBeInflected.add(Grammar.PartOfSpeech.valueOf(pos.toUpperCase().replace('-', '_'))); + posToBeInflected.add(Grammar.PartOfSpeech.valueOf( + Grammar.normalizeGrammemeKey(pos).toUpperCase(Locale.ROOT).replace('-', '_'))); } optionsUsedToInvoke.add(ParserOptions.INFLECTION_TYPES); @@ -202,48 +256,138 @@ private static void printUsage() { optionsUsedToInvoke.add(ParserOptions.ADD_SOUND); optionsUsedToInvoke.add(soundGrammemeTypes); } else { - sourceFilenames.add(arg); + printUsage(); + throw new IllegalArgumentException("Unknown option: " + arg); } } addSound = !claimsToSound.isEmpty(); - if (sourceFilenames.isEmpty()) { + if (sourceInputs.isEmpty()) { printUsage(); - throw new IllegalArgumentException(); + throw new IllegalArgumentException("At least one input file must be specified via --wikidata or --dmlex"); } } + private int consumeSourceFiles(String[] args, int i, String flagPrefix, SourceFormat format) { + String arg = args[i]; + int startCount = sourceInputs.size(); + if (arg.startsWith(flagPrefix + "=")) { + String inlineVal = arg.substring((flagPrefix + "=").length()); + for (String part : inlineVal.split(",")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + sourceInputs.add(new SourceInput(trimmed, format, sourceInputs.size())); + sourceFilenames.add(trimmed); + } + } + } + while (i + 1 < args.length && args[i + 1] != null && !args[i + 1].startsWith("--")) { + String rawArg = args[++i]; + for (String part : rawArg.split(",")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + sourceInputs.add(new SourceInput(trimmed, format, sourceInputs.size())); + sourceFilenames.add(trimmed); + } + } + } + if (sourceInputs.size() == startCount) { + printUsage(); + throw new IllegalArgumentException(flagPrefix + " requires at least one file argument"); + } + return i; + } + + /** + * Records per-run ignore rules from {@code --ignore-property} or {@code --ignore-entries-with-grammemes}. + * Known grammeme names or single-enum shorthand aliases (e.g. {@code "countable"}, {@code "shortForm"}, + * {@code "masc"}) are resolved to their non-{@link Grammar.Ignorable} enum and recorded in + * {@link #ignoredGrammemes} so any Q-ID or DMLex label mapping to that enum is replaced with + * {@code ignorable} at lookup time. Raw Wikidata Q-IDs (e.g. {@code "Q55194613"}) and unmapped + * DMLex tags (e.g. {@code "archaic"}, {@code "dialectal"}) are recorded in {@link #customTypeMap}. + */ void setIgnoreProperty(String[] grammemes, Grammar.Ignorable ignorable) { var ignorableSet = EnumSet.of(ignorable); for (String grammeme : grammemes) { - if (grammeme.matches("Q\\d*")) { - TYPEMAP.put(grammeme, ignorableSet); - } else { - for (Map.Entry>> entry : TYPEMAP.entrySet()) { - for (var grammemeEnum : entry.getValue()) { - String name = grammemeEnum.name(); - if (name.equalsIgnoreCase(grammeme)) { - if (entry.getValue().size() == 1) { - entry.setValue(ignorableSet); - } else { - entry.getValue().remove(grammemeEnum); - ArrayList> clone = new ArrayList<>(entry.getValue()); - clone.add(ignorable); - entry.setValue(new HashSet<>(clone)); - } - break; - } + Enum enumVal = null; + if (!grammeme.matches("Q\\d*")) { + enumVal = DEFAULTMAP.get(grammeme); + if (enumVal == null) { + enumVal = DEFAULTMAP.get(Grammar.normalizeGrammemeKey(grammeme)); + } + if (enumVal == null) { + Set> mapped = Grammar.getMappedGrammemes(grammeme); + if (mapped != null && mapped.size() == 1) { + enumVal = mapped.iterator().next(); } } } + if (enumVal != null && !(enumVal instanceof Grammar.Ignorable)) { + ignoredGrammemes.put(enumVal, ignorable); + } else { + customTypeMap.put(grammeme, ignorableSet); + customTypeMap.put(Grammar.normalizeGrammemeKey(grammeme), ignorableSet); + } + } + } + + /** + * Resolves a Wikidata Q-ID or DMLex grammatical label to its mapped {@link Grammar} enums, + * applying any per-run CLI overrides ({@code --map-grammeme}, {@code --ignore-property}, + * {@code --ignore-entries-with-grammemes}) without mutating static {@link Grammar#TYPEMAP}. + */ + Set> getMappedGrammemes(String grammeme) { + if (grammeme == null) { + return null; + } + // 1. Check per-run string overrides (--map-grammeme or Q-ID --ignore-* rules). + Set> mapped = customTypeMap.get(grammeme); + if (mapped == null && !customTypeMap.isEmpty()) { + mapped = customTypeMap.get(Grammar.normalizeGrammemeKey(grammeme)); + } + // 2. Fall back to the read-only static Grammar lookup table. + if (mapped == null) { + mapped = Grammar.getMappedGrammemes(grammeme); + } + if (mapped == null || ignoredGrammemes.isEmpty()) { + return mapped; + } + // 3. Replace any resolved enum present in ignoredGrammemes with its Ignorable token. + boolean hasIgnored = false; + for (Enum e : mapped) { + if (ignoredGrammemes.containsKey(e)) { + hasIgnored = true; + break; + } + } + if (!hasIgnored) { + return mapped; + } + if (mapped.size() == 1) { + return EnumSet.of(ignoredGrammemes.get(mapped.iterator().next())); + } + Set> adjusted = new HashSet<>(mapped.size()); + for (Enum e : mapped) { + Grammar.Ignorable replacement = ignoredGrammemes.get(e); + adjusted.add(replacement != null ? replacement : e); } + return adjusted; } TreeSet> toEnumSet(String grammemes) { TreeSet> grammemeSet = new TreeSet<>(Inflection.ENUM_COMPARATOR); for (var grammeme : grammemes.split(",")) { var grammemeEnum = DEFAULTMAP.get(grammeme); + if (grammemeEnum == null) { + grammemeEnum = DEFAULTMAP.get(Grammar.normalizeGrammemeKey(grammeme)); + } + if (grammemeEnum == null && !grammeme.matches("Q\\d*")) { + Set> mapped = Grammar.getMappedGrammemes(grammeme); + if (mapped != null && mapped.size() == 1) { + grammemeEnum = mapped.iterator().next(); + } + } if (grammemeEnum == null) { throw new NullPointerException(grammeme + " is not a valid grammeme"); } diff --git a/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java b/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java index cf057c899..4de8692fc 100644 --- a/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java +++ b/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java @@ -7,8 +7,12 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Map; import java.util.Objects; +import java.util.stream.Collectors; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.exc.MismatchedInputException; import org.junit.jupiter.api.Assertions; @@ -18,12 +22,16 @@ class ParseWikidataTest { private static final String lexiconSource = Objects .requireNonNull(MethodHandles.lookup().lookupClass().getClassLoader().getResource("sourceLexicon.json")).getFile(); private static final String baseDir = lexiconSource.substring(0, lexiconSource.lastIndexOf("/")); - private static final String inflectionalFile = baseDir+"/LexiconInflectOut.xml"; - private static final String dictionaryFile = baseDir+"/LexiconInflectOut.lst"; - private static final String expectedOutputFile = baseDir+ "/lexiconCorrectOut.txt"; - private static final String fatalErrorFile = baseDir+ "/lexiconFatalError.json"; - private static final String missingGrammemeErrorFile = baseDir+ "/lexiconMissingGrammemeError.json"; - private static final String caseInsensitiveGrammemeFile = baseDir+ "/lexiconcaseInsenstiveGrammeme.json"; + private static final String inflectionalFile = baseDir + "/LexiconInflectOut.xml"; + private static final String dictionaryFile = baseDir + "/LexiconInflectOut.lst"; + private static final String expectedOutputFile = baseDir + "/lexiconCorrectOut.txt"; + private static final String fatalErrorFile = baseDir + "/lexiconFatalError.json"; + private static final String missingGrammemeErrorFile = baseDir + "/lexiconMissingGrammemeError.json"; + private static final String caseInsensitiveGrammemeFile = baseDir + "/lexiconcaseInsenstiveGrammeme.json"; + private static final String dmlexLexiconFile = baseDir + "/dmlexLexicon.json"; + private static final String dmlexOverlayFile = baseDir + "/dmlexOverlay.json"; + private static final String dmlexPatchFile = baseDir + "/dmlexPatch.json"; + private static final String dmlexInvalidSchemaFile = baseDir + "/dmlexInvalidSchema.json"; private static final String LINE_END = "=============================================="; private void compareOutputs(String actual, String expected) { @@ -44,9 +52,18 @@ private String getParserOutput(String[] args) throws Exception { return Files.readString(Paths.get(dictionaryFile), StandardCharsets.UTF_8); } + private Map parseDictionaryEntries(String output) { + return Arrays.stream(output.split("\n")) + .takeWhile(line -> !line.equals(LINE_END)) + .filter(line -> line.contains(":")) + .collect(Collectors.toMap( + line -> line.substring(0, line.indexOf(':')), + line -> line.substring(line.indexOf(':') + 1).trim())); + } + @Test public void missingInputFile() { - Assertions.assertThrows(Exception.class,() -> ParseWikidata.main(new String[1])); + Assertions.assertThrows(Exception.class, () -> ParseWikidata.main(new String[1])); } @Test @@ -58,29 +75,625 @@ public void enumTest() { @Test public void fatalErrorHandlerTest() { // Malformed JSON syntax cannot be parsed by the Wikidata reader. - String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, fatalErrorFile}; - Assertions.assertThrows(JsonParseException.class,() -> ParseWikidata.main(args)); + String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", fatalErrorFile}; + Assertions.assertThrows(JsonParseException.class, () -> ParseWikidata.main(args)); } @Test public void missingGrammemeTest() { // The grammatical features are not in the expected array form, so the reader rejects the input. - String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, missingGrammemeErrorFile}; - Assertions.assertThrows(MismatchedInputException.class,() -> ParseWikidata.main(args)); + String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", missingGrammemeErrorFile}; + Assertions.assertThrows(MismatchedInputException.class, () -> ParseWikidata.main(args)); } @Test public void caseInsensitiveGrammemeTest() { - String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, caseInsensitiveGrammemeFile}; + String[] args = {"--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", caseInsensitiveGrammemeFile}; Assertions.assertDoesNotThrow(() -> ParseWikidata.main(args)); } @Test public void lexiconParserTest() throws Exception { String[] args = {"--inflection-types", "noun,adjective,proper-noun", "--add-sound", "vowel-start", - "--inflections", inflectionalFile, "--dictionary", dictionaryFile, lexiconSource}; + "--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--wikidata", lexiconSource}; String actual = getParserOutput(args); String expected = Files.readString(Paths.get(expectedOutputFile), StandardCharsets.UTF_8); compareOutputs(actual, expected); } + + @Test + public void standaloneDMLexParserTest() throws Exception { + String[] args = { + "--inflection-types", "noun,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexLexiconFile + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // cat and bat share pattern 1 (count=2, most frequent) + Assertions.assertEquals("singular masculine noun inflection=1", entries.get("cat")); + Assertions.assertEquals("plural masculine noun inflection=1", entries.get("cats")); + Assertions.assertEquals("singular masculine noun inflection=1", entries.get("bat")); + Assertions.assertEquals("plural masculine noun inflection=1", entries.get("bats")); + + // apple has IPA pronunciation matching vowel-start + Assertions.assertEquals("singular vowel-start noun inflection=2", entries.get("apple")); + Assertions.assertEquals("plural vowel-start noun inflection=2", entries.get("apples")); + + // London uses camelCase properNoun normalized to proper-noun + Assertions.assertEquals("singular proper-noun inflection=3", entries.get("London")); + + // wow has no inflectedForms array, synthesized default form + Assertions.assertEquals("interjection", entries.get("wow")); + + // French entry "bonjour" must be filtered out because default --language is en + Assertions.assertFalse(entries.containsKey("bonjour")); + + // Verify inflectional XML was generated + String xml = Files.readString(Paths.get(inflectionalFile), StandardCharsets.UTF_8); + Assertions.assertTrue(xml.contains("")); + Assertions.assertTrue(xml.contains("")); + } + + @Test + public void multiSourceWikidataAndDMLexOverwriteTest() throws Exception { + // Default --merge-conflicts=false: later source (dmlexOverlayFile) overrides same-POS conflicts + String[] args = { + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", lexiconSource, + "--dmlex", dmlexOverlayFile + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // 1. New lemma "book" from DMLex is added + Assertions.assertTrue(entries.containsKey("book")); + Assertions.assertTrue(entries.get("book").startsWith("singular noun inflection=")); + Assertions.assertTrue(entries.containsKey("book's")); + Assertions.assertTrue(entries.containsKey("books")); + Assertions.assertTrue(entries.containsKey("books'")); + + // 2. "hey" was interjection in Wikidata and noun in DMLex -> both POS are preserved + String heyEntry = entries.get("hey"); + Assertions.assertNotNull(heyEntry); + Assertions.assertTrue(heyEntry.contains("noun"), "Expected noun in: " + heyEntry); + Assertions.assertTrue(heyEntry.contains("interjection"), "Expected interjection in: " + heyEntry); + Assertions.assertTrue(entries.containsKey("heys")); + + // 3. "ciel" was masculine noun (ciel/ciels/cieux) in Wikidata, overridden by feminine noun (ciel/cieux) in DMLex + String cielEntry = entries.get("ciel"); + Assertions.assertNotNull(cielEntry); + Assertions.assertTrue(cielEntry.contains("feminine"), "Expected feminine in: " + cielEntry); + Assertions.assertFalse(cielEntry.contains("masculine"), "Did not expect masculine in: " + cielEntry); + + String cieuxEntry = entries.get("cieux"); + Assertions.assertNotNull(cieuxEntry); + Assertions.assertTrue(cieuxEntry.contains("feminine"), "Expected feminine in: " + cieuxEntry); + Assertions.assertFalse(cieuxEntry.contains("masculine"), "Did not expect masculine in: " + cieuxEntry); + + // Orphaned Wikidata form "ciels" must not be present when ciel(noun) was superseded + Assertions.assertFalse(entries.containsKey("ciels"), "Expected orphaned form 'ciels' to be removed"); + + // 4. Verify inflectional XML: preempting "ciel" (L7, L8) at Tier 1 leaves only "idéal" (L9, L10) + // in the -l/-ls/-ux pattern, so its word count is words="2" (instead of words="3" or words="4") + // and "idéal" references a single merged pattern instead of two split patterns. + String idealEntry = entries.get("idéal"); + Assertions.assertNotNull(idealEntry); + Assertions.assertEquals(1, idealEntry.split("inflection=").length - 1, "Expected 1 merged inflection pattern in: " + idealEntry); + + String xml = Files.readString(Paths.get(inflectionalFile), StandardCharsets.UTF_8); + Assertions.assertTrue(xml.contains("")); + Assertions.assertFalse(xml.contains("words=\"3\"")); + Assertions.assertFalse(xml.contains("words=\"4\"")); + } + + @Test + public void multiSourceWikidataAndDMLexMergeConflictsTrueTest() throws Exception { + // --merge-conflicts=true: conflicting entries of the same POS across files are merged + String[] args = { + "--merge-conflicts=true", + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", lexiconSource, + "--dmlex", dmlexOverlayFile + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // In merge mode, "ciel" merges Wikidata (masculine) and DMLex (feminine), keeping both patterns and "ciels" + String cielEntry = entries.get("ciel"); + Assertions.assertNotNull(cielEntry); + Assertions.assertTrue(cielEntry.contains("masculine"), "Expected masculine in: " + cielEntry); + Assertions.assertTrue(cielEntry.contains("feminine"), "Expected feminine in: " + cielEntry); + // Should have two inflection=... references (one from Wikidata pattern, one from DMLex pattern) + Assertions.assertEquals(2, cielEntry.split("inflection=").length - 1, "Expected 2 inflection patterns in: " + cielEntry); + + // "ciels" from Wikidata is preserved in merge mode + Assertions.assertTrue(entries.containsKey("ciels")); + Assertions.assertTrue(entries.get("ciels").contains("plural masculine noun")); + } + + @Test + public void multiSourcePartialMultiPosOverrideTest() throws Exception { + // In sourceLexicon.json, "Élie" (L2) has Q147276 which maps to BOTH noun and proper-noun (plural masculine). + // When a later DMLex file overrides ONLY proper-noun on "Élie" (singular feminine proper-noun), + // the noun slice from Wikidata (plural masculine noun) must be preserved while proper-noun is replaced. + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-partial-pos-", ".json"); + try { + String json = """ + { + "langCode": "en", + "entries": [ + { + "id": "elie-override", + "headword": "Élie", + "partOfSpeech": ["proper-noun"], + "tag": "feminine", + "labels": ["rare"], + "pronunciations": [ + { + "transcriptions": [ + { "text": "e.li", "scheme": "IPA" } + ] + } + ], + "inflectedForms": [ + { + "value": "Élie", + "tag": "singular", + "labels": ["rare"] + } + ] + } + ] + } + """; + Files.writeString(tempDmlex, json, StandardCharsets.UTF_8); + + String[] args = { + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", lexiconSource, + "--dmlex=" + tempDmlex + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + String elieEntry = entries.get("Élie"); + Assertions.assertNotNull(elieEntry); + // Both noun (plural masculine from Wikidata) and proper-noun (singular feminine vowel-start from DMLex) are present, + // with 2 distinct inflection patterns (non-rare noun pattern first, rare proper-noun pattern second). + Assertions.assertEquals( + "singular plural masculine feminine vowel-start noun proper-noun inflection=3 inflection=8", + elieEntry); + } finally { + Files.deleteIfExists(tempDmlex); + } + } + + @Test + public void multiFileDMLexSpaceAndCommaSeparatedTest() throws Exception { + String[] spaceArgs = { + "--inflection-types", "noun,proper-noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexLexiconFile, dmlexPatchFile + }; + String spaceOutput = getParserOutput(spaceArgs); + + String[] commaArgs = { + "--inflection-types", "noun,proper-noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexLexiconFile + "," + dmlexPatchFile + }; + String commaOutput = getParserOutput(commaArgs); + + compareOutputs(spaceOutput, commaOutput); + + Map entries = parseDictionaryEntries(spaceOutput); + // "cat" (noun) was overridden by dmlexPatch.json: feminine, plural "kitties", and "cats" removed + Assertions.assertTrue(entries.get("cat").contains("singular feminine noun")); + Assertions.assertFalse(entries.get("cat").contains("masculine")); + Assertions.assertTrue(entries.containsKey("kitties")); + Assertions.assertFalse(entries.containsKey("cats")); + + // "bats" surface form from lemma "bat" (masculine noun) was overridden at Tier 2 by lemma "batte" (feminine noun) + Assertions.assertTrue(entries.get("bats").contains("plural feminine noun")); + Assertions.assertFalse(entries.get("bats").contains("masculine")); + + // "wow" keeps interjection from dmlexLexicon.json and adds verb from dmlexPatch.json + Assertions.assertEquals("interjection verb", entries.get("wow")); + Assertions.assertEquals("verb", entries.get("wowed")); + } + + @Test + public void invalidDMLexSchemaAndUnknownOptionTest() { + // 1. JSON object missing required "langCode" / "entries" + String[] invalidObjArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", dmlexInvalidSchemaFile + }; + Assertions.assertThrows(MismatchedInputException.class, () -> ParseWikidata.main(invalidObjArgs)); + + // 2. Passing Wikidata JSON array to --dmlex + String[] wikidataAsDmlexArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", lexiconSource + }; + Assertions.assertThrows(MismatchedInputException.class, () -> ParseWikidata.main(wikidataAsDmlexArgs)); + + // 3. Bare positional file without --wikidata or --dmlex is rejected + String[] bareFileArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + lexiconSource + }; + Assertions.assertThrows(IllegalArgumentException.class, () -> ParseWikidata.main(bareFileArgs)); + + // 4. --dmlex with no file argument is rejected + String[] emptyDmlexArgs = { + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex" + }; + Assertions.assertThrows(IllegalArgumentException.class, () -> ParseWikidata.main(emptyDmlexArgs)); + } + + @Test + public void dmlexCliOptionsCompatibilityTest() throws Exception { + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-opts-", ".json"); + try { + String json = """ + { + "langCode": "ro", + "entries": [ + { + "id": "ro-1", + "headword": "casă", + "partOfSpeech": ["noun"], + "labels": ["feminine", "countable", "dialectal"], + "inflectedForms": [ + { "value": "casă", "labels": ["singular", "customNomAcc"] }, + { "value": "case", "labels": ["plural", "customNomAcc"] }, + { "value": "casă-short", "labels": ["singular", "short-form"] }, + { "value": "casă-archaic", "labels": ["singular", "archaic"] } + ] + }, + { + "id": "ro-md-1", + "headword": "moldovanWord", + "langCode": "ro-md", + "partOfSpeech": ["noun"], + "inflectedForms": [ + { "value": "moldovanWord", "labels": ["singular"] } + ] + } + ] + } + """; + Files.writeString(tempDmlex, json, StandardCharsets.UTF_8); + + String[] args = { + "--language", "ro", + "--exclude-language", "ro-md", + "--inflection-types", "noun,properNoun", + "--map-grammeme", "customNomAcc,nominative,accusative", + "--expand-grammemes", "singular,nominative,accusative,feminine,noun:indefinite", + "--ignore-property", "countable,dialectal", + "--ignore-entries-with-grammemes", "shortForm,archaic", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", tempDmlex.toString() + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // ro-md entry excluded via --exclude-language + Assertions.assertFalse(entries.containsKey("moldovanWord")); + // short-form and unmapped "archaic" surface forms ignored via --ignore-entries-with-grammemes + Assertions.assertFalse(entries.containsKey("casă-short")); + Assertions.assertFalse(entries.containsKey("casă-archaic")); + // countable and unmapped "dialectal" ignored via --ignore-property; + // customNomAcc mapped to nominative,accusative; singular expanded with indefinite via --expand-grammemes + Assertions.assertEquals("singular accusative nominative indefinite feminine noun", entries.get("casă")); + Assertions.assertEquals("plural accusative nominative feminine noun", entries.get("case")); + } finally { + Files.deleteIfExists(tempDmlex); + } + } + + @Test + public void dmlexEntriesBeforeLangCodeMultiFieldObjectsAndReverseOrderTest() throws Exception { + // Tests: + // 1. "entries" appearing before "langCode" in the root DMLex JSON object. + // 2. Single-object multi-field labels ({"tag": "singular", "label": "Singular number"}) and + // pronunciations ({"label": "US", "transcription": "ˈæp.əl"}). + // 3. Reverse source order (--dmlex --wikidata ), where Wikidata overrides DMLex. + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-reverse-", ".json"); + try { + String json = """ + { + "entries": [ + { + "id": "dmlex-apple", + "headword": "apple", + "partOfSpeech": [ + { "label": "Common noun", "tag": "noun" } + ], + "labels": [ + { "tag": "nonHuman", "label": "Non-human animacy" } + ], + "pronunciations": [ + { "label": "US", "transcription": "ˈæp.əl" } + ], + "inflectedForms": [ + { + "value": "apple", + "labels": [ + { "tag": "singular", "label": "Singular number" } + ] + }, + { + "value": "apples", + "labels": [ + { "label": "Plural number", "tag": "plural" } + ] + } + ] + }, + { + "id": "dmlex-ciel-early", + "headword": "ciel", + "partOfSpeech": ["noun"], + "labels": ["feminine"], + "inflectedForms": [ + { "value": "ciel", "labels": ["singular"] }, + { "value": "ciel-obsolete", "labels": ["plural"] } + ] + }, + { + "id": "dmlex-posless-early", + "headword": "uh-oh", + "inflectedForms": [ + { "value": "uh-oh", "labels": ["singular"] }, + { "value": "uh-oh-obsolete", "labels": ["plural"] } + ] + } + ], + "langCode": "en" + } + """; + Files.writeString(tempDmlex, json, StandardCharsets.UTF_8); + + Path tempDmlex2 = Files.createTempFile(Paths.get(baseDir), "dmlex-posless-late-", ".json"); + try { + String json2 = """ + { + "langCode": "en", + "entries": [ + { + "id": "dmlex-posless-late", + "headword": "uh-oh", + "inflectedForms": [ + { "value": "uh-oh", "labels": ["plural"] } + ] + } + ] + } + """; + Files.writeString(tempDmlex2, json2, StandardCharsets.UTF_8); + + String[] args = { + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--dmlex", tempDmlex.toString(), + "--wikidata", lexiconSource, + "--dmlex", tempDmlex2.toString() + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + // "apple" from DMLex is parsed with multi-field objects and deferred "langCode" + Assertions.assertTrue(entries.get("apple").startsWith("singular nonhuman vowel-start noun inflection=")); + Assertions.assertTrue(entries.get("apples").startsWith("plural nonhuman vowel-start noun inflection=")); + + // Because --wikidata came AFTER --dmlex, Wikidata's masculine "ciel" (ciel/ciels/cieux) overrides DMLex's feminine "ciel", + // and DMLex's orphaned form "ciel-obsolete" is preempted at Tier 1. + Assertions.assertTrue(entries.get("ciel").contains("masculine")); + Assertions.assertFalse(entries.get("ciel").contains("feminine")); + Assertions.assertTrue(entries.containsKey("ciels")); + Assertions.assertTrue(entries.containsKey("cieux")); + Assertions.assertFalse(entries.containsKey("ciel-obsolete")); + + // POS-less lemma "uh-oh" from tempDmlex was preempted at Tier 1 by tempDmlex2, removing "uh-oh-obsolete". + Assertions.assertEquals("plural", entries.get("uh-oh")); + Assertions.assertFalse(entries.containsKey("uh-oh-obsolete")); + } finally { + Files.deleteIfExists(tempDmlex2); + } + } finally { + Files.deleteIfExists(tempDmlex); + } + } + + @Test + public void deferredLexemeMultiVariantAndSupplementalMergeTest() throws Exception { + // filter_fr.properties defines L27261=L1566399 (organisateur = organisatrice). + // Tests: + // 1. Primary Wikidata file (sourceIndex == 0) with multi-variant L27261 ("fr" and "fr-x-Q56352306") + // and L1566399 merges both variants and both partner lexemes without throwing "Duplicate lexeme". + // 2. Supplemental Wikidata file (sourceIndex == 1) containing only L27261 (without L1566399) + // does not throw "L1566399: id not found" and overrides the earlier paradigm. + // 3. Duplicate top-level Lexeme objects with the same deferred ID in one file still throw IllegalStateException. + Path primaryWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-primary-", ".json"); + Path supplementalWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-supp-", ".json"); + Path duplicateWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-dup-", ".json"); + try { + String primaryJson = """ + [ + { + "type": "lexeme", + "id": "L27261", + "lemmas": { + "fr": { "language": "fr", "value": "organisateur" }, + "fr-x-Q56352306": { "language": "fr-x-Q56352306", "value": "organisateur" } + }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { + "fr": { "language": "fr", "value": "organisateur" } + }, + "grammaticalFeatures": ["Q110786", "Q499327"], + "claims": {} + }, + { + "id": "L27261-F2", + "representations": { + "fr": { "language": "fr", "value": "organisateurs-old" } + }, + "grammaticalFeatures": ["Q146786", "Q499327"], + "claims": {} + } + ], + "senses": [] + }, + { + "type": "lexeme", + "id": "L1566399", + "lemmas": { + "fr": { "language": "fr", "value": "organisatrice" } + }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L1566399-F1", + "representations": { + "fr": { "language": "fr", "value": "organisatrice" } + }, + "grammaticalFeatures": ["Q110786", "Q1775415"], + "claims": {} + } + ], + "senses": [] + } + ] + """; + String supplementalJson = """ + [ + { + "type": "lexeme", + "id": "L27261", + "lemmas": { + "fr": { "language": "fr", "value": "organisateur" } + }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { + "fr": { "language": "fr", "value": "organisateur" } + }, + "grammaticalFeatures": ["Q110786", "Q499327"], + "claims": {} + }, + { + "id": "L27261-F2", + "representations": { + "fr": { "language": "fr", "value": "organisateurs" } + }, + "grammaticalFeatures": ["Q146786", "Q499327"], + "claims": {} + } + ], + "senses": [] + } + ] + """; + String duplicateJson = """ + [ + { + "type": "lexeme", + "id": "L27261", + "lemmas": { "fr": { "language": "fr", "value": "organisateur" } }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { "fr": { "language": "fr", "value": "organisateur" } }, + "grammaticalFeatures": ["Q110786"], + "claims": {} + } + ], + "senses": [] + }, + { + "type": "lexeme", + "id": "L27261", + "lemmas": { "fr": { "language": "fr", "value": "organisateur" } }, + "lexicalCategory": "Q1084", + "claims": {}, + "forms": [ + { + "id": "L27261-F1", + "representations": { "fr": { "language": "fr", "value": "organisateur" } }, + "grammaticalFeatures": ["Q110786"], + "claims": {} + } + ], + "senses": [] + } + ] + """; + Files.writeString(primaryWikidata, primaryJson, StandardCharsets.UTF_8); + Files.writeString(supplementalWikidata, supplementalJson, StandardCharsets.UTF_8); + Files.writeString(duplicateWikidata, duplicateJson, StandardCharsets.UTF_8); + + String[] args = { + "--language", "fr", + "--inflection-types", "noun", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", primaryWikidata.toString(), supplementalWikidata.toString() + }; + String actual = getParserOutput(args); + Map entries = parseDictionaryEntries(actual); + + Assertions.assertTrue(entries.containsKey("organisateur")); + Assertions.assertTrue(entries.containsKey("organisateurs")); + Assertions.assertFalse(entries.containsKey("organisateurs-old")); + + String[] dupArgs = { + "--language", "fr", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", duplicateWikidata.toString() + }; + Assertions.assertThrows(IllegalStateException.class, () -> ParseWikidata.main(dupArgs)); + } finally { + Files.deleteIfExists(primaryWikidata); + Files.deleteIfExists(supplementalWikidata); + Files.deleteIfExists(duplicateWikidata); + } + } } diff --git a/tools/dictionary-parser/src/test/resources/dmlexInvalidSchema.json b/tools/dictionary-parser/src/test/resources/dmlexInvalidSchema.json new file mode 100644 index 000000000..ec1f4c8f9 --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexInvalidSchema.json @@ -0,0 +1,3 @@ +{ + "unexpectedField": "not a valid DMLex document" +} diff --git a/tools/dictionary-parser/src/test/resources/dmlexLexicon.json b/tools/dictionary-parser/src/test/resources/dmlexLexicon.json new file mode 100644 index 000000000..7fc818ca9 --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexLexicon.json @@ -0,0 +1,92 @@ +{ + "uri": "http://example.org/lexicon/en", + "langCode": "en", + "entries": [ + { + "id": "dmlex-1", + "headword": "cat", + "partOfSpeech": ["noun"], + "labels": ["masculine"], + "inflectedForms": [ + { + "id": "dmlex-1-f1", + "value": "cat", + "labels": ["singular"] + }, + { + "id": "dmlex-1-f2", + "value": "cats", + "labels": ["plural"] + } + ] + }, + { + "id": "dmlex-2", + "headword": "bat", + "partOfSpeech": ["noun"], + "labels": ["masculine"], + "inflectedForms": [ + { + "id": "dmlex-2-f1", + "value": "bat", + "labels": ["singular"] + }, + { + "id": "dmlex-2-f2", + "value": "bats", + "labels": ["plural"] + } + ] + }, + { + "id": "dmlex-3", + "headword": "apple", + "partOfSpeech": "noun", + "pronunciations": [ + { + "transcription": "ˈæp.əl", + "scheme": "IPA" + } + ], + "inflectedForms": [ + { + "id": "dmlex-3-f1", + "value": "apple", + "labels": [ + { "tag": "singular" } + ] + }, + { + "id": "dmlex-3-f2", + "value": "apples", + "labels": [ + { "tag": "plural" } + ] + } + ] + }, + { + "id": "dmlex-4", + "headword": "wow", + "partOfSpeech": ["interjection"] + }, + { + "id": "dmlex-5", + "headword": "London", + "partOfSpeech": ["properNoun"], + "inflectedForms": [ + { + "id": "dmlex-5-f1", + "value": "London", + "labels": ["singular"] + } + ] + }, + { + "id": "dmlex-fr-ignored", + "headword": "bonjour", + "langCode": "fr", + "partOfSpeech": ["interjection"] + } + ] +} diff --git a/tools/dictionary-parser/src/test/resources/dmlexOverlay.json b/tools/dictionary-parser/src/test/resources/dmlexOverlay.json new file mode 100644 index 000000000..c77b9093c --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexOverlay.json @@ -0,0 +1,60 @@ +{ + "uri": "http://example.org/lexicon/en-overlay", + "langCode": "en", + "entries": [ + { + "id": "overlay-new", + "headword": "book", + "partOfSpeech": ["noun"], + "inflectedForms": [ + { + "value": "book", + "labels": ["singular"] + }, + { + "value": "book's", + "labels": ["singular", "genitive"] + }, + { + "value": "books", + "labels": ["plural"] + }, + { + "value": "books'", + "labels": ["plural", "genitive"] + } + ] + }, + { + "id": "overlay-diff-pos", + "headword": "hey", + "partOfSpeech": ["noun"], + "inflectedForms": [ + { + "value": "hey", + "labels": ["singular"] + }, + { + "value": "heys", + "labels": ["plural"] + } + ] + }, + { + "id": "overlay-same-pos-conflict", + "headword": "ciel", + "partOfSpeech": ["noun"], + "labels": ["feminine"], + "inflectedForms": [ + { + "value": "ciel", + "labels": ["singular"] + }, + { + "value": "cieux", + "labels": ["plural"] + } + ] + } + ] +} diff --git a/tools/dictionary-parser/src/test/resources/dmlexPatch.json b/tools/dictionary-parser/src/test/resources/dmlexPatch.json new file mode 100644 index 000000000..1769f0afd --- /dev/null +++ b/tools/dictionary-parser/src/test/resources/dmlexPatch.json @@ -0,0 +1,53 @@ +{ + "uri": "http://example.org/lexicon/en-patch", + "langCode": "en", + "entries": [ + { + "id": "patch-1", + "headword": "cat", + "partOfSpeech": ["noun"], + "labels": ["feminine"], + "inflectedForms": [ + { + "value": "cat", + "labels": ["singular"] + }, + { + "value": "kitties", + "labels": ["plural"] + } + ] + }, + { + "id": "patch-2", + "headword": "wow", + "partOfSpeech": ["verb"], + "inflectedForms": [ + { + "value": "wow", + "labels": [] + }, + { + "value": "wowed", + "labels": [] + } + ] + }, + { + "id": "patch-3", + "headword": "batte", + "partOfSpeech": ["noun"], + "labels": ["feminine"], + "inflectedForms": [ + { + "value": "batte", + "labels": ["singular"] + }, + { + "value": "bats", + "labels": ["plural"] + } + ] + } + ] +} From 013f0d10e7cea7d3b900a11658c22bbd0f92e8d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Neboj=C5=A1a=20=C4=86iri=C4=87?= Date: Thu, 15 Oct 2026 18:13:50 +0000 Subject: [PATCH 2/4] Inflection-208 Address PR review feedback and simplify conflict resolution --- tools/dictionary-parser/README.md | 4 +- .../java/org/unicode/wikidata/DMLexEntry.java | 19 +- .../wikidata/DMLexStringListDeserializer.java | 170 ++------- .../org/unicode/wikidata/DictionaryEntry.java | 264 ++++--------- .../org/unicode/wikidata/DocumentState.java | 22 +- .../java/org/unicode/wikidata/Grammar.java | 110 +----- .../org/unicode/wikidata/ParseWikidata.java | 348 ++++-------------- .../org/unicode/wikidata/ParserOptions.java | 157 ++------ .../unicode/wikidata/ParseWikidataTest.java | 268 ++++++++++---- .../src/test/resources/dmlexLexicon.json | 2 +- 10 files changed, 458 insertions(+), 906 deletions(-) diff --git a/tools/dictionary-parser/README.md b/tools/dictionary-parser/README.md index be1369a66..7e76f58f9 100644 --- a/tools/dictionary-parser/README.md +++ b/tools/dictionary-parser/README.md @@ -43,6 +43,6 @@ These tools generate files that describes the grammatical properties of words fr --dmlex [ ...] ``` - **Conflict Resolution (`--merge-conflicts`)**: - - By default (`--merge-conflicts=false`), later source files take higher precedence per `(surfaceForm, PartOfSpeech)` and `(lemma, PartOfSpeech)` (overwriting conflicting entries of the same part of speech from earlier files while preserving distinct parts of speech). - - Pass `--merge-conflicts=true` (or `--merge-conflicts`) to merge conflicting entries across files instead of overwriting. + - By default (`--merge-conflicts=false`), files are processed in CLI order and a conflicting surface form in a later source file overrides all previous grammemes and inflection patterns for that surface form. + - Pass `--merge-conflicts=true` (or `--merge-conflicts`) to merge grammemes and inflection patterns across files, placing the later source's conflicting pattern first in priority. diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java index 4241d5318..0f89a2ae4 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexEntry.java @@ -16,11 +16,19 @@ */ @JsonIgnoreProperties(ignoreUnknown = true) public class DMLexEntry { + @JsonIgnoreProperties(ignoreUnknown = true) + public static class Lexicon { + public String langCode; + public List entries; + } + public String id; public String headword; public String langCode; - public List partOfSpeech = new ArrayList<>(); + @JsonAlias({"pos"}) + @JsonDeserialize(using = DMLexStringListDeserializer.class) + public List partOfSpeech; public List labels = new ArrayList<>(); @@ -30,15 +38,6 @@ public class DMLexEntry { @JsonAlias({"forms"}) public List inflectedForms; - @JsonSetter("partOfSpeech") - @JsonAlias({"pos"}) - @JsonDeserialize(using = DMLexStringListDeserializer.class) - public void addPartOfSpeech(List incoming) { - if (incoming != null) { - this.partOfSpeech.addAll(incoming); - } - } - @JsonSetter("labels") @JsonAlias({"grammaticalFeatures", "features", "tag", "tags"}) @JsonDeserialize(using = DMLexStringListDeserializer.class) diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java index 26b32ee9c..bf8342c0d 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DMLexStringListDeserializer.java @@ -4,9 +4,9 @@ package org.unicode.wikidata; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.exc.MismatchedInputException; import java.io.IOException; @@ -14,148 +14,54 @@ import java.util.List; /** - * Deserializes OASIS DMLex string or tag/label/pronunciation lists, accepting: - *

    - *
  • a single string (e.g. {@code "noun"})
    • - *
  • an array of strings (e.g. {@code ["singular", "nominative"]})
    • - *
  • an object or array of objects containing {@code "tag"}, {@code "value"}, - * {@code "transcription"}, {@code "text"}, or {@code "label"}
    • - *
+ * Deserializes OASIS DMLex string or tag/label/pronunciation lists, accepting a single string, + * an array of strings, or an object/array of objects containing {@code "transcription"}, + * {@code "tag"}, {@code "value"}, {@code "label"}, or {@code "text"}. */ public class DMLexStringListDeserializer extends JsonDeserializer> { + private static final String[] OBJECT_VALUE_KEYS = { + "transcription", "tag", "value", "label", "text" + }; + @Override public List deserialize(JsonParser jsonParser, DeserializationContext ctxt) throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); List result = new ArrayList<>(); - JsonToken currentToken = jsonParser.currentToken(); - if (currentToken == JsonToken.VALUE_STRING) { - String text = jsonParser.getText(); - if (text != null && !text.isBlank()) { - result.add(text); - } - return result; - } - if (currentToken == JsonToken.START_OBJECT) { - readValuesFromObject(jsonParser, result); - return result; - } - if (currentToken != JsonToken.START_ARRAY) { - throw MismatchedInputException.from( - jsonParser, - List.class, - "Expected string, object, or array for DMLex list field, got " + currentToken); - } - - while (jsonParser.nextToken() != JsonToken.END_ARRAY) { - JsonToken elemToken = jsonParser.currentToken(); - if (elemToken == JsonToken.VALUE_STRING) { - String text = jsonParser.getText(); - if (text != null && !text.isBlank()) { - result.add(text); - } - } else if (elemToken == JsonToken.START_OBJECT) { - readValuesFromObject(jsonParser, result); - } else { - throw MismatchedInputException.from( - jsonParser, - String.class, - "Expected string or object element in DMLex array, got " + elemToken); - } - } + extractValues(node, result, jsonParser); return result; } - /** - * Extracts value(s) from a single JSON object. When an object carries multiple scalar string - * fields (for example {@code {"tag": "singular", "label": "Singular number"}} or - * {@code {"label": "US", "transcription": "ˈæp.əl"}}), only the highest-priority non-blank - * scalar field ({@code "transcription"} → {@code "tag"} → {@code "value"} → - * {@code "label"} → {@code "text"}) is selected, while also supporting nested - * {@code "transcriptions"} arrays/objects. - */ - private static void readValuesFromObject(JsonParser jsonParser, List result) throws IOException { - String transcription = null; - String tag = null; - String value = null; - String label = null; - String text = null; - List nestedTranscriptions = null; - - while (jsonParser.nextToken() != JsonToken.END_OBJECT) { - String fieldName = jsonParser.currentName(); - JsonToken valueToken = jsonParser.nextToken(); - if (valueToken == JsonToken.VALUE_STRING) { - String str = jsonParser.getText(); - if (str != null && !str.isBlank()) { - switch (fieldName) { - case "transcription" -> { - if (transcription == null) { - transcription = str; - } - } - case "tag" -> { - if (tag == null) { - tag = str; - } - } - case "value" -> { - if (value == null) { - value = str; - } - } - case "label" -> { - if (label == null) { - label = str; - } - } - case "text" -> { - if (text == null) { - text = str; - } - } - default -> {} - } - } - } else if ("transcriptions".equals(fieldName) || "transcription".equals(fieldName)) { - if (nestedTranscriptions == null) { - nestedTranscriptions = new ArrayList<>(); - } - if (valueToken == JsonToken.START_OBJECT) { - readValuesFromObject(jsonParser, nestedTranscriptions); - } else if (valueToken == JsonToken.START_ARRAY) { - while (jsonParser.nextToken() != JsonToken.END_ARRAY) { - JsonToken elemToken = jsonParser.currentToken(); - if (elemToken == JsonToken.VALUE_STRING) { - String str = jsonParser.getText(); - if (str != null && !str.isBlank()) { - nestedTranscriptions.add(str); - } - } else if (elemToken == JsonToken.START_OBJECT) { - readValuesFromObject(jsonParser, nestedTranscriptions); - } else if (elemToken == JsonToken.START_ARRAY) { - jsonParser.skipChildren(); - } + private static void extractValues(JsonNode node, List result, JsonParser jsonParser) throws IOException { + if (node == null || node.isNull()) { + return; + } + if (node.isTextual()) { + String text = node.textValue(); + if (!text.isBlank()) { + result.add(text); + } + } else if (node.isArray()) { + for (JsonNode elem : node) { + extractValues(elem, result, jsonParser); + } + } else if (node.isObject()) { + JsonNode transcriptions = node.get("transcriptions"); + if (transcriptions != null) { + extractValues(transcriptions, result, jsonParser); + } else { + for (String key : OBJECT_VALUE_KEYS) { + JsonNode field = node.get(key); + if (field != null && field.isTextual() && !field.textValue().isBlank()) { + result.add(field.textValue()); + break; } } - } else if (valueToken == JsonToken.START_OBJECT || valueToken == JsonToken.START_ARRAY) { - jsonParser.skipChildren(); - } - } - - if (transcription != null) { - result.add(transcription); - if (nestedTranscriptions != null) { - result.addAll(nestedTranscriptions); } - } else if (nestedTranscriptions != null && !nestedTranscriptions.isEmpty()) { - result.addAll(nestedTranscriptions); - } else if (tag != null) { - result.add(tag); - } else if (value != null) { - result.add(value); - } else if (label != null) { - result.add(label); - } else if (text != null) { - result.add(text); + } else { + throw MismatchedInputException.from( + jsonParser, + List.class, + "Expected string, object, or array for DMLex list field, got " + node.getNodeType()); } } } diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java index 5260ddb5c..2cacfed1a 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java @@ -5,9 +5,9 @@ package org.unicode.wikidata; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; -import java.util.EnumSet; -import java.util.LinkedHashMap; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -15,56 +15,20 @@ import javax.annotation.Nonnull; /** - * A dictionary entry represents a single surface form ({@code phrase}) in the generated dictionary, - * along with its combined grammatical properties and references to inflection patterns. - * - *

To support multi-source inputs with fine-grained conflict resolution, a {@code DictionaryEntry} - * partitions its grammemes and inflection patterns into per-{@link Grammar.PartOfSpeech} slices - * ({@link #posSlices}) rather than storing a single flat set. This allows a higher-priority source - * file (higher {@code sourceIndex}) to override a word's reading for one part of speech (e.g., {@code noun}) - * without clobbering readings for other parts of speech (e.g., {@code verb} or {@code adjective}) - * defined in earlier source files. - * - *

Rare inflections are sorted last when serialized so that they remain accessible, - * but are not directly referenceable by default when multiple inflections share the same + * A dictionary entry has a lemma and a set of inflections. The lemma may have additional part of speech information + * that should be included in the inflections. Rare inflections are sorted last so that they're accessible, + * but not directly referencable by default, which is important when there may be more than one inflection with the same * set of grammemes. */ public class DictionaryEntry implements Comparable { - /** - * Holds the grammatical features and inflection patterns contributed for a single - * {@link Grammar.PartOfSpeech} (or for a POS-less entry) from a specific {@link #sourceIndex}. - */ - private static final class PosSlice { - final int sourceIndex; - final TreeSet> grammemes; - final List inflectionalPatterns = new ArrayList<>(2); - final List rareInflectionalPatterns = new ArrayList<>(); - - PosSlice(int sourceIndex, Comparator> comparator) { - this.sourceIndex = sourceIndex; - this.grammemes = new TreeSet<>(comparator); - } - - PosSlice copy() { - PosSlice copy = new PosSlice(this.sourceIndex, this.grammemes.comparator()); - copy.grammemes.addAll(this.grammemes); - copy.inflectionalPatterns.addAll(this.inflectionalPatterns); - copy.rareInflectionalPatterns.addAll(this.rareInflectionalPatterns); - return copy; - } - } - final String phrase; - private final Comparator> comparator; - // LinkedHashMap preserves PartOfSpeech insertion order so tie-breaking between - // equal-frequency InflectionPatterns remains deterministic. - private final Map posSlices = new LinkedHashMap<>(); - // Holds grammemes and patterns for entries that have no PartOfSpeech tag. - private PosSlice poslessSlice = null; - // Holds POS-less supplemental grammemes injected via --add-extra-grammemes, - // which are always unioned regardless of source precedence. - private final TreeSet> extraGrammemes; + private final TreeSet lemmas; + private final TreeSet> grammemes; + private final List inflectionalPatterns; + private final List rareInflectionalPatterns = new ArrayList<>(); + private int sourceIndex; + private Map patternSourceIndices = null; public static final Comparator INFLECTION_PATTERN_COMPARATOR = Comparator .comparing(InflectionPattern::getCount) @@ -80,121 +44,85 @@ public DictionaryEntry(String phrase, String lemmas, boolean isLemmaRare, TreeSe public DictionaryEntry(String phrase, String lemmas, boolean isLemmaRare, TreeSet> grammemes, InflectionPattern inflectionalPattern, int sourceIndex) { this.phrase = phrase; - this.comparator = Inflection.ENUM_COMPARATOR; - this.extraGrammemes = new TreeSet<>(this.comparator); - - // Separate PartOfSpeech values from all other grammatical categories (gender, number, case, etc.) - // so that each PosSlice only stores its own PartOfSpeech plus the shared non-POS grammemes. - // If a multi-POS entry (e.g. noun + proper-noun) later has only one POS overridden by a - // higher-priority source, the remaining POS slice will not retain a stale POS tag. - EnumSet posSet = EnumSet.noneOf(Grammar.PartOfSpeech.class); - TreeSet> nonPosGrammemes = new TreeSet<>(this.comparator); - for (Enum g : grammemes) { - if (g instanceof Grammar.PartOfSpeech pos) { - posSet.add(pos); - } else { - nonPosGrammemes.add(g); + this.lemmas = new TreeSet<>(Collections.singleton(lemmas)); + this.grammemes = new TreeSet<>(grammemes.comparator()); + this.grammemes.addAll(grammemes); + this.sourceIndex = sourceIndex; + if (inflectionalPattern != null) { + this.inflectionalPatterns = new ArrayList<>(2); + this.inflectionalPatterns.add(inflectionalPattern); + if (isLemmaRare) { + rareInflectionalPatterns.add(inflectionalPattern); } } - - if (posSet.isEmpty()) { - this.poslessSlice = new PosSlice(sourceIndex, this.comparator); - this.poslessSlice.grammemes.addAll(nonPosGrammemes); - if (inflectionalPattern != null) { - this.poslessSlice.inflectionalPatterns.add(inflectionalPattern); - if (isLemmaRare) { - this.poslessSlice.rareInflectionalPatterns.add(inflectionalPattern); - } - } - } else { - for (Grammar.PartOfSpeech pos : posSet) { - PosSlice slice = new PosSlice(sourceIndex, this.comparator); - slice.grammemes.addAll(nonPosGrammemes); - slice.grammemes.add(pos); - if (inflectionalPattern != null) { - slice.inflectionalPatterns.add(inflectionalPattern); - if (isLemmaRare) { - slice.rareInflectionalPatterns.add(inflectionalPattern); - } - } - this.posSlices.put(pos, slice); - } + else { + this.inflectionalPatterns = new ArrayList<>(); } } - /** - * Unions the grammemes and deduplicates inflection patterns from {@code incoming} into {@code target}. - * Used when two entries for the same surface form and {@link Grammar.PartOfSpeech} originate from - * the same source file (or when {@code --merge-conflicts=true} is enabled). - */ - private static void mergeSlice(PosSlice target, PosSlice incoming) { - target.grammemes.addAll(incoming.grammemes); - for (InflectionPattern inflectionalPattern : incoming.inflectionalPatterns) { - if (!target.inflectionalPatterns.contains(inflectionalPattern)) { - target.inflectionalPatterns.add(inflectionalPattern); - } - } - for (InflectionPattern inflectionalPattern : incoming.rareInflectionalPatterns) { - if (!target.rareInflectionalPatterns.contains(inflectionalPattern)) { - target.rareInflectionalPatterns.add(inflectionalPattern); - } + private int getPatternSourceIndex(InflectionPattern pattern) { + if (patternSourceIndices != null) { + return patternSourceIndices.getOrDefault(pattern, sourceIndex); } + return sourceIndex; } - /** - * Merges another {@code DictionaryEntry} for the same surface form into this entry. - * - *

Conflict resolution is performed independently for each {@link Grammar.PartOfSpeech} slice: - *

    - *
  1. New POS or higher-priority source ({@code incomingSlice.sourceIndex > existingSlice.sourceIndex}): - * Replaces the existing slice for that POS with the incoming slice, leaving other POS slices intact.
    2. - *
  2. Same-priority source ({@code incomingSlice.sourceIndex == existingSlice.sourceIndex}, - * or when {@code --merge-conflicts=true} normalizes all indices to {@code 0}): - * Unions grammemes and inflection patterns via {@link #mergeSlice(PosSlice, PosSlice)}, preserving - * intra-file homograph and syncretism merging.
    3. - *
  3. Lower-priority source ({@code incomingSlice.sourceIndex < existingSlice.sourceIndex}): - * Ignored for that POS.
    4. - *
- */ public void merge(DictionaryEntry entry) { - for (Map.Entry incomingEntry : entry.posSlices.entrySet()) { - Grammar.PartOfSpeech pos = incomingEntry.getKey(); - PosSlice incomingSlice = incomingEntry.getValue(); - PosSlice existingSlice = this.posSlices.get(pos); - if (existingSlice == null || incomingSlice.sourceIndex > existingSlice.sourceIndex) { - this.posSlices.put(pos, incomingSlice.copy()); - } else if (incomingSlice.sourceIndex == existingSlice.sourceIndex) { - mergeSlice(existingSlice, incomingSlice); + merge(entry, false); + } + + public void merge(DictionaryEntry entry, boolean mergeConflicts) { + if (!mergeConflicts) { + if (entry.sourceIndex > this.sourceIndex) { + this.sourceIndex = entry.sourceIndex; + this.grammemes.clear(); + this.grammemes.addAll(entry.grammemes); + this.inflectionalPatterns.clear(); + this.inflectionalPatterns.addAll(entry.inflectionalPatterns); + this.rareInflectionalPatterns.clear(); + this.rareInflectionalPatterns.addAll(entry.rareInflectionalPatterns); + this.lemmas.clear(); + this.lemmas.addAll(entry.lemmas); + this.patternSourceIndices = null; + return; } + if (entry.sourceIndex < this.sourceIndex) { + return; + } + } else if (entry.sourceIndex != this.sourceIndex || this.patternSourceIndices != null || entry.patternSourceIndices != null) { + if (this.patternSourceIndices == null) { + this.patternSourceIndices = new IdentityHashMap<>(); + for (InflectionPattern pattern : this.inflectionalPatterns) { + this.patternSourceIndices.put(pattern, this.sourceIndex); + } + } + for (InflectionPattern pattern : entry.inflectionalPatterns) { + this.patternSourceIndices.merge(pattern, entry.getPatternSourceIndex(pattern), Math::max); + } + this.sourceIndex = Math.max(this.sourceIndex, entry.sourceIndex); } - if (entry.poslessSlice != null) { - if (this.poslessSlice == null || entry.poslessSlice.sourceIndex > this.poslessSlice.sourceIndex) { - this.poslessSlice = entry.poslessSlice.copy(); - } else if (entry.poslessSlice.sourceIndex == this.poslessSlice.sourceIndex) { - mergeSlice(this.poslessSlice, entry.poslessSlice); + + this.grammemes.addAll(entry.grammemes); + for (InflectionPattern inflectionalPattern : entry.inflectionalPatterns) { + boolean isRareInEntry = entry.rareInflectionalPatterns.contains(inflectionalPattern); + if (!this.inflectionalPatterns.contains(inflectionalPattern)) { + this.inflectionalPatterns.add(inflectionalPattern); + if (isRareInEntry) { + this.rareInflectionalPatterns.add(inflectionalPattern); + } + } else if (!isRareInEntry) { + this.rareInflectionalPatterns.remove(inflectionalPattern); } } - this.extraGrammemes.addAll(entry.extraGrammemes); + this.lemmas.addAll(entry.lemmas); } - public void addExtraGrammemes(Set> grammemes) { - this.extraGrammemes.addAll(grammemes); + public Set> getGrammemes() { + return grammemes; } - /** - * Recombines grammemes across all active {@link PosSlice}s, {@link #poslessSlice}, and - * {@link #extraGrammemes} into a single sorted set ordered by {@link Inflection#ENUM_COMPARATOR}. - */ - public Set> getGrammemes() { - TreeSet> combined = new TreeSet<>(this.comparator); - for (PosSlice slice : posSlices.values()) { - combined.addAll(slice.grammemes); - } - if (poslessSlice != null) { - combined.addAll(poslessSlice.grammemes); - } - combined.addAll(extraGrammemes); - return combined; + List getInflectionalPatterns() { + return inflectionalPatterns; } @Override @@ -202,52 +130,22 @@ public String toString() { return this.toString(true); } - /** - * Formats this entry as a single line for the output {@code dictionary_*.lst} file: - * {@code : [inflection= ...]}. - * - *

When {@code isInflectional} is {@code true}, inflection patterns from all active slices - * are deduplicated in slice insertion order, sorted by descending usage frequency - * ({@link #INFLECTION_PATTERN_COMPARATOR}), and emitted with non-rare patterns preceding rare patterns. - */ public String toString(boolean isInflectional) { StringBuilder sb = new StringBuilder(256); sb.append(phrase).append(':'); - for (Enum enumVal : getGrammemes()) { + for (Enum enumVal : grammemes) { sb.append(' ').append(enumVal); } if (isInflectional) { - List inflectionalPatterns = new ArrayList<>(); - List rareInflectionalPatterns = new ArrayList<>(); - for (PosSlice slice : posSlices.values()) { - for (InflectionPattern pattern : slice.inflectionalPatterns) { - if (!inflectionalPatterns.contains(pattern)) { - inflectionalPatterns.add(pattern); - } - } - for (InflectionPattern pattern : slice.rareInflectionalPatterns) { - if (!rareInflectionalPatterns.contains(pattern)) { - rareInflectionalPatterns.add(pattern); - } - } - } - if (poslessSlice != null) { - for (InflectionPattern pattern : poslessSlice.inflectionalPatterns) { - if (!inflectionalPatterns.contains(pattern)) { - inflectionalPatterns.add(pattern); - } - } - for (InflectionPattern pattern : poslessSlice.rareInflectionalPatterns) { - if (!rareInflectionalPatterns.contains(pattern)) { - rareInflectionalPatterns.add(pattern); - } - } - } - + Comparator patternComparator = (patternSourceIndices == null) + ? INFLECTION_PATTERN_COMPARATOR + : Comparator.comparing( + p -> patternSourceIndices.getOrDefault(p, sourceIndex), Comparator.reverseOrder()) + .thenComparing(INFLECTION_PATTERN_COMPARATOR); if (inflectionalPatterns.size() > 1) { - inflectionalPatterns.sort(INFLECTION_PATTERN_COMPARATOR); + inflectionalPatterns.sort(patternComparator); if (rareInflectionalPatterns.size() > 1) { - rareInflectionalPatterns.sort(INFLECTION_PATTERN_COMPARATOR); + rareInflectionalPatterns.sort(patternComparator); } } for (InflectionPattern inflectionPattern : inflectionalPatterns) { diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java index b43586cac..be3304cc3 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DocumentState.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; +import java.util.IdentityHashMap; import java.util.List; import java.util.Locale; import java.util.Map; @@ -49,17 +50,31 @@ private void sortInflectionPatterns(ArrayList inflectionPatte } public void addDictionaryEntry(DictionaryEntry dictionaryEntry) { + addDictionaryEntry(dictionaryEntry, false); + } + + public void addDictionaryEntry(DictionaryEntry dictionaryEntry, boolean mergeConflicts) { String phrase = dictionaryEntry.phrase; DictionaryEntry existingDictionaryEntry = dictionary.get(phrase); if (existingDictionaryEntry == null) { dictionary.put(phrase, dictionaryEntry); } else { mergedCount++; - existingDictionaryEntry.merge(dictionaryEntry); + existingDictionaryEntry.merge(dictionaryEntry, mergeConflicts); } } void printDocument(ParserOptions parserOptions, long startTime) throws FileNotFoundException { + if (!inflectionPatterns.isEmpty()) { + Set referencedPatterns = Collections.newSetFromMap(new IdentityHashMap<>()); + for (DictionaryEntry dictionaryEntry : dictionary.values()) { + if (!dictionaryEntry.getGrammemes().isEmpty()) { + referencedPatterns.addAll(dictionaryEntry.getInflectionalPatterns()); + } + } + inflectionPatterns.removeIf(pattern -> !referencedPatterns.contains(pattern)); + } + TreeMap, Integer> grammemeCounts = new TreeMap<>(EnumComparator.ENUM_COMPARATOR); int unclassifiedTerms = 0; if (isInflectional()) { @@ -79,15 +94,14 @@ void printDocument(ParserOptions parserOptions, long startTime) throws FileNotFo new FileOutputStream(parserOptions.lexicalDictionaryFilename), StandardCharsets.UTF_8))) { for (Map.Entry entry : dictionary.entrySet()) { DictionaryEntry dictionaryEntry = entry.getValue(); - Set> grammemes = dictionaryEntry.getGrammemes(); - if (grammemes.isEmpty()) { + if (dictionaryEntry.getGrammemes().isEmpty()) { // We don't care about only known words. We need grammeme data unclassifiedTerms++; continue; } // Print the dictionary entry to the .lst file. lexicalDictionaryStream.println(dictionaryEntry.toString(isInflectional())); - for (Enum grammeme : grammemes) { + for (Enum grammeme : dictionaryEntry.getGrammemes()) { grammemeCounts.merge(grammeme, 1, Integer::sum); } } diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java index 4e55dc6ce..4f72f4375 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/Grammar.java @@ -8,7 +8,6 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; -import java.util.Locale; import java.util.Map; import java.util.Set; @@ -1177,114 +1176,17 @@ public String toString() { TYPEMAP.put("Q694268", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // collective TYPEMAP.put("Q1450795", EnumSet.of(Ignorable.IGNORABLE_PROPERTY)); // singulative - // Seed TYPEMAP with all canonical grammeme names from DEFAULTMAP (e.g. "noun", "singular", - // "proper-noun") so both Wikidata Q-IDs and DMLex string labels can be resolved through - // a single read-only lookup table. - DEFAULTMAP.forEach((key, value) -> TYPEMAP.putIfAbsent(key, Set.of(value))); - - // Standard OASIS DMLex and Universal Dependencies / morphological shorthand tag aliases. - // Part-of-speech abbreviations: - TYPEMAP.putIfAbsent("propn", EnumSet.of(PartOfSpeech.PROPER_NOUN)); - TYPEMAP.putIfAbsent("adj", EnumSet.of(PartOfSpeech.ADJECTIVE)); - TYPEMAP.putIfAbsent("adv", EnumSet.of(PartOfSpeech.ADVERB)); - TYPEMAP.putIfAbsent("adp", EnumSet.of(PartOfSpeech.ADPOSITION)); - TYPEMAP.putIfAbsent("prep", EnumSet.of(PartOfSpeech.PREPOSITION)); - TYPEMAP.putIfAbsent("conj", EnumSet.of(PartOfSpeech.CONJUNCTION)); - TYPEMAP.putIfAbsent("cconj", EnumSet.of(PartOfSpeech.CONJUNCTION)); - TYPEMAP.putIfAbsent("sconj", EnumSet.of(PartOfSpeech.CONJUNCTION)); - TYPEMAP.putIfAbsent("det", EnumSet.of(PartOfSpeech.DETERMINER)); - TYPEMAP.putIfAbsent("intj", EnumSet.of(PartOfSpeech.INTERJECTION)); - TYPEMAP.putIfAbsent("num", EnumSet.of(PartOfSpeech.NUMERAL)); - TYPEMAP.putIfAbsent("part", EnumSet.of(PartOfSpeech.PARTICLE)); - TYPEMAP.putIfAbsent("pron", EnumSet.of(PartOfSpeech.PRONOUN)); - - // Person aliases: - TYPEMAP.putIfAbsent("first-person", EnumSet.of(Person.FIRST)); - TYPEMAP.putIfAbsent("1st-person", EnumSet.of(Person.FIRST)); - TYPEMAP.putIfAbsent("second-person", EnumSet.of(Person.SECOND)); - TYPEMAP.putIfAbsent("2nd-person", EnumSet.of(Person.SECOND)); - TYPEMAP.putIfAbsent("third-person", EnumSet.of(Person.THIRD)); - TYPEMAP.putIfAbsent("3rd-person", EnumSet.of(Person.THIRD)); - - // Number abbreviations: - TYPEMAP.putIfAbsent("sg", EnumSet.of(Number.SINGULAR)); - TYPEMAP.putIfAbsent("sing", EnumSet.of(Number.SINGULAR)); - TYPEMAP.putIfAbsent("pl", EnumSet.of(Number.PLURAL)); - TYPEMAP.putIfAbsent("plur", EnumSet.of(Number.PLURAL)); - - // Gender abbreviations: - TYPEMAP.putIfAbsent("masc", EnumSet.of(Gender.MASCULINE)); - TYPEMAP.putIfAbsent("fem", EnumSet.of(Gender.FEMININE)); - TYPEMAP.putIfAbsent("neut", EnumSet.of(Gender.NEUTER)); - TYPEMAP.putIfAbsent("common-gender", EnumSet.of(Gender.COMMON)); - - // Case abbreviations: - TYPEMAP.putIfAbsent("nom", EnumSet.of(Case.NOMINATIVE)); - TYPEMAP.putIfAbsent("acc", EnumSet.of(Case.ACCUSATIVE)); - TYPEMAP.putIfAbsent("gen", EnumSet.of(Case.GENITIVE)); - TYPEMAP.putIfAbsent("dat", EnumSet.of(Case.DATIVE)); - TYPEMAP.putIfAbsent("ins", EnumSet.of(Case.INSTRUMENTAL)); - TYPEMAP.putIfAbsent("inst", EnumSet.of(Case.INSTRUMENTAL)); - TYPEMAP.putIfAbsent("loc", EnumSet.of(Case.LOCATIVE)); - TYPEMAP.putIfAbsent("voc", EnumSet.of(Case.VOCATIVE)); - TYPEMAP.putIfAbsent("abl", EnumSet.of(Case.ABLATIVE)); - - // Definiteness abbreviations: - TYPEMAP.putIfAbsent("def", EnumSet.of(Definiteness.DEFINITE)); - TYPEMAP.putIfAbsent("indef", EnumSet.of(Definiteness.INDEFINITE)); - TYPEMAP.putIfAbsent("construct-state", EnumSet.of(Definiteness.CONSTRUCT)); - - // Hyphenated aliases for compound enums declared without underscores: - TYPEMAP.putIfAbsent("non-finite", EnumSet.of(VerbType.NONFINITE)); - TYPEMAP.putIfAbsent("non-human", EnumSet.of(Animacy.NONHUMAN)); - } - - /** - * Normalizes a grammeme label into canonical lowercase hyphenated form (e.g., converting - * camelCase {@code "properNoun"}, snake_case {@code "proper_noun"}, or uppercase - * {@code "PROPER_NOUN"} to {@code "proper-noun"}). - */ - static String normalizeGrammemeKey(String grammeme) { - if (grammeme == null) { - return ""; + for (var entry : DEFAULTMAP.entrySet()) { + @SuppressWarnings({"unchecked", "rawtypes"}) + Set> enumSet = EnumSet.of((Enum) entry.getValue()); + TYPEMAP.putIfAbsent(entry.getKey(), enumSet); } - String trimmed = grammeme.trim(); - StringBuilder sb = new StringBuilder(trimmed.length() + 4); - for (int i = 0; i < trimmed.length(); i++) { - char c = trimmed.charAt(i); - if (c == '_' || Character.isWhitespace(c)) { - if (sb.length() > 0 && sb.charAt(sb.length() - 1) != '-') { - sb.append('-'); - } - } else if (Character.isUpperCase(c)) { - if (i > 0) { - char prev = trimmed.charAt(i - 1); - if (Character.isLowerCase(prev) || Character.isDigit(prev)) { - sb.append('-'); - } - } - sb.append(Character.toLowerCase(c)); - } else { - sb.append(Character.toLowerCase(c)); - } - } - return sb.toString().toLowerCase(Locale.ROOT); } - /** - * Looks up the default mapped grammeme enums for a Wikidata Q-ID or DMLex grammatical label - * in the read-only {@link #TYPEMAP} table. Checks for an exact match first (fast path for - * Q-IDs like {@code "Q1084"} and canonical strings like {@code "proper-noun"}), then falls - * back to a normalized key lookup via {@link #normalizeGrammemeKey(String)}. - */ static Set> getMappedGrammemes(String grammeme) { if (grammeme == null) { - return null; - } - Set> mapped = TYPEMAP.get(grammeme); - if (mapped != null) { - return mapped; + throw new IllegalArgumentException("Grammeme cannot be null"); } - return TYPEMAP.get(normalizeGrammemeKey(grammeme)); + return TYPEMAP.get(grammeme); } } diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java index d7e82375f..3356f358a 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParseWikidata.java @@ -25,17 +25,16 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; -import java.util.EnumMap; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.HashMap; import java.util.TreeSet; +import java.util.Arrays; import static org.unicode.wikidata.Grammar.Gender; import static org.unicode.wikidata.Grammar.Ignorable; import static org.unicode.wikidata.Grammar.PartOfSpeech; @@ -66,12 +65,17 @@ static class Lemma { final TreeSet> grammemes = new TreeSet<>(EnumComparator.ENUM_COMPARATOR); final List inflections = new ArrayList<>(64); + void reset() { + value = null; + isRare = false; + grammemes.clear(); + inflections.clear(); + } + private Lemma() { } } - private record StagedLemma(Lemma lemma, int sourceIndex) {} - private final ParserOptions parserOptions; private final DocumentState documentState = new DocumentState(); private final TreeSet rareLemmas = new TreeSet<>(); @@ -79,14 +83,6 @@ private record StagedLemma(Lemma lemma, int sourceIndex) {} private final Map> mergeMap = new HashMap<>(); private final TreeSet deferredLexemes = new TreeSet<>(); private final Map deferredLemmaMap = new HashMap<>(); - // Holds normalized lemmas in encounter order when multi-source overwrite mode is active - // (--merge-conflicts=false with >1 source file). - private final List stagedLemmas = new ArrayList<>(); - // Tracks the highest sourceIndex that defines a given (lemma.value, PartOfSpeech) pair - // so earlier paradigms for the same lemma and POS can be preempted before pattern generation. - private final Map> highestSourceForLemmaPos = new HashMap<>(); - // Tracks the highest sourceIndex that defines a POS-less lemma for a given headword. - private final Map highestSourceForPoslessLemma = new HashMap<>(); ParseWikidata(ParserOptions parserOptions) { this.parserOptions = parserOptions; @@ -131,102 +127,9 @@ private record StagedLemma(Lemma lemma, int sourceIndex) {} static final String VARIANT_SEPARATOR = "-x-"; - /** - * Returns {@code true} when multiple input files are provided and {@code --merge-conflicts} is {@code false} - * (the default). In this mode, later files on the CLI override conflicting entries of the same - * {@link PartOfSpeech} from earlier files via a two-tier strategy: - *

    - *
  • Tier 1 (Lemma/Paradigm level): {@link #recordOrAnalyzeLemma} stages lemmas and - * {@link #processStagedLemmas} strips any {@code (lemma.value, PartOfSpeech)} superseded by a later - * source before {@link #analyzeInflections} runs. This prevents removed inflected surface forms - * (e.g., an obsolete plural dropped in a patch file) from lingering as orphans in {@code dictionary.lst} - * and prevents superseded paradigms from inflating {@link InflectionPattern} counts in {@code inflectional.xml}.
    • - *
  • Tier 2 (Surface-form level): {@link DictionaryEntry#merge} resolves remaining surface-form - * collisions per {@link PartOfSpeech} using {@code sourceIndex}.
    • - *
- */ - private boolean isMultiSourceOverwriteMode() { - return !parserOptions.mergeConflicts && parserOptions.sourceInputs.size() > 1; - } - - /** - * Either analyzes a normalized lemma immediately (single-source or {@code --merge-conflicts=true}, using - * {@code effectiveSourceIndex = 0} so all sources merge identically) or stages it for Tier 1 - * cross-source lemma preemption when {@link #isMultiSourceOverwriteMode()} is active. - */ - private void recordOrAnalyzeLemma(Lemma lemma, int sourceIndex) { - if (!isMultiSourceOverwriteMode()) { - analyzeLemma(lemma, 0); - return; - } - stagedLemmas.add(new StagedLemma(lemma, sourceIndex)); - boolean hasPos = false; - for (Enum g : lemma.grammemes) { - if (g instanceof PartOfSpeech pos) { - hasPos = true; - highestSourceForLemmaPos - .computeIfAbsent(lemma.value, k -> new EnumMap<>(PartOfSpeech.class)) - .merge(pos, sourceIndex, Math::max); - } - } - if (!hasPos) { - highestSourceForPoslessLemma.merge(lemma.value, sourceIndex, Math::max); - } - } - - /** - * Executes Tier 1 lemma preemption across all staged lemmas after all source files have been read. - * For each staged lemma, any {@link PartOfSpeech} (or POS-less reading) that appears in a later - * source file for the same headword ({@code highestSource > sourceIndex}) is stripped from the - * lemma and its inflections; if all of the lemma's parts of speech were superseded, the earlier - * lemma is skipped entirely. - */ - private void processStagedLemmas() { - for (StagedLemma staged : stagedLemmas) { - Lemma lemma = staged.lemma(); - int sourceIndex = staged.sourceIndex(); - EnumSet originalPos = EnumSet.noneOf(PartOfSpeech.class); - for (Enum g : lemma.grammemes) { - if (g instanceof PartOfSpeech pos) { - originalPos.add(pos); - } - } - if (originalPos.isEmpty()) { - Integer highestPoslessSource = highestSourceForPoslessLemma.get(lemma.value); - if (highestPoslessSource != null && highestPoslessSource > sourceIndex) { - // This POS-less lemma was superseded by a later POS-less entry for the same headword. - continue; - } - } else { - EnumMap posMap = highestSourceForLemmaPos.get(lemma.value); - if (posMap != null) { - EnumSet supersededPos = EnumSet.noneOf(PartOfSpeech.class); - for (PartOfSpeech pos : originalPos) { - Integer highestSource = posMap.get(pos); - if (highestSource != null && highestSource > sourceIndex) { - supersededPos.add(pos); - } - } - if (!supersededPos.isEmpty()) { - if (supersededPos.size() == originalPos.size()) { - // Every PartOfSpeech of this lemma was superseded by a later source. - continue; - } - lemma.grammemes.removeAll(supersededPos); - for (Inflection inflection : lemma.inflections) { - inflection.getGrammemeSet().removeAll(supersededPos); - } - } - } - } - analyzeLemma(lemma, sourceIndex); - } - } - /** * Finalizes a single surface-form inflection with lemma grammeme propagation, ignorable filtering, - * and rarity extraction. Shared by both Wikidata and DMLex paths. {@code --expand-grammemes} - * expansion is deferred to {@link #expandAndCountInflections(Lemma)} at the start of {@link #analyzeLemma}. + * rarity extraction, and {@code --expand-grammemes} expansion. Shared by both Wikidata and DMLex paths. * * @return {@code false} if the entire lemma must be aborted due to {@code IGNORABLE_LEMMA}; {@code true} otherwise. */ @@ -234,14 +137,11 @@ private boolean finalizeAndAddInflection( Lemma lemma, @Nullable TreeSet> genderlessLemmaGrammemes, Inflection currentInflection) { - // 1. Propagate lemma-level grammemes to the surface form. If the surface form specifies its - // own Gender, inherit genderlessLemmaGrammemes so form-level Gender overrides lemma Gender. if (genderlessLemmaGrammemes == null || countGrammemeType(currentInflection.grammemeSet, Gender.class) == 0) { currentInflection.grammemeSet.addAll(lemma.grammemes); } else { currentInflection.grammemeSet.addAll(genderlessLemmaGrammemes); } - // 2. Check for grammemes configured to discard the entire lemma or this specific surface form. if (currentInflection.grammemeSet.contains(Ignorable.IGNORABLE_LEMMA)) { documentState.unusableLemmaCount++; return false; @@ -250,23 +150,35 @@ private boolean finalizeAndAddInflection( documentState.unusableSurfaceFormCount++; return true; } - // 3. Extract Usage.RARE into the boolean flag on Inflection and strip IGNORABLE_PROPERTY. currentInflection.rareUsage = currentInflection.grammemeSet.contains(Grammar.Usage.RARE); if (currentInflection.rareUsage) { currentInflection.grammemeSet.remove(Grammar.Usage.RARE); } currentInflection.grammemeSet.remove(Ignorable.IGNORABLE_PROPERTY); - lemma.inflections.add(currentInflection); + var grammemeExpansion = parserOptions.expandGramemes != null + ? parserOptions.expandGramemes.get(currentInflection.grammemeSet) + : null; + if (grammemeExpansion == null) { + lemma.inflections.add(currentInflection); + } else { + for (var grammemeSet : grammemeExpansion) { + var expandedInflection = new Inflection(currentInflection.inflection, + currentInflection.rareUsage); + expandedInflection.grammemeSet.addAll(currentInflection.grammemeSet); + expandedInflection.grammemeSet.addAll(grammemeSet); + lemma.inflections.add(expandedInflection); + } + } return true; } /** - * Performs post-form lemma cleanup shared by Wikidata and DMLex: extracts lemma-level - * {@link Grammar.Usage#RARE}, rejects lemmas with no usable surface forms, and strips {@link Gender} - * from {@code lemma.grammemes} (since forms have already inherited gender and {@code } - * elements in {@code inflectional.xml} do not include gender). + * Performs post-form lemma cleanup shared by Wikidata and DMLex: counts incoming surface forms, + * extracts lemma-level {@link Grammar.Usage#RARE}, rejects lemmas with no usable surface forms, + * and strips {@link Gender} from {@code lemma.grammemes}. */ private boolean finalizeLemmaAfterForms(Lemma lemma) { + documentState.incomingSurfaceForm += lemma.inflections.size(); lemma.isRare = lemma.grammemes.contains(Grammar.Usage.RARE); if (lemma.isRare) { lemma.grammemes.remove(Grammar.Usage.RARE); @@ -284,11 +196,12 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme, int sourceIndex) { // We really don't want this junk. return; } + Lemma lemma = new Lemma(); Set> partOfSpeechSet = null; boolean addedToDeferredInThisLexeme = false; for (var lemmaEntry : lexeme.lemmas.entrySet()) { var currentLemmaLanguage = lemmaEntry.getKey(); - Lemma lemma = new Lemma(); + lemma.reset(); documentState.lemmaCount++; LexemeRepresentation lemmaRepresentation = lemmaEntry.getValue(); lemma.value = lemmaRepresentation.value; @@ -297,7 +210,7 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme, int sourceIndex) { // The languages can have wierd Q entry after the desired language. // A spelling variant is informative. Most of the rest are irrelevant. var additionalCategory = currentLemmaLanguage.substring(qVariantIdx + VARIANT_SEPARATOR.length()); - var variant = parserOptions.getMappedGrammemes(additionalCategory); + var variant = Grammar.getMappedGrammemes(additionalCategory); if (variant == null) { if (parserOptions.debug) { System.err.println("Line " + lineNumber + ": " + additionalCategory @@ -313,7 +226,7 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme, int sourceIndex) { lemma.grammemes.addAll(variant); } if (partOfSpeechSet == null) { - partOfSpeechSet = parserOptions.getMappedGrammemes(lexeme.lexicalCategory); + partOfSpeechSet = Grammar.getMappedGrammemes(lexeme.lexicalCategory); if (partOfSpeechSet == null) { throw new IllegalArgumentException(lexeme.lexicalCategory + " is not a known part of speech grammeme for " + lexeme.id + "(" + lemma.value + ")"); @@ -362,7 +275,7 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme, int sourceIndex) { } } if (!finalizeLemmaAfterForms(lemma)) { - continue; + return; } if (deferredLexemes.contains(lexeme.id)) { if (addedToDeferredInThisLexeme) { @@ -373,19 +286,22 @@ private void analyzeLexeme(int lineNumber, Lexeme lexeme, int sourceIndex) { throw new IllegalStateException("Duplicate lexeme " + lexeme.id); } else { addedToDeferredInThisLexeme = true; + lemma = new Lemma(); } } else { - recordOrAnalyzeLemma(lemma, sourceIndex); + analyzeLemma(lemma, sourceIndex); } } } /** * Converts a single OASIS DMLex {@link DMLexEntry} into the internal {@link Lemma} representation - * and routes it through the shared normalization and conflict-resolution pipeline. + * and routes it through the shared normalization and analysis pipeline. */ - private void analyzeDMLexEntry(int lineNumber, DMLexEntry entry, String lexiconLangCode, int sourceIndex) { - // 1. Filter by language and --exclude-language options (allowing entry-level langCode override). + private void analyzeDMLexEntry(Lemma lemma, DMLexEntry entry, String lexiconLangCode, int sourceIndex) { + if (entry == null) { + throw new IllegalArgumentException("Null DMLex entry"); + } String entryLang = (entry.langCode != null && !entry.langCode.isBlank()) ? entry.langCode : lexiconLangCode; if (!LexemesJsonDeserializer.isContained(entryLang)) { return; @@ -394,22 +310,21 @@ private void analyzeDMLexEntry(int lineNumber, DMLexEntry entry, String lexiconL if (omitLemmas.contains(entryId)) { return; } + lemma.reset(); documentState.lemmaCount++; if (entry.headword == null || entry.headword.isBlank()) { throw new IllegalArgumentException("Missing headword for DMLex entry " + entryId); } - Lemma lemma = new Lemma(); lemma.value = entry.headword; - // 2. Extract any "-x-" subtag from the language code (matching Wikidata behavior). int qVariantIdx = entryLang.indexOf(VARIANT_SEPARATOR); if (qVariantIdx >= 0) { var additionalCategory = entryLang.substring(qVariantIdx + VARIANT_SEPARATOR.length()); - var variant = parserOptions.getMappedGrammemes(additionalCategory); + var variant = Grammar.getMappedGrammemes(additionalCategory); if (variant == null) { if (parserOptions.debug) { - System.err.println("Line " + lineNumber + ": " + additionalCategory + System.err.println(additionalCategory + " is not a known grammeme for the language variant " + entryId + "(" + lemma.value + ")"); } return; @@ -420,10 +335,9 @@ private void analyzeDMLexEntry(int lineNumber, DMLexEntry entry, String lexiconL lemma.grammemes.addAll(variant); } - // 3. Map entry-level partOfSpeech, rarity filter, grammatical labels, and IPA pronunciations. if (entry.partOfSpeech != null) { for (String posStr : entry.partOfSpeech) { - var mappedPos = parserOptions.getMappedGrammemes(posStr); + var mappedPos = Grammar.getMappedGrammemes(posStr); if (mappedPos == null) { throw new IllegalArgumentException(posStr + " is not a known part of speech grammeme for " + entryId + "(" + lemma.value + ")"); @@ -436,7 +350,7 @@ private void analyzeDMLexEntry(int lineNumber, DMLexEntry entry, String lexiconL } if (entry.labels != null) { for (String labelStr : entry.labels) { - var mappedLabel = parserOptions.getMappedGrammemes(labelStr); + var mappedLabel = Grammar.getMappedGrammemes(labelStr); if (mappedLabel == null) { throw new IllegalArgumentException(labelStr + " is not a known grammeme for " + entryId + "(" + lemma.value + ")"); @@ -461,8 +375,6 @@ private void analyzeDMLexEntry(int lineNumber, DMLexEntry entry, String lexiconL removeGrammemeType(genderlessLemmaGrammemes, Gender.class); } - // 4. If a DMLex entry has no explicit inflectedForms (e.g. an uninflected particle or noun - // entry listing only its headword), synthesize a single default form from the headword. List forms = entry.inflectedForms; if (forms == null || forms.isEmpty()) { DMLexInflectedForm defaultForm = new DMLexInflectedForm(); @@ -470,7 +382,6 @@ private void analyzeDMLexEntry(int lineNumber, DMLexEntry entry, String lexiconL forms = List.of(defaultForm); } - // 5. Convert each DMLexInflectedForm and run shared form normalization. for (DMLexInflectedForm form : forms) { if (form == null || form.value == null || form.value.isBlank()) { throw new IllegalArgumentException("Missing inflected form value for " + entryId + "(" + lemma.value + ")"); @@ -478,7 +389,7 @@ private void analyzeDMLexEntry(int lineNumber, DMLexEntry entry, String lexiconL Inflection currentInflection = new Inflection(form.value); if (form.labels != null) { for (String labelStr : form.labels) { - Set> values = parserOptions.getMappedGrammemes(labelStr); + Set> values = Grammar.getMappedGrammemes(labelStr); if (values == null) { throw new IllegalArgumentException(labelStr + " is not a known grammeme for " + entryId + "(" + lemma.value + ")"); @@ -497,17 +408,9 @@ private void analyzeDMLexEntry(int lineNumber, DMLexEntry entry, String lexiconL if (!finalizeLemmaAfterForms(lemma)) { return; } - recordOrAnalyzeLemma(lemma, sourceIndex); + analyzeLemma(lemma, sourceIndex); } - /** - * Merges deferred Wikidata lexemes configured in {@code filter_.properties} at the end - * of a Wikidata file stream. For the primary Wikidata dump ({@code sourceIndex == 0}), missing - * partner IDs throw {@link IllegalArgumentException}. For supplemental Wikidata files - * ({@code sourceIndex > 0}), whichever partner IDs are present in that file are merged and any - * remaining deferred lemmas are still analyzed. Clears {@link #deferredLemmaMap} afterward so - * deferred state does not bleed into subsequent files. - */ private void processAndMergeLexemes(int sourceIndex) { if (sourceIndex == 0) { for (Map.Entry> entry : mergeMap.entrySet()) { @@ -522,7 +425,7 @@ private void processAndMergeLexemes(int sourceIndex) { lemma.isRare |= lemmaToMerge.isRare; // The lemma grammemes should already have the relevant POS data. } - recordOrAnalyzeLemma(lemma, sourceIndex); + analyzeLemma(lemma, sourceIndex); } } else { for (Map.Entry> entry : mergeMap.entrySet()) { @@ -539,11 +442,11 @@ private void processAndMergeLexemes(int sourceIndex) { } } if (lemma != null) { - recordOrAnalyzeLemma(lemma, sourceIndex); + analyzeLemma(lemma, sourceIndex); } } for (Lemma remaining : deferredLemmaMap.values()) { - recordOrAnalyzeLemma(remaining, sourceIndex); + analyzeLemma(remaining, sourceIndex); } } deferredLemmaMap.clear(); @@ -586,7 +489,7 @@ private void removeConflicts(TreeSet> grammemes, Class grammemeType) private void convertGrammemes(LexemeForm form, Inflection currentInflection, String id, String lemma) { for (var feature : form.grammaticalFeatures) { - Set> values = parserOptions.getMappedGrammemes(feature); + Set> values = Grammar.getMappedGrammemes(feature); if (values == null) { throw new IllegalArgumentException(feature + " is not a known grammeme for " + id + "(" + lemma + ")"); } @@ -614,7 +517,7 @@ private boolean extractImportantProperties(Map> claims, Tre var claim = claimEntry.getValue(); conflicts = conflicts || claim.size() > 1; for (var grammemeStr : claim) { - var grammemeEnum = parserOptions.getMappedGrammemes(grammemeStr); + var grammemeEnum = Grammar.getMappedGrammemes(grammemeStr); if (grammemeEnum != null) { grammemes.addAll(grammemeEnum); } else if (parserOptions.debug) { @@ -799,8 +702,10 @@ private void analyzeInflections(Lemma lemma, List inputInflections, String phrase = inflection.getInflection(); InflectionPattern inflectionPatternForDict = nonEmptyInflectionIndices.contains(i) ? inflectionPattern : null; - documentState.addDictionaryEntry(new DictionaryEntry(phrase, lemma.value, lemma.isRare, - inflection.getGrammemeSet(), inflectionPatternForDict, sourceIndex)); + documentState.addDictionaryEntry( + new DictionaryEntry(phrase, lemma.value, lemma.isRare, + inflection.getGrammemeSet(), inflectionPatternForDict, sourceIndex), + parserOptions.mergeConflicts); } } @@ -865,50 +770,12 @@ private void mergeAdditionalGrammemes() { DictionaryEntry existingEntry = documentState.dictionary.putIfAbsent(entry.getKey(), newDictionaryEntry); if (existingEntry != null) { // Add instead of replace. - existingEntry.addExtraGrammemes(grammemes); - } - } - } - - /** - * Applies any {@code --expand-grammemes} rules to {@code lemma.inflections} and records the - * resulting surface-form count in {@link DocumentState#incomingSurfaceForm}. - * - *

Running this at the start of {@link #analyzeLemma} (after Tier 1 preemption in - * {@link #processStagedLemmas}) ensures that: - *

    - *
  1. lemmas completely superseded by a later source file do not inflate - * {@code incomingSurfaceForm}, and
    2. - *
  2. {@code --expand-grammemes} keys are matched against each inflection's surviving - * {@link PartOfSpeech} set after any superseded POS has been stripped.
    3. - *
- */ - private void expandAndCountInflections(Lemma lemma) { - if (parserOptions.expandGramemes != null) { - List expanded = new ArrayList<>(lemma.inflections.size()); - for (Inflection currentInflection : lemma.inflections) { - var grammemeExpansion = parserOptions.expandGramemes.get(currentInflection.grammemeSet); - if (grammemeExpansion == null) { - expanded.add(currentInflection); - } else { - for (var grammemeSet : grammemeExpansion) { - var expandedInflection = new Inflection( - currentInflection.inflection, - currentInflection.rareUsage); - expandedInflection.grammemeSet.addAll(currentInflection.grammemeSet); - expandedInflection.grammemeSet.addAll(grammemeSet); - expanded.add(expandedInflection); - } - } + existingEntry.getGrammemes().addAll(grammemes); } - lemma.inflections.clear(); - lemma.inflections.addAll(expanded); } - documentState.incomingSurfaceForm += lemma.inflections.size(); } private void analyzeLemma(Lemma lemma, int sourceIndex) { - expandAndCountInflections(lemma); analyzeInflections(lemma, lemma.inflections, sourceIndex); if (parserOptions.includeLemmasWithoutWords) { @@ -921,10 +788,6 @@ public void writeInflectionPatterns(long startTime) throws FileNotFoundException documentState.printDocument(parserOptions, startTime); } - /** - * Streams a Wikidata lexeme JSON array (`[ { "id": "L...", ... }, ... ]`), analyzing each - * {@link Lexeme} one by one and running per-file deferred lexeme merges at the end of the array. - */ private void parseWikidataStream(ObjectMapper objectMapper, JsonParser parser, int sourceIndex) throws IOException { JsonToken currToken; while ((currToken = parser.nextToken()) != JsonToken.START_OBJECT && currToken != null) { @@ -945,100 +808,24 @@ private void parseWikidataStream(ObjectMapper objectMapper, JsonParser parser, i } } - /** - * Streams an OASIS DMLex v1.0 JSON lexicon object (`{ "langCode": "...", "entries": [ ... ] }`), - * validating that the root is a JSON object containing both required top-level {@code "langCode"} - * and {@code "entries"} fields. - * - *

Entries are deserialized and analyzed in streaming fashion as they are encountered. In the - * unusual case where {@code "entries"} appears before {@code "langCode"} in the JSON key order, - * entries are buffered in {@code entriesBeforeLangCode} until {@code "langCode"} is read. - */ private void parseDMLexStream( ObjectMapper objectMapper, JsonParser parser, ParserOptions.SourceInput sourceInput) throws IOException { - JsonToken firstToken = parser.nextToken(); - if (firstToken != JsonToken.START_OBJECT) { + DMLexEntry.Lexicon lexicon = objectMapper.readValue(parser, DMLexEntry.Lexicon.class); + if (lexicon == null || lexicon.langCode == null || lexicon.langCode.isBlank() || lexicon.entries == null) { throw MismatchedInputException.from( parser, - DMLexEntry.class, - "Expected JSON object at root of DMLex file: " + sourceInput.filename()); - } - - String lexiconLangCode = null; - boolean sawEntries = false; - record DeferredDMLexEntry(int lineNumber, DMLexEntry entry) {} - List entriesBeforeLangCode = new ArrayList<>(); - - while (parser.nextToken() != JsonToken.END_OBJECT) { - if (parser.currentToken() == null) { - throw MismatchedInputException.from( - parser, - DMLexEntry.class, - "Unexpected end of DMLex file: " + sourceInput.filename()); - } - String fieldName = parser.currentName(); - JsonToken valueToken = parser.nextToken(); - if ("langCode".equals(fieldName)) { - if (valueToken != JsonToken.VALUE_STRING) { - throw MismatchedInputException.from( - parser, - String.class, - "Expected string for DMLex 'langCode' in " + sourceInput.filename()); - } - String langVal = parser.getText(); - if (langVal != null && !langVal.isBlank()) { - lexiconLangCode = langVal; - } - } else if ("entries".equals(fieldName)) { - if (valueToken != JsonToken.START_ARRAY) { - throw MismatchedInputException.from( - parser, - List.class, - "Expected array for DMLex 'entries' in " + sourceInput.filename()); - } - sawEntries = true; - while (parser.nextToken() != JsonToken.END_ARRAY) { - if (parser.currentToken() != JsonToken.START_OBJECT) { - throw MismatchedInputException.from( - parser, - DMLexEntry.class, - "Expected object in DMLex 'entries' array in " + sourceInput.filename()); - } - int lineNr = parser.currentLocation().getLineNr(); - DMLexEntry entry = objectMapper.readValue(parser, DMLexEntry.class); - if (lexiconLangCode != null) { - try { - analyzeDMLexEntry(lineNr, entry, lexiconLangCode, sourceInput.sourceIndex()); - } catch (IllegalArgumentException e) { - documentState.unusableLemmaCount++; - System.err.println("Line " + lineNr + ": " + e.getMessage()); - } - } else { - // "entries" appeared before "langCode" in the JSON object; defer until "langCode" is read. - entriesBeforeLangCode.add(new DeferredDMLexEntry(lineNr, entry)); - } - } - } else if (valueToken == JsonToken.START_OBJECT || valueToken == JsonToken.START_ARRAY) { - // Skip any other top-level metadata arrays/objects in the DMLex lexicon. - parser.skipChildren(); - } - } - - if (lexiconLangCode == null || !sawEntries) { - throw MismatchedInputException.from( - parser, - DMLexEntry.class, + DMLexEntry.Lexicon.class, "Invalid DMLex file (missing required 'langCode' or 'entries'): " + sourceInput.filename()); } - - for (DeferredDMLexEntry deferred : entriesBeforeLangCode) { + Lemma lemma = new Lemma(); + for (DMLexEntry entry : lexicon.entries) { try { - analyzeDMLexEntry(deferred.lineNumber(), deferred.entry(), lexiconLangCode, sourceInput.sourceIndex()); + analyzeDMLexEntry(lemma, entry, lexicon.langCode, sourceInput.sourceIndex()); } catch (IllegalArgumentException e) { documentState.unusableLemmaCount++; - System.err.println("Line " + deferred.lineNumber() + ": " + e.getMessage()); + System.err.println(e.getMessage()); } } } @@ -1057,7 +844,6 @@ public static void main(String[] args) throws Exception { // being interpreted as malformed UTF-8. // We use a large buffer because we're reading a large file, and we're // frequently reading file data. - // Iterate through all configured source files (--wikidata and --dmlex) in CLI order. for (ParserOptions.SourceInput sourceInput : parserOptions.sourceInputs) { String sourceFilename = sourceInput.filename(); try (InputStream fileInputStream = new FileInputStream(sourceFilename)) { @@ -1076,12 +862,6 @@ public static void main(String[] args) throws Exception { } } - // In multi-source overwrite mode (--merge-conflicts=false with >1 source file), - // preempt superseded (lemma, PartOfSpeech) paradigms and analyze the surviving lemmas now. - if (lexParser.isMultiSourceOverwriteMode()) { - lexParser.processStagedLemmas(); - } - if (!lexParser.parserOptions.additionalGrammemesDict.isEmpty()) { lexParser.mergeAdditionalGrammemes(); } diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java index 17893cff6..2109eef90 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java @@ -13,7 +13,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; @@ -26,6 +25,7 @@ import java.util.regex.Pattern; import static org.unicode.wikidata.Grammar.DEFAULTMAP; +import static org.unicode.wikidata.Grammar.TYPEMAP; /** * The options to extract the data from the data source. @@ -65,14 +65,6 @@ record SourceInput(String filename, SourceFormat format, int sourceIndex) {} TreeMap>, List>>> expandGramemes; TreeMap> additionalGrammemesDict; TreeMap> claimsToSound; - // Per-run string-to-grammeme overrides configured via --map-grammeme or Q-ID arguments to - // --ignore-property / --ignore-entries-with-grammemes. Stored per ParserOptions instance so - // static Grammar.TYPEMAP is never mutated in place. - private final Map>> customTypeMap = new HashMap<>(); - // Per-run enum overrides configured via --ignore-property or --ignore-entries-with-grammemes - // when specified by grammeme name (e.g. "countable", "shortForm"). Any resolved grammeme enum - // present in this map is replaced with its configured Grammar.Ignorable token during lookup. - private final Map, Grammar.Ignorable> ignoredGrammemes = new HashMap<>(); ArrayList sourceFilenames; ArrayList sourceInputs; @@ -87,7 +79,7 @@ private static void printUsage() { System.err.println("\nOPTIONS"); System.err.println(WIKIDATA_FILE + " [ ...]\tOne or more Wikidata lexeme JSON/bz2 files (space- or comma-separated)"); System.err.println(DMLEX_FILE + " [ ...]\tOne or more OASIS DMLex JSON files (space- or comma-separated)"); - System.err.println(MERGE_CONFLICTS + "[={true|false}]\tWhen true, merge conflicting cross-file entries of the same part of speech instead of overwriting, default: false"); + System.err.println(MERGE_CONFLICTS + "[={true|false}]\tWhen true, merge conflicting cross-file surface forms (with the later source's pattern first) instead of overwriting, default: false"); System.err.println(INFLECTIONS_FILE + " \tthe file for the inflectional patterns to be generated, default: inflectional.xml"); System.err.println(DICTIONARY_FILE + " \tthe file for the lexical dictionary to be generated, default: dictionary.lst"); System.err.println(ADD_EXTRA_GRAMMEMES + " \tFile containing words with the extra grammemes to be added, provide path relative to tools/dictionary-parser/src/main/resources/org/unicode/wikidata/ (only to be used for a temporary grammeme addition)"); @@ -121,19 +113,14 @@ private static void printUsage() { } else if (ParserOptions.DMLEX_FILE.equals(arg) || arg.startsWith(ParserOptions.DMLEX_FILE + "=")) { i = consumeSourceFiles(args, i, ParserOptions.DMLEX_FILE, SourceFormat.DMLEX); } else if (ParserOptions.MERGE_CONFLICTS.equals(arg) || arg.startsWith(ParserOptions.MERGE_CONFLICTS + "=")) { - String valueStr; + String valueStr = "true"; if (arg.startsWith(ParserOptions.MERGE_CONFLICTS + "=")) { - valueStr = arg.substring((ParserOptions.MERGE_CONFLICTS + "=").length()).trim(); - } else if (i + 1 < args.length && args[i + 1] != null - && ("true".equalsIgnoreCase(args[i + 1]) || "false".equalsIgnoreCase(args[i + 1]))) { + valueStr = arg.substring(ParserOptions.MERGE_CONFLICTS.length() + 1).trim(); + } else if (i + 1 < args.length && ("true".equalsIgnoreCase(args[i + 1]) || "false".equalsIgnoreCase(args[i + 1]))) { valueStr = args[++i].trim(); - } else { - valueStr = "true"; } - if ("true".equalsIgnoreCase(valueStr)) { - mergeConflicts = true; - } else if ("false".equalsIgnoreCase(valueStr)) { - mergeConflicts = false; + if ("true".equalsIgnoreCase(valueStr) || "false".equalsIgnoreCase(valueStr)) { + mergeConflicts = Boolean.parseBoolean(valueStr); } else { printUsage(); throw new IllegalArgumentException("Invalid value for " + ParserOptions.MERGE_CONFLICTS + ": " + valueStr); @@ -164,9 +151,7 @@ private static void printUsage() { } else if (ParserOptions.MAP_GRAMMEME.equals(arg)) { String mapGrammeme = args[++i]; String[] split = mapGrammeme.split(",", 2); - TreeSet> mappedSet = toEnumSet(split[1]); - customTypeMap.put(split[0], mappedSet); - customTypeMap.put(Grammar.normalizeGrammemeKey(split[0]), mappedSet); + TYPEMAP.put(split[0], toEnumSet(split[1])); optionsUsedToInvoke.add(ParserOptions.MAP_GRAMMEME); optionsUsedToInvoke.add(mapGrammeme); @@ -194,8 +179,7 @@ private static void printUsage() { posToBeInflected.clear(); for (String pos : inflectionTypes.split(",")) { - posToBeInflected.add(Grammar.PartOfSpeech.valueOf( - Grammar.normalizeGrammemeKey(pos).toUpperCase(Locale.ROOT).replace('-', '_'))); + posToBeInflected.add(Grammar.PartOfSpeech.valueOf(pos.toUpperCase(Locale.ROOT).replace('-', '_'))); } optionsUsedToInvoke.add(ParserOptions.INFLECTION_TYPES); @@ -270,27 +254,12 @@ private static void printUsage() { } private int consumeSourceFiles(String[] args, int i, String flagPrefix, SourceFormat format) { - String arg = args[i]; int startCount = sourceInputs.size(); - if (arg.startsWith(flagPrefix + "=")) { - String inlineVal = arg.substring((flagPrefix + "=").length()); - for (String part : inlineVal.split(",")) { - String trimmed = part.trim(); - if (!trimmed.isEmpty()) { - sourceInputs.add(new SourceInput(trimmed, format, sourceInputs.size())); - sourceFilenames.add(trimmed); - } - } + if (args[i].startsWith(flagPrefix + "=")) { + addSourceFiles(args[i].substring(flagPrefix.length() + 1), format); } while (i + 1 < args.length && args[i + 1] != null && !args[i + 1].startsWith("--")) { - String rawArg = args[++i]; - for (String part : rawArg.split(",")) { - String trimmed = part.trim(); - if (!trimmed.isEmpty()) { - sourceInputs.add(new SourceInput(trimmed, format, sourceInputs.size())); - sourceFilenames.add(trimmed); - } - } + addSourceFiles(args[++i], format); } if (sourceInputs.size() == startCount) { printUsage(); @@ -299,95 +268,49 @@ private int consumeSourceFiles(String[] args, int i, String flagPrefix, SourceFo return i; } - /** - * Records per-run ignore rules from {@code --ignore-property} or {@code --ignore-entries-with-grammemes}. - * Known grammeme names or single-enum shorthand aliases (e.g. {@code "countable"}, {@code "shortForm"}, - * {@code "masc"}) are resolved to their non-{@link Grammar.Ignorable} enum and recorded in - * {@link #ignoredGrammemes} so any Q-ID or DMLex label mapping to that enum is replaced with - * {@code ignorable} at lookup time. Raw Wikidata Q-IDs (e.g. {@code "Q55194613"}) and unmapped - * DMLex tags (e.g. {@code "archaic"}, {@code "dialectal"}) are recorded in {@link #customTypeMap}. - */ + private void addSourceFiles(String rawArg, SourceFormat format) { + for (String part : rawArg.split(",")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) { + sourceInputs.add(new SourceInput(trimmed, format, sourceInputs.size())); + sourceFilenames.add(trimmed); + } + } + } + void setIgnoreProperty(String[] grammemes, Grammar.Ignorable ignorable) { var ignorableSet = EnumSet.of(ignorable); for (String grammeme : grammemes) { - Enum enumVal = null; + boolean matched = false; if (!grammeme.matches("Q\\d*")) { - enumVal = DEFAULTMAP.get(grammeme); - if (enumVal == null) { - enumVal = DEFAULTMAP.get(Grammar.normalizeGrammemeKey(grammeme)); - } - if (enumVal == null) { - Set> mapped = Grammar.getMappedGrammemes(grammeme); - if (mapped != null && mapped.size() == 1) { - enumVal = mapped.iterator().next(); + for (Map.Entry>> entry : TYPEMAP.entrySet()) { + for (var grammemeEnum : entry.getValue()) { + if (grammemeEnum.name().equalsIgnoreCase(grammeme) + || grammemeEnum.toString().equalsIgnoreCase(grammeme)) { + matched = true; + if (entry.getValue().size() == 1) { + entry.setValue(ignorableSet); + } else { + entry.getValue().remove(grammemeEnum); + ArrayList> clone = new ArrayList<>(entry.getValue()); + clone.add(ignorable); + entry.setValue(new HashSet<>(clone)); + } + break; + } } } } - if (enumVal != null && !(enumVal instanceof Grammar.Ignorable)) { - ignoredGrammemes.put(enumVal, ignorable); - } else { - customTypeMap.put(grammeme, ignorableSet); - customTypeMap.put(Grammar.normalizeGrammemeKey(grammeme), ignorableSet); - } - } - } - - /** - * Resolves a Wikidata Q-ID or DMLex grammatical label to its mapped {@link Grammar} enums, - * applying any per-run CLI overrides ({@code --map-grammeme}, {@code --ignore-property}, - * {@code --ignore-entries-with-grammemes}) without mutating static {@link Grammar#TYPEMAP}. - */ - Set> getMappedGrammemes(String grammeme) { - if (grammeme == null) { - return null; - } - // 1. Check per-run string overrides (--map-grammeme or Q-ID --ignore-* rules). - Set> mapped = customTypeMap.get(grammeme); - if (mapped == null && !customTypeMap.isEmpty()) { - mapped = customTypeMap.get(Grammar.normalizeGrammemeKey(grammeme)); - } - // 2. Fall back to the read-only static Grammar lookup table. - if (mapped == null) { - mapped = Grammar.getMappedGrammemes(grammeme); - } - if (mapped == null || ignoredGrammemes.isEmpty()) { - return mapped; - } - // 3. Replace any resolved enum present in ignoredGrammemes with its Ignorable token. - boolean hasIgnored = false; - for (Enum e : mapped) { - if (ignoredGrammemes.containsKey(e)) { - hasIgnored = true; - break; + if (!matched) { + TYPEMAP.put(grammeme, ignorableSet); } } - if (!hasIgnored) { - return mapped; - } - if (mapped.size() == 1) { - return EnumSet.of(ignoredGrammemes.get(mapped.iterator().next())); - } - Set> adjusted = new HashSet<>(mapped.size()); - for (Enum e : mapped) { - Grammar.Ignorable replacement = ignoredGrammemes.get(e); - adjusted.add(replacement != null ? replacement : e); - } - return adjusted; } TreeSet> toEnumSet(String grammemes) { TreeSet> grammemeSet = new TreeSet<>(Inflection.ENUM_COMPARATOR); for (var grammeme : grammemes.split(",")) { var grammemeEnum = DEFAULTMAP.get(grammeme); - if (grammemeEnum == null) { - grammemeEnum = DEFAULTMAP.get(Grammar.normalizeGrammemeKey(grammeme)); - } - if (grammemeEnum == null && !grammeme.matches("Q\\d*")) { - Set> mapped = Grammar.getMappedGrammemes(grammeme); - if (mapped != null && mapped.size() == 1) { - grammemeEnum = mapped.iterator().next(); - } - } if (grammemeEnum == null) { throw new NullPointerException(grammeme + " is not a valid grammeme"); } diff --git a/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java b/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java index 4de8692fc..158ba53d7 100644 --- a/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java +++ b/tools/dictionary-parser/src/test/java/org/unicode/wikidata/ParseWikidataTest.java @@ -10,8 +10,15 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.exc.MismatchedInputException; @@ -33,6 +40,8 @@ class ParseWikidataTest { private static final String dmlexPatchFile = baseDir + "/dmlexPatch.json"; private static final String dmlexInvalidSchemaFile = baseDir + "/dmlexInvalidSchema.json"; private static final String LINE_END = "=============================================="; + private static final Pattern XML_PATTERN_NAME = Pattern.compile(" parseDictionaryEntries(String output) { line -> line.substring(line.indexOf(':') + 1).trim())); } + private Set assertAllXmlPatternsReferencedInDictionary(String dictionaryOutput) throws Exception { + String xml = Files.readString(Paths.get(inflectionalFile), StandardCharsets.UTF_8); + Set xmlPatternIds = new TreeSet<>(); + Matcher xmlMatcher = XML_PATTERN_NAME.matcher(xml); + while (xmlMatcher.find()) { + Assertions.assertTrue(xmlPatternIds.add(xmlMatcher.group(1)), + "Duplicate pattern name in inflectional XML: " + xmlMatcher.group(1)); + } + Set dictPatternIds = new TreeSet<>(); + for (String line : dictionaryOutput.split("\n")) { + if (line.equals(LINE_END)) { + break; + } + Matcher dictMatcher = DICT_INFLECTION_ID.matcher(line); + while (dictMatcher.find()) { + dictPatternIds.add(dictMatcher.group(1)); + } + } + Assertions.assertEquals(dictPatternIds, xmlPatternIds, + "Every in inflectional.xml must be referenced as inflection=X in dictionary.lst"); + return xmlPatternIds; + } + @Test public void missingInputFile() { Assertions.assertThrows(Exception.class, () -> ParseWikidata.main(new String[1])); @@ -70,6 +102,7 @@ public void missingInputFile() { public void enumTest() { String fieldValue = Grammar.Alternate.SPELLING.toString(); Assertions.assertEquals("spelling", fieldValue); + Assertions.assertThrows(IllegalArgumentException.class, () -> Grammar.getMappedGrammemes(null)); } @Test @@ -123,7 +156,7 @@ public void standaloneDMLexParserTest() throws Exception { Assertions.assertEquals("singular vowel-start noun inflection=2", entries.get("apple")); Assertions.assertEquals("plural vowel-start noun inflection=2", entries.get("apples")); - // London uses camelCase properNoun normalized to proper-noun + // London uses proper-noun Assertions.assertEquals("singular proper-noun inflection=3", entries.get("London")); // wow has no inflectedForms array, synthesized default form @@ -140,7 +173,9 @@ public void standaloneDMLexParserTest() throws Exception { @Test public void multiSourceWikidataAndDMLexOverwriteTest() throws Exception { - // Default --merge-conflicts=false: later source (dmlexOverlayFile) overrides same-POS conflicts + // Default --merge-conflicts=false (Option 4 + Option 3): normal processing happens for earlier + // files, and when a surface form is encountered in a later file, the new pattern and grammemes + // override all previous data for that surface form. String[] args = { "--inflection-types", "noun,adjective,proper-noun", "--add-sound", "vowel-start", @@ -159,43 +194,36 @@ public void multiSourceWikidataAndDMLexOverwriteTest() throws Exception { Assertions.assertTrue(entries.containsKey("books")); Assertions.assertTrue(entries.containsKey("books'")); - // 2. "hey" was interjection in Wikidata and noun in DMLex -> both POS are preserved + // 2. "hey" was interjection in Wikidata and singular noun in DMLex -> DMLex overrides "hey" String heyEntry = entries.get("hey"); Assertions.assertNotNull(heyEntry); - Assertions.assertTrue(heyEntry.contains("noun"), "Expected noun in: " + heyEntry); - Assertions.assertTrue(heyEntry.contains("interjection"), "Expected interjection in: " + heyEntry); + Assertions.assertTrue(heyEntry.startsWith("singular noun inflection="), "Expected singular noun override in: " + heyEntry); + Assertions.assertFalse(heyEntry.contains("interjection"), "Did not expect interjection after override in: " + heyEntry); Assertions.assertTrue(entries.containsKey("heys")); - // 3. "ciel" was masculine noun (ciel/ciels/cieux) in Wikidata, overridden by feminine noun (ciel/cieux) in DMLex + // 3. "ciel" and "cieux" were masculine noun in Wikidata, overridden by feminine noun in DMLex String cielEntry = entries.get("ciel"); Assertions.assertNotNull(cielEntry); Assertions.assertTrue(cielEntry.contains("feminine"), "Expected feminine in: " + cielEntry); Assertions.assertFalse(cielEntry.contains("masculine"), "Did not expect masculine in: " + cielEntry); + Assertions.assertEquals(1, cielEntry.split("inflection=").length - 1, "Expected 1 override pattern in: " + cielEntry); String cieuxEntry = entries.get("cieux"); Assertions.assertNotNull(cieuxEntry); Assertions.assertTrue(cieuxEntry.contains("feminine"), "Expected feminine in: " + cieuxEntry); Assertions.assertFalse(cieuxEntry.contains("masculine"), "Did not expect masculine in: " + cieuxEntry); + Assertions.assertEquals(1, cieuxEntry.split("inflection=").length - 1, "Expected 1 override pattern in: " + cieuxEntry); - // Orphaned Wikidata form "ciels" must not be present when ciel(noun) was superseded - Assertions.assertFalse(entries.containsKey("ciels"), "Expected orphaned form 'ciels' to be removed"); - - // 4. Verify inflectional XML: preempting "ciel" (L7, L8) at Tier 1 leaves only "idéal" (L9, L10) - // in the -l/-ls/-ux pattern, so its word count is words="2" (instead of words="3" or words="4") - // and "idéal" references a single merged pattern instead of two split patterns. - String idealEntry = entries.get("idéal"); - Assertions.assertNotNull(idealEntry); - Assertions.assertEquals(1, idealEntry.split("inflection=").length - 1, "Expected 1 merged inflection pattern in: " + idealEntry); - - String xml = Files.readString(Paths.get(inflectionalFile), StandardCharsets.UTF_8); - Assertions.assertTrue(xml.contains("")); - Assertions.assertFalse(xml.contains("words=\"3\"")); - Assertions.assertFalse(xml.contains("words=\"4\"")); + // "ciels" was processed in Wikidata and not overridden in DMLex (Option 4) + Assertions.assertTrue(entries.containsKey("ciels")); + Assertions.assertTrue(entries.get("ciels").startsWith("plural masculine noun inflection=")); + assertAllXmlPatternsReferencedInDictionary(actual); } @Test public void multiSourceWikidataAndDMLexMergeConflictsTrueTest() throws Exception { - // --merge-conflicts=true: conflicting entries of the same POS across files are merged + // --merge-conflicts=true (Option 4 + Option 2): conflicting entries across files mix in + // their grammemes and patterns, and the later source's conflicting pattern comes first. String[] args = { "--merge-conflicts=true", "--inflection-types", "noun,adjective,proper-noun", @@ -208,36 +236,54 @@ public void multiSourceWikidataAndDMLexMergeConflictsTrueTest() throws Exception String actual = getParserOutput(args); Map entries = parseDictionaryEntries(actual); - // In merge mode, "ciel" merges Wikidata (masculine) and DMLex (feminine), keeping both patterns and "ciels" + // "hey" merges Wikidata (interjection) and DMLex (singular noun) + String heyEntry = entries.get("hey"); + Assertions.assertNotNull(heyEntry); + Assertions.assertTrue(heyEntry.contains("noun"), "Expected noun in: " + heyEntry); + Assertions.assertTrue(heyEntry.contains("interjection"), "Expected interjection in: " + heyEntry); + + // "ciel" merges Wikidata (masculine) and DMLex (feminine), keeping both patterns, + // with the DMLex conflict pattern placed FIRST before the higher-count Wikidata pattern! String cielEntry = entries.get("ciel"); Assertions.assertNotNull(cielEntry); Assertions.assertTrue(cielEntry.contains("masculine"), "Expected masculine in: " + cielEntry); Assertions.assertTrue(cielEntry.contains("feminine"), "Expected feminine in: " + cielEntry); - // Should have two inflection=... references (one from Wikidata pattern, one from DMLex pattern) - Assertions.assertEquals(2, cielEntry.split("inflection=").length - 1, "Expected 2 inflection patterns in: " + cielEntry); - - // "ciels" from Wikidata is preserved in merge mode - Assertions.assertTrue(entries.containsKey("ciels")); - Assertions.assertTrue(entries.get("ciels").contains("plural masculine noun")); + String[] cielParts = cielEntry.split("inflection="); + Assertions.assertEquals(3, cielParts.length, "Expected 2 inflection patterns in: " + cielEntry); + String firstPatternId = cielParts[1].trim(); + String secondPatternId = cielParts[2].trim(); + + // "ciels" only exists in Wikidata, so its pattern ID is the Wikidata pattern ID. + // Verify that on "ciel", the Wikidata pattern ID is second and the DMLex override pattern ID is first! + String cielsEntry = entries.get("ciels"); + Assertions.assertNotNull(cielsEntry); + String wikidataPatternId = cielsEntry.split("inflection=")[1].trim(); + Assertions.assertEquals(wikidataPatternId, secondPatternId, + "Expected Wikidata pattern to be second after DMLex override pattern in: " + cielEntry); + Assertions.assertNotEquals(firstPatternId, secondPatternId); + assertAllXmlPatternsReferencedInDictionary(actual); } @Test - public void multiSourcePartialMultiPosOverrideTest() throws Exception { - // In sourceLexicon.json, "Élie" (L2) has Q147276 which maps to BOTH noun and proper-noun (plural masculine). - // When a later DMLex file overrides ONLY proper-noun on "Élie" (singular feminine proper-noun), - // the noun slice from Wikidata (plural masculine noun) must be preserved while proper-noun is replaced. - Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-partial-pos-", ".json"); + public void multiSourceSameFileHomographAndMergePriorityTest() throws Exception { + // Tests: + // 1. When a later DMLex file defines multiple entries for the same surface form ("Élie" as both + // proper-noun and verb in the same file), under --merge-conflicts=false (Option 4 + Option 3) + // the first entry overrides Wikidata's "Élie" and the second entry from the same DMLex file + // merges with it. + // 2. Under --merge-conflicts=true (Option 4 + Option 2), both Wikidata and DMLex grammemes/patterns + // are mixed in. + Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-homograph-", ".json"); try { String json = """ { "langCode": "en", "entries": [ { - "id": "elie-override", + "id": "elie-proper-noun", "headword": "Élie", "partOfSpeech": ["proper-noun"], "tag": "feminine", - "labels": ["rare"], "pronunciations": [ { "transcriptions": [ @@ -248,8 +294,17 @@ public void multiSourcePartialMultiPosOverrideTest() throws Exception { "inflectedForms": [ { "value": "Élie", - "tag": "singular", - "labels": ["rare"] + "tag": "singular" + } + ] + }, + { + "id": "elie-verb", + "headword": "Élie", + "partOfSpeech": ["verb"], + "inflectedForms": [ + { + "value": "Élie" } ] } @@ -258,7 +313,7 @@ public void multiSourcePartialMultiPosOverrideTest() throws Exception { """; Files.writeString(tempDmlex, json, StandardCharsets.UTF_8); - String[] args = { + String[] overwriteArgs = { "--inflection-types", "noun,adjective,proper-noun", "--add-sound", "vowel-start", "--inflections", inflectionalFile, @@ -266,16 +321,67 @@ public void multiSourcePartialMultiPosOverrideTest() throws Exception { "--wikidata", lexiconSource, "--dmlex=" + tempDmlex }; - String actual = getParserOutput(args); - Map entries = parseDictionaryEntries(actual); - - String elieEntry = entries.get("Élie"); - Assertions.assertNotNull(elieEntry); - // Both noun (plural masculine from Wikidata) and proper-noun (singular feminine vowel-start from DMLex) are present, - // with 2 distinct inflection patterns (non-rare noun pattern first, rare proper-noun pattern second). - Assertions.assertEquals( - "singular plural masculine feminine vowel-start noun proper-noun inflection=3 inflection=8", - elieEntry); + String overwriteOutput = getParserOutput(overwriteArgs); + Map overwriteEntries = parseDictionaryEntries(overwriteOutput); + String elieOverwrite = overwriteEntries.get("Élie"); + Assertions.assertNotNull(elieOverwrite); + // Wikidata's "plural masculine noun proper-noun" was overridden by tempDmlex's first entry + // ("singular feminine vowel-start proper-noun"), and tempDmlex's second entry ("verb") merged with it. + Assertions.assertTrue(elieOverwrite.startsWith("singular feminine vowel-start proper-noun verb inflection="), + "Unexpected overwrite entry: " + elieOverwrite); + Assertions.assertEquals(1, elieOverwrite.split("inflection=").length - 1); + // Because Wikidata's "Élie" was the only surface form referencing its inflection pattern, + // overriding "Élie" must prune that superseded pattern from inflectional.xml so all XML patterns + // are referenced in dictionary.lst (7 patterns total instead of 8). + Set overwriteXmlPatternIds = assertAllXmlPatternsReferencedInDictionary(overwriteOutput); + Assertions.assertEquals(7, overwriteXmlPatternIds.size()); + + String[] mergeArgs = { + "--merge-conflicts=true", + "--inflection-types", "noun,adjective,proper-noun", + "--add-sound", "vowel-start", + "--inflections", inflectionalFile, + "--dictionary", dictionaryFile, + "--wikidata", lexiconSource, + "--dmlex=" + tempDmlex + }; + String mergeOutput = getParserOutput(mergeArgs); + Map mergeEntries = parseDictionaryEntries(mergeOutput); + String elieMerge = mergeEntries.get("Élie"); + Assertions.assertNotNull(elieMerge); + Assertions.assertTrue( + elieMerge.startsWith("singular plural masculine feminine vowel-start noun proper-noun verb inflection="), + "Unexpected merge entry: " + elieMerge); + Assertions.assertEquals(2, elieMerge.split("inflection=").length - 1); + // Under --merge-conflicts=true, "Élie" retains both the DMLex pattern and the Wikidata pattern, + // so all 8 patterns remain referenced in dictionary.lst and present in inflectional.xml. + Set mergeXmlPatternIds = assertAllXmlPatternsReferencedInDictionary(mergeOutput); + Assertions.assertEquals(8, mergeXmlPatternIds.size()); + + // Verify that if a pattern was rare in an earlier source (sourceIndex=0) alongside a non-rare + // pattern, and a later source (sourceIndex=1) uses that same pattern non-rarely, it is promoted + // to non-rare and sorted first under --merge-conflicts=true. + TreeSet> grammemes = new TreeSet<>(EnumComparator.ENUM_COMPARATOR); + grammemes.add(Grammar.PartOfSpeech.NOUN); + Inflection suffixS = new Inflection("s", false); + suffixS.addGrammemes(List.of(Grammar.PartOfSpeech.NOUN, Grammar.Number.PLURAL)); + Inflection suffixEs = new Inflection("es", false); + suffixEs.addGrammemes(List.of(Grammar.PartOfSpeech.NOUN, Grammar.Number.PLURAL)); + InflectionPattern pattern1 = new InflectionPattern(1, "", grammemes, List.of(suffixS)); + InflectionPattern pattern2 = new InflectionPattern(2, "", grammemes, List.of(suffixEs)); + // Give pattern2 a higher count than pattern1 + pattern2.merge(pattern2); + + DictionaryEntry baseRareEntry = new DictionaryEntry("word", "word", true, grammemes, pattern1, 0); + DictionaryEntry baseNonRareEntry = new DictionaryEntry("word", "word", false, grammemes, pattern2, 0); + baseRareEntry.merge(baseNonRareEntry, true); + // Before override, pattern1 is rare so pattern2 (non-rare) comes first: + Assertions.assertEquals("word: noun inflection=2 inflection=1", baseRareEntry.toString(true)); + + // Now merge a later source (sourceIndex=1) that uses pattern1 as non-rare: + DictionaryEntry overrideNonRareEntry = new DictionaryEntry("word", "word", false, grammemes, pattern1, 1); + baseRareEntry.merge(overrideNonRareEntry, true); + Assertions.assertEquals("word: noun inflection=1 inflection=2", baseRareEntry.toString(true)); } finally { Files.deleteIfExists(tempDmlex); } @@ -302,19 +408,21 @@ public void multiFileDMLexSpaceAndCommaSeparatedTest() throws Exception { compareOutputs(spaceOutput, commaOutput); Map entries = parseDictionaryEntries(spaceOutput); - // "cat" (noun) was overridden by dmlexPatch.json: feminine, plural "kitties", and "cats" removed + // "cat" surface form was overridden by dmlexPatch.json: feminine, plural "kitties" added, + // while "cats" from dmlexLexicon.json remains from normal processing (Option 4 + Option 3). Assertions.assertTrue(entries.get("cat").contains("singular feminine noun")); Assertions.assertFalse(entries.get("cat").contains("masculine")); Assertions.assertTrue(entries.containsKey("kitties")); - Assertions.assertFalse(entries.containsKey("cats")); + Assertions.assertTrue(entries.containsKey("cats")); - // "bats" surface form from lemma "bat" (masculine noun) was overridden at Tier 2 by lemma "batte" (feminine noun) + // "bats" surface form from lemma "bat" (masculine noun) was overridden by lemma "batte" (feminine noun) Assertions.assertTrue(entries.get("bats").contains("plural feminine noun")); Assertions.assertFalse(entries.get("bats").contains("masculine")); - // "wow" keeps interjection from dmlexLexicon.json and adds verb from dmlexPatch.json - Assertions.assertEquals("interjection verb", entries.get("wow")); + // "wow" surface form was overridden by dmlexPatch.json (verb) + Assertions.assertEquals("verb", entries.get("wow")); Assertions.assertEquals("verb", entries.get("wowed")); + assertAllXmlPatternsReferencedInDictionary(spaceOutput); } @Test @@ -355,6 +463,10 @@ public void invalidDMLexSchemaAndUnknownOptionTest() { @Test public void dmlexCliOptionsCompatibilityTest() throws Exception { Path tempDmlex = Files.createTempFile(Paths.get(baseDir), "dmlex-opts-", ".json"); + Map>> savedTypeMap = new HashMap<>(Grammar.TYPEMAP.size()); + for (var entry : Grammar.TYPEMAP.entrySet()) { + savedTypeMap.put(entry.getKey(), new HashSet<>(entry.getValue())); + } try { String json = """ { @@ -389,11 +501,11 @@ public void dmlexCliOptionsCompatibilityTest() throws Exception { String[] args = { "--language", "ro", "--exclude-language", "ro-md", - "--inflection-types", "noun,properNoun", + "--inflection-types", "noun,proper-noun", "--map-grammeme", "customNomAcc,nominative,accusative", "--expand-grammemes", "singular,nominative,accusative,feminine,noun:indefinite", "--ignore-property", "countable,dialectal", - "--ignore-entries-with-grammemes", "shortForm,archaic", + "--ignore-entries-with-grammemes", "short-form,archaic", "--inflections", inflectionalFile, "--dictionary", dictionaryFile, "--dmlex", tempDmlex.toString() @@ -411,6 +523,8 @@ public void dmlexCliOptionsCompatibilityTest() throws Exception { Assertions.assertEquals("singular accusative nominative indefinite feminine noun", entries.get("casă")); Assertions.assertEquals("plural accusative nominative feminine noun", entries.get("case")); } finally { + Grammar.TYPEMAP.clear(); + Grammar.TYPEMAP.putAll(savedTypeMap); Files.deleteIfExists(tempDmlex); } } @@ -434,7 +548,7 @@ public void dmlexEntriesBeforeLangCodeMultiFieldObjectsAndReverseOrderTest() thr { "label": "Common noun", "tag": "noun" } ], "labels": [ - { "tag": "nonHuman", "label": "Non-human animacy" } + { "tag": "nonhuman", "label": "Non-human animacy" } ], "pronunciations": [ { "label": "US", "transcription": "ˈæp.əl" } @@ -459,17 +573,25 @@ public void dmlexEntriesBeforeLangCodeMultiFieldObjectsAndReverseOrderTest() thr "headword": "ciel", "partOfSpeech": ["noun"], "labels": ["feminine"], + "pronunciations": [ + { + "label": "en-US", + "transcriptions": [ + { "text": "sjɛl", "scheme": "IPA" } + ] + } + ], "inflectedForms": [ { "value": "ciel", "labels": ["singular"] }, - { "value": "ciel-obsolete", "labels": ["plural"] } + { "value": "cieux", "labels": ["plural"] }, + { "value": "ciel-dmlex-only", "labels": ["plural"] } ] }, { "id": "dmlex-posless-early", "headword": "uh-oh", "inflectedForms": [ - { "value": "uh-oh", "labels": ["singular"] }, - { "value": "uh-oh-obsolete", "labels": ["plural"] } + { "value": "uh-oh", "labels": ["singular"] } ] } ], @@ -508,21 +630,24 @@ public void dmlexEntriesBeforeLangCodeMultiFieldObjectsAndReverseOrderTest() thr String actual = getParserOutput(args); Map entries = parseDictionaryEntries(actual); - // "apple" from DMLex is parsed with multi-field objects and deferred "langCode" + // "apple" from DMLex is parsed with multi-field objects and "entries" before "langCode" Assertions.assertTrue(entries.get("apple").startsWith("singular nonhuman vowel-start noun inflection=")); Assertions.assertTrue(entries.get("apples").startsWith("plural nonhuman vowel-start noun inflection=")); - // Because --wikidata came AFTER --dmlex, Wikidata's masculine "ciel" (ciel/ciels/cieux) overrides DMLex's feminine "ciel", - // and DMLex's orphaned form "ciel-obsolete" is preempted at Tier 1. + // Because --wikidata came AFTER --dmlex, Wikidata's masculine "ciel" overrides DMLex's feminine "ciel" Assertions.assertTrue(entries.get("ciel").contains("masculine")); Assertions.assertFalse(entries.get("ciel").contains("feminine")); Assertions.assertTrue(entries.containsKey("ciels")); - Assertions.assertTrue(entries.containsKey("cieux")); - Assertions.assertFalse(entries.containsKey("ciel-obsolete")); - - // POS-less lemma "uh-oh" from tempDmlex was preempted at Tier 1 by tempDmlex2, removing "uh-oh-obsolete". + Assertions.assertTrue(entries.get("cieux").contains("masculine")); + Assertions.assertFalse(entries.get("cieux").contains("feminine")); + // "ciel-dmlex-only" was not in Wikidata, and its pronunciation container's "label": "en-US" + // must not be mistaken for a vowel-start transcription when "transcriptions" is present. + Assertions.assertTrue(entries.get("ciel-dmlex-only").startsWith("plural feminine noun inflection=")); + Assertions.assertFalse(entries.get("ciel-dmlex-only").contains("vowel-start")); + + // POS-less surface form "uh-oh" from tempDmlex was overridden by tempDmlex2 Assertions.assertEquals("plural", entries.get("uh-oh")); - Assertions.assertFalse(entries.containsKey("uh-oh-obsolete")); + assertAllXmlPatternsReferencedInDictionary(actual); } finally { Files.deleteIfExists(tempDmlex2); } @@ -538,7 +663,7 @@ public void deferredLexemeMultiVariantAndSupplementalMergeTest() throws Exceptio // 1. Primary Wikidata file (sourceIndex == 0) with multi-variant L27261 ("fr" and "fr-x-Q56352306") // and L1566399 merges both variants and both partner lexemes without throwing "Duplicate lexeme". // 2. Supplemental Wikidata file (sourceIndex == 1) containing only L27261 (without L1566399) - // does not throw "L1566399: id not found" and overrides the earlier paradigm. + // does not throw "L1566399: id not found" and overrides the "organisateur" surface form. // 3. Duplicate top-level Lexeme objects with the same deferred ID in one file still throw IllegalStateException. Path primaryWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-primary-", ".json"); Path supplementalWikidata = Files.createTempFile(Paths.get(baseDir), "wikidata-fr-supp-", ".json"); @@ -681,7 +806,12 @@ public void deferredLexemeMultiVariantAndSupplementalMergeTest() throws Exceptio Assertions.assertTrue(entries.containsKey("organisateur")); Assertions.assertTrue(entries.containsKey("organisateurs")); - Assertions.assertFalse(entries.containsKey("organisateurs-old")); + Assertions.assertTrue(entries.containsKey("organisatrice")); + // "organisateur" was overridden by supplementalWikidata, so it shares the same pattern as "organisateurs" + Assertions.assertEquals( + entries.get("organisateurs").split("inflection=")[1], + entries.get("organisateur").split("inflection=")[1]); + assertAllXmlPatternsReferencedInDictionary(actual); String[] dupArgs = { "--language", "fr", diff --git a/tools/dictionary-parser/src/test/resources/dmlexLexicon.json b/tools/dictionary-parser/src/test/resources/dmlexLexicon.json index 7fc818ca9..3bb27cb19 100644 --- a/tools/dictionary-parser/src/test/resources/dmlexLexicon.json +++ b/tools/dictionary-parser/src/test/resources/dmlexLexicon.json @@ -73,7 +73,7 @@ { "id": "dmlex-5", "headword": "London", - "partOfSpeech": ["properNoun"], + "partOfSpeech": ["proper-noun"], "inflectedForms": [ { "id": "dmlex-5-f1", From 0f717a363418151f7b39b556d468f8a2b15679d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Neboj=C5=A1a=20=C4=86iri=C4=87?= Date: Tue, 20 Oct 2026 19:09:35 +0000 Subject: [PATCH 3/4] Inflection-208 Preserve same-source rare pattern merging --- .../org/unicode/wikidata/DictionaryEntry.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java index 2cacfed1a..8f5f583fb 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/DictionaryEntry.java @@ -97,21 +97,25 @@ public void merge(DictionaryEntry entry, boolean mergeConflicts) { } } for (InflectionPattern pattern : entry.inflectionalPatterns) { - this.patternSourceIndices.merge(pattern, entry.getPatternSourceIndex(pattern), Math::max); + int existingSource = this.patternSourceIndices.getOrDefault(pattern, this.sourceIndex); + int incomingSource = entry.getPatternSourceIndex(pattern); + if (incomingSource > existingSource && !entry.rareInflectionalPatterns.contains(pattern)) { + this.rareInflectionalPatterns.remove(pattern); + } + this.patternSourceIndices.put(pattern, Math.max(existingSource, incomingSource)); } this.sourceIndex = Math.max(this.sourceIndex, entry.sourceIndex); } this.grammemes.addAll(entry.grammemes); for (InflectionPattern inflectionalPattern : entry.inflectionalPatterns) { - boolean isRareInEntry = entry.rareInflectionalPatterns.contains(inflectionalPattern); if (!this.inflectionalPatterns.contains(inflectionalPattern)) { this.inflectionalPatterns.add(inflectionalPattern); - if (isRareInEntry) { - this.rareInflectionalPatterns.add(inflectionalPattern); - } - } else if (!isRareInEntry) { - this.rareInflectionalPatterns.remove(inflectionalPattern); + } + } + for (InflectionPattern inflectionalPattern : entry.rareInflectionalPatterns) { + if (!this.rareInflectionalPatterns.contains(inflectionalPattern)) { + this.rareInflectionalPatterns.add(inflectionalPattern); } } this.lemmas.addAll(entry.lemmas); From da5ddd5b2c5ebd24f3b37c18d0b953cf704706a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Neboj=C5=A1a=20=C4=86iri=C4=87?= Date: Fri, 23 Oct 2026 07:58:03 +0000 Subject: [PATCH 4/4] Inflection-208 Use grammemeEnum.toString() in setIgnoreProperty --- .../src/main/java/org/unicode/wikidata/ParserOptions.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java index 2109eef90..65eee98c6 100644 --- a/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java +++ b/tools/dictionary-parser/src/main/java/org/unicode/wikidata/ParserOptions.java @@ -285,8 +285,7 @@ void setIgnoreProperty(String[] grammemes, Grammar.Ignorable ignorable) { if (!grammeme.matches("Q\\d*")) { for (Map.Entry>> entry : TYPEMAP.entrySet()) { for (var grammemeEnum : entry.getValue()) { - if (grammemeEnum.name().equalsIgnoreCase(grammeme) - || grammemeEnum.toString().equalsIgnoreCase(grammeme)) { + if (grammemeEnum.toString().equalsIgnoreCase(grammeme)) { matched = true; if (entry.getValue().size() == 1) { entry.setValue(ignorableSet);