user_word: study_choice and unfit_reason replace user_preference

tools/delete_bookmarks_word_not_in_context.py

@@ -24,6 +24,7 @@
 app.app_context().push()
 
 from zeeguu.core.model.bookmark import Bookmark
+from zeeguu.core.model.study_choice import UnfitReason
 import unicodedata
 
 
@@ -180,7 +181,7 @@ def delete_bookmarks(bookmark_ids, dry_run=True):
                 else:
                     # No other bookmarks exist - mark as unfit for study AND clear schedule
                     # (Keep UserWord for historical data)
-                    user_word.set_unfit_for_study(db.session)
+                    user_word.set_unfit_for_study(db.session, UnfitReason.NO_USABLE_CONTEXT)
                     userwords_updated += 1
 
             # Delete the bookmark

tools/migrations/26-10-25--user_word_study_choice.sql

@@ -0,0 +1,54 @@
+-- user_word.user_preference -> study_choice + unfit_reason (roadmap 0b).
+-- user_preference itself stays, unused, until a later migration drops it.
+-- After this, run tools/recompute_fit_after_study_choice.py for the words
+-- that lose their forced "use in exercises" (step 3).
+
+ALTER TABLE user_word
+    ADD COLUMN study_choice ENUM('study','skip','known') NULL DEFAULT NULL,
+    ADD COLUMN unfit_reason ENUM('rare_for_beginner','same_as_translation','too_short',
+        'too_many_words','arbitrary_multi_word','bad_contexts','separated_mwe',
+        'validator_gave_up','no_usable_context',
+        'validator_invalid','replaced_by_fix','duplicate_meaning') NULL DEFAULT NULL;
+
+-- 1. The learner's skips and "I know this"
+UPDATE user_word SET study_choice = 'skip' WHERE user_preference = '-1';
+UPDATE user_word SET study_choice = 'known' WHERE user_preference IN ('-6', '-4', '-2');
+
+-- 2. -100: #896's write on the old word of a validator fix, else a learner's
+-- "bad translation", which stays out as a skip until reports exist
+UPDATE user_word uw
+SET uw.unfit_reason = 'replaced_by_fix'
+WHERE uw.user_preference = '-100'
+  AND EXISTS (SELECT 1 FROM validation_log v
+              WHERE v.action = 'fixed' AND v.user_word_id = uw.id AND v.meaning_id = uw.meaning_id);
+UPDATE user_word SET study_choice = 'skip' WHERE user_preference = '-100' AND unfit_reason IS NULL;
+
+-- 3. 1 is the learner's "study", except the opt-ins #895's bug forced on the
+-- target of a validator fix with no sign of the learner choosing it: no star,
+-- not added by them, no "set to study" event (roadmap item 0)
+UPDATE user_word uw
+JOIN meaning m ON m.id = uw.meaning_id
+JOIN phrase p ON p.id = m.origin_id
+SET uw.study_choice = 'study'
+WHERE uw.user_preference = '1'
+  AND NOT (
+      uw.is_user_added = 0
+      AND EXISTS (SELECT 1 FROM validation_log v
+                  WHERE v.action = 'fixed' AND v.new_meaning_id = uw.meaning_id)
+      AND NOT EXISTS (SELECT 1 FROM bookmark b
+                      WHERE b.user_word_id = uw.id
+                        AND (b.starred = 1 OR b.translation_source = 'user_added'))
+      AND NOT EXISTS (SELECT 1 FROM user_activity_data a
+                      WHERE a.user_id = uw.user_id
+                        AND a.event = 'USER SET WORD TO STUDY'
+                        AND a.value = p.content)
+  );
+
+-- 4. Words the validator found wrong with no correction: a sticky verdict.
+-- Other unfit words get their reason the next time they are recomputed.
+UPDATE user_word uw
+SET uw.unfit_reason = 'validator_invalid'
+WHERE uw.fit_for_study = 0
+  AND uw.unfit_reason IS NULL
+  AND EXISTS (SELECT 1 FROM validation_log v
+              WHERE v.action = 'invalid' AND v.user_word_id = uw.id AND v.meaning_id = uw.meaning_id);

tools/recompute_fit_after_study_choice.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+"""
+Run once after migration 26-10-25--user_word_study_choice.sql. The words whose
+forced "use in exercises" (#895) was not carried over as the learner's study
+choice are judged by the quality checks again; those that fail them leave
+study. Prints what changes; pass --apply to write.
+"""
+import sys
+import os
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from zeeguu.api.app import create_app_for_scripts
+from zeeguu.core.model import db
+
+app = create_app_for_scripts()
+app.app_context().push()
+
+from sqlalchemy import text
+
+from zeeguu.core.model.user_word import UserWord
+
+apply = "--apply" in sys.argv
+
+# user_preference is no longer on the model; the column is still in the table
+ids = [
+    row[0]
+    for row in db.session.execute(
+        text("SELECT id FROM user_word WHERE user_preference = '1' AND study_choice IS NULL")
+    )
+]
+print(f"{len(ids)} words lost a forced opt-in")
+
+leaving = 0
+for user_word_id in ids:
+    user_word = db.session.get(UserWord, user_word_id)
+    was_fit = user_word.fit_for_study
+    user_word.update_fit_for_study(db.session)
+    if was_fit and not user_word.fit_for_study:
+        leaving += 1
+        print(f"  {user_word.id} {user_word.meaning.origin.content!r} -> "
+              f"{user_word.meaning.translation.content!r}: {user_word.unfit_reason}")
+    if apply:
+        db.session.commit()
+    else:
+        db.session.rollback()
+
+print(f"{leaving} leave study{'' if apply else ' (dry run; --apply to write)'}")

zeeguu/api/endpoints/bookmarks_and_words.py

@@ -8,7 +8,7 @@
 
 from zeeguu.core.model import User, Article, Bookmark
 from zeeguu.core.model.context_identifier import ContextIdentifier
-from zeeguu.core.model.bookmark_user_preference import UserWordExPreference
+from zeeguu.core.model.study_choice import StudyChoice, UnfitReason
 from zeeguu.core.model.example_sentence_context import ExampleSentenceContext
 from . import api, db_session
 from zeeguu.api.utils.json_result import json_result
@@ -268,10 +268,10 @@ def delete_bookmark(bookmark_id):
                 else:
                     # preserve UserWord but mark it as not fit for study
                     # in the future we can generate an example for this user word with the help of the robots!
-                    user_word.set_unfit_for_study(db_session)
+                    user_word.set_unfit_for_study(db_session, UnfitReason.NO_USABLE_CONTEXT)
             else:
                 # No other bookmarks exist - ALWAYS keep the user_word for historical data
-                user_word.set_unfit_for_study(db_session)
+                user_word.set_unfit_for_study(db_session, UnfitReason.NO_USABLE_CONTEXT)
 
         # Delete any ExampleSentenceContext records that reference this bookmark
         ExampleSentenceContext.query.filter(
@@ -297,9 +297,7 @@ def set_user_word_exercise_preference(bookmark_id):
     bookmark = get_bookmark_if_owned(bookmark_id)
     user_word = bookmark.user_word
 
-    # Fix: set preference on UserWord, not Bookmark
-    user_word.user_preference = UserWordExPreference.USE_IN_EXERCISES
-    user_word.update_fit_for_study(db_session)
+    user_word.choose(db_session, StudyChoice.STUDY)
 
     # Ensure a schedule exists for this UserWord
     if user_word.fit_for_study:
@@ -321,21 +319,21 @@ def set_user_word_exercise_dislike(bookmark_id):
     reason = request.form.get("reason", "")
 
     if reason == "declared_known":
-        user_word.user_preference = UserWordExPreference.DECLARED_KNOWN
+        user_word.choose(db_session, StudyChoice.KNOWN)
         UserWordInteractionHistory.log(
             db_session, user_word,
             UserWordInteractionHistory.DECLARED_KNOWN,
         )
     elif reason == "bad_translation":
-        user_word.user_preference = UserWordExPreference.BAD_TRANSLATION
+        # Until reports exist (roadmap 0b): kept out like any skip
+        user_word.choose(db_session, StudyChoice.SKIP)
         UserWordInteractionHistory.log(
             db_session, user_word,
             UserWordInteractionHistory.BAD_TRANSLATION,
         )
     else:
-        user_word.user_preference = UserWordExPreference.DONT_USE_IN_EXERCISES
+        user_word.choose(db_session, StudyChoice.SKIP)
 
-    user_word.update_fit_for_study(db_session)
     BasicSRSchedule.clear_user_word_schedule(db_session, user_word)
     db_session.commit()
     return "OK"

zeeguu/api/endpoints/generated_examples.py

@@ -14,7 +14,7 @@
 from zeeguu.core.llm_services.llm_service import prepare_learning_card
 from zeeguu.core.model import UserWord, User, Bookmark, Language, Meaning, MeaningReport, TranslationService
 from zeeguu.core.model.ai_generator import AIGenerator
-from zeeguu.core.model.bookmark_user_preference import UserWordExPreference
+from zeeguu.core.model.study_choice import StudyChoice
 from zeeguu.core.model.context_identifier import ContextIdentifier
 from zeeguu.core.model.context_type import ContextType
 from zeeguu.core.model.example_sentence import ExampleSentence
@@ -576,8 +576,7 @@ def add_word_to_learning():
             db_session, user, meaning, is_user_added=True
         )
 
-        user_word.user_preference = UserWordExPreference.USE_IN_EXERCISES
-        user_word.update_fit_for_study(db_session)
+        user_word.choose(db_session, StudyChoice.STUDY)
 
         # Validate word position in example
         validation_result = validate_single_occurrence(final_word, final_example, origin_lang)

zeeguu/core/account_management/user_account_merge.py

@@ -311,7 +311,7 @@ def _user_word_rows(user_id):
         row.meaning_id: row
         for row in _exec(
             """
-            SELECT uw.id, uw.meaning_id, uw.fit_for_study, uw.user_preference,
+            SELECT uw.id, uw.meaning_id, uw.fit_for_study, uw.study_choice, uw.unfit_reason,
                    uw.learned_time, uw.level, uw.is_user_added,
                    uw.preferred_bookmark_id,
                    (SELECT COUNT(*) FROM bookmark b WHERE b.user_word_id = uw.id)
@@ -357,14 +357,16 @@ def _merge_user_word_pair(keep, drop):
         """
         UPDATE user_word SET
             fit_for_study = COALESCE(fit_for_study, :fit),
-            user_preference = COALESCE(user_preference, :pref),
+            study_choice = COALESCE(study_choice, :choice),
+            unfit_reason = COALESCE(unfit_reason, :reason),
             learned_time = COALESCE(learned_time, :learned),
             is_user_added = CASE WHEN :added = 1 THEN 1 ELSE is_user_added END,
             preferred_bookmark_id = COALESCE(preferred_bookmark_id, :bm)
         WHERE id = :keep
         """,
         fit=drop.fit_for_study,
-        pref=drop.user_preference,
+        choice=drop.study_choice,
+        reason=drop.unfit_reason,
         learned=drop.learned_time,
         added=1 if drop.is_user_added else 0,
         bm=drop.preferred_bookmark_id,

zeeguu/core/bookmark_operations/add_word_to_study.py

@@ -11,7 +11,7 @@
 anything is created.
 """
 
-from zeeguu.core.model.bookmark_user_preference import UserWordExPreference
+from zeeguu.core.model.study_choice import StudyChoice
 from zeeguu.core.model.context_identifier import ContextIdentifier
 from zeeguu.core.model.context_type import ContextType
 from zeeguu.core.word_scheduling import BasicSRSchedule
@@ -81,8 +81,7 @@ def add_word_to_study(
         if existing is not None and (
             BasicSRSchedule.find(existing)
             or existing.is_learned()
-            or UserWordExPreference.of(existing.user_preference)
-            != UserWordExPreference.NO_PREFERENCE
+            or existing.study_choice is not None
         ):
             return None, existing, None
 
@@ -130,8 +129,7 @@ def add_word_to_study(
     # checks that hide looked-up words ("too short", ...) do not apply. Set
     # only now, with the bookmark in place: a word left half-made by a failure
     # above keeps no preference, and the next lesson tries it again.
-    user_word.user_preference = UserWordExPreference.USE_IN_EXERCISES
-    user_word.update_fit_for_study(db_session)
+    user_word.choose(db_session, StudyChoice.STUDY)
 
     if from_audio_lesson and (
         BasicSRSchedule.scheduled_user_words_count(user, from_lang)

zeeguu/core/bookmark_operations/update_bookmark.py

@@ -30,7 +30,7 @@ def as_dict(self):
             "word": self.word,
             "context": self.context,
         }
-from zeeguu.core.model.bookmark_user_preference import UserWordExPreference
+from zeeguu.core.model.study_choice import StudyChoice
 from zeeguu.core.word_scheduling.basicSR.basicSR import BasicSRSchedule
 from zeeguu.core.word_scheduling.basicSR.four_levels_per_word import FourLevelsPerWord
 
@@ -235,7 +235,6 @@ def preserve_learning_progress(db_session, old_user_word, new_user_word):
         new_user_word: New UserWord
     """
     old_fit_for_study = old_user_word.fit_for_study
-    old_user_preference = old_user_word.user_preference
     old_level = old_user_word.level
 
     if not old_fit_for_study:
@@ -246,10 +245,9 @@ def preserve_learning_progress(db_session, old_user_word, new_user_word):
     print(f"[UPDATE_BOOKMARK] Preserving level {old_level} from old UserWord")
     new_user_word.level = old_level
 
-    if UserWordExPreference.of(old_user_preference) == UserWordExPreference.USE_IN_EXERCISES:
+    if old_user_word.study_choice == StudyChoice.STUDY:
         print(f"[UPDATE_BOOKMARK] Carrying over the learner's opt-in to exercises")
-        new_user_word.user_preference = UserWordExPreference.USE_IN_EXERCISES
-        new_user_word.update_fit_for_study(db_session)
+        new_user_word.choose(db_session, StudyChoice.STUDY)
 
 
 def transfer_learning_progress(db_session, old_user_word, new_user_word, bookmark):

zeeguu/core/bookmark_quality/__init__.py

@@ -1,2 +1,2 @@
 from .positive_qualities import quality_meaning
-from .negative_qualities import bad_quality_meaning
+from .negative_qualities import bad_quality_meaning, bad_quality_reason

zeeguu/core/bookmark_quality/fit_for_study.py

@@ -1,32 +1,40 @@
-from zeeguu.core.bookmark_quality import quality_meaning
-from zeeguu.core.model.bookmark_user_preference import UserWordExPreference
+from zeeguu.core.bookmark_quality import bad_quality_reason
+from zeeguu.core.model.study_choice import StudyChoice, UnfitReason
 
 
-def fit_for_study(user_word):
-    preference = UserWordExPreference.of(user_word.user_preference)
+def unfit_reason(user_word):
+    """
+    Why the word can't be studied now, or None. A sticky verdict stays;
+    anything else is worked out again from the word, which also clears a
+    reason that only holds until the word is judged again (UnfitReason).
+    """
+    if user_word.unfit_reason in UnfitReason.STICKY:
+        return user_word.unfit_reason
 
     # Separated MWEs (multi-word expressions with words between parts) are not fit for study
     # because fill-in-the-blank exercises don't work well with non-adjacent words
     if _is_separated_mwe(user_word):
+        return UnfitReason.SEPARATED_MWE
+
+    return bad_quality_reason(user_word)
+
+
+def fit_given(study_choice, reason):
+    """The learner's choice, then the system's reason: a STUDY choice lifts
+    only the quality heuristics."""
+    if study_choice in (StudyChoice.SKIP, StudyChoice.KNOWN):
         return False
+    if reason is None:
+        return True
+    return reason in UnfitReason.LIFTABLE and study_choice == StudyChoice.STUDY
+
 
-    return (
-        quality_meaning(user_word)
-        or preference == UserWordExPreference.USE_IN_EXERCISES
-    ) and preference >= UserWordExPreference.NO_PREFERENCE
+def fit_for_study(user_word):
+    return fit_given(user_word.study_choice, unfit_reason(user_word))
 
 
 def _is_separated_mwe(user_word):
     """Check if the user_word's preferred bookmark is a separated MWE."""
     if user_word.preferred_bookmark is None:
         return False
     return user_word.preferred_bookmark.mwe_partner_token_i is not None
-
-
-def feedback_prevents_further_study(exercise_log):
-    last_outcome = exercise_log.latest_exercise_outcome()
-
-    if not last_outcome:
-        return False
-
-    return last_outcome.free_text_feedback() or last_outcome.too_easy()

zeeguu/core/bookmark_quality/negative_qualities.py

@@ -1,4 +1,5 @@
 from zeeguu.core.model.meaning import MeaningFrequency, PhraseType
+from zeeguu.core.model.study_choice import UnfitReason
 
 
 def bad_quality_bookmark(bookmark):
@@ -27,16 +28,26 @@ def uncommon_word_for_beginner_user(user_word):
     return False
 
 
-def bad_quality_meaning(user_word):
+def bad_quality_reason(user_word):
+    """The first quality check the word fails, as an UnfitReason, or None."""
+    if uncommon_word_for_beginner_user(user_word):
+        return UnfitReason.RARE_FOR_BEGINNER
+    if arbitrary_multi_word_translation(user_word):
+        return UnfitReason.ARBITRARY_MULTI_WORD
+    if origin_same_as_translation(user_word):
+        return UnfitReason.SAME_AS_TRANSLATION
+    if origin_has_too_many_words(user_word):
+        return UnfitReason.TOO_MANY_WORDS
+    if origin_is_a_very_short_word(user_word):
+        return UnfitReason.TOO_SHORT
     bookmarks = user_word.bookmarks()
-    return (
-        uncommon_word_for_beginner_user(user_word)
-        or arbitrary_multi_word_translation(user_word)
-        or origin_same_as_translation(user_word)
-        or origin_has_too_many_words(user_word)
-        or origin_is_a_very_short_word(user_word)
-        or (bookmarks and all([bad_quality_bookmark(b) for b in bookmarks]))
-    )
+    if bookmarks and all(bad_quality_bookmark(b) for b in bookmarks):
+        return UnfitReason.BAD_CONTEXTS
+    return None
+
+
+def bad_quality_meaning(user_word):
+    return bad_quality_reason(user_word) is not None
 
 
 def arbitrary_multi_word_translation(user_word):