diff --git a/creation/management/hst.py b/creation/management/hst.py index a41d5dbe..39907873 100644 --- a/creation/management/hst.py +++ b/creation/management/hst.py @@ -1,11 +1,11 @@ FOSS_FOLDER = { - 183: "Pregnancy-Breastfeeding-and-Growth-Monitoring", - 184: "6-to-24-Months-Complementary-Feeding", - 185: "Teens-to-Adults-Nutrients-Insulin-and-Recipes", + 170: "Pregnancy-Breastfeeding-and-Growth-Monitoring", + 171: "6-to-24-Months-Complementary-Feeding", + 172: "Teens-to-Adults-Nutrients-Insulin-and-Recipes", } TOPICS_BY_FOSS = { - 183: [ + 170: [ "Cross cradle hold for breastfeeding", "Breastfeeding latching", "Importance of Breastfeeding", @@ -42,7 +42,7 @@ "WHO length-for-age Z score percentile growth charts", "Measurement of children's weight and height", ], - 184: [ + 171: [ "Essential Nutrition Actions for Young Children", "Type 1 and type 2 nutrients", "Junk food", @@ -66,7 +66,7 @@ "Nutritious Vegetarian Recipes for Kids Parties", "Delicious recipes for kids parties", ], - 185: [ + 172: [ "Vegetarian Recipes for Adolescents", "Non-vegetarian Recipes for Adolescents", "Pre-pregnancy Nutrition", diff --git a/shell-scripts/generate_hst_videos.sh b/shell-scripts/generate_hst_videos.sh index 01e1e848..7364f451 100644 --- a/shell-scripts/generate_hst_videos.sh +++ b/shell-scripts/generate_hst_videos.sh @@ -1,76 +1,189 @@ +#!/bin/bash set -e echo "=================================" echo "Starting HST → Spoken migration" echo "=================================" -HST_ROOT="./test_hst/Media/Content/Tutorial" - -# HST_ROOT="./Media/Content/Tutorial" - -SPOKEN_MEDIA="./media/videos" - -VENV_PATH="/beta_st/django_spoken.test/env_py3" - -if [ -f "$VENV_PATH/bin/activate" ]; then -source "$VENV_PATH/bin/activate" -fi +# ==================== SERVER CONFIG (COMMENTED OUT) ==================== +# HST_SERVER="Ayisha@beta.spoken-tutorial.org" +HST_ROOT="/beta_st/tomcat.new/health_data/Media/Content/Tutorial" +SPOKEN_MEDIA="/beta_st/django_spoken.test/spoken-website/media/videos" +VENV_PATH="/beta_st/django_spoken.test/env_py3" +if [ -f "$VENV_PATH/bin/activate" ]; then + source "$VENV_PATH/bin/activate" +fi PYTHON_BIN="$VENV_PATH/bin/python" -mkdir -p "$SPOKEN_MEDIA" +# ==================== LOCAL CONFIG ==================== +# HST_ROOT="./test_hst/Media/Content/Tutorial" +# SPOKEN_MEDIA="./media/videos" -copy_file () { +# source /mnt/d/EDUPYRAMIDS/spoken-website/Python-3.6.15/venv36/bin/activate +# PYTHON_BIN=$(which python) -SRC="$1" -DST="$2" - -if [ -f "$SRC" ]; then - -mkdir -p "$(dirname "$DST")" - -cp "$SRC" "$DST" +mkdir -p "$SPOKEN_MEDIA" -echo "✓ $(basename "$SRC")" +echo "" +echo -e "\033[1;34m[STEP 1]\033[0m Generating tutorial mapping..." +echo -e "\033[1;33mPython:\033[0m $PYTHON_BIN" +echo -e "\033[1;33mPWD:\033[0m $(pwd)" + +# Run Python: stdout → mapping file, stderr → debug log +"$PYTHON_BIN" manage.py shell <<'PYEOF' 2>/tmp/hst_debug.log > /tmp/hst_mapping.txt +import sys +import re +import os + +print("\033[92m[PYTHON STARTED]\033[0m", file=sys.stderr) + +from django.db import connections + +print("\033[92m[Django imports OK]\033[0m", file=sys.stderr) + +from creation.management.hst import FOSS_FOLDER + +print("\033[92m[FOSS_FOLDER imported]\033[0m", file=sys.stderr) + +source = connections["healthdb"] +target = connections["default"] + +print("\033[92m[DB connections created]\033[0m", file=sys.stderr) + +VALID_FOSS = tuple(FOSS_FOLDER.keys()) +topic_map = {} + +print("\033[92m[Loading Spoken tutorials]\033[0m", file=sys.stderr) + +# Load Spoken tutorials +with target.cursor() as cur: + + print("\033[93m[DEBUG 1] Running Spoken query\033[0m", file=sys.stderr) + + cur.execute(""" + SELECT + id, + tutorial, + foss_id + FROM creation_tutorialdetail + WHERE foss_id IN %s + """, [VALID_FOSS]) + + spoken_rows = cur.fetchall() + + print("\033[92m[DEBUG 2] Spoken rows={}\033[0m".format(len(spoken_rows)),file=sys.stderr) + + for tutorial_detail_id, tutorial, foss_id in spoken_rows: + + if tutorial: + topic_map[ + tutorial.strip().lower() + ] = ( + tutorial_detail_id, + foss_id + ) + +print("\033[92m[DEBUG 3] Topic map={}\033[0m".format(len(topic_map)),file=sys.stderr) + +print("\033[92m[DEBUG 7] Loading healthdb\033[0m", file=sys.stderr) + +# Load HST tutorial_resource rows with video column +with source.cursor() as cur: + cur.execute(""" + SELECT tutorial_id, topic_name, video + FROM tutorial_resource + WHERE topic_name IS NOT NULL + """) + rows = cur.fetchall() + +print("\033[92m[DEBUG 8] healthdb rows = {}\033[0m".format(len(rows)), file=sys.stderr) + +matched = 0 +skipped = 0 + +print("\033[92m[DEBUG 9] Starting matching\033[0m", file=sys.stderr) + +for tutorial_id, topic_name, video_path in rows: + if not topic_name: + skipped += 1 + continue + + # Extract full name from video path + # e.g., "Media/Content/Tutorial/1/Video/Cross Cradle Hold for Breastfeeding - English.mp4" + full_name = None + + if video_path: + # Get filename from path + filename = os.path.basename(video_path) # "Cross Cradle Hold for Breastfeeding - English.mp4" + # Remove extension + name_no_ext = os.path.splitext(filename)[0] # "Cross Cradle Hold for Breastfeeding - English" + # Remove language suffix + full_name = re.sub(r'\s*[-–]\s*English\s*$', '', name_no_ext, flags=re.IGNORECASE).strip() + + # Try full name from video first + if full_name: + match_key = full_name.lower() + if match_key in topic_map: + td_id, foss_id = topic_map[match_key] + print("\033[92mMATCH {}\033[0m".format(tutorial_id),file=sys.stderr) + print(f"{tutorial_id}\t{td_id}\t{foss_id}") + matched += 1 + continue + + # Fallback: try topic_name + key = topic_name.strip().lower() + if key in topic_map: + td_id, foss_id = topic_map[key] + print(f"{tutorial_id}\t{td_id}\t{foss_id}") + matched += 1 + continue + + skipped += 1 + +print(f"Matched: {matched}, Skipped: {skipped}", file=sys.stderr) +PYEOF +echo "" +echo "=== MAPPING FILE ===" +cat /tmp/hst_mapping.txt +echo "" +echo "=== LINE COUNT ===" +wc -l /tmp/hst_mapping.txt +echo "" +echo -e "\033[1;35m=== DEBUG LOG ===\033[0m" +cat /tmp/hst_debug.log + +# Check if mapping file has content +if [ ! -s /tmp/hst_mapping.txt ]; then + echo "" + echo "ERROR: Mapping file is empty! Nothing to copy." + rm -f /tmp/hst_mapping.txt /tmp/hst_debug.log + exit 1 fi -} - -echo "Generating tutorial mapping..." - -"$PYTHON_BIN" manage.py shell <<'EOF' > /tmp/hst_mapping.txt - -from creation.models import TutorialDetail - -for obj in TutorialDetail.objects.all(): - - print( - f"{obj.id}\t{obj.id}\t{obj.foss_id}" - ) - -EOF - +echo "" +echo "Copying files..." while IFS=$'\t' read -r tutorial_id tutorial_detail_id foss_id do + # Skip empty lines + [ -z "$tutorial_id" ] && continue -echo "" -echo "Tutorial: $tutorial_id" - -SRC="$HST_ROOT/$tutorial_id" - -DEST="$SPOKEN_MEDIA/$foss_id/$tutorial_detail_id" - -mkdir -p "$DEST/resources" + echo "" + echo "HST Tutorial → $tutorial_id" + echo "Tutorial Detail → $tutorial_detail_id" + echo "FOSS → $foss_id" -if [ ! -d "$SRC" ]; then + SRC="$HST_ROOT/$tutorial_id" + DEST="$SPOKEN_MEDIA/$foss_id/$tutorial_detail_id" -echo "Skipping: $SRC not found" - -continue - -fi + if [ ! -d "$SRC" ]; then + echo "Skipping: $SRC not found" + continue + fi + mkdir -p "$DEST" + mkdir -p "$DEST/resources" find "$SRC" -type f | while read file do @@ -79,35 +192,43 @@ filename=$(basename "$file") case "$file" in + */Slide/*) -copy_file \ -"$file" \ -"$DEST/resources/$filename" +cp -f "$file" "$DEST/resources/" + +echo -e "\033[92m✓ Slide:\033[0m $filename" ;; -*/Script/*) -copy_file \ -"$file" \ -"$DEST/$filename" +*/Script/*|*/TimeScript/*|*/Video/*) -;; +# ONLY exact English files +if [[ "$filename" =~ ([-[:space:]]English)\.(mp4|pdf|odt|webm|ogv)$ ]] +then -*/TimeScript/*) +new_filename=$( +echo "$filename" \ +| sed \ +-e 's/ - /-/g' \ +-e 's/ /-/g' +) -copy_file \ +cp -f \ "$file" \ -"$DEST/$filename" +"$DEST/$new_filename" -;; +echo -e "\033[92m✓ COPIED:\033[0m" -*/Video/*) +echo "OLD → $filename" +echo "NEW → $new_filename" -copy_file \ -"$file" \ -"$DEST/$filename" +else + +echo -e "\033[31mSKIPPED:\033[0m $filename" + +fi ;; @@ -117,13 +238,11 @@ done done < /tmp/hst_mapping.txt - -rm -f /tmp/hst_mapping.txt - +rm -f /tmp/hst_mapping.txt /tmp/hst_debug.log echo "" echo "=================================" echo "Migration completed" -echo "Output → media/videos" +echo "Stored under media/videos/" echo "================================="