From a8cfc53f92c8a5b88b395c1320ed11a8d80de034 Mon Sep 17 00:00:00 2001
From: ayishanishana21 <ayishanishana21@gmail.com>
Date: Fri, 19 Jun 2026 13:21:11 +0530
Subject: [PATCH 1/5] added logiv by mapping topic name

---
 shell-scripts/generate_hst_videos.sh | 255 ++++++++++++++++-----------
 1 file changed, 154 insertions(+), 101 deletions(-)

diff --git a/shell-scripts/generate_hst_videos.sh b/shell-scripts/generate_hst_videos.sh
index 01e1e848..63c861e3 100644
--- a/shell-scripts/generate_hst_videos.sh
+++ b/shell-scripts/generate_hst_videos.sh
@@ -1,129 +1,182 @@
+#!/bin/bash
 set -e
 
 echo "================================="
 echo "Starting HST → Spoken migration"
 echo "================================="
 
-HST_ROOT="./test_hst/Media/Content/Tutorial"
-
-# HST_ROOT="./Media/Content/Tutorial"
-
-SPOKEN_MEDIA="./media/videos"
-
-VENV_PATH="/beta_st/django_spoken.test/env_py3" 
-
-if [ -f "$VENV_PATH/bin/activate" ]; then 
-source "$VENV_PATH/bin/activate" 
-fi 
+# ==================== SERVER CONFIG (COMMENTED OUT) ====================
+HST_SERVER="Ayisha@beta.spoken-tutorial.org"
+HST_ROOT="/beta_st/tomcat.new/health_data/Media/Content/Tutorial"
+SPOKEN_MEDIA="/beta_st/django_spoken.test/spoken-website/media/videos"
+VENV_PATH="/beta_st/django_spoken.test/env_py3"
+if [ -f "$VENV_PATH/bin/activate" ]; then
+    source "$VENV_PATH/bin/activate"
+fi
 PYTHON_BIN="$VENV_PATH/bin/python"
 
-mkdir -p "$SPOKEN_MEDIA"
-
-copy_file () {
+# ==================== LOCAL CONFIG ====================
+# HST_ROOT="./test_hst/Media/Content/Tutorial"
+# SPOKEN_MEDIA="./media/videos"
 
-SRC="$1"
-DST="$2"
-
-if [ -f "$SRC" ]; then
-
-mkdir -p "$(dirname "$DST")"
-
-cp "$SRC" "$DST"
-
-echo "✓ $(basename "$SRC")"
-
-fi
+# source /mnt/d/EDUPYRAMIDS/spoken-website/Python-3.6.15/venv36/bin/activate
+# PYTHON_BIN=$(which python)
 
-}
+mkdir -p "$SPOKEN_MEDIA"
 
+echo ""
 echo "Generating tutorial mapping..."
 
-"$PYTHON_BIN" manage.py shell <<'EOF' > /tmp/hst_mapping.txt
-
-from creation.models import TutorialDetail
-
-for obj in TutorialDetail.objects.all():
-
-    print(
-        f"{obj.id}\t{obj.id}\t{obj.foss_id}"
-    )
-
-EOF
-
-
-while IFS=$'\t' read -r tutorial_id tutorial_detail_id foss_id
-do
+# Run Python: stdout → mapping file, stderr → debug log
+"$PYTHON_BIN" manage.py shell <<'PYEOF' 2>/tmp/hst_debug.log > /tmp/hst_mapping.txt
+import sys
+import re
+import os
+from django.db import connections
+from creation.management.hst import FOSS_FOLDER
+
+source = connections["healthdb"]
+target = connections["default"]
+
+VALID_FOSS = tuple(FOSS_FOLDER.keys())
+topic_map = {}
+
+# Load Spoken tutorials
+with target.cursor() as cur:
+    cur.execute("""
+        SELECT id, tutorial, foss_id
+        FROM creation_tutorialdetail
+        WHERE foss_id IN %s
+    """, [VALID_FOSS])
+
+    for tutorial_detail_id, tutorial, foss_id in cur.fetchall():
+        if tutorial:
+            topic_map[tutorial.strip().lower()] = (tutorial_detail_id, foss_id)
+
+# Load HST tutorial_resource rows with video column
+with source.cursor() as cur:
+    cur.execute("""
+        SELECT tutorial_id, topic_name, video
+        FROM tutorial_resource
+        WHERE topic_name IS NOT NULL
+    """)
+    rows = cur.fetchall()
+
+matched = 0
+skipped = 0
+
+for tutorial_id, topic_name, video_path in rows:
+    if not topic_name:
+        skipped += 1
+        continue
+
+    # Extract full name from video path
+    # e.g., "Media/Content/Tutorial/1/Video/Cross Cradle Hold for Breastfeeding - English.mp4"
+    full_name = None
+    
+    if video_path:
+        # Get filename from path
+        filename = os.path.basename(video_path)  # "Cross Cradle Hold for Breastfeeding - English.mp4"
+        # Remove extension
+        name_no_ext = os.path.splitext(filename)[0]  # "Cross Cradle Hold for Breastfeeding - English"
+        # Remove language suffix
+        full_name = re.sub(r'\s*[-–]\s*English\s*$', '', name_no_ext, flags=re.IGNORECASE).strip()
+    
+    # Try full name from video first
+    if full_name:
+        match_key = full_name.lower()
+        if match_key in topic_map:
+            td_id, foss_id = topic_map[match_key]
+            print(f"{tutorial_id}\t{td_id}\t{foss_id}")
+            matched += 1
+            continue
+    
+    # Fallback: try topic_name
+    key = topic_name.strip().lower()
+    if key in topic_map:
+        td_id, foss_id = topic_map[key]
+        print(f"{tutorial_id}\t{td_id}\t{foss_id}")
+        matched += 1
+        continue
+
+    skipped += 1
+
+print(f"Matched: {matched}, Skipped: {skipped}", file=sys.stderr)
+PYEOF
 
 echo ""
-echo "Tutorial: $tutorial_id"
-
-SRC="$HST_ROOT/$tutorial_id"
-
-DEST="$SPOKEN_MEDIA/$foss_id/$tutorial_detail_id"
-
-mkdir -p "$DEST/resources"
-
-if [ ! -d "$SRC" ]; then
-
-echo "Skipping: $SRC not found"
-
-continue
-
+echo "=== MAPPING FILE ==="
+cat /tmp/hst_mapping.txt
+echo ""
+echo "=== LINE COUNT ==="
+wc -l /tmp/hst_mapping.txt
+echo ""
+echo "=== DEBUG LOG ==="
+cat /tmp/hst_debug.log
+
+# Check if mapping file has content
+if [ ! -s /tmp/hst_mapping.txt ]; then
+    echo ""
+    echo "ERROR: Mapping file is empty! Nothing to copy."
+    rm -f /tmp/hst_mapping.txt /tmp/hst_debug.log
+    exit 1
 fi
 
+echo ""
+echo "Copying files..."
 
-find "$SRC" -type f | while read file
+while IFS=$'\t' read -r tutorial_id tutorial_detail_id foss_id
 do
-
-filename=$(basename "$file")
-
-case "$file" in
-
-*/Slide/*)
-
-copy_file \
-"$file" \
-"$DEST/resources/$filename"
-
-;;
-
-*/Script/*)
-
-copy_file \
-"$file" \
-"$DEST/$filename"
-
-;;
-
-*/TimeScript/*)
-
-copy_file \
-"$file" \
-"$DEST/$filename"
-
-;;
-
-*/Video/*)
-
-copy_file \
-"$file" \
-"$DEST/$filename"
-
-;;
-
-esac
-
-done
+    # Skip empty lines
+    [ -z "$tutorial_id" ] && continue
+
+    echo ""
+    echo "HST Tutorial → $tutorial_id"
+    echo "Tutorial Detail → $tutorial_detail_id"
+    echo "FOSS → $foss_id"
+
+    SRC="$HST_ROOT/$tutorial_id"
+    DEST="$SPOKEN_MEDIA/$foss_id/$tutorial_detail_id"
+
+    if [ ! -d "$SRC" ]; then
+        echo "Skipping: $SRC not found"
+        continue
+    fi
+
+    mkdir -p "$DEST"
+    mkdir -p "$DEST/resources"
+
+    find "$SRC" -type f | while read file
+    do
+        filename=$(basename "$file")
+
+        case "$file" in
+            */Slide/*)
+                cp -f "$file" "$DEST/resources/"
+                echo "  ✓ Slide: $filename"
+                ;;
+            */Script/*)
+                cp -f "$file" "$DEST/"
+                echo "  ✓ Script: $filename"
+                ;;
+            */TimeScript/*)
+                cp -f "$file" "$DEST/"
+                echo "  ✓ TimeScript: $filename"
+                ;;
+            */Video/*)
+                cp -f "$file" "$DEST/"
+                echo "  ✓ Video: $filename"
+                ;;
+        esac
+    done
 
 done < /tmp/hst_mapping.txt
 
-
-rm -f /tmp/hst_mapping.txt
-
+rm -f /tmp/hst_mapping.txt /tmp/hst_debug.log
 
 echo ""
 echo "================================="
 echo "Migration completed"
-echo "Output → media/videos"
+echo "Stored under media/videos/"
 echo "================================="
 

From 544bd221225759b4c034ceb956e9322598f5a52a Mon Sep 17 00:00:00 2001
From: ayishanishana21 <ayishanishana21@gmail.com>
Date: Mon, 22 Jun 2026 10:28:49 +0530
Subject: [PATCH 2/5] added print statement along with the fix

---
 shell-scripts/generate_hst_videos.sh | 73 +++++++++++++++++++++-------
 1 file changed, 56 insertions(+), 17 deletions(-)

diff --git a/shell-scripts/generate_hst_videos.sh b/shell-scripts/generate_hst_videos.sh
index 63c861e3..cb71dc3b 100644
--- a/shell-scripts/generate_hst_videos.sh
+++ b/shell-scripts/generate_hst_videos.sh
@@ -6,52 +6,86 @@ echo "Starting HST → Spoken migration"
 echo "================================="
 
 # ==================== SERVER CONFIG (COMMENTED OUT) ====================
-HST_SERVER="Ayisha@beta.spoken-tutorial.org"
-HST_ROOT="/beta_st/tomcat.new/health_data/Media/Content/Tutorial"
-SPOKEN_MEDIA="/beta_st/django_spoken.test/spoken-website/media/videos"
-VENV_PATH="/beta_st/django_spoken.test/env_py3"
-if [ -f "$VENV_PATH/bin/activate" ]; then
-    source "$VENV_PATH/bin/activate"
-fi
-PYTHON_BIN="$VENV_PATH/bin/python"
+# HST_SERVER="Ayisha@beta.spoken-tutorial.org"
+# HST_ROOT="/beta_st/tomcat.new/health_data/Media/Content/Tutorial"
+# SPOKEN_MEDIA="/beta_st/django_spoken.test/spoken-website/media/videos"
+# VENV_PATH="/beta_st/django_spoken.test/env_py3"
+# if [ -f "$VENV_PATH/bin/activate" ]; then
+#     source "$VENV_PATH/bin/activate"
+# fi
+# PYTHON_BIN="$VENV_PATH/bin/python"
 
 # ==================== LOCAL CONFIG ====================
-# HST_ROOT="./test_hst/Media/Content/Tutorial"
-# SPOKEN_MEDIA="./media/videos"
+HST_ROOT="./test_hst/Media/Content/Tutorial"
+SPOKEN_MEDIA="./media/videos"
 
-# source /mnt/d/EDUPYRAMIDS/spoken-website/Python-3.6.15/venv36/bin/activate
-# PYTHON_BIN=$(which python)
+source /mnt/d/EDUPYRAMIDS/spoken-website/Python-3.6.15/venv36/bin/activate
+PYTHON_BIN=$(which python)
 
 mkdir -p "$SPOKEN_MEDIA"
 
 echo ""
-echo "Generating tutorial mapping..."
+echo -e "\033[1;34m[STEP 1]\033[0m Generating tutorial mapping..."
+echo -e "\033[1;33mPython:\033[0m $PYTHON_BIN"
+echo -e "\033[1;33mPWD:\033[0m $(pwd)"
 
 # Run Python: stdout → mapping file, stderr → debug log
 "$PYTHON_BIN" manage.py shell <<'PYEOF' 2>/tmp/hst_debug.log > /tmp/hst_mapping.txt
 import sys
 import re
 import os
+
+print("\033[92m[PYTHON STARTED]\033[0m", file=sys.stderr)
+
 from django.db import connections
+
+print("\033[92m[Django imports OK]\033[0m", file=sys.stderr)
+
 from creation.management.hst import FOSS_FOLDER
 
+print("\033[92m[FOSS_FOLDER imported]\033[0m", file=sys.stderr)
+
 source = connections["healthdb"]
 target = connections["default"]
 
+print("\033[92m[DB connections created]\033[0m", file=sys.stderr)
+
 VALID_FOSS = tuple(FOSS_FOLDER.keys())
 topic_map = {}
 
+print("\033[92m[Loading Spoken tutorials]\033[0m", file=sys.stderr)
+
 # Load Spoken tutorials
 with target.cursor() as cur:
+
+    print("\033[93m[DEBUG 1] Running Spoken query\033[0m", file=sys.stderr)
+
     cur.execute("""
-        SELECT id, tutorial, foss_id
+        SELECT
+            id,
+            tutorial,
+            foss_id
         FROM creation_tutorialdetail
         WHERE foss_id IN %s
     """, [VALID_FOSS])
 
-    for tutorial_detail_id, tutorial, foss_id in cur.fetchall():
+    spoken_rows = cur.fetchall()
+
+    print("\033[92m[DEBUG 2] Spoken rows={}\033[0m".format(len(spoken_rows)),file=sys.stderr)
+
+    for tutorial_detail_id, tutorial, foss_id in spoken_rows:
+
         if tutorial:
-            topic_map[tutorial.strip().lower()] = (tutorial_detail_id, foss_id)
+            topic_map[
+                tutorial.strip().lower()
+            ] = (
+                tutorial_detail_id,
+                foss_id
+            )
+
+print("\033[92m[DEBUG 3] Topic map={}\033[0m".format(len(topic_map)),file=sys.stderr)
+
+print("\033[92m[DEBUG 7] Loading healthdb\033[0m", file=sys.stderr)
 
 # Load HST tutorial_resource rows with video column
 with source.cursor() as cur:
@@ -62,9 +96,13 @@ with source.cursor() as cur:
     """)
     rows = cur.fetchall()
 
+print("\033[92m[DEBUG 8] healthdb rows = {}\033[0m".format(len(rows)), file=sys.stderr)
+
 matched = 0
 skipped = 0
 
+print("\033[92m[DEBUG 9] Starting matching\033[0m", file=sys.stderr)
+
 for tutorial_id, topic_name, video_path in rows:
     if not topic_name:
         skipped += 1
@@ -87,6 +125,7 @@ for tutorial_id, topic_name, video_path in rows:
         match_key = full_name.lower()
         if match_key in topic_map:
             td_id, foss_id = topic_map[match_key]
+            print("\033[92mMATCH {}\033[0m".format(tutorial_id),file=sys.stderr)
             print(f"{tutorial_id}\t{td_id}\t{foss_id}")
             matched += 1
             continue
@@ -111,7 +150,7 @@ echo ""
 echo "=== LINE COUNT ==="
 wc -l /tmp/hst_mapping.txt
 echo ""
-echo "=== DEBUG LOG ==="
+echo -e "\033[1;35m=== DEBUG LOG ===\033[0m"
 cat /tmp/hst_debug.log
 
 # Check if mapping file has content

From ad5bd136cc2c7045972224d699a0b001e193a8a8 Mon Sep 17 00:00:00 2001
From: ayishanishana21 <ayishanishana21@gmail.com>
Date: Mon, 22 Jun 2026 10:37:05 +0530
Subject: [PATCH 3/5] added foss name acc beta

---
 creation/management/hst.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/creation/management/hst.py b/creation/management/hst.py
index a41d5dbe..39907873 100644
--- a/creation/management/hst.py
+++ b/creation/management/hst.py
@@ -1,11 +1,11 @@
 FOSS_FOLDER = {
-    183: "Pregnancy-Breastfeeding-and-Growth-Monitoring",
-    184: "6-to-24-Months-Complementary-Feeding",
-    185: "Teens-to-Adults-Nutrients-Insulin-and-Recipes",
+    170: "Pregnancy-Breastfeeding-and-Growth-Monitoring",
+    171: "6-to-24-Months-Complementary-Feeding",
+    172: "Teens-to-Adults-Nutrients-Insulin-and-Recipes",
 }
 
 TOPICS_BY_FOSS = {
-    183: [
+    170: [
         "Cross cradle hold for breastfeeding",
         "Breastfeeding latching",
         "Importance of Breastfeeding",
@@ -42,7 +42,7 @@
         "WHO length-for-age Z score percentile growth charts",
         "Measurement of children's weight and height",
     ],
-    184: [
+    171: [
         "Essential Nutrition Actions for Young Children",
         "Type 1 and type 2 nutrients",
         "Junk food",
@@ -66,7 +66,7 @@
         "Nutritious Vegetarian Recipes for Kids Parties",
         "Delicious recipes for kids parties",
     ],
-    185: [
+    172: [
         "Vegetarian Recipes for Adolescents",
         "Non-vegetarian Recipes for Adolescents",
         "Pre-pregnancy Nutrition",

From 4d400c933b3d4718740c4252404c4b26df71d425 Mon Sep 17 00:00:00 2001
From: ayishanishana21 <ayishanishana21@gmail.com>
Date: Mon, 22 Jun 2026 10:38:33 +0530
Subject: [PATCH 4/5] given server path

---
 shell-scripts/generate_hst_videos.sh | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/shell-scripts/generate_hst_videos.sh b/shell-scripts/generate_hst_videos.sh
index cb71dc3b..d4aece3d 100644
--- a/shell-scripts/generate_hst_videos.sh
+++ b/shell-scripts/generate_hst_videos.sh
@@ -7,20 +7,20 @@ echo "================================="
 
 # ==================== SERVER CONFIG (COMMENTED OUT) ====================
 # HST_SERVER="Ayisha@beta.spoken-tutorial.org"
-# HST_ROOT="/beta_st/tomcat.new/health_data/Media/Content/Tutorial"
-# SPOKEN_MEDIA="/beta_st/django_spoken.test/spoken-website/media/videos"
-# VENV_PATH="/beta_st/django_spoken.test/env_py3"
-# if [ -f "$VENV_PATH/bin/activate" ]; then
-#     source "$VENV_PATH/bin/activate"
-# fi
-# PYTHON_BIN="$VENV_PATH/bin/python"
+HST_ROOT="/beta_st/tomcat.new/health_data/Media/Content/Tutorial"
+SPOKEN_MEDIA="/beta_st/django_spoken.test/spoken-website/media/videos"
+VENV_PATH="/beta_st/django_spoken.test/env_py3"
+if [ -f "$VENV_PATH/bin/activate" ]; then
+    source "$VENV_PATH/bin/activate"
+fi
+PYTHON_BIN="$VENV_PATH/bin/python"
 
 # ==================== LOCAL CONFIG ====================
-HST_ROOT="./test_hst/Media/Content/Tutorial"
-SPOKEN_MEDIA="./media/videos"
+# HST_ROOT="./test_hst/Media/Content/Tutorial"
+# SPOKEN_MEDIA="./media/videos"
 
-source /mnt/d/EDUPYRAMIDS/spoken-website/Python-3.6.15/venv36/bin/activate
-PYTHON_BIN=$(which python)
+# source /mnt/d/EDUPYRAMIDS/spoken-website/Python-3.6.15/venv36/bin/activate
+# PYTHON_BIN=$(which python)
 
 mkdir -p "$SPOKEN_MEDIA"
 

From f8baf54e995bd89e8324a92c4f371ecbd153b2d8 Mon Sep 17 00:00:00 2001
From: ayishanishana21 <ayishanishana21@gmail.com>
Date: Tue, 23 Jun 2026 15:34:09 +0530
Subject: [PATCH 5/5] filtered out english tutorial

---
 shell-scripts/generate_hst_videos.sh | 73 +++++++++++++++++++---------
 1 file changed, 50 insertions(+), 23 deletions(-)

diff --git a/shell-scripts/generate_hst_videos.sh b/shell-scripts/generate_hst_videos.sh
index d4aece3d..7364f451 100644
--- a/shell-scripts/generate_hst_videos.sh
+++ b/shell-scripts/generate_hst_videos.sh
@@ -185,29 +185,56 @@ do
     mkdir -p "$DEST"
     mkdir -p "$DEST/resources"
 
-    find "$SRC" -type f | while read file
-    do
-        filename=$(basename "$file")
-
-        case "$file" in
-            */Slide/*)
-                cp -f "$file" "$DEST/resources/"
-                echo "  ✓ Slide: $filename"
-                ;;
-            */Script/*)
-                cp -f "$file" "$DEST/"
-                echo "  ✓ Script: $filename"
-                ;;
-            */TimeScript/*)
-                cp -f "$file" "$DEST/"
-                echo "  ✓ TimeScript: $filename"
-                ;;
-            */Video/*)
-                cp -f "$file" "$DEST/"
-                echo "  ✓ Video: $filename"
-                ;;
-        esac
-    done
+find "$SRC" -type f | while read file
+do
+
+filename=$(basename "$file")
+
+case "$file" in
+
+
+*/Slide/*)
+
+cp -f "$file" "$DEST/resources/"
+
+echo -e "\033[92m✓ Slide:\033[0m $filename"
+
+;;
+
+
+*/Script/*|*/TimeScript/*|*/Video/*)
+
+# ONLY exact English files
+if [[ "$filename" =~ ([-[:space:]]English)\.(mp4|pdf|odt|webm|ogv)$ ]]
+then
+
+new_filename=$(
+echo "$filename" \
+| sed \
+-e 's/ - /-/g' \
+-e 's/ /-/g'
+)
+
+cp -f \
+"$file" \
+"$DEST/$new_filename"
+
+echo -e "\033[92m✓ COPIED:\033[0m"
+
+echo "OLD → $filename"
+echo "NEW → $new_filename"
+
+else
+
+echo -e "\033[31mSKIPPED:\033[0m $filename"
+
+fi
+
+;;
+
+esac
+
+done
 
 done < /tmp/hst_mapping.txt