diff --git a/CHANGELOG.md b/CHANGELOG.md index 69d2919..b6fa0ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ uses: qBraid/upload-course-action@v0.1.0-beta ## [Unreleased] ### Fixed +- Report invalid mapping JSON arrays and scalars as CLI validation errors without tracebacks or file changes. +- Preserve `chapterQuestions` and `sectionQuestions` in deployment payloads so copied question references can trigger an existing course rebuild. - Fixed JSend response parsing in deploy and polling scripts to support `{status, data}` envelope format - Added guard in `action.yml` to fail early with a clear error when `course_custom_id` is empty after create/update step - Replaced silent `pass` with warning logs when API response parsing fails @@ -37,6 +39,7 @@ uses: qBraid/upload-course-action@v0.1.0-beta - Treat active pre-existing kernels as a successful deploy outcome for rerun-safe custom kernel workflows ### Added +- Add `prepare_question_copy.py` to apply a saved MCP copy mapping to destination notebooks and derive matching course question references, with preview, write, and check modes. - Unit tests for JSend response handling in course creation and polling - Forward the optional `durationWeeks` field (author-declared course length in weeks, integer 1-52) from `course.json` to the deploy payload; previously the validate step's model round-trip silently dropped it. An out-of-range or non-integer value (including a numeric string such as `"3"`) now fails validation with a clear message, and an absent field serializes as `null`, which the qBraid API treats as not declared - Unit tests for shared deploy payload/response handling in `deploy_common.py` diff --git a/docs/question-copy-workflow.md b/docs/question-copy-workflow.md new file mode 100644 index 0000000..12d2b4f --- /dev/null +++ b/docs/question-copy-workflow.md @@ -0,0 +1,92 @@ +# Copy questions into a GitHub-backed course + +Use an agent with qBraid MCP and GitHub repository access to copy questions, +prepare a course PR, and deploy the reviewed commit. This procedure assumes the +API's question-copy feature is deployed. + +## Save the copy result + +1. Select the source questions and destination organization through MCP. +2. Call `copy_qbook_questions`. For course-based authorization, supply both + `sourceCourseId` and `targetCourseId`. +3. Save the decoded JSON result to `.qbraid/question-copy.json` in the destination + repository. Save only the result, without access tokens or HTTP headers. + +The result contains a `questions` array. Each item includes `sourceCustomId`, +`customId`, and `type`. The helper also accepts a REST response with this array +under `data`. Retain the saved mapping for retries; do not copy again because a +repository edit or deployment failed. + +## Prepare the destination PR + +Check out a reviewed revision of `qBraid/upload-course-action` that includes +`src/scripts/prepare_question_copy.py`. Install that revision's Python +dependencies. Set `UPLOAD_ACTION_DIR` to the absolute path of that checkout. + +From the destination course repository root: + +1. Create a feature branch. +2. Add the source question widgets at the requested positions in the destination + notebook. If you copied a whole lesson, its existing widgets already provide + these positions. Put repeated widgets of the same type in separate cells. +3. Preview the reference changes: + + ```sh + python "$UPLOAD_ACTION_DIR/src/scripts/prepare_question_copy.py" \ + --mapping .qbraid/question-copy.json + ``` + +4. Apply the changes: + + ```sh + python "$UPLOAD_ACTION_DIR/src/scripts/prepare_question_copy.py" \ + --mapping .qbraid/question-copy.json --write + ``` + +5. Verify that all changes are applied: + + ```sh + python "$UPLOAD_ACTION_DIR/src/scripts/prepare_question_copy.py" \ + --mapping .qbraid/question-copy.json --check + ``` + + Expect `"files": []`. Use the same command in PR validation. It does not + authenticate, create backend questions, deploy, or modify files. + +6. Review and commit the changed notebooks, complete `course.json`, and saved + mapping. Open a PR that identifies the destination course and organization. + +Use `--course-json path/to/course.json` for a nondefault manifest path. Notebook +paths remain relative to the repository root, matching the deployment Action. +The helper updates every occurrence of a mapped widget in notebooks listed by +that manifest. It preserves unrelated references and derives both MCQ and FRQ +placement lists. Existing CQ lists remain unchanged. + +The helper refuses unused mappings and ambiguous mappings before writing. It +does not insert cells, select lesson positions, check backend permissions, or +verify that an unrelated question belongs to the destination organization. The +agent must verify destination question access before opening the PR. + +## Deploy the reviewed commit + +Merge the course PR and run its deployment workflow with: + +- A pinned Action revision containing this helper and the reference-preserving + validator. An older Action silently drops the reference fields. +- `mode: update` and the existing destination `course-custom-id`. +- The destination organization's API key from GitHub secrets. +- `force-duplicate-questions: 'false'`. +- `repo-read-token` with access to the destination repository. + +For a dedicated API preview, set `QBRAID_API_BASE_URL` in the Action step's +environment to that preview's `/api/v1` URL. Point the qBook preview at the same +API preview. Keep the normal workflow on the intended production or staging API. + +The Action must receive the complete course manifest. The helper-generated +`chapterQuestions` and `sectionQuestions` fields survive validation and make a +question-reference change visible to API redeployment detection. The processor +then regenerates the lesson JSON from the merged repository commit. + +Open the deployed lesson and answer both copied question types. Rerun the +deployment and confirm IDs and points remain unchanged. A processed status by +itself does not prove the lesson renders. diff --git a/src/scripts/prepare_question_copy.py b/src/scripts/prepare_question_copy.py new file mode 100644 index 0000000..2c89624 --- /dev/null +++ b/src/scripts/prepare_question_copy.py @@ -0,0 +1,196 @@ +# Copyright (C) 2026 qBraid + +"""Apply a saved MCP copy result to a destination course checkout before a PR.""" + +import argparse +import json +import re +from collections import Counter +from pathlib import Path +from typing import Literal + +from pydantic import BaseModel, Field, field_validator + +MARKERS = {"mcq": "MultipleChoiceQuestion", "frq": "FreeResponseQuestion"} +PATTERNS = { + kind: re.compile(rf"{marker}\(\s*(['\"])([^'\"]+)\1\s*\)") + for kind, marker in MARKERS.items() +} +CUSTOM_ID = r"^[\w.\-:]{10,200}$" + + +class CopiedQuestion(BaseModel): + """The reference mapping returned by copy_qbook_questions.""" + + sourceCustomId: str = Field(pattern=CUSTOM_ID) + customId: str = Field(pattern=CUSTOM_ID) + type: Literal["mcq", "frq"] + + +class CopyResult(BaseModel): + """A saved, successful MCP result or the data from the REST response.""" + + questions: list[CopiedQuestion] = Field(min_length=1, max_length=100) + + @field_validator("questions") + @classmethod + def unique_mapping(cls, questions: list[CopiedQuestion]) -> list[CopiedQuestion]: + """Reject ambiguous mappings so applying the result twice is safe.""" + sources = [question.sourceCustomId for question in questions] + targets = [question.customId for question in questions] + if len(set(sources)) != len(sources) or len(set(targets)) != len(targets): + raise ValueError("Source and destination customIds must each be unique") + if set(sources) & set(targets): + raise ValueError("Source and destination customIds must not overlap") + return questions + + +def repository_path(root: Path, path: str) -> Path: + """Resolve an existing file without allowing writes outside the checkout.""" + resolved = (root / path).resolve(strict=True) + if not resolved.is_relative_to(root): + raise ValueError(f"Path is outside the destination repository: {path}") + if not resolved.is_file(): + raise ValueError(f"Expected a file: {path}") + return resolved + + +def rewrite_notebook( + notebook: dict, questions: list[CopiedQuestion] +) -> tuple[dict, set]: + """Rewrite widget arguments and derive the processor's placement lists. + + The processor records one widget per type per cell and numbers repeated + placements within each file. Refuse extra widgets instead of silently losing + a reference when the course is deployed. + """ + mapping = { + kind: {q.sourceCustomId: q.customId for q in questions if q.type == kind} + for kind in MARKERS + } + targets = { + kind: {q.customId for q in questions if q.type == kind} for kind in MARKERS + } + references = {"MCQ": [], "FRQ": []} + seen = set() + counts = {kind: Counter() for kind in MARKERS} + for cell in notebook["cells"]: + source = cell["source"] + text = "".join(source) if isinstance(source, list) else source + original_text = text + for kind, pattern in PATTERNS.items(): + if len(list(pattern.finditer(text))) > 1: + raise ValueError( + f"Put each {MARKERS[kind]} in a separate notebook cell" + ) + + def replace(match, question_type=kind): + old = match.group(2) + new = mapping[question_type].get(old, old) + if new in targets[question_type]: + seen.add((question_type, new)) + occurrence = counts[question_type][new] + counts[question_type][new] += 1 + references[question_type.upper()].append(f"{new}:{occurrence}") + # Preserve quote style, whitespace, and surrounding source. + start = match.start(2) - match.start() + end = match.end(2) - match.start() + return match.group(0)[:start] + new + match.group(0)[end:] + + text = pattern.sub(replace, text) + if text != original_text: + cell["source"] = ( + text.splitlines(keepends=True) if isinstance(source, list) else text + ) + return references, seen + + +def prepare_copy( + root: Path, course_json: str, mapping_file: str, *, write: bool = False +) -> list[str]: + """Validate all changes before writing; reuse the saved mapping on retry. + + Notebook paths are relative to the repository root, matching the upload + Action. This command neither copies backend records nor deploys a course. + """ + root = root.resolve() + course_path = repository_path(root, course_json) + mapping_path = repository_path(root, mapping_file) + saved = json.loads(mapping_path.read_text(encoding="utf-8")) + payload = saved.get("data", saved) if isinstance(saved, dict) else saved + result = CopyResult.model_validate(payload) + original_course = json.loads(course_path.read_text(encoding="utf-8")) + course = json.loads(json.dumps(original_course)) + pending = {} + seen = set() + notebooks = {} + for chapter in course["content"]: + items = [(chapter, "chapterQuestions")] + items.extend( + (section, "sectionQuestions") for section in chapter.get("sections") or [] + ) + for item, field in items: + notebook_path = repository_path(root, item["baseFilePath"]) + if notebook_path not in notebooks: + original = json.loads(notebook_path.read_text(encoding="utf-8")) + notebook = json.loads(json.dumps(original)) + references, used = rewrite_notebook(notebook, result.questions) + notebooks[notebook_path] = references + seen.update(used) + if notebook != original: + pending[notebook_path] = notebook + # This operation owns MCQ and FRQ; retain any CQ references. + item[field] = {**(item.get(field) or {}), **notebooks[notebook_path]} + unused = {(q.type, q.customId) for q in result.questions} - seen + if unused: + raise ValueError( + "Copied questions are missing from the destination notebooks: " + + ", ".join(sorted(custom_id for _, custom_id in unused)) + ) + if course != original_course: + pending[course_path] = course + if write: + for path, document in pending.items(): + path.write_text( + json.dumps(document, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + return [str(path.relative_to(root)) for path in pending] + + +def main() -> None: + """Preview changes by default; write or check them only when requested.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--course-json", default="course.json") + parser.add_argument( + "--mapping", + required=True, + help="Saved MCP result or REST response, without credentials", + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "--write", action="store_true", help="Update notebooks and course.json" + ) + mode.add_argument( + "--check", + action="store_true", + help="Fail if notebooks or course.json still need changes", + ) + args = parser.parse_args() + try: + changed = prepare_copy( + Path.cwd(), args.course_json, args.mapping, write=args.write + ) + except (OSError, ValueError, KeyError, TypeError) as error: + parser.exit(2, f"Cannot prepare question copy: {error}\n") + print( + json.dumps({"files": changed, "written": args.write, "courseDeployed": False}) + ) + if args.check and changed: + parser.exit( + 1, "Apply the saved mapping with --write and commit the changed files.\n" + ) + + +if __name__ == "__main__": + main() diff --git a/src/scripts/validate_course.py b/src/scripts/validate_course.py index f03f336..2265a25 100644 --- a/src/scripts/validate_course.py +++ b/src/scripts/validate_course.py @@ -5,10 +5,10 @@ import sys import urllib.request from pathlib import Path -from typing import List, Optional, Set +from typing import Any, Dict, List, Optional, Set from common import Config, setup_logging -from pydantic import BaseModel, Field, ValidationError, field_validator +from pydantic import BaseModel, ConfigDict, Field, ValidationError, field_validator logger = setup_logging(__name__) @@ -50,6 +50,16 @@ class ImageLink(BaseModel): lightLogo: str +class QuestionReferences(BaseModel): + """Question placements from notebooks, including occurrence suffixes.""" + + model_config = ConfigDict(extra="forbid") + + MCQ: List[str] = Field(default_factory=list) + FRQ: List[str] = Field(default_factory=list) + CQ: Optional[List[str]] = None + + class Section(BaseModel): """Model for course sections.""" @@ -58,6 +68,7 @@ class Section(BaseModel): baseFilePath: Path kernelName: str kernelId: str + sectionQuestions: Optional[QuestionReferences] = None @field_validator("baseFilePath") @classmethod @@ -84,6 +95,7 @@ class Chapter(BaseModel): chapterNumber: float kernelName: str kernelId: str + chapterQuestions: Optional[QuestionReferences] = None sections: Optional[List[Section]] = [] @field_validator("baseFilePath") @@ -118,6 +130,21 @@ class Course(BaseModel): # would otherwise coerce "3" to 3 and forward a value the API rejects. durationWeeks: Optional[int] = Field(None, ge=1, le=52, strict=True) + def to_payload(self) -> Dict[str, Any]: + """Preserve supplied references without adding nulls to existing courses.""" + payload = self.model_dump(mode="json") + for chapter in payload["content"]: + items = [(chapter, "chapterQuestions")] + items.extend( + (section, "sectionQuestions") for section in chapter["sections"] or [] + ) + for item, key in items: + if item[key] is None: + item.pop(key) + elif item[key]["CQ"] is None: + item[key].pop("CQ") + return payload + @field_validator("deployedTo") @classmethod def check_domains(cls, v: List[str]) -> List[str]: @@ -202,7 +229,7 @@ def validate(self) -> None: # the same as an absent field. try: with open(Config.COURSE_DATA_FILE_NAME, "w") as f: - json.dump(course.model_dump(mode="json"), f) + json.dump(course.to_payload(), f) except IOError as e: logger.error(f"Failed to write {Config.COURSE_DATA_FILE_NAME}: {e}") sys.exit(1) diff --git a/test/unit/test_prepare_question_copy.py b/test/unit/test_prepare_question_copy.py new file mode 100644 index 0000000..7cf33dd --- /dev/null +++ b/test/unit/test_prepare_question_copy.py @@ -0,0 +1,255 @@ +"""Exercise the saved MCP mapping through notebook edits and Action payloads.""" + +import json +import subprocess +import sys +from pathlib import Path + +import nbformat +import pytest +from common import Config +from create_course import CourseCreator +from prepare_question_copy import prepare_copy +from update_course import CourseUpdater +from validate_course import Course, validate_course_json +from verify_notebooks import NotebookVerifier + +SCRIPT = Path(__file__).resolve().parents[2] / "src/scripts/prepare_question_copy.py" +MAPPING = { + "questions": [ + { + "sourceCustomId": "source-question-mcq", + "customId": "copied-question-mcq", + "type": "mcq", + }, + { + "sourceCustomId": "source-question-frq", + "customId": "copied-question-frq", + "type": "frq", + }, + ] +} + + +@pytest.fixture +def checkout(tmp_path, sample_course_json, monkeypatch): + monkeypatch.chdir(tmp_path) + chapter = sample_course_json["content"][0] + chapter["baseFilePath"] = "chapter1.ipynb" + chapter["chapterQuestions"] = {"CQ": ["existing-coding-question:0"]} + chapter["sections"] = [ + { + "sectionName": "Section", + "sectionNumber": 1, + "baseFilePath": "section.ipynb", + "kernelName": "qbraid_python", + "kernelId": "qbraid_python", + } + ] + (tmp_path / "course.json").write_text(json.dumps(sample_course_json)) + (tmp_path / "question-copy.json").write_text(json.dumps(MAPPING)) + chapter_nb = nbformat.v4.new_notebook( + cells=[ + nbformat.v4.new_markdown_cell( + "Keep source-question-mcq in this explanation." + ), + nbformat.v4.new_code_cell( + "MultipleChoiceQuestion( 'source-question-mcq' )\n" + ), + nbformat.v4.new_code_cell('MultipleChoiceQuestion("source-question-mcq")'), + nbformat.v4.new_code_cell('MultipleChoiceQuestion("unrelated-question")'), + ] + ) + # Exercise the multiline list encoding used by checked-in .ipynb files. + chapter_nb["cells"][1]["source"] = [ + "MultipleChoiceQuestion( ", + "'source-question-mcq' )\n", + ] + (tmp_path / "chapter1.ipynb").write_text(json.dumps(chapter_nb)) + nbformat.write( + nbformat.v4.new_notebook( + cells=[ + nbformat.v4.new_code_cell('FreeResponseQuestion("source-question-frq")') + ] + ), + tmp_path / "section.ipynb", + ) + return tmp_path + + +def read(root, path): + return json.loads((root / path).read_text()) + + +def test_preview_then_write_is_repeatable_and_preserves_other_content(checkout): + originals = {path.name: path.read_bytes() for path in checkout.iterdir()} + changes = prepare_copy(checkout, "course.json", "question-copy.json") + assert set(changes) == {"chapter1.ipynb", "section.ipynb", "course.json"} + assert originals == {path.name: path.read_bytes() for path in checkout.iterdir()} + prepare_copy(checkout, "course.json", "question-copy.json", write=True) + notebook = read(checkout, "chapter1.ipynb") + assert ( + notebook["cells"][0]["source"] + == "Keep source-question-mcq in this explanation." + ) + assert notebook["cells"][1]["source"] == [ + "MultipleChoiceQuestion( 'copied-question-mcq' )\n" + ] + assert notebook["cells"][1]["outputs"] == [] + assert ( + notebook["cells"][3]["source"] == 'MultipleChoiceQuestion("unrelated-question")' + ) + chapter = read(checkout, "course.json")["content"][0] + assert chapter["chapterQuestions"] == { + "MCQ": [ + "copied-question-mcq:0", + "copied-question-mcq:1", + "unrelated-question:0", + ], + "FRQ": [], + "CQ": ["existing-coding-question:0"], + } + assert chapter["sections"][0]["sectionQuestions"] == { + "MCQ": [], + "FRQ": ["copied-question-frq:0"], + } + assert prepare_copy(checkout, "course.json", "question-copy.json", write=True) == [] + + +def test_action_validation_and_create_update_payloads_keep_the_mapping(checkout): + before = Course(**read(checkout, "course.json")).to_payload() + prepare_copy(checkout, "course.json", "question-copy.json", write=True) + validate_course_json("course.json") + NotebookVerifier().run() + validated = read(checkout, Config.COURSE_DATA_FILE_NAME) + assert validated["content"] != before["content"] + creator = CourseCreator( + "fixture-key", + "fixture-token", + "https://github.com/qBraid/demo", + "a" * 40, + force_duplicate_questions=False, + ) + updater = CourseUpdater( + "fixture-key", + "destination-course", + "fixture-token", + "https://github.com/qBraid/demo", + "b" * 40, + force_duplicate_questions=False, + ) + create_payload = creator.get_common_payload() + update_payload = updater.get_common_payload() + assert create_payload["data"] == update_payload["data"] == validated + assert create_payload["forceDuplicateQuestions"] is False + assert ( + update_payload["data"]["content"][0]["chapterQuestions"]["MCQ"][0] + == "copied-question-mcq:0" + ) + assert update_payload["data"]["content"][0]["sections"][0]["sectionQuestions"][ + "FRQ" + ] == ["copied-question-frq:0"] + # A deployment retry uses the same mapping and manifest, without recopying. + assert prepare_copy(checkout, "course.json", "question-copy.json", write=True) == [] + validate_course_json("course.json") + assert updater.get_common_payload()["data"] == validated + + +def test_existing_courses_without_reference_fields_keep_their_payload(checkout): + course = read(checkout, "course.json") + course["content"][0].pop("chapterQuestions") + payload = Course(**course).to_payload() + assert "chapterQuestions" not in payload["content"][0] + assert "sectionQuestions" not in payload["content"][0]["sections"][0] + assert payload["durationWeeks"] is None + + +@pytest.mark.parametrize( + "mapping", + [ + {"questions": []}, + {"questions": [MAPPING["questions"][0]] * 2}, + {"questions": [{**MAPPING["questions"][0], "customId": "source-question-mcq"}]}, + {"questions": [{**MAPPING["questions"][0], "type": "cq"}]}, + {"questions": [{**MAPPING["questions"][0], "customId": 'bad"reference'}]}, + { + "questions": [ + {**MAPPING["questions"][0], "sourceCustomId": "absent-question"} + ] + }, + {"questions": [{**MAPPING["questions"][0], "type": "frq"}]}, + ], +) +def test_invalid_or_unused_mappings_fail_before_writes(checkout, mapping): + (checkout / "question-copy.json").write_text(json.dumps(mapping)) + originals = {path.name: path.read_bytes() for path in checkout.iterdir()} + with pytest.raises(ValueError): + prepare_copy(checkout, "course.json", "question-copy.json", write=True) + assert originals == {path.name: path.read_bytes() for path in checkout.iterdir()} + + +def test_rest_envelope_is_accepted(checkout): + (checkout / "question-copy.json").write_text( + json.dumps({"status": "success", "data": MAPPING}) + ) + assert prepare_copy(checkout, "course.json", "question-copy.json", write=True) + + +def test_missing_file_fails_before_any_notebook_is_written(checkout): + (checkout / "section.ipynb").unlink() + before = (checkout / "chapter1.ipynb").read_bytes() + with pytest.raises(FileNotFoundError): + prepare_copy(checkout, "course.json", "question-copy.json", write=True) + assert (checkout / "chapter1.ipynb").read_bytes() == before + + +def test_symlink_cannot_rewrite_a_source_repository(checkout, tmp_path_factory): + outside = tmp_path_factory.mktemp("source") / "original.ipynb" + outside.write_bytes((checkout / "chapter1.ipynb").read_bytes()) + (checkout / "chapter1.ipynb").unlink() + (checkout / "chapter1.ipynb").symlink_to(outside) + original = outside.read_bytes() + with pytest.raises(ValueError, match="outside"): + prepare_copy(checkout, "course.json", "question-copy.json", write=True) + assert outside.read_bytes() == original + + +def test_multiple_widgets_of_same_type_in_one_cell_are_rejected(checkout): + notebook = read(checkout, "chapter1.ipynb") + notebook["cells"][2]["source"] *= 2 + (checkout / "chapter1.ipynb").write_text(json.dumps(notebook)) + with pytest.raises(ValueError, match="separate notebook cell"): + prepare_copy(checkout, "course.json", "question-copy.json", write=True) + + +def test_cli_check_fails_until_saved_mapping_is_applied(checkout): + command = [sys.executable, str(SCRIPT), "--mapping", "question-copy.json"] + preview = subprocess.run( + command, cwd=checkout, capture_output=True, text=True, check=True + ) + assert json.loads(preview.stdout)["written"] is False + pending = subprocess.run( + command + ["--check"], cwd=checkout, capture_output=True, text=True + ) + assert pending.returncode == 1 + subprocess.run(command + ["--write"], cwd=checkout, capture_output=True, check=True) + ready = subprocess.run( + command + ["--check"], cwd=checkout, capture_output=True, text=True, check=True + ) + assert json.loads(ready.stdout)["files"] == [] + + +@pytest.mark.parametrize("mapping", [[], None, 1, "questions", True]) +def test_cli_rejects_non_object_mapping_without_traceback_or_writes(checkout, mapping): + (checkout / "question-copy.json").write_text(json.dumps(mapping)) + originals = {path.name: path.read_bytes() for path in checkout.iterdir()} + result = subprocess.run( + [sys.executable, str(SCRIPT), "--mapping", "question-copy.json", "--write"], + cwd=checkout, + capture_output=True, + text=True, + ) + assert result.returncode == 2 + assert result.stderr.startswith("Cannot prepare question copy:") + assert "Traceback" not in result.stderr + assert originals == {path.name: path.read_bytes() for path in checkout.iterdir()}