feature/SOF-7961: add scoped material bridge helpers

config.yml

@@ -8,9 +8,35 @@ default:
     packages_pyodide:
       - mat3ra-notebooks-utils
 notebooks:
+  # Materials Designer REPL: `made` minus ipywidgets/plotly/nbformat, which it never renders (~7s).
+  # Not named `made-repl` — names are regexes matched inside the request, so that would also match
+  # `made` and merge both lists. Profiles can only add packages, never subtract.
+  - name: repl
+    packages_pyodide:
+      - lzma
+      - sqlite3
+      - ssl
+      - annotated_types>=0.6.0
+      - networkx==3.2.1
+      - monty==2023.11.3
+      - scipy==1.11.2
+      - tabulate==0.9.0
+      - sympy==1.12
+      - uncertainties==3.1.6
+      - ase==3.25.0
+      - emfs:/drive/packages/pymatgen-2024.4.13-py3-none-any.whl
+      - emfs:/drive/packages/spglib-2.0.2-py3-none-any.whl
+      - emfs:/drive/packages/ruamel.yaml-0.17.32-py3-none-any.whl
+      - emfs:/drive/packages/pydantic_core-2.18.2-py3-none-any.whl
+      - emfs:/drive/packages/pydantic-2.7.1-py3-none-any.whl
+      - pymatgen-analysis-defects<=2024.4.23
+      - mat3ra-periodic-table
+      - mat3ra-made
   - name: made
     packages_pyodide:
       - lzma
+      - sqlite3
+      - ssl
       - annotated_types>=0.6.0
       - networkx==3.2.1
       - monty==2023.11.3

src/py/mat3ra/notebooks_utils/core/entity/material/io.py

@@ -7,7 +7,7 @@
 from mat3ra.made.tools.build_components import MaterialWithBuildMetadata
 from mat3ra.utils.array import convert_to_array_if_not
 
-from ....io import get_data, set_data
+from ....io import get_data, send_data, set_data
 from ....primitive.enums import SeverityLevelEnum
 from ....primitive.logger import log
 from ....settings import UPLOADS_FOLDER
@@ -64,6 +64,40 @@ def set_materials(materials: List[Any], folder_path: str = UPLOADS_FOLDER):
     )
 
 
+def sync_materials(globals_dict: dict, sync_scope: str = "python-repl") -> None:
+    """Send the complete set of public Material bindings owned by a REPL sync scope.
+
+    Lists, tuples, and dictionary values are inspected one level deep. The host-provided input
+    bindings are deliberately excluded so merely running a cell does not echo all inputs back.
+    """
+    reserved_names = {"materials_in", "material"}
+    entities = []
+
+    for name, value in globals_dict.items():
+        if name.startswith("_") or name in reserved_names:
+            continue
+
+        if isinstance(value, Material):
+            materials = [value]
+        elif isinstance(value, (list, tuple)):
+            materials = [item for item in value if isinstance(item, Material)]
+        elif isinstance(value, dict):
+            materials = [item for item in value.values() if isinstance(item, Material)]
+        else:
+            continue
+
+        for material in materials:
+            entities.append(
+                {
+                    "type": "material",
+                    "name": name,
+                    "config": json.loads(material.to_json()),
+                }
+            )
+
+    send_data({"syncScope": sync_scope, "entities": entities})
+
+
 def load_materials_from_folder(folder_path: Optional[str] = None, verbose: bool = True) -> List[Any]:
     """
     Load materials from the specified folder or from the UPLOADS_FOLDER by default.

src/py/mat3ra/notebooks_utils/io.py

@@ -4,7 +4,7 @@
 from .ipython.io import download_content_to_file
 from .primitive.enums import EnvironmentsEnum
 from .primitive.environment import ENVIRONMENT
-from .pyodide.io import get_data_pyodide, read_from_url_pyodide, set_data_pyodide
+from .pyodide.io import get_data_pyodide, read_from_url_pyodide, send_data_pyodide, set_data_pyodide
 from .settings import UPLOADS_FOLDER
 
 
@@ -37,6 +37,12 @@ def set_data(key: str, value: Any, folder_path: str = UPLOADS_FOLDER):
         set_data_python(key, value, folder_path=folder_path)
 
 
+def send_data(payload: Dict[str, Any]):
+    """Send a complete bridge payload. This operation is meaningful only in Pyodide."""
+    if ENVIRONMENT == EnvironmentsEnum.PYODIDE:
+        send_data_pyodide(payload)
+
+
 async def read_from_url(url: str, as_bytes: bool = False) -> Union[str, bytes]:
     """
     Read content from a URL, routing to the pyodide or Python implementation.
@@ -53,4 +59,4 @@ async def read_from_url(url: str, as_bytes: bool = False) -> Union[str, bytes]:
     return read_from_url_python(url, as_bytes)
 
 
-__all__ = ["download_content_to_file", "get_data", "read_from_url", "set_data"]
+__all__ = ["download_content_to_file", "get_data", "read_from_url", "send_data", "set_data"]

src/py/mat3ra/notebooks_utils/ipython/io.py

@@ -1,6 +1,10 @@
 import json
 
-from IPython.display import Javascript, display
+try:
+    from IPython.display import Javascript, display
+except ImportError:
+    Javascript = None
+    display = None
 
 
 def download_content_to_file(content: dict, filename: str):
@@ -11,6 +15,9 @@ def download_content_to_file(content: dict, filename: str):
         content (dict): The content to download.
         filename (str): The name of the file to download.
     """
+    if Javascript is None or display is None:
+        raise RuntimeError("IPython is required to download content from a notebook")
+
     if isinstance(content, dict):
         content_str = json.dumps(content, indent=4)
     else:

src/py/mat3ra/notebooks_utils/preamble/__init__.py

@@ -0,0 +1 @@
+"""Ready-to-execute namespace preambles for interactive Python environments."""

src/py/mat3ra/notebooks_utils/preamble/material.py

@@ -0,0 +1,20 @@
+"""Imports exposed by the Materials Designer Python REPL."""
+
+from mat3ra.made.material import Material
+from mat3ra.made.tools.build.defective_structures.zero_dimensional.point_defect.point_defect_type_enum import (
+    PointDefectTypeEnum,
+)
+from mat3ra.made.tools.build.pristine_structures.zero_dimensional.nanoparticle.enums import NanoparticleShapesEnum
+from mat3ra.made.tools.build_components.entities.reusable.zero_dimensional.coordinates_shape_enum import (
+    CoordinatesShapeEnum,
+)
+from mat3ra.made.tools.helpers import *  # noqa: F403
+from mat3ra.made.tools.helpers import __all__ as _HELPER_NAMES
+
+__all__ = [
+    "CoordinatesShapeEnum",
+    "Material",
+    "NanoparticleShapesEnum",
+    "PointDefectTypeEnum",
+    *_HELPER_NAMES,
+]

src/py/mat3ra/notebooks_utils/pyodide/io.py

@@ -3,11 +3,26 @@
 import os
 from typing import Any, Dict, Optional, Union
 
-from IPython.display import Javascript, display
-
 from ..core.io import set_data_python
 from ..primitive.logger import log
 
+try:
+    from IPython.display import Javascript, display
+except ImportError:
+    Javascript = None
+    display = None
+
+try:
+    from js import JSON, sendDataToHost  # type: ignore
+except ImportError:
+    JSON = None
+    sendDataToHost = None
+
+try:
+    from pyodide.http import pyfetch  # type: ignore
+except ImportError:
+    pyfetch = None
+
 
 async def read_from_url_pyodide(url: str, as_bytes: bool = False) -> Union[str, bytes]:
     """
@@ -20,16 +35,29 @@ async def read_from_url_pyodide(url: str, as_bytes: bool = False) -> Union[str,
     Returns:
         str or bytes: The content.
     """
-    # `http` is a Pyodide module that will be installed in the Pyodide environment by default.
-    from pyodide.http import pyfetch  # type: ignore
+    if pyfetch is None:
+        raise RuntimeError("pyfetch is available only in Pyodide")
 
-    # Per https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch
     response = await pyfetch(url)
     if as_bytes:
         return await response.bytes()
     return await response.string()
 
 
+def send_data_pyodide(payload: Dict[str, Any]):
+    """Send a bridge payload to the host application."""
+    serialized_data = json.dumps(payload)
+
+    if JSON is not None and sendDataToHost is not None:
+        sendDataToHost(JSON.parse(serialized_data))
+        return
+
+    if Javascript is None or display is None:
+        raise RuntimeError("IPython is required to send data from JupyterLite")
+
+    display(Javascript(f"window.sendDataToHost({serialized_data});"))
+
+
 def set_data_pyodide(key: str, value: Any):
     """
     Take a Python object, serialize it to JSON, and send it to the host environment
@@ -39,18 +67,7 @@ def set_data_pyodide(key: str, value: Any):
         key (str): The name under which data will be sent.
         value (Any): The value to send to the host environment.
     """
-    serialized_data = json.dumps({key: value})
-    js_code = f"""
-      (function() {{
-          if (window.sendDataToHost) {{
-              window.sendDataToHost({serialized_data});
-              console.log('Data sent to host:', {serialized_data});
-          }} else {{
-              console.error('sendDataToHost function is not defined on the window object.');
-          }}
-      }})();
-      """
-    display(Javascript(js_code))
+    send_data_pyodide({key: value})
     log(f"Data for {key} sent to host.")
     set_data_python(key, value)
 

src/py/mat3ra/notebooks_utils/pyodide/packages/install.py

@@ -114,8 +114,12 @@ def package_has_version_specifier(pkg: str) -> bool:
     return any(op in spec for op in VERSION_SPECIFIERS)
 
 
-def should_reinstall_package(pkg: str, profile_changed: bool) -> bool:
-    return profile_changed and package_has_version_specifier(pkg) and not is_url_package(remove_nodeps_prefix(pkg))
+def should_reinstall_package(pkg: str, previous_packages: List[str]) -> bool:
+    package_name = get_package_name(pkg)
+    if not package_name or not package_has_version_specifier(pkg) or is_url_package(remove_nodeps_prefix(pkg)):
+        return False
+    previous_spec = next((item for item in previous_packages if get_package_name(item) == package_name), None)
+    return previous_spec is not None and previous_spec != pkg
 
 
 def get_package_name(pkg: str) -> Union[str, None]:
@@ -187,17 +191,18 @@ async def install_packages_pyodide(notebook_name_pattern: str, verbose: bool = T
     packages = await get_package_list_from_config(get_config_yml_file_path(""), notebook_name_pattern)
     requirements_hash = str(hash(json.dumps(packages)))
     previous_hash = os.environ.get("requirements_hash")
-    profile_changed = previous_hash is not None and previous_hash != requirements_hash
+    previous_packages = json.loads(os.environ.get("requirements_packages", "[]"))
     if should_install_packages(previous_hash, requirements_hash):
         for pkg in packages:
             await install_package_pyodide(
                 pkg,
                 verbose,
-                reinstall=should_reinstall_package(pkg, profile_changed),
+                reinstall=should_reinstall_package(pkg, previous_packages),
             )
         if verbose:
             log("Packages installed successfully.", force_verbose=verbose)
         os.environ["requirements_hash"] = requirements_hash
+        os.environ["requirements_packages"] = json.dumps(packages)
     else:
         if verbose:
             log("Packages are already installed.", force_verbose=verbose)

tests/py/unit/core/entity/test_material_io.py

@@ -0,0 +1,48 @@
+from unittest.mock import patch
+
+from mat3ra.made.material import Material
+from mat3ra.notebooks_utils.core.entity.material.io import sync_materials
+from mat3ra.standata.materials import Materials
+
+
+def material(name: str) -> Material:
+    return Material.create({**Materials.get_by_name_first_match("Silicon"), "name": name})
+
+
+def test_sync_materials_collects_public_bindings_and_one_container_level():
+    direct = material("direct")
+    listed = material("listed")
+    mapped = material("mapped")
+
+    namespace = {
+        "direct": direct,
+        "group": [listed, 3, [material("too-deep")]],
+        "mapping": {"value": mapped},
+        "materials_in": [material("input")],
+        "material": material("selected"),
+        "_private": material("private"),
+        "number": 4,
+    }
+
+    with patch("mat3ra.notebooks_utils.core.entity.material.io.send_data") as send:
+        sync_materials(namespace)
+
+    payload = send.call_args.args[0]
+    assert payload["syncScope"] == "python-repl"
+    assert [(entity["type"], entity["name"]) for entity in payload["entities"]] == [
+        ("material", "direct"),
+        ("material", "group"),
+        ("material", "mapping"),
+    ]
+    assert [entity["config"]["name"] for entity in payload["entities"]] == [
+        "direct",
+        "listed",
+        "mapped",
+    ]
+
+
+def test_sync_materials_sends_an_empty_batch_to_clear_the_scope():
+    with patch("mat3ra.notebooks_utils.core.entity.material.io.send_data") as send:
+        sync_materials({"x": 1}, sync_scope="test-scope")
+
+    send.assert_called_once_with({"syncScope": "test-scope", "entities": []})

tests/py/unit/test_pyodide_io.py

@@ -0,0 +1,32 @@
+import json
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from mat3ra.notebooks_utils.pyodide.io import send_data_pyodide, set_data_pyodide
+
+
+def test_send_data_pyodide_calls_same_page_bridge_directly():
+    received = []
+    with patch("mat3ra.notebooks_utils.pyodide.io.JSON", SimpleNamespace(parse=json.loads)):
+        with patch("mat3ra.notebooks_utils.pyodide.io.sendDataToHost", received.append):
+            send_data_pyodide({"syncScope": "python-repl", "entities": []})
+
+    assert received == [{"syncScope": "python-repl", "entities": []}]
+
+
+def test_send_data_pyodide_uses_display_for_jupyterlite():
+    with patch("mat3ra.notebooks_utils.pyodide.io.JSON", None):
+        with patch("mat3ra.notebooks_utils.pyodide.io.Javascript", side_effect=lambda source: source):
+            with patch("mat3ra.notebooks_utils.pyodide.io.display") as display:
+                send_data_pyodide({"syncScope": "python-repl", "entities": []})
+
+    assert display.call_args.args[0] == 'window.sendDataToHost({"syncScope": "python-repl", "entities": []});'
+
+
+def test_set_data_pyodide_updates_python_data():
+    with patch("mat3ra.notebooks_utils.pyodide.io.JSON", SimpleNamespace(parse=json.loads)):
+        with patch("mat3ra.notebooks_utils.pyodide.io.sendDataToHost"):
+            with patch("mat3ra.notebooks_utils.pyodide.io.set_data_python") as set_data_python:
+                set_data_pyodide("materials", [{"name": "Si"}])
+
+    set_data_python.assert_called_once_with("materials", [{"name": "Si"}])

tests/py/unit/test_pyodide_packages_install.py

@@ -0,0 +1,16 @@
+from mat3ra.notebooks_utils.pyodide.packages.install import should_reinstall_package
+
+
+def test_reinstalls_only_when_the_same_package_version_changes():
+    previous = ["networkx==3.2.1", "scipy==1.11.2"]
+
+    assert should_reinstall_package("networkx==3.2.2", previous)
+    assert not should_reinstall_package("networkx==3.2.1", previous)
+    assert not should_reinstall_package("tabulate==0.9.0", previous)
+
+
+def test_does_not_reinstall_url_or_emfs_requirements():
+    assert not should_reinstall_package(
+        "emfs:/drive/packages/example.whl",
+        ["emfs:/drive/packages/old.whl"],
+    )