diff --git a/config.yml b/config.yml index 5a5c5477..3f1d2ea3 100644 --- a/config.yml +++ b/config.yml @@ -8,9 +8,35 @@ default: packages_pyodide: - mat3ra-notebooks-utils notebooks: + # Materials Designer REPL: `made` minus ipywidgets/plotly/nbformat, which it never renders (~7s). + # Not named `made-repl` — names are regexes matched inside the request, so that would also match + # `made` and merge both lists. Profiles can only add packages, never subtract. + - name: repl + packages_pyodide: + - lzma + - sqlite3 + - ssl + - annotated_types>=0.6.0 + - networkx==3.2.1 + - monty==2023.11.3 + - scipy==1.11.2 + - tabulate==0.9.0 + - sympy==1.12 + - uncertainties==3.1.6 + - ase==3.25.0 + - emfs:/drive/packages/pymatgen-2024.4.13-py3-none-any.whl + - emfs:/drive/packages/spglib-2.0.2-py3-none-any.whl + - emfs:/drive/packages/ruamel.yaml-0.17.32-py3-none-any.whl + - emfs:/drive/packages/pydantic_core-2.18.2-py3-none-any.whl + - emfs:/drive/packages/pydantic-2.7.1-py3-none-any.whl + - pymatgen-analysis-defects<=2024.4.23 + - mat3ra-periodic-table + - mat3ra-made - name: made packages_pyodide: - lzma + - sqlite3 + - ssl - annotated_types>=0.6.0 - networkx==3.2.1 - monty==2023.11.3 diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/io.py b/src/py/mat3ra/notebooks_utils/core/entity/material/io.py index 5c60dd5f..cd18f9ed 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/material/io.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/io.py @@ -7,7 +7,7 @@ from mat3ra.made.tools.build_components import MaterialWithBuildMetadata from mat3ra.utils.array import convert_to_array_if_not -from ....io import get_data, set_data +from ....io import get_data, send_data, set_data from ....primitive.enums import SeverityLevelEnum from ....primitive.logger import log from ....settings import UPLOADS_FOLDER @@ -64,6 +64,40 @@ def set_materials(materials: List[Any], folder_path: str = UPLOADS_FOLDER): ) +def sync_materials(globals_dict: dict, sync_scope: str = "python-repl") -> None: + """Send the complete set of public Material bindings owned by a REPL sync scope. + + Lists, tuples, and dictionary values are inspected one level deep. The host-provided input + bindings are deliberately excluded so merely running a cell does not echo all inputs back. + """ + reserved_names = {"materials_in", "material"} + entities = [] + + for name, value in globals_dict.items(): + if name.startswith("_") or name in reserved_names: + continue + + if isinstance(value, Material): + materials = [value] + elif isinstance(value, (list, tuple)): + materials = [item for item in value if isinstance(item, Material)] + elif isinstance(value, dict): + materials = [item for item in value.values() if isinstance(item, Material)] + else: + continue + + for material in materials: + entities.append( + { + "type": "material", + "name": name, + "config": json.loads(material.to_json()), + } + ) + + send_data({"syncScope": sync_scope, "entities": entities}) + + def load_materials_from_folder(folder_path: Optional[str] = None, verbose: bool = True) -> List[Any]: """ Load materials from the specified folder or from the UPLOADS_FOLDER by default. diff --git a/src/py/mat3ra/notebooks_utils/io.py b/src/py/mat3ra/notebooks_utils/io.py index 1e7edd47..6f8794ea 100644 --- a/src/py/mat3ra/notebooks_utils/io.py +++ b/src/py/mat3ra/notebooks_utils/io.py @@ -4,7 +4,7 @@ from .ipython.io import download_content_to_file from .primitive.enums import EnvironmentsEnum from .primitive.environment import ENVIRONMENT -from .pyodide.io import get_data_pyodide, read_from_url_pyodide, set_data_pyodide +from .pyodide.io import get_data_pyodide, read_from_url_pyodide, send_data_pyodide, set_data_pyodide from .settings import UPLOADS_FOLDER @@ -37,6 +37,12 @@ def set_data(key: str, value: Any, folder_path: str = UPLOADS_FOLDER): set_data_python(key, value, folder_path=folder_path) +def send_data(payload: Dict[str, Any]): + """Send a complete bridge payload. This operation is meaningful only in Pyodide.""" + if ENVIRONMENT == EnvironmentsEnum.PYODIDE: + send_data_pyodide(payload) + + async def read_from_url(url: str, as_bytes: bool = False) -> Union[str, bytes]: """ Read content from a URL, routing to the pyodide or Python implementation. @@ -53,4 +59,4 @@ async def read_from_url(url: str, as_bytes: bool = False) -> Union[str, bytes]: return read_from_url_python(url, as_bytes) -__all__ = ["download_content_to_file", "get_data", "read_from_url", "set_data"] +__all__ = ["download_content_to_file", "get_data", "read_from_url", "send_data", "set_data"] diff --git a/src/py/mat3ra/notebooks_utils/ipython/io.py b/src/py/mat3ra/notebooks_utils/ipython/io.py index 900fe765..0bb3dc4c 100644 --- a/src/py/mat3ra/notebooks_utils/ipython/io.py +++ b/src/py/mat3ra/notebooks_utils/ipython/io.py @@ -1,6 +1,10 @@ import json -from IPython.display import Javascript, display +try: + from IPython.display import Javascript, display +except ImportError: + Javascript = None + display = None def download_content_to_file(content: dict, filename: str): @@ -11,6 +15,9 @@ def download_content_to_file(content: dict, filename: str): content (dict): The content to download. filename (str): The name of the file to download. """ + if Javascript is None or display is None: + raise RuntimeError("IPython is required to download content from a notebook") + if isinstance(content, dict): content_str = json.dumps(content, indent=4) else: diff --git a/src/py/mat3ra/notebooks_utils/preamble/__init__.py b/src/py/mat3ra/notebooks_utils/preamble/__init__.py new file mode 100644 index 00000000..83c5b39e --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/preamble/__init__.py @@ -0,0 +1 @@ +"""Ready-to-execute namespace preambles for interactive Python environments.""" diff --git a/src/py/mat3ra/notebooks_utils/preamble/material.py b/src/py/mat3ra/notebooks_utils/preamble/material.py new file mode 100644 index 00000000..935927fe --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/preamble/material.py @@ -0,0 +1,20 @@ +"""Imports exposed by the Materials Designer Python REPL.""" + +from mat3ra.made.material import Material +from mat3ra.made.tools.build.defective_structures.zero_dimensional.point_defect.point_defect_type_enum import ( + PointDefectTypeEnum, +) +from mat3ra.made.tools.build.pristine_structures.zero_dimensional.nanoparticle.enums import NanoparticleShapesEnum +from mat3ra.made.tools.build_components.entities.reusable.zero_dimensional.coordinates_shape_enum import ( + CoordinatesShapeEnum, +) +from mat3ra.made.tools.helpers import * # noqa: F403 +from mat3ra.made.tools.helpers import __all__ as _HELPER_NAMES + +__all__ = [ + "CoordinatesShapeEnum", + "Material", + "NanoparticleShapesEnum", + "PointDefectTypeEnum", + *_HELPER_NAMES, +] diff --git a/src/py/mat3ra/notebooks_utils/pyodide/io.py b/src/py/mat3ra/notebooks_utils/pyodide/io.py index 9dbabfeb..ead45d94 100644 --- a/src/py/mat3ra/notebooks_utils/pyodide/io.py +++ b/src/py/mat3ra/notebooks_utils/pyodide/io.py @@ -3,11 +3,26 @@ import os from typing import Any, Dict, Optional, Union -from IPython.display import Javascript, display - from ..core.io import set_data_python from ..primitive.logger import log +try: + from IPython.display import Javascript, display +except ImportError: + Javascript = None + display = None + +try: + from js import JSON, sendDataToHost # type: ignore +except ImportError: + JSON = None + sendDataToHost = None + +try: + from pyodide.http import pyfetch # type: ignore +except ImportError: + pyfetch = None + async def read_from_url_pyodide(url: str, as_bytes: bool = False) -> Union[str, bytes]: """ @@ -20,16 +35,29 @@ async def read_from_url_pyodide(url: str, as_bytes: bool = False) -> Union[str, Returns: str or bytes: The content. """ - # `http` is a Pyodide module that will be installed in the Pyodide environment by default. - from pyodide.http import pyfetch # type: ignore + if pyfetch is None: + raise RuntimeError("pyfetch is available only in Pyodide") - # Per https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch response = await pyfetch(url) if as_bytes: return await response.bytes() return await response.string() +def send_data_pyodide(payload: Dict[str, Any]): + """Send a bridge payload to the host application.""" + serialized_data = json.dumps(payload) + + if JSON is not None and sendDataToHost is not None: + sendDataToHost(JSON.parse(serialized_data)) + return + + if Javascript is None or display is None: + raise RuntimeError("IPython is required to send data from JupyterLite") + + display(Javascript(f"window.sendDataToHost({serialized_data});")) + + def set_data_pyodide(key: str, value: Any): """ Take a Python object, serialize it to JSON, and send it to the host environment @@ -39,18 +67,7 @@ def set_data_pyodide(key: str, value: Any): key (str): The name under which data will be sent. value (Any): The value to send to the host environment. """ - serialized_data = json.dumps({key: value}) - js_code = f""" - (function() {{ - if (window.sendDataToHost) {{ - window.sendDataToHost({serialized_data}); - console.log('Data sent to host:', {serialized_data}); - }} else {{ - console.error('sendDataToHost function is not defined on the window object.'); - }} - }})(); - """ - display(Javascript(js_code)) + send_data_pyodide({key: value}) log(f"Data for {key} sent to host.") set_data_python(key, value) diff --git a/src/py/mat3ra/notebooks_utils/pyodide/packages/install.py b/src/py/mat3ra/notebooks_utils/pyodide/packages/install.py index 5a8e10ca..31f00739 100644 --- a/src/py/mat3ra/notebooks_utils/pyodide/packages/install.py +++ b/src/py/mat3ra/notebooks_utils/pyodide/packages/install.py @@ -114,8 +114,12 @@ def package_has_version_specifier(pkg: str) -> bool: return any(op in spec for op in VERSION_SPECIFIERS) -def should_reinstall_package(pkg: str, profile_changed: bool) -> bool: - return profile_changed and package_has_version_specifier(pkg) and not is_url_package(remove_nodeps_prefix(pkg)) +def should_reinstall_package(pkg: str, previous_packages: List[str]) -> bool: + package_name = get_package_name(pkg) + if not package_name or not package_has_version_specifier(pkg) or is_url_package(remove_nodeps_prefix(pkg)): + return False + previous_spec = next((item for item in previous_packages if get_package_name(item) == package_name), None) + return previous_spec is not None and previous_spec != pkg def get_package_name(pkg: str) -> Union[str, None]: @@ -187,17 +191,18 @@ async def install_packages_pyodide(notebook_name_pattern: str, verbose: bool = T packages = await get_package_list_from_config(get_config_yml_file_path(""), notebook_name_pattern) requirements_hash = str(hash(json.dumps(packages))) previous_hash = os.environ.get("requirements_hash") - profile_changed = previous_hash is not None and previous_hash != requirements_hash + previous_packages = json.loads(os.environ.get("requirements_packages", "[]")) if should_install_packages(previous_hash, requirements_hash): for pkg in packages: await install_package_pyodide( pkg, verbose, - reinstall=should_reinstall_package(pkg, profile_changed), + reinstall=should_reinstall_package(pkg, previous_packages), ) if verbose: log("Packages installed successfully.", force_verbose=verbose) os.environ["requirements_hash"] = requirements_hash + os.environ["requirements_packages"] = json.dumps(packages) else: if verbose: log("Packages are already installed.", force_verbose=verbose) diff --git a/tests/py/unit/core/entity/test_material_io.py b/tests/py/unit/core/entity/test_material_io.py new file mode 100644 index 00000000..f824ad81 --- /dev/null +++ b/tests/py/unit/core/entity/test_material_io.py @@ -0,0 +1,48 @@ +from unittest.mock import patch + +from mat3ra.made.material import Material +from mat3ra.notebooks_utils.core.entity.material.io import sync_materials +from mat3ra.standata.materials import Materials + + +def material(name: str) -> Material: + return Material.create({**Materials.get_by_name_first_match("Silicon"), "name": name}) + + +def test_sync_materials_collects_public_bindings_and_one_container_level(): + direct = material("direct") + listed = material("listed") + mapped = material("mapped") + + namespace = { + "direct": direct, + "group": [listed, 3, [material("too-deep")]], + "mapping": {"value": mapped}, + "materials_in": [material("input")], + "material": material("selected"), + "_private": material("private"), + "number": 4, + } + + with patch("mat3ra.notebooks_utils.core.entity.material.io.send_data") as send: + sync_materials(namespace) + + payload = send.call_args.args[0] + assert payload["syncScope"] == "python-repl" + assert [(entity["type"], entity["name"]) for entity in payload["entities"]] == [ + ("material", "direct"), + ("material", "group"), + ("material", "mapping"), + ] + assert [entity["config"]["name"] for entity in payload["entities"]] == [ + "direct", + "listed", + "mapped", + ] + + +def test_sync_materials_sends_an_empty_batch_to_clear_the_scope(): + with patch("mat3ra.notebooks_utils.core.entity.material.io.send_data") as send: + sync_materials({"x": 1}, sync_scope="test-scope") + + send.assert_called_once_with({"syncScope": "test-scope", "entities": []}) diff --git a/tests/py/unit/test_pyodide_io.py b/tests/py/unit/test_pyodide_io.py new file mode 100644 index 00000000..1b9626c8 --- /dev/null +++ b/tests/py/unit/test_pyodide_io.py @@ -0,0 +1,32 @@ +import json +from types import SimpleNamespace +from unittest.mock import patch + +from mat3ra.notebooks_utils.pyodide.io import send_data_pyodide, set_data_pyodide + + +def test_send_data_pyodide_calls_same_page_bridge_directly(): + received = [] + with patch("mat3ra.notebooks_utils.pyodide.io.JSON", SimpleNamespace(parse=json.loads)): + with patch("mat3ra.notebooks_utils.pyodide.io.sendDataToHost", received.append): + send_data_pyodide({"syncScope": "python-repl", "entities": []}) + + assert received == [{"syncScope": "python-repl", "entities": []}] + + +def test_send_data_pyodide_uses_display_for_jupyterlite(): + with patch("mat3ra.notebooks_utils.pyodide.io.JSON", None): + with patch("mat3ra.notebooks_utils.pyodide.io.Javascript", side_effect=lambda source: source): + with patch("mat3ra.notebooks_utils.pyodide.io.display") as display: + send_data_pyodide({"syncScope": "python-repl", "entities": []}) + + assert display.call_args.args[0] == 'window.sendDataToHost({"syncScope": "python-repl", "entities": []});' + + +def test_set_data_pyodide_updates_python_data(): + with patch("mat3ra.notebooks_utils.pyodide.io.JSON", SimpleNamespace(parse=json.loads)): + with patch("mat3ra.notebooks_utils.pyodide.io.sendDataToHost"): + with patch("mat3ra.notebooks_utils.pyodide.io.set_data_python") as set_data_python: + set_data_pyodide("materials", [{"name": "Si"}]) + + set_data_python.assert_called_once_with("materials", [{"name": "Si"}]) diff --git a/tests/py/unit/test_pyodide_packages_install.py b/tests/py/unit/test_pyodide_packages_install.py new file mode 100644 index 00000000..dd0b2f44 --- /dev/null +++ b/tests/py/unit/test_pyodide_packages_install.py @@ -0,0 +1,16 @@ +from mat3ra.notebooks_utils.pyodide.packages.install import should_reinstall_package + + +def test_reinstalls_only_when_the_same_package_version_changes(): + previous = ["networkx==3.2.1", "scipy==1.11.2"] + + assert should_reinstall_package("networkx==3.2.2", previous) + assert not should_reinstall_package("networkx==3.2.1", previous) + assert not should_reinstall_package("tabulate==0.9.0", previous) + + +def test_does_not_reinstall_url_or_emfs_requirements(): + assert not should_reinstall_package( + "emfs:/drive/packages/example.whl", + ["emfs:/drive/packages/old.whl"], + )