diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 3cad54bfb..57ca0dc87 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -28,8 +28,38 @@ jobs: python -m pip install pre-commit pre-commit run --all-files --show-diff-on-failure + run-py-tests: + needs: run-linter + runs-on: ubuntu-24.04 + strategy: + matrix: + python-version: + - 3.10.x + - 3.11.x + - 3.12.x + + steps: + - name: Checkout this repository + uses: actions/checkout@v4 + with: + lfs: true + + - name: Checkout actions repository + uses: actions/checkout@v4 + with: + repository: Exabyte-io/actions + token: ${{ secrets.BOT_GITHUB_TOKEN }} + path: actions + + - name: Run python unit tests + uses: ./actions/py/pytest + with: + python-version: ${{ matrix.python-version }} + unit-test-directory: tests/py/unit + bot-ssh-key: ${{ secrets.BOT_GITHUB_KEY }} + publish-py-package: - needs: [run-linter] + needs: [run-linter, run-py-tests] runs-on: ubuntu-latest if: github.ref_name == 'main' diff --git a/pyproject.toml b/pyproject.toml index d888c0712..d3216a1f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,38 +1,67 @@ [project] -name = "mat3ra-api-examples" +name = "mat3ra-notebooks-utils" dynamic = ["version"] -description = "Mat3ra API Examples" +description = "Mat3ra notebooks utilities." readme = "README.md" requires-python = ">=3.10" -dependencies = [ +dependencies = [] + +[project.optional-dependencies] +auxilary = [ + "pydantic", + "matplotlib>=3.4.1", + "pandas>=1.5.3", +] +jupyterlite = [ + "pyyaml", + "matplotlib>=3.4.1", + "plotly>=5.18", + "ipython>=8.0", +] +utils_standata = [ + "mat3ra-utils", + "mat3ra-standata", +] +api = [ + "mat3ra-api-client", +] +materials = [ # ase >=3.25.0 is required for supercell to be generated, # otherwise on 3.21.1 we encountered negative number of atoms during supercell generation + "mat3ra-notebooks-utils[auxilary]", + "mat3ra-notebooks-utils[utils_standata]", "ase>=3.25.0", - "matplotlib>=3.4.1", - "pandas>=1.5.3", "pymatgen==2024.4.13", "mat3ra-made>=2026.4.2.post0", - "mat3ra-utils>=2026.3.6.post0", +] +workflows = [ + "mat3ra-notebooks-utils[materials]", "mat3ra-wode", "mat3ra-prode", "mat3ra-ide", - "mat3ra-api-client", - "mat3ra-standata" + "mar3ra-notebooks-utils[api]", ] -[project.optional-dependencies] # Install all above dependencies in colab -colab = ["mat3ra-api-examples"] +colab = ["mat3ra-notebooks-utils"] + +# ToDo: figure out necessary packages jupyterlab = [ "jupyterlab>=3.0.17", "nbconvert>=6.0.7", ] # Install colab + jupyterlab on localhost -localhost = ["mat3ra-api-examples[jupyterlab]"] +localhost = ["mat3ra-notebooks-utils[jupyterlab]"] dev = [ "pre-commit>=3.3.3", "pip-tools>=6.13.0", ] +tests = [ + "pydantic", + "pytest", + "pytest-asyncio", + "pytest-cov", +] docs = [ "mkdocs>=1.4.3", "mkdocs-material>=9.1.17", @@ -64,18 +93,18 @@ build-backend = "setuptools.build_meta" git_describe_command = "git describe --tags --long" [tool.setuptools.packages.find] -include = ["utils"] +where = ["src/py"] [tool.setuptools.package-data] -utils = [ - "settings.json", - "web/renderjson.*", +"mat3ra.notebooks_utils" = [ + "core/api/settings.json", + "ipython/web/renderjson.*", ] [tool.black] line-length = 120 -target-version = ['py38'] +target-version = ['py310'] # 'extend-exclude' excludes files or directories in addition to the defaults extend-exclude = ''' ( @@ -98,6 +127,10 @@ profile = "black" multi_line_output = 3 include_trailing_comma = true +[tool.mypy] +python_version = "3.10" +explicit_package_bases = true + [[tool.mypy.overrides]] module = "yaml" ignore_missing_imports = true diff --git a/utils/__init__.py b/src/py/mat3ra/__init__.py similarity index 100% rename from utils/__init__.py rename to src/py/mat3ra/__init__.py diff --git a/src/py/mat3ra/notebooks_utils/README.md b/src/py/mat3ra/notebooks_utils/README.md new file mode 100644 index 000000000..7673f47d3 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/README.md @@ -0,0 +1,94 @@ +# mat3ra.notebooks_utils — Developer Guide + +## Architecture + +Four progressive layers, each building on the previous: + +``` +primitive/ → core/ → ipython/ → pyodide/ +``` + +Top-level files (`auth.py`, `io.py`, `ui.py`, `plot.py`, `settings.py`, `material.py`) are thin +routing/re-export adapters that keep notebook code environment-agnostic. + +--- + +## Layers + +### `primitive/` +Only Python stdlib. No third-party packages, no domain knowledge. +Enums, environment detection, logger, CLI prompt helpers. + +### `core/` +Third-party packages allowed (`mat3ra.api_client`, `requests`, `numpy`, …). +No IPython, no ipywidgets, no browser/pyodide APIs. + +``` +core/api/ — Platform credentials and OIDC auth. +core/io.py — Plain-Python file IO and HTTP. +core/entity// — One folder per domain (material, workflow, job, compute, property). + api.py — REST API calls only. + io.py — Local filesystem reads/writes only. + analysis.py — Pure computation (numpy/pymatgen/ase). No API, no display. + job.py — Cross-entity helpers (e.g. property derived from job data). +``` + +### `ipython/` +Imports `IPython.display`, `ipywidgets`, or generates HTML/JS for notebook output. +Must work in JupyterLab, Colab, and VS Code notebooks — not browser/pyodide specific. + +``` +ipython/ui.py — Generic cell output: display_JSON, image grid, viewer HTML/JS. +ipython/io.py — Browser file-download helpers. +ipython/plot/ — Domain-agnostic plot primitives (_plotly.py, _matplotlib.py). +ipython/entity// — Domain-specific display. + visualize.py — Renders domain objects into notebook cells. + plot.py — Domain-specific charts (RDF, strain, EOS, …). +``` + +### `pyodide/` +Calls `micropip`, `pyodide.http.pyfetch`, `BroadcastChannel`, or `js` (Emscripten/WASM APIs). +JupyterLite-specific overrides of `core/` or `ipython/` capabilities. + +``` +pyodide/io.py — JS data bridge: kernel↔host shared state, file writes. +pyodide/ui.py — Async input widgets (PyodideFuture / BroadcastChannel). +pyodide/runtime.py — Interruptible loop and abort controller. +pyodide/packages/ — micropip installation and config.yml parsing. +pyodide/api/ — Host-to-kernel token injection. +``` + +--- + +## Dependency Rules + +``` +primitive/ → (nothing from this package) +core/ → primitive/ +ipython/ → primitive/, core/ +pyodide/ → primitive/, core/, ipython/ +top-level → any layer (routing only, no new logic) +``` + +Within `core/entity/`: domains must not import each other; `api.py`, `io.py`, and `analysis.py` +must stay separate (no cross-imports within the same entity folder). + +Within `ipython/entity/`: may import from `core/entity//` and `ipython/` primitives, +but not from other entity domains. + +--- + +## Where does my code go? + +| Question | Destination | +|---|---| +| Uses only stdlib? | `primitive/` | +| Calls a 3rd-party package, produces no output? | `core/` | +| Reads/writes domain objects from disk? | `core/entity//io.py` | +| Calls the REST API? | `core/entity//api.py` | +| Computes/transforms domain data (no display)? | `core/entity//analysis.py` | +| Renders into a notebook cell (IPython/HTML)? | `ipython/` | +| Domain-specific chart? | `ipython/entity//plot.py` | +| Domain-specific visualiser? | `ipython/entity//visualize.py` | +| Calls micropip / pyfetch / BroadcastChannel? | `pyodide/` | +| Branches on environment to pick implementation? | top-level routing file | diff --git a/src/py/mat3ra/notebooks_utils/__init__.py b/src/py/mat3ra/notebooks_utils/__init__.py new file mode 100644 index 000000000..7ab0324fb --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/__init__.py @@ -0,0 +1,29 @@ +from .core.api.settings import ( + ACCOUNT_ID, + AUTH_TOKEN, + ENDPOINT_ARGS, + HOST, + MATERIALS_PROJECT_API_KEY, + ORGANIZATION_ID, + PORT, + SECURE, + VERSION, + absolute_path_to_settings_json_file, + settings_json_config, +) +from .settings import UPLOADS_FOLDER + +__all__ = [ + "absolute_path_to_settings_json_file", + "settings_json_config", + "ACCOUNT_ID", + "AUTH_TOKEN", + "MATERIALS_PROJECT_API_KEY", + "ORGANIZATION_ID", + "PORT", + "SECURE", + "VERSION", + "HOST", + "ENDPOINT_ARGS", + "UPLOADS_FOLDER", +] diff --git a/src/py/mat3ra/notebooks_utils/auth.py b/src/py/mat3ra/notebooks_utils/auth.py new file mode 100644 index 000000000..b8c3cb634 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/auth.py @@ -0,0 +1,28 @@ +import inspect +import os + +from mat3ra.api_client import ACCESS_TOKEN_ENV_VAR + +from .core.api.auth import authenticate_oidc +from .io import get_data +from .ipython.ui import show_device_flow_popup +from .primitive.environment import ENVIRONMENT, EnvironmentsEnum +from .pyodide.api.auth import authenticate_jupyterlite + +REFRESH_TOKEN_ENV_VAR = "OIDC_REFRESH_TOKEN" + + +async def authenticate(force=False, globals_dict=None): + if globals_dict is None: + frame = inspect.currentframe() + try: + globals_dict = frame.f_back.f_globals # type: ignore + finally: + del frame + if ENVIRONMENT == EnvironmentsEnum.PYODIDE: + get_data("data_from_host", globals_dict) + data_from_host = globals_dict.get("data_from_host") + if data_from_host: + await authenticate_jupyterlite(data_from_host) + elif ACCESS_TOKEN_ENV_VAR not in os.environ or force: + await authenticate_oidc(show_popup=show_device_flow_popup) diff --git a/src/py/mat3ra/notebooks_utils/core/__init__.py b/src/py/mat3ra/notebooks_utils/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/core/api/__init__.py b/src/py/mat3ra/notebooks_utils/core/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/utils/auth.py b/src/py/mat3ra/notebooks_utils/core/api/auth.py similarity index 56% rename from utils/auth.py rename to src/py/mat3ra/notebooks_utils/core/api/auth.py index 4b16fd8cd..91d21f7cc 100644 --- a/utils/auth.py +++ b/src/py/mat3ra/notebooks_utils/core/api/auth.py @@ -1,16 +1,10 @@ import asyncio -import inspect -import json import os import time -from typing import Optional +from typing import Callable, Optional import requests -from IPython.display import Javascript, display from mat3ra.api_client import ACCESS_TOKEN_ENV_VAR, CLIENT_ID, SCOPE, APIEnv, build_oidc_base_url -from mat3ra.utils.jupyterlite.environment import ENVIRONMENT, EnvironmentsEnum - -from utils.jupyterlite import get_data REFRESH_TOKEN_ENV_VAR = "OIDC_REFRESH_TOKEN" @@ -24,22 +18,14 @@ def request_device_flow_state(oidc_base_url: str, client_id: str, scope: str) -> """ Request an OAuth/OIDC Device Authorization flow state. - This calls the authorization server's device endpoint and returns the values needed to: - - display a user-facing verification URL + code - - poll the token endpoint until authorization completes - Args: oidc_base_url: Base OIDC URL. client_id: OAuth client identifier for the device flow. scope: Space-separated scopes to request (e.g. "openid profile email"). Returns: - A dict with: - - device_code: Device code used to poll for token issuance. - - user_code: Short code the user enters in the verification UI. - - verification_uri_complete: URL to open for user authorization. - - polling_interval_seconds: Recommended polling interval. - - expires_in_seconds: Device code lifetime. + A dict with device_code, user_code, verification_uri_complete, + polling_interval_seconds, expires_in_seconds. """ device_response = requests.post( f"{oidc_base_url}/device/auth", @@ -58,20 +44,6 @@ def request_device_flow_state(oidc_base_url: str, client_id: str, scope: str) -> } -def show_device_flow_popup(verification_uri_complete: str, user_code: str) -> None: - from IPython.display import HTML - - display( - HTML( - f"
" - f"Authentication Required
" - f"Enter this code: {user_code}" - f"
" - ) - ) - display(Javascript(f"window.open({verification_uri_complete!r}, '_blank');")) - - def store_token_data_in_environment(token_data: dict) -> None: os.environ[ACCESS_TOKEN_ENV_VAR] = token_data["access_token"] if "refresh_token" in token_data: @@ -110,11 +82,13 @@ async def authenticate_oidc( oidc_base_url: Optional[str] = None, client_id: str = CLIENT_ID, scope: str = SCOPE, + show_popup: Optional[Callable[[str, str], None]] = None, ) -> dict: if oidc_base_url is None: oidc_base_url = get_oidc_base_url() device_flow_state = request_device_flow_state(oidc_base_url, client_id, scope) - show_device_flow_popup(device_flow_state["verification_uri_complete"], device_flow_state["user_code"]) + if show_popup is not None: + show_popup(device_flow_state["verification_uri_complete"], device_flow_state["user_code"]) token_data = await _poll_for_token_data( oidc_base_url=oidc_base_url, client_id=client_id, @@ -124,32 +98,3 @@ async def authenticate_oidc( ) store_token_data_in_environment(token_data) return token_data - - -async def authenticate_jupyterlite(data_from_host: dict) -> None: - apiConfig = data_from_host.get("apiConfig") - os.environ.update(data_from_host.get("environ", {})) - os.environ.update( - dict( - ACCOUNT_ID=apiConfig.get("accountId"), # type: ignore - AUTH_TOKEN=apiConfig.get("authToken"), # type: ignore - ORGANIZATION_ID=apiConfig.get("organizationId", ""), # type: ignore - CLUSTERS=json.dumps(apiConfig.get("clusters", [])), # type: ignore - ) - ) - - -async def authenticate(force=False, globals_dict=None): - if globals_dict is None: - frame = inspect.currentframe() - try: - globals_dict = frame.f_back.f_globals # type: ignore - finally: - del frame - if ENVIRONMENT == EnvironmentsEnum.PYODIDE: - get_data("data_from_host", globals_dict) - data_from_host = globals_dict.get("data_from_host") - if data_from_host: - await authenticate_jupyterlite(data_from_host) - elif ACCESS_TOKEN_ENV_VAR not in os.environ or force: - await authenticate_oidc() diff --git a/utils/settings.json b/src/py/mat3ra/notebooks_utils/core/api/settings.json similarity index 100% rename from utils/settings.json rename to src/py/mat3ra/notebooks_utils/core/api/settings.json diff --git a/utils/settings.py b/src/py/mat3ra/notebooks_utils/core/api/settings.py similarity index 92% rename from utils/settings.py rename to src/py/mat3ra/notebooks_utils/core/api/settings.py index da0552b0a..847ff6dcd 100644 --- a/utils/settings.py +++ b/src/py/mat3ra/notebooks_utils/core/api/settings.py @@ -4,12 +4,6 @@ import json import os -# General settings. For how the notebooks operate. - -# use_interactive_JSON_viewer: Whether to use the IPython interactive viewer, or print in plaintext. - -use_interactive_JSON_viewer = True - # Account settings. Need a one-time adjustment for examples to work. These should be set in settings.json # ACCOUNT_ID: Account ID. See get_authentication_params.ipynb example for more information. diff --git a/src/py/mat3ra/notebooks_utils/core/entity/__init__.py b/src/py/mat3ra/notebooks_utils/core/entity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/core/entity/compute/__init__.py b/src/py/mat3ra/notebooks_utils/core/entity/compute/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/core/entity/compute/api.py b/src/py/mat3ra/notebooks_utils/core/entity/compute/api.py new file mode 100644 index 000000000..767ec32d8 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/entity/compute/api.py @@ -0,0 +1,7 @@ +import json +import os + + +def get_cluster_name(name: str = "cluster-001") -> str: + clusters = json.loads(os.environ.get("CLUSTERS", "[]") or "[]") + return clusters[0] if clusters else name diff --git a/src/py/mat3ra/notebooks_utils/core/entity/job/__init__.py b/src/py/mat3ra/notebooks_utils/core/entity/job/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/utils/equation_of_state.py b/src/py/mat3ra/notebooks_utils/core/entity/job/analysis.py similarity index 100% rename from utils/equation_of_state.py rename to src/py/mat3ra/notebooks_utils/core/entity/job/analysis.py diff --git a/src/py/mat3ra/notebooks_utils/core/entity/job/api.py b/src/py/mat3ra/notebooks_utils/core/entity/job/api.py new file mode 100644 index 000000000..4b18b0634 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/entity/job/api.py @@ -0,0 +1,93 @@ +import urllib.request +from typing import List, Optional, Union + +from mat3ra.api_client import APIClient, JobEndpoints + + +def save_files(job_id: str, job_endpoint: JobEndpoints, filename_on_cloud: str, filename_on_disk: str) -> None: + """ + Saves a file to disk, overwriting any files with the same name as filename_on_disk. + + Args: + job_id (str): ID of the job + job_endpoint (JobEndpoints): Job endpoint object from the Exabyte API Client + filename_on_cloud (str): Name of the file on the server + filename_on_disk (str): Name the file will be saved to + """ + files = job_endpoint.list_files(job_id) + file_metadata = next(f for f in files if filename_on_cloud in f["key"]) + signed_url = file_metadata["signedUrl"] + server_response = urllib.request.urlopen(signed_url) + with open(filename_on_disk, "wb") as outp: + outp.write(server_response.read()) + + +def get_jobs_statuses_by_ids(endpoint: JobEndpoints, job_ids: List[str]) -> List[str]: + """ + Gets jobs statues by their IDs. + + Args: + endpoint (JobEndpoints): Job endpoint object from the Exabyte API Client + job_ids (list): list of job IDs to get the status for + + Returns: + list: list of job statuses + """ + jobs = endpoint.list({"_id": {"$in": job_ids}}, {"fields": {"status": 1}}) + return [job["status"] for job in jobs] + + +def create_job( + api_client: APIClient, + material_dicts: List[dict], + job_workflow_dict: dict, + project_id: str, + owner_id: str, + prefix: str, + compute: Optional[dict] = None, +) -> Union[dict, List[dict]]: + """ + Creates jobs using pre-serialised material and workflow dicts. + + Args: + api_client (APIClient): API client instance carrying the authorization context. + material_dicts (list[dict]): Serialised material dicts. + job_workflow_dict (dict): Serialised workflow dict. + project_id (str): Project ID. + owner_id (str): Account ID. + prefix (str): Job name prefix. + compute (dict, optional): Compute configuration dict. + + Returns: + dict | list[dict]: Created job(s). + """ + job_workflow_dict.pop("_id", None) + is_multimaterial = job_workflow_dict.get("isMultiMaterial", False) + + config: dict = { + "_project": {"_id": project_id}, + "workflow": job_workflow_dict, + "owner": {"_id": owner_id}, + "name": prefix, + "_material": {"_id": material_dicts[0]["_id"]}, + } + + if is_multimaterial: + config["_materials"] = [{"_id": m["_id"]} for m in material_dicts] + + if compute: + config["compute"] = compute + + return api_client.jobs.create(config) + + +def submit_jobs(endpoint: JobEndpoints, job_ids: List[str]) -> None: + """ + Submits jobs by IDs. + + Args: + endpoint (JobEndpoints): Job endpoint object from the Exabyte API Client. + job_ids (list[str]): Job IDs to submit. + """ + for job_id in job_ids: + endpoint.submit(job_id) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/__init__.py b/src/py/mat3ra/notebooks_utils/core/entity/material/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/utils/material.py b/src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py similarity index 52% rename from utils/material.py rename to src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py index 8ef2fb99d..d7156261c 100644 --- a/utils/material.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/analysis.py @@ -1,6 +1,4 @@ -import functools -import urllib.request -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any, Dict, Tuple, Union import ase.build import ase.constraints @@ -8,103 +6,12 @@ import pymatgen.core.surface import pymatgen.io.ase import pymatgen.symmetry.analyzer -from mat3ra.api_client.endpoints.jobs import JobEndpoints from mat3ra.made.material import Material -def get_bulk_material(api_client: Any, slab_material: Material, owner_id: str): - slab_dict = slab_material.to_dict() - metadata = slab_dict.get("metadata") or {} - bulk_crystal = None - - if metadata.get("bulkId") is not None: - bulk_query = {"_id": metadata["bulkId"]} - else: - for build_step in reversed(metadata.get("build") or []): - try: - bulk_crystal = build_step["configuration"]["stack_components"][0]["crystal"] - break - except (KeyError, IndexError, TypeError): - continue - - if bulk_crystal is None: - raise ValueError( - "No metadata.build[*].configuration.stack_components[0].crystal entry was found on the slab." - ) - - if bulk_crystal.get("_id") is not None: - bulk_query = {"_id": bulk_crystal["_id"]} - elif bulk_crystal.get("scaledHash") is not None: - bulk_query = {"scaledHash": bulk_crystal["scaledHash"]} - elif bulk_crystal.get("hash") is not None: - bulk_query = {"hash": bulk_crystal["hash"]} - else: - try: - bulk_query = {"hash": Material.create(bulk_crystal).hash} - except Exception as exc: - raise ValueError("Could not resolve a bulk query from the slab metadata.") from exc - - matches = api_client.materials.list(bulk_query) - bulk_material_response = next( - (item for item in matches if item.get("owner", {}).get("_id") == owner_id), - None, - ) or (matches[0] if matches else None) - - if bulk_material_response is None: - raise ValueError( - "The bulk material resolved from slab metadata is not present on the platform. " - "Run the Total Energy notebook for that bulk material first, then rerun this notebook." - ) - - print(f"Found exact bulk material: {bulk_material_response['_id']}") - - return bulk_query, bulk_material_response, Material.create(bulk_material_response) - - -def download_file_by_name(job_id: str, job_endpoint: JobEndpoints, target: str, pattern: str) -> None: - """ - Downloads a file from S3 and writes it to the local disk. - - Args: - job_id (str): The ID string for the job that the file is to be downloaded from. - job_endpoint (JobEndpoints): Job endpoint object from the Exabyte API Client - target (str): Target filename that the file will be written to. - pattern (str): The name of the file as it appears on the S3. For example, to download a file named - "CONTCAR", the pattern would be provided as "CONTCAR" - Notes: - This function does not check whether the file specified in `target` exists or not, and will always - be overwritten it if the file is successfully downloaded. - """ - # Get a list of files for each - job_files = job_endpoint.list_files(job_id) - - # Find the file - for file in job_files: - if file["name"] == pattern: - file_metadata = file - - # Get a download URL for the file - file_signed_url = file_metadata["signedUrl"] - - # Download the file to memory - server_response = urllib.request.urlopen(file_signed_url) - - # Write it to disk - with open(target, "wb") as outp: - outp.write(server_response.read()) - - -download_contcar = functools.partial(download_file_by_name, pattern="CONTCAR") - - -# Pymatgen - - def is_symmetric(slab: pymatgen.core.structure.Structure) -> bool: """ - Checks whether a slab is in a point group with inversion symmetry, which includes the following groups: - -1, 2/m, mmm, 4/m, 4/mmm,-3, -3m, 6/m, 6/mmm, m-3, m-3m - This technique is borrowed from the `nonstoichiometric_symmetrized_slab` method in PyMatGen's SlabGenerator. + Checks whether a slab is in a point group with inversion symmetry. Args: slab (pymatgen.core.structure.Structure): Slab of interest @@ -112,28 +19,24 @@ def is_symmetric(slab: pymatgen.core.structure.Structure) -> bool: Returns: True if the slab's spacegroup has inversion symmetry, otherwise False. """ - # Create a Spacegroup analyzer object with the slab spacegroup = pymatgen.symmetry.analyzer.SpacegroupAnalyzer(slab) - # Check for inversion symmetry - is_symmetric = spacegroup.is_laue() - return is_symmetric + return spacegroup.is_laue() def get_all_slabs_and_terms( crystal: pymatgen.core.structure.Structure, thickness: Union[int, float], is_by_layers: bool ) -> Dict[str, Dict[str, Dict[str, Any]]]: """ - Gets all slabs and terminations for a given crystal, forcing a specific number of layers in the resultant slab. + Gets all slabs and terminations for a given crystal. Args: - crystal (pymatgen.core.structure.Structure): Crystal of interest - thickness (int or float): How thick the slab is supposed to be, by either angstroms or number of layers - is_by_layers (bool): Whether thickness is by the number of layers or by angstroms + crystal (pymatgen.core.structure.Structure): Crystal of interest + thickness (int or float): How thick the slab should be + is_by_layers (bool): Whether thickness is by number of layers or angstroms Returns: Dict """ - # First, get a list of all non-symetrically-equivalent indices that the crystal has all_indices = pymatgen.core.surface.get_symmetrically_distinct_miller_indices(crystal, max_index=3) slabs = {} @@ -141,8 +44,6 @@ def get_all_slabs_and_terms( slab_generator = pymatgen.core.surface.SlabGenerator( crystal, plane, min_slab_size=thickness, min_vacuum_size=10, center_slab=True, in_unit_planes=is_by_layers ) - - # Generate all surface terminations, and add them to the list of slabs returned all_terminations = slab_generator.get_slabs() term_dict = {} symmetric_terminations = filter(is_symmetric, all_terminations) @@ -153,31 +54,22 @@ def get_all_slabs_and_terms( return slabs -# ASE - - def get_bulk_bottom_and_top_frac_coords(slab: ase.Atoms, layers: int = 3) -> Tuple[float, float]: """ - Finds the top and bottom of the bulk, in fractional coordinates + Finds the top and bottom of the bulk, in fractional coordinates. Args: slab (ase.Atoms): The slab of interest layers (int): How many layers are in the slab Returns: - A tuple containing, in order, the fractional coordintes of the bottom and top of the bulk portion of the slab + Tuple of (bulk_bottom, bulk_top) fractional coordinates. """ - # Determine how large the slab actually is c_direction_coords = [atom.scaled_position[2] for atom in slab] slab_range = max(c_direction_coords) - min(c_direction_coords) - - # Do some math to figure out where the bottom of the bulk starts layer_size = slab_range / layers bulk_bottom = min(c_direction_coords) + layer_size - - # Figure out where the top of the bulk starts bulk_top = min(c_direction_coords) + 2 * layer_size - return (bulk_bottom, bulk_top) @@ -187,73 +79,77 @@ def freeze_center_bulk(slab: ase.Atoms) -> None: Args: slab (ase.Atoms): The slab of interest - - Returns: - None, this function changes the slab in-place. """ - # Get the fractional coordinates for the bottom and top of the bulk bulk_bottom, bulk_top = get_bulk_bottom_and_top_frac_coords(slab) - - # Filter to get the atoms between the bottom/top of the bulk - that is, the bulk atoms frozen_atoms = filter(lambda atom: bulk_bottom <= atom.scaled_position[2] <= bulk_top, slab) - - # Get the indices of the bulk atoms, and apply the constraint to the Atoms object frozen_atoms_indices = [atom.index for atom in frozen_atoms] fix_atoms_constraint = ase.constraints.FixAtoms(indices=frozen_atoms_indices) slab.set_constraint(fix_atoms_constraint) -def get_vasp_total_energy(job_id: str, jobs_endpoint: JobEndpoints) -> Optional[float]: - """ - This function takes in a VASP Job ID, reads the OUTCAR, and returns the final energy reported in the run. - - Args: - job_id (str): The ID of the job of interest - jobs_endpoint (JobEndpoints): A job endpoint for interacting with the Exabyte platform - - Returns: - The electronic energy reported by the VASP job or None - """ - # Get the URL for the OUTCAR - files = jobs_endpoint.list_files(job_id) - file_metadata = None - for file in files: - if file["name"] == "OUTCAR": - file_metadata = file - - # Get a download URL for each CONTCAR - cell_outcar_signed_url = file_metadata["signedUrl"] # type: ignore - - # Download the outcar to memory - cell_response = urllib.request.urlopen(cell_outcar_signed_url) - - # And iterate through it, finding the last electronic energy reported - outcar = cell_response.read().decode("utf-8") - outcar = outcar.split("\n") - unit_cell_energy = None - for line in outcar: - if "sigma->0" in line: - unit_cell_energy = float(line.strip().split()[-1]) - return unit_cell_energy - - def get_surface_energy(e_slab: float, e_bulk: float, n_slab: float, n_bulk: float, a: float) -> float: """ - Calculates the slab energy according to the following formula: + Calculates the slab surface energy: (E_Slab - E_bulk * (N_Slab / N_Bulk)) / (2A) """ - surface_energy = (e_slab - e_bulk * (n_slab / n_bulk)) / (2 * a) - return surface_energy + return (e_slab - e_bulk * (n_slab / n_bulk)) / (2 * a) def get_slab_area(a_vector: np.ndarray, b_vector: np.ndarray) -> float: """ Gets the area of a slab defined by the two unit vectors. - The magnitude of the cross product of the vectors is the area of the parallelogram they enclose. Args: - a_vector + a_vector: First lattice vector. + b_vector: Second lattice vector. """ crossprod = np.cross(a_vector, b_vector) - magnitude = np.linalg.norm(crossprod) - return magnitude + return np.linalg.norm(crossprod) + + +def get_bulk_material(api_client: Any, slab_material: Material, owner_id: str): + slab_dict = slab_material.to_dict() + metadata = slab_dict.get("metadata") or {} + bulk_crystal = None + + if metadata.get("bulkId") is not None: + bulk_query = {"_id": metadata["bulkId"]} + else: + for build_step in reversed(metadata.get("build") or []): + try: + bulk_crystal = build_step["configuration"]["stack_components"][0]["crystal"] + break + except (KeyError, IndexError, TypeError): + continue + + if bulk_crystal is None: + raise ValueError( + "No metadata.build[*].configuration.stack_components[0].crystal entry was found on the slab." + ) + + if bulk_crystal.get("_id") is not None: + bulk_query = {"_id": bulk_crystal["_id"]} + elif bulk_crystal.get("scaledHash") is not None: + bulk_query = {"scaledHash": bulk_crystal["scaledHash"]} + elif bulk_crystal.get("hash") is not None: + bulk_query = {"hash": bulk_crystal["hash"]} + else: + try: + bulk_query = {"hash": Material.create(bulk_crystal).hash} + except Exception as exc: + raise ValueError("Could not resolve a bulk query from the slab metadata.") from exc + + matches = api_client.materials.list(bulk_query) + bulk_material_response = next( + (item for item in matches if item.get("owner", {}).get("_id") == owner_id), + None, + ) or (matches[0] if matches else None) + + if bulk_material_response is None: + raise ValueError( + "The bulk material resolved from slab metadata is not present on the platform. " + "Run the Total Energy notebook for that bulk material first, then rerun this notebook." + ) + + print(f"Found exact bulk material: {bulk_material_response['_id']}") + return bulk_query, bulk_material_response, Material.create(bulk_material_response) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/api.py b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py new file mode 100644 index 000000000..a82127b1b --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/api.py @@ -0,0 +1,23 @@ +from mat3ra.api_client import APIClient + + +def get_or_create_material(api_client: APIClient, material, owner_id: str) -> dict: + """ + Returns an existing material from the collection if one with the same structural hash + exists under the given owner, otherwise creates a new one. + + Args: + api_client (APIClient): API client instance carrying the authorization context. + material: mat3ra-made Material object (must have a .hash property). + owner_id (str): Account ID under which to search and create. + + Returns: + dict: The material dict (existing or newly created). + """ + existing = api_client.materials.list({"hash": material.hash, "owner._id": owner_id}) + if existing: + print(f"♻️ Reusing already existing Material: {existing[0]['_id']}") + return existing[0] + created = api_client.materials.create(material.to_dict(), owner_id=owner_id) + print(f"✅ Material created: {created['_id']}") + return created diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/io.py b/src/py/mat3ra/notebooks_utils/core/entity/material/io.py new file mode 100644 index 000000000..83055716a --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/io.py @@ -0,0 +1,159 @@ +import inspect +import json +import os +from typing import Any, List, Optional + +from mat3ra.made.material import Material +from mat3ra.made.tools.build_components import MaterialWithBuildMetadata + +from ....io import get_data, set_data +from ....primitive.enums import SeverityLevelEnum +from ....primitive.logger import log +from ....settings import UPLOADS_FOLDER + + +def get_materials(globals_dict: Optional[dict] = None) -> List[Any]: + """ + Retrieve materials from the environment and assign them to globals_dict["materials_in"]. + + Args: + globals_dict (dict, optional): The globals dictionary to populate. + + Returns: + List[Material]: A list of Material objects. + """ + if globals_dict is None: + frame = inspect.currentframe() + try: + caller_frame = frame.f_back # type: ignore + caller_globals = caller_frame.f_globals # type: ignore + globals_dict = caller_globals + finally: + del frame # Avoid reference cycles + get_data("materials_in", globals_dict) + + if "materials_in" in globals_dict and globals_dict["materials_in"]: + materials = [] + for item in globals_dict["materials_in"]: + try: + materials.append(MaterialWithBuildMetadata.create(item)) + except Exception: + materials.append(Material.create(item)) + log(f"Retrieved {len(materials)} materials.") + return materials + else: + log(f"No input materials found. Loading from the {UPLOADS_FOLDER} folder.") + return load_materials_from_folder() + + +def set_materials(materials: List[Any]): + """ + Serialize and send a list of Material objects to the environment. + + Args: + materials (List[Material]): The list of Material objects to send. + """ + from mat3ra.utils.array import convert_to_array_if_not + + materials = convert_to_array_if_not(materials) + materials_data = [json.loads(material.to_json()) for material in materials] + set_data("materials", materials_data) + + +def load_materials_from_folder(folder_path: Optional[str] = None, verbose: bool = True) -> List[Any]: + """ + Load materials from the specified folder or from the UPLOADS_FOLDER by default. + + Args: + folder_path (Optional[str]): The path to the folder containing material files. + If not provided, defaults to the UPLOADS_FOLDER. + verbose (bool): Whether to log verbose messages. + + Returns: + List[Material]: A list of Material objects loaded from the folder. + """ + folder_path = folder_path or UPLOADS_FOLDER + + if not os.path.exists(folder_path): + log(f"Folder '{folder_path}' does not exist.", SeverityLevelEnum.ERROR, force_verbose=verbose) + return [] + + data_from_host = [] + try: + index = 0 + for filename in sorted(os.listdir(folder_path)): + if filename.endswith(".json"): + file_path = os.path.join(folder_path, filename) + try: + with open(file_path, "r") as file: + data = json.load(file) + except (json.JSONDecodeError, OSError) as error: + log( + f"Skipping invalid JSON file '{file_path}': {error}", + SeverityLevelEnum.WARNING, + force_verbose=verbose, + ) + continue + name = os.path.splitext(filename)[0] + log(f"{index}: {name}", SeverityLevelEnum.INFO, force_verbose=verbose) + index += 1 + data_from_host.append(data) + except FileNotFoundError: + log(f"No data found in the '{folder_path}' folder.", SeverityLevelEnum.ERROR, force_verbose=verbose) + return [] + + try: + materials = [MaterialWithBuildMetadata.create(item) for item in data_from_host] + except Exception: + materials = [Material.create(item) for item in data_from_host] + + if materials: + log( + f"Successfully loaded {len(materials)} materials from folder '{folder_path}'", + SeverityLevelEnum.INFO, + force_verbose=verbose, + ) + else: + log(f"No materials found in folder '{folder_path}'", SeverityLevelEnum.WARNING, force_verbose=verbose) + + return materials + + +def load_material_from_folder(folder_path: str, name: str, verbose: bool = True) -> Optional[Any]: + """ + Load a single material from the specified folder by matching a substring of the name or filename. + + Args: + folder_path (str): The path to the folder containing material files. + name (str): The substring to match against material names or filenames (case-insensitive). + verbose (bool): Whether to log verbose messages. + + Returns: + Optional[Material]: The first Material object that matches, or None if not found. + """ + name_lower = name.lower() + resulting_material = None + + for filename in sorted(os.listdir(folder_path)): + if filename.endswith(".json") and name_lower in os.path.splitext(filename)[0].lower(): + with open(os.path.join(folder_path, filename), "r") as file: + data = json.load(file) + try: + resulting_material = MaterialWithBuildMetadata.create(data) + except Exception: + resulting_material = Material.create(data) + break + + if not resulting_material: + materials = load_materials_from_folder(folder_path, verbose=verbose) + for material in materials: + if name_lower in material.name.lower(): + resulting_material = material + break + + if resulting_material: + log(f"Found: '{resulting_material.name}'", SeverityLevelEnum.INFO, force_verbose=verbose) + return resulting_material + + log(f"No material containing '{name}' found in '{folder_path}'.", SeverityLevelEnum.WARNING, force_verbose=verbose) + return None diff --git a/src/py/mat3ra/notebooks_utils/core/entity/property/__init__.py b/src/py/mat3ra/notebooks_utils/core/entity/property/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/core/entity/property/api.py b/src/py/mat3ra/notebooks_utils/core/entity/property/api.py new file mode 100644 index 000000000..090373fc4 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/entity/property/api.py @@ -0,0 +1,91 @@ +from typing import List, Optional + +from mat3ra.api_client import APIClient, PropertiesEndpoints +from mat3ra.prode import PropertyName + +from .job import get_fermi_energy_flowchart_id + +FERMI_ENERGY_PROPERTIES = { + PropertyName.non_scalar.band_structure.value, + PropertyName.non_scalar.density_of_states.value, +} + + +def get_properties_for_job(client: APIClient, job_id: str, property_name: Optional[str] = None) -> List[dict]: + """ + Fetch properties for a job, automatically enriching band_structure/DOS results with fermiEnergy. + Use instead of client.properties.get_for_job when passing results to visualize_properties. + """ + job = client.jobs.get(job_id) + properties = client.properties.get_for_job(job_id, property_name) + if property_name not in FERMI_ENERGY_PROPERTIES: + return properties + flowchart_id = get_fermi_energy_flowchart_id(job) + fermi_energy = None + if flowchart_id: + fe_props = client.properties.get_for_job(job_id, PropertyName.scalar.fermi_energy.value, flowchart_id) + if fe_props: + fermi_energy = fe_props[0].get("value") + return [{**prop, "fermiEnergy": fermi_energy} for prop in properties] + + +def get_property_holder_for_job( + client: APIClient, job_id: str, property_name: str, unit_id: Optional[str] = None +) -> dict: + """ + Fetch the first full property holder for a job/property pair. + + Args: + client (APIClient): API client instance. + job_id (str): Job ID. + property_name (str): Property name. + unit_id (str, optional): Unit flowchart ID. + + Returns: + dict: Full property holder document. + """ + query = { + "source.info.jobId": job_id, + "data.name": property_name, + } + if unit_id: + query["source.info.unitId"] = unit_id + holders = client.properties.list(query=query) + if not holders: + raise ValueError(f"Property '{property_name}' not found for job '{job_id}'") + return holders[0] + + +def update_property_holder_value(client: APIClient, property_holder_id: str, value: float) -> dict: + """ + Update a scalar property's data.value. + + Args: + client (APIClient): API client instance. + property_holder_id (str): Property holder ID. + value (float): New scalar value. + + Returns: + dict: Server response payload. + """ + return client.properties.update(property_holder_id, {"$set": {"data.value": value}}) + + +def get_property_by_subworkflow_and_unit_indicies( + endpoint: PropertiesEndpoints, property_name: str, job: dict, subworkflow_index: int, unit_index: int +) -> dict: + """ + Returns the property extracted in the given unit of the job's subworkflow. + + Args: + endpoint (PropertiesEndpoints): an instance of PropertiesEndpoints class. + property_name (str): name of property to extract. + job (dict): job config to extract the property from. + subworkflow_index (int): index of subworkflow to extract the property from. + unit_index (int): index of unit to extract the property from. + + Returns: + dict: extracted property + """ + unit_flowchart_id = job["workflow"]["subworkflows"][subworkflow_index]["units"][unit_index]["flowchartId"] + return endpoint.get_property(job["_id"], unit_flowchart_id, property_name) diff --git a/utils/job_properties.py b/src/py/mat3ra/notebooks_utils/core/entity/property/job.py similarity index 100% rename from utils/job_properties.py rename to src/py/mat3ra/notebooks_utils/core/entity/property/job.py diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/__init__.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py new file mode 100644 index 000000000..8694cfced --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py @@ -0,0 +1,39 @@ +from mat3ra.api_client import APIClient, BankWorkflowEndpoints +from mat3ra.wode import Workflow + + +def get_or_create_workflow(api_client: APIClient, workflow: Workflow, owner_id: str) -> dict: + """ + Creates a workflow in the collection if none with the same hash exists under the given owner. + + Args: + api_client (APIClient): API client instance carrying the authorization context. + workflow: mat3ra-wode Workflow object. + owner_id (str): Account ID under which to search and create. + + Returns: + dict: The workflow dict (existing or newly created). + """ + existing = api_client.workflows.list({"hash": workflow.hash, "owner._id": owner_id}) + if existing: + print(f"♻️ Reusing already existing Workflow: {existing[0]['_id']}") + return existing[0] + created = api_client.workflows.create(workflow.to_dict_without_special_keys(), owner_id=owner_id) + print(f"✅ Workflow created: {created['_id']}") + return created + + +def copy_bank_workflow_by_system_name(endpoint: BankWorkflowEndpoints, system_name: str, account_id: str) -> dict: + """ + Copies a bank workflow with given system name into the account's workflows. + + Args: + endpoint (BankWorkflowEndpoints): an instance of BankWorkflowEndpoints class + system_name (str): workflow system name. + account_id (str): ID of account to copy the bank workflow into. + + Returns: + dict: new account's workflow + """ + bank_workflow_id = endpoint.list({"systemName": system_name})[0]["_id"] + return endpoint.copy(bank_workflow_id, account_id)["_id"] diff --git a/src/py/mat3ra/notebooks_utils/core/io.py b/src/py/mat3ra/notebooks_utils/core/io.py new file mode 100644 index 000000000..34a440739 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/io.py @@ -0,0 +1,69 @@ +import json +import os +import urllib.request +from typing import Any, Dict, Optional + +from ..primitive.logger import log +from ..settings import UPLOADS_FOLDER + + +def get_data_python(key: str, globals_dict: Optional[Dict] = None): + """ + Read data from the `uploads` folder in a JupyterLab environment. + + Args: + key (str): The name under which data is expected to be received. + globals_dict (dict, optional): A dictionary to store the received data. Defaults to None. + """ + try: + data_from_host = [] + index = 0 + for filename in sorted(os.listdir(UPLOADS_FOLDER)): + if filename.endswith(".json"): + with open(os.path.join(UPLOADS_FOLDER, filename), "r") as file: + data = json.load(file) + name = os.path.splitext(filename)[0] + log(f"{index}: {name}") + index += 1 + data_from_host.append(data) + if globals_dict is not None: + globals_dict[key] = data_from_host + return data_from_host + except FileNotFoundError: + print("No data found in the 'uploads' folder.") + + +def set_data_python(key: str, value: Any): + """ + Write data to the `uploads` folder in a JupyterLab environment. + + Args: + key (str): The name under which data will be written. + value (Any): The value to write to the `uploads` folder. + """ + if not os.path.exists(UPLOADS_FOLDER): + os.makedirs(UPLOADS_FOLDER) + for item in value: + safe_name = item["name"].replace("%", "pct").replace("/", ":") + file_path = os.path.join(UPLOADS_FOLDER, f"{safe_name}.json") + with open(file_path, "w") as file: + json.dump(item, file) + log(f"Data for {key} written to {file_path}") + + +def read_from_url_python(url: str, as_bytes: bool = False): + """ + Fetch, read, and decode content from a URL in a Python environment. + + Args: + url (str): The URL to fetch from. + as_bytes (bool): Whether to return the content as bytes. + + Returns: + str or bytes: The content. + """ + with urllib.request.urlopen(url) as response: + body = response.read() + if as_bytes: + return body + return body.decode("utf-8") diff --git a/src/py/mat3ra/notebooks_utils/core/prompt.py b/src/py/mat3ra/notebooks_utils/core/prompt.py new file mode 100644 index 000000000..70ae3b531 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/prompt.py @@ -0,0 +1,30 @@ +from typing import Dict + + +async def select_coordination_threshold_emscripten(distribution: Dict[int, int], default_threshold: int) -> int: + """ + Select the coordination threshold from the given distribution. Works in Pyodide environment. + + Args: + distribution: The distribution of coordination numbers. + default_threshold: The default threshold value. + + Returns: + int: The selected coordination threshold. + """ + coordination_threshold = default_threshold + coordination_numbers = list(distribution.keys()) + prompt_text = f"\nCoordination numbers distribution: {distribution}" f"\nEnter coordination threshold value: " + while True: + try: + value_str = await input(prompt_text) # type: ignore + value = int(value_str) + if value in coordination_numbers: + coordination_threshold = value + break + else: + print(f"Invalid value. Please enter one of these coordination numbers: {coordination_numbers}") + break + except ValueError: + print(f"Please enter a valid integer value from: {coordination_numbers}") + return coordination_threshold diff --git a/src/py/mat3ra/notebooks_utils/io.py b/src/py/mat3ra/notebooks_utils/io.py new file mode 100644 index 000000000..1bcc27c87 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/io.py @@ -0,0 +1,50 @@ +from typing import Any, Dict, Optional, Union + +from .core.io import get_data_python, read_from_url_python, set_data_python +from .primitive.enums import EnvironmentsEnum +from .primitive.environment import ENVIRONMENT +from .pyodide.io import get_data_pyodide, read_from_url_pyodide, set_data_pyodide + + +def get_data(key: str, globals_dict: Optional[Dict] = None): + """ + Switch between the two functions `get_data_pyodide` and `get_data_python` based on the environment. + + Args: + key (str): The name under which data is expected to be received. + globals_dict (dict, optional): A dictionary to store the received data. Defaults to None. + """ + if ENVIRONMENT == EnvironmentsEnum.PYODIDE: + get_data_pyodide(key, globals_dict) + elif ENVIRONMENT == EnvironmentsEnum.PYTHON: + get_data_python(key, globals_dict) + + +def set_data(key: str, value: Any): + """ + Switch between the two functions `set_data_pyodide` and `set_data_python` based on the environment. + + Args: + key (str): The name under which data will be written or sent. + value (Any): The value to write or send. + """ + if ENVIRONMENT == EnvironmentsEnum.PYODIDE: + set_data_pyodide(key, value) + elif ENVIRONMENT == EnvironmentsEnum.PYTHON: + set_data_python(key, value) + + +async def read_from_url(url: str, as_bytes: bool = False) -> Union[str, bytes]: + """ + Read content from a URL, routing to the pyodide or Python implementation. + + Args: + url (str): The URL to fetch from. + as_bytes (bool): Whether to return the content as bytes. + + Returns: + str or bytes: The content. + """ + if ENVIRONMENT == EnvironmentsEnum.PYODIDE: + return await read_from_url_pyodide(url, as_bytes) + return read_from_url_python(url, as_bytes) diff --git a/src/py/mat3ra/notebooks_utils/ipython/__init__.py b/src/py/mat3ra/notebooks_utils/ipython/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/utils/notebook.py b/src/py/mat3ra/notebooks_utils/ipython/_collab.py similarity index 81% rename from utils/notebook.py rename to src/py/mat3ra/notebooks_utils/ipython/_collab.py index 26ee4360e..23c2485e2 100644 --- a/utils/notebook.py +++ b/src/py/mat3ra/notebooks_utils/ipython/_collab.py @@ -5,15 +5,14 @@ import requests +# TODO: remove alongside the COLLAB environment in NBs def get_notebook_info() -> dict: """ Get the information about a currently running notebook in Google Colab. - Args: - None + Return: a dict with notebook info. """ - # ip = socket.gethostbyname(socket.gethostname()) # 172.28.0.12 ip = os.getenv("COLAB_JUPYTER_IP") response = requests.get(f"http://{ip}:9000/api/sessions").json()[0] @@ -36,11 +35,5 @@ def print_notebook_path() -> None: """ A proxy function used for a single string return when the corresponding entry-point script in 'setup.py' is called. - - Args: - None - Return: - None """ - # 'return get_notebook_info()["notebook_path"]' returns a non-zero exit code; using 'print' instead. print(get_notebook_info()["notebook_path"]) diff --git a/src/py/mat3ra/notebooks_utils/ipython/entity/__init__.py b/src/py/mat3ra/notebooks_utils/ipython/entity/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/ipython/entity/material/__init__.py b/src/py/mat3ra/notebooks_utils/ipython/entity/material/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/utils/plot.py b/src/py/mat3ra/notebooks_utils/ipython/entity/material/plot.py similarity index 58% rename from utils/plot.py rename to src/py/mat3ra/notebooks_utils/ipython/entity/material/plot.py index 8ddfc63ad..314ea8942 100644 --- a/utils/plot.py +++ b/src/py/mat3ra/notebooks_utils/ipython/entity/material/plot.py @@ -1,10 +1,11 @@ -from typing import Dict, List, Tuple, Union +from typing import Dict, List, Union from mat3ra.made.material import Material from mat3ra.made.tools.analyze.interface import ZSLMatchHolder from mat3ra.made.tools.analyze.rdf import RadialDistributionFunction -from mat3ra.utils.jupyterlite.plot import plot_distribution_function, render_figure, scatter_plot_2d -from matplotlib import pyplot as plt + +from ...plot._matplotlib import plot_distribution_function +from ...plot._plotly import create_scatter_plot_2d, render_figure def plot_strain_vs_area(matches: List["ZSLMatchHolder"], settings: Dict[str, Union[str, int]]) -> None: @@ -14,7 +15,6 @@ def plot_strain_vs_area(matches: List["ZSLMatchHolder"], settings: Dict[str, Uni Args: matches: List of interface matches to plot. settings: Plot settings. - """ x_values = [] y_values = [] @@ -39,7 +39,7 @@ def plot_strain_vs_area(matches: List["ZSLMatchHolder"], settings: Dict[str, Uni "legend_title": "Interfaces Indices", } - fig = scatter_plot_2d(x_values, y_values, hover_texts, plot_settings, trace_names) + fig = create_scatter_plot_2d(x_values, y_values, hover_texts, plot_settings, trace_names) render_figure(fig) @@ -50,10 +50,10 @@ def plot_twisted_interface_solutions(interfaces: List["Material"]) -> None: Args: interfaces: List of interfaces to plot. """ - x_values = [] - y_values = [] - hover_texts = [] - trace_names = [] + x_values: List[Union[float, int]] = [] + y_values: List[Union[float, int]] = [] + hover_texts: List[str] = [] + trace_names: List[str] = [] for i, interface in enumerate(interfaces): angle = interface.metadata.get("actual_twist_angle", 0) @@ -66,7 +66,7 @@ def plot_twisted_interface_solutions(interfaces: List["Material"]) -> None: plot_settings = {"x_title": "Twist Angle (°)", "y_title": "Number of Atoms", "title": "Twisted Interface Solutions"} - fig = scatter_plot_2d(x_values, y_values, hover_texts, plot_settings, trace_names) + fig = create_scatter_plot_2d(x_values, y_values, hover_texts, plot_settings, trace_names) render_figure(fig) @@ -78,42 +78,3 @@ def plot_rdf(material: "Material", cutoff: float = 10.0, bin_size: float = 0.1) plot_distribution_function( rdf.bin_centers, rdf.rdf, xlabel="Distance (Å)", ylabel="g(r)", title="Radial Distribution Function (RDF)" ) - - -def plot_series( - series: List[Dict], - x_key: str, - y_key: str, - xlabel: str, - ylabel: str, - title: str, - figsize: Tuple[int, int] = (8, 5), - marker: str = "o", - rotation: int = 45, -) -> None: - """ - Plot a series of data points with configurable parameters. - - Args: - series: List of dictionaries containing data points. - x_key: Key to extract x values from series items. - y_key: Key to extract y values from series items. - xlabel: Label for x-axis. - ylabel: Label for y-axis. - title: Title of the plot. - figsize: Size of the figure. - marker: Marker style for data points. - rotation: Rotation angle for x-axis labels. - """ - x_labels = [str(item[x_key]) for item in series] - y_values = [item[y_key] for item in series] - x_indices = list(range(len(series))) - figure, ax = plt.subplots(figsize=figsize) - ax.plot(x_indices, y_values, marker=marker) - ax.set_xticks(x_indices) - ax.set_xticklabels(x_labels, rotation=rotation, ha="right") - ax.set_xlabel(xlabel) - ax.set_ylabel(ylabel) - ax.set_title(title) - plt.tight_layout() - render_figure(figure) diff --git a/src/py/mat3ra/notebooks_utils/ipython/entity/material/visualize.py b/src/py/mat3ra/notebooks_utils/ipython/entity/material/visualize.py new file mode 100644 index 000000000..34aa32ebd --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/ipython/entity/material/visualize.py @@ -0,0 +1,184 @@ +import io +import time +from enum import Enum +from typing import Dict, List, Optional, Tuple, Union + +import ipywidgets as widgets +from ase.build import make_supercell +from ase.io import write +from IPython.display import HTML, Javascript, display +from mat3ra.made.material import Material +from mat3ra.made.tools.convert import to_ase +from mat3ra.utils.array import convert_to_array_if_not + +from ...ui import MaterialViewProperties, create_responsive_image_grid, get_viewer_html, get_viewer_js + + +class ViewersEnum(str, Enum): + wave = "wave" + ase = "ase" + + +def get_material_image(material: Material, title: str, rotation="0x,0y,0z", repetitions=[1, 1, 1]): + """ + Returns an image of the material structure with the specified title. + + Args: + material (Material): Material object to visualize. + title (str): Title of the image. + rotation (str): Rotation of the image. + repetitions (list): Repetitions alongside a,b,c lattice vectors. + + Returns: + tuple: Tuple containing the image bytes and the title. + """ + ase_atoms = to_ase(material) + supercell_matrix = [[repetitions[0], 0, 0], [0, repetitions[1], 0], [0, 0, repetitions[2]]] + material_repeat = make_supercell(ase_atoms, supercell_matrix) + text = f"{ase_atoms.get_chemical_formula()} - {title} - rotation: {rotation}" + + buf = io.BytesIO() + write(buf, material_repeat, format="png", rotation=rotation) + buf.seek(0) + return buf.read(), text + + +def get_wave_viewer(material, div_id, width, height, title): + size = min(width, height) + html = get_viewer_html( + div_id=div_id, + width=size, + height=size, + title=title, + custom_styles="border:1px solid #333;", + ) + js = get_viewer_js( + data_json=material.to_json(), + div_id=div_id, + bundle_url="https://exabyte-io.github.io/wave.js/main.js", + render_function="renderThreeDEditor", + data_var_name="materialConfig", + css_url="https://exabyte-io.github.io/wave.js/main.css", + ) + return html, js + + +def render_wave(material, properties, width=600, height=600): + timestamp = time.time() + div_id = f"wave-{timestamp}" + html, js = get_wave_viewer(material, div_id, width, height, properties.title) + display(HTML(html)) + display(Javascript(js)) + + +def render_wave_grid( + materials: List[Material], + list_of_properties_configs: List[MaterialViewProperties], + width=400, + height=400, + max_columns=3, +): + html_items = [] + js_items = [] + timestamp = time.time() + + for i, material in enumerate(materials): + properties_config = list_of_properties_configs[i] + div_id = f"wave-{i}-{timestamp}" + html, js = get_wave_viewer(material, div_id, width, height, properties_config.title) + html_items.append(widgets.HTML(html)) + js_items.append(Javascript(js)) + + grid = widgets.GridBox( + html_items, + layout=widgets.Layout( + grid_template_columns=f"repeat({max_columns}, 1fr)", + grid_gap="10px", + width="100%", + ), + ) + display(grid) + for js in js_items: + display(js) + + +def _process_material_entry( + material_entry: Union[Material, Dict], default_properties: MaterialViewProperties +) -> Tuple[Material, MaterialViewProperties]: + """ + Process the material entry and return the material and properties. + + Args: + material_entry: Material or a dictionary containing the material and properties. + default_properties: Default properties to use if not specified in the material entry. + + Returns: + Tuple[Material, MaterialViewProperties]: Material and properties. + """ + if isinstance(material_entry, Material): + material = material_entry + properties = default_properties + elif isinstance(material_entry, dict) and "material" in material_entry: + material = material_entry["material"] + properties = MaterialViewProperties( + title=material_entry.get("title", default_properties.title), + repetitions=material_entry.get("repetitions", default_properties.repetitions), + rotation=material_entry.get("rotation", default_properties.rotation), + ) + else: + raise ValueError("Invalid material entry") + return material, properties + + +def visualize_materials( + materials: Union[List[Material], List[Dict[str, Union[Material, dict]]]], + repetitions: Optional[List[int]] = [1, 1, 1], + rotation: Optional[str] = "0x,0y,0z", + title: Optional[str] = "Material", + viewer: ViewersEnum = ViewersEnum.ase, +) -> None: + """ + Visualize the material(s) in the output cell. + + Args: + materials: List of Materials or list of dicts with "material", "title", "repetitions", "rotation". + repetitions (Optional[List[int]]): Repetitions alongside a, b, c lattice vectors. + rotation (Optional[str]): Rotation of the image. + title (Optional[str]): Title of the image. + viewer (ViewersEnum): Viewer to use for visualization. "ase" or "wave". + """ + materials = convert_to_array_if_not(materials) + + if not materials: + print("No materials to visualize.") + return + + default_properties = MaterialViewProperties( + title=title if title is not None else MaterialViewProperties.title, + repetitions=repetitions if repetitions is not None else MaterialViewProperties.repetitions, + rotation=rotation if rotation is not None else MaterialViewProperties.rotation, + ) + + if viewer == ViewersEnum.wave: + wave_materials = [] + wave_properties_list = [] + for material_entry in materials: + material, material_properties = _process_material_entry(material_entry, default_properties) + wave_materials.append(material) + wave_properties_list.append(material_properties) + if len(wave_materials) == 1: + render_wave(wave_materials[0], properties=wave_properties_list[0]) + else: + render_wave_grid(materials=wave_materials, list_of_properties_configs=wave_properties_list) + + else: + items = [] + for material_entry in materials: + material, properties = _process_material_entry(material_entry, default_properties) + if material: + image_data, image_title = get_material_image( + material, title=properties.title, rotation=properties.rotation, repetitions=properties.repetitions + ) + items.append((image_data, image_title)) + + display(create_responsive_image_grid(items)) diff --git a/src/py/mat3ra/notebooks_utils/ipython/entity/property/__init__.py b/src/py/mat3ra/notebooks_utils/ipython/entity/property/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/ipython/entity/property/visualize.py b/src/py/mat3ra/notebooks_utils/ipython/entity/property/visualize.py new file mode 100644 index 000000000..905f07f5c --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/ipython/entity/property/visualize.py @@ -0,0 +1,46 @@ +import json +import time + +from IPython.display import HTML, Javascript, display + +from ...ui import get_viewer_html, get_viewer_js + + +def visualize_properties(results, width=900, title="Properties", extra_config=None): + """ + Visualize properties using a Prove viewer. + + Args: + results: List[dict] of property JSON objects (or a single dict). + width: Container width in pixels. + title: Title displayed above the viewer. + extra_config: Optional dict with materials, components, callbacks, etc. + """ + if isinstance(results, dict): + results = [results] + + DATA_KEYS = {"value", "values", "xDataArray"} + results = [r for r in results if DATA_KEYS & r.keys()] + + timestamp = time.time() + div_id = f"prove-{timestamp}" + results_json = json.dumps(results) + extra_config_json = json.dumps(extra_config) if extra_config else "undefined" + + html = get_viewer_html( + div_id=div_id, + width=width, + title=title, + custom_styles="border:1px solid #ddd; padding:12px; background:#fff; color:#111;", + ) + js = get_viewer_js( + data_json=results_json, + div_id=div_id, + bundle_url="https://exabyte-io.github.io/prove/main.js", + render_function="renderResults", + data_var_name="results", + extra_config_json=extra_config_json, + ) + + display(HTML(html)) + display(Javascript(js)) diff --git a/src/py/mat3ra/notebooks_utils/ipython/entity/workflow/__init__.py b/src/py/mat3ra/notebooks_utils/ipython/entity/workflow/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/ipython/entity/workflow/visualize.py b/src/py/mat3ra/notebooks_utils/ipython/entity/workflow/visualize.py new file mode 100644 index 000000000..be14f95b4 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/ipython/entity/workflow/visualize.py @@ -0,0 +1,13 @@ +from ...ui import display_JSON + + +def visualize_workflow(workflow, level: int = 2) -> None: + """ + Visualize a workflow by displaying its JSON configuration. + + Args: + workflow: Workflow object with a to_dict() method + level: Expansion level for the JSON viewer (default: 2) + """ + workflow_config = workflow.to_dict() + display_JSON(workflow_config, level=level) diff --git a/src/py/mat3ra/notebooks_utils/ipython/io.py b/src/py/mat3ra/notebooks_utils/ipython/io.py new file mode 100644 index 000000000..900fe765d --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/ipython/io.py @@ -0,0 +1,30 @@ +import json + +from IPython.display import Javascript, display + + +def download_content_to_file(content: dict, filename: str): + """ + Download content to a file with the given filename. + + Args: + content (dict): The content to download. + filename (str): The name of the file to download. + """ + if isinstance(content, dict): + content_str = json.dumps(content, indent=4) + else: + content_str = str(content) + + js_code = f""" + var content = `{content_str}`; + var filename = `{filename}`; + var blob = new Blob([content], {{ type: 'application/json' }}); + var link = document.createElement('a'); + link.href = window.URL.createObjectURL(blob); + link.download = filename; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + """ + display(Javascript(js_code)) diff --git a/src/py/mat3ra/notebooks_utils/ipython/packages/__init__.py b/src/py/mat3ra/notebooks_utils/ipython/packages/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/ipython/packages/install.py b/src/py/mat3ra/notebooks_utils/ipython/packages/install.py new file mode 100644 index 000000000..97f5597bf --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/ipython/packages/install.py @@ -0,0 +1,57 @@ +import re +from typing import List + +from ...primitive.environment import ENVIRONMENT + + +def install_package_python(pkg: str, verbose: bool = True): + """ + Install a package in a standard Python environment. + + Args: + pkg (str): The name of the package to install. + verbose (bool): Whether to print the name of the installed package. + """ + # NOTE: in a regular Python environment packages should be installed via pip, + # not programmatically from config.yml. Direct user to do so. + print('To install packages, run `pip install ".[all]"` in the terminal') + + +def get_packages_list(requirements_dict: dict, notebook_name_pattern: str = "") -> List[str]: + """ + Get the list of packages to install based on the requirements dict. + + Args: + requirements_dict (dict): The dictionary containing the requirements. + notebook_name_pattern (str): The pattern of the notebook name. + + Returns: + List[str]: The list of packages to install. + """ + packages_default_common = requirements_dict.get("default", {}).get("packages_common", []) + packages_default_environment_specific = requirements_dict.get("default", {}).get( + f"packages_{ENVIRONMENT.value}", [] + ) + + matching_notebook_requirements_list = [ + cfg for cfg in requirements_dict.get("notebooks", []) if re.search(cfg.get("name"), notebook_name_pattern) + ] + packages_notebook_common = [] + packages_notebook_environment_specific = [] + + for notebook_requirements in matching_notebook_requirements_list: + packages_common = notebook_requirements.get("packages_common", []) + packages_environment_specific = notebook_requirements.get(f"packages_{ENVIRONMENT.value}", []) + if packages_common: + packages_notebook_common.extend(packages_common) + if packages_environment_specific: + packages_notebook_environment_specific.extend(packages_environment_specific) + + # Note: environment specific packages have to be installed first, + # because in Pyodide common packages might depend on them + return [ + *packages_default_environment_specific, + *packages_notebook_environment_specific, + *packages_default_common, + *packages_notebook_common, + ] diff --git a/src/py/mat3ra/notebooks_utils/ipython/plot/__init__.py b/src/py/mat3ra/notebooks_utils/ipython/plot/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/ipython/plot/_matplotlib.py b/src/py/mat3ra/notebooks_utils/ipython/plot/_matplotlib.py new file mode 100644 index 000000000..6a52544cd --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/ipython/plot/_matplotlib.py @@ -0,0 +1,83 @@ +import io +from typing import Dict, List, Tuple + +import numpy as np +from IPython.display import Image, display +from matplotlib import pyplot as plt +from matplotlib.figure import Figure as MatplotlibFigure + + +def display_matplotlib_figure(figure: MatplotlibFigure) -> None: + buffer = io.BytesIO() + figure.savefig(buffer, format="png") + buffer.seek(0) + display(Image(buffer.read())) + plt.close(figure) + + +def plot_distribution_function( + bin_centers: np.ndarray, + distribution: np.ndarray, + xlabel: str = "Distance", + ylabel: str = "g(r)", + title: str = "Distribution Function", + figsize: Tuple[int, int] = (8, 5), +) -> None: + """ + Plot a generic distribution function. + + Args: + bin_centers: The bin centers. + distribution: The distribution values. + xlabel: The x-axis label. + ylabel: The y-axis label. + title: The title of the plot. + figsize: The size of the figure. + """ + figure = plt.figure(figsize=figsize) + plt.plot(bin_centers, distribution, label=title) + plt.xlabel(xlabel) + plt.ylabel(ylabel) + plt.title(title) + plt.legend() + plt.grid() + display_matplotlib_figure(figure) + + +def plot_series( + series: List[Dict], + x_key: str, + y_key: str, + xlabel: str, + ylabel: str, + title: str, + figsize: Tuple[int, int] = (8, 5), + marker: str = "o", + rotation: int = 45, +) -> None: + """ + Plot a series of data points with configurable parameters. + + Args: + series: List of dictionaries containing data points. + x_key: Key to extract x values from series items. + y_key: Key to extract y values from series items. + xlabel: Label for x-axis. + ylabel: Label for y-axis. + title: Title of the plot. + figsize: Size of the figure. + marker: Marker style for data points. + rotation: Rotation angle for x-axis labels. + """ + x_labels = [str(item[x_key]) for item in series] + y_values = [item[y_key] for item in series] + x_indices = list(range(len(series))) + figure, ax = plt.subplots(figsize=figsize) + ax.plot(x_indices, y_values, marker=marker) + ax.set_xticks(x_indices) + ax.set_xticklabels(x_labels, rotation=rotation, ha="right") + ax.set_xlabel(xlabel) + ax.set_ylabel(ylabel) + ax.set_title(title) + plt.tight_layout() + display_matplotlib_figure(figure) diff --git a/src/py/mat3ra/notebooks_utils/ipython/plot/_plotly.py b/src/py/mat3ra/notebooks_utils/ipython/plot/_plotly.py new file mode 100644 index 000000000..5099382d3 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/ipython/plot/_plotly.py @@ -0,0 +1,186 @@ +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import plotly.graph_objs as go +from IPython.display import clear_output, display +from plotly.subplots import make_subplots + +from ...plot import configure_matplotlib_renderer +from ...primitive.environment import is_pyodide_environment + +configure_matplotlib_renderer() + + +def render_figure(figure: Union[go.Figure, go.FigureWidget]) -> None: + if is_pyodide_environment(): + display(figure) + return + figure.show() + + +def create_scatter_plot_2d( + x_values: List[Union[float, int]], + y_values: List[Union[float, int]], + hover_texts: List[str], + settings: Dict[str, Any], + trace_names: Optional[List[str]] = None, +) -> go.Figure: + """ + Create a generic 2D scatter plot. + + Args: + x_values: List of x-coordinates + y_values: List of y-coordinates + hover_texts: List of hover texts for each point + settings: Plot settings including scales, height, and titles + trace_names: Optional list of names for each trace + """ + data = [] + for i in range(len(x_values)): + trace = go.Scatter( + x=[x_values[i]], + y=[y_values[i]], + text=[hover_texts[i]], + mode="markers", + hoverinfo="text", + name=trace_names[i] if trace_names else f"Point {i}", + ) + data.append(trace) + + layout = go.Layout( + xaxis=dict(title=settings.get("x_title", "X"), type=settings.get("x_scale", "linear")), + yaxis=dict(title=settings.get("y_title", "Y"), type=settings.get("y_scale", "linear")), + hovermode="closest", + height=settings.get("height", 600), + title=settings.get("title", ""), + legend_title_text=settings.get("legend_title", ""), + ) + + return go.Figure(data=data, layout=layout) + + +def create_realtime_plot(title: str = "Real-time Progress", x_label: str = "Step", y_label: str = "Value") -> go.Figure: + """ + Create a real-time updating plot. + """ + fig = make_subplots(rows=1, cols=1, specs=[[{"type": "scatter"}]]) + scatter = go.Scatter(x=[], y=[], mode="lines+markers", name="Progress") + fig.add_trace(scatter) + fig.update_layout(title_text=title, xaxis_title=x_label, yaxis_title=y_label) + return fig + + +def create_update_callback( + dynamic_object: Any, + value_getter: Union[Callable, Any], + figure: go.FigureWidget, + steps: List[int], + values: List[float], + value_attr: Optional[str] = None, + step_attr: str = "nsteps", + print_format: str = "Step: {}, Value: {:.4f}", +): + """ + Create a general update callback for real-time plotting. + + Args: + dynamic_object: Object containing step information + value_getter: Function to retrieve the measured value + figure: Plotly figure to update + steps: List to store step values + values: List to store measured values + step_attr: Attribute name for step count in dynamic_object + value_attr: Optional attribute name for retrieving value from dynamic_object + print_format: Format string for progress printing + """ + + def update(): + step = getattr(dynamic_object, step_attr, len(steps)) + + if callable(value_getter): + value = value_getter() + elif value_attr: + value = getattr(dynamic_object, value_attr, None) + else: + raise ValueError("Either value_getter (function) or value_attr (object attribute) must be provided.") + + steps.append(step) + values.append(value) + + print(print_format.format(step, value)) + + figure.data[0].x = steps + figure.data[0].y = values + + clear_output(wait=True) + render_figure(figure) + + return update + + +def plot_3d_surface( + x_matrix: np.ndarray, + y_matrix: np.ndarray, + z_matrix: np.ndarray, + optimal_point: Optional[Tuple[float, float]] = None, + title: str = "Surface Plot", + labels: Optional[Dict[str, str]] = None, +) -> None: + """ + Create a 3D surface plot with optional optimal point. + """ + if labels is None: + labels = {"x": "X", "y": "Y", "z": "Z"} + + fig = go.Figure(data=[go.Surface(x=x_matrix, y=y_matrix, z=z_matrix, colorscale="Viridis")]) + + if optimal_point is not None: + x_opt, y_opt = optimal_point + z_opt = np.min(z_matrix) + fig.add_trace( + go.Scatter3d( + x=[x_opt], y=[y_opt], z=[z_opt], mode="markers", marker=dict(size=8, color="red"), name="Optimal Point" + ) + ) + + fig.update_layout( + title=title, + scene=dict(xaxis_title=labels["x"], yaxis_title=labels["y"], zaxis_title=labels["z"]), + width=800, + height=800, + ) + render_figure(fig) + + +def plot_2d_heatmap( + x_values: np.ndarray, + y_values: np.ndarray, + z_matrix: np.ndarray, + optimal_point: Optional[Tuple[float, float]] = None, + title: str = "Heatmap", + labels: Optional[Dict[str, str]] = None, +) -> None: + """ + Create a 2D heatmap with optional optimal point. + """ + if labels is None: + labels = {"x": "X", "y": "Y", "z": "Z"} + + fig = go.Figure( + data=go.Heatmap(x=x_values, y=y_values, z=z_matrix, colorscale="Viridis", colorbar=dict(title=labels["z"])) + ) + + if optimal_point is not None: + x_opt, y_opt = optimal_point + fig.add_trace( + go.Scatter( + x=[x_opt], + y=[y_opt], + mode="markers", + marker=dict(size=12, color="red", symbol="x"), + name="Optimal Point", + ) + ) + + fig.update_layout(title=title, xaxis_title=labels["x"], yaxis_title=labels["y"], width=800, height=600) + render_figure(fig) diff --git a/src/py/mat3ra/notebooks_utils/ipython/ui.py b/src/py/mat3ra/notebooks_utils/ipython/ui.py new file mode 100644 index 000000000..a50f9685f --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/ipython/ui.py @@ -0,0 +1,198 @@ +import json +import os +import uuid +from typing import List + +import ipywidgets as widgets +from IPython.display import HTML, Javascript, display +from pandas import DataFrame +from pandas.io.formats.style import Styler +from pydantic import BaseModel + +from ..settings import use_interactive_JSON_viewer + +_WEB_DIR = os.path.join(os.path.dirname(__file__), "web") + + +def display_JSON(obj, interactive_viewer: bool = use_interactive_JSON_viewer, level: int = 2) -> None: + """ + Displays JSON, either interactively or via a text dump to Stdout. + + The interactive viewer is based on https://github.com/mljar/mercury/blob/main/mercury/widgets/json.py. + + Args: + obj (dict): Object to display as nicely-formatted JSON + interactive_viewer (bool): Whether to use the interactive viewer or not + level (int): The level to which the JSON should be expanded by default + """ + if interactive_viewer: + if isinstance(obj, (dict, list)): + json_str = json.dumps(obj) + else: + json_str = obj + + id = str(uuid.uuid4()) + + with open(os.path.join(_WEB_DIR, "renderjson.css")) as fp: + css = fp.read() + + with open(os.path.join(_WEB_DIR, "renderjson.js")) as fp: + js = fp.read() + + display(HTML(f'
')) + display( + HTML( + f"' + ) + ) + else: + print(json.dumps(obj, indent=4)) + + +def dataframe_to_html(df: DataFrame, text_align: str = "center") -> Styler: + """ + Converts Pandas dataframe to HTML. + + Args: + df (pd.DataFrame): Pandas dataframe. + text_align (str): text align. Defaults to center. + """ + styles = [ + dict(selector="th", props=[("text-align", text_align)]), + dict(selector="td", props=[("text-align", text_align)]), + ] + return df.style.set_table_styles(styles) + + +class MaterialViewProperties(BaseModel): + repetitions: List[int] = [1, 1, 1] + rotation: str = "0x,0y,0z" + title: str = "Material" + + +def create_image_widget(image_data, format="png", object_fit="contain"): + """ + Creates an Image widget with specified layout settings. + + Args: + image_data (bytes): The image data to be displayed. + format (str): The format of the image, default is 'png'. + object_fit (str): CSS object-fit property value, default is 'contain'. + + Returns: + widgets.Image: A configured image widget. + """ + image = widgets.Image(value=image_data, format=format) + image.layout.object_fit = object_fit + return image + + +def create_responsive_image_grid(image_tuples, max_columns=3): + """ + Create a responsive image grid. Limits the grid to a maximum of three columns. + + Args: + image_tuples (list): List of tuples where each tuple contains an image and a title. + max_columns (int): Maximum number of columns in the grid. + """ + items = [ + widgets.VBox( + [ + widgets.Label(value=title, layout=widgets.Layout(height="30px", align_self="center")), + create_image_widget(image, object_fit="contain"), + ], + layout=widgets.Layout(align_items="center", padding="0px 0px 10px 0px"), + ) + for image, title in image_tuples + ] + + column_width = f"minmax(100px, {100 / max_columns}%)" + grid = widgets.GridBox( + items, + layout=widgets.Layout( + grid_template_columns=f"repeat({max_columns}, {column_width})", + grid_gap="10px", + width="100%", + ), + ) + return grid + + +def get_viewer_html(div_id, width, height=None, title="Viewer", custom_styles=""): + """ + Generate HTML container for a viewer. + + Args: + div_id: Unique ID for the container div + width: Width in pixels + height: Height in pixels (optional, if not provided only width is set) + title: Title to display above the viewer + custom_styles: Additional inline CSS styles + """ + if height is not None: + size_style = f"width:{width}px; height:{height}px;" + else: + size_style = f"width:{width}px;" + + return f""" +

{title}

+
+ """ + + +def get_viewer_js( + data_json, + div_id, + bundle_url, + render_function, + data_var_name="data", + extra_config_json=None, + css_url=None, +): + """ + Generate JavaScript to load and render a viewer bundle. + + Args: + data_json: JSON string of data to render + div_id: Container div ID + bundle_url: URL to the JS bundle + render_function: Name of the window function to call + data_var_name: Variable name for the data + extra_config_json: Optional extra config as JSON string + css_url: Optional CSS file URL to load + """ + extra_config_arg = f", {extra_config_json}" if extra_config_json else "" + css_loader = ( + f""" + document.head.insertAdjacentHTML( + 'beforeend', + ''); + """ + if css_url + else "" + ) + + return f""" + const {data_var_name}={data_json}; + const container = document.getElementById('{div_id}'); + (async function() {{ + await import('{bundle_url}'); + window.{render_function}({data_var_name}, container{extra_config_arg}); + }})(); + {css_loader} + """ + + +def show_device_flow_popup(verification_uri_complete: str, user_code: str) -> None: + display( + HTML( + f"
" + f"Authentication Required
" + f"Enter this code: {user_code}" + f"
" + ) + ) + display(Javascript(f"window.open({verification_uri_complete!r}, '_blank');")) diff --git a/utils/web/renderjson.css b/src/py/mat3ra/notebooks_utils/ipython/web/renderjson.css similarity index 100% rename from utils/web/renderjson.css rename to src/py/mat3ra/notebooks_utils/ipython/web/renderjson.css diff --git a/utils/web/renderjson.js b/src/py/mat3ra/notebooks_utils/ipython/web/renderjson.js similarity index 100% rename from utils/web/renderjson.js rename to src/py/mat3ra/notebooks_utils/ipython/web/renderjson.js diff --git a/src/py/mat3ra/notebooks_utils/job.py b/src/py/mat3ra/notebooks_utils/job.py new file mode 100644 index 000000000..a8b80a910 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/job.py @@ -0,0 +1,101 @@ +import datetime +from collections import Counter +from functools import wraps +from typing import Any, Callable, List + +from mat3ra.api_client import APIClient, JobEndpoints +from mat3ra.made.material import Material +from mat3ra.utils.extra.tabulate import pretty_print +from mat3ra.wode import Workflow + +from .core.entity.job.api import create_job as _create_job +from .core.entity.job.api import get_jobs_statuses_by_ids, save_files, submit_jobs +from .pyodide.runtime import interruptible_polling_loop + + +# TODO: place to mat3ra-made +def convert_material_args(fn: Callable) -> Callable: + """Converts any Material or List[Material] arg/kwarg to dict(s).""" + + @wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + def convert(v: Any) -> Any: + if isinstance(v, Material): + return v.to_dict() + if isinstance(v, list): + return [i.to_dict() if isinstance(i, Material) else i for i in v] + return v + + return fn(*[convert(a) for a in args], **{k: convert(v) for k, v in kwargs.items()}) + + return wrapper + + +# TODO: place to mat3ra-wode +def convert_workflow_args(fn: Callable) -> Callable: + """Converts any Workflow arg/kwarg to a dict.""" + + @wraps(fn) + def wrapper(*args: Any, **kwargs: Any) -> Any: + convert = lambda v: v.to_dict() if isinstance(v, Workflow) else v # noqa: E731 + return fn(*[convert(a) for a in args], **{k: convert(v) for k, v in kwargs.items()}) + + return wrapper + + +create_job = convert_workflow_args(convert_material_args(_create_job)) + + +def get_convergence_series(client: APIClient, job_id: str, subworkflow_index: int = 0) -> List[dict]: + """ + Returns the convergence series from a finished convergence job. + + Args: + client: API client instance. + job_id: ID of the finished convergence job. + subworkflow_index: Index of the convergence subworkflow (default 0). + + Returns: + List of dicts with keys "x", "parameter", "y". + """ + finished_job = client.jobs.get(job_id) + job_workflow = Workflow.create(finished_job["workflow"]) + subworkflow = job_workflow.subworkflows[subworkflow_index] + return subworkflow.convergence_series(finished_job.get("scopeTrack")) + + +@interruptible_polling_loop() +def wait_for_jobs_to_finish_async(endpoint: JobEndpoints, job_ids: List[str]) -> bool: + """ + Waits for jobs to finish and prints their statuses. + A job is considered finished if it is not in "pre-submission", "submitted", or "active" status. + + Args: + endpoint (JobEndpoints): Job endpoint object from the Exabyte API Client + job_ids (list): list of job IDs to wait for + """ + statuses = get_jobs_statuses_by_ids(endpoint, job_ids) + counts = Counter(statuses) + headers = ["TIME", "SUBMITTED-JOBS", "ACTIVE-JOBS", "FINISHED-JOBS", "ERRORED-JOBS"] + now = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") + row = [ + now, + counts.get("submitted", 0) + counts.get("queued", 0), + counts.get("active", 0), + counts.get("finished", 0), + counts.get("error", 0), + ] + pretty_print([row], headers, tablefmt="grid", stralign="center") + + active_statuses = {"pre-submission", "submitted", "queued", "active"} + return not statuses or any(status in active_statuses for status in statuses) + + +__all__ = [ + "create_job", + "get_convergence_series", + "get_jobs_statuses_by_ids", + "save_files", + "submit_jobs", + "wait_for_jobs_to_finish_async", +] diff --git a/src/py/mat3ra/notebooks_utils/material.py b/src/py/mat3ra/notebooks_utils/material.py new file mode 100644 index 000000000..69cb41cfd --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/material.py @@ -0,0 +1,8 @@ +from .core.entity.material.io import get_materials, load_material_from_folder, load_materials_from_folder, set_materials + +__all__ = [ + "get_materials", + "set_materials", + "load_materials_from_folder", + "load_material_from_folder", +] diff --git a/src/py/mat3ra/notebooks_utils/packages.py b/src/py/mat3ra/notebooks_utils/packages.py new file mode 100644 index 000000000..948c455a7 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/packages.py @@ -0,0 +1,68 @@ +import importlib +import json +import os +from typing import List + +from .ipython.packages.install import get_packages_list, install_package_python +from .primitive.enums import EnvironmentsEnum +from .primitive.environment import ENVIRONMENT, is_pyodide_environment +from .primitive.logger import log +from .pyodide.packages.install import get_config_yml_file_path, install_init, install_package_pyodide + + +async def install_package(pkg: str, verbose: bool = True): + """ + Install a package in the current environment. + + Args: + pkg (str): The name of the package to install. + verbose (bool): Whether to print the name of the installed package. + """ + if ENVIRONMENT == EnvironmentsEnum.PYODIDE: + await install_package_pyodide(pkg, verbose) + elif ENVIRONMENT == EnvironmentsEnum.PYTHON: + install_package_python(pkg, verbose) + + +async def install_packages_with_hashing(packages: List[str], verbose: bool = True): + """ + Install the given packages, skipping if the list is unchanged since last run. + + Args: + packages (List[str]): The list of packages to install. + verbose (bool): Whether to print the names of the installed packages. + """ + requirements_hash = str(hash(json.dumps(packages))) + if os.environ.get("requirements_hash") != requirements_hash: + for pkg in packages: + await install_package(pkg, verbose) + if verbose: + log("Packages installed successfully.", force_verbose=verbose) + os.environ["requirements_hash"] = requirements_hash + else: + if verbose: + log("Packages are already installed.", force_verbose=verbose) + + +async def install_packages(notebook_name_pattern: str, config_file_path: str = "", verbose: bool = True): + """ + Install the packages listed in config.yml for the given notebook name pattern. + + Usage in notebooks: + from mat3ra.notebooks_utils.packages import install_packages + await install_packages("my_notebook") + + Args: + notebook_name_pattern (str): Pattern matched against notebook names in config.yml. + config_file_path (str): Path to config.yml; empty string uses the JupyterLite default (/drive/config.yml). + verbose (bool): Whether to print install progress. + """ + if is_pyodide_environment(): + await install_init() + + yaml = importlib.import_module("yaml") + with open(get_config_yml_file_path(config_file_path), "r") as f: + requirements_dict = yaml.safe_load(f) + + packages = get_packages_list(requirements_dict, notebook_name_pattern) + await install_packages_with_hashing(packages, verbose) diff --git a/src/py/mat3ra/notebooks_utils/plot.py b/src/py/mat3ra/notebooks_utils/plot.py new file mode 100644 index 000000000..0302881c5 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/plot.py @@ -0,0 +1,8 @@ +from matplotlib import pyplot as plt + +from .primitive.environment import is_pyodide_environment + + +def configure_matplotlib_renderer() -> None: + if is_pyodide_environment(): + plt.switch_backend("Agg") diff --git a/src/py/mat3ra/notebooks_utils/primitive/__init__.py b/src/py/mat3ra/notebooks_utils/primitive/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/primitive/enums.py b/src/py/mat3ra/notebooks_utils/primitive/enums.py new file mode 100644 index 000000000..a18491004 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/primitive/enums.py @@ -0,0 +1,12 @@ +from enum import Enum + + +class EnvironmentsEnum(Enum): + PYODIDE = "pyodide" + PYTHON = "python" + + +class SeverityLevelEnum(Enum): + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" diff --git a/src/py/mat3ra/notebooks_utils/primitive/environment.py b/src/py/mat3ra/notebooks_utils/primitive/environment.py new file mode 100644 index 000000000..207b43753 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/primitive/environment.py @@ -0,0 +1,10 @@ +import os + +from .enums import EnvironmentsEnum + +# default value for env.HOME from https://pyodide.org/en/stable/usage/api/js-api.html +ENVIRONMENT = EnvironmentsEnum.PYODIDE if os.environ.get("HOME") == "/home/pyodide" else EnvironmentsEnum.PYTHON + + +def is_pyodide_environment() -> bool: + return ENVIRONMENT == EnvironmentsEnum.PYODIDE diff --git a/src/py/mat3ra/notebooks_utils/primitive/logger.py b/src/py/mat3ra/notebooks_utils/primitive/logger.py new file mode 100644 index 000000000..5cad9a03c --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/primitive/logger.py @@ -0,0 +1,33 @@ +import inspect +import os +from typing import Optional + +from .enums import SeverityLevelEnum + + +def log(message: str, level: Optional[SeverityLevelEnum] = None, force_verbose: Optional[bool] = None): + """ + Log a message based on the VERBOSE flag in the caller's globals(). + + Args: + message (str): The message to log. + level (SeverityLevelEnum, optional): The severity level of the message (e.g., INFO, WARNING, ERROR). + force_verbose (bool, optional): If True, log the message regardless of the VERBOSE flag in globals(). + """ + if force_verbose is True: + should_log = True + elif force_verbose is False: + should_log = False + else: + frame = inspect.currentframe() + try: + caller_frame = frame.f_back # type: ignore + caller_globals = caller_frame.f_globals # type: ignore + should_log = caller_globals.get("VERBOSE", os.environ.get("VERBOSE", True)) + finally: + del frame # Avoid reference cycles + if should_log: + if level is None: + print(message) + else: + print(f"{level.value}: {message}") diff --git a/utils/io.py b/src/py/mat3ra/notebooks_utils/primitive/prompt.py similarity index 53% rename from utils/io.py rename to src/py/mat3ra/notebooks_utils/primitive/prompt.py index 3a8221c20..0d900f424 100644 --- a/utils/io.py +++ b/src/py/mat3ra/notebooks_utils/primitive/prompt.py @@ -1,4 +1,3 @@ -import sys from typing import Any, Dict, List, Optional, Union @@ -47,7 +46,7 @@ def ui_prompt_select_array_element_by_index( Prompt the user to select an element from an array by index and return the chosen element. Args: array (List[Any]): The array (list) of elements to select from. - element_name (str): The name of the element to be used in the prompt text (e.g., "transformation", "interface") + element_name (str): The name of the element to be used in the prompt text. prompt_head: The prompt text to be displayed before the list of elements. Returns: @@ -63,58 +62,6 @@ def ui_prompt_select_array_element_by_index( return result -async def ui_prompt_select_array_element_by_index_pyodide( - array: List[Any], element_name: str = "element", prompt_head: Optional[str] = None -) -> Any: - """ - Prompt the user to select an element from an array by index and return the chosen element. - Used in Pyodide environment, needs to be awaited since the return of input() is type of PyodideFuture. - Args: - array (List[Any]): The array (list) of elements to select from. - element_name (str): The name of the element to be used in the prompt text (e.g., "transformation", "interface") - prompt_head: The prompt text to be displayed before the list of elements. - - Returns: - Any: The selected element from the array. - """ - prompt_text = create_prompt_text(array, element_name, prompt_head) - # `input()` in Pyodide returns PyodideFuture, which is not compatible with `int` in regular Python environment - selected_index_str = await input(prompt_text) # type: ignore - index = get_integer_from_input(selected_index_str, array) - if index is None: - return None - result = array[index] - print(f"Selected {element_name}: ", array[index]) - return result - - -async def select_coordination_threshold_emscripten(distribution: Dict[int, int], default_threshold: int) -> int: - """ - Select the coordination threshold from the given distribution. Works in Pyodide environment. - Args: - distribution: The distribution of coordination numbers. - default_threshold: The default threshold value. - Returns: - int: The selected coordination threshold. - """ - coordination_threshold = default_threshold - coordination_numbers = list(distribution.keys()) - prompt_text = f"\nCoordination numbers distribution: {distribution}" f"\nEnter coordination threshold value: " - while True: - try: - value_str = await input(prompt_text) # type: ignore - value = int(value_str) - if value in coordination_numbers: - coordination_threshold = value - break - else: - print(f"Invalid value. Please enter one of these coordination numbers: {coordination_numbers}") - break - except ValueError: - print(f"Please enter a valid integer value from: {coordination_numbers}") - return coordination_threshold - - def select_coordination_threshold_python(distribution: Dict[int, int], default_threshold: int) -> int: """ Select the coordination threshold from the given distribution. Works in regular Python environment. @@ -140,18 +87,3 @@ def select_coordination_threshold_python(distribution: Dict[int, int], default_t except ValueError: print(f"Please enter a valid integer value from: {coordination_numbers}") return coordination_threshold - - -async def select_coordination_threshold(distribution: Dict[int, int], default_threshold: int) -> int: - """ - Select the coordination threshold from the given distribution. - Args: - distribution: The distribution of coordination numbers. - default_threshold: The default threshold value. - Returns: - int: The selected coordination threshold. - """ - if sys.platform == "emscripten": - return await select_coordination_threshold_emscripten(distribution, default_threshold) - else: - return select_coordination_threshold_python(distribution, default_threshold) diff --git a/src/py/mat3ra/notebooks_utils/pyodide/__init__.py b/src/py/mat3ra/notebooks_utils/pyodide/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/pyodide/api/__init__.py b/src/py/mat3ra/notebooks_utils/pyodide/api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/pyodide/api/auth.py b/src/py/mat3ra/notebooks_utils/pyodide/api/auth.py new file mode 100644 index 000000000..400d01e30 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/pyodide/api/auth.py @@ -0,0 +1,15 @@ +import json +import os + + +async def authenticate_jupyterlite(data_from_host: dict) -> None: + apiConfig = data_from_host.get("apiConfig") + os.environ.update(data_from_host.get("environ", {})) + os.environ.update( + dict( + ACCOUNT_ID=apiConfig.get("accountId"), # type: ignore + AUTH_TOKEN=apiConfig.get("authToken"), # type: ignore + ORGANIZATION_ID=apiConfig.get("organizationId", ""), # type: ignore + CLUSTERS=json.dumps(apiConfig.get("clusters", [])), # type: ignore + ) + ) diff --git a/src/py/mat3ra/notebooks_utils/pyodide/io.py b/src/py/mat3ra/notebooks_utils/pyodide/io.py new file mode 100644 index 000000000..9dbabfeba --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/pyodide/io.py @@ -0,0 +1,93 @@ +import io +import json +import os +from typing import Any, Dict, Optional, Union + +from IPython.display import Javascript, display + +from ..core.io import set_data_python +from ..primitive.logger import log + + +async def read_from_url_pyodide(url: str, as_bytes: bool = False) -> Union[str, bytes]: + """ + Fetch and read content from a URL in a Pyodide environment. + + Args: + url (str): The URL to fetch from. + as_bytes (bool): Whether to return the content as bytes. + + Returns: + str or bytes: The content. + """ + # `http` is a Pyodide module that will be installed in the Pyodide environment by default. + from pyodide.http import pyfetch # type: ignore + + # Per https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API/Using_Fetch + response = await pyfetch(url) + if as_bytes: + return await response.bytes() + return await response.string() + + +def set_data_pyodide(key: str, value: Any): + """ + Take a Python object, serialize it to JSON, and send it to the host environment + through a JavaScript function defined in the JupyterLite extension `data_bridge`. + + Args: + key (str): The name under which data will be sent. + value (Any): The value to send to the host environment. + """ + serialized_data = json.dumps({key: value}) + js_code = f""" + (function() {{ + if (window.sendDataToHost) {{ + window.sendDataToHost({serialized_data}); + console.log('Data sent to host:', {serialized_data}); + }} else {{ + console.error('sendDataToHost function is not defined on the window object.'); + }} + }})(); + """ + display(Javascript(js_code)) + log(f"Data for {key} sent to host.") + set_data_python(key, value) + + +def get_data_pyodide(key: str, globals_dict: Optional[Dict] = None): + """ + Load data from the host environment into globals()[key] variable. + + Args: + key (str): Global variable name to store the received data. + globals_dict (dict, optional): globals() dictionary of the current scope. + """ + if globals_dict is not None: + globals_dict[key] = globals_dict.get("data_from_host", None) + + +async def write_to_file(file_name: str, file_content, mode: str = "wb"): + """ + Write content to a file, handling both Python and Pyodide environments. + + Args: + file_name (str): The name of the file to write. + file_content (str | bytes | io.StringIO | io.BytesIO): The content to write. + mode (str): The mode to open the file in. Defaults to "wb" (write bytes). + + Returns: + str: The absolute path of the saved file. + """ + if isinstance(file_content, io.StringIO): + file_content = file_content.getvalue().encode("utf-8") + elif isinstance(file_content, io.BytesIO): + file_content = file_content.getvalue() + + if "b" in mode and isinstance(file_content, str): + file_content = file_content.encode("utf-8") + + with open(file_name, mode) as file: + file.write(file_content) + + return os.path.abspath(file_name) diff --git a/src/py/mat3ra/notebooks_utils/pyodide/packages/__init__.py b/src/py/mat3ra/notebooks_utils/pyodide/packages/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/pyodide/packages/install.py b/src/py/mat3ra/notebooks_utils/pyodide/packages/install.py new file mode 100644 index 000000000..084f94ad2 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/pyodide/packages/install.py @@ -0,0 +1,63 @@ +import importlib +import os +import sys + +from ...primitive.logger import log + +try: + import micropip # type: ignore +except ImportError: + micropip = None # type: ignore + +PYODIDE_INIT_PACKAGES = ["pyyaml"] +PYODIDE_INIT_MODULES = ["yaml"] + + +def get_config_yml_file_path(config_file_path: str) -> str: + """ + Resolve the absolute path to the config.yml file. + + Args: + config_file_path (str): Relative path override; empty string uses the JupyterLite default (/drive/config.yml). + + Returns: + str: Absolute path to the config file. + """ + config_file_full_path = os.path.normpath(os.path.join("/drive/", "./config.yml")) + if config_file_path != "": + config_file_full_path = os.path.normpath(os.path.join(os.getcwd(), config_file_path)) + return config_file_full_path + + +async def install_init(): + if sys.platform != "emscripten": + return + + await micropip.install(PYODIDE_INIT_PACKAGES) + for module in PYODIDE_INIT_MODULES: + importlib.import_module(module) + + +async def install_package_pyodide(pkg: str, verbose: bool = True): + """ + Install a package in a Pyodide environment. + + Args: + pkg (str): The name of the package to install. Can be prefixed with 'nodeps:' to skip dependencies. + verbose (bool): Whether to print the name of the installed package. + + Examples: + await install_package_pyodide("numpy") # installs with deps + await install_package_pyodide("nodeps:e3nn==0.4.4") # installs without deps + """ + if pkg.startswith("nodeps:"): + pkg = pkg.replace("nodeps:", "") + are_dependencies_installed = False + else: + is_url = pkg.startswith("http://") or pkg.startswith("https://") or pkg.startswith("emfs:/") + are_dependencies_installed = not is_url + + await micropip.install(pkg, deps=are_dependencies_installed) + pkg_name = pkg.split("/")[-1].split("-")[0] if "://" in pkg else pkg.split("==")[0] + if verbose: + log(f"Installed {pkg_name}", force_verbose=verbose) diff --git a/utils/torch_pyodide.py b/src/py/mat3ra/notebooks_utils/pyodide/packages/torch.py similarity index 99% rename from utils/torch_pyodide.py rename to src/py/mat3ra/notebooks_utils/pyodide/packages/torch.py index 838c6ccde..3aa373ac2 100644 --- a/utils/torch_pyodide.py +++ b/src/py/mat3ra/notebooks_utils/pyodide/packages/torch.py @@ -5,7 +5,7 @@ in Pyodide's WASM environment, organized by functionality. Usage: - from utils.torch_pyodide import ( + from mat3ra.notebooks_utils.other.torch_pyodide import ( patch_torch_linalg, patch_torch_testing, patch_matscipy, diff --git a/src/py/mat3ra/notebooks_utils/pyodide/runtime.py b/src/py/mat3ra/notebooks_utils/pyodide/runtime.py new file mode 100644 index 000000000..efac1ffa6 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/pyodide/runtime.py @@ -0,0 +1,209 @@ +import asyncio +import inspect +import uuid +from functools import wraps +from typing import Any, Awaitable, Callable + +from pydantic import BaseModel + +from ..primitive.environment import ENVIRONMENT, EnvironmentsEnum + +try: + from IPython.display import HTML, display # type: ignore +except Exception: + HTML = None + display = None + + +class UserAbortError(RuntimeError): + pass + + +def display_abort_controls_in_current_cell_output( + channel_name: str = "mat3ra_abort_channel", + abort_button_text: str = "Stop polling", +) -> None: + """ + Shows: + [Stop polling] Press ESC to abort + + Only works in notebook frontends that support HTML output. + Safe no-op otherwise. + """ + if HTML is None or display is None: + return + + element_id = f"abort_controls_{uuid.uuid4().hex}" + + display( + HTML( + f""" +
+ + + Press ESC to abort + +
+ + + """ + ) + ) + + +class BroadcastChannelAbortController(BaseModel): + """ + WebWorker-side receiver. Works only in pyodide (emscripten). + In regular Python: start() does nothing and is_aborted stays False. + """ + + channel_name: str = "mat3ra_abort_channel" + is_aborted: bool = False + + def model_post_init(self, __context: Any) -> None: + self._broadcast_channel = None + self._on_message_proxy = None + + def start(self) -> None: + if ENVIRONMENT != EnvironmentsEnum.PYODIDE: + return + if self._broadcast_channel is not None: + return + + import js # type: ignore + from pyodide.ffi import create_proxy # type: ignore + + self._broadcast_channel = js.BroadcastChannel.new(self.channel_name) + + def on_message(event) -> None: + message = getattr(event, "data", None) + if message and getattr(message, "type", None) == "abort": + self.is_aborted = True + + self._on_message_proxy = create_proxy(on_message) + self._broadcast_channel.onmessage = self._on_message_proxy # type: ignore + + def stop(self) -> None: + if self._broadcast_channel is None: + return + + self._broadcast_channel.close() + self._broadcast_channel = None + + if self._on_message_proxy is not None: + self._on_message_proxy.destroy() + self._on_message_proxy = None + + +async def run_interruptible_loop_async( + loop_body: Callable[[], Awaitable[bool]], + poll_interval_seconds: float, + *, + channel_name: str = "mat3ra_abort_channel", + check_interval_seconds: float = 0.05, + show_controls: bool = True, +) -> None: + """ + Wraps an async loop around a "poll" function that returns True to continue, False to stop. + + loop_body(): + - do one "poll" iteration + - return True to keep looping, False to stop normally + + Between iterations we sleep in small slices so: + - pyodide: ESC/button can be received and stop the loop + - regular Python: yields control (Ctrl+C/Stop works where supported) + """ + broadcast_channel_abort_controller = BroadcastChannelAbortController(channel_name=channel_name) + broadcast_channel_abort_controller.start() + + if show_controls and ENVIRONMENT == EnvironmentsEnum.PYODIDE: + display_abort_controls_in_current_cell_output(channel_name=channel_name, abort_button_text="Abort") + + try: + while True: + should_continue = await loop_body() + if not should_continue: + return + + remaining_seconds = float(poll_interval_seconds) + while remaining_seconds > 0: + if broadcast_channel_abort_controller.is_aborted: + raise UserAbortError("Aborted by user.") + await asyncio.sleep(min(check_interval_seconds, remaining_seconds)) + remaining_seconds -= check_interval_seconds + + finally: + broadcast_channel_abort_controller.stop() + + +def interruptible_polling_loop( + poll_interval_kwarg_name: str = "poll_interval", + *, + default_poll_interval_seconds: float = 10.0, + channel_name: str = "mat3ra_abort_channel", + check_interval_seconds: float = 0.05, + show_controls: bool = True, +): + """ + Turns a poll-step function into an async loop. Wrapped fn returns True to continue, False to stop. + Sleeps in small slices so ESC/Abort (notebooks) or Ctrl+C can raise UserAbortError. + Poll interval: kwarg poll_interval_kwarg_name, else default_poll_interval_seconds. + """ + + def decorator(poll_step_function: Callable[..., Any]) -> Callable[..., Any]: + @wraps(poll_step_function) + async def wrapped(*args: Any, **kwargs: Any) -> None: + poll_interval_seconds = float(kwargs.pop(poll_interval_kwarg_name, default_poll_interval_seconds)) + + async def loop_body() -> bool: + result = poll_step_function(*args, **kwargs) + should_continue = await result if inspect.isawaitable(result) else result + return bool(should_continue) + + await run_interruptible_loop_async( + loop_body, + poll_interval_seconds, + channel_name=channel_name, + check_interval_seconds=check_interval_seconds, + show_controls=show_controls, + ) + + return wrapped + + return decorator diff --git a/src/py/mat3ra/notebooks_utils/pyodide/ui.py b/src/py/mat3ra/notebooks_utils/pyodide/ui.py new file mode 100644 index 000000000..27b556bb6 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/pyodide/ui.py @@ -0,0 +1,29 @@ +from typing import Any, List, Optional + +from ..primitive.prompt import create_prompt_text, get_integer_from_input + + +async def ui_prompt_select_array_element_by_index_pyodide( + array: List[Any], element_name: str = "element", prompt_head: Optional[str] = None +) -> Any: + """ + Prompt the user to select an element from an array by index and return the chosen element. + Used in Pyodide environment, needs to be awaited since the return of input() is type of PyodideFuture. + + Args: + array (List[Any]): The array (list) of elements to select from. + element_name (str): The name of the element to be used in the prompt text. + prompt_head: The prompt text to be displayed before the list of elements. + + Returns: + Any: The selected element from the array. + """ + prompt_text = create_prompt_text(array, element_name, prompt_head) + # `input()` in Pyodide returns PyodideFuture, which is not compatible with `int` in regular Python environment + selected_index_str = await input(prompt_text) # type: ignore + index = get_integer_from_input(selected_index_str, array) + if index is None: + return None + result = array[index] + print(f"Selected {element_name}: ", array[index]) + return result diff --git a/src/py/mat3ra/notebooks_utils/settings.py b/src/py/mat3ra/notebooks_utils/settings.py new file mode 100644 index 000000000..5245b5d22 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/settings.py @@ -0,0 +1,10 @@ +# General settings. For how the notebooks operate. + +from .core.api.settings import ACCOUNT_ID, AUTH_TOKEN, MATERIALS_PROJECT_API_KEY, ORGANIZATION_ID + +# use_interactive_JSON_viewer: Whether to use the IPython interactive viewer, or print in plaintext. +use_interactive_JSON_viewer = True + +UPLOADS_FOLDER = "uploads" + +__all__ = ["ACCOUNT_ID", "AUTH_TOKEN", "MATERIALS_PROJECT_API_KEY", "ORGANIZATION_ID"] diff --git a/src/py/mat3ra/notebooks_utils/ui.py b/src/py/mat3ra/notebooks_utils/ui.py new file mode 100644 index 000000000..2b934cd86 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/ui.py @@ -0,0 +1,26 @@ +from typing import Dict + +from .core.prompt import select_coordination_threshold_emscripten +from .ipython.ui import dataframe_to_html, display_JSON +from .primitive.environment import is_pyodide_environment +from .primitive.prompt import select_coordination_threshold_python + + +async def select_coordination_threshold(distribution: Dict[int, int], default_threshold: int) -> int: + """ + Select the coordination threshold from the given distribution. + + Args: + distribution: The distribution of coordination numbers. + default_threshold: The default threshold value. + + Returns: + int: The selected coordination threshold. + """ + if is_pyodide_environment(): + return await select_coordination_threshold_emscripten(distribution, default_threshold) + else: + return select_coordination_threshold_python(distribution, default_threshold) + + +__all__ = ["dataframe_to_html", "display_JSON"] diff --git a/tests/py/unit/__init__.py b/tests/py/unit/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/py/unit/test_jupyterlite_interrupts.py b/tests/py/unit/test_jupyterlite_interrupts.py new file mode 100644 index 000000000..9b072e2b1 --- /dev/null +++ b/tests/py/unit/test_jupyterlite_interrupts.py @@ -0,0 +1,50 @@ +import asyncio + +import pytest +from mat3ra.notebooks_utils.pyodide.runtime import ( + UserAbortError, + interruptible_polling_loop, + run_interruptible_loop_async, +) + +POLL_INTERVAL_SECONDS = 0.01 +CHECK_INTERVAL_SECONDS = 0.005 + + +@pytest.mark.asyncio +async def test_run_interruptible_loop_async_stops_when_body_returns_false(): + call_count = 0 + + async def loop_body(): + nonlocal call_count + call_count += 1 + return call_count < 3 + + await run_interruptible_loop_async( + loop_body, + POLL_INTERVAL_SECONDS, + show_controls=False, + check_interval_seconds=CHECK_INTERVAL_SECONDS, + ) + assert call_count == 3 + + +@pytest.mark.asyncio +async def test_interruptible_polling_loop_decorator_returns_coroutine_and_runs_until_false(): + call_count = 0 + + @interruptible_polling_loop(show_controls=False) + def poll_step(): + nonlocal call_count + call_count += 1 + return call_count < 2 + + assert asyncio.iscoroutinefunction(poll_step) + await poll_step(poll_interval=POLL_INTERVAL_SECONDS) + assert call_count == 2 + + +def test_user_abort_error_is_runtime_error(): + error = UserAbortError("test message") + assert isinstance(error, RuntimeError) + assert str(error) == "test message" diff --git a/utils/api.py b/utils/api.py deleted file mode 100644 index d5ccb69d8..000000000 --- a/utils/api.py +++ /dev/null @@ -1,333 +0,0 @@ -import datetime -import json -import os -import urllib.request -from collections import Counter -from typing import List, Optional, Union - -from mat3ra.api_client import APIClient -from mat3ra.api_client.endpoints.bank_workflows import BankWorkflowEndpoints -from mat3ra.api_client.endpoints.jobs import JobEndpoints -from mat3ra.api_client.endpoints.properties import PropertiesEndpoints -from mat3ra.made.material import Material -from mat3ra.prode import PropertyName -from mat3ra.utils.extra.tabulate import pretty_print -from mat3ra.utils.jupyterlite.interrupts import interruptible_polling_loop -from mat3ra.wode import Workflow - -from .job_properties import get_fermi_energy_flowchart_id - - -def save_files(job_id: str, job_endpoint: JobEndpoints, filename_on_cloud: str, filename_on_disk: str) -> None: - """ - Saves a file to disk, overwriting any files with the same name as filename_on_disk - - Args: - job_id (str): ID of the job - job_endpoint (JobEndpoints): Job endpoint object from the Exabyte API Client - filename_on_cloud (str): Name of the file on the server - filename_on_disk (str): Name the file will be saved to - - Returns: - None - """ - files = job_endpoint.list_files(job_id) - for file in files: - if filename_on_cloud in file["key"]: - file_metadata = file - - # Get a download URL for the CONTCAR - signed_url = file_metadata["signedUrl"] - - # Download the contcar to memory - server_response = urllib.request.urlopen(signed_url) - - # Write it to disk - with open(filename_on_disk, "wb") as outp: - outp.write(server_response.read()) - - -def get_jobs_statuses_by_ids(endpoint: JobEndpoints, job_ids: List[str]) -> List[str]: - """ - Gets jobs statues by their IDs. - - Args: - endpoint (JobEndpoints): Job endpoint object from the Exabyte API Client - job_ids (list): list of job IDs to get the status for - - Returns: - list: list of job statuses - """ - jobs = endpoint.list({"_id": {"$in": job_ids}}, {"fields": {"status": 1}}) - return [job["status"] for job in jobs] - - -@interruptible_polling_loop() -def wait_for_jobs_to_finish_async(endpoint: JobEndpoints, job_ids: List[str]) -> bool: - """ - Waits for jobs to finish and prints their statuses. - A job is considered finished if it is not in "pre-submission", "submitted", or, "active" status. - - Args: - endpoint (JobEndpoints): Job endpoint object from the Exabyte API Client - job_ids (list): list of job IDs to wait for - """ - statuses = get_jobs_statuses_by_ids(endpoint, job_ids) - counts = Counter(statuses) - headers = ["TIME", "SUBMITTED-JOBS", "ACTIVE-JOBS", "FINISHED-JOBS", "ERRORED-JOBS"] - now = datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S") - row = [ - now, - counts.get("submitted", 0) + counts.get("queued", 0), - counts.get("active", 0), - counts.get("finished", 0), - counts.get("error", 0), - ] - pretty_print([row], headers, tablefmt="grid", stralign="center") - - active_statuses = {"pre-submission", "submitted", "queued", "active"} - return not statuses or any(status in active_statuses for status in statuses) - - -def copy_bank_workflow_by_system_name(endpoint: BankWorkflowEndpoints, system_name: str, account_id: str) -> dict: - """ - Copies a bank workflow with given ID into the account's workflows. - - Args: - endpoint (endpoints.bank_workflows.BankWorkflowEndpoints): an instance of BankWorkflowEndpoints class - system_name (str): workflow system name. - account_id (str): ID of account to copy the bank workflow into. - - Returns: - dict: new account's workflow - """ - bank_workflow_id = endpoint.list({"systemName": system_name})[0]["_id"] - return endpoint.copy(bank_workflow_id, account_id)["_id"] - - -def get_property_by_subworkflow_and_unit_indicies( - endpoint: PropertiesEndpoints, property_name: str, job: dict, subworkflow_index: int, unit_index: int -) -> dict: - """ - Returns the property extracted in the given unit of the job's subworkflow. - - Args: - endpoint (endpoints.properties.PropertiesEndpoints): an instance of PropertiesEndpoints class. - property_name (str): name of property to extract. - job (dict): job config to extract the property from. - subworkflow_index (int): index of subworkflow to extract the property from. - unit_index (int): index of unit to extract the property from. - - Returns: - dict: extracted property - """ - unit_flowchart_id = job["workflow"]["subworkflows"][subworkflow_index]["units"][unit_index]["flowchartId"] - return endpoint.get_property(job["_id"], unit_flowchart_id, property_name) - - -def get_cluster_name(name: str = "cluster-001") -> str: - clusters = json.loads(os.environ.get("CLUSTERS", "[]") or "[]") - return clusters[0] if clusters else name - - -def get_or_create_material(api_client: APIClient, material, owner_id: str) -> dict: - """ - Returns an existing material from the collection if one with the same structural hash - exists under the given owner, otherwise creates a new one. - Uses the client-side hash (mat3ra-made Material.hash) to avoid unnecessary DB writes. - - Args: - api_client (APIClient): API client instance carrying the authorization context. - material: mat3ra-made Material object (must have a .hash property). - owner_id (str): Account ID under which to search and create. - - Returns: - dict: The material dict (existing or newly created). - """ - existing = api_client.materials.list({"hash": material.hash, "owner._id": owner_id}) - if existing: - print(f"♻️ Reusing already existing Material: {existing[0]['_id']}") - return existing[0] - created = api_client.materials.create(material.to_dict(), owner_id=owner_id) - print(f"✅ Material created: {created['_id']}") - return created - - -def get_or_create_workflow(api_client: APIClient, workflow: Workflow, owner_id: str) -> dict: - """ - Creates a workflow in the collection if none with the same hash exists under the given owner. - Unit-level context (important settings) is stripped before saving so the base workflow - stays clean and reusable. - - Args: - api_client (APIClient): API client instance carrying the authorization context. - workflow: mat3ra-wode Workflow object. - owner_id (str): Account ID under which to search and create. - - Returns: - dict: The workflow dict (existing or newly created). - """ - existing = api_client.workflows.list({"hash": workflow.hash, "owner._id": owner_id}) - if existing: - print(f"♻️ Reusing already existing Workflow: {existing[0]['_id']}") - return existing[0] - created = api_client.workflows.create(workflow.to_dict_without_special_keys(), owner_id=owner_id) - print(f"✅ Workflow created: {created['_id']}") - return created - - -FERMI_ENERGY_PROPERTIES = { - PropertyName.non_scalar.band_structure.value, - PropertyName.non_scalar.density_of_states.value, -} - - -def get_properties_for_job(client: APIClient, job_id: str, property_name: Optional[str] = None) -> List[dict]: - """ - Fetch properties for a job, automatically enriching band_structure/DOS results with fermiEnergy. - Use instead of client.properties.get_for_job when passing results to visualize_properties. - """ - job = client.jobs.get(job_id) - properties = client.properties.get_for_job(job_id, property_name) - if property_name not in FERMI_ENERGY_PROPERTIES: - return properties - flowchart_id = get_fermi_energy_flowchart_id(job) - fermi_energy = None - if flowchart_id: - fe_props = client.properties.get_for_job(job_id, PropertyName.scalar.fermi_energy.value, flowchart_id) - if fe_props: - fermi_energy = fe_props[0].get("value") - return [{**prop, "fermiEnergy": fermi_energy} for prop in properties] - - -def get_property_holder_for_job( - client: APIClient, job_id: str, property_name: str, unit_id: Optional[str] = None -) -> dict: - """ - Fetch the first full property holder for a job/property pair. - - Args: - client (APIClient): API client instance. - job_id (str): Job ID. - property_name (str): Property name. - unit_id (str, optional): Unit flowchart ID. - - Returns: - dict: Full property holder document. - """ - query = { - "source.info.jobId": job_id, - "data.name": property_name, - } - if unit_id: - query["source.info.unitId"] = unit_id - holders = client.properties.list(query=query) - if not holders: - raise ValueError(f"Property '{property_name}' not found for job '{job_id}'") - return holders[0] - - -def update_property_holder_value(client: APIClient, property_holder_id: str, value: float) -> dict: - """ - Update a scalar property's data.value. - - Args: - client (APIClient): API client instance. - property_holder_id (str): Property holder ID. - value (float): New scalar value. - - Returns: - dict: Server response payload. - """ - return client.properties.update(property_holder_id, {"$set": {"data.value": value}}) - - -def create_job( - api_client: APIClient, - materials: List[Union[dict, Material]], - workflow: Union[dict, Workflow], - project_id: str, - owner_id: str, - prefix: str, - compute: Optional[dict] = None, -) -> Union[dict, List[dict]]: - """ - Creates jobs for each material using an embedded workflow with any context (important settings) - already applied. The workflow _id is stripped so the server uses the embedded dict as-is, - preserving unit-level context (kpath, kgrid, cutoffs, etc.) without saving them to the - workflow collection. - - Args: - api_client (APIClient): API client instance carrying the authorization context. - materials (list): List of material dicts or mat3ra-made Material objects. - workflow: Workflow dict or Workflow object with important settings already applied. - project_id (str): Project ID. - owner_id (str): Account ID. - prefix (str): Job name prefix. - compute (dict, optional): Compute configuration dict. - - Returns: - list[dict]: List of created job dicts. - """ - material_dicts = [] - for material in materials: - if isinstance(material, Material): - material_dicts.append(material.to_dict()) - else: - material_dicts.append(material) - - job_workflow_dict = workflow.to_dict() if isinstance(workflow, Workflow) else workflow - # Strip _id so the server uses the embedded workflow as-is instead of fetching from DB, - # which would discard any unit-level context (kpath, kgrid, cutoffs, etc.). - job_workflow_dict.pop("_id", None) - is_multimaterial = job_workflow_dict.get("isMultiMaterial", False) - - config = { - "_project": {"_id": project_id}, - "workflow": job_workflow_dict, - "owner": {"_id": owner_id}, - "name": prefix, - } - - if is_multimaterial: - # Some API environments still validate `_material._id` even for - # multi-material workflows, so provide the first material as a - # compatibility fallback while preserving the full ordered list. - config["_material"] = {"_id": material_dicts[0]["_id"]} - config["_materials"] = [{"_id": m["_id"]} for m in material_dicts] - else: - config["_material"] = {"_id": material_dicts[0]["_id"]} - - if compute: - config["compute"] = compute - return api_client.jobs.create(config) - - -def get_convergence_series(client: APIClient, job_id: str, subworkflow_index: int = 0) -> List[dict]: - """ - Returns the convergence series from a finished convergence job. - - Args: - client: API client instance. - job_id: ID of the finished convergence job. - subworkflow_index: Index of the convergence subworkflow (default 0). - - Returns: - List of dicts with keys "x", "parameter", "y". - """ - finished_job = client.jobs.get(job_id) - job_workflow = Workflow.create(finished_job["workflow"]) - subworkflow = job_workflow.subworkflows[subworkflow_index] - return subworkflow.convergence_series(finished_job.get("scopeTrack")) - - -def submit_jobs(endpoint: JobEndpoints, job_ids: List[str]) -> None: - """ - Submits jobs by IDs. - - Args: - endpoint (JobEndpoints): Job endpoint object from the Exabyte API Client. - job_ids (list[str]): Job IDs to submit. - """ - for job_id in job_ids: - endpoint.submit(job_id) diff --git a/utils/generic.py b/utils/generic.py deleted file mode 100644 index d9c86f17a..000000000 --- a/utils/generic.py +++ /dev/null @@ -1,40 +0,0 @@ -import json -import os -from types import SimpleNamespace - - -def update_json_file_kwargs(path_to_json_file: str = "settings.json", **kwargs) -> None: - """ - This function updates settings.json for a given kwargs if kwargs - contains variables different from those already in json - - Args: - path_to_json_file (str): the path to the json file to be updated - **kwargs (dict): A dict of keyword arguments - - Returns: - None - """ - - # 1. Assert the json file is where we think it is - assert os.path.isfile(path_to_json_file) - - # 2. Load settings.json - with open(path_to_json_file) as settings_json_file: - variables = json.load(settings_json_file) - - # 3. Update json file if kwargs contains new variables - if kwargs != variables: - updated_variables = {**variables, **kwargs} - with open(path_to_json_file, "w") as settings_json_file: - json.dump(updated_variables, settings_json_file, indent=4) - - -# Helper function to convert dictionaries to SimpleNamespace objects for dot notation access -def dict_to_namespace(obj): - if isinstance(obj, dict): - return SimpleNamespace(**{k: dict_to_namespace(v) for k, v in obj.items()}) - elif isinstance(obj, list): - return [dict_to_namespace(item) for item in obj] - else: - return obj diff --git a/utils/jupyterlite.py b/utils/jupyterlite.py deleted file mode 100644 index bf5c8949c..000000000 --- a/utils/jupyterlite.py +++ /dev/null @@ -1,335 +0,0 @@ -import inspect -import json -import os -from typing import Any, Dict, List, Optional - -from IPython.display import Javascript, display -from mat3ra.utils.jupyterlite.environment import ENVIRONMENT, EnvironmentsEnum -from mat3ra.utils.jupyterlite.logger import SeverityLevelEnum, log -from mat3ra.utils.jupyterlite.settings import UPLOADS_FOLDER - - -def set_data_pyodide(key: str, value: Any): - """ - Take a Python object, serialize it to JSON, and send it to the host environment - through a JavaScript function defined in the JupyterLite extension `data_bridge`. - - Args: - key (str): The name under which data will be sent. - value (Any): The value to send to the host environment. - """ - serialized_data = json.dumps({key: value}) - js_code = f""" - (function() {{ - if (window.sendDataToHost) {{ - window.sendDataToHost({serialized_data}); - console.log('Data sent to host:', {serialized_data}); - }} else {{ - console.error('sendDataToHost function is not defined on the window object.'); - }} - }})(); - """ - display(Javascript(js_code)) - log(f"Data for {key} sent to host.") - set_data_python(key, value) - - -def set_data_python(key: str, value: Any): - """ - Write data to the `uploads` folder in a JupyterLab environment. - - Args: - key (str): The name under which data will be written. - value (Any): The value to write to the `uploads` folder. - """ - if not os.path.exists(UPLOADS_FOLDER): - os.makedirs(UPLOADS_FOLDER) - for item in value: - safe_name = item["name"].replace("%", "pct").replace("/", ":") - file_path = os.path.join(UPLOADS_FOLDER, f"{safe_name}.json") - with open(file_path, "w") as file: - json.dump(item, file) - log(f"Data for {key} written to {file_path}") - - -def set_data(key: str, value: Any): - """ - Switch between the two functions `set_data_pyodide` and `set_data_python` based on the environment. - - Args: - key (str): The name under which data will be written or sent. - value (Any): The value to write or send. - """ - if ENVIRONMENT == EnvironmentsEnum.PYODIDE: - set_data_pyodide(key, value) - elif ENVIRONMENT == EnvironmentsEnum.PYTHON: - set_data_python(key, value) - - -def get_data_pyodide(key: str, globals_dict: Optional[Dict] = None): - """ - Load data from the host environment into globals()[key] variable. - - Args: - key (str): Global variable name to store the received data. - globals_dict (dict, optional): globals() dictionary of the current scope. - """ - if globals_dict is not None: - globals_dict[key] = globals_dict.get("data_from_host", None) - - -def get_data_python(key: str, globals_dict: Optional[Dict] = None): - """ - Read data from the `uploads` folder in a JupyterLab environment. - - Args: - key (str): The name under which data is expected to be received. - globals_dict (dict, optional): A dictionary to store the received data. Defaults to None. - """ - try: - data_from_host = [] - index = 0 - for filename in sorted(os.listdir(UPLOADS_FOLDER)): - if filename.endswith(".json"): - with open(os.path.join(UPLOADS_FOLDER, filename), "r") as file: - data = json.load(file) - name = os.path.splitext(filename)[0] - log(f"{index}: {name}") - index += 1 - data_from_host.append(data) - if globals_dict is not None: - globals_dict[key] = data_from_host - return data_from_host - except FileNotFoundError: - print("No data found in the 'uploads' folder.") - - -def get_data(key: str, globals_dict: Optional[Dict] = None): - """ - Switch between the two functions `get_data_pyodide` and `get_data_python` based on the environment. - - Args: - key (str): The name under which data is expected to be received. - globals_dict (dict, optional): A dictionary to store the received data. Defaults to None. - """ - if ENVIRONMENT == EnvironmentsEnum.PYODIDE: - get_data_pyodide(key, globals_dict) - elif ENVIRONMENT == EnvironmentsEnum.PYTHON: - get_data_python(key, globals_dict) - - -def get_materials(globals_dict: Optional[Dict] = None) -> List[Any]: - """ - Retrieve materials from the environment and assign them to globals_dict["materials_in"]. - - Args: - globals_dict (dict, optional): The globals dictionary to populate. - - Returns: - List[Material]: A list of Material objects. - """ - from mat3ra.made.material import Material - from mat3ra.made.tools.build_components import MaterialWithBuildMetadata - - if globals_dict is None: - # Get the globals of the caller for correct variable assignment during the execution of data_bridge extension - frame = inspect.currentframe() - try: - caller_frame = frame.f_back # type: ignore - caller_globals = caller_frame.f_globals # type: ignore - globals_dict = caller_globals - finally: - del frame # Avoid reference cycles - get_data("materials_in", globals_dict) - - if "materials_in" in globals_dict and globals_dict["materials_in"]: - materials = [] - for item in globals_dict["materials_in"]: - try: - materials.append(MaterialWithBuildMetadata.create(item)) - except Exception: - materials.append(Material.create(item)) - log(f"Retrieved {len(materials)} materials.") - return materials - else: - # Fallback to load materials from the UPLOADS_FOLDER if launched outside of Materials Designer - log(f"No input materials found. Loading from the {UPLOADS_FOLDER} folder.") - return load_materials_from_folder() - - -def set_materials(materials: List[Any]): - """ - Serialize and send a list of Material objects to the environment. - - Args: - materials (List[Material]): The list of Material objects to send. - """ - from mat3ra.utils.array import convert_to_array_if_not - - materials = convert_to_array_if_not(materials) - materials_data = [json.loads(material.to_json()) for material in materials] - set_data("materials", materials_data) - - -def load_materials_from_folder(folder_path: Optional[str] = None, verbose: bool = True) -> List[Any]: - """ - Load materials from the specified folder or from the UPLOADS_FOLDER by default. - - Args: - folder_path (Optional[str]): The path to the folder containing material files. - If not provided, defaults to the UPLOADS_FOLDER. - verbose (bool): Whether to log verbose messages. - - Returns: - List[Material]: A list of Material objects loaded from the folder. - """ - from mat3ra.made.material import Material - from mat3ra.made.tools.build_components import MaterialWithBuildMetadata - - folder_path = folder_path or UPLOADS_FOLDER - - if not os.path.exists(folder_path): - log(f"Folder '{folder_path}' does not exist.", SeverityLevelEnum.ERROR, force_verbose=verbose) - return [] - - data_from_host = [] - try: - index = 0 - for filename in sorted(os.listdir(folder_path)): - if filename.endswith(".json"): - file_path = os.path.join(folder_path, filename) - try: - with open(file_path, "r") as file: - data = json.load(file) - except (json.JSONDecodeError, OSError) as error: - log( - f"Skipping invalid JSON file '{file_path}': {error}", - SeverityLevelEnum.WARNING, - force_verbose=verbose, - ) - continue - name = os.path.splitext(filename)[0] - log(f"{index}: {name}", SeverityLevelEnum.INFO, force_verbose=verbose) - index += 1 - data_from_host.append(data) - except FileNotFoundError: - log(f"No data found in the '{folder_path}' folder.", SeverityLevelEnum.ERROR, force_verbose=verbose) - return [] - try: - materials = [MaterialWithBuildMetadata.create(item) for item in data_from_host] - except Exception: - materials = [Material.create(item) for item in data_from_host] - - if materials: - log( - f"Successfully loaded {len(materials)} materials from folder '{folder_path}'", - SeverityLevelEnum.INFO, - force_verbose=verbose, - ) - else: - log(f"No materials found in folder '{folder_path}'", SeverityLevelEnum.WARNING, force_verbose=verbose) - - return materials - - -def load_material_from_folder(folder_path: str, name: str, verbose: bool = True) -> Optional[Any]: - """ - Load a single material from the specified folder by matching a substring of the name or filename. - - Args: - folder_path (str): The path to the folder containing material files. - name (str): The substring to match against material names or filenames (case-insensitive). - verbose (bool): Whether to log verbose messages. - - Returns: - Optional[Material]: The first Material object that matches, or None if not found. - """ - from mat3ra.made.material import Material - from mat3ra.made.tools.build_components import MaterialWithBuildMetadata - - name_lower = name.lower() - resulting_material = None - - for filename in sorted(os.listdir(folder_path)): - if filename.endswith(".json") and name_lower in os.path.splitext(filename)[0].lower(): - with open(os.path.join(folder_path, filename), "r") as file: - data = json.load(file) - try: - resulting_material = MaterialWithBuildMetadata.create(data) - except Exception: - resulting_material = Material.create(data) - break - - if not resulting_material: - materials = load_materials_from_folder(folder_path, verbose=verbose) - for material in materials: - if name_lower in material.name.lower(): - resulting_material = material - break - - if resulting_material: - log(f"Found: '{resulting_material.name}'", SeverityLevelEnum.INFO, force_verbose=verbose) - return resulting_material - - log(f"No material containing '{name}' found in '{folder_path}'.", SeverityLevelEnum.WARNING, force_verbose=verbose) - return None - - -def write_materials_to_folder(materials: List[Any], folder_path: Optional[str] = None, verbose: bool = True): - """ - Write materials to the specified folder or to the UPLOADS_FOLDER by default. - - Args: - materials (List[Material]): The list of Material objects to write to the folder. - folder_path (Optional[str]): The path to the folder where the materials will be written. - If not provided, defaults to the UPLOADS_FOLDER. - verbose (bool): Whether to log verbose messages. - """ - from mat3ra.utils.array import convert_to_array_if_not - - folder_path = folder_path or UPLOADS_FOLDER - materials = convert_to_array_if_not(materials) - - if not os.path.exists(folder_path): - os.makedirs(folder_path) - if verbose: - log(f"Created folder '{folder_path}'.", SeverityLevelEnum.INFO) - - for material in materials: - safe_name = material.name.replace("%", "pct").replace("/", ":") - file_path = os.path.join(folder_path, f"{safe_name}.json") - with open(file_path, "w") as file: - json.dump(material.to_dict(), file) - log(f"Material '{material.name}' written to '{file_path}'", SeverityLevelEnum.INFO, force_verbose=verbose) - - -def download_content_to_file(content: Any, filename: str): - """ - Download content to a file with the given filename. - - Args: - content (Any): The content to download. - filename (str): The name of the file to download. - """ - from mat3ra.made.material import Material - from mat3ra.made.tools.build_components import MaterialWithBuildMetadata - - if isinstance(content, dict): - content = json.dumps(content, indent=4) - - if isinstance(content, (Material, MaterialWithBuildMetadata)): - content = content.to_json() - content = json.dumps(content, indent=4) - - js_code = f""" - var content = `{content}`; - var filename = `{filename}`; - var blob = new Blob([content], {{ type: 'application/json' }}); - var link = document.createElement('a'); - link.href = window.URL.createObjectURL(blob); - link.download = filename; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - """ - display(Javascript(js_code)) diff --git a/utils/visualize.py b/utils/visualize.py deleted file mode 100644 index 5d791e944..000000000 --- a/utils/visualize.py +++ /dev/null @@ -1,430 +0,0 @@ -import io -import json -import os -import time -import uuid -from enum import Enum -from typing import Dict, List, Optional, Tuple, Union - -import ipywidgets as widgets -from ase.build import make_supercell -from ase.io import write -from IPython.display import HTML, Javascript, display -from mat3ra.made.material import Material -from mat3ra.made.tools.convert import to_ase -from mat3ra.utils.array import convert_to_array_if_not -from pandas import DataFrame -from pandas.io.formats.style import Styler -from pydantic import BaseModel - -from utils import settings - - -class ViewersEnum(str, Enum): - wave = "wave" - ase = "ase" - - -def dataframe_to_html(df: DataFrame, text_align: str = "center") -> Styler: - """ - Converts Pandas dataframe to HTML. - See https://pandas.pydata.org/pandas-docs/stable/style.html for more information about styling. - - Args: - df (pd.DataFrame): Pandas dataframe. - text_align (str): text align. Defaults to center. - """ - styles = [ - dict(selector="th", props=[("text-align", text_align)]), - dict(selector="td", props=[("text-align", text_align)]), - ] - return df.style.set_table_styles(styles) - - -def display_JSON( - obj: Union[dict, list], interactive_viewer: bool = settings.use_interactive_JSON_viewer, level: int = 2 -) -> None: - """ - Displays JSON, either interactively or via a text dump to Stdout. - - The interactive viewer is based on https://github.com/mljar/mercury/blob/main/mercury/widgets/json.py. - - Args: - obj (dict): Object to display as nicely-formatted JSON - interactive_viewer (bool): Whether to use the interactive viewer or not - level (int): The level to which the JSON should be expanded by default - """ - if interactive_viewer: - if isinstance(obj, (dict, list)): - json_str = json.dumps(obj) - else: - json_str = obj - - id = str(uuid.uuid4()) - - web_dir = os.path.join(os.path.dirname(__file__), "web") - - with open(os.path.join(web_dir, "renderjson.css")) as fp: - css = fp.read() - - with open(os.path.join(web_dir, "renderjson.js")) as fp: - js = fp.read() - - display(HTML(f'
')) - display( - HTML( - f"' - ) - ) - else: - print(json.dumps(obj, indent=4)) - - -def get_material_image(material: Material, title: str, rotation="0x,0y,0z", repetitions=[1, 1, 1]): - """ - Returns an image of the material structure with the specified title. - - Args: - material (Material): Material object to visualize. - title (str): Title of the image. - rotation (str): Rotation of the image, in degrees around the x, y, and z axes (e.g., "-90x,90y,0z"). - repetitions (list): Repetitions alongside a,b,c lattice vectors. - - Returns: - tuple: Tuple containing the image and the title. - """ - - ase_atoms = to_ase(material) - # Create supercell for visualization - supercell_matrix = [[repetitions[0], 0, 0], [0, repetitions[1], 0], [0, 0, repetitions[2]]] - material_repeat = make_supercell(ase_atoms, supercell_matrix) - text = f"{ase_atoms.get_chemical_formula()} - {title} - rotation: {rotation}" - - # Write image to a buffer to display in HTML - buf = io.BytesIO() - write(buf, material_repeat, format="png", rotation=rotation) - buf.seek(0) - return buf.read(), text - - -def create_image_widget(image_data, format="png", object_fit="contain"): - """ - Creates an Image widget with specified layout settings. - - Args: - image_data (bytes): The image data to be displayed. - format (str): The format of the image, default is 'png'. - object_fit (str): CSS object-fit property value, default is 'contain'. - - Returns: - widgets.Image: A configured image widget. - """ - image = widgets.Image(value=image_data, format=format) - image.layout.object_fit = object_fit - return image - - -def create_responsive_image_grid(image_tuples, max_columns=3): - """ - Create a responsive image grid that can display images from a specified folder. - Ensures images are displayed at their true sizes and limits the grid to a maximum of three columns. - - Args: - image_tuples (list): List of tuples where each tuple contains an image and a title. - max_columns (int): Maximum number of columns in the grid. - """ - items = [ - widgets.VBox( - [ - widgets.Label(value=title, layout=widgets.Layout(height="30px", align_self="center")), - create_image_widget(image, object_fit="contain"), - ], - layout=widgets.Layout(align_items="center", padding="0px 0px 10px 0px"), - ) - for image, title in image_tuples - ] - - column_width = f"minmax(100px, {100 / max_columns}%)" - grid = widgets.GridBox( - items, - layout=widgets.Layout( - grid_template_columns=f"repeat({max_columns}, {column_width})", - grid_gap="10px", - width="100%", - ), - ) - return grid - - -class MaterialViewProperties(BaseModel): - repetitions: List[int] = [1, 1, 1] - rotation: str = "0x,0y,0z" - title: str = "Material" - - -def get_viewer_html(div_id, width, height=None, title="Viewer", custom_styles=""): - """ - Generate HTML container for a viewer. - - Args: - div_id: Unique ID for the container div - width: Width in pixels - height: Height in pixels (optional, if not provided only width is set) - title: Title to display above the viewer - custom_styles: Additional inline CSS styles - """ - if height is not None: - size_style = f"width:{width}px; height:{height}px;" - else: - size_style = f"width:{width}px;" - - return f""" -

{title}

-
- """ - - -def get_viewer_js( - data_json, - div_id, - bundle_url, - render_function, - data_var_name="data", - extra_config_json=None, - css_url=None, -): - """ - Generate JavaScript to load and render a viewer bundle. - - Args: - data_json: JSON string of data to render - div_id: Container div ID - bundle_url: URL to the JS bundle - render_function: Name of the window function to call (e.g., 'renderThreeDEditor', 'renderResults') - data_var_name: Variable name for the data (e.g., 'materialConfig', 'results') - extra_config_json: Optional extra config as JSON string - css_url: Optional CSS file URL to load - """ - extra_config_arg = f", {extra_config_json}" if extra_config_json else "" - css_loader = ( - f""" - document.head.insertAdjacentHTML( - 'beforeend', - ''); - """ - if css_url - else "" - ) - - return f""" - const {data_var_name}={data_json}; - const container = document.getElementById('{div_id}'); - (async function() {{ - await import('{bundle_url}'); - window.{render_function}({data_var_name}, container{extra_config_arg}); - }})(); - {css_loader} - """ - - -def get_wave_viewer(material, div_id, width, height, title): - size = min(width, height) - html = get_viewer_html( - div_id=div_id, - width=size, - height=size, - title=title, - custom_styles="border:1px solid #333;", - ) - js = get_viewer_js( - data_json=material.to_json(), - div_id=div_id, - bundle_url="https://exabyte-io.github.io/wave.js/main.js", - render_function="renderThreeDEditor", - data_var_name="materialConfig", - css_url="https://exabyte-io.github.io/wave.js/main.css", - ) - return html, js - - -def render_wave(material, properties, width=600, height=600): - timestamp = time.time() - div_id = f"wave-{timestamp}" - html, js = get_wave_viewer(material, div_id, width, height, properties.title) - display(HTML(html)) - display(Javascript(js)) - - -def render_wave_grid( - materials: List[Material], - list_of_properties_configs: List[MaterialViewProperties], - width=400, - height=400, - max_columns=3, -): - html_items = [] - js_items = [] - timestamp = time.time() - - for i, material in enumerate(materials): - properties_config = list_of_properties_configs[i] - div_id = f"wave-{i}-{timestamp}" - html, js = get_wave_viewer(material, div_id, width, height, properties_config.title) - html_items.append(widgets.HTML(html)) - js_items.append(Javascript(js)) - - grid = widgets.GridBox( - html_items, - layout=widgets.Layout( - grid_template_columns=f"repeat({max_columns}, 1fr)", - grid_gap="10px", - width="100%", - ), - ) - display(grid) - for js in js_items: - display(js) - - -def process_material_entry( - material_entry: Union[Material, Dict], default_properties: MaterialViewProperties -) -> Tuple[Material, MaterialViewProperties]: - """ - Process the material entry and return the material and properties. - Args: - material_entry: Material or a dictionary containing the material and properties. - default_properties: Default properties to use if not specified in the material entry. - - Returns: - Tuple[Material, MaterialViewProperties]: Material and properties. - - """ - if isinstance(material_entry, Material): - material = material_entry - properties = default_properties - elif isinstance(material_entry, dict) and "material" in material_entry: - material = material_entry["material"] - properties = MaterialViewProperties( - title=material_entry.get("title", default_properties.title), - repetitions=material_entry.get("repetitions", default_properties.repetitions), - rotation=material_entry.get("rotation", default_properties.rotation), - ) - else: - raise ValueError("Invalid material entry") - return material, properties - - -def visualize_materials( - materials: Union[List[Material], List[Dict[str, Union[Material, dict]]]], - repetitions: Optional[List[int]] = [1, 1, 1], - rotation: Optional[str] = "0x,0y,0z", - title: Optional[str] = "Material", - viewer: ViewersEnum = ViewersEnum.ase, -) -> None: - """ - Visualize the material(s) in the output cell. - Args: - materials: Mist of Materials or a list of dictionaries: - {"material": Material, "title": str,"repetitions": List[int], "rotation": str}. - repetitions (Optional[List[int]]): Repetitions alongside a, b, c lattice vectors. - rotation (Optional[str]): Rotation of the image, in degrees around the x, y, and z axes (e.g., "-90x,90y,0z"). - title (Optional[str]): Title of the image. - viewer (ViewersEnum): Viewer to use for visualization. "ase" or "wave". - - Returns: - None - """ - materials = convert_to_array_if_not(materials) - - if not materials: - print("No materials to visualize.") - return - - default_properties = MaterialViewProperties( - title=title if title is not None else MaterialViewProperties.title, - repetitions=repetitions if repetitions is not None else MaterialViewProperties.repetitions, - rotation=rotation if rotation is not None else MaterialViewProperties.rotation, - ) - - if viewer == ViewersEnum.wave: - wave_materials = [] - wave_properties_list = [] - for material_entry in materials: - material, material_properties = process_material_entry(material_entry, default_properties) - wave_materials.append(material) - wave_properties_list.append(material_properties) - if len(wave_materials) == 1: - # Render single material in the wave viewer, larger size and hotkeys working - render_wave(wave_materials[0], properties=wave_properties_list[0]) - else: - render_wave_grid(materials=wave_materials, list_of_properties_configs=wave_properties_list) - - else: - items = [] - for material_entry in materials: - material, properties = process_material_entry(material_entry, default_properties) - if material: - image_data, image_title = get_material_image( - material, title=properties.title, rotation=properties.rotation, repetitions=properties.repetitions - ) - items.append((image_data, image_title)) - - display(create_responsive_image_grid(items)) - - -def visualize_workflow(workflow, level: int = 2) -> None: - """ - Visualize a workflow by displaying its JSON configuration. - - Args: - workflow: Workflow object with a to_dict() method - level: Expansion level for the JSON viewer (default: 2) - - Returns: - None - """ - workflow_config = workflow.to_dict() - display_JSON(workflow_config, level=level) - - -def visualize_properties(results, width=900, title="Properties", extra_config=None): - """ - Visualize properties using a Prove viewer. - - Args: - results: List[dict] of property JSON objects (or a single dict). - width: Container width in pixels. - title: Title displayed above the viewer. - extra_config: Optional dict with materials, components, callbacks, etc. - """ - if isinstance(results, dict): - results = [results] - - DATA_KEYS = {"value", "values", "xDataArray"} - results = [r for r in results if DATA_KEYS & r.keys()] - - timestamp = time.time() - div_id = f"prove-{timestamp}" - results_json = json.dumps(results) - extra_config_json = json.dumps(extra_config) if extra_config else "undefined" - - html = get_viewer_html( - div_id=div_id, - width=width, - title=title, - custom_styles="border:1px solid #ddd; padding:12px; background:#fff; color:#111;", - ) - js = get_viewer_js( - data_json=results_json, - div_id=div_id, - bundle_url="https://exabyte-io.github.io/prove/main.js", - render_function="renderResults", - data_var_name="results", - extra_config_json=extra_config_json, - ) - - display(HTML(html)) - display(Javascript(js))