feature/SOF 7894 Feature: create Notebooks Utils

.github/workflows/cicd.yml

@@ -28,8 +28,38 @@ jobs:
           python -m pip install pre-commit
           pre-commit run --all-files --show-diff-on-failure
 
+  run-py-tests:
+    needs: run-linter
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        python-version:
+          - 3.10.x
+          - 3.11.x
+          - 3.12.x
+
+    steps:
+      - name: Checkout this repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      - name: Checkout actions repository
+        uses: actions/checkout@v4
+        with:
+          repository: Exabyte-io/actions
+          token: ${{ secrets.BOT_GITHUB_TOKEN }}
+          path: actions
+
+      - name: Run python unit tests
+        uses: ./actions/py/pytest
+        with:
+          python-version: ${{ matrix.python-version }}
+          unit-test-directory: tests/py/unit
+          bot-ssh-key: ${{ secrets.BOT_GITHUB_KEY }}
+
   publish-py-package:
-    needs: [run-linter]
+    needs: [run-linter, run-py-tests]
     runs-on: ubuntu-latest
     if: github.ref_name == 'main'
 

pyproject.toml

@@ -1,38 +1,67 @@
 [project]
-name = "mat3ra-api-examples"
+name = "mat3ra-notebooks-utils"
 dynamic = ["version"]
-description = "Mat3ra API Examples"
+description = "Mat3ra notebooks utilities."
 readme = "README.md"
 requires-python = ">=3.10"
-dependencies = [
+dependencies = []
+
+[project.optional-dependencies]
+auxilary = [
+  "pydantic",
+  "matplotlib>=3.4.1",
+  "pandas>=1.5.3",
+]
+jupyterlite = [
+  "pyyaml",
+  "matplotlib>=3.4.1",
+  "plotly>=5.18",
+  "ipython>=8.0",
+]
+utils_standata = [
+  "mat3ra-utils",
+  "mat3ra-standata",
+]
+api = [
+  "mat3ra-api-client",
+]
+materials = [
   # ase >=3.25.0 is required for supercell to be generated,
   # otherwise on 3.21.1 we encountered negative number of atoms during supercell generation
+  "mat3ra-notebooks-utils[auxilary]",
+  "mat3ra-notebooks-utils[utils_standata]",
   "ase>=3.25.0",
-  "matplotlib>=3.4.1",
-  "pandas>=1.5.3",
   "pymatgen==2024.4.13",
   "mat3ra-made>=2026.4.2.post0",
-  "mat3ra-utils>=2026.3.6.post0",
+]
+workflows = [
+  "mat3ra-notebooks-utils[materials]",
   "mat3ra-wode",
   "mat3ra-prode",
   "mat3ra-ide",
-  "mat3ra-api-client",
-  "mat3ra-standata"
+  "mar3ra-notebooks-utils[api]",
 ]
 
-[project.optional-dependencies]
 # Install all above dependencies in colab
-colab = ["mat3ra-api-examples"]
+colab = ["mat3ra-notebooks-utils"]
+
+# ToDo: figure out necessary packages
 jupyterlab = [
   "jupyterlab>=3.0.17",
   "nbconvert>=6.0.7",
 ]
 # Install colab + jupyterlab on localhost
-localhost = ["mat3ra-api-examples[jupyterlab]"]
+localhost = ["mat3ra-notebooks-utils[jupyterlab]"]
 dev = [
   "pre-commit>=3.3.3",
   "pip-tools>=6.13.0",
 ]
+tests = [
+  "pydantic",
+  "pytest",
+  "pytest-asyncio",
+  "pytest-cov",
+]
 docs = [
   "mkdocs>=1.4.3",
   "mkdocs-material>=9.1.17",
@@ -64,18 +93,18 @@ build-backend = "setuptools.build_meta"
 git_describe_command = "git describe --tags --long"
 
 [tool.setuptools.packages.find]
-include = ["utils"]
+where = ["src/py"]
 
 [tool.setuptools.package-data]
-utils = [
-  "settings.json",
-  "web/renderjson.*",
+"mat3ra.notebooks_utils" = [
+  "core/api/settings.json",
+  "ipython/web/renderjson.*",
 ]
 
 
 [tool.black]
 line-length = 120
-target-version = ['py38']
+target-version = ['py310']
 # 'extend-exclude' excludes files or directories in addition to the defaults
 extend-exclude = '''
 (
@@ -98,6 +127,10 @@ profile = "black"
 multi_line_output = 3
 include_trailing_comma = true
 
+[tool.mypy]
+python_version = "3.10"
+explicit_package_bases = true
+
 [[tool.mypy.overrides]]
 module = "yaml"
 ignore_missing_imports = true

src/py/mat3ra/notebooks_utils/README.md

@@ -0,0 +1,94 @@
+# mat3ra.notebooks_utils — Developer Guide
+
+## Architecture
+
+Four progressive layers, each building on the previous:
+
+```
+primitive/  →  core/  →  ipython/  →  pyodide/
+```
+
+Top-level files (`auth.py`, `io.py`, `ui.py`, `plot.py`, `settings.py`, `material.py`) are thin
+routing/re-export adapters that keep notebook code environment-agnostic.
+
+---
+
+## Layers
+
+### `primitive/`
+Only Python stdlib. No third-party packages, no domain knowledge.
+Enums, environment detection, logger, CLI prompt helpers.
+
+### `core/`
+Third-party packages allowed (`mat3ra.api_client`, `requests`, `numpy`, …).
+No IPython, no ipywidgets, no browser/pyodide APIs.
+
+```
+core/api/              — Platform credentials and OIDC auth.
+core/io.py             — Plain-Python file IO and HTTP.
+core/entity/<domain>/  — One folder per domain (material, workflow, job, compute, property).
+    api.py             — REST API calls only.
+    io.py              — Local filesystem reads/writes only.
+    analysis.py        — Pure computation (numpy/pymatgen/ase). No API, no display.
+    job.py             — Cross-entity helpers (e.g. property derived from job data).
+```
+
+### `ipython/`
+Imports `IPython.display`, `ipywidgets`, or generates HTML/JS for notebook output.
+Must work in JupyterLab, Colab, and VS Code notebooks — not browser/pyodide specific.
+
+```
+ipython/ui.py              — Generic cell output: display_JSON, image grid, viewer HTML/JS.
+ipython/io.py              — Browser file-download helpers.
+ipython/plot/              — Domain-agnostic plot primitives (_plotly.py, _matplotlib.py).
+ipython/entity/<domain>/   — Domain-specific display.
+    visualize.py           — Renders domain objects into notebook cells.
+    plot.py                — Domain-specific charts (RDF, strain, EOS, …).
+```
+
+### `pyodide/`
+Calls `micropip`, `pyodide.http.pyfetch`, `BroadcastChannel`, or `js` (Emscripten/WASM APIs).
+JupyterLite-specific overrides of `core/` or `ipython/` capabilities.
+
+```
+pyodide/io.py          — JS data bridge: kernel↔host shared state, file writes.
+pyodide/ui.py          — Async input widgets (PyodideFuture / BroadcastChannel).
+pyodide/runtime.py     — Interruptible loop and abort controller.
+pyodide/packages/      — micropip installation and config.yml parsing.
+pyodide/api/           — Host-to-kernel token injection.
+```
+
+---
+
+## Dependency Rules
+
+```
+primitive/  →  (nothing from this package)
+core/       →  primitive/
+ipython/    →  primitive/, core/
+pyodide/    →  primitive/, core/, ipython/
+top-level   →  any layer (routing only, no new logic)
+```
+
+Within `core/entity/`: domains must not import each other; `api.py`, `io.py`, and `analysis.py`
+must stay separate (no cross-imports within the same entity folder).
+
+Within `ipython/entity/`: may import from `core/entity/<same domain>/` and `ipython/` primitives,
+but not from other entity domains.
+
+---
+
+## Where does my code go?
+
+| Question | Destination |
+|---|---|
+| Uses only stdlib? | `primitive/` |
+| Calls a 3rd-party package, produces no output? | `core/` |
+| Reads/writes domain objects from disk? | `core/entity/<domain>/io.py` |
+| Calls the REST API? | `core/entity/<domain>/api.py` |
+| Computes/transforms domain data (no display)? | `core/entity/<domain>/analysis.py` |
+| Renders into a notebook cell (IPython/HTML)? | `ipython/` |
+| Domain-specific chart? | `ipython/entity/<domain>/plot.py` |
+| Domain-specific visualiser? | `ipython/entity/<domain>/visualize.py` |
+| Calls micropip / pyfetch / BroadcastChannel? | `pyodide/` |
+| Branches on environment to pick implementation? | top-level routing file |

src/py/mat3ra/notebooks_utils/__init__.py

@@ -0,0 +1,29 @@
+from .core.api.settings import (
+    ACCOUNT_ID,
+    AUTH_TOKEN,
+    ENDPOINT_ARGS,
+    HOST,
+    MATERIALS_PROJECT_API_KEY,
+    ORGANIZATION_ID,
+    PORT,
+    SECURE,
+    VERSION,
+    absolute_path_to_settings_json_file,
+    settings_json_config,
+)
+from .settings import UPLOADS_FOLDER
+
+__all__ = [
+    "absolute_path_to_settings_json_file",
+    "settings_json_config",
+    "ACCOUNT_ID",
+    "AUTH_TOKEN",
+    "MATERIALS_PROJECT_API_KEY",
+    "ORGANIZATION_ID",
+    "PORT",
+    "SECURE",
+    "VERSION",
+    "HOST",
+    "ENDPOINT_ARGS",
+    "UPLOADS_FOLDER",
+]

src/py/mat3ra/notebooks_utils/auth.py

@@ -0,0 +1,28 @@
+import inspect
+import os
+
+from mat3ra.api_client import ACCESS_TOKEN_ENV_VAR
+
+from .core.api.auth import authenticate_oidc
+from .io import get_data
+from .ipython.ui import show_device_flow_popup
+from .primitive.environment import ENVIRONMENT, EnvironmentsEnum
+from .pyodide.api.auth import authenticate_jupyterlite
+
+REFRESH_TOKEN_ENV_VAR = "OIDC_REFRESH_TOKEN"
+
+
+async def authenticate(force=False, globals_dict=None):
+    if globals_dict is None:
+        frame = inspect.currentframe()
+        try:
+            globals_dict = frame.f_back.f_globals  # type: ignore
+        finally:
+            del frame
+    if ENVIRONMENT == EnvironmentsEnum.PYODIDE:
+        get_data("data_from_host", globals_dict)
+    data_from_host = globals_dict.get("data_from_host")
+    if data_from_host:
+        await authenticate_jupyterlite(data_from_host)
+    elif ACCESS_TOKEN_ENV_VAR not in os.environ or force:
+        await authenticate_oidc(show_popup=show_device_flow_popup)