diff --git a/Workflows/pyiron_table_potential_scan.py b/Workflows/pyiron_table_potential_scan.py new file mode 100644 index 00000000..30364b75 --- /dev/null +++ b/Workflows/pyiron_table_potential_scan.py @@ -0,0 +1,53 @@ +""" +Reproduces the "Data mining using pyiron tables" notebook +(potential_scan project: equilibrium lattice parameter / bulk modulus per +potential, from the Murnaghan jobs) using BuildTable from +pyiron_nodes.databases.pyiron_tables - no pyiron_base / pyiron_atomistics +dependency, reads the project directory and job HDF5 files directly. +""" + +import sys + +from core import Workflow +from pyiron_nodes.databases.pyiron_tables import ( + AddBulkModulus, + AddLatticeParameter, + AddPotential, + BuildTable, + DbFilterFunction, +) + + +def make_workflow( + project_path: str = "/u/pchilaka/1_Work/1_My_Notebooks/1_Beginners_Guide/DONE/potential_scan", +) -> Workflow: + wf = Workflow("pyiron_table_potential_scan") + + wf.DbFilterFunction = DbFilterFunction(hamilton="Murnaghan") + + # each node takes the previous one's output back in as `functions` and + # grows the same dict, like AddPristine chains a StructureContainer - no + # separate merge node needed. Each node wraps one named function + # (get_bulk_modulus, get_potential, get_lattice_parameter) from + # pyiron_tables.py. + wf.BulkModulusFunction = AddBulkModulus() + wf.PotentialFunction = AddPotential(functions=wf.BulkModulusFunction) + wf.LatticeParameterFunction = AddLatticeParameter(functions=wf.PotentialFunction) + + wf.BuildTable = BuildTable( + project_path=project_path, + functions=wf.LatticeParameterFunction, + status=["finished"], + db_filter_function=wf.DbFilterFunction, + ) + return wf + + +# module-level graph so UIs that exec() this file (e.g. pyironflow) can find it +wf = make_workflow() + +if __name__ == "__main__": + if len(sys.argv) > 1: + wf = make_workflow(project_path=sys.argv[1]) + out = wf.run() + print(out) diff --git a/databases/pyiron_tables.md b/databases/pyiron_tables.md new file mode 100644 index 00000000..2ec420c5 --- /dev/null +++ b/databases/pyiron_tables.md @@ -0,0 +1,203 @@ +# pyiron_tables.py — how to add a new table property + +This document explains the pattern used in +`pyiron_nodes/databases/pyiron_tables.py` for turning a single job property +into a `pyiron_nodes` node that can be daisy-chained into `BuildTable`. It is +written so another LLM can add a new property node without re-deriving the +design from scratch. + +## What this module is + +A reimplementation of pyiron_base's job-table / `PyironTable` data mining +(`pyiron_base.database.filetable.FileTable` + +`pyiron_base.jobs.datamining.PyironTable`/`TableJob`) as plain functions that +read directly from a project directory and each job's HDF5 file. **It has +zero dependency on pyiron_base or pyiron_atomistics** — no job objects, no +SQL database. Only `pyfileindex` (filesystem indexing) and `h5io_browser` +(HDF5 reads) are used. This constraint is intentional and must not be +reintroduced: never write a node/function here that takes a `job` object +with `__getitem__`; always take `(file_name, job_name)` strings. + +## The calling convention every property function must follow + +Every function that extracts one value from a job has this exact signature: + +```python +def get_(file_name: str, job_name: str): + return read_value(file_name, job_name, "") +``` + +- `file_name`: path to the job's `.h5` file on disk. +- `job_name`: the job's name — the top-level HDF5 group inside that file. +- `read_value(file_name, job_name, path)` (defined in this module) reads + `/` out of the HDF5 file. `path` is exactly what you'd put + inside `job["..."]` in real pyiron, e.g. `"output/generic/energy_pot"`. + +Function names spell the property out in full — no abbreviations (`get_bm` +was renamed to `get_bulk_modulus` for exactly this reason). Existing examples +in this file: + +```python +def get_bulk_modulus(file_name: str, job_name: str): + return read_value(file_name, job_name, "output/equilibrium_bulk_modulus") + + +def get_potential(file_name: str, job_name: str): + return read_value(file_name, job_name, "Al_ref/input/potential_inp/potential/Name") + + +def get_lattice_parameter(file_name: str, job_name: str): + return read_value(file_name, job_name, "output/equilibrium_volume") ** (1 / 3) +``` + +`get_lattice_parameter` shows that any transform (unit conversion, `** power`, +etc.) is just plain Python applied to the value returned by `read_value` — +there is no separate "power" parameter or config system; the transform lives +directly in the function body. + +If you don't know the exact HDF5 path for a property, inspect a real job +file directly, e.g.: + +```python +import h5py +with h5py.File(file_name, "r") as f: + print(list(f[job_name].keys())) # top-level groups: input, output, server, ... + print(list(f[job_name]["output"].keys())) +``` + +## The node that wraps each function: the "Add" pattern + +Every property function gets exactly one matching `@as_function_node` +wrapper, named `Add`, with this exact shape: + +```python +@as_function_node("functions") +def Add(functions: dict = None) -> dict: + """ + Add {"": get_} to a functions dict and return it, + pluggable directly into BuildTable(functions=...). Daisy-chains the same + way AddBulkModulus does. + """ + from pyiron_nodes.databases.pyiron_tables import get_ + + functions = dict(functions) if functions is not None else {} + functions[""] = get_ + return functions +``` + +Rules, all load-bearing — do not deviate: + +1. **One input**, `functions: dict = None`. `None` means "start a fresh + dict"; a non-`None` dict means "grow this dict". +2. **Copy, don't mutate**: `functions = dict(functions) if functions is not + None else {}` — the incoming dict is never mutated in place, a fresh copy + is returned. This makes the node pure/side-effect-free, matching every + other accumulator node in this codebase (e.g. `AddPristine` in + `pyiron_nodes/atomistic/structure/container_new.py`). +3. **No shared helper function.** Each node inlines its own two-line + copy-and-insert. There is deliberately no `add_table_function(func, + functions)` utility — this was tried and explicitly rejected in favor of + every node being self-contained. Do not reintroduce a shared accumulator + helper. +4. **The import is local, inside the function body** + (`from pyiron_nodes.databases.pyiron_tables import get_`), not + a module-level import — this matches the existing style of every node in + this file (`IndexProject`, `BuildTable`, `AddBulkModulus`, ...). +5. **Output dict key is the bare property name, never the function name.** + The `get_` prefix must never appear in the table: use `functions["bulk_modulus"] + = get_bulk_modulus`, not `functions[get_bulk_modulus.__name__] = ...` (which + would put `"get_bulk_modulus"` in the table). The dict key becomes the + resulting table's column name, so it must be exactly the clean property + name — full word, no abbreviation (`"bulk_modulus"`, not `"bm"`) and no + `get_` prefix. The function name and the label are kept in sync by + naming the function `get_` — e.g. property + `"bulk_modulus"` → function `get_bulk_modulus`, property + `"lattice_parameter"` → function `get_lattice_parameter` — so there's only + one property name to invent per node, just written two ways. +6. There is **no generic parametrized node** (no `AddFunction(column_label, + hdf_path, power=...)`) — that approach existed earlier and was removed in + favor of one named function + one node per property, because it's more + explicit and each property's HDF5 path/transform lives in readable, + greppable Python rather than as a string argument at the call site. + +## Wiring nodes into a workflow + +Nodes only resolve their upstream inputs correctly when connected through a +`core.Workflow` graph — instantiating and chaining nodes directly as plain +Python objects outside a `Workflow` does **not** auto-pull upstream node +outputs (you'd get the raw `Node` object passed to `dict(...)` and a +`TypeError`). Always wire like this: + +```python +from core import Workflow +from pyiron_nodes.databases.pyiron_tables import ( + AddBulkModulus, AddPotential, AddLatticeParameter, BuildTable, DbFilterFunction, +) + +wf = Workflow("my_table") + +wf.DbFilterFunction = DbFilterFunction(hamilton="Murnaghan") + +wf.BulkModulusFunction = AddBulkModulus() +wf.PotentialFunction = AddPotential(functions=wf.BulkModulusFunction) +wf.LatticeParameterFunction = AddLatticeParameter(functions=wf.PotentialFunction) + +wf.BuildTable = BuildTable( + project_path="/path/to/pyiron/project", + functions=wf.LatticeParameterFunction, + status=["finished"], + db_filter_function=wf.DbFilterFunction, +) + +result = wf.run() # or wf.BuildTable.pull() for just that node +``` + +Each `Add*` node takes the *previous* node's output back in as `functions` +— this is the same accumulator-chaining pattern `AddPristine` uses for +`StructureContainer` in `container_new.py`: no separate merge node is ever +needed, you just keep threading the growing dict through the chain. Order of +chaining doesn't matter for correctness (dict keys don't collide unless two +properties reuse the same function name), only for readability. + +The fully worked reference implementation is +`pyiron_nodes/Workflows/pyiron_table_potential_scan.py`. + +## What `BuildTable` does with the dict + +`functions` ends up as a plain `dict` of `{"": callable}`, e.g. +`{"bulk_modulus": get_bulk_modulus, "potential": get_potential, +"lattice_parameter": get_lattice_parameter}`. +`BuildTable`/`build_table` walks every job in the project (via +`index_project` + `filter_job_table`), and for each job calls every function +in the dict as `func(file_name, job_name)` (see `apply_functions_to_job`). +A function that raises for a given job has its value set to `None` for that +row rather than aborting the whole table build — write property functions +assuming this: no need for your own broad `try/except`, just read the value +and let a genuine failure produce `None` for that row. + +Each output row is built as `{"job_id": ..., "job": ..., "hamilton": ...}` +first, then merged with the function results — so `job_id`, `job`, +`hamilton` are always the first three columns, followed by one column per +property in the order its `Add*` node was chained in. + +`job_id` is the **real** pyiron database job ID, read directly out of the +job's own HDF5 file via `get_job_id` (`/job_id` inside the file). +It is not a synthetic per-scan counter — do not reintroduce +`enumerate(..., start=1)` style numbering for it. + +## Checklist for adding a new property + +1. Find the HDF5 path for the value (inspect a real job file if unsure). +2. Pick a clean, full-word property name (no abbreviations) — this is both + the eventual table column name and, prefixed with `get_`, the function + name. +3. Add `def get_(file_name, job_name): return read_value(file_name, job_name, "")` + (plus any transform) near the other `get_*` functions. +4. Add `Add(functions: dict = None) -> dict` immediately after it, + copying the exact shape of `AddBulkModulus`/`AddPotential`/ + `AddLatticeParameter` — remember the dict key is `""`, not + `get_.__name__`. +5. Wire it into a workflow: `wf.X = Add(functions=wf.)`. +6. Verify against a real project directory — `wf.BuildTable.pull()` (or + `wf.run()`) and check the new column appears with sane values, not + all-`None` (which usually means the HDF5 path is wrong). diff --git a/databases/pyiron_tables.py b/databases/pyiron_tables.py new file mode 100644 index 00000000..bc5273d1 --- /dev/null +++ b/databases/pyiron_tables.py @@ -0,0 +1,328 @@ +""" +Recreates pyiron_base's job-table / PyironTable data mining functionality +(pyiron_base.database.filetable.FileTable + pyiron_base.jobs.datamining.PyironTable/TableJob) +as plain functions, reading directly from the project directory and job HDF5 files. + +No dependency on pyiron_base or pyiron_atomistics - only on the lightweight libraries +pyiron_base itself builds on: pyfileindex (file-system indexing) and h5io_browser +(HDF5 read access). +""" + +from core import as_function_node + +# --------------------------------------------------------------------------- +# low level HDF5 / file-system helpers +# --------------------------------------------------------------------------- + + +def _read_hdf_value(file_name: str, h5_path: str): + from h5io_browser.base import _read_hdf + + return _read_hdf(hdf_filehandle=file_name, h5_path=h5_path) + + +def _parse_job_type(type_string: str) -> str: + # type_string looks like "" + return type_string.split(".")[-1].split("'")[0] + + +def get_job_status(file_name: str, job_name: str): + import os + + if not os.path.exists(file_name): + return None + try: + return _read_hdf_value(file_name, job_name + "/status") + except (KeyError, OSError): + return None + + +def get_job_type(file_name: str, job_name: str): + try: + return _parse_job_type(_read_hdf_value(file_name, job_name + "/TYPE")) + except (KeyError, OSError): + return None + + +def get_job_id(file_name: str, job_name: str): + try: + return _read_hdf_value(file_name, job_name + "/job_id") + except (KeyError, OSError): + return None + + +def read_value(file_name: str, job_name: str, path: str): + """ + Read an arbitrary value out of a job's HDF5 file, analogous to `job[""]` + in pyiron_base, e.g. read_value(file_name, job_name, "output/generic/energy_pot") + """ + return _read_hdf_value(file_name, job_name + "/" + path) + + +def get_bulk_modulus(file_name: str, job_name: str): + return read_value(file_name, job_name, "output/equilibrium_bulk_modulus") + + +def get_potential(file_name: str, job_name: str): + return read_value(file_name, job_name, "Al_ref/input/potential_inp/potential/Name") + + +def get_lattice_parameter(file_name: str, job_name: str): + return read_value(file_name, job_name, "output/equilibrium_volume") ** (1 / 3) + + +# --------------------------------------------------------------------------- +# job table (replaces Project.job_table() / FileTable) +# --------------------------------------------------------------------------- + + +def index_project(project_path: str, recursive: bool = True): + """ + Walk a pyiron project directory and build a job table purely from the job + HDF5 files on disk - no SQL database involved. + + Mirrors pyiron_base.database.filetable.FileTable.init_table / get_extract. + """ + import os + + import pandas as pd + from pyfileindex import PyFileIndex + + def _is_h5(file_name: str) -> bool: + return file_name.endswith(".h5") + + fileindex = PyFileIndex(path=project_path, filter_function=_is_h5) + df_files = fileindex.dataframe + df_files = df_files[~df_files.is_directory] + + if not recursive: + project_path_abs = os.path.abspath(project_path) + df_files = df_files[ + df_files.path.apply(lambda p: os.path.dirname(p) == project_path_abs) + ] + + rows = [] + for path in df_files.path.values: + job_name = os.path.splitext(os.path.basename(path))[0] + status = get_job_status(path, job_name) + if status is None: + # not a pyiron job hdf5 file (e.g. no top-level "/status" node) + continue + rows.append( + { + "id": get_job_id(path, job_name), + "job": job_name, + "project": os.path.dirname(path) + "/", + "path": path, + "status": status, + "hamilton": get_job_type(path, job_name), + } + ) + + return pd.DataFrame( + rows, columns=["id", "job", "project", "path", "status", "hamilton"] + ) + + +def filter_job_table(job_table, status=("finished",), db_filter_function=None): + """ + Args: + job_table (pandas.DataFrame): as returned by index_project + status (list/tuple of str): only keep jobs with one of these status values + db_filter_function (callable/None): function(job_table) -> bool pandas.Series, + same signature as pyiron_base's `TableJob.db_filter_function` + """ + df = job_table[job_table.status.isin(status)] + if db_filter_function is not None: + df = df[db_filter_function(df)] + return df + + +# --------------------------------------------------------------------------- +# applying user analysis functions to jobs (replaces PyironTable._iterate_over_job_lst) +# --------------------------------------------------------------------------- + + +def apply_functions_to_job(file_name: str, job_name: str, functions: dict): + """ + Args: + file_name (str): path to the job's hdf5 file + job_name (str): name of the job (top level group in the hdf5 file) + functions (dict): {label: callable(file_name, job_name) -> value} + + Returns: + dict: {label: value}, functions that raise are set to None + """ + result = {} + for label, func in functions.items(): + try: + result[label] = func(file_name, job_name) + except Exception: + result[label] = None + return result + + +def build_table( + project_path: str, + functions: dict, + status=("finished",), + db_filter_function=None, + recursive: bool = True, +): + """ + End to end replacement for: + + table = pr.create.table("table") + table.db_filter_function = db_filter_function + table.add["label"] = func + table.run() + table.get_dataframe() + + Args: + project_path (str): root directory of the pyiron project to scan + functions (dict): {label: callable(file_name, job_name) -> value} + status (list/tuple of str): job status values to include, default ("finished",) + db_filter_function (callable/None): function(job_table) -> bool pandas.Series + recursive (bool): include jobs in sub-projects + + Returns: + pandas.DataFrame + """ + import pandas as pd + + job_table = index_project(project_path=project_path, recursive=recursive) + job_table = filter_job_table( + job_table, status=status, db_filter_function=db_filter_function + ) + + rows = [] + for _, row in job_table.iterrows(): + values = { + "job_id": row["id"], + "job": row["job"], + "hamilton": row["hamilton"], + } + values.update( + apply_functions_to_job( + file_name=row["path"], job_name=row["job"], functions=functions + ) + ) + rows.append(values) + + return pd.DataFrame(rows) + + +# --------------------------------------------------------------------------- +# pyiron_nodes wrappers +# --------------------------------------------------------------------------- + + +@as_function_node("job_table") +def IndexProject(project_path: str, recursive: bool = True): + from pyiron_nodes.databases.pyiron_tables import index_project + + return index_project(project_path=project_path, recursive=recursive) + + +@as_function_node("table") +def BuildTable( + project_path: str, + functions: dict, + status: list = ["finished"], + db_filter_function=None, + recursive: bool = True, +): + from pyiron_nodes.databases.pyiron_tables import build_table + + return build_table( + project_path=project_path, + functions=functions, + status=status, + db_filter_function=db_filter_function, + recursive=recursive, + ) + + +@as_function_node("db_filter_function") +def DbFilterFunction(hamilton=None, status=None, job_name_contains: str = None): + """ + Build a `db_filter_function(job_table) -> bool Series`, pluggable directly into + BuildTable(db_filter_function=...). Mirrors pyiron_base's JobFilters.job_type / + JobFilters.job_name_contains, plus a status filter. + + Args: + hamilton (str/list/None): keep only rows whose job type ("hamilton") matches + one of these, e.g. "Murnaghan" or ["Murnaghan", "Lammps"] + status (str/list/None): keep only rows whose status matches one of these + job_name_contains (str/None): keep only rows whose job name contains this substring + """ + + def _db_filter_function(job_table): + import pandas as pd + + mask = pd.Series(True, index=job_table.index) + if status is not None: + status_lst = status if isinstance(status, (list, tuple)) else [status] + mask = mask & job_table.status.isin(status_lst) + if hamilton is not None: + hamilton_lst = ( + hamilton if isinstance(hamilton, (list, tuple)) else [hamilton] + ) + mask = mask & job_table.hamilton.isin(hamilton_lst) + if job_name_contains is not None: + mask = mask & job_table.job.str.contains(job_name_contains) + return mask + + return _db_filter_function + + +@as_function_node("functions") +def AddBulkModulus(functions: dict = None) -> dict: + """ + Add {"bulk_modulus": get_bulk_modulus} to a functions dict and return it, + pluggable directly into BuildTable(functions=...). Daisy-chains the same + way AddPristine chains StructureContainer: pass a previous node's dict + output back in as `functions` to keep growing the same dict, so no + separate merge node is needed: + + wf.A = AddBulkModulus() + wf.B = AddPotential(functions=wf.A) + wf.C = AddLatticeParameter(functions=wf.B) + wf.BuildTable = BuildTable(project_path=..., functions=wf.C) + + The dict key ("bulk_modulus") becomes the resulting table's column + name - it is the property name, not the function name (get_bulk_modulus). + """ + from pyiron_nodes.databases.pyiron_tables import get_bulk_modulus + + functions = dict(functions) if functions is not None else {} + functions["bulk_modulus"] = get_bulk_modulus + return functions + + +@as_function_node("functions") +def AddPotential(functions: dict = None) -> dict: + """ + Add {"potential": get_potential} to a functions dict and return it, + pluggable directly into BuildTable(functions=...). Daisy-chains the same + way AddBulkModulus does. + """ + from pyiron_nodes.databases.pyiron_tables import get_potential + + functions = dict(functions) if functions is not None else {} + functions["potential"] = get_potential + return functions + + +@as_function_node("functions") +def AddLatticeParameter(functions: dict = None) -> dict: + """ + Add {"lattice_parameter": get_lattice_parameter} to a functions dict and + return it, pluggable directly into BuildTable(functions=...). + Daisy-chains the same way AddBulkModulus does. + """ + from pyiron_nodes.databases.pyiron_tables import get_lattice_parameter + + functions = dict(functions) if functions is not None else {} + functions["lattice_parameter"] = get_lattice_parameter + return functions