From 8dd89413a0b57f109f424a10581fe9df23bc8913 Mon Sep 17 00:00:00 2001 From: Aliaksandr Yakutovich Date: Wed, 24 Apr 2024 14:17:24 +0000 Subject: [PATCH 1/5] Add openbis connection --- aiidalab_widgets_base/nodes.py | 7 +++++++ notebooks/eln_import.ipynb | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/aiidalab_widgets_base/nodes.py b/aiidalab_widgets_base/nodes.py index 9f4bf766a..26840d82b 100644 --- a/aiidalab_widgets_base/nodes.py +++ b/aiidalab_widgets_base/nodes.py @@ -36,6 +36,13 @@ "parameter_name": "structure_uuid", "description": "Optimize atomic positions and/or unit cell employing Quantum ESPRESSO. Quantum ESPRESSO is preferable for small structures with no cell dimensions larger than 15 Å. Additionally, you can choose to compute electronic properties of the material such as band structure and density of states.", }, + { + "name": "surfaces", + "calculation_type": "geo_opt", + "notebook": "submit_geometry_optimization.ipynb", + "parameter_name": "structure_uuid", + "description": "Optimize atomic positions and/or unit cell employing CP2K. CP2K is preferable for large structures with cell dimensions larger than 15 Å.", + }, { "name": "aiidalab-lsmo", "calculation_type": "geo_opt", diff --git a/notebooks/eln_import.ipynb b/notebooks/eln_import.ipynb index 51e566821..b443c2307 100644 --- a/notebooks/eln_import.ipynb +++ b/notebooks/eln_import.ipynb @@ -69,7 +69,7 @@ "outputs": [], "source": [ "object_displayed = AiidaNodeViewWidget()\n", - "open_in_app = OpenAiidaNodeInAppWidget()\n", + "open_in_app = OpenAiidaNodeInAppWidget(path_to_root=\"../../\")\n", "\n", "_ = dlink((eln_widget, 'node'), (object_displayed, 'node'))\n", "_ = dlink((eln_widget, 'node'), (open_in_app, 'node'))" From ba5b16915bb37bbeca975a7e3b3458eefa06c979 Mon Sep 17 00:00:00 2001 From: Aliaksandr Yakutovich Date: Thu, 25 Apr 2024 11:36:48 +0200 Subject: [PATCH 2/5] small fix --- notebooks/eln_import.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/eln_import.ipynb b/notebooks/eln_import.ipynb index b443c2307..295df7b93 100644 --- a/notebooks/eln_import.ipynb +++ b/notebooks/eln_import.ipynb @@ -58,7 +58,7 @@ "url = urlparse.urlsplit(jupyter_notebook_url)\n", "parsed_url = urlparse.parse_qs(url.query)\n", "params = {key:value[0] for key, value in parsed_url.items()}\n", - "eln_widget = ElnImportWidget(**params)" + "eln_widget = ElnImportWidget(path_to_root=\"../../\", **params)" ] }, { From 5bb6900d4bed84cc04add26e8bab8d54ee25f5d4 Mon Sep 17 00:00:00 2001 From: Fabio Lopes Date: Tue, 7 May 2024 15:36:36 +0200 Subject: [PATCH 3/5] Connection to openBIS --- aiidalab_widgets_base/elns.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/aiidalab_widgets_base/elns.py b/aiidalab_widgets_base/elns.py index 08c34f7df..badf3a04e 100644 --- a/aiidalab_widgets_base/elns.py +++ b/aiidalab_widgets_base/elns.py @@ -141,10 +141,39 @@ def _observe_node(self, _=None): ) info = q.all(flat=True)[0] except IndexError: - info = {} + structures, _ = self.get_all_structures_and_geoopts(self.node) + info = structures[-1].base.extras.all["eln"] self.eln.set_sample_config(**info) + def get_all_structures_and_geoopts(self, node): + """Get all atomistic models that led to the one used in the simulation""" + current_node = node + all_structures = [] + all_geoopts = [] + + while current_node is not None: + if isinstance(current_node, orm.StructureData): + all_structures.append(current_node) + current_node = current_node.creator + + elif isinstance(current_node, orm.CalcJobNode): + current_node = current_node.caller + + elif isinstance(current_node, orm.CalcFunctionNode): + current_node = current_node.inputs.source_structure + + elif isinstance(current_node, orm.WorkChainNode): + if "GeoOpt" in current_node.label: + all_geoopts.append(current_node) + current_node = current_node.inputs.structure + elif "ORBITALS" in current_node.label or "STM" in current_node.label: + current_node = current_node.inputs.structure + else: + current_node = current_node.caller + + return all_structures, all_geoopts + def send_to_eln(self, _=None): if self.eln and self.eln.is_connected: self.message.value = f"\u29d7 Sending data to {self.eln.eln_instance}..." From 0530453d41a2a393064ee61c09c0c15a3c09b368 Mon Sep 17 00:00:00 2001 From: Fabio Lopes Date: Wed, 12 Jun 2024 14:50:34 +0200 Subject: [PATCH 4/5] Read and validate JSON-LD using JSON-Schema --- notebooks/eln_import.ipynb | 138 +++++++++++++++++++++++++++++++++---- 1 file changed, 124 insertions(+), 14 deletions(-) diff --git a/notebooks/eln_import.ipynb b/notebooks/eln_import.ipynb index 295df7b93..79dd9dd27 100644 --- a/notebooks/eln_import.ipynb +++ b/notebooks/eln_import.ipynb @@ -45,7 +45,11 @@ "from aiidalab_widgets_base import AiidaNodeViewWidget, OpenAiidaNodeInAppWidget, ElnImportWidget\n", "import urllib.parse as urlparse\n", "from aiidalab_widgets_base import viewer\n", - "from traitlets import dlink" + "from traitlets import dlink\n", + "\n", + "from jsonschema import validate, RefResolver, Draft7Validator, ValidationError\n", + "import json\n", + "from pyld import jsonld" ] }, { @@ -54,30 +58,134 @@ "id": "4", "metadata": {}, "outputs": [], + "source": [ + "def load_json(filepath: str) -> dict:\n", + " return json.load(open(filepath, \"r\"))\n", + "\n", + "def find_refs(obj, refs=None) -> list[str]:\n", + " if refs is None:\n", + " refs = []\n", + " \n", + " if isinstance(obj, dict):\n", + " for key, value in obj.items():\n", + " if key == '$ref':\n", + " refs.append(value[2:])\n", + " else:\n", + " find_refs(value, refs)\n", + " \n", + " return refs\n", + "\n", + "def find_ref_schemas(schemas_filenames: list[str], refs = None):\n", + " if refs is None:\n", + " refs = []\n", + " \n", + " for schema_filename in schemas_filenames:\n", + " schema = load_json(f\"/home/jovyan/aiida-openbis/Notebooks/New_Metadata_Schemas/{schema_filename}\")\n", + " ref_schemas_filenames = find_refs(schema)\n", + " \n", + " if schema_filename not in refs:\n", + " refs.append(schema_filename)\n", + " find_ref_schemas(ref_schemas_filenames, refs)\n", + " \n", + " return refs\n", + "\n", + "def set_up_validator(json_schema_filename):\n", + " schemas_filenames = [json_schema_filename]\n", + " all_schemas_filenames = find_ref_schemas(schemas_filenames)\n", + "\n", + " first_schema = load_json(f\"/home/jovyan/aiida-openbis/Notebooks/New_Metadata_Schemas/{all_schemas_filenames[0]}\")\n", + " resolver = RefResolver(base_uri=\"http://example.com/\", referrer = first_schema)\n", + "\n", + " for referenced_schema_filename in all_schemas_filenames:\n", + " referenced_schema = load_json(f\"/home/jovyan/aiida-openbis/Notebooks/New_Metadata_Schemas/{referenced_schema_filename}\")\n", + " resolver.store[referenced_schema[\"$id\"]] = referenced_schema\n", + "\n", + " validator = Draft7Validator(first_schema, resolver=resolver)\n", + " \n", + " return validator\n", + "\n", + "# Function to expand context terms\n", + "def expand_context(base_context):\n", + " expanded_context = {}\n", + " for key, value in base_context.items():\n", + " if isinstance(value, str) and ':' in value:\n", + " prefix, suffix = value.split(':', 1)\n", + " if prefix in base_context:\n", + " expanded_context[key] = base_context[prefix] + suffix\n", + " else:\n", + " expanded_context[key] = value\n", + " else:\n", + " expanded_context[key] = value\n", + " return expanded_context\n", + "\n", + "def replace_keys_recursive(data, old_key, new_key):\n", + " if isinstance(data, dict):\n", + " for key in list(data.keys()): # Create a copy of keys to avoid RuntimeError\n", + " if key == old_key:\n", + " key = new_key\n", + " data[new_key] = data.pop(old_key)\n", + " replace_keys_recursive(data[key], old_key, new_key)\n", + " elif isinstance(data, list):\n", + " for item in data:\n", + " replace_keys_recursive(item, old_key, new_key)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], "source": [ "url = urlparse.urlsplit(jupyter_notebook_url)\n", "parsed_url = urlparse.parse_qs(url.query)\n", "params = {key:value[0] for key, value in parsed_url.items()}\n", - "eln_widget = ElnImportWidget(path_to_root=\"../../\", **params)" + "\n", + "molecule_json = json.loads(params[\"molecule_info\"])\n", + "\n", + "molecule_validator = set_up_validator(\"molecule.schema.json\")\n", + "schema_context = molecule_validator.schema[\"@context\"]\n", + "schema_context = expand_context(schema_context)\n", + "json_context = expand_context(molecule_json[\"@context\"])\n", + "\n", + "if \"@context\" in molecule_json:\n", + " _ = molecule_json.pop(\"@context\")\n", + " \n", + "for key1, value1 in schema_context.items():\n", + " for key2, value2 in json_context.items():\n", + " if value1 == value2:\n", + " replace_keys_recursive(molecule_json, key2, key1)\n", + " \n", + "molecule_info_valid = False\n", + "\n", + "try:\n", + " molecule_validator.validate(instance = molecule_json)\n", + " molecule_info_valid = True\n", + " params[\"molecule_info\"] = json.dumps(molecule_json)\n", + " eln_widget = ElnImportWidget(path_to_root=\"../../\", **params)\n", + "except ValidationError as e:\n", + " message = e.schema[\"error_msg\"] if \"error_msg\" in e.schema else e.message\n", + " print(f'Invalid data: {message}')" ] }, { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "6", "metadata": {}, "outputs": [], "source": [ - "object_displayed = AiidaNodeViewWidget()\n", - "open_in_app = OpenAiidaNodeInAppWidget(path_to_root=\"../../\")\n", + "if molecule_info_valid:\n", + " object_displayed = AiidaNodeViewWidget()\n", + " open_in_app = OpenAiidaNodeInAppWidget(path_to_root=\"../../\")\n", "\n", - "_ = dlink((eln_widget, 'node'), (object_displayed, 'node'))\n", - "_ = dlink((eln_widget, 'node'), (open_in_app, 'node'))" + " _ = dlink((eln_widget, 'node'), (object_displayed, 'node'))\n", + " _ = dlink((eln_widget, 'node'), (open_in_app, 'node'))" ] }, { "cell_type": "markdown", - "id": "6", + "id": "7", "metadata": {}, "source": [ "## Selected object:" @@ -86,17 +194,18 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "8", "metadata": {}, "outputs": [], "source": [ - "display(object_displayed)\n", - "display(eln_widget)" + "if molecule_info_valid:\n", + " display(object_displayed)\n", + " display(eln_widget)" ] }, { "cell_type": "markdown", - "id": "8", + "id": "9", "metadata": {}, "source": [ "## What's next?" @@ -105,11 +214,12 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "10", "metadata": {}, "outputs": [], "source": [ - "display(open_in_app)" + "if molecule_info_valid:\n", + " display(open_in_app)" ] } ], From 59471bb29a72d6b828d03f27f8fe9155cf9d93c3 Mon Sep 17 00:00:00 2001 From: Fabio Lopes Date: Fri, 26 Jul 2024 10:18:59 +0200 Subject: [PATCH 5/5] Correct connection to openBIS --- notebooks/eln_import.ipynb | 133 ++++++++++++------------------------- 1 file changed, 42 insertions(+), 91 deletions(-) diff --git a/notebooks/eln_import.ipynb b/notebooks/eln_import.ipynb index 79dd9dd27..3b12310d0 100644 --- a/notebooks/eln_import.ipynb +++ b/notebooks/eln_import.ipynb @@ -47,9 +47,11 @@ "from aiidalab_widgets_base import viewer\n", "from traitlets import dlink\n", "\n", - "from jsonschema import validate, RefResolver, Draft7Validator, ValidationError\n", "import json\n", - "from pyld import jsonld" + "from pyld import jsonld\n", + "\n", + "import sys\n", + "import os" ] }, { @@ -59,75 +61,16 @@ "metadata": {}, "outputs": [], "source": [ - "def load_json(filepath: str) -> dict:\n", - " return json.load(open(filepath, \"r\"))\n", - "\n", - "def find_refs(obj, refs=None) -> list[str]:\n", - " if refs is None:\n", - " refs = []\n", - " \n", - " if isinstance(obj, dict):\n", - " for key, value in obj.items():\n", - " if key == '$ref':\n", - " refs.append(value[2:])\n", - " else:\n", - " find_refs(value, refs)\n", - " \n", - " return refs\n", - "\n", - "def find_ref_schemas(schemas_filenames: list[str], refs = None):\n", - " if refs is None:\n", - " refs = []\n", - " \n", - " for schema_filename in schemas_filenames:\n", - " schema = load_json(f\"/home/jovyan/aiida-openbis/Notebooks/New_Metadata_Schemas/{schema_filename}\")\n", - " ref_schemas_filenames = find_refs(schema)\n", - " \n", - " if schema_filename not in refs:\n", - " refs.append(schema_filename)\n", - " find_ref_schemas(ref_schemas_filenames, refs)\n", - " \n", - " return refs\n", - "\n", - "def set_up_validator(json_schema_filename):\n", - " schemas_filenames = [json_schema_filename]\n", - " all_schemas_filenames = find_ref_schemas(schemas_filenames)\n", - "\n", - " first_schema = load_json(f\"/home/jovyan/aiida-openbis/Notebooks/New_Metadata_Schemas/{all_schemas_filenames[0]}\")\n", - " resolver = RefResolver(base_uri=\"http://example.com/\", referrer = first_schema)\n", - "\n", - " for referenced_schema_filename in all_schemas_filenames:\n", - " referenced_schema = load_json(f\"/home/jovyan/aiida-openbis/Notebooks/New_Metadata_Schemas/{referenced_schema_filename}\")\n", - " resolver.store[referenced_schema[\"$id\"]] = referenced_schema\n", - "\n", - " validator = Draft7Validator(first_schema, resolver=resolver)\n", - " \n", - " return validator\n", - "\n", - "# Function to expand context terms\n", - "def expand_context(base_context):\n", - " expanded_context = {}\n", - " for key, value in base_context.items():\n", - " if isinstance(value, str) and ':' in value:\n", - " prefix, suffix = value.split(':', 1)\n", - " if prefix in base_context:\n", - " expanded_context[key] = base_context[prefix] + suffix\n", - " else:\n", - " expanded_context[key] = value\n", - " else:\n", - " expanded_context[key] = value\n", - " return expanded_context\n", - "\n", - "def replace_keys_recursive(data, old_key, new_key):\n", - " if isinstance(data, dict):\n", - " for key in list(data.keys()): # Create a copy of keys to avoid RuntimeError\n", - " if key == old_key:\n", - " key = new_key\n", - " data[new_key] = data.pop(old_key)\n", - " replace_keys_recursive(data[key], old_key, new_key)\n", - " elif isinstance(data, list):\n", - " for item in data:\n", - " replace_keys_recursive(item, old_key, new_key)" + "#sys.path.append(os.path.dirname(\"/home/jovyan/aiida-openbis/Notebooks/Metadata_Schemas_LinkML/\"))\n", + "# from materialMLinfo import Molecule, slots\n", + "# def extract_molecule_data_with_linkml(molecule_jsonld):\n", + "# expanded = jsonld.expand(molecule_jsonld)\n", + "# molecule_data = {}\n", + "# for obj in jsonld_object:\n", + "# if '@type' in obj and str(Molecule.class_class_uri) in obj['@type']:\n", + "# molecule_data['name'] = obj.get(str(slots.name.uri), [None])[0]['@value']\n", + "# molecule_data['smiles'] = obj.get(str(slots.smiles.uri), [None])[0]['@value']\n", + "# return molecule_data" ] }, { @@ -137,35 +80,43 @@ "metadata": {}, "outputs": [], "source": [ + "# Extract data from the converted JSON-LD\n", + "def extract_molecule_data(jsonld_object, context):\n", + " expanded = jsonld.expand(molecule_jsonld)\n", + " compacted = jsonld.compact(expanded, context)\n", + " return {\"name\": compacted[\"name\"], \"smiles\": compacted[\"smiles\"]}\n", + "\n", + "def read_json(filepath: str) -> dict:\n", + " return json.load(open(filepath, \"r\"))\n", + "\n", "url = urlparse.urlsplit(jupyter_notebook_url)\n", "parsed_url = urlparse.parse_qs(url.query)\n", "params = {key:value[0] for key, value in parsed_url.items()}\n", + "molecule_jsonld = json.loads(params[\"molecule_info\"])\n", "\n", - "molecule_json = json.loads(params[\"molecule_info\"])\n", + "molecule_info_valid = False\n", "\n", - "molecule_validator = set_up_validator(\"molecule.schema.json\")\n", - "schema_context = molecule_validator.schema[\"@context\"]\n", - "schema_context = expand_context(schema_context)\n", - "json_context = expand_context(molecule_json[\"@context\"])\n", + "jsonld_context_filename = os.path.join(\n", + " os.sep, \n", + " \"home\", \n", + " \"jovyan\", \n", + " \"aiida-openbis\", \n", + " \"Notebooks\", \n", + " \"Metadata_Schemas_LinkML\",\n", + " \"materialMLinfoContext.jsonld\"\n", + ")\n", "\n", - "if \"@context\" in molecule_json:\n", - " _ = molecule_json.pop(\"@context\")\n", - " \n", - "for key1, value1 in schema_context.items():\n", - " for key2, value2 in json_context.items():\n", - " if value1 == value2:\n", - " replace_keys_recursive(molecule_json, key2, key1)\n", - " \n", - "molecule_info_valid = False\n", + "\n", + "aiida_context = read_json(jsonld_context_filename)\n", "\n", "try:\n", - " molecule_validator.validate(instance = molecule_json)\n", - " molecule_info_valid = True\n", - " params[\"molecule_info\"] = json.dumps(molecule_json)\n", + " molecule_data = extract_molecule_data(molecule_jsonld, aiida_context)\n", + " params[\"molecule_info\"] = json.dumps(molecule_data)\n", " eln_widget = ElnImportWidget(path_to_root=\"../../\", **params)\n", - "except ValidationError as e:\n", - " message = e.schema[\"error_msg\"] if \"error_msg\" in e.schema else e.message\n", - " print(f'Invalid data: {message}')" + " molecule_info_valid = True\n", + "except ValueError as e:\n", + " print(e)\n", + "\n" ] }, {