Standardize code layout, add some missing details

Author: blokhin

README.md

@@ -1,6 +1,6 @@
 # Metis scientific backend
 
-<p align="center"><img src="https://github.com/basf/metis-backend/blob/master/metis-logo.png" width="300" /></p>
+<p align="center"><img src="https://github.com/basf/metis-backend/blob/master/logo.png" width="300" /></p>
 
 **This is the third part of the whole Metis infra: [GUI](https://github.com/basf/metis-gui) &rlarr; [BFF](https://github.com/basf/metis-bff) &rlarr; [backend](https://github.com/basf/metis-backend).**
 

index.py

@@ -6,8 +6,8 @@
 from flask import Flask
 from netius.servers import WSGIServer
 
-from i_data.bp_data import bp_data
-from i_calculations.bp_calculations import bp_calculations
+from metis_backend.datasources.bp_data import bp_data
+from metis_backend.calculations.bp_calculations import bp_calculations
 
 
 app = Flask(__name__)

metis-logo.png renamed to logo.png

@@ -8,10 +8,10 @@
 
 from flask import Response, current_app, request
 
-from i_data import Data_storage
+from metis_backend.datasources import Data_storage
 
 
-CONFIG_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), 'conf/env.ini'))
+CONFIG_PATH = os.path.realpath(os.path.join(os.path.dirname(__file__), '../conf/env.ini'))
 assert os.path.exists(CONFIG_PATH)
 config = ConfigParser()
 config.read(CONFIG_PATH)

utils.py renamed to metis_backend/__init__.py

@@ -4,10 +4,10 @@
 
 from yascheduler import Yascheduler
 
-from i_calculations.xrpd import get_pattern
-from i_data import Data_type
-from i_structures.topas import ase_to_topas
-from i_structures.fullprof import ase_to_fullprof
+from metis_backend.calculations.xrpd import get_pattern
+from metis_backend.datasources import Data_type
+from metis_backend.structures.topas import ase_to_topas
+from metis_backend.structures.fullprof import ase_to_fullprof
 
 
 _scheduler_status_mapping = {

i_calculations/__init__.py renamed to metis_backend/calculations/__init__.py

@@ -11,7 +11,7 @@
 import requests
 from yascheduler import Yascheduler
 
-from utils import (
+from metis_backend import (
     get_data_storage,
     fmt_msg,
     key_auth,
@@ -24,11 +24,11 @@
     TMP_PHASEID_DIR,
     TMP_PHASEID_URL,
 )
-from i_calculations import Calc_setup, _scheduler_status_mapping
-from i_data import Data_type
-from i_structures import html_formula
-from i_structures.struct_utils import ase_unserialize
-from i_phaseid import (
+from metis_backend.calculations import Calc_setup, _scheduler_status_mapping
+from metis_backend.datasources import Data_type
+from metis_backend.structures import html_formula
+from metis_backend.structures.struct_utils import ase_unserialize
+from metis_backend.phaseid import (
     WAVELENGTH, MIN_Q, MAX_Q, N_BINS, N_BEST_MATCHES,
     background,
     get_best_match,

i_calculations/bp_calculations.py renamed to metis_backend/calculations/bp_calculations.py

@@ -1,7 +1,7 @@
 import random
 import string
 
-from i_data import Data_type
+from metis_backend.datasources import Data_type
 
 
 def get_pattern(resource):

i_calculations/schemata/dummy.json renamed to metis_backend/calculations/schemata/dummy.json

@@ -1,7 +1,7 @@
 """
 A simple persistence layer implementation;
-only a few Postgres tables are used;
-of course we use SQL here only and nowhere else
+a few Postgres tables are used
+(of course we must use raw SQL here only and nowhere else)
 """
 import json
 import logging
@@ -185,7 +185,7 @@ def drop_item(self, uuid):
 
     def get_refdis(self, els, strict=False):
 
-        from i_phaseid import MAX_PATT_LEN
+        from metis_backend.phaseid import MAX_PATT_LEN
 
         pg_arrays, pattern_ids, names = [], [], []
         els.sort()
@@ -220,7 +220,7 @@ def import_item(self, ext_id):
         # FIXME only allow from a certain phase ID session
         # FIXME specify parent node
 
-        from i_structures.struct_utils import provider_to_ase, ase_serialize
+        from metis_backend.structures.struct_utils import provider_to_ase, ase_serialize
 
         self.cursor.execute(f"SELECT provider, name, content FROM {REFSTRS_TABLE} WHERE ext_id = '{ext_id}';")
         row = self.cursor.fetchone()

i_calculations/schemata/fullprof.json renamed to metis_backend/calculations/schemata/fullprof.json

@@ -7,23 +7,23 @@
 from flask import Blueprint, current_app, request, abort, Response, send_file
 from ase import io as ase_io
 
-from i_data import Data_type
-from i_data.fmt import detect_format
-from i_data.xrpd import extract_pattern
+from metis_backend import MAX_UPLOAD_SIZE, get_data_storage, fmt_msg, key_auth, is_plain_text, is_valid_uuid
 
-from i_structures import html_formula
-from i_structures.cif_utils import cif_to_ase
-from i_structures.struct_utils import (
+from metis_backend.datasources import Data_type
+from metis_backend.datasources.fmt import detect_format
+from metis_backend.datasources.xrpd import extract_pattern
+
+from metis_backend.structures import html_formula
+from metis_backend.structures.cif_utils import cif_to_ase
+from metis_backend.structures.struct_utils import (
     poscar_to_ase,
     optimade_to_ase,
     refine,
     get_formula,
     ase_serialize,
     ase_unserialize,
 )
-from i_calculations.xrpd import get_pattern, get_pattern_name
-
-from utils import MAX_UPLOAD_SIZE, get_data_storage, fmt_msg, key_auth, is_plain_text, is_valid_uuid
+from metis_backend.calculations.xrpd import get_pattern, get_pattern_name
 
 
 bp_data = Blueprint("data", __name__, url_prefix="/data")

i_calculations/schemata/topas.json renamed to metis_backend/calculations/schemata/topas.json

@@ -1,16 +1,16 @@
 
-#from utils import is_plain_text
-
-
 def is_ascii(test):
-    try: test.decode("utf-8")
-    except: return False
-    else: return True
+    try:
+        test.decode("utf-8")
+    except:
+        return False
+    else:
+        return True
 
 
 def detect_format(string):
     """
-    Detect data format checking the most common features
+    Detect data format, checking the most common features
     """
     try: string = string.encode("utf-8")
     except: pass # keep strings as bytes FIXME

i_calculations/templates/dummy.tpl renamed to metis_backend/calculations/templates/dummy.tpl

@@ -4,8 +4,8 @@
 #import numpy as np
 import xylib
 
-from i_phaseid import MAX_PATT_LEN
-from i_data import Data_type
+from metis_backend.phaseid import MAX_PATT_LEN
+from metis_backend.datasources import Data_type
 
 
 def extract_pattern(binary):

i_calculations/templates/fullprof.tpl renamed to metis_backend/calculations/templates/fullprof.tpl

@@ -1,9 +1,9 @@
 
 import numpy as np
 
-from i_phaseid.el_groups import groups_abbreviations, chemical_symbols_and_groups, get_elements_or_groups
-from i_phaseid.background import background
-from i_phaseid.histogram import get_best_match
+from metis_backend.phaseid.el_groups import groups_abbreviations, chemical_symbols_and_groups, get_elements_or_groups
+from metis_backend.phaseid.background import background
+from metis_backend.phaseid.histogram import get_best_match
 
 
 WAVELENGTH = 0.1610184395  # experimental, switch between wavelengths based on 2theta

i_calculations/templates/topas.tpl renamed to metis_backend/calculations/templates/topas.tpl

@@ -0,0 +1,4 @@
+TRUNCATE TABLE backend_data_links;
+TRUNCATE TABLE backend_data_nodes CASCADE;
+-- TRUNCATE TABLE yascheduler_nodes;
+TRUNCATE TABLE yascheduler_tasks;

i_calculations/xrpd.py renamed to metis_backend/calculations/xrpd.py

@@ -10,10 +10,10 @@
 from phase_utils import assign_phase, save_as_phase
 
 import set_path
-from i_data import Data_type, NODE_TABLE, PHASE_TABLE
-from i_structures import html_formula
-from i_structures.struct_utils import get_formula, sgn_to_crsystem, crsystem_to_sgn, ase_unserialize
-from utils import get_data_storage
+from metis_backend import get_data_storage
+from metis_backend.datasources import Data_type, NODE_TABLE, PHASE_TABLE
+from metis_backend.structures import html_formula
+from metis_backend.structures.struct_utils import get_formula, sgn_to_crsystem, crsystem_to_sgn, ase_unserialize
 
 
 accuracy = 1E-02

i_data/__init__.py renamed to metis_backend/datasources/__init__.py

@@ -3,9 +3,9 @@
 import sys
 
 import set_path
-from i_data import detect_format
-from i_structures.struct_utils import poscar_to_ase, optimade_to_ase, refine, get_formula
-from i_structures.cif_utils import cif_to_ase
+from metis_backend.datasources.fmt import detect_format
+from metis_backend.structures.struct_utils import poscar_to_ase, optimade_to_ase, refine, get_formula
+from metis_backend.structures.cif_utils import cif_to_ase
 
 
 structure = open(sys.argv[1]).read()

i_data/bp_data.py renamed to metis_backend/datasources/bp_data.py

@@ -3,10 +3,11 @@
 import sys
 
 import set_path
-from i_structures.struct_utils import detect_format, poscar_to_ase, optimade_to_ase, refine, get_formula
-from i_structures.cif_utils import cif_to_ase
-from i_structures.topas import ase_to_topas
-from i_calculations import Calc_setup
+from metis_backend.datasources.fmt import detect_format
+from metis_backend.structures.struct_utils import poscar_to_ase, optimade_to_ase, refine, get_formula
+from metis_backend.structures.cif_utils import cif_to_ase
+from metis_backend.structures.topas import ase_to_topas
+from metis_backend.calculations import Calc_setup
 
 
 structure = open(sys.argv[1]).read()

i_data/fmt.py renamed to metis_backend/datasources/fmt.py

@@ -4,7 +4,7 @@
 import random # TODO
 
 import set_path
-from i_structures.chemical_formulae import parse_formula
+from metis_backend.structures.chemical_formulae import parse_formula
 
 
 BFF_PREFIX = 'bff_'
@@ -66,7 +66,7 @@ def get_or_create_tag(conn, cursor, user_id, type_id, title, descr):
 
 def save_as_phase(db, node_uuid, phase):
 
-    logging.warning(f"Assing phase {phase} to node {node_uuid}")
+    logging.warning(f"Assigning phase {phase} to node {node_uuid}")
 
     db.cursor.execute("""SELECT id FROM {} WHERE uuid = '{}';""".format(
         BFF_TABLES['datasources'], node_uuid

i_data/migration.sql renamed to metis_backend/datasources/migration.sql

@@ -6,7 +6,7 @@
 import httplib2
 
 import set_path
-from utils import API_KEY
+from metis_backend import API_KEY
 
 
 # see BFF server

i_data/xrpd.py renamed to metis_backend/datasources/xrpd.py

@@ -1,31 +1,60 @@
 #!/usr/bin/env python
 """
-NB
+NB this is equivalent to the curl calls, e.g.
 curl -XPOST -HKey:... http://localhost:7050/data/create -d 'content={"attributes":{"immutable_id":42,"species":[{"chemical_symbols":["Au"]}],"cartesian_site_positions":[[0,0,0]],"lattice_vectors":[[0,2,2],[2,0,2],[2,2,0]]}}'
-curl -XPOST -HKey:... http://localhost:7050/calculations/create -d 'engine=dummy&uuid=0ee24493-5f4a-41f6-a5d7-171e408f4048'
+curl -XPOST -HKey:... http://localhost:7050/calculations/create -d 'engine=dummy&uuid=$UUID'
+curl -XPOST -HKey:... http://localhost:7050/calculations/phaseid -d 'uuid=$UUID&els=Al-K-O'
+curl -XPOST -HKey:... http://localhost:7050/data/import -d 'id=S1254081'
 """
+
 import sys
+import os.path
+import base64
 
 from common import make_request
+
 import set_path
-from utils import API_KEY
+from metis_backend import API_KEY
+from metis_backend.datasources.fmt import detect_format
 
 
 host = 'http://localhost:7050'
-#API_KEY = 'XXX' # for production server
+#API_KEY = 'XXX'  # redefine for production
 
-try: content = open(sys.argv[1]).read()
-except IndexError: content = """{"attributes":{"immutable_id":42, "species":[{"chemical_symbols":
+try:
+    fname = sys.argv[1]
+except IndexError:
+    fname = None
+    content = """{"attributes":{"immutable_id":42, "species":[{"chemical_symbols":
 ["Au"]}],"cartesian_site_positions":[[0,0,0]],"lattice_vectors":[[0,2,2],[2,0,2],[2,2,0]]}}"""
 
+try:
+    content = open(fname).read()
+except UnicodeDecodeError:
+    content = open(fname, "rb").read()
+except TypeError:
+    pass
+
+if fname:
+    fname = os.path.basename(fname)
+
+fmt = detect_format(content)
+if fmt == 'raw':
+    content = b'base64,' + base64.b64encode(content)
+
 try: engine = sys.argv[2]
-except IndexError: engine = 'topas'
+except IndexError: engine = None
+
+print(f"Requested calculation: {engine or 'NO'}")
 
-_, answer = make_request(host + '/data/create', {'content': content}, 'POST', headers={'Key': API_KEY})
+#_, answer = make_request(host + '/data/create', {'content': content, 'name': fname}, 'POST', headers={'Key': API_KEY})
+_, answer = make_request(host + '/data/create', {'content': content, 'fmt': fmt, 'name': fname}, 'POST', headers={'Key': API_KEY})
 print(answer)
 print('=' * 100 + 'Data uploaded correctly')
 
-_, answer = make_request(host + '/calculations/create', {'uuid': answer['uuid'], 'engine': engine}, 'POST', headers={'Key': API_KEY})
-print(answer)
-print('=' * 100 + f'Calc via {engine} engine submitted correctly')
+if engine:
+    _, answer = make_request(host + '/calculations/create', {'uuid': answer['uuid'], 'engine': engine}, 'POST', headers={'Key': API_KEY})
+    print(answer)
+    print('=' * 100 + f'Calc via {engine} engine submitted correctly')
+
 print('=' * 100 + 'Test passed')

i_phaseid/__init__.py renamed to metis_backend/phaseid/__init__.py

@@ -1,9 +1,9 @@
 import random
 
 import set_path
-from i_data import Data_type
-from i_structures.chemical_formulae import common_chem_elements
-from i_calculations.xrpd import get_pattern_name
+from metis_backend.datasources import Data_type
+from metis_backend.structures.chemical_formulae import common_chem_elements
+from metis_backend.calculations.xrpd import get_pattern_name
 
 
 common_chem_elements = common_chem_elements.split()

i_phaseid/background.py renamed to metis_backend/phaseid/background.py

@@ -7,8 +7,8 @@
 from helpers import gen_data_item
 
 import set_path
-from i_data import Data_type
-from utils import get_data_storage
+from metis_backend import get_data_storage
+from metis_backend.datasources import Data_type
 
 
 FAKE_NODES = 5000

i_phaseid/el_groups.py renamed to metis_backend/phaseid/el_groups.py

@@ -7,7 +7,7 @@
 from helpers import POSSIBLE_CONTENT
 
 import set_path
-from utils import get_data_storage
+from metis_backend import get_data_storage
 
 
 db = get_data_storage()