diff --git a/config/common/geography_tree.json b/config/common/geography_tree.json index 6792f658bb4..a9b6dd84050 100644 --- a/config/common/geography_tree.json +++ b/config/common/geography_tree.json @@ -1,5 +1,5 @@ { - "geography_tree": { + "tree": { "treedef": { "levels": [ { diff --git a/config/common/geologictimeperiod_tree.json b/config/common/geologictimeperiod_tree.json new file mode 100644 index 00000000000..16e952d1f46 --- /dev/null +++ b/config/common/geologictimeperiod_tree.json @@ -0,0 +1,44 @@ +{ + "tree": { + "treedef": { + "levels": [ + { + "name": "Time Root", + "rank": 0, + "enforced": true, + "infullname": false, + "fullnameseparator": ", " + }, + { + "name": "Erathem/Era", + "rank": 100, + "enforced": false, + "infullname": false, + "fullnameseparator": ", " + }, + { + "name": "System/Period", + "rank": 200, + "enforced": false, + "infullname": false, + "fullnameseparator": ", " + }, + { + "name": "Series/Epoch", + "rank": 300, + "enforced": false, + "infullname": true, + "fullnameseparator": ", " + }, + { + "name": "Stage/Age", + "rank": 400, + "enforced": false, + "infullname": true, + "fullnameseparator": ", " + } + ] + }, + "nodes": [] + } +} \ No newline at end of file diff --git a/config/common/lithostrat_tree.json b/config/common/lithostrat_tree.json new file mode 100644 index 00000000000..9f006679ca8 --- /dev/null +++ b/config/common/lithostrat_tree.json @@ -0,0 +1,45 @@ +{ + "tree": { + "treedef": { + "levels": [ + { + "name": "Surface", + "enforced": true, + "infullname": false, + "rank": 0 + }, + { + "name": "Super Group", + "enforced": false, + "infullname": false, + "rank": 100 + }, + { + "name": "Litho Group", + "enforced": false, + "infullname": false, + "rank": 200 + }, + { + "name": "Formation", + "enforced": false, + "infullname": false, + "rank": 300 + }, + { + "name": "Member", + "enforced": false, + "infullname": false, + "rank": 400 + }, + { + "name": "Bed", + "enforced": false, + "infullname": false, + "rank": 500 + } + ] + }, + "nodes": [] + } +} \ No newline at end of file diff --git a/config/common/storage_tree.json b/config/common/storage_tree.json index b6ab8843805..3c0de68b307 100644 --- a/config/common/storage_tree.json +++ b/config/common/storage_tree.json @@ -1,5 +1,5 @@ { - "storage_tree": { + "tree": { "treedef": { "levels": [ { @@ -53,7 +53,7 @@ { "name": "Rack", "enforced": false, - "infullname": "Rack", + "infullname": false, "rank": 450 }, { diff --git a/config/common/tectonicunit_tree.json b/config/common/tectonicunit_tree.json new file mode 100644 index 00000000000..1d0537607db --- /dev/null +++ b/config/common/tectonicunit_tree.json @@ -0,0 +1,45 @@ +{ + "tree": { + "treedef": { + "levels": [ + { + "name": "Root", + "enforced": true, + "infullname": false, + "rank": 0 + }, + { + "name": "Superstructure", + "enforced": false, + "infullname": false, + "rank": 100 + }, + { + "name": "Tectonic Domain", + "enforced": false, + "infullname": false, + "rank": 200 + }, + { + "name": "Tectonic Subdomain", + "enforced": false, + "infullname": false, + "rank": 300 + }, + { + "name": "Tectonic Unit", + "enforced": false, + "infullname": true, + "rank": 400 + }, + { + "name": "Tectonic Subunit", + "enforced": false, + "infullname": true, + "rank": 500 + } + ] + }, + "nodes": [] + } +} \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/api.py b/specifyweb/backend/setup_tool/api.py index 359d3b98eba..f59c974c6c0 100644 --- a/specifyweb/backend/setup_tool/api.py +++ b/specifyweb/backend/setup_tool/api.py @@ -17,7 +17,7 @@ from specifyweb.backend.setup_tool.prep_type_defaults import create_default_prep_types from specifyweb.backend.setup_tool.setup_tasks import setup_database_background, get_active_setup_task, get_last_setup_error, set_last_setup_error from specifyweb.celery_tasks import MissingWorkerError -from specifyweb.backend.setup_tool.tree_defaults import create_default_tree, update_tree_scoping +from specifyweb.backend.setup_tool.tree_defaults import start_default_tree_from_configuration, update_tree_scoping from specifyweb.specify.models import Institution, Discipline from specifyweb.backend.businessrules.uniqueness_rules import apply_default_uniqueness_rules from specifyweb.specify.management.commands.run_key_migration_functions import fix_cots @@ -193,6 +193,8 @@ def create_discipline(data): existing_discipline = Discipline.objects.filter(id=existing_id).first() if existing_discipline: return {"discipline_id": existing_discipline.id} + + is_first_discipline = Discipline.objects.count() == 0 # Resolve division division_url = data.get('division') @@ -215,14 +217,13 @@ def create_discipline(data): # Assign a taxon tree. Not required, but its eventually needed for collection object type. taxontreedef_url = data.get('taxontreedef', None) taxontreedef = resolve_uri_or_fallback(taxontreedef_url, None, Taxontreedef) - if taxontreedef is not None: + if taxontreedef_url and taxontreedef is not None: data['taxontreedef_id'] = taxontreedef.id data.update({ 'datatype_id': datatype.id, 'geographytreedef_id': geographytreedef.id, - 'geologictimeperiodtreedef_id': geologictimeperiodtreedef.id, - 'taxontreedef_id': taxontreedef.id if taxontreedef else None + 'geologictimeperiodtreedef_id': geologictimeperiodtreedef.id }) # Assign new Discipline ID @@ -247,6 +248,12 @@ def create_discipline(data): update_tree_scoping(geographytreedef, new_discipline.id) update_tree_scoping(geologictimeperiodtreedef, new_discipline.id) + # Create a default taxon tree if the database is already set up. + if not is_first_discipline: + create_taxon_tree({ + 'discipline_id': new_discipline.id + }) + return {"discipline_id": new_discipline.id} except Exception as e: @@ -365,9 +372,6 @@ def create_tectonicunit_tree(data): return create_tree('Tectonicunit', data) def create_tree(name: str, data: dict) -> dict: - # TODO: Use trees/create_default_trees - # https://github.com/specify/specify7/pull/6429 - # Figure out which scoping field should be used. use_institution = False use_discipline = True @@ -392,9 +396,9 @@ def create_tree(name: str, data: dict) -> dict: # Get tree configuration ranks = data.pop('ranks', dict()) - # Pre-load Default Tree - # TODO: trees/create_default_trees - preload_tree = data.pop('default', None) + # Properties for pre-loading default tree. Pre-loading should be done once setup is complete. + preload_tree = data.pop('preload', None) + preload_tree_file = data.pop('preloadFile', None) try: kwargs = {} @@ -404,7 +408,7 @@ def create_tree(name: str, data: dict) -> dict: if use_discipline and discipline is not None: kwargs['discipline'] = discipline - treedef = create_default_tree(name, kwargs, ranks, preload_tree) + treedef = start_default_tree_from_configuration(name, kwargs, ranks) # Set as the primary tree in the discipline if its the first one if use_discipline and discipline: diff --git a/specifyweb/backend/setup_tool/schema_defaults.py b/specifyweb/backend/setup_tool/schema_defaults.py index 13154336513..c52fb490e8e 100644 --- a/specifyweb/backend/setup_tool/schema_defaults.py +++ b/specifyweb/backend/setup_tool/schema_defaults.py @@ -1,8 +1,21 @@ from typing import Optional -from specifyweb.specify.models_utils.models_by_table_id import model_names_by_table_id -from specifyweb.specify.migration_utils.update_schema_config import update_table_schema_config_with_defaults +from django.db import transaction +from django.apps import apps as global_apps +from django.core.exceptions import MultipleObjectsReturned +from specifyweb.specify.migration_utils.update_schema_config import ( + camel_to_spaced_title_case, + datamodel_type_to_schematype, + FieldSchemaConfig, + HIDDEN_FIELDS, + uncapitilize, +) from .utils import load_json_from_file -from specifyweb.specify.models import Discipline +from specifyweb.specify.models import ( + Discipline, + datamodel, +) +from typing import NamedTuple, Literal +from specifyweb.specify.models_utils.models_by_table_id import model_names_by_table_id from pathlib import Path @@ -40,19 +53,215 @@ def apply_schema_defaults(discipline: Discipline): continue defaults.setdefault(table_name, {})[key] = v - # Update the schema for each table individually. - for model_name in model_names_by_table_id.values(): - logger.debug(f'Applying schema defaults for {model_name}. Using defaults: {overrides is not None}.') + create_default_table_schema_config( + discipline_id=discipline.id, + defaults=defaults, + ) + +def create_default_table_schema_config( + discipline_id: int, + apps = global_apps, + defaults: dict = None, +): + """Creates all schema config localization records. Assumes none exist.""" + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + with transaction.atomic(): + container_batch = [] + item_batch = [] + str_batch = [] + + # These seemingly redundant loops are used for much needed batching + + # Create table containers # + for table_name in model_names_by_table_id.values(): + container_batch.append(create_table_container( + table_name, + discipline_id + )) + Splocalecontainer.objects.bulk_create(container_batch, ignore_conflicts=True) + + # Create a map to reference the saved parent containers later + saved_containers = Splocalecontainer.objects.filter( + name__in=[c.name for c in container_batch], + discipline_id=discipline_id + ) + container_map = {c.name: c for c in saved_containers} + + # Create items for all fields in every table # + for table_name in model_names_by_table_id.values(): + table = datamodel.get_table(table_name) + table_defaults = defaults.get(table_name.lower()) if defaults is not None else dict() + if table_defaults is None: + table_defaults = dict() + + container = container_map[table_name.lower()] + + for field in table.all_fields: + field_defaults = None + if table_defaults.get('items'): + field_defaults = table_defaults['items'].get(field.name.lower()) + + item_batch.append(create_field_item( + field.name, + table_name, + container, + field_defaults + )) + + Splocalecontaineritem.objects.bulk_create(item_batch, ignore_conflicts=True) + + saved_items = Splocalecontaineritem.objects.filter( + container__name__in=[c.name for c in container_batch], + container__discipline_id=discipline_id + ) + item_map = {(i.container.name.lower(), i.name.lower()): i for i in saved_items} + + # Create strings for names and descriptions belonging all tables and their fields # + for table_name in model_names_by_table_id.values(): + table = datamodel.get_table(table_name) + table_defaults = defaults.get(table_name.lower()) if defaults is not None else dict() + if table_defaults is None: + table_defaults = dict() + + container = container_map[table.name.lower()] + + str_batch.extend(create_table_strings( + table_name, + container, + table_defaults + )) + + for field in table.all_fields: + field_name = field.name.lower() + + field_defaults = None + if table_defaults.get('items'): + field_defaults = table_defaults['items'].get(field_name) + + item_key = (container.name.lower(), field_name.lower()) + item = item_map[item_key] + + str_batch.extend( + create_field_strings( + field_name, + table_name, + item, + field_defaults, + ) + ) + Splocaleitemstr.objects.bulk_create(str_batch, ignore_conflicts=True) + +def create_table_container( + table_name: str, + discipline_id: int, + apps = global_apps, +): + Splocalecontainer = apps.get_model('specify', 'Splocalecontainer') + table = datamodel.get_table(table_name) + + return Splocalecontainer( + name=table.name.lower(), + discipline_id=discipline_id, + schematype=0, + ishidden=False, + issystem=table.system, + version=0, + ) + +def create_field_item( + field_name: str, + table_name: str, + container, + field_defaults: dict = None, + apps = global_apps, +): + Splocalecontaineritem = apps.get_model('specify', 'Splocalecontaineritem') + + table = datamodel.get_table(table_name) + field = table.get_field_strict(field_name) + field_name_str = camel_to_spaced_title_case(field.name) + field_desc_str = camel_to_spaced_title_case(field.name) + field_hidden = field_name.lower() in HIDDEN_FIELDS + field_required = field.required + picklist_name = None + if field_defaults is not None: + field_name_str = field_defaults.get('name', field_name_str) + field_desc_str = field_defaults.get('desc', field_desc_str) + field_hidden = field_defaults.get('ishidden', field_hidden) + field_required = field_defaults.get('isrequired', field_required) + picklist_name = field_defaults.get('picklistname', picklist_name) + + field_config = FieldSchemaConfig( + name=field_name, + column=field.column, + java_type=datamodel_type_to_schematype(field.type) if field.is_relationship else field.type, + description=field_desc_str, + language="en" + ) + + return Splocalecontaineritem( + name=field_config.name, + container=container, + type=field_config.java_type, + ishidden=field_hidden, + isrequired=field_required, + issystem=table.system, + version=0, + picklistname=picklist_name + ) + +def create_table_strings( + table_name: str, + container, + table_defaults: dict = None, + apps = global_apps, +) -> list: + table = datamodel.get_table(table_name) + table_name_str = table_defaults.get('name', camel_to_spaced_title_case(uncapitilize(table.name))) + table_desc_str = table_defaults.get('desc', camel_to_spaced_title_case(uncapitilize(table.name))) + + return create_strings('container', container, table_name_str, table_desc_str, apps) - # Table information - table_defaults = defaults.get(model_name.lower()) - table_description = None - if table_defaults: - table_description = table_defaults.get('desc') - - update_table_schema_config_with_defaults( - table_name=model_name, - discipline_id=discipline.id, - description=table_description, - defaults=table_defaults, - ) \ No newline at end of file +def create_field_strings( + field_name: str, + table_name: str, + item, + field_defaults: dict = None, + apps = global_apps, +): + table = datamodel.get_table(table_name) + field = table.get_field_strict(field_name) + field_name = field.name + field = table.get_field_strict(field_name) + field_name_str = camel_to_spaced_title_case(field.name) + field_desc_str = camel_to_spaced_title_case(field.name) + if field_defaults is not None: + field_name_str = field_defaults.get('name', field_name_str) + field_desc_str = field_defaults.get('desc', field_desc_str) + + return create_strings('item', item, field_name_str, field_desc_str, apps) + +def create_strings( + parent_type: Literal['item', 'container'], + item, + name_str, + desc_str, + apps = global_apps, +): + Splocaleitemstr = apps.get_model('specify', 'Splocaleitemstr') + strings = [] + for k, text in { + f"{parent_type}name": name_str, + f"{parent_type}desc": desc_str, + }.items(): + itm_str = { + 'text': text, + 'language': 'en', + 'version': 0, + } + itm_str[k] = item + strings.append(Splocaleitemstr(**itm_str)) + return strings \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/setup_tasks.py b/specifyweb/backend/setup_tool/setup_tasks.py index 9f2af7fdcb5..f9b1ad6a86c 100644 --- a/specifyweb/backend/setup_tool/setup_tasks.py +++ b/specifyweb/backend/setup_tool/setup_tasks.py @@ -8,6 +8,7 @@ from celery.result import AsyncResult from specifyweb.backend.setup_tool import api from specifyweb.backend.setup_tool.app_resource_defaults import create_app_resource_defaults +from specifyweb.backend.setup_tool.tree_defaults import start_preload_default_tree from specifyweb.specify.management.commands.run_key_migration_functions import fix_schema_config from specifyweb.specify.models_utils.model_extras import PALEO_DISCIPLINES, GEOLOGY_DISCIPLINES from specifyweb.celery_tasks import is_worker_alive, MissingWorkerError @@ -80,67 +81,79 @@ def update_progress(): logger.debug('## SETTING UP DATABASE WITH SETTINGS:##') logger.debug(data) - logger.debug('Creating institution') + logger.info('Creating institution') api.create_institution(data['institution']) update_progress() - logger.debug('Creating storage tree') + logger.info('Creating storage tree') api.create_storage_tree(data['storagetreedef']) update_progress() - logger.debug('Creating division') + logger.info('Creating division') api.create_division(data['division']) update_progress() discipline_type = data['discipline'].get('type', '') is_paleo_geo = discipline_type in PALEO_DISCIPLINES or discipline_type in GEOLOGY_DISCIPLINES default_tree = { - 'fullnamedirection': 1, - 'ranks': { - '0': True - } + 'ranks': {} } # if is_paleo_geo: # Create an empty chronostrat tree no matter what because discipline needs it. - logger.debug('Creating Chronostratigraphy tree') + logger.info('Creating Chronostratigraphy tree') default_chronostrat_tree = default_tree.copy() default_chronostrat_tree['fullnamedirection'] = -1 - api.create_geologictimeperiod_tree(default_chronostrat_tree) + chronostrat_result = api.create_geologictimeperiod_tree(default_chronostrat_tree) + chronostrat_treedef_id = chronostrat_result.get('treedef_id') - logger.debug('Creating geography tree') + logger.info('Creating geography tree') uses_global_geography_tree = data['institution'].get('issinglegeographytree', False) - api.create_geography_tree(data['geographytreedef'], global_tree=uses_global_geography_tree) + geography_result = api.create_geography_tree(data['geographytreedef'].copy(), global_tree=uses_global_geography_tree) + geography_treedef_id = geography_result.get('treedef_id') - logger.debug('Creating discipline') + logger.info('Creating discipline') discipline_result = api.create_discipline(data['discipline']) discipline_id = discipline_result.get('discipline_id') default_tree['discipline_id'] = discipline_id update_progress() if is_paleo_geo: - logger.debug('Creating Lithostratigraphy tree') + logger.info('Creating Lithostratigraphy tree') api.create_lithostrat_tree(default_tree.copy()) - logger.debug('Creating Tectonic Unit tree') + logger.info('Creating Tectonic Unit tree') api.create_tectonicunit_tree(default_tree.copy()) - logger.debug('Creating taxon tree') + logger.info('Creating taxon tree') if data['taxontreedef'].get('discipline_id') is None: data['taxontreedef']['discipline_id'] = discipline_id - api.create_taxon_tree(data['taxontreedef']) + taxon_result = api.create_taxon_tree(data['taxontreedef'].copy()) + taxon_treedef_id = taxon_result.get('treedef_id') update_progress() - logger.debug('Creating collection') - api.create_collection(data['collection']) + logger.info('Creating collection') + collection_result = api.create_collection(data['collection']) + collection_id = collection_result.get('collection_id') update_progress() - logger.debug('Creating specify user') - api.create_specifyuser(data['specifyuser']) + logger.info('Creating specify user') + specifyuser_result = api.create_specifyuser(data['specifyuser']) + specifyuser_id = specifyuser_result.get('user_id') - logger.debug('Finalizing database') + logger.info('Finalizing database') fix_schema_config() create_app_resource_defaults() + + # Pre-load trees + logger.info('Starting default tree downloads') + if is_paleo_geo: + start_preload_default_tree('Geologictimeperiod', discipline_id, collection_id, chronostrat_treedef_id, specifyuser_id) + if data['geographytreedef'].get('preload'): + start_preload_default_tree('Geography', discipline_id, collection_id, geography_treedef_id, specifyuser_id, data['geographytreedef'].get('preloadfile')) + if data['taxontreedef'].get('preload'): + start_preload_default_tree('Taxon', discipline_id, collection_id, taxon_treedef_id, specifyuser_id, data['taxontreedef'].get('preloadfile')) + update_progress() except Exception as e: logger.exception(f'Error setting up database: {e}') diff --git a/specifyweb/backend/setup_tool/tree_defaults.py b/specifyweb/backend/setup_tool/tree_defaults.py index 4cabae84742..8bdb992b410 100644 --- a/specifyweb/backend/setup_tool/tree_defaults.py +++ b/specifyweb/backend/setup_tool/tree_defaults.py @@ -1,60 +1,122 @@ -from django.db import transaction from django.db.models import Model as DjangoModel from typing import Type, Optional +from pathlib import Path +from uuid import uuid4 +import requests -from ..trees.utils import get_models +from .utils import load_json_from_file +from specifyweb.backend.trees.defaults import initialize_default_tree, create_default_tree_task import logging logger = logging.getLogger(__name__) -def create_default_tree(name: str, kwargs: dict, ranks: dict, preload_tree: Optional[str]): - """Creates an initial empty tree. This should not be used outside of the initial database setup.""" - with transaction.atomic(): - tree_def_model, tree_rank_model, tree_node_model = get_models(name) +DEFAULT_TREE_RANKS_FILES = { + 'Storage': Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'storage_tree.json', + 'Geography': Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'geography_tree.json', + 'Taxon': Path(__file__).parent.parent.parent.parent / 'config' / 'mammal' / 'taxon_mammal_tree.json', + 'Geologictimeperiod': Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'geologictimeperiod_tree.json', + 'Lithostrat': Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'lithostrat_tree.json', + 'Tectonicunit': Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'tectonicunit_tree.json' +} +DEFAULT_TAXON_TREE_LIST_URL = 'https://files.specifysoftware.org/taxonfiles/taxonfiles.json' +DEFAULT_TREE_URLS = { + 'Geography': 'https://files.specifysoftware.org/geographyfiles/geonames.csv', + 'Geologictimeperiod': 'https://files.specifysoftware.org/chronostratfiles/GeologicTimePeriod.csv', +} +DEFAULT_TREE_MAPPING_URLS = { + 'Geography': 'https://files.specifysoftware.org/treerows/geography.json', + 'Geologictimeperiod': 'https://files.specifysoftware.org/treerows/geologictimeperiod.json', +} - if tree_def_model.objects.count() > 0: - raise RuntimeError(f'Tree {name} already exists, cannot create default.') +def start_default_tree_from_configuration(tree_type: str, kwargs: dict, user_rank_cfg: dict): + """Starts the creation of an initial empty tree. This should not be used outside of the initial database setup.""" + # Load all default ranks for this type of tree + if tree_type == 'Taxon': + discipline = kwargs.get('discipline') + if discipline: + taxon_tree_discipline = discipline.type + rank_data = load_json_from_file(Path(__file__).parent.parent.parent.parent / 'config' / taxon_tree_discipline / f'taxon_{taxon_tree_discipline}_tree.json') + else: + rank_data = load_json_from_file(DEFAULT_TREE_RANKS_FILES.get(tree_type)) + if rank_data is None: + raise Exception(f'Could not load default rank JSON file for tree type {tree_type}.') - # Create tree definition - treedef = tree_def_model.objects.create( - name=name, - **kwargs, - ) + # list[RankConfiguration] + default_rank_cfg = rank_data.get('tree',{}).get('treedef',{}).get('levels') + if default_rank_cfg is None: + logger.debug(rank_data) + raise Exception(f'No default ranks found in the {tree_type} rank JSON file.') - # Create tree ranks - previous_tree_def_item = None - rank_list = list(int(rank_id) for rank_id, enabled in ranks.items() if enabled) - rank_list.sort() - for rank_id in rank_list: - previous_tree_def_item = tree_rank_model.objects.create( - treedef=treedef, - name=str(rank_id), # TODO: allow rank name configuration - rankid=rank_id, - parent=previous_tree_def_item, - ) - root_tree_def_item, create = tree_rank_model.objects.get_or_create( - treedef=treedef, - rankid=0 - ) + # Override default configuration with user's configuration + configurable_fields = {'title', 'enforced', 'infullname', 'fullnameseparator'} - # Create root node - # TODO: Avoid having duplicated code from add_root endpoint - root_node = tree_node_model.objects.create( - name="Root", - isaccepted=1, - nodenumber=1, - rankid=0, - parent=None, - definition=treedef, - definitionitem=root_tree_def_item, - fullname="Root" - ) + rank_cfg = [] + for rank in default_rank_cfg: + name = rank.get('name', '').lower() + user_rank = user_rank_cfg.get(name) + + # Initially assume all ranks should be included, except those explicitly set to False + rank_included = not (user_rank == False) + + if isinstance(user_rank, dict): + # The user configured this rank's properties + rank_included = user_rank.get('include', False) + + for field in configurable_fields: + rank[field] = user_rank.get(field, rank.get(field)) + + if not rank_included: + # The user disabled this rank. + continue + # Add this rank to the final rank configuration + rank_cfg.append(rank) + + if tree_type == 'Storage': + discipline_or_institution = kwargs.get('institution') + else: + discipline_or_institution = kwargs.get('discipline') - # TODO: Preload tree - if preload_tree is not None: - pass + tree_def = initialize_default_tree(tree_type.lower(), discipline_or_institution, tree_type.title(), rank_cfg, kwargs['fullnamedirection']) - return treedef + return tree_def + +def start_preload_default_tree(tree_type: str, discipline_id: Optional[int], collection_id: Optional[int], tree_def_id: int, specify_user_id: Optional[int], preload_file = None): + """Starts a populated default tree import without user input.""" + try: + # Tree download config: + tree_discipline_name = tree_type.lower() + tree_name = tree_type.title() + row_count = 1000000 # dummy value, only to allow progress checking + # Tree file urls + url = DEFAULT_TREE_URLS.get(tree_type) + mapping_url = DEFAULT_TREE_MAPPING_URLS.get(tree_type) + + if tree_type.lower() == 'taxon' and preload_file is not None: + # Schema described in CreateTree.tsx + logger.debug(f'Using tree configuration provided for taxon tree {tree_discipline_name}') + url = preload_file.get('file') + mapping_url = preload_file.get('mappingFile', preload_file.get('mappingfile')) + tree_name = preload_file.get('title', tree_name) + row_count = preload_file.get('rows', row_count) + + if not url or not mapping_url: + logger.warning(f'Can\'t preload tree, no default tree URLs for {tree_discipline_name} tree.') + return + + resp = requests.get(mapping_url) + resp.raise_for_status() + tree_cfg = resp.json() + + task_id = str(uuid4()) + create_default_tree_task.apply_async( + args=[url, discipline_id, tree_discipline_name, collection_id, specify_user_id, tree_cfg, row_count, tree_name, tree_def_id], + task_id=f"create_default_tree_{tree_type}_{task_id}", + taskid=task_id + ) + except Exception as e: + # Give up if there's an error to avoid resetting the entire setup. + logger.warning(f'Error trying to preload {tree_type} tree: {e}') + return def update_tree_scoping(treedef: Type[DjangoModel], discipline_id: int): """Trees may be created before a discipline is created. This will update their discipline.""" diff --git a/specifyweb/backend/trees/defaults.py b/specifyweb/backend/trees/defaults.py new file mode 100644 index 00000000000..8b9a2877a5c --- /dev/null +++ b/specifyweb/backend/trees/defaults.py @@ -0,0 +1,474 @@ +from typing import Any, Callable, List, Dict, Iterator, Optional, TypedDict, NotRequired +import json +import requests +import csv +import time +from requests.exceptions import ChunkedEncodingError, ConnectionError + +from django.db import transaction + +from specifyweb.backend.notifications.models import Message +from specifyweb.celery_tasks import LogErrorsTask, app +import specifyweb.specify.models as spmodels +from specifyweb.backend.trees.utils import get_models, SPECIFY_TREES, TREE_ROOT_NODES +from specifyweb.backend.trees.extras import renumber_tree, set_fullnames + +import logging +logger = logging.getLogger(__name__) + +class RankConfiguration(TypedDict): + name: str + title: NotRequired[str] + enforced: bool + infullname: bool + fullnameseparator: NotRequired[str] + rank: int # rank id + +def initialize_default_tree(tree_type: str, discipline_or_institution, tree_name: str, rank_cfg: list[RankConfiguration], full_name_direction: int=1): + """Creates an initial empty tree.""" + with transaction.atomic(): + tree_def_model, tree_rank_model, tree_node_model = get_models(tree_type) + + # Uniquify name + tree_def = None + unique_tree_name = tree_name + if tree_def_model.objects.filter(name=tree_name).exists(): + i = 1 + while tree_def_model.objects.filter(name=f"{tree_name}_{i}").exists(): + i += 1 + unique_tree_name = f"{tree_name}_{i}" + + # Create tree definition + scope = {} + if discipline_or_institution: + if tree_type == 'storage': + scope = { + 'institution': discipline_or_institution + } + else: + scope = { + 'discipline': discipline_or_institution + } + + tree_def, _ = tree_def_model.objects.get_or_create( + name=unique_tree_name, + fullnamedirection=full_name_direction, + **scope + ) + + # Create tree ranks + treedefitems_bulk = [] + rank_id = 0 + for rank in rank_cfg: + treedefitems_bulk.append( + tree_rank_model( + treedef=tree_def, + name=rank.get('name'), + title=(rank.get('title') or rank.get('name').title()), + rankid=int(rank.get('rank', rank_id)), + isenforced=rank.get('enforced', True), + isinfullname=rank.get('infullname', False), + fullnameseparator=rank.get('fullnameseparator', ' ') + ) + ) + rank_id += 10 + if treedefitems_bulk: + tree_rank_model.objects.bulk_create(treedefitems_bulk, ignore_conflicts=False) + + # Create a root node + created_items = list( + tree_rank_model.objects.filter(treedef=tree_def).order_by('rankid') + ) + + parent_item = None + for item in created_items: + item.parent = parent_item + parent_item = item + + tree_rank_model.objects.bulk_update(created_items, ['parent']) + + # Create a root node for non-taxon trees + # New taxon trees are expected to be empty + if tree_type != 'taxon': + create_default_root(tree_def, tree_type) + + return tree_def + +def create_default_root(tree_def, tree_type: str): + """Create root node""" + # TODO: Avoid having duplicated code from add_root endpoint + tree_def_model, tree_rank_model, tree_node_model = get_models(tree_type) + root_rank = tree_rank_model.objects.get(treedef=tree_def, rankid=0) + + # Don't create a root if one already exists + existing_root = tree_node_model.objects.filter( + definition=tree_def, + definitionitem=root_rank + ).first() + + if existing_root: + return existing_root + + tree_node, _ = tree_node_model.objects.get_or_create( + name=TREE_ROOT_NODES.get(tree_type, "Root"), + fullname=TREE_ROOT_NODES.get(tree_type, "Root"), + nodenumber=1, + definition=tree_def, + definitionitem=root_rank, + parent=None + ) + return tree_node + +class RankMappingConfiguration(TypedDict): + name: str + column: str + enforced: NotRequired[bool] + rank: NotRequired[int] + infullname: NotRequired[bool] + fullnameseparator: NotRequired[str] + fields: Dict[str, str] + +class DefaultTreeContext(): + """Context for a default tree creation task""" + def __init__(self, tree_type: str, tree_name: str): + self.tree_type = tree_type + self.tree_name = tree_name + + self.tree_def_model, self.tree_rank_model, self.tree_node_model = get_models(tree_type) + + self.tree_def = self.tree_def_model.objects.get(name=tree_name) + + self.create_rank_map() + self.root_parent = self.tree_node_model.objects.filter( + definitionitem__rankid=0, + definition=self.tree_def + ).first() + + self.counter = 0 + self.batch_size = 1000 + + def create_rank_map(self): + """Rank lookup map to reduce queries""" + ranks = list(self.tree_rank_model.objects.filter(treedef=self.tree_def)) + self.tree_def_item_map = {rank.name: rank for rank in ranks} + # Buffers for batches + self.rankid_map = {rank.rankid: rank for rank in ranks} + self.buffers = {rank.rankid: {} for rank in ranks} + self.created = {rank.rankid: {} for rank in ranks} + + def add_node_to_buffer(self, node, rank_id, row_id): + """Add node to the current batch of nodes to be created""" + if rank_id not in self.buffers: + self.buffers[rank_id] = {} + self.created[rank_id] = {} + self.buffers[rank_id][row_id] = node + return node + + def get_node_in_buffer(self, rank_id: int, name: str): + """Gets a node if its already in the current batch's buffer. Prevents duplication within a batch.""" + # Check for node in buffer, return node + buffer = self.buffers.get(rank_id, {}) + for node in buffer.values(): + if node.name == name: + return node + return None + + def get_existing_node_id(self, rank_id: int, name: str) -> Optional[int]: + """Gets a node's id if it has already been created. Prevents duplication across an entire import.""" + # Check for existing id, return id + created_in_rank = self.created.get(rank_id) + if created_in_rank: + return created_in_rank.get(name) + return None + + def flush(self, force=False): + """Flushes this batch's buffer if the batch is complete. Bulk creates the nodes in a complete batch.""" + self.counter += 1 + if not (force or self.counter > self.batch_size): + return + logger.debug(f"Batch creating {self.batch_size} rows.") + + # Go through ranks in ascending order and bulk create nodes + ordered_rank_ids = sorted(self.buffers.keys()) + for rank_id in ordered_rank_ids: + logger.debug(f"On rank {rank_id}") + buffer = self.buffers.get(rank_id, {}) + + rank = self.rankid_map.get(rank_id) + if rank is None: + # Can't create nodes because this rank doesn't exist + continue + + nodes_to_create = [] + # Update the nodes' parents to a saved version of their parents + for row_id, node in list(buffer.items()): + parent = getattr(node, 'parent', None) + parent_id = getattr(node, 'parent_id', None) + if parent is not None and getattr(parent, 'pk', None) is None: + saved_parent_id = self.created[parent.rankid].get(parent.name) + # Handle root + if not saved_parent_id and parent.name == getattr(self.root_parent, 'name', None): + saved_parent_id = self.root_parent.id + if saved_parent_id: + node.parent = None + node.parent_id = saved_parent_id + + # Create node if its parent has been created + if getattr(node.parent, 'pk', None) is not None or getattr(node, 'parent_id', None) is not None: + nodes_to_create.append(node) + else: + logger.warning(f"Could not create {node.name} because a valid parent could not be resolved. {parent_id}, {str(parent)}") + + if nodes_to_create: + self.tree_node_model.objects.bulk_create(nodes_to_create, ignore_conflicts=True) + + # Store the ids of the nodes were created in this batch + created_names = [n.name for n in nodes_to_create] + created_nodes = self.tree_node_model.objects.filter( + definition=self.tree_def, + definitionitem=rank, + name__in=created_names + ) + self.created[rank_id].update({n.name: n.id for n in created_nodes}) + + self.buffers[rank_id] = {} + + self.counter = 0 + +def add_default_tree_record(context: DefaultTreeContext, row: dict, tree_cfg: dict[str, RankMappingConfiguration], row_id: int): + """ + Given one CSV row and a column mapping / rank configuration dictionary, + walk through the 'ranks' in order, creating or updating each tree record and linking + it to its parent. + """ + tree_node_model = context.tree_node_model + tree_def = context.tree_def + parent = context.root_parent + parent_id = None + rank_id = 10 + + for rank_mapping in tree_cfg['ranks']: + rank_name = rank_mapping['name'] + fields_mapping = rank_mapping['fields'] + + record_name = row.get(rank_mapping.get('column', rank_name)) # Record's name is in the column. + + if not record_name: + continue # This row doesn't contain a record for this rank. + + defaults = {} + for model_field, csv_col in fields_mapping.items(): + if model_field == 'name': + continue + v = row.get(csv_col) + if v: + defaults[model_field] = v + + rank_title = rank_mapping.get('title', rank_name.capitalize()) + + # Get the rank by the column name. + # Skip creating on this rank if it doesn't exist + tree_def_item = context.tree_def_item_map.get(rank_name) + + if tree_def_item is None: + continue + + # Create the node at this rank if it isn't already there. + buffered = context.get_node_in_buffer(tree_def_item.rankid, record_name) + existing_id = context.get_existing_node_id(tree_def_item.rankid, record_name) + if existing_id is not None: + parent_id = existing_id + parent = None + elif buffered is not None: + parent_id = None + parent = buffered + else: + data = { + 'name': record_name, + 'fullname': record_name, + 'definition': tree_def, + 'definitionitem': tree_def_item, + 'rankid': tree_def_item.rankid, + **defaults + } + if parent is not None: + data['parent'] = parent + elif parent_id is not None: + data['parent_id'] = parent_id + obj = tree_node_model(**data) + obj = context.add_node_to_buffer(obj, tree_def_item.rankid, row_id) + + parent = obj + parent_id = None + rank_id += 10 + +@app.task(base=LogErrorsTask, bind=True) +def create_default_tree_task(self, url: str, discipline_id: int, tree_discipline_name: str, specify_collection_id: Optional[int], + specify_user_id: Optional[int], tree_cfg: dict, row_count: Optional[int], initial_tree_name: str, + existing_tree_def_id = None): + logger.info(f'starting task {str(self.request.id)}') + + discipline = None + if discipline_id: + discipline = spmodels.Discipline.objects.get(id=discipline_id) + tree_name = initial_tree_name # Name will be uniquified on tree creation + + if specify_user_id and specify_collection_id: + specify_user = spmodels.Specifyuser.objects.get(id=specify_user_id) + Message.objects.create( + user=specify_user, + content=json.dumps({ + 'type': 'create-default-tree-starting', + 'name': initial_tree_name, + 'taskid': str(self.request.id), + 'collection_id': specify_collection_id, + }) + ) + + current = 0 + total = 1 + def progress(cur: int, additional_total: int=0) -> None: + nonlocal current, total + current += cur + total += additional_total + if current > total: + current = total + self.update_state(state='RUNNING', meta={'current': current, 'total': total}) + + try: + with transaction.atomic(): + tree_type = 'taxon' + if tree_discipline_name in SPECIFY_TREES: + # non-taxon tree + tree_type = tree_discipline_name + + tree_def = None + if existing_tree_def_id: + # Import into exisiting tree + tree_def_model, tree_rank_model, tree_node_model = get_models(tree_type) + tree_def = tree_def_model.objects.filter(pk=existing_tree_def_id).first() + + if tree_def is None: + # Create a new empty tree. Get rank configuration from the mapping. + full_name_direction = 1 + if tree_type in ('geologictimeperiod',): + full_name_direction = -1 + + rank_cfg = [{ + 'name': 'Root', + 'enforced': True, + 'rank': 0, + **tree_cfg.get('root', {}) + }] + auto_rank_id = 10 + for rank in tree_cfg['ranks']: + rank_cfg.append({ + 'name': rank['name'], + 'enforced': rank.get('enforced', True), + 'infullname': rank.get('infullname', False), + 'fullnameseparator': rank.get('fullnameseparator', ' '), + 'rank': rank.get('rank', auto_rank_id) + }) + auto_rank_id += 10 + tree_def = initialize_default_tree(tree_type, discipline, initial_tree_name, rank_cfg, full_name_direction) + + create_default_root(tree_def, tree_type) + tree_name = tree_def.name + + # Start importing CSV data + context = DefaultTreeContext(tree_type, tree_name) + + total_rows = 0 + if row_count: + total_rows = row_count-2 + progress(0, total_rows) + + for row in stream_csv_from_url(url): + add_default_tree_record(context, row, tree_cfg, current) + context.flush() + progress(1, 0) + context.flush(force=True) + + # Finalize Tree + renumber_tree(tree_type) + set_fullnames(tree_def) + except Exception as e: + if specify_user_id and specify_collection_id: + Message.objects.create( + user=specify_user, + content=json.dumps({ + 'type': 'create-default-tree-failed', + 'name': tree_name, + 'taskid': str(self.request.id), + 'collection_id': specify_collection_id, + # 'error': str(e) + }) + ) + raise + + if specify_user_id and specify_collection_id: + Message.objects.create( + user=specify_user, + content=json.dumps({ + 'type': 'create-default-tree-completed', + 'name': tree_name, + 'taskid': str(self.request.id), + 'collection_id': specify_collection_id, + }) + ) + +def stream_csv_from_url(url: str) -> Iterator[Dict[str, str]]: + """ + Streams a taxon CSV from a URL. Yields each row. + """ + chunk_size = 8192 + max_retries = 10 + + def lines_iter() -> Iterator[str]: + # Streams data from the server in -chunks-, yields -lines-. + buffer = b"" + bytes_downloaded = 0 + retries = 0 + + headers = {} + while True: + # Request data starting from the last downloaded bytes + if bytes_downloaded > 0: + headers['Range'] = f'bytes={bytes_downloaded}-' + + try: + with requests.get(url, stream=True, timeout=(5, 30), headers=headers) as resp: + resp.raise_for_status() + for chunk in resp.iter_content(chunk_size=chunk_size): + chunk_length = len(chunk) + if chunk_length == 0: + continue + buffer += chunk + bytes_downloaded += chunk_length + + # Extract all lines from chunk + while True: + new_line_index = buffer.find(b'\n') + if new_line_index == -1: break + line = buffer[:new_line_index + 1] # extract line + buffer = buffer[new_line_index + 1 :] # clear read buffer + yield line.decode('utf-8-sig', errors='replace') + + if buffer: + # yield last line + yield buffer.decode('utf-8-sig', errors='replace') + return + except (ChunkedEncodingError, ConnectionError) as e: + # Trigger retry + if retries < max_retries: + retries += 1 + time.sleep(2 ** retries) + continue + raise + except Exception: + raise + + reader = csv.DictReader(lines_iter()) + + for row in reader: + yield row \ No newline at end of file diff --git a/specifyweb/backend/trees/utils.py b/specifyweb/backend/trees/utils.py index 506fc8610b6..621571c5304 100644 --- a/specifyweb/backend/trees/utils.py +++ b/specifyweb/backend/trees/utils.py @@ -1,15 +1,5 @@ -from typing import Any, Callable, Dict, Iterator, Optional, TypedDict, NotRequired -import json -import requests -import csv -import time -from requests.exceptions import ChunkedEncodingError, ConnectionError - -from django.db import transaction from django.db.models import Q, Count, Model -from specifyweb.backend.notifications.models import Message -from specifyweb.celery_tasks import LogErrorsTask, app import specifyweb.specify.models as spmodels from specifyweb.specify.datamodel import datamodel, Table @@ -107,289 +97,3 @@ def get_models(name: str): return tree_def_model, tree_rank_model, tree_node_model -class RankConfiguration(TypedDict): - name: str - title: NotRequired[str] - enforced: bool - infullname: bool - fullnameseparator: NotRequired[str] - rank: int # rank id - -def initialize_default_tree(tree_type: str, discipline, tree_name: str, rank_cfg: list[RankConfiguration], full_name_direction: int=1): - """Creates an initial empty tree.""" - with transaction.atomic(): - tree_def_model, tree_rank_model, tree_node_model = get_models(tree_type) - - # Uniquify name - tree_def = None - unique_tree_name = tree_name - if tree_def_model.objects.filter(name=tree_name).exists(): - i = 1 - while tree_def_model.objects.filter(name=f"{tree_name}_{i}").exists(): - i += 1 - unique_tree_name = f"{tree_name}_{i}" - - # Create tree definition - tree_def, _ = tree_def_model.objects.get_or_create( - name=unique_tree_name, - discipline=discipline, - fullnamedirection=full_name_direction - ) - - # Create tree ranks - treedefitems_bulk = [] - rank_id = 0 - for rank in rank_cfg: - treedefitems_bulk.append( - tree_rank_model( - treedef=tree_def, - name=rank.get('name'), - title=rank.get('title', rank['name'].title()), - rankid=int(rank.get('rank', rank_id)), - isenforced=rank.get('enforced', True), - isinfullname=rank.get('infullname', False), - fullnameseparator=rank.get('fullnameseparator', ' ') - ) - ) - rank_id += 10 - if treedefitems_bulk: - tree_rank_model.objects.bulk_create(treedefitems_bulk, ignore_conflicts=False) - - # Create root node - # TODO: Avoid having duplicated code from add_root endpoint - root_rank = tree_rank_model.objects.get(treedef=tree_def, rankid=0) - tree_node, _ = tree_node_model.objects.get_or_create( - name=TREE_ROOT_NODES.get(tree_type, "Root"), - fullname=TREE_ROOT_NODES.get(tree_type, "Root"), - nodenumber=1, - definition=tree_def, - definitionitem=root_rank, - parent=None - ) - - return tree_def.name - -class RankMappingConfiguration(TypedDict): - name: str - column: str - enforced: NotRequired[bool] - rank: NotRequired[int] - infullname: NotRequired[bool] - fullnameseparator: NotRequired[str] - fields: Dict[str, str] - -def add_default_tree_record(tree_type: str, row: dict, tree_name: str, tree_cfg: dict[str, RankMappingConfiguration]): - """ - Given one CSV row and a column mapping / rank configuration dictionary, - walk through the 'ranks' in order, creating or updating each tree record and linking - it to its parent. - """ - tree_def_model, tree_rank_model, tree_node_model = get_models(tree_type) - tree_def = tree_def_model.objects.get(name=tree_name) - parent = tree_node_model.objects.filter(definitionitem__rankid=0, definition=tree_def).first() - rank_id = 10 - - for rank_map in tree_cfg['ranks']: - rank_name = rank_map['name'] - fields_map = rank_map['fields'] - - record_name = row.get(rank_map.get('column', rank_name)) # Record's name is in the column. - - if not record_name: - continue # This row doesn't contain a record for this rank. - - defaults = {} - for model_field, csv_col in fields_map.items(): - if model_field == 'name': - continue - v = row.get(csv_col) - if v: - defaults[model_field] = v - - rank_title = rank_map.get('title', rank_name.capitalize()) - - # Get the rank by the column name. - # It should already exist by this point, but worst case it will be generated here. - treedef_item, _ = tree_rank_model.objects.get_or_create( - name=rank_name, - treedef=tree_def, - defaults={ - 'title': rank_title, - 'rankid': rank_id - } - ) - - # Create the record at this rank if it isn't already there. - obj = tree_node_model.objects.filter( - name=record_name, - fullname=record_name, - definition=tree_def, - definitionitem=treedef_item, - parent=parent, - ).first() - if obj is None: - data = { - 'name': record_name, - 'fullname': record_name, - 'definition': tree_def, - 'definitionitem': treedef_item, - 'parent': parent, - 'rankid': treedef_item.rankid, - **defaults - } - obj = tree_node_model(**data) - obj.save(skip_tree_extras=True) - - # if not taxon_obj and defaults: - # for f, v in defaults.items(): - # setattr(taxon_obj, f, v) - # taxon_obj.save() - - parent = obj - rank_id += 10 - -@app.task(base=LogErrorsTask, bind=True) -def create_default_tree_task(self, url: str, discipline_id: int, tree_discipline_name: str, specify_collection_id: int, - specify_user_id: int, tree_cfg: dict, row_count: Optional[int], initial_tree_name: str): - logger.info(f'starting task {str(self.request.id)}') - - specify_user = spmodels.Specifyuser.objects.get(id=specify_user_id) - discipline = spmodels.Discipline.objects.get(id=discipline_id) - tree_name = initial_tree_name # Name will be uniquified on tree creation - - Message.objects.create( - user=specify_user, - content=json.dumps({ - 'type': 'create-default-tree-starting', - 'name': initial_tree_name, - 'taskid': str(self.request.id), - 'collection_id': specify_collection_id, - }) - ) - - current = 0 - total = 1 - def progress(cur: int, additional_total: int=0) -> None: - nonlocal current, total - current += cur - total += additional_total - if current > total: - current = total - self.update_state(state='RUNNING', meta={'current': current, 'total': total}) - - try: - with transaction.atomic(): - tree_type = 'taxon' - if tree_discipline_name in SPECIFY_TREES: - # non-taxon tree - tree_type = tree_discipline_name - - # Create a new empty tree. Get rank configuration from the mapping. - full_name_direction = 1 - if tree_type in ('geologictimeperiod'): - full_name_direction = -1 - - rank_cfg = [{ - 'name': 'Root', - 'enforced': True, - 'rank': 0, - **tree_cfg.get('root', {}) - }] - auto_rank_id = 10 - for rank in tree_cfg['ranks']: - rank_cfg.append({ - 'name': rank['name'], - 'enforced': rank.get('enforced', True), - 'infullname': rank.get('infullname', False), - 'fullnameseparator': rank.get('fullnameseparator', ' '), - 'rank': rank.get('rank', auto_rank_id) - }) - auto_rank_id += 10 - tree_name = initialize_default_tree(tree_type, discipline, initial_tree_name, rank_cfg, full_name_direction) - - # Start importing CSV data - total_rows = 0 - if row_count: - total_rows = row_count-2 - progress(0, total_rows) - for row in stream_csv_from_url(url): - add_default_tree_record(tree_type, row, tree_name, tree_cfg) - progress(1, 0) - except Exception as e: - Message.objects.create( - user=specify_user, - content=json.dumps({ - 'type': 'create-default-tree-failed', - 'name': tree_name, - 'taskid': str(self.request.id), - 'collection_id': specify_collection_id, - # 'error': str(e) - }) - ) - raise - - Message.objects.create( - user=specify_user, - content=json.dumps({ - 'type': 'create-default-tree-completed', - 'name': tree_name, - 'taskid': str(self.request.id), - 'collection_id': specify_collection_id, - }) - ) - -def stream_csv_from_url(url: str) -> Iterator[Dict[str, str]]: - """ - Streams a taxon CSV from a URL. Yields each row. - """ - chunk_size = 8192 - max_retries = 10 - - def lines_iter() -> Iterator[str]: - # Streams data from the server in -chunks-, yields -lines-. - buffer = b"" - bytes_downloaded = 0 - retries = 0 - - headers = {} - while True: - # Request data starting from the last downloaded bytes - if bytes_downloaded > 0: - headers['Range'] = f'bytes={bytes_downloaded}-' - - try: - with requests.get(url, stream=True, timeout=(5, 30), headers=headers) as resp: - resp.raise_for_status() - for chunk in resp.iter_content(chunk_size=chunk_size): - chunk_length = len(chunk) - if chunk_length == 0: - continue - buffer += chunk - bytes_downloaded += chunk_length - - # Extract all lines from chunk - while True: - new_line_index = buffer.find(b'\n') - if new_line_index == -1: break - line = buffer[:new_line_index + 1] # extract line - buffer = buffer[new_line_index + 1 :] # clear read buffer - yield line.decode('utf-8-sig', errors='replace') - - if buffer: - # yield last line - yield buffer.decode('utf-8-sig', errors='replace') - return - except (ChunkedEncodingError, ConnectionError) as e: - # Trigger retry - if retries < max_retries: - retries += 1 - time.sleep(2 ** retries) - continue - raise - except Exception: - raise - - reader = csv.DictReader(lines_iter()) - - for row in reader: - yield row \ No newline at end of file diff --git a/specifyweb/backend/trees/views.py b/specifyweb/backend/trees/views.py index d8be1198b3e..eba1f68a17e 100644 --- a/specifyweb/backend/trees/views.py +++ b/specifyweb/backend/trees/views.py @@ -28,7 +28,8 @@ from specifyweb.backend.stored_queries.group_concat import group_concat from specifyweb.backend.notifications.models import Message -from specifyweb.backend.trees.utils import add_default_tree_record, create_default_tree_task, get_search_filters, stream_csv_from_url +from specifyweb.backend.trees.utils import get_search_filters +from specifyweb.backend.trees.defaults import create_default_tree_task from specifyweb.specify.utils.field_change_info import FieldChangeInfo from specifyweb.backend.trees.ranks import tree_rank_count from . import extras @@ -660,6 +661,10 @@ def has_tree_read_permission(tree: TREE_TABLE) -> bool: "type": "integer", "description": "The total number of rows contained in the CSV file. Only used for progress tracking." }, + "treeDefId": { + "type": "integer", + "description": "(optional) The ID of the existing tree to import into." + } }, "required": ["url", "disciplineName"], "oneOf": [ @@ -737,6 +742,9 @@ def create_default_tree_view(request): if not url: return http.JsonResponse({'error': 'Tree not found.'}, status=404) + # Import into an existing tree + tree_def_id = data.get('treeDefId') + # CSV mapping. Accept the mapping directly or a url to a JSON file containing the mapping. tree_cfg = data.get('mapping', None) mapping_url = data.get('mappingUrl', None) @@ -754,7 +762,7 @@ def create_default_tree_view(request): task_id = str(uuid4()) async_result = create_default_tree_task.apply_async( - args=[url, discipline.id, tree_discipline_name, request.specify_collection.id, request.specify_user.id, tree_cfg, row_count, tree_name], + args=[url, discipline.id, tree_discipline_name, request.specify_collection.id, request.specify_user.id, tree_cfg, row_count, tree_name, tree_def_id], task_id=f"create_default_tree_{tree_discipline_name}_{task_id}", taskid=task_id ) @@ -883,4 +891,4 @@ def abort_default_tree_creation(request, task_id: str) -> http.HttpResponse: return http.HttpResponse('', status=204) except Exception as e: - return http.JsonResponse({'error': str(e)}, status=400) \ No newline at end of file + return http.JsonResponse({'error': str(e)}, status=400) diff --git a/specifyweb/frontend/js_src/lib/components/Login/index.tsx b/specifyweb/frontend/js_src/lib/components/Login/index.tsx index 23bed97e967..ae4c47afef3 100644 --- a/specifyweb/frontend/js_src/lib/components/Login/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/Login/index.tsx @@ -51,8 +51,17 @@ export function Login(): JSX.Element { if (setupProgress === undefined) return ; - if (setupProgress.busy || (setupProgress.hasOwnProperty('resources') && Object.values(setupProgress.resources).includes(false))) { - return ; + if ( + setupProgress.busy || + (setupProgress.hasOwnProperty('resources') && + Object.values(setupProgress.resources).includes(false)) + ) { + return ( + + ); } return providers.length > 0 ? ( diff --git a/specifyweb/frontend/js_src/lib/components/Notifications/NotificationRenderers.tsx b/specifyweb/frontend/js_src/lib/components/Notifications/NotificationRenderers.tsx index 7548864725b..074dff359a3 100644 --- a/specifyweb/frontend/js_src/lib/components/Notifications/NotificationRenderers.tsx +++ b/specifyweb/frontend/js_src/lib/components/Notifications/NotificationRenderers.tsx @@ -367,7 +367,7 @@ export const notificationRenderers: IR<

{treeText.defaultTreeTaskStarted()}

{notification.payload.name} - ) + ); }, 'create-default-tree-failed'(notification) { return ( @@ -375,7 +375,7 @@ export const notificationRenderers: IR<

{treeText.defaultTreeTaskFailed()}

{notification.payload.name} - ) + ); }, 'create-default-tree-cancelled'(notification) { return ( @@ -383,7 +383,7 @@ export const notificationRenderers: IR<

{treeText.defaultTreeTaskCancelled()}

{notification.payload.name} - ) + ); }, 'create-default-tree-completed'(notification) { return ( @@ -391,7 +391,7 @@ export const notificationRenderers: IR<

{treeText.defaultTreeTaskCompleted()}

{notification.payload.name} - ) + ); }, default(notification) { console.error('Unknown notification type', { notification }); diff --git a/specifyweb/frontend/js_src/lib/components/SetupTool/SetupForm.tsx b/specifyweb/frontend/js_src/lib/components/SetupTool/SetupForm.tsx index 8747b774958..2b5627ed5c1 100644 --- a/specifyweb/frontend/js_src/lib/components/SetupTool/SetupForm.tsx +++ b/specifyweb/frontend/js_src/lib/components/SetupTool/SetupForm.tsx @@ -5,12 +5,19 @@ import React from 'react'; import type { LocalizedString } from 'typesafe-i18n'; +import { useBooleanState } from '../../hooks/useBooleanState'; import { commonText } from '../../localization/common'; +import { setupToolText } from '../../localization/setupTool'; +import { treeText } from '../../localization/tree'; import { userText } from '../../localization/user'; import { type RA } from '../../utils/types'; import { H3 } from '../Atoms'; +import { Button } from '../Atoms/Button'; import { Input, Label, Select } from '../Atoms/Form'; +import { Dialog } from '../Molecules/Dialog'; import { MIN_PASSWORD_LENGTH } from '../Security/SetPassword'; +import type { TaxonFileDefaultDefinition } from '../TreeView/CreateTree'; +import { PopulatedTreeList } from '../TreeView/CreateTree'; import type { FieldConfig, ResourceConfig } from './setupResources'; import { FIELD_MAX_LENGTH, resources } from './setupResources'; import type { ResourceFormData } from './types'; @@ -58,7 +65,7 @@ export function renderFormFieldFactory({ readonly currentStep: number; readonly handleChange: ( name: string, - newValue: LocalizedString | boolean + newValue: LocalizedString | TaxonFileDefaultDefinition | boolean ) => void; readonly temporaryFormData: ResourceFormData; readonly setTemporaryFormData: ( @@ -66,9 +73,13 @@ export function renderFormFieldFactory({ ) => void; readonly formRef: React.MutableRefObject; }) { + const [isTreeDialogOpen, handleTreeDialogOpen, handleTreeDialogClose] = + useBooleanState(false); + const renderFormField = ( field: FieldConfig, - parentName?: string + parentName?: string, + inTable: boolean = false ): JSX.Element => { const { name, @@ -79,11 +90,20 @@ export function renderFormFieldFactory({ options, fields, passwordRepeat, + width, + isTable, } = field; const fieldName = parentName === undefined ? name : `${parentName}.${name}`; - const colSpan = type === 'object' ? 2 : 1; + const colSpan = + width === undefined + ? type === 'object' + ? 'col-span-4' + : 'col-span-2' + : `col-span-${width}`; + + const verticalSpacing = width !== undefined && width < 2 ? '-mb-2' : 'mb-2'; const disciplineTypeValue = resources[currentStep].resourceName === 'discipline' @@ -94,10 +114,12 @@ export function renderFormFieldFactory({ fieldName === 'name' && (disciplineTypeValue === undefined || disciplineTypeValue === ''); + const showTreeSelector = getFormValue(formData, currentStep, 'preload'); + return ( -
+
{type === 'boolean' ? ( -
+
- {label} + {!inTable && label}
) : type === 'select' && Array.isArray(options) ? ( @@ -118,32 +140,30 @@ export function renderFormFieldFactory({ className="mb-4" key={`${resources[currentStep].resourceName}.${fieldName}`} > - - {label} - handleChange(fieldName, value)} + > + + {options.map((option) => ( + - {options.map((option) => ( - - ))} - - + ))} +
) : type === 'password' ? ( <> - {label} + {!inTable && label} - {passwordRepeat.label} + {!inTable && passwordRepeat.label} {label} - {fields ? renderFormFields(fields, name) : null} + {fields + ? renderFormFields(fields, fieldName, isTable === true) + : null}
+ ) : type === 'tree' ? ( + showTreeSelector ? ( + // Taxon tree selection + + {label} + + {setupToolText.selectATree()} + + {(() => { + // Display the selected tree + const selectedTree = getFormValue( + formData, + currentStep, + fieldName + ); + if (selectedTree && typeof selectedTree === 'object') { + const tree = selectedTree as TaxonFileDefaultDefinition; + return ( +
+
+
{tree.title}
+
+ {tree.description} +
+
{`${treeText.source()}: ${tree.src}`}
+
+
+ ); + } + return null; + })()} + {isTreeDialogOpen ? ( + + { + handleChange(fieldName, resource); + handleChange('preload', true); + handleTreeDialogClose(); + }} + /> + + ) : null} + + ) : null ) : ( - {label} + {!inTable && label} , parentName?: string) => ( -
- {fields.map((field) => renderFormField(field, parentName))} -
- ); + const renderFormFields = ( + fields: RA, + parentName?: string, + isTable: boolean = false + ): JSX.Element => { + if (isTable && fields.length > 0 && fields[0].fields) { + // Table format specifically for tree rank configuration + return ( +
+ + + + + {fields[0].fields.map((subField) => ( + + ))} + + + + {fields.map((field) => ( + + + {field.fields!.map((subField) => ( + + ))} + + ))} + +
+ {setupToolText.treeRanks()} + + {subField.label} +
+ {field.label} + + {renderFormField( + subField, + parentName === undefined + ? field.name + : `${parentName}.${field.name}`, + true + )} +
+
+ ); + } + // Otherwise, lay out fields normally + return ( +
+ {fields.map((field) => renderFormField(field, parentName))} +
+ ); + }; return { renderFormField, renderFormFields }; } diff --git a/specifyweb/frontend/js_src/lib/components/SetupTool/SetupOverview.tsx b/specifyweb/frontend/js_src/lib/components/SetupTool/SetupOverview.tsx index 27803308ffb..872457a40ac 100644 --- a/specifyweb/frontend/js_src/lib/components/SetupTool/SetupOverview.tsx +++ b/specifyweb/frontend/js_src/lib/components/SetupTool/SetupOverview.tsx @@ -46,7 +46,7 @@ export function SetupOverview({ if (field.type === 'object') { // Construct a sub list of properties field.fields?.map((child_field) => - fieldDisplay(child_field, field.name) + fieldDisplay(child_field, fieldName) ); return ( {field.fields?.map((child) => ( {fieldDisplay( child, @@ -81,8 +81,11 @@ export function SetupOverview({ (option) => String(option.value) === value ); value = match ? (match.label ?? match.value) : value; - } else if (field.type == 'boolean') { + } else if (field.type === 'boolean') { value = rawValue === true ? queryText.yes() : commonText.no(); + } else if (field.type === 'tree') { + value = + typeof rawValue === 'object' ? String(rawValue.title) : '-'; } return ( diff --git a/specifyweb/frontend/js_src/lib/components/SetupTool/index.tsx b/specifyweb/frontend/js_src/lib/components/SetupTool/index.tsx index b8495478b84..3fe879705d2 100644 --- a/specifyweb/frontend/js_src/lib/components/SetupTool/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/SetupTool/index.tsx @@ -8,8 +8,7 @@ import { setupToolText } from '../../localization/setupTool'; import { ajax } from '../../utils/ajax'; import { Http } from '../../utils/ajax/definitions'; import { type RA, localized } from '../../utils/types'; -import { Container, H2, H3 } from '../Atoms'; -import { Progress } from '../Atoms'; +import { Container, H2, H3, Progress } from '../Atoms'; import { Button } from '../Atoms/Button'; import { Form } from '../Atoms/Form'; import { dialogIcons } from '../Atoms/Icons'; @@ -17,6 +16,7 @@ import { Link } from '../Atoms/Link'; import { Submit } from '../Atoms/Submit'; import { LoadingContext } from '../Core/Contexts'; import { loadingBar } from '../Molecules'; +import type { TaxonFileDefaultDefinition } from '../TreeView/CreateTree'; import { checkFormCondition, renderFormFieldFactory } from './SetupForm'; import { SetupOverview } from './SetupOverview'; import type { FieldConfig, ResourceConfig } from './setupResources'; @@ -29,6 +29,8 @@ import type { } from './types'; import { flattenAllResources } from './utils'; +const SETUP_POLLING_INTERVAL = 3000; + export const stepOrder: RA = [ 'institution', 'storageTreeDef', @@ -60,14 +62,17 @@ function findNextStep( return currentStep; } -function useFormDefaults( +function applyFormDefaults( resource: ResourceConfig, setFormData: (data: ResourceFormData) => void, currentStep: number ): void { const resourceName = resources[currentStep].resourceName; const defaultFormData: ResourceFormData = {}; - const applyFieldDefaults = (field: FieldConfig, parentName?: string) => { + const applyFieldDefaults = ( + field: FieldConfig, + parentName?: string + ): void => { const fieldName = parentName === undefined ? field.name : `${parentName}.${field.name}`; if (field.type === 'object' && field.fields !== undefined) @@ -75,7 +80,7 @@ function useFormDefaults( if (field.default !== undefined) defaultFormData[fieldName] = field.default; }; resource.fields.forEach((field) => applyFieldDefaults(field)); - setFormData((previous: any) => ({ + setFormData((previous: ResourceFormData) => ({ ...previous, [resourceName]: { ...defaultFormData, @@ -105,14 +110,14 @@ export function SetupTool({ const [currentStep, setCurrentStep] = React.useState(0); React.useEffect(() => { - useFormDefaults(resources[currentStep], setFormData, currentStep); + applyFormDefaults(resources[currentStep], setFormData, currentStep); }, [currentStep]); const [saveBlocked, setSaveBlocked] = React.useState(false); React.useEffect(() => { const formValid = formRef.current?.checkValidity(); - setSaveBlocked(!formValid); + setSaveBlocked(formValid !== true); }, [formData, temporaryFormData, currentStep]); const SubmitComponent = saveBlocked ? Submit.Danger : Submit.Save; @@ -122,7 +127,9 @@ export function SetupTool({ ); // Is the database currrently being created? - const [inProgress, setInProgress] = React.useState(setupProgress.busy); + const [inProgress, setInProgress] = React.useState( + setupProgress.busy + ); const nextIncompleteStep = stepOrder.findIndex( (resourceName) => !setupProgress.resources[resourceName] ); @@ -146,7 +153,7 @@ export function SetupTool({ console.error('Failed to fetch setup progress:', error); return undefined; }), - 3000 + SETUP_POLLING_INTERVAL ); return () => clearInterval(interval); @@ -154,7 +161,7 @@ export function SetupTool({ const loading = React.useContext(LoadingContext); - const startSetup = async (data: ResourceFormData): Promise => + const startSetup = async (data: ResourceFormData): Promise => ajax('/setup_tool/setup_database/create/', { method: 'POST', headers: { @@ -182,9 +189,9 @@ export function SetupTool({ const handleChange = ( name: string, - newValue: LocalizedString | boolean + newValue: LocalizedString | TaxonFileDefaultDefinition | boolean ): void => { - setFormData((previous) => { + setFormData((previous: ResourceFormData) => { const resourceName = resources[currentStep].resourceName; const previousResourceData = previous[resourceName]; const updates: Record = { @@ -197,7 +204,7 @@ export function SetupTool({ (option) => option.value === newValue ); updates.name = matchingType - ? matchingType.label ?? String(matchingType.value) + ? (matchingType.label ?? String(matchingType.value)) : ''; } @@ -219,7 +226,7 @@ export function SetupTool({ loading( startSetup(formData) .then((data) => { - setSetupProgress(data.setup_progress as SetupProgress); + setSetupProgress(data.setup_progress); setInProgress(true); }) .catch((error) => { @@ -252,7 +259,11 @@ export function SetupTool({
- +

{setupToolText.guidedSetup()}

diff --git a/specifyweb/frontend/js_src/lib/components/SetupTool/setupResources.ts b/specifyweb/frontend/js_src/lib/components/SetupTool/setupResources.ts index 819456039fa..4ba5d2bb68e 100644 --- a/specifyweb/frontend/js_src/lib/components/SetupTool/setupResources.ts +++ b/specifyweb/frontend/js_src/lib/components/SetupTool/setupResources.ts @@ -1,7 +1,9 @@ import type { LocalizedString } from 'typesafe-i18n'; +import { commonText } from '../../localization/common'; import { formsText } from '../../localization/forms'; import { setupToolText } from '../../localization/setupTool'; +import { statsText } from '../../localization/stats'; import type { RA } from '../../utils/types'; // Default for max field length. @@ -10,7 +12,6 @@ export const FIELD_MAX_LENGTH = 64; export type ResourceConfig = { readonly resourceName: string; readonly label: LocalizedString; - readonly endpoint: string; readonly description?: LocalizedString; readonly condition?: Record< string, @@ -28,7 +29,13 @@ type Option = { export type FieldConfig = { readonly name: string; readonly label: string; - readonly type?: 'boolean' | 'object' | 'password' | 'select' | 'text'; + readonly type?: + | 'boolean' + | 'object' + | 'password' + | 'select' + | 'text' + | 'tree'; readonly required?: boolean; readonly default?: boolean | number | string; readonly description?: string; @@ -40,6 +47,8 @@ export type FieldConfig = { readonly description: string; }; readonly maxLength?: number; + readonly width?: number; + readonly isTable?: boolean; }; // Discipline list from backend/context/app_resource.py @@ -57,14 +66,19 @@ export const disciplineTypeOptions = [ { value: 'geology', label: 'Geology' }, ]; -// Must match config/backstop/uiformatters.xml -// TODO: Fetch uiformatters.xml from the backend instead and use UIFormatter.placeholder +/* + * Must match config/backstop/uiformatters.xml + * TODO: Fetch uiformatters.xml from the backend instead and use UIFormatter.placeholder + */ const currentYear = new Date().getFullYear(); const catalogNumberFormats = [ { value: 'CatalogNumber', label: `CatalogNumber (${currentYear}-######)` }, - { value: 'CatalogNumberAlphaNumByYear', label: `CatalogNumberAlphaNumByYear (${currentYear}-######)` }, + { + value: 'CatalogNumberAlphaNumByYear', + label: `CatalogNumberAlphaNumByYear (${currentYear}-######)`, + }, { value: 'CatalogNumberNumeric', label: 'CatalogNumberNumeric (#########)' }, - { value: 'CatalogNumberString', label: 'None' }, + { value: 'CatalogNumberString', label: commonText.none() }, ]; const fullNameDirections = [ @@ -72,12 +86,64 @@ const fullNameDirections = [ { value: -1, label: formsText.reverse() }, ]; +function generateTreeRankFields( + rankNames: RA, + enabled: RA, + enforced: RA, + inFullName: RA, + separator: string = ' ' +): RA { + return rankNames.map( + (rankName, index) => + ({ + name: rankName.toLowerCase(), + label: rankName, + type: 'object', + fields: [ + { + name: 'include', + label: setupToolText.include(), + description: setupToolText.includeDescription(), + type: 'boolean', + default: index === 0 || enabled.includes(rankName), + required: index === 0, + width: 1, + }, + { + name: 'enforced', + label: setupToolText.enforced(), + description: setupToolText.enforcedDescription(), + type: 'boolean', + default: index === 0 || enforced.includes(rankName), + required: index === 0, + width: 1, + }, + { + name: 'infullname', + label: setupToolText.inFullName(), + description: setupToolText.inFullNameDescription(), + type: 'boolean', + default: inFullName.includes(rankName), + width: 1, + }, + { + name: 'fullnameseparator', + label: setupToolText.fullNameSeparator(), + description: setupToolText.fullNameSeparatorDescription(), + type: 'text', + default: separator, + width: 1, + }, + ], + }) as FieldConfig + ); +} + export const resources: RA = [ { resourceName: 'institution', label: setupToolText.institution(), description: setupToolText.institutionDescription(), - endpoint: '/setup_tool/institution/create/', documentationUrl: 'https://discourse.specifysoftware.org/t/guided-setup/3234', fields: [ @@ -151,47 +217,37 @@ export const resources: RA = [ description: setupToolText.institutionIsAccessionGlobalDescription(), type: 'boolean', }, - /* - * { - * name: 'isSingleGeographyTree', - * label: setupToolText_institutionIsSingleGeographyTree(), // underscore in comment to avoid failing test - * description: - * setupToolText_institutionIsSingleGeographyTreeDescription(), - * type: 'boolean', - * default: false, - * }, - */ ], }, { resourceName: 'storageTreeDef', label: setupToolText.storageTree(), - endpoint: '/setup_tool/storagetreedef/create/', fields: [ { name: 'ranks', label: setupToolText.treeRanks(), required: false, type: 'object', + isTable: true, // TODO: Rank fields should be generated from a .json file. - fields: [ - { - name: '0', - label: 'Site', - type: 'boolean', - default: true, - required: true, - }, - { name: '100', label: 'Building', type: 'boolean', default: true }, - { name: '150', label: 'Collection', type: 'boolean', default: true }, - { name: '200', label: 'Room', type: 'boolean', default: true }, - { name: '250', label: 'Aisle', type: 'boolean', default: true }, - { name: '300', label: 'Cabinet', type: 'boolean', default: true }, - { name: '350', label: 'Shelf', type: 'boolean' }, - { name: '400', label: 'Box', type: 'boolean' }, - { name: '450', label: 'Rack', type: 'boolean' }, - { name: '500', label: 'Vial', type: 'boolean' }, - ], + fields: generateTreeRankFields( + [ + 'Site', + 'Building', + 'Collection', + 'Room', + 'Aisle', + 'Cabinet', + 'Shelf', + 'Box', + 'Rack', + 'Vial', + ], + ['Site', 'Building', 'Collection', 'Room', 'Aisle', 'Cabinet'], + [], + ['Building', 'Collection', 'Room', 'Aisle', 'Cabinet'], + ', ' + ), }, // TODO: This should be name direction. Each rank should have configurable formats, too., { @@ -207,7 +263,6 @@ export const resources: RA = [ { resourceName: 'division', label: setupToolText.division(), - endpoint: '/setup_tool/division/create/', fields: [ { name: 'name', label: setupToolText.divisionName(), required: true }, { name: 'abbrev', label: setupToolText.divisionAbbrev(), required: true }, @@ -216,7 +271,6 @@ export const resources: RA = [ { resourceName: 'discipline', label: setupToolText.discipline(), - endpoint: '/setup_tool/discipline/create/', fields: [ { name: 'type', @@ -236,26 +290,20 @@ export const resources: RA = [ { resourceName: 'geographyTreeDef', label: setupToolText.geographyTree(), - endpoint: '/setup_tool/geographytreedef/create/', fields: [ { name: 'ranks', label: setupToolText.treeRanks(), required: false, type: 'object', - fields: [ - { - name: '0', - label: 'Earth', - type: 'boolean', - default: true, - required: true, - }, - { name: '100', label: 'Continent', type: 'boolean', default: true }, - { name: '200', label: 'Country', type: 'boolean', default: true }, - { name: '300', label: 'State', type: 'boolean', default: true }, - { name: '400', label: 'County', type: 'boolean', default: true }, - ], + isTable: true, + fields: generateTreeRankFields( + ['Earth', 'Continent', 'Country', 'State', 'County'], + ['Earth', 'Continent', 'Country', 'State', 'County'], + ['Earth', 'Continent', 'Country', 'State', 'County'], + ['Country', 'State', 'County'], + ', ' + ), }, { name: 'fullNameDirection', @@ -265,69 +313,94 @@ export const resources: RA = [ required: true, default: fullNameDirections[0].value.toString(), }, - /* - * { - * name: 'default', - * label: setupToolText_defaultTree(), // underscore in comment to avoid failing test - * type: 'boolean', - * }, - */ + { + name: 'preload', + label: setupToolText.preloadTree(), + description: setupToolText.preloadTreeDescription(), + type: 'boolean', + width: 4, + }, ], }, { resourceName: 'taxonTreeDef', label: setupToolText.taxonTree(), - endpoint: '/setup_tool/taxontreedef/create/', fields: [ - { - name: 'ranks', - label: setupToolText.treeRanks(), - required: false, - type: 'object', - fields: [ - { - name: '0', - label: 'Life', - type: 'boolean', - default: true, - required: true, - }, - { name: '10', label: 'Kingdom', type: 'boolean', default: true }, - { name: '30', label: 'Phylum', type: 'boolean', default: true }, - { name: '40', label: 'Subphylum', type: 'boolean', default: false }, - { name: '60', label: 'Class', type: 'boolean', default: true }, - { name: '70', label: 'Subclass', type: 'boolean', default: false }, - { name: '90', label: 'Superorder', type: 'boolean', default: false }, - { name: '100', label: 'Order', type: 'boolean', default: true }, - { name: '140', label: 'Family', type: 'boolean', default: true }, - { name: '150', label: 'Subfamily', type: 'boolean', default: false }, - { name: '180', label: 'Genus', type: 'boolean', default: true }, - { name: '220', label: 'Species', type: 'boolean', default: true }, - { name: '230', label: 'Subspecies', type: 'boolean', default: false }, - ], - }, - { - name: 'fullNameDirection', - label: setupToolText.fullNameDirection(), - type: 'select', - options: fullNameDirections, - required: true, - default: fullNameDirections[0].value.toString(), - }, /* - * TODO: Select which Taxon tree to import (Re-use dialog from default tree creation in tree viewer) * { - * name: 'default', - * label: setupToolText_defaultTree(), // underscore in comment to avoid failing test - * type: 'boolean', + * name: 'ranks', + * label: setupToolText.treeRanks(), + * required: false, + * type: 'object', + * isTable: true, + * fields: generateTreeRankFields( + * [ + * 'Life', + * 'Kingdom', + * 'Phylum', + * 'Subphylum', + * 'Class', + * 'Subclass', + * 'Superorder', + * 'Order', + * 'Family', + * 'Subfamily', + * 'Genus', + * 'Species', + * 'Subspecies', + * ], + * [ + * 'Life', + * 'Kingdom', + * 'Phylum', + * 'Class', + * 'Order', + * 'Family', + * 'Genus', + * 'Species', + * ], + * [ + * 'Life', + * 'Kingdom', + * 'Phylum', + * 'Class', + * 'Order', + * 'Family', + * 'Genus', + * 'Species', + * ], + * ['Genus', 'Species'], + * ' ' + * ), + * }, + * { + * name: 'fullNameDirection', + * label: setupToolText.fullNameDirection(), + * type: 'select', + * options: fullNameDirections, + * required: true, + * default: fullNameDirections[0].value.toString(), * }, */ + { + name: 'preload', + label: setupToolText.preloadTree(), + description: setupToolText.preloadTreeDescription(), + type: 'boolean', + width: 4, + }, + { + name: 'preloadFile', + label: setupToolText.treeToPreload(), + description: setupToolText.preloadTreeDescription(), + type: 'tree', + width: 4, + }, ], }, { resourceName: 'collection', - label: setupToolText.collection(), - endpoint: '/setup_tool/collection/create/', + label: statsText.collection(), fields: [ { name: 'collectionName', @@ -354,7 +427,6 @@ export const resources: RA = [ { resourceName: 'specifyUser', label: setupToolText.specifyUser(), - endpoint: '/setup_tool/specifyuser/create/', fields: [ { name: 'firstname', diff --git a/specifyweb/frontend/js_src/lib/components/SetupTool/utils.ts b/specifyweb/frontend/js_src/lib/components/SetupTool/utils.ts index f46de738669..9853087e175 100644 --- a/specifyweb/frontend/js_src/lib/components/SetupTool/utils.ts +++ b/specifyweb/frontend/js_src/lib/components/SetupTool/utils.ts @@ -1,11 +1,31 @@ // Turn 'table.field' keys to nested objects to send to the backend +function setNested( + object: Record, + path: string, + value: any +): void { + const index = path.indexOf('.'); + if (index === -1) { + object[path] = value; + return; + } + const head = path.slice(0, index); + const rest = path.slice(index + 1); + if ( + object[head] === undefined || + typeof object[head] !== 'object' || + Array.isArray(object[head]) + ) { + object[head] = {}; + } + setNested(object[head], rest, value); +} + function flattenToNested(data: Record): Record { const result: Record = {}; Object.entries(data).forEach(([key, value]) => { if (key.includes('.')) { - const [prefix, field] = key.split('.', 2); - result[prefix] ||= {}; - result[prefix][field] = value; + setNested(result, key, value); } else { result[key] = value; } @@ -13,7 +33,9 @@ function flattenToNested(data: Record): Record { return result; } -export function flattenAllResources(data: Record): Record { +export function flattenAllResources( + data: Record +): Record { const result: Record = {}; Object.entries(data).forEach(([key, value]) => { result[key] = flattenToNested(value); diff --git a/specifyweb/frontend/js_src/lib/components/SystemConfigurationTool/Hierarchy.tsx b/specifyweb/frontend/js_src/lib/components/SystemConfigurationTool/Hierarchy.tsx index 700ad912c0f..b5e02510671 100644 --- a/specifyweb/frontend/js_src/lib/components/SystemConfigurationTool/Hierarchy.tsx +++ b/specifyweb/frontend/js_src/lib/components/SystemConfigurationTool/Hierarchy.tsx @@ -30,6 +30,7 @@ import { stepOrder } from '../SetupTool'; import { renderFormFieldFactory } from '../SetupTool/SetupForm'; import { resources } from '../SetupTool/setupResources'; import type { ResourceFormData } from '../SetupTool/types'; +import type { TaxonFileDefaultDefinition } from '../TreeView/CreateTree'; import { CollapsibleSection } from './CollapsibleSection'; import type { InstitutionData } from './Utils'; @@ -143,7 +144,10 @@ function HierarchyDiagram({ const zoomBehavior = d3Zoom() .scaleExtent([0.5, 8]) - .wheelDelta((wheelEvent: WheelEvent) => -wheelEvent.deltaY * (wheelEvent.deltaMode === 1 ? 0.05 : 0.002)) + .wheelDelta( + (wheelEvent: WheelEvent) => + -wheelEvent.deltaY * (wheelEvent.deltaMode === 1 ? 0.05 : 0.002) + ) .on('zoom', (event) => { setTransform(event.transform); }); @@ -245,14 +249,16 @@ function HierarchyDiagram({ width={NODE_WIDTH} y={6} /> - +
{node.data.name}
-
- {textByKind[node.data.kind]} -
+
{textByKind[node.data.kind]}
 @@ -273,7 +279,13 @@ type DialogFormProps = { readonly refreshAllInfo: () => Promise; }; -function DialogForm({ open, onClose, title, step, refreshAllInfo }: DialogFormProps) { +function DialogForm({ + open, + onClose, + title, + step, + refreshAllInfo, +}: DialogFormProps) { const id = useId('config-tool'); if (!open) return null; @@ -287,7 +299,10 @@ function DialogForm({ open, onClose, title, step, refreshAllInfo }: DialogFormPr const [temporaryFormData, setTemporaryFormData] = React.useState({}); - const handleChange = (name: string, newValue: LocalizedString | boolean) => { + const handleChange = ( + name: string, + newValue: LocalizedString | TaxonFileDefaultDefinition | boolean + ) => { const resourceName = resources[5].resourceName; setFormData((previous) => ({ ...previous, @@ -439,7 +454,10 @@ export function Hierarchy({ className="flex items-center gap-2 flex-wrap bg-[color:var(--background)] rounded px-2 py-1" key={collection.id} > -

{`${tableLabel('Collection')}:`}

+

{`${tableLabel('Collection')}:`}

{collection.name}

{handleEditResource( @@ -467,7 +485,10 @@ export function Hierarchy({
-

{`${tableLabel('Discipline')}:`}

+

{`${tableLabel('Discipline')}:`}

{discipline.name}

@@ -564,7 +585,10 @@ export function Hierarchy({
-

{`${tableLabel('Division')}:`}

+

{`${tableLabel('Division')}:`}

{division.name}

@@ -613,7 +637,10 @@ export function Hierarchy({ title={
-

{`${tableLabel('Institution')}:`}

+

{`${tableLabel('Institution')}:`}

{institution.name}

diff --git a/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx b/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx index 690e72402e4..c15db718789 100644 --- a/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx +++ b/specifyweb/frontend/js_src/lib/components/TreeView/CreateTree.tsx @@ -28,7 +28,7 @@ import { userInformation } from '../InitialContext/userInformation'; import { Dialog } from '../Molecules/Dialog'; import { defaultTreeDefs } from './defaults'; -type TaxonFileDefaultDefinition = { +export type TaxonFileDefaultDefinition = { readonly discipline: string; readonly title: string; readonly coverage: string; @@ -39,15 +39,28 @@ type TaxonFileDefaultDefinition = { readonly rows: number; readonly description: string; }; -type TaxonFileDefaultList = RA; +export type TaxonFileDefaultList = RA; type TreeCreationInfo = { readonly message: string; readonly task_id?: string; -} +}; type TreeCreationProgressInfo = { readonly taskstatus: string; readonly taskprogress: any; readonly taskid: string; +}; + +export async function fetchDefaultTrees(): Promise { + const response = await fetch( + 'https://files.specifysoftware.org/taxonfiles/taxonfiles.json' + ); + if (!response.ok) { + throw new Error( + `Failed to fetch default trees: ${response.status} ${response.statusText}` + ); + } + const data = await response.json(); + return data as TaxonFileDefaultList; } export function CreateTree< @@ -64,33 +77,22 @@ export function CreateTree< const loading = React.useContext(LoadingContext); const [isActive, setIsActive] = React.useState(0); - const [isTreeCreationStarted, setIsTreeCreationStarted] = React.useState(false); - const [treeCreationTaskId, setTreeCreationTaskId] = React.useState(undefined); + const [isTreeCreationStarted, setIsTreeCreationStarted] = + React.useState(false); + const [treeCreationTaskId, setTreeCreationTaskId] = React.useState< + string | undefined + >(undefined); const [selectedResource, setSelectedResource] = React.useState< SpecifyResource | undefined >(undefined); - const [treeOptions, setTreeOptions] = React.useState< - TaxonFileDefaultList | undefined - >(undefined); - - // Fetch list of available default trees. - React.useEffect(() => { - fetch('https://files.specifysoftware.org/taxonfiles/taxonfiles.json') - .then(async (response) => response.json()) - .then((data: TaxonFileDefaultList) => { - setTreeOptions(data); - }) - .catch((error) => { - console.error('Failed to fetch tree options:', error); - }); - }, []); - const connectedCollection = getSystemInfo().collection; // Start default tree creation - const handleClick = async (resource: TaxonFileDefaultDefinition): Promise => { + const handleClick = async ( + resource: TaxonFileDefaultDefinition + ): Promise => { setIsTreeCreationStarted(true); return ajax('/trees/create_default_tree/', { method: 'POST', @@ -109,7 +111,10 @@ export function CreateTree< console.log(`${resource.title} tree created successfully:`, data); } else if (status === Http.ACCEPTED) { // Tree is being created in the background. - console.log(`${resource.title} tree creation started successfully:`, data); + console.log( + `${resource.title} tree creation started successfully:`, + data + ); setTreeCreationTaskId(data.task_id); } }) @@ -117,7 +122,7 @@ export function CreateTree< console.error(`Request failed for ${resource.file}:`, error); throw error; }); - } + }; const handleClickEmptyTree = ( resource: DeepPartial> @@ -158,44 +163,14 @@ export function CreateTree< >
-
    -

    {treeText.populatedTrees()}

    - {treeOptions === undefined - ? undefined - : treeOptions.map((resource, index) => ( -
  • - { - loading( - handleClick(resource).catch(console.error) - ); - }} - > - {localized(resource.title)} - -
    - {resource.description} -
    -
    - {`Source: ${resource.src}`} -
    -
    • - ))} -
+ { + loading(handleClick(resource).catch(console.error)); + }} + />
-
    -

    {treeText.emptyTrees()}

    - {defaultTreeDefs.map((resource, index) => ( -
  • - handleClickEmptyTree(resource)} - > - {localized(resource.name)} - -
    • - ))} -
+
<> @@ -232,6 +207,169 @@ export function CreateTree< ); } +export function ImportTree({ + tableName, + treeDefId, +}: { + readonly tableName: SCHEMA['tableName']; + readonly treeDefId: number; +}): JSX.Element { + const loading = React.useContext(LoadingContext); + const [isActive, setIsActive] = React.useState(0); + const [isTreeCreationStarted, setIsTreeCreationStarted] = + React.useState(false); + const [treeCreationTaskId, setTreeCreationTaskId] = React.useState< + string | undefined + >(undefined); + + const connectedCollection = getSystemInfo().collection; + + const handleClick = async ( + resource: TaxonFileDefaultDefinition + ): Promise => { + setIsTreeCreationStarted(true); + return ajax('/trees/create_default_tree/', { + method: 'POST', + headers: { Accept: 'application/json' }, + body: { + url: resource.file, + mappingUrl: resource.mappingFile, + collection: connectedCollection, + disciplineName: resource.discipline, + rowCount: resource.rows, + treeName: resource.title, + treeDefId, + }, + }) + .then(({ data, status }) => { + if (status === Http.OK) { + console.log(`${resource.title} tree created successfully:`, data); + } else if (status === Http.ACCEPTED) { + // Tree is being created in the background. + console.log( + `${resource.title} tree creation started successfully:`, + data + ); + setTreeCreationTaskId(data.task_id); + } + }) + .catch((error) => { + console.error(`Request failed for ${resource.file}:`, error); + throw error; + }); + }; + + return ( + <> + {tableName === 'Taxon' && userInformation.isadmin ? ( + { + setIsActive(1); + }} + /> + ) : null} + {isActive === 1 ? ( + + {commonText.cancel()} + + } + header={commonText.import()} + onClose={() => setIsActive(0)} + > +
+ { + loading(handleClick(resource).catch(console.error)); + }} + /> +
+ <> + {isTreeCreationStarted && treeCreationTaskId ? ( + { + setIsTreeCreationStarted(false); + setTreeCreationTaskId(undefined); + setIsActive(0); + }} + onStopped={() => { + setIsTreeCreationStarted(false); + setTreeCreationTaskId(undefined); + }} + /> + ) : undefined} + + + ) : null} + + ); +} + +function EmptyTreeList({ + handleClick, +}: { + readonly handleClick: ( + resource: DeepPartial> + ) => void; +}): JSX.Element { + return ( +
    +

    {treeText.emptyTrees()}

    + {defaultTreeDefs.map((resource, index) => ( +
  • + handleClick(resource)}> + {localized(resource.name)} + +
    • + ))} +
+ ); +} + +export function PopulatedTreeList({ + handleClick, +}: { + readonly handleClick: (resource: TaxonFileDefaultDefinition) => void; +}): JSX.Element { + const [treeOptions, setTreeOptions] = React.useState< + TaxonFileDefaultList | undefined + >(undefined); + + // Fetch list of available default trees. + React.useEffect(() => { + fetchDefaultTrees() + .then((data) => setTreeOptions(data)) + .catch((error) => { + console.error('Failed to fetch tree options:', error); + }); + }, []); + + return ( +
    +

    {treeText.populatedTrees()}

    + {treeOptions === undefined + ? undefined + : treeOptions.map((resource, index) => ( +
  • + handleClick(resource)}> + {localized(resource.title)} + +
    + {resource.description} +
    +
    + {`${treeText.source()}: ${resource.src}`} +
    +
    • + ))} +
+ ); +} + export function TreeCreationProgressDialog({ taskId, onClose, @@ -246,40 +384,38 @@ export function TreeCreationProgressDialog({ const [progressTotal, setProgressTotal] = React.useState(1); const handleStop = async (): Promise => { - ping( - `/trees/create_default_tree/abort/${taskId}/`, - { - method: 'POST', - body: {}, + ping(`/trees/create_default_tree/abort/${taskId}/`, { + method: 'POST', + body: {}, + }).then((status) => { + if (status === Http.NO_CONTENT) { + onStopped(); } - ) - .then((status) => { - if (status === Http.NO_CONTENT) { - onStopped(); - } - }) - } + }); + }; // Poll for tree creation progress React.useEffect(() => { const interval = setInterval( async () => - ajax(`/trees/create_default_tree/status/${taskId}/`, { - method: 'GET', - headers: { Accept: 'application/json' }, - errorMode: 'silent', - }) - .then(({ data }) => { - if (data.taskstatus === 'RUNNING') { - setProgress(data.taskprogress.current ?? 0); - setProgressTotal(data.taskprogress.total ?? 1); - } else if (data.taskstatus === 'FAILURE') { - onStopped(); - throw data.taskprogress; - } else if (data.taskstatus === 'SUCCESS') { - globalThis.location.reload(); - } - }), + ajax( + `/trees/create_default_tree/status/${taskId}/`, + { + method: 'GET', + headers: { Accept: 'application/json' }, + errorMode: 'silent', + } + ).then(({ data }) => { + if (data.taskstatus === 'RUNNING') { + setProgress(data.taskprogress.current ?? 0); + setProgressTotal(data.taskprogress.total ?? 1); + } else if (data.taskstatus === 'FAILURE') { + onStopped(); + throw data.taskprogress; + } else if (data.taskstatus === 'SUCCESS') { + globalThis.location.reload(); + } + }), 5000 ); return () => clearInterval(interval); @@ -289,7 +425,11 @@ export function TreeCreationProgressDialog({ - {loading(handleStop())}}> + { + loading(handleStop()); + }} + > {commonText.cancel()} @@ -312,4 +452,4 @@ export function TreeCreationProgressDialog({ {treeText.defaultTreeCreationStartedDescription()} ); -} \ No newline at end of file +} diff --git a/specifyweb/frontend/js_src/lib/components/TreeView/Tree.tsx b/specifyweb/frontend/js_src/lib/components/TreeView/Tree.tsx index b343a725731..d92315976c1 100644 --- a/specifyweb/frontend/js_src/lib/components/TreeView/Tree.tsx +++ b/specifyweb/frontend/js_src/lib/components/TreeView/Tree.tsx @@ -23,6 +23,7 @@ import { hasTablePermission } from '../Permissions/helpers'; import { useHighContrast } from '../Preferences/Hooks'; import { userPreferences } from '../Preferences/userPreferences'; import { AddRank } from './AddRank'; +import { ImportTree } from './CreateTree'; import type { Conformations, Row, Stats } from './helpers'; import { fetchStats } from './helpers'; import { TreeRow } from './Row'; @@ -218,11 +219,16 @@ export function Tree<
{rows.length === 0 ? ( - +
+ + {treeDefId ? ( + + ) : null} +
) : undefined}
    {rows.map((row, index) => ( diff --git a/specifyweb/frontend/js_src/lib/localization/setupTool.ts b/specifyweb/frontend/js_src/lib/localization/setupTool.ts index 7fceb81c5ba..28af2ac82ad 100644 --- a/specifyweb/frontend/js_src/lib/localization/setupTool.ts +++ b/specifyweb/frontend/js_src/lib/localization/setupTool.ts @@ -95,7 +95,7 @@ export const setupToolText = createDictionary({ 'en-us': 'The city where the institution is located.', }, addressState: { - 'en-us': 'State', + 'en-us': 'Province/State', }, addressStateDescription: { 'en-us': 'The state or province.', @@ -126,6 +126,45 @@ export const setupToolText = createDictionary({ fullNameDirection: { 'en-us': 'Full Name Direction', }, + preloadTree: { + 'en-us': 'Populate tree with default records', + }, + preloadTreeDescription: { + 'en-us': 'Download default records for this tree.', + }, + treeToPreload: { + 'en-us': 'Tree to download:', + }, + selectATree: { + 'en-us': 'Select a tree', + }, + include: { + 'en-us': 'Include', + }, + includeDescription: { + 'en-us': 'Include places the Level in the tree definition.', + }, + enforced: { + 'en-us': 'Enforced', + }, + enforcedDescription: { + 'en-us': + 'Is Enforced ensures that the level can not be skipped when adding nodes lower down the tree.', + }, + inFullName: { + 'en-us': 'In Full Name', + }, + inFullNameDescription: { + 'en-us': + 'Is in Full Name includes the level when building a full name expression, which can be queried and used in reports.', + }, + fullNameSeparator: { + 'en-us': 'Separator', + }, + fullNameSeparatorDescription: { + 'en-us': + 'Separator refers to the character that separates the levels when displaying the full name.', + }, // Storage Tree storageTree: { @@ -139,11 +178,6 @@ export const setupToolText = createDictionary({ taxonTree: { 'en-us': 'Taxon Tree', }, - /* - * DefaultTree: { - * 'en-us': 'Pre-load Default Tree' - * }, - */ // Division division: { @@ -168,9 +202,6 @@ export const setupToolText = createDictionary({ }, // Collection - collection: { - 'en-us': 'Collection', - }, collectionName: { 'en-us': 'Collection Name', }, @@ -214,8 +245,7 @@ export const setupToolText = createDictionary({ 'en-us': 'Last Name', }, specifyUserLastNameDescription: { - 'en-us': - 'The last name of the agent associated with the account. Optional.', + 'en-us': 'The last name of the agent associated with the account.', }, taxonTreeSetUp: { diff --git a/specifyweb/frontend/js_src/lib/localization/tree.ts b/specifyweb/frontend/js_src/lib/localization/tree.ts index b5613d643d9..1891fd5e8a7 100644 --- a/specifyweb/frontend/js_src/lib/localization/tree.ts +++ b/specifyweb/frontend/js_src/lib/localization/tree.ts @@ -736,6 +736,9 @@ export const treeText = createDictionary({ 'en-us': 'Creating tree record {current:number|formatted}/{total:number|formatted}', }, + source: { + 'en-us': 'Source', + }, treeManagement: { 'en-us': 'Tree Management', 'de-ch': 'Baumpflege', diff --git a/specifyweb/specify/migration_utils/update_schema_config.py b/specifyweb/specify/migration_utils/update_schema_config.py index 0507398b969..eb22e401520 100644 --- a/specifyweb/specify/migration_utils/update_schema_config.py +++ b/specifyweb/specify/migration_utils/update_schema_config.py @@ -84,7 +84,6 @@ class FieldSchemaConfig(NamedTuple): def uncapitilize(string: str) -> str: return string.lower() if len(string) <= 1 else string[0].lower() + string[1:] - def update_table_schema_config_with_defaults( table_name, discipline_id: int,