From 99429cdd42d0c43a707d9e2c930548d2023b4cff Mon Sep 17 00:00:00 2001 From: Greeshma Date: Mon, 11 Aug 2025 15:31:16 +0530 Subject: [PATCH 1/3] Code changes --- .../bg_import/api_call_utils.py | 9 ++++++ .../bg_import/business_glossary_export_v2.py | 32 ++++++++++--------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py index f8f783f5..3d7ed5a8 100644 --- a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py +++ b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py @@ -130,14 +130,20 @@ def fetch_api_response( Returns: Dictionary with response and error if any. """ + + logger.debug( + f'Calling {method.__name__.upper()} {url} with project_id: {project_id} and request_body: {request_body}' + ) data, error_msg = None, None method_name = 'GET' if method == requests.get else 'POST' try: res = method(url, headers=_get_header(project_id), json=request_body) + logger.debug(f'Response status: {res.status_code}, Response text: {res.text}') try: data = res.json() except requests.exceptions.JSONDecodeError: error_msg = f'{method_name} call to {url} returned non valid JSON.' + logger.debug(f'Error: {error_msg}') return { 'json': None, 'error_msg': error_msg @@ -145,16 +151,19 @@ def fetch_api_response( if not res.ok: # If the response is an error, capture the error message from the JSON error_msg = data.get('error', {}).get('message') or f'{method_name} call to {url} returned HTTP {res.status_code}.' + logger.debug(f'Error: {error_msg}') return { 'json': data, 'error_msg': error_msg } + logger.debug(f'Successful response JSON: {data}') return { 'json': data, 'error_msg': None } except requests.exceptions.RequestException as err: error_msg = create_error_message(method_name, url, err) + logger.debug(f'Exception occurred: {error_msg}') return { 'json': None, 'error_msg': error_msg diff --git a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py index 9fd0fe02..8be60c20 100644 --- a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py +++ b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py @@ -161,37 +161,36 @@ def compute_ancestors( map_entry_id_to_entry_type: Dict[str, str], ) -> List[Dict[str, str]]: """ - Build the ancestors array for an entry (using entry IDs). - For each ancestor in the chain, construct its export resource using get_export_resource_by_id. - If no belongs_to exists, assume the glossary is the only ancestor. + Builds an ancestors array containing only the root glossary and the immediate parent. Args: child_id (str): The entry id of the child. parent_mapping (Dict[str, str]): The parent mapping. map_entry_id_to_entry_type (Dict[str, str]): The mapping of entry id to entry type. Returns: - List[Dict[str, str]]: The ancestors array. + List[Dict[str, str]]: The ancestors array with at most two elements. """ ancestors: List[Dict[str, str]] = [] if child_id in parent_mapping: - current = parent_mapping[child_id] - while True: - current_type = map_entry_id_to_entry_type.get(current, "glossary") - resource = get_export_resource_by_id(current, current_type) - glossary_child_entry_name = f"{DATAPLEX_ENTRY_GROUP}/entries/{resource}" - ancestors.append({"name": glossary_child_entry_name, "type": get_entry_type_name(current_type)}) - if current not in parent_mapping: - break - current = parent_mapping[current] + parent_id = parent_mapping[child_id] + parent_type = map_entry_id_to_entry_type.get(parent_id, "glossary_category") + resource = get_export_resource_by_id(parent_id, parent_type) + parent_entry_name = f"{DATAPLEX_ENTRY_GROUP}/entries/{resource}" + ancestors.append({ + "name": parent_entry_name, + "type": get_entry_type_name(parent_type) + }) glossary_entry_name = ( f"{DATAPLEX_ENTRY_GROUP}/entries/projects/{PROJECT}/locations/{GLOSSARY_EXPORT_LOCATION}/glossaries/{GLOSSARY}" ) - ancestors.append({"name": glossary_entry_name, "type": get_entry_type_name("glossary")}) + ancestors.append({ + "name": glossary_entry_name, + "type": get_entry_type_name("glossary") + }) ancestors.reverse() return ancestors - def process_entry( entry: Dict[str, Any], parent_mapping: Dict[str, str], @@ -486,6 +485,9 @@ def process_term_entry_links(entry: Dict[str, Any]) -> List[Dict[str, Any]]: rel_url = f"https://datacatalog.googleapis.com/v2/{relative_resource_name}/relationships" response = api_call_utils.fetch_api_response(requests.get, rel_url, USER_PROJECT) logger.debug(f"Relationships response for {rel_url}: {response}") + if not response or not response.get("json"): + logger.warning(f"No relationships found for {relative_resource_name}") + continue relationships = response.get("json", {}).get("relationships", []) for rel in relationships: From ec2cd9cff0cfcfe7b2ea1c2978c2112dbfd65c9f Mon Sep 17 00:00:00 2001 From: Greeshma Date: Mon, 11 Aug 2025 15:44:23 +0530 Subject: [PATCH 2/3] Resolved comments --- .../bg_import/api_call_utils.py | 22 +++++++++++-------- .../bg_import/business_glossary_export_v2.py | 16 +++++++------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py index 3d7ed5a8..8fc18e44 100644 --- a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py +++ b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py @@ -130,28 +130,32 @@ def fetch_api_response( Returns: Dictionary with response and error if any. """ + data, error_msg = None, None + method_name = 'GET' if method == requests.get else 'POST' + context = f'[{method_name} {url}]' logger.debug( - f'Calling {method.__name__.upper()} {url} with project_id: {project_id} and request_body: {request_body}' + f'{context} Initiating call with project_id: {project_id} and request_body: {request_body}' ) - data, error_msg = None, None - method_name = 'GET' if method == requests.get else 'POST' + try: res = method(url, headers=_get_header(project_id), json=request_body) - logger.debug(f'Response status: {res.status_code}, Response text: {res.text}') + logger.debug(f'{context} Response status: {res.status_code}, Response text: {res.text}') + try: data = res.json() except requests.exceptions.JSONDecodeError: - error_msg = f'{method_name} call to {url} returned non valid JSON.' - logger.debug(f'Error: {error_msg}') + error_msg = 'Call returned non-valid JSON.' + logger.debug(f'{context} Json decode error: {error_msg}') return { 'json': None, 'error_msg': error_msg } + if not res.ok: # If the response is an error, capture the error message from the JSON - error_msg = data.get('error', {}).get('message') or f'{method_name} call to {url} returned HTTP {res.status_code}.' - logger.debug(f'Error: {error_msg}') + error_msg = data.get('error', {}).get('message') or f'Call returned HTTP {res.status_code}.' + logger.debug(f'{context} Bad response error: {error_msg}') return { 'json': data, 'error_msg': error_msg @@ -163,7 +167,7 @@ def fetch_api_response( } except requests.exceptions.RequestException as err: error_msg = create_error_message(method_name, url, err) - logger.debug(f'Exception occurred: {error_msg}') + logger.debug(f'{context} Exception occurred: {error_msg}') return { 'json': None, 'error_msg': error_msg diff --git a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py index 8be60c20..f96f704e 100644 --- a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py +++ b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py @@ -171,6 +171,14 @@ def compute_ancestors( List[Dict[str, str]]: The ancestors array with at most two elements. """ ancestors: List[Dict[str, str]] = [] + glossary_entry_name = ( + f"{DATAPLEX_ENTRY_GROUP}/entries/projects/{PROJECT}/locations/{GLOSSARY_EXPORT_LOCATION}/glossaries/{GLOSSARY}" + ) + ancestors.append({ + "name": glossary_entry_name, + "type": get_entry_type_name("glossary") + }) + if child_id in parent_mapping: parent_id = parent_mapping[child_id] parent_type = map_entry_id_to_entry_type.get(parent_id, "glossary_category") @@ -181,14 +189,6 @@ def compute_ancestors( "type": get_entry_type_name(parent_type) }) - glossary_entry_name = ( - f"{DATAPLEX_ENTRY_GROUP}/entries/projects/{PROJECT}/locations/{GLOSSARY_EXPORT_LOCATION}/glossaries/{GLOSSARY}" - ) - ancestors.append({ - "name": glossary_entry_name, - "type": get_entry_type_name("glossary") - }) - ancestors.reverse() return ancestors def process_entry( From da6c5dc925f238ff1990fed02e839c4577d1d595 Mon Sep 17 00:00:00 2001 From: Greeshma Date: Mon, 11 Aug 2025 16:26:38 +0530 Subject: [PATCH 3/3] Minor changes --- .../python/business-glossary-import/bg_import/api_call_utils.py | 2 +- .../bg_import/business_glossary_export_v2.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py index 8fc18e44..7537a461 100644 --- a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py +++ b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/api_call_utils.py @@ -160,7 +160,7 @@ def fetch_api_response( 'json': data, 'error_msg': error_msg } - logger.debug(f'Successful response JSON: {data}') + logger.debug(f'{context} Successful response JSON: {data},') return { 'json': data, 'error_msg': None diff --git a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py index f96f704e..ed97a2d4 100644 --- a/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py +++ b/dataplex-quickstart-labs/00-resources/scripts/python/business-glossary-import/bg_import/business_glossary_export_v2.py @@ -669,6 +669,7 @@ def main(): if args.export_mode != "entry_links_only": map_entry_id_to_entry_type = {get_entry_id(e["name"]): e.get("entryType", "") for e in entries} parent_mapping = build_parent_mapping(entries, relationships_data) + logger.debug(f"Parent mapping: {parent_mapping}") else: map_entry_id_to_entry_type = {} parent_mapping = {}