|
| 1 | +"""Data Loader CLI STAC_API Ingestion Tool.""" |
| 2 | +import json |
| 3 | +import os |
| 4 | + |
| 5 | +import click |
| 6 | +import requests |
| 7 | + |
| 8 | + |
| 9 | +def load_data(data_dir, filename): |
| 10 | + """Load json data from a file within the specified data directory.""" |
| 11 | + filepath = os.path.join(data_dir, filename) |
| 12 | + if not os.path.exists(filepath): |
| 13 | + click.secho(f"File not found: {filepath}", fg="red", err=True) |
| 14 | + raise click.Abort() |
| 15 | + with open(filepath) as file: |
| 16 | + return json.load(file) |
| 17 | + |
| 18 | + |
| 19 | +def load_collection(base_url, collection_id, data_dir): |
| 20 | + """Load a STAC collection into the database.""" |
| 21 | + collection = load_data(data_dir, "collection.json") |
| 22 | + collection["id"] = collection_id |
| 23 | + try: |
| 24 | + resp = requests.post(f"{base_url}/collections", json=collection) |
| 25 | + if resp.status_code == 200: |
| 26 | + click.echo(f"Status code: {resp.status_code}") |
| 27 | + click.echo(f"Added collection: {collection['id']}") |
| 28 | + elif resp.status_code == 409: |
| 29 | + click.echo(f"Status code: {resp.status_code}") |
| 30 | + click.echo(f"Collection: {collection['id']} already exists") |
| 31 | + except requests.ConnectionError: |
| 32 | + click.secho("Failed to connect", fg="red", err=True) |
| 33 | + |
| 34 | + |
| 35 | +def load_items(base_url, collection_id, use_bulk, data_dir): |
| 36 | + """Load STAC items into the database based on the method selected.""" |
| 37 | + # Attempt to dynamically find a suitable feature collection file |
| 38 | + feature_files = [ |
| 39 | + file |
| 40 | + for file in os.listdir(data_dir) |
| 41 | + if file.endswith(".json") and file != "collection.json" |
| 42 | + ] |
| 43 | + if not feature_files: |
| 44 | + click.secho( |
| 45 | + "No feature collection files found in the specified directory.", |
| 46 | + fg="red", |
| 47 | + err=True, |
| 48 | + ) |
| 49 | + raise click.Abort() |
| 50 | + feature_collection_file = feature_files[ |
| 51 | + 0 |
| 52 | + ] # Use the first found feature collection file |
| 53 | + feature_collection = load_data(data_dir, feature_collection_file) |
| 54 | + |
| 55 | + load_collection(base_url, collection_id, data_dir) |
| 56 | + if use_bulk: |
| 57 | + load_items_bulk_insert(base_url, collection_id, feature_collection, data_dir) |
| 58 | + else: |
| 59 | + load_items_one_by_one(base_url, collection_id, feature_collection, data_dir) |
| 60 | + |
| 61 | + |
| 62 | +def load_items_one_by_one(base_url, collection_id, feature_collection, data_dir): |
| 63 | + """Load STAC items into the database one by one.""" |
| 64 | + for feature in feature_collection["features"]: |
| 65 | + try: |
| 66 | + feature["collection"] = collection_id |
| 67 | + resp = requests.post( |
| 68 | + f"{base_url}/collections/{collection_id}/items", json=feature |
| 69 | + ) |
| 70 | + if resp.status_code == 200: |
| 71 | + click.echo(f"Status code: {resp.status_code}") |
| 72 | + click.echo(f"Added item: {feature['id']}") |
| 73 | + elif resp.status_code == 409: |
| 74 | + click.echo(f"Status code: {resp.status_code}") |
| 75 | + click.echo(f"Item: {feature['id']} already exists") |
| 76 | + except requests.ConnectionError: |
| 77 | + click.secho("Failed to connect", fg="red", err=True) |
| 78 | + |
| 79 | + |
| 80 | +def load_items_bulk_insert(base_url, collection_id, feature_collection, data_dir): |
| 81 | + """Load STAC items into the database via bulk insert.""" |
| 82 | + try: |
| 83 | + for i, _ in enumerate(feature_collection["features"]): |
| 84 | + feature_collection["features"][i]["collection"] = collection_id |
| 85 | + resp = requests.post( |
| 86 | + f"{base_url}/collections/{collection_id}/items", json=feature_collection |
| 87 | + ) |
| 88 | + if resp.status_code == 200: |
| 89 | + click.echo(f"Status code: {resp.status_code}") |
| 90 | + click.echo("Bulk inserted items successfully.") |
| 91 | + elif resp.status_code == 204: |
| 92 | + click.echo(f"Status code: {resp.status_code}") |
| 93 | + click.echo("Bulk update successful, no content returned.") |
| 94 | + elif resp.status_code == 409: |
| 95 | + click.echo(f"Status code: {resp.status_code}") |
| 96 | + click.echo("Conflict detected, some items might already exist.") |
| 97 | + except requests.ConnectionError: |
| 98 | + click.secho("Failed to connect", fg="red", err=True) |
| 99 | + |
| 100 | + |
| 101 | +@click.command() |
| 102 | +@click.option("--base-url", required=True, help="Base URL of the STAC API") |
| 103 | +@click.option( |
| 104 | + "--collection-id", |
| 105 | + default="test-collection", |
| 106 | + help="ID of the collection to which items are added", |
| 107 | +) |
| 108 | +@click.option("--use-bulk", is_flag=True, help="Use bulk insert method for items") |
| 109 | +@click.option( |
| 110 | + "--data-dir", |
| 111 | + type=click.Path(exists=True), |
| 112 | + default="sample_data/", |
| 113 | + help="Directory containing collection.json and feature collection file", |
| 114 | +) |
| 115 | +def main(base_url, collection_id, use_bulk, data_dir): |
| 116 | + """Load STAC items into the database.""" |
| 117 | + load_items(base_url, collection_id, use_bulk, data_dir) |
| 118 | + |
| 119 | + |
| 120 | +if __name__ == "__main__": |
| 121 | + main() |
0 commit comments