update with new osn pod paths

hytest-org · Jan 2, 2024 · 56a3b7b · 56a3b7b
1 parent eae5c62
commit 56a3b7b
Show file tree

Hide file tree

Showing 21 changed files with 1,521 additions and 12,049 deletions.
diff --git a/.buildinfo b/.buildinfo
@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: a3dca5c4b0daf018db378b9a012159d3
+config: a1ec04d33cfa74d9bd97f0658f3f6c2f
 tags: 645f666f9bcd5a90fca523b33c5a78b7
diff --git a/_sources/dataset_access/conus404_point_selection.ipynb b/_sources/dataset_access/conus404_point_selection.ipynb
@@ -13,7 +13,9 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "547b851f-3dd9-4b9f-988b-54c743a43bbe",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "import fsspec\n",
@@ -190,7 +192,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "points_df = pd.read_csv('https://usgs.osn.mghpcc.org/usgspod-testbucket/trends_and_drivers/filtered_temperature_sites.csv')\n",
+    "points_df = hytest_cat['pointsample-tutorial-sites-osn'].read()\n",
     "print(len(points_df))\n",
     "points_df.head()"
    ]
@@ -465,7 +467,7 @@
    "source": [
     "fs_write = fsspec.filesystem(\n",
     "    's3',\n",
-    "    profile='osn-usgs',  ## This is the profile name you configured above.\n",
+    "    profile='osn-hytest',  ## This is the name of the AWS profile with credentials to write to the output bucket\n",
     "    client_kwargs={'endpoint_url': 'https://usgs.osn.mghpcc.org/'}\n",
     ")"
    ]
@@ -477,9 +479,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fs_write.ls('usgspod-testbucket')\n",
-    "outfile = f'usgspod-testbucket/trends_and_drivers/daily_skintemp_at_filtered_temperature_sites.nc'\n",
-    "local_file = f'./daily_{var}_at_filtered_temperature_sites.nc'"
+    "fs_write.ls('hytest')\n",
+    "outfile = hytest_cat['pointsample-tutorial-output-osn'].urlpath\n",
+    "local_file = hytest_cat['pointsample-tutorial-output-osn'].urlpath.split('/')[-1]"
    ]
   },
   {
@@ -508,8 +510,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# %%time\n",
-    "# fs_write.upload(local_file, outfile)"
+    "%%time\n",
+    "fs_write.upload(local_file, outfile)"
    ]
   },
   {
@@ -519,7 +521,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "fs_write.ls('usgspod-testbucket/trends_and_drivers/')"
+    "# check that file has been written\n",
+    "fs_write.ls(hytest_cat['pointsample-tutorial-output-osn'].urlpath.split(local_file)[0])"
    ]
   },
   {
@@ -560,26 +563,6 @@
    "source": [
     "ds_final"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "9230a094-4dcd-4552-852e-be62cb29e15d",
-   "metadata": {
-    "tags": []
-   },
-   "source": [
-    "## Shutdown cluster"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c3aff08b-975d-47e1-ac51-6bb8e8b1adbc",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "#client.close(); cluster.shutdown()"
-   ]
   }
  ],
  "metadata": {
@@ -598,7 +581,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {

diff --git a/_sources/dataset_preprocessing/tutorials/rechunking/ReChunkingData.ipynb b/_sources/dataset_preprocessing/tutorials/rechunking/ReChunkingData.ipynb
@@ -951,7 +951,7 @@
     "import intake\n",
     "url = 'https://raw.githubusercontent.com/hytest-org/hytest/main/dataset_catalog/hytest_intake_catalog.yml'\n",
     "cat = intake.open_catalog(url)\n",
-    "sampleData = cat['rechunking-tutorial-cloud'].to_dask()\n",
+    "sampleData = cat['rechunking-tutorial-osn'].to_dask()\n",
     "sampleData"
    ]
   },
@@ -4880,7 +4880,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {

diff --git a/_sources/dataset_preprocessing/tutorials/rechunking/ReChunkingData_Cloud.ipynb b/_sources/dataset_preprocessing/tutorials/rechunking/ReChunkingData_Cloud.ipynb
@@ -397,7 +397,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {

diff --git a/_sources/evaluation/tutorials/nwm_streamflow/01_Data_Prep.ipynb b/_sources/evaluation/tutorials/nwm_streamflow/01_Data_Prep.ipynb
@@ -78,7 +78,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "Modeled data is within the HyTEST `intake` catalog, with name \"nwm21-streamflow-usgs-gages-cloud\":"
+    "Modeled data is within the HyTEST `intake` catalog, with name \"nwm21-streamflow-usgs-gages-osn\":"
    ]
   },
   {
@@ -89,7 +89,7 @@
    "source": [
     "cat = intake.open_catalog(r'https://raw.githubusercontent.com/hytest-org/hytest/main/dataset_catalog/hytest_intake_catalog.yml')\n",
     "\n",
-    "modeled = cat['nwm21-streamflow-usgs-gages-cloud'].to_dask()\n",
+    "modeled = cat['nwm21-streamflow-usgs-gages-osn'].to_dask()\n",
     "\n",
     "modeled"
    ]
@@ -483,7 +483,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {

diff --git a/_sources/evaluation/tutorials/nwm_streamflow/02_Analysis_DScore.ipynb b/_sources/evaluation/tutorials/nwm_streamflow/02_Analysis_DScore.ipynb
@@ -86,11 +86,11 @@
     "you can load data directly from S3 or onprem. \n",
     "If you load data from a source other than this list, you can jump to Step 2: Restrict to a Domain\n",
     "\n",
-    "Note that the interesting datasets in the cataloged dataset above are duplicated: Some are `-onprem` \n",
-    "and some are `-cloud`. Same data, but the storage location and access protocol will be different. You \n",
+    "Note that the interesting datasets in the cataloged dataset above are duplicated: Some are `-onprem` , some are `-cloud`, and some are some are `-osn`. Same data, but the storage location and access protocol will be different. You \n",
     "will definitely want to use the correct copy of the data for your computing environment.  \n",
     "* `onprem` : Direct access via the `caldera` filesystem from _denali_ or _tallgrass_\n",
     "* `cloud` : Network access via S3 bucket, suitable for consumption on cloud-hosted jupyter servers. You could also access using any network-attached computer, but the amount of data will likely saturate your connection.  Use in the cloud (e.g. ESIP QHub)\n",
+    "* `osn` : Network access via OSN pod, which uses the S3 API, suitable for consumption on any jupyter server.\n",
     "\n",
     "So... are you on-prem? "
    ]
@@ -108,9 +108,9 @@
     "    obs_data_src='nwis-streamflow-usgs-gages-onprem'\n",
     "    mod_data_src='nwm21-streamflow-usgs-gages-onprem'\n",
     "else:\n",
-    "    print(\"Not onprem; use '-cloud' data source\")\n",
-    "    obs_data_src='nwis-streamflow-usgs-gages-cloud'\n",
-    "    mod_data_src='nwm21-streamflow-usgs-gages-cloud'\n",
+    "    print(\"Not onprem; use '-osn' data source\")\n",
+    "    obs_data_src='nwis-streamflow-usgs-gages-osn'\n",
+    "    mod_data_src='nwm21-streamflow-usgs-gages-osn'\n",
     "print(\"Observed : \", obs_data_src)\n",
     "print(\"Modeled  : \", mod_data_src)"
    ]
@@ -440,7 +440,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {

diff --git a/_sources/evaluation/tutorials/nwm_streamflow/02_Analysis_StdSuite.ipynb b/_sources/evaluation/tutorials/nwm_streamflow/02_Analysis_StdSuite.ipynb
@@ -81,6 +81,7 @@
     "will definitely want to use the correct copy of the data for your computing environment.  \n",
     "* `onprem` : Direct access via the `caldera` filesystem from _denali_ or _tallgrass_\n",
     "* `cloud` : Network access via S3 bucket, suitable for consumption on cloud-hosted jupyter servers. You could also access using any network-attached computer, but the amount of data will likely saturate your connection.  Use in the cloud (e.g. ESIP QHub)\n",
+    "* `osn` : Network access via OSN pod, which uses the S3 API, suitable for consumption on any jupyter server.\n",
     "\n",
     "So... are you on-prem? "
    ]
@@ -98,9 +99,9 @@
     "    obs_data_src='nwis-streamflow-usgs-gages-onprem'\n",
     "    mod_data_src='nwm21-streamflow-usgs-gages-onprem'\n",
     "else:\n",
-    "    print(\"Not onprem; use '-cloud' data source\")\n",
-    "    obs_data_src='nwis-streamflow-usgs-gages-cloud'\n",
-    "    mod_data_src='nwm21-streamflow-usgs-gages-cloud'\n",
+    "    print(\"Not onprem; use '-osn' data source\")\n",
+    "    obs_data_src='nwis-streamflow-usgs-gages-osn'\n",
+    "    mod_data_src='nwm21-streamflow-usgs-gages-osn'\n",
     "print(\"Observed : \", obs_data_src)\n",
     "print(\"Modeled  : \", mod_data_src)"
    ]
@@ -437,7 +438,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.8"
+   "version": "3.10.13"
   },
   "vscode": {
    "interpreter": {