Merge pull request #365 from junefish/merge-namespaces

update merging namespaces notebook

Author: junefish

learn/experimental/merge-namespaces/merge-namespaces.ipynb

@@ -32,6 +32,17 @@
     "%pip install -qU pinecone-notebooks pinecone-client[grpc]"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pinecone_notebooks.colab import Authenticate\n",
+    "\n",
+    "Authenticate()"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -64,13 +75,20 @@
     "dimension = index.describe_index_stats()['dimension']\n",
     "# Function to fetch all vectors from a namespace\n",
     "def fetch_all_vectors(namespace):\n",
-    "    count = index.describe_index_stats()['namespaces'][namespace]['vector_count']\n",
+    "    try:\n",
+    "      count = index.describe_index_stats()['namespaces'][namespace]['vector_count']\n",
+    "      if count > 10000:\n",
+    "          raise ValueError(\"Namespaces larger than 10000 vectors need to be handled iteratively.\")\n",
+    "    except ValueError as e:\n",
+    "        print(f\"Error: {e}\")\n",
+    "        return []\n",
     "    random_vecs = [random.random() for _ in range(dimension)]\n",
     "    response = index.query(\n",
     "        namespace=namespace,\n",
     "        vector=random_vecs,\n",
     "        top_k=count,\n",
-    "        include_values=True\n",
+    "        include_values=True,\n",
+    "        include_metadata=True\n",
     "    )\n",
     "    return response['matches']\n",
     "\n",
@@ -87,7 +105,9 @@
    "source": [
     "# Convert fetched vectors to the required upsert format\n",
     "def format_vectors_for_upsert(fetched_vectors):\n",
-    "    return [{\"id\": match['id'], \"values\": match['values']} for match in fetched_vectors]\n",
+    "    return [{\"id\": match['id'], \n",
+    "            \"values\": match['values'], \n",
+    "            \"metadata\": match['metadata']} for match in fetched_vectors]\n",
     "\n",
     "formatted_vectors_ns1 = format_vectors_for_upsert(vectors_ns1)\n",
     "formatted_vectors_ns2 = format_vectors_for_upsert(vectors_ns2)\n",
@@ -131,8 +151,14 @@
   }
  ],
  "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
   "language_info": {
-   "name": "python"
+   "name": "python",
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,