admin-valentin/openai-cookbook
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added notes where users can consider adding thread pool to speed up upsert operation

Browse source
This commit is contained in:
Colin Jarvis 2023-01-11 04:04:16 -08:00
parent 21ca5a9555
commit fb18b7f6c2
1 changed files with 5 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
examples/vector_databases/Vector_db_introduction.ipynb
View file

@ -476,6 +476,7 @@
], ],
"source": [ "source": [
"# Upsert content vectors in content namespace\n", "# Upsert content vectors in content namespace\n",
"# NOTE: Using a thread pool here can accelerate this upsert operation\n",
"print(\"Uploading vectors to content namespace..\")\n", "print(\"Uploading vectors to content namespace..\")\n",
"for batch_df in df_batcher(article_df):\n", "for batch_df in df_batcher(article_df):\n",
" index.upsert(vectors=zip(batch_df.vector_id, batch_df.content_vector), namespace='content')" " index.upsert(vectors=zip(batch_df.vector_id, batch_df.content_vector), namespace='content')"
@ -497,6 +498,7 @@
], ],
"source": [ "source": [
"# Upsert title vectors in title namespace\n", "# Upsert title vectors in title namespace\n",
"# NOTE: Using a thread pool here can accelerate this upsert operation\n",
"print(\"Uploading vectors to title namespace..\")\n", "print(\"Uploading vectors to title namespace..\")\n",
"for batch_df in df_batcher(article_df):\n", "for batch_df in df_batcher(article_df):\n",
" index.upsert(vectors=zip(batch_df.vector_id, batch_df.title_vector), namespace='title')" " index.upsert(vectors=zip(batch_df.vector_id, batch_df.title_vector), namespace='title')"
@ -841,13 +843,14 @@
" data_objects.append((v['title'],v['text'],v['title_vector'],v['vector_id']))\n", " data_objects.append((v['title'],v['text'],v['title_vector'],v['vector_id']))\n",
"\n", "\n",
"# Upsert into article schema\n", "# Upsert into article schema\n",
"# NOTE: Using a thread pool here can accelerate this upsert operation\n",
"print(\"Uploading vectors to article schema..\")\n", "print(\"Uploading vectors to article schema..\")\n",
"uuids = []\n", "uuids = []\n",
"for articles in data_objects:\n", "for articles in data_objects:\n",
" uuid = client.data_object.create(\n", " uuid = client.data_object.create(\n",
" {\n", " {\n",
" \"title\": articles[0],\n", " \"title\": articles[0],\n",
" \"content\": articles[1]\n", " \"content\": articles[1]\n",
" },\n", " },\n",
" \"Article\",\n", " \"Article\",\n",
" vector=articles[2]\n", " vector=articles[2]\n",