From b7cb21689abbc92fe145fd100b5ecaca9a7b7d52 Mon Sep 17 00:00:00 2001 From: Colin Jarvis Date: Thu, 5 Jan 2023 01:54:46 -0800 Subject: [PATCH 01/13] Initial commit of vector database example with new embeddings --- .../Vector_db_introduction.ipynb | 1187 +++++++++++++++++ .../weaviate/docker-compose.yaml | 20 + 2 files changed, 1207 insertions(+) create mode 100644 examples/vector_databases/Vector_db_introduction.ipynb create mode 100644 examples/vector_databases/weaviate/docker-compose.yaml diff --git a/examples/vector_databases/Vector_db_introduction.ipynb b/examples/vector_databases/Vector_db_introduction.ipynb new file mode 100644 index 0000000..aca539d --- /dev/null +++ b/examples/vector_databases/Vector_db_introduction.ipynb @@ -0,0 +1,1187 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cb1537e6", + "metadata": {}, + "source": [ + "# Vector Database Introduction\n", + "\n", + "This notebook takes you through a simple flow to download some data, embed it, and then index and search it using a selection of vector databases. This is a common requirement for customers who want to store and search our embeddings with their own data in a secure environment to support production use cases such as chatbots, topic modelling and more.\n", + "\n", + "The demo flow is:\n", + "- **Setup**: Import packages and set any required variables\n", + "- **Load data**: Load a dataset and embed it using OpenAI embeddings\n", + "- **Pinecone**\n", + " - *Setup*: Here we setup the Python client for Pinecone. For more details go [here](https://docs.pinecone.io/docs/quickstart)\n", + " - *Index Data*: We'll create an index with namespaces for __titles__ and __content__\n", + " - *Search Data*: We'll test out both namespaces with search queries to confirm it works\n", + "- **Weaviate**\n", + " - *Setup*: Here we setup the Python client for Weaviate. For more details go [here](https://weaviate.io/developers/weaviate/current/client-libraries/python.html)\n", + " - *Index Data*: We'll create an index with __title__ search vectors in it\n", + " - *Search Data*: We'll run a few searches to confirm it works\n", + "\n", + "Once you've run through this notebook you should have a basic understanding of how to setup and use vector databases, and can move on to more complex use cases making use of our embeddings" + ] + }, + { + "cell_type": "markdown", + "id": "e2b59250", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "Here we import the required libraries and set the embedding model that we'd like to use" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "5be94df6", + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "import tiktoken\n", + "from tenacity import retry, wait_random_exponential, stop_after_attempt\n", + "from typing import List, Iterator\n", + "import concurrent\n", + "from tqdm import tqdm\n", + "import pandas as pd\n", + "from datasets import load_dataset\n", + "import numpy as np\n", + "import os\n", + "\n", + "# Pinecone's client library for Python\n", + "import pinecone\n", + "\n", + "# Weaviate's client library for Python\n", + "import weaviate\n", + "\n", + "# I've set this to our new embeddings model, this can be changed to the embedding model of your choice\n", + "MODEL = \"text-embedding-ada-002\"" + ] + }, + { + "cell_type": "markdown", + "id": "e5d9d2e1", + "metadata": {}, + "source": [ + "## Load data\n", + "\n", + "In this section we'll source the data for this task, embed it and format it for insertion into a vector database\n", + "\n", + "*Thanks to Ryan Greene for the template used for the batch ingestion" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "bd99e08e", + "metadata": {}, + "outputs": [], + "source": [ + "@retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6))\n", + "\n", + "# Simple function to take in a list of text objects and return them as a list of embeddings\n", + "def get_embeddings(input: List):\n", + " response = openai.Embedding.create(\n", + " input=input,\n", + " model=MODEL,\n", + " )[\"data\"]\n", + " return [data[\"embedding\"] for data in response]\n", + "\n", + "# Function for batching and parallel processing the embeddings\n", + "def embed_corpus(\n", + " corpus: List[str],\n", + " batch_size=64,\n", + " num_workers=8,\n", + " max_context_len=8191,\n", + "):\n", + " def batchify(iterable, n=1):\n", + " l = len(iterable)\n", + " for ndx in range(0, l, n):\n", + " yield iterable[ndx : min(ndx + n, l)]\n", + "\n", + " # Encode the corpus, truncating to max_context_len\n", + " encoding = tiktoken.get_encoding(\"cl100k_base\")\n", + " encoded_corpus = [\n", + " encoded_article[:max_context_len] for encoded_article in encoding.encode_batch(corpus)\n", + " ]\n", + "\n", + " # Calculate corpus statistics: the number of inputs, the total number of tokens, and the estimated cost to embed\n", + " num_tokens = sum(len(article) for article in encoded_corpus)\n", + " cost_to_embed_tokens = num_tokens / 1_000 * 0.0004\n", + " print(\n", + " f\"num_articles={len(encoded_corpus)}, num_tokens={num_tokens}, est_embedding_cost={cost_to_embed_tokens:.2f} USD\"\n", + " )\n", + "\n", + " # Embed the corpus\n", + " with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:\n", + " futures = [\n", + " executor.submit(get_embeddings, text_batch)\n", + " for text_batch in batchify(encoded_corpus, batch_size)\n", + " ]\n", + "\n", + " with tqdm(total=len(encoded_corpus)) as pbar:\n", + " for _ in concurrent.futures.as_completed(futures):\n", + " pbar.update(batch_size)\n", + "\n", + " embeddings = []\n", + " for future in futures:\n", + " data = future.result()\n", + " embeddings.extend(data)\n", + " return embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "0c1c73cb", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset wikipedia (/Users/colin.jarvis/.cache/huggingface/datasets/wikipedia/20220301.simple/2.0.0/aa542ed919df55cc5d3347f42dd4521d05ca68751f50dbc32bae2a7f1e167559)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "edbff2615b964463be20d0a2ac33e4ab", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/1 [00:00\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idurltitletexttitle_vectorcontent_vectorvector_id
01https://simple.wikipedia.org/wiki/AprilAprilApril is the fourth month of the year in the J...[0.00107035250402987, -0.02077057771384716, -0...[-0.011253940872848034, -0.013491976074874401,...0
12https://simple.wikipedia.org/wiki/AugustAugustAugust (Aug.) is the eighth month of the year ...[0.0010461278725415468, 0.0008924593566916883,...[0.0003609954728744924, 0.007262262050062418, ...1
26https://simple.wikipedia.org/wiki/ArtArtArt is a creative activity that expresses imag...[0.0033627033699303865, 0.006122018210589886, ...[-0.004959689453244209, 0.015772193670272827, ...2
38https://simple.wikipedia.org/wiki/AAA or a is the first letter of the English alph...[0.015406121499836445, -0.013689860701560974, ...[0.024894846603274345, -0.022186409682035446, ...3
49https://simple.wikipedia.org/wiki/AirAirAir refers to the Earth's atmosphere. Air is a...[0.022219523787498474, -0.020443666726350784, ...[0.021524671465158463, 0.018522677943110466, -...4
\n", + "" + ], + "text/plain": [ + " id url title \\\n", + "0 1 https://simple.wikipedia.org/wiki/April April \n", + "1 2 https://simple.wikipedia.org/wiki/August August \n", + "2 6 https://simple.wikipedia.org/wiki/Art Art \n", + "3 8 https://simple.wikipedia.org/wiki/A A \n", + "4 9 https://simple.wikipedia.org/wiki/Air Air \n", + "\n", + " text \\\n", + "0 April is the fourth month of the year in the J... \n", + "1 August (Aug.) is the eighth month of the year ... \n", + "2 Art is a creative activity that expresses imag... \n", + "3 A or a is the first letter of the English alph... \n", + "4 Air refers to the Earth's atmosphere. Air is a... \n", + "\n", + " title_vector \\\n", + "0 [0.00107035250402987, -0.02077057771384716, -0... \n", + "1 [0.0010461278725415468, 0.0008924593566916883,... \n", + "2 [0.0033627033699303865, 0.006122018210589886, ... \n", + "3 [0.015406121499836445, -0.013689860701560974, ... \n", + "4 [0.022219523787498474, -0.020443666726350784, ... \n", + "\n", + " content_vector vector_id \n", + "0 [-0.011253940872848034, -0.013491976074874401,... 0 \n", + "1 [0.0003609954728744924, 0.007262262050062418, ... 1 \n", + "2 [-0.004959689453244209, 0.015772193670272827, ... 2 \n", + "3 [0.024894846603274345, -0.022186409682035446, ... 3 \n", + "4 [0.021524671465158463, 0.018522677943110466, -... 4 " + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# We then store the result in another dataframe, and prep the data for insertion into a vector DB\n", + "article_df = pd.DataFrame(dataset)\n", + "article_df['title_vector'] = title_embeddings\n", + "article_df['content_vector'] = dataset_embeddings\n", + "article_df['vector_id'] = article_df.index\n", + "article_df['vector_id'] = article_df['vector_id'].apply(str)\n", + "article_df.head()" + ] + }, + { + "cell_type": "markdown", + "id": "ed32fc87", + "metadata": {}, + "source": [ + "## Pinecone\n", + "\n", + "Now we'll look to index these embedded documents in a vector database and search them. The first option we'll look at is **Pinecone**, a managed vector database which offers a cloud-native option.\n", + "\n", + "Before you proceed with this step you'll need to navigate to [Pinecone](pinecone.io), sign up and then save your API key as an environment variable titled ```PINECONE_API_KEY```.\n", + "\n", + "For section we will:\n", + "- Create an index with multiple namespaces for article titles and content\n", + "- Store our data in the index with separate searchable \"namespaces\" for article **titles** and **content**\n", + "- Fire some similarity search queries to verify our setup is working" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "92e6152a", + "metadata": {}, + "outputs": [], + "source": [ + "api_key = os.getenv(\"PINECONE_API_KEY\")\n", + "pinecone.init(api_key=api_key)" + ] + }, + { + "cell_type": "markdown", + "id": "63b28543", + "metadata": {}, + "source": [ + "### Create Index\n", + "\n", + "First we need to create an index, which we'll call `wikipedia-articles`. Once we have an index, we can create multiple namespaces, which can make a single index searchable for various use cases. For more details, consult [this article](https://docs.pinecone.io/docs/namespaces#:~:text=Pinecone%20allows%20you%20to%20partition,different%20subsets%20of%20your%20index.)." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "0a71c575", + "metadata": {}, + "outputs": [], + "source": [ + "class BatchGenerator:\n", + " \"\"\" Models a simple batch generator that make chunks out of an input DataFrame. \"\"\"\n", + " \n", + " def __init__(self, batch_size: int = 10) -> None:\n", + " self.batch_size = batch_size\n", + " \n", + " def to_batches(self, df: pd.DataFrame) -> Iterator[pd.DataFrame]:\n", + " \"\"\" Makes chunks out of an input DataFrame. \"\"\"\n", + " splits = self.splits_num(df.shape[0])\n", + " if splits <= 1:\n", + " yield df\n", + " else:\n", + " for chunk in np.array_split(df, splits):\n", + " yield chunk\n", + " \n", + " def splits_num(self, elements: int) -> int:\n", + " \"\"\" Determines how many chunks DataFrame contians. \"\"\"\n", + " return round(elements / self.batch_size)\n", + " \n", + " __call__ = to_batches\n", + "\n", + "df_batcher = BatchGenerator(300)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "7ea9ad46", + "metadata": {}, + "outputs": [], + "source": [ + "# Pick a name for the new index\n", + "index_name = 'wikipedia-articles'" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "3ff8eca1", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/core/client/rest.py:45: DeprecationWarning: HTTPResponse.getheader() is deprecated and will be removed in urllib3 v2.1.0. Instead use HTTResponse.headers.get(name, default).\n", + " return self.urllib3_response.getheader(name, default)\n", + "/var/folders/bs/rjtxlzk512103d0h0b1t18b40000gp/T/ipykernel_13361/2813989476.py:2: ResourceWarning: unclosed \n", + " if index_name in pinecone.list_indexes():\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/var/folders/bs/rjtxlzk512103d0h0b1t18b40000gp/T/ipykernel_13361/2813989476.py:3: ResourceWarning: unclosed \n", + " pinecone.delete_index(index_name)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + } + ], + "source": [ + "# Check whether the index with the same name already exists\n", + "if index_name in pinecone.list_indexes():\n", + " pinecone.delete_index(index_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "35cb853d", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/manage.py:133: ResourceWarning: unclosed \n", + " status = _get_status(name)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/manage.py:133: ResourceWarning: unclosed \n", + " status = _get_status(name)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/manage.py:133: ResourceWarning: unclosed \n", + " status = _get_status(name)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/manage.py:133: ResourceWarning: unclosed \n", + " status = _get_status(name)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/manage.py:133: ResourceWarning: unclosed \n", + " status = _get_status(name)\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", + "/var/folders/bs/rjtxlzk512103d0h0b1t18b40000gp/T/ipykernel_13361/3257515604.py:1: ResourceWarning: unclosed \n", + " pinecone.create_index(name=index_name, dimension=len(article_df['content_vector'][0]))\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + } + ], + "source": [ + "pinecone.create_index(name=index_name, dimension=len(article_df['content_vector'][0]))\n", + "index = pinecone.Index(index_name=index_name)" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "1328ddaf", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/var/folders/bs/rjtxlzk512103d0h0b1t18b40000gp/T/ipykernel_13361/2524688261.py:1: ResourceWarning: unclosed \n", + " pinecone.list_indexes()\n", + "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" + ] + }, + { + "data": { + "text/plain": [ + "['wikipedia-articles']" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Confirm our index was created\n", + "pinecone.list_indexes()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "5daeba00", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Uploading vectors to content namespace..\n" + ] + } + ], + "source": [ + "# Upsert content vectors in content namespace\n", + "print(\"Uploading vectors to content namespace..\")\n", + "for batch_df in df_batcher(article_df):\n", + " index.upsert(vectors=zip(batch_df.vector_id, batch_df.content_vector), namespace='content')" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "5fc1b083", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Uploading vectors to title namespace..\n" + ] + } + ], + "source": [ + "# Upsert title vectors in title namespace\n", + "print(\"Uploading vectors to title namespace..\")\n", + "for batch_df in df_batcher(article_df):\n", + " index.upsert(vectors=zip(batch_df.vector_id, batch_df.title_vector), namespace='title')" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "f90c7fba", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'dimension': 1536,\n", + " 'index_fullness': 0.2,\n", + " 'namespaces': {'content': {'vector_count': 50000},\n", + " 'title': {'vector_count': 50000}},\n", + " 'total_vector_count': 100000}" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check index size for each namespace\n", + "index.describe_index_stats()" + ] + }, + { + "cell_type": "markdown", + "id": "2da40a69", + "metadata": {}, + "source": [ + "### Search data\n", + "\n", + "Now we'll enter some dummy searches and check we get decent results back" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "d701b3c7", + "metadata": {}, + "outputs": [], + "source": [ + "# First we'll create dictionaries mapping vector IDs to their outputs so we can retrieve the text for our search results\n", + "titles_mapped = dict(zip(article_df.vector_id,article_df.title))\n", + "content_mapped = dict(zip(article_df.vector_id,article_df.text))" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "3c8c2aa1", + "metadata": {}, + "outputs": [], + "source": [ + "def query_article(query, namespace, top_k=5):\n", + " '''Queries an article using its title in the specified\n", + " namespace and prints results.'''\n", + "\n", + " # Create vector embeddings based on the title column\n", + " embedded_query = openai.Embedding.create(\n", + " input=query,\n", + " model=MODEL,\n", + " )[\"data\"][0]['embedding']\n", + "\n", + " # Query namespace passed as parameter using title vector\n", + " query_result = index.query(embedded_query, \n", + " namespace=namespace, \n", + " top_k=top_k)\n", + "\n", + " # Print query results \n", + " print(f'\\nMost similar results querying {query} in \"{namespace}\" namespace:\\n')\n", + " if not query_result.matches:\n", + " print('no query result')\n", + " \n", + " matches = query_result.matches\n", + " ids = [res.id for res in matches]\n", + " scores = [res.score for res in matches]\n", + " df = pd.DataFrame({'id':ids, \n", + " 'score':scores,\n", + " 'title': [titles_mapped[_id] for _id in ids],\n", + " 'content': [content_mapped[_id] for _id in ids],\n", + " })\n", + " \n", + " counter = 0\n", + " for k,v in df.iterrows():\n", + " counter += 1\n", + " print(f'Result {counter} with a score of {v.score} is {v.title}')\n", + " \n", + " print('\\n')\n", + "\n", + " return df" + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "id": "67b3584d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Most similar results querying modern art in Europe in \"title\" namespace:\n", + "\n", + "Result 1 with a score of 0.891034067 is Early modern Europe\n", + "Result 2 with a score of 0.87504226 is Museum of Modern Art\n", + "Result 3 with a score of 0.867497 is Western Europe\n", + "Result 4 with a score of 0.864146471 is Renaissance art\n", + "Result 5 with a score of 0.860363305 is Pop art\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/core/client/rest.py:45: DeprecationWarning: HTTPResponse.getheader() is deprecated and will be removed in urllib3 v2.1.0. Instead use HTTResponse.headers.get(name, default).\n", + " return self.urllib3_response.getheader(name, default)\n" + ] + } + ], + "source": [ + "query_output = query_article('modern art in Europe','title')\n", + "#query_output" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "3e7ac79b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Most similar results querying Famous battles in Scottish history in \"content\" namespace:\n", + "\n", + "Result 1 with a score of 0.869324744 is Battle of Bannockburn\n", + "Result 2 with a score of 0.861479 is Wars of Scottish Independence\n", + "Result 3 with a score of 0.852555931 is 1651\n", + "Result 4 with a score of 0.84969604 is First War of Scottish Independence\n", + "Result 5 with a score of 0.846192539 is Robert I of Scotland\n", + "\n", + "\n" + ] + } + ], + "source": [ + "content_query_output = query_article(\"Famous battles in Scottish history\",'content')\n", + "#content_query_output" + ] + }, + { + "cell_type": "markdown", + "id": "d939342f", + "metadata": {}, + "source": [ + "## Weaviate\n", + "\n", + "The other vector database option we'll explore here is **Weaviate**, which offers both a managed, SaaS option like Pinecone, as well as a self-hosted option. As we've already looked at a cloud vector database, we'll try the self-hosted option here.\n", + "\n", + "For this we will:\n", + "- Set up a local deployment of Weaviate\n", + "- Create indices in Weaviate\n", + "- Store our data there\n", + "- Fire some similarity search queries\n", + "- Try a real use case" + ] + }, + { + "cell_type": "markdown", + "id": "bfdfe260", + "metadata": {}, + "source": [ + "### Setup\n", + "\n", + "To get Weaviate running locally we used Docker and followed the instructions contained in this article: https://weaviate.io/developers/weaviate/current/installation/docker-compose.html\n", + "\n", + "For an example docker-compose.yaml file please refer to `./weaviate/docker-compose.yaml` in this repo\n", + "\n", + "You can start Weaviate up locally by navigating to this directory and running `docker-compose up -d `" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "b9ea472d", + "metadata": {}, + "outputs": [], + "source": [ + "client = weaviate.Client(\"http://localhost:8080/\")" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "13be220d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'classes': []}" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.schema.delete_all()\n", + "client.schema.get()" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "73d33184", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.is_ready()" + ] + }, + { + "cell_type": "markdown", + "id": "03a926b9", + "metadata": {}, + "source": [ + "### Index data\n", + "\n", + "In Weaviate you create __schemas__ to capture each of the entities you will be searching. \n", + "\n", + "In this case we'll create a schema called **Article** with the **title** vector from above included for us to search by.\n", + "\n", + "The next few steps closely follow the documents Weaviate provides [here](https://weaviate.io/developers/weaviate/current/tutorials/how-to-use-weaviate-without-modules.htm)" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "e868d143", + "metadata": {}, + "outputs": [], + "source": [ + "class_obj = {\n", + " \"class\": \"Article\",\n", + " \"vectorizer\": \"none\", # explicitly tell Weaviate not to vectorize anything, we are providing the vectors ourselves through our BERT model\n", + " \"properties\": [{\n", + " \"name\": \"title\",\n", + " \"description\": \"Title of the article\",\n", + " \"dataType\": [\"text\"]\n", + " },\n", + " {\n", + " \"name\": \"content\",\n", + " \"description\": \"Contents of the article\",\n", + " \"dataType\": [\"text\"]\n", + " }]\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "d8d430d0", + "metadata": {}, + "outputs": [], + "source": [ + "client.schema.create_class(class_obj)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "856f20f9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'classes': [{'class': 'Article',\n", + " 'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n", + " 'cleanupIntervalSeconds': 60,\n", + " 'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n", + " 'properties': [{'dataType': ['text'],\n", + " 'description': 'Title of the article',\n", + " 'name': 'title',\n", + " 'tokenization': 'word'},\n", + " {'dataType': ['text'],\n", + " 'description': 'Contents of the article',\n", + " 'name': 'content',\n", + " 'tokenization': 'word'}],\n", + " 'shardingConfig': {'virtualPerPhysical': 128,\n", + " 'desiredCount': 1,\n", + " 'actualCount': 1,\n", + " 'desiredVirtualCount': 128,\n", + " 'actualVirtualCount': 128,\n", + " 'key': '_id',\n", + " 'strategy': 'hash',\n", + " 'function': 'murmur3'},\n", + " 'vectorIndexConfig': {'skip': False,\n", + " 'cleanupIntervalSeconds': 300,\n", + " 'maxConnections': 64,\n", + " 'efConstruction': 128,\n", + " 'ef': -1,\n", + " 'dynamicEfMin': 100,\n", + " 'dynamicEfMax': 500,\n", + " 'dynamicEfFactor': 8,\n", + " 'vectorCacheMaxObjects': 2000000,\n", + " 'flatSearchCutoff': 40000,\n", + " 'distance': 'cosine'},\n", + " 'vectorIndexType': 'hnsw',\n", + " 'vectorizer': 'none'}]}" + ] + }, + "execution_count": 34, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.schema.get()" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "e6f48f6f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.batch.configure(\n", + " # `batch_size` takes an `int` value to enable auto-batching\n", + " # (`None` is used for manual batching)\n", + " batch_size=100, \n", + " # dynamically update the `batch_size` based on import speed\n", + " dynamic=False,\n", + " # `timeout_retries` takes an `int` value to retry on time outs\n", + " timeout_retries=3,\n", + " # checks for batch-item creation errors\n", + " # this is the default in weaviate-client >= 3.6.0\n", + " callback=weaviate.util.check_batch_result,\n", + ")\n", + "#result = client.batch.create_objects(batch)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "7c40c204", + "metadata": {}, + "outputs": [], + "source": [ + "# Make a list of tuples\n", + "data_objects = []\n", + "for k,v in article_df.iterrows():\n", + " data_objects.append((v['title'],v['text'],v['title_vector'],v['vector_id']))" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "bb5eb2c1", + "metadata": {}, + "outputs": [], + "source": [ + "# Template function for setting up parallel upload process\n", + "def transcription_extractor(audio_filepath):\n", + " response = call_asr(openai.api_key,audio_filepath)\n", + " return(response)" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "786d437f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Uploading vectors to article_schema..\n" + ] + } + ], + "source": [ + "# Upsert into article schema\n", + "print(\"Uploading vectors to article schema..\")\n", + "uuids = []\n", + "for articles in data_objects:\n", + " uuid = client.data_object.create(\n", + " {\n", + " \"title\": articles[0],\n", + " \"content\": articles[1]\n", + " },\n", + " \"Article\",\n", + " vector=articles[2]\n", + " )\n", + " uuids.append(uuid)" + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "3658693c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'content': 'Sociedade Esportiva Palmeiras, usually called Palmeiras, is a Brazilian football team. They are from São Paulo, Brazil. The team was founded by an Italian-speaking community on August 26, 1914, as Palestra Itália. They changed to the name used now on September 14, 1942.\\n\\nThey play in green shirts, white shorts and green socks and are one of the most popular and traditional Brazilian clubs.\\n\\nPalmeiras plays at the Palestra Itália stadium, which has seats for 32,000. But in the past, local derbies against São Paulo or Corinthians were usually played in Morumbi stadium. However, the Arena Palestra Itália is under construction with capacity for 45,000 people, expected to be finalized in 2013.\\n\\nName \\n 1914–1942 S.S. Palestra Italia\\n 1942–present S.E. Palmeiras\\n\\nMain titles \\n Copa Rio: 1951\\n Libertadores Cup: 1999 and 2020\\n Copa Mercosul: 1998\\n Campeonato Brasileiro: 1960, 1967, 1967, 1969, 1972, 1973, 1993, 1994, 2016 and 2018 – greatest champion\\n Copa do Brasil: 1998, 2012, 2015 and 2020/21\\n Copa dos Campeões: 2000\\n Campeão do Século\\n Torneio Rio-SP: 1933, 1951, 1965, 1993 and 2000\\n Campeonato Paulista: 1920, 1926 (unbeaten), 1927, 1932 (unbeaten), 1933, 1934, 1936, 1940, 1942, 1944, 1947,1950, 1959 (super champions), 1963, 1966, 1972 (unbeaten), 1974, 1976, 1993, 1994, 1996, 2008 and 2020.\\n Campeonato Paulista Extra: 1926 (unbeaten) and 1938\\n\\nRelated pages\\n List of Brazilian football teams\\n\\nOther websites \\n Palmeiras official site \\n\\nFootball clubs in São Paulo (state)\\n1914 establishments in Brazil',\n", + " 'title': 'Sociedade Esportiva Palmeiras'}" + ] + }, + "execution_count": 48, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "client.data_object.get()['objects'][0]['properties']" + ] + }, + { + "cell_type": "markdown", + "id": "46050ca9", + "metadata": {}, + "source": [ + "### Search Data" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "5acd5437", + "metadata": {}, + "outputs": [], + "source": [ + "def query_weaviate(query, schema, top_k=20):\n", + " '''Queries an article using its title in the specified\n", + " namespace and prints results.'''\n", + "\n", + " # Create vector embeddings based on the title column\n", + " embedded_query = openai.Embedding.create(\n", + " input=query,\n", + " model=MODEL,\n", + " )[\"data\"][0]['embedding']\n", + " \n", + " near_vector = {\"vector\": embedded_query}\n", + "\n", + " # Query namespace passed as parameter using title vector\n", + " query_result = client.query.get(schema,[\"title\",\"content\", \"_additional {certainty}\"]) \\\n", + " .with_near_vector(near_vector) \\\n", + " .with_limit(top_k) \\\n", + " .do()\n", + " \n", + " return query_result\n", + " # Print query results " + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "id": "15def653", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1. Title: Early modern Europe Certainty: 0.9454971551895142\n", + "2. Title: Museum of Modern Art Certainty: 0.9375567138195038\n", + "3. Title: Western Europe Certainty: 0.9336977899074554\n", + "4. Title: Renaissance art Certainty: 0.9321110248565674\n", + "5. Title: Pop art Certainty: 0.9302356243133545\n", + "6. Title: Art exhibition Certainty: 0.9281864166259766\n", + "7. Title: History of Europe Certainty: 0.9278470575809479\n", + "8. Title: Northern Europe Certainty: 0.9273118078708649\n", + "9. Title: Concert of Europe Certainty: 0.9268475472927094\n", + "10. Title: Hellenistic art Certainty: 0.9264660775661469\n", + "11. Title: Piet Mondrian Certainty: 0.9235712587833405\n", + "12. Title: Modernist literature Certainty: 0.9235587120056152\n", + "13. Title: European Capital of Culture Certainty: 0.9228664338588715\n", + "14. Title: Art film Certainty: 0.9217151403427124\n", + "15. Title: Europa Certainty: 0.9216068089008331\n", + "16. Title: Art rock Certainty: 0.9212885200977325\n", + "17. Title: Central Europe Certainty: 0.9212862849235535\n", + "18. Title: Art Certainty: 0.9208334386348724\n", + "19. Title: European Certainty: 0.92069211602211\n", + "20. Title: Byzantine art Certainty: 0.920437216758728\n" + ] + } + ], + "source": [ + "query_result = query_weaviate('modern art in Europe','Article')\n", + "counter = 0\n", + "for article in query_result['data']['Get']['Article']:\n", + " counter += 1\n", + " print(f\"{counter}. Title: {article['title']} Certainty: {article['_additional']['certainty']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "id": "93c4a696", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1. Title: Historic Scotland Certainty: 0.9464837908744812\n", + "2. Title: First War of Scottish Independence Certainty: 0.9461104869842529\n", + "3. Title: Battle of Bannockburn Certainty: 0.9455609619617462\n", + "4. Title: Wars of Scottish Independence Certainty: 0.944368839263916\n", + "5. Title: Second War of Scottish Independence Certainty: 0.9395008385181427\n", + "6. Title: List of Scottish monarchs Certainty: 0.9366503059864044\n", + "7. Title: Kingdom of Scotland Certainty: 0.935274213552475\n", + "8. Title: Scottish Borders Certainty: 0.9317866265773773\n", + "9. Title: List of rivers of Scotland Certainty: 0.9296278059482574\n", + "10. Title: Braveheart Certainty: 0.9294214248657227\n", + "11. Title: John of Scotland Certainty: 0.9292325675487518\n", + "12. Title: Duncan II of Scotland Certainty: 0.9291643798351288\n", + "13. Title: Bannockburn Certainty: 0.9291241466999054\n", + "14. Title: The Scotsman Certainty: 0.9280610680580139\n", + "15. Title: Flag of Scotland Certainty: 0.9270428121089935\n", + "16. Title: Banff and Macduff Certainty: 0.9267247915267944\n", + "17. Title: Guardians of Scotland Certainty: 0.9260919094085693\n", + "18. Title: Scottish Parliament Certainty: 0.9252097904682159\n", + "19. Title: Holyrood Abbey Certainty: 0.925055593252182\n", + "20. Title: Scottish Certainty: 0.9249534606933594\n" + ] + } + ], + "source": [ + "query_result = query_weaviate('Famous battles in Scottish history','Article')\n", + "counter = 0\n", + "for article in query_result['data']['Get']['Article']:\n", + " counter += 1\n", + " print(f\"{counter}. Title: {article['title']} Certainty: {article['_additional']['certainty']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "ad74202e", + "metadata": {}, + "source": [ + "Thanks for following along, you're now equipped to set up your own vector databases and use embeddings to do all kinds of cool things - enjoy! For more complex use cases please continue to work through the cookbook examples here:\n", + "\n", + "TODO: Make other cool things to link to" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/vector_databases/weaviate/docker-compose.yaml b/examples/vector_databases/weaviate/docker-compose.yaml new file mode 100644 index 0000000..ff02bb3 --- /dev/null +++ b/examples/vector_databases/weaviate/docker-compose.yaml @@ -0,0 +1,20 @@ +version: '3.4' +services: + weaviate: + image: semitechnologies/weaviate:1.14.0 + restart: on-failure:0 + ports: + - "8080:8080" + environment: + QUERY_DEFAULTS_LIMIT: 20 + AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true' + PERSISTENCE_DATA_PATH: "./data" + DEFAULT_VECTORIZER_MODULE: text2vec-transformers + ENABLE_MODULES: text2vec-transformers + TRANSFORMERS_INFERENCE_API: http://t2v-transformers:8080 + CLUSTER_HOSTNAME: 'node1' + t2v-transformers: + image: semitechnologies/transformers-inference:sentence-transformers-msmarco-distilroberta-base-v2 + environment: + ENABLE_CUDA: 0 # set to 1 to enable + # NVIDIA_VISIBLE_DEVICES: all # enable if running with CUDA \ No newline at end of file From f2567b62a51dd99f641d5b74e917df4dc7f023ae Mon Sep 17 00:00:00 2001 From: Colin Jarvis Date: Wed, 11 Jan 2023 03:55:23 -0800 Subject: [PATCH 02/13] Pushing updated Vector DB introduction notebook with PR changes --- .../Vector_db_introduction.ipynb | 463 ++++++------------ 1 file changed, 152 insertions(+), 311 deletions(-) diff --git a/examples/vector_databases/Vector_db_introduction.ipynb b/examples/vector_databases/Vector_db_introduction.ipynb index aca539d..6f9d50d 100644 --- a/examples/vector_databases/Vector_db_introduction.ipynb +++ b/examples/vector_databases/Vector_db_introduction.ipynb @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 98, "id": "5be94df6", "metadata": {}, "outputs": [], @@ -60,7 +60,13 @@ "import weaviate\n", "\n", "# I've set this to our new embeddings model, this can be changed to the embedding model of your choice\n", - "MODEL = \"text-embedding-ada-002\"" + "MODEL = \"text-embedding-ada-002\"\n", + "\n", + "# Ignore unclosed SSL socket warnings - optional in case you get these errors\n", + "import warnings\n", + "\n", + "warnings.filterwarnings(action=\"ignore\", message=\"unclosed\", category=ResourceWarning)\n", + "warnings.filterwarnings(\"ignore\", category=DeprecationWarning) " ] }, { @@ -77,13 +83,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 116, "id": "bd99e08e", "metadata": {}, "outputs": [], "source": [ - "@retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6))\n", - "\n", "# Simple function to take in a list of text objects and return them as a list of embeddings\n", "def get_embeddings(input: List):\n", " response = openai.Embedding.create(\n", @@ -92,17 +96,19 @@ " )[\"data\"]\n", " return [data[\"embedding\"] for data in response]\n", "\n", + "def batchify(iterable, n=1):\n", + " l = len(iterable)\n", + " for ndx in range(0, l, n):\n", + " yield iterable[ndx : min(ndx + n, l)]\n", + "\n", "# Function for batching and parallel processing the embeddings\n", + "@retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6))\n", "def embed_corpus(\n", " corpus: List[str],\n", " batch_size=64,\n", " num_workers=8,\n", " max_context_len=8191,\n", "):\n", - " def batchify(iterable, n=1):\n", - " l = len(iterable)\n", - " for ndx in range(0, l, n):\n", - " yield iterable[ndx : min(ndx + n, l)]\n", "\n", " # Encode the corpus, truncating to max_context_len\n", " encoding = tiktoken.get_encoding(\"cl100k_base\")\n", @@ -119,50 +125,37 @@ "\n", " # Embed the corpus\n", " with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:\n", - " futures = [\n", - " executor.submit(get_embeddings, text_batch)\n", - " for text_batch in batchify(encoded_corpus, batch_size)\n", - " ]\n", + " \n", + " try:\n", + " futures = [\n", + " executor.submit(get_embeddings, text_batch)\n", + " for text_batch in batchify(encoded_corpus, batch_size)\n", + " ]\n", "\n", - " with tqdm(total=len(encoded_corpus)) as pbar:\n", - " for _ in concurrent.futures.as_completed(futures):\n", - " pbar.update(batch_size)\n", + " with tqdm(total=len(encoded_corpus)) as pbar:\n", + " for _ in concurrent.futures.as_completed(futures):\n", + " pbar.update(batch_size)\n", "\n", - " embeddings = []\n", - " for future in futures:\n", - " data = future.result()\n", - " embeddings.extend(data)\n", - " return embeddings" + " embeddings = []\n", + " for future in futures:\n", + " data = future.result()\n", + " embeddings.extend(data)\n", + " \n", + " return embeddings\n", + " \n", + " except Exception as e:\n", + " print('Get embeddings failed, returning exception')\n", + " \n", + " return e\n", + " " ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "0c1c73cb", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found cached dataset wikipedia (/Users/colin.jarvis/.cache/huggingface/datasets/wikipedia/20220301.simple/2.0.0/aa542ed919df55cc5d3347f42dd4521d05ca68751f50dbc32bae2a7f1e167559)\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "edbff2615b964463be20d0a2ac33e4ab", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/1 [00:00 None:\n", " self.batch_size = batch_size\n", " \n", + " # Makes chunks out of an input DataFrame\n", " def to_batches(self, df: pd.DataFrame) -> Iterator[pd.DataFrame]:\n", - " \"\"\" Makes chunks out of an input DataFrame. \"\"\"\n", " splits = self.splits_num(df.shape[0])\n", " if splits <= 1:\n", " yield df\n", " else:\n", " for chunk in np.array_split(df, splits):\n", " yield chunk\n", - " \n", + "\n", + " # Determines how many chunks DataFrame contains\n", " def splits_num(self, elements: int) -> int:\n", - " \"\"\" Determines how many chunks DataFrame contians. \"\"\"\n", " return round(elements / self.batch_size)\n", " \n", " __call__ = to_batches\n", @@ -435,112 +429,40 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 99, "id": "7ea9ad46", "metadata": {}, - "outputs": [], - "source": [ - "# Pick a name for the new index\n", - "index_name = 'wikipedia-articles'" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "id": "3ff8eca1", - "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/core/client/rest.py:45: DeprecationWarning: HTTPResponse.getheader() is deprecated and will be removed in urllib3 v2.1.0. Instead use HTTResponse.headers.get(name, default).\n", - " return self.urllib3_response.getheader(name, default)\n", - "/var/folders/bs/rjtxlzk512103d0h0b1t18b40000gp/T/ipykernel_13361/2813989476.py:2: ResourceWarning: unclosed \n", - " if index_name in pinecone.list_indexes():\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", - "/var/folders/bs/rjtxlzk512103d0h0b1t18b40000gp/T/ipykernel_13361/2813989476.py:3: ResourceWarning: unclosed \n", - " pinecone.delete_index(index_name)\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" - ] - } - ], - "source": [ - "# Check whether the index with the same name already exists\n", - "if index_name in pinecone.list_indexes():\n", - " pinecone.delete_index(index_name)" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "35cb853d", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/manage.py:133: ResourceWarning: unclosed \n", - " status = _get_status(name)\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/manage.py:133: ResourceWarning: unclosed \n", - " status = _get_status(name)\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/manage.py:133: ResourceWarning: unclosed \n", - " status = _get_status(name)\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/manage.py:133: ResourceWarning: unclosed \n", - " status = _get_status(name)\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", - "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/manage.py:133: ResourceWarning: unclosed \n", - " status = _get_status(name)\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n", - "/var/folders/bs/rjtxlzk512103d0h0b1t18b40000gp/T/ipykernel_13361/3257515604.py:1: ResourceWarning: unclosed \n", - " pinecone.create_index(name=index_name, dimension=len(article_df['content_vector'][0]))\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" - ] - } - ], - "source": [ - "pinecone.create_index(name=index_name, dimension=len(article_df['content_vector'][0]))\n", - "index = pinecone.Index(index_name=index_name)" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "1328ddaf", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/bs/rjtxlzk512103d0h0b1t18b40000gp/T/ipykernel_13361/2524688261.py:1: ResourceWarning: unclosed \n", - " pinecone.list_indexes()\n", - "ResourceWarning: Enable tracemalloc to get the object allocation traceback\n" - ] - }, { "data": { "text/plain": [ "['wikipedia-articles']" ] }, - "execution_count": 17, + "execution_count": 99, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "# Pick a name for the new index\n", + "index_name = 'wikipedia-articles'\n", + "\n", + "# Check whether the index with the same name already exists - if so, delete it\n", + "if index_name in pinecone.list_indexes():\n", + " pinecone.delete_index(index_name)\n", + " \n", + "# Creates new index\n", + "pinecone.create_index(name=index_name, dimension=len(article_df['content_vector'][0]))\n", + "index = pinecone.Index(index_name=index_name)\n", + "\n", "# Confirm our index was created\n", "pinecone.list_indexes()" ] }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 100, "id": "5daeba00", "metadata": {}, "outputs": [ @@ -561,7 +483,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 101, "id": "5fc1b083", "metadata": {}, "outputs": [ @@ -582,7 +504,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 102, "id": "f90c7fba", "metadata": {}, "outputs": [ @@ -596,7 +518,7 @@ " 'total_vector_count': 100000}" ] }, - "execution_count": 20, + "execution_count": 102, "metadata": {}, "output_type": "execute_result" } @@ -618,7 +540,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 103, "id": "d701b3c7", "metadata": {}, "outputs": [], @@ -630,7 +552,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 104, "id": "3c8c2aa1", "metadata": {}, "outputs": [], @@ -676,7 +598,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 105, "id": "67b3584d", "metadata": {}, "outputs": [ @@ -687,32 +609,23 @@ "\n", "Most similar results querying modern art in Europe in \"title\" namespace:\n", "\n", - "Result 1 with a score of 0.891034067 is Early modern Europe\n", - "Result 2 with a score of 0.87504226 is Museum of Modern Art\n", - "Result 3 with a score of 0.867497 is Western Europe\n", - "Result 4 with a score of 0.864146471 is Renaissance art\n", - "Result 5 with a score of 0.860363305 is Pop art\n", + "Result 1 with a score of 0.890994787 is Early modern Europe\n", + "Result 2 with a score of 0.875286043 is Museum of Modern Art\n", + "Result 3 with a score of 0.867404044 is Western Europe\n", + "Result 4 with a score of 0.864250064 is Renaissance art\n", + "Result 5 with a score of 0.860506058 is Pop art\n", "\n", "\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/colin.jarvis/Documents/dev/vectordb_cookbook/vectordb/lib/python3.10/site-packages/pinecone/core/client/rest.py:45: DeprecationWarning: HTTPResponse.getheader() is deprecated and will be removed in urllib3 v2.1.0. Instead use HTTResponse.headers.get(name, default).\n", - " return self.urllib3_response.getheader(name, default)\n" - ] } ], "source": [ - "query_output = query_article('modern art in Europe','title')\n", - "#query_output" + "query_output = query_article('modern art in Europe','title')" ] }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 106, "id": "3e7ac79b", "metadata": {}, "outputs": [ @@ -734,8 +647,7 @@ } ], "source": [ - "content_query_output = query_article(\"Famous battles in Scottish history\",'content')\n", - "#content_query_output" + "content_query_output = query_article(\"Famous battles in Scottish history\",'content')" ] }, { @@ -771,7 +683,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 107, "id": "b9ea472d", "metadata": {}, "outputs": [], @@ -781,7 +693,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 108, "id": "13be220d", "metadata": {}, "outputs": [ @@ -791,7 +703,7 @@ "{'classes': []}" ] }, - "execution_count": 30, + "execution_count": 108, "metadata": {}, "output_type": "execute_result" } @@ -803,7 +715,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 109, "id": "73d33184", "metadata": {}, "outputs": [ @@ -813,7 +725,7 @@ "True" ] }, - "execution_count": 31, + "execution_count": 109, "metadata": {}, "output_type": "execute_result" } @@ -838,42 +750,9 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 110, "id": "e868d143", "metadata": {}, - "outputs": [], - "source": [ - "class_obj = {\n", - " \"class\": \"Article\",\n", - " \"vectorizer\": \"none\", # explicitly tell Weaviate not to vectorize anything, we are providing the vectors ourselves through our BERT model\n", - " \"properties\": [{\n", - " \"name\": \"title\",\n", - " \"description\": \"Title of the article\",\n", - " \"dataType\": [\"text\"]\n", - " },\n", - " {\n", - " \"name\": \"content\",\n", - " \"description\": \"Contents of the article\",\n", - " \"dataType\": [\"text\"]\n", - " }]\n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "id": "d8d430d0", - "metadata": {}, - "outputs": [], - "source": [ - "client.schema.create_class(class_obj)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "id": "856f20f9", - "metadata": {}, "outputs": [ { "data": { @@ -913,77 +792,37 @@ " 'vectorizer': 'none'}]}" ] }, - "execution_count": 34, + "execution_count": 110, "metadata": {}, "output_type": "execute_result" } ], "source": [ + "class_obj = {\n", + " \"class\": \"Article\",\n", + " \"vectorizer\": \"none\", # explicitly tell Weaviate not to vectorize anything, we are providing the vectors ourselves through our BERT model\n", + " \"properties\": [{\n", + " \"name\": \"title\",\n", + " \"description\": \"Title of the article\",\n", + " \"dataType\": [\"text\"]\n", + " },\n", + " {\n", + " \"name\": \"content\",\n", + " \"description\": \"Contents of the article\",\n", + " \"dataType\": [\"text\"]\n", + " }]\n", + "}\n", + "\n", + "# Create the schema in Weaviate\n", + "client.schema.create_class(class_obj)\n", + "\n", + "# Check that we've created it as intended\n", "client.schema.get()" ] }, { "cell_type": "code", - "execution_count": 35, - "id": "e6f48f6f", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "client.batch.configure(\n", - " # `batch_size` takes an `int` value to enable auto-batching\n", - " # (`None` is used for manual batching)\n", - " batch_size=100, \n", - " # dynamically update the `batch_size` based on import speed\n", - " dynamic=False,\n", - " # `timeout_retries` takes an `int` value to retry on time outs\n", - " timeout_retries=3,\n", - " # checks for batch-item creation errors\n", - " # this is the default in weaviate-client >= 3.6.0\n", - " callback=weaviate.util.check_batch_result,\n", - ")\n", - "#result = client.batch.create_objects(batch)" - ] - }, - { - "cell_type": "code", - "execution_count": 37, - "id": "7c40c204", - "metadata": {}, - "outputs": [], - "source": [ - "# Make a list of tuples\n", - "data_objects = []\n", - "for k,v in article_df.iterrows():\n", - " data_objects.append((v['title'],v['text'],v['title_vector'],v['vector_id']))" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "id": "bb5eb2c1", - "metadata": {}, - "outputs": [], - "source": [ - "# Template function for setting up parallel upload process\n", - "def transcription_extractor(audio_filepath):\n", - " response = call_asr(openai.api_key,audio_filepath)\n", - " return(response)" - ] - }, - { - "cell_type": "code", - "execution_count": 39, + "execution_count": 111, "id": "786d437f", "metadata": {}, "outputs": [ @@ -991,11 +830,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "Uploading vectors to article_schema..\n" + "Uploading vectors to article schema..\n" ] } ], "source": [ + "# Convert DF into a list of tuples\n", + "data_objects = []\n", + "for k,v in article_df.iterrows():\n", + " data_objects.append((v['title'],v['text'],v['title_vector'],v['vector_id']))\n", + "\n", "# Upsert into article schema\n", "print(\"Uploading vectors to article schema..\")\n", "uuids = []\n", @@ -1013,18 +857,18 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 112, "id": "3658693c", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'content': 'Sociedade Esportiva Palmeiras, usually called Palmeiras, is a Brazilian football team. They are from São Paulo, Brazil. The team was founded by an Italian-speaking community on August 26, 1914, as Palestra Itália. They changed to the name used now on September 14, 1942.\\n\\nThey play in green shirts, white shorts and green socks and are one of the most popular and traditional Brazilian clubs.\\n\\nPalmeiras plays at the Palestra Itália stadium, which has seats for 32,000. But in the past, local derbies against São Paulo or Corinthians were usually played in Morumbi stadium. However, the Arena Palestra Itália is under construction with capacity for 45,000 people, expected to be finalized in 2013.\\n\\nName \\n 1914–1942 S.S. Palestra Italia\\n 1942–present S.E. Palmeiras\\n\\nMain titles \\n Copa Rio: 1951\\n Libertadores Cup: 1999 and 2020\\n Copa Mercosul: 1998\\n Campeonato Brasileiro: 1960, 1967, 1967, 1969, 1972, 1973, 1993, 1994, 2016 and 2018 – greatest champion\\n Copa do Brasil: 1998, 2012, 2015 and 2020/21\\n Copa dos Campeões: 2000\\n Campeão do Século\\n Torneio Rio-SP: 1933, 1951, 1965, 1993 and 2000\\n Campeonato Paulista: 1920, 1926 (unbeaten), 1927, 1932 (unbeaten), 1933, 1934, 1936, 1940, 1942, 1944, 1947,1950, 1959 (super champions), 1963, 1966, 1972 (unbeaten), 1974, 1976, 1993, 1994, 1996, 2008 and 2020.\\n Campeonato Paulista Extra: 1926 (unbeaten) and 1938\\n\\nRelated pages\\n List of Brazilian football teams\\n\\nOther websites \\n Palmeiras official site \\n\\nFootball clubs in São Paulo (state)\\n1914 establishments in Brazil',\n", - " 'title': 'Sociedade Esportiva Palmeiras'}" + "{'content': 'Eddie Cantor (January 31, 1892 - October 10, 1964) was an American comedian, singer, actor, songwriter. Familiar to Broadway, radio and early television audiences, this \"Apostle of Pep\" was regarded almost as a family member by millions because his top-rated radio shows revealed intimate stories and amusing anecdotes about his wife Ida and five daughters. His eye-rolling song-and-dance routines eventually led to his nickname, Banjo Eyes, and in 1933, the artist Frederick J. Garner caricatured Cantor with large round and white eyes resembling the drum-like pot of a banjo. Cantor\\'s eyes became his trademark, often exaggerated in illustrations, and leading to his appearance on Broadway in the musical Banjo Eyes (1941). He was the original singer of 1929 hit song \"Makin\\' Whoopie\".\\n\\nReferences\\n\\nPresidents of the Screen Actors Guild\\nAmerican stage actors\\nComedians from New York City\\nAmerican Jews\\nActors from New York City\\nSingers from New York City\\nAmerican television actors\\nAmerican radio actors\\n1892 births\\n1964 deaths',\n", + " 'title': 'Eddie Cantor'}" ] }, - "execution_count": 48, + "execution_count": 112, "metadata": {}, "output_type": "execute_result" } @@ -1038,21 +882,21 @@ "id": "46050ca9", "metadata": {}, "source": [ - "### Search Data" + "### Search Data\n", + "\n", + "As above, we'll fire some queries at our new Index and get back results based on the closeness to our existing vectors" ] }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 113, "id": "5acd5437", "metadata": {}, "outputs": [], "source": [ "def query_weaviate(query, schema, top_k=20):\n", - " '''Queries an article using its title in the specified\n", - " namespace and prints results.'''\n", "\n", - " # Create vector embeddings based on the title column\n", + " # Creates embedding vector from user query\n", " embedded_query = openai.Embedding.create(\n", " input=query,\n", " model=MODEL,\n", @@ -1060,19 +904,18 @@ " \n", " near_vector = {\"vector\": embedded_query}\n", "\n", - " # Query namespace passed as parameter using title vector\n", + " # Queries input schema with vectorised user query\n", " query_result = client.query.get(schema,[\"title\",\"content\", \"_additional {certainty}\"]) \\\n", " .with_near_vector(near_vector) \\\n", " .with_limit(top_k) \\\n", " .do()\n", " \n", - " return query_result\n", - " # Print query results " + " return query_result" ] }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 114, "id": "15def653", "metadata": {}, "outputs": [ @@ -1081,25 +924,25 @@ "output_type": "stream", "text": [ "1. Title: Early modern Europe Certainty: 0.9454971551895142\n", - "2. Title: Museum of Modern Art Certainty: 0.9375567138195038\n", - "3. Title: Western Europe Certainty: 0.9336977899074554\n", - "4. Title: Renaissance art Certainty: 0.9321110248565674\n", - "5. Title: Pop art Certainty: 0.9302356243133545\n", - "6. Title: Art exhibition Certainty: 0.9281864166259766\n", - "7. Title: History of Europe Certainty: 0.9278470575809479\n", - "8. Title: Northern Europe Certainty: 0.9273118078708649\n", + "2. Title: Museum of Modern Art Certainty: 0.9376430511474609\n", + "3. Title: Western Europe Certainty: 0.9337018430233002\n", + "4. Title: Renaissance art Certainty: 0.932124525308609\n", + "5. Title: Pop art Certainty: 0.9302527010440826\n", + "6. Title: Art exhibition Certainty: 0.9282020926475525\n", + "7. Title: History of Europe Certainty: 0.927833616733551\n", + "8. Title: Northern Europe Certainty: 0.9273514151573181\n", "9. Title: Concert of Europe Certainty: 0.9268475472927094\n", - "10. Title: Hellenistic art Certainty: 0.9264660775661469\n", - "11. Title: Piet Mondrian Certainty: 0.9235712587833405\n", + "10. Title: Hellenistic art Certainty: 0.9264959394931793\n", + "11. Title: Piet Mondrian Certainty: 0.9235787093639374\n", "12. Title: Modernist literature Certainty: 0.9235587120056152\n", - "13. Title: European Capital of Culture Certainty: 0.9228664338588715\n", - "14. Title: Art film Certainty: 0.9217151403427124\n", - "15. Title: Europa Certainty: 0.9216068089008331\n", + "13. Title: European Capital of Culture Certainty: 0.9227772951126099\n", + "14. Title: Art film Certainty: 0.9217384457588196\n", + "15. Title: Europa Certainty: 0.9216940104961395\n", "16. Title: Art rock Certainty: 0.9212885200977325\n", - "17. Title: Central Europe Certainty: 0.9212862849235535\n", - "18. Title: Art Certainty: 0.9208334386348724\n", - "19. Title: European Certainty: 0.92069211602211\n", - "20. Title: Byzantine art Certainty: 0.920437216758728\n" + "17. Title: Central Europe Certainty: 0.9212715923786163\n", + "18. Title: Art Certainty: 0.9207542240619659\n", + "19. Title: European Certainty: 0.9207191467285156\n", + "20. Title: Byzantine art Certainty: 0.9204496443271637\n" ] } ], @@ -1113,7 +956,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": 115, "id": "93c4a696", "metadata": {}, "outputs": [ @@ -1121,24 +964,24 @@ "name": "stdout", "output_type": "stream", "text": [ - "1. Title: Historic Scotland Certainty: 0.9464837908744812\n", + "1. Title: Historic Scotland Certainty: 0.9465253949165344\n", "2. Title: First War of Scottish Independence Certainty: 0.9461104869842529\n", - "3. Title: Battle of Bannockburn Certainty: 0.9455609619617462\n", + "3. Title: Battle of Bannockburn Certainty: 0.9455604553222656\n", "4. Title: Wars of Scottish Independence Certainty: 0.944368839263916\n", - "5. Title: Second War of Scottish Independence Certainty: 0.9395008385181427\n", + "5. Title: Second War of Scottish Independence Certainty: 0.9394940435886383\n", "6. Title: List of Scottish monarchs Certainty: 0.9366503059864044\n", - "7. Title: Kingdom of Scotland Certainty: 0.935274213552475\n", - "8. Title: Scottish Borders Certainty: 0.9317866265773773\n", + "7. Title: Kingdom of Scotland Certainty: 0.9353288412094116\n", + "8. Title: Scottish Borders Certainty: 0.9317235946655273\n", "9. Title: List of rivers of Scotland Certainty: 0.9296278059482574\n", "10. Title: Braveheart Certainty: 0.9294214248657227\n", "11. Title: John of Scotland Certainty: 0.9292325675487518\n", "12. Title: Duncan II of Scotland Certainty: 0.9291643798351288\n", - "13. Title: Bannockburn Certainty: 0.9291241466999054\n", - "14. Title: The Scotsman Certainty: 0.9280610680580139\n", + "13. Title: Bannockburn Certainty: 0.929103285074234\n", + "14. Title: The Scotsman Certainty: 0.9280981719493866\n", "15. Title: Flag of Scotland Certainty: 0.9270428121089935\n", "16. Title: Banff and Macduff Certainty: 0.9267247915267944\n", - "17. Title: Guardians of Scotland Certainty: 0.9260919094085693\n", - "18. Title: Scottish Parliament Certainty: 0.9252097904682159\n", + "17. Title: Guardians of Scotland Certainty: 0.9260668158531189\n", + "18. Title: Scottish Parliament Certainty: 0.9251855313777924\n", "19. Title: Holyrood Abbey Certainty: 0.925055593252182\n", "20. Title: Scottish Certainty: 0.9249534606933594\n" ] @@ -1157,9 +1000,7 @@ "id": "ad74202e", "metadata": {}, "source": [ - "Thanks for following along, you're now equipped to set up your own vector databases and use embeddings to do all kinds of cool things - enjoy! For more complex use cases please continue to work through the cookbook examples here:\n", - "\n", - "TODO: Make other cool things to link to" + "Thanks for following along, you're now equipped to set up your own vector databases and use embeddings to do all kinds of cool things - enjoy! For more complex use cases please continue to work through other cookbook examples in this repo" ] } ], From 21ca5a955572207d65bda7a275a07749149c2542 Mon Sep 17 00:00:00 2001 From: Colin Jarvis Date: Wed, 11 Jan 2023 04:01:29 -0800 Subject: [PATCH 03/13] Removed extra whitespace within Weaviate query function --- examples/vector_databases/Vector_db_introduction.ipynb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/vector_databases/Vector_db_introduction.ipynb b/examples/vector_databases/Vector_db_introduction.ipynb index 6f9d50d..723dbe4 100644 --- a/examples/vector_databases/Vector_db_introduction.ipynb +++ b/examples/vector_databases/Vector_db_introduction.ipynb @@ -563,14 +563,14 @@ "\n", " # Create vector embeddings based on the title column\n", " embedded_query = openai.Embedding.create(\n", - " input=query,\n", - " model=MODEL,\n", + " input=query,\n", + " model=MODEL,\n", " )[\"data\"][0]['embedding']\n", "\n", " # Query namespace passed as parameter using title vector\n", " query_result = index.query(embedded_query, \n", - " namespace=namespace, \n", - " top_k=top_k)\n", + " namespace=namespace, \n", + " top_k=top_k)\n", "\n", " # Print query results \n", " print(f'\\nMost similar results querying {query} in \"{namespace}\" namespace:\\n')\n", From fb18b7f6c21673dcce2843c9652d9507cf308a8e Mon Sep 17 00:00:00 2001 From: Colin Jarvis Date: Wed, 11 Jan 2023 04:04:16 -0800 Subject: [PATCH 04/13] Added notes where users can consider adding thread pool to speed up upsert operation --- examples/vector_databases/Vector_db_introduction.ipynb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/vector_databases/Vector_db_introduction.ipynb b/examples/vector_databases/Vector_db_introduction.ipynb index 723dbe4..9eebfe0 100644 --- a/examples/vector_databases/Vector_db_introduction.ipynb +++ b/examples/vector_databases/Vector_db_introduction.ipynb @@ -476,6 +476,7 @@ ], "source": [ "# Upsert content vectors in content namespace\n", + "# NOTE: Using a thread pool here can accelerate this upsert operation\n", "print(\"Uploading vectors to content namespace..\")\n", "for batch_df in df_batcher(article_df):\n", " index.upsert(vectors=zip(batch_df.vector_id, batch_df.content_vector), namespace='content')" @@ -497,6 +498,7 @@ ], "source": [ "# Upsert title vectors in title namespace\n", + "# NOTE: Using a thread pool here can accelerate this upsert operation\n", "print(\"Uploading vectors to title namespace..\")\n", "for batch_df in df_batcher(article_df):\n", " index.upsert(vectors=zip(batch_df.vector_id, batch_df.title_vector), namespace='title')" @@ -841,13 +843,14 @@ " data_objects.append((v['title'],v['text'],v['title_vector'],v['vector_id']))\n", "\n", "# Upsert into article schema\n", + "# NOTE: Using a thread pool here can accelerate this upsert operation\n", "print(\"Uploading vectors to article schema..\")\n", "uuids = []\n", "for articles in data_objects:\n", " uuid = client.data_object.create(\n", " {\n", - " \"title\": articles[0],\n", - " \"content\": articles[1]\n", + " \"title\": articles[0],\n", + " \"content\": articles[1]\n", " },\n", " \"Article\",\n", " vector=articles[2]\n", From 80b8e5fc3f07d0d6cfc67ae02e4ab63973b2d2ab Mon Sep 17 00:00:00 2001 From: Colin Jarvis Date: Mon, 16 Jan 2023 07:41:10 -0800 Subject: [PATCH 05/13] Pushing updated version responding to PR comments --- .../Vector_db_introduction.ipynb | 386 ++++++++++-------- 1 file changed, 217 insertions(+), 169 deletions(-) diff --git a/examples/vector_databases/Vector_db_introduction.ipynb b/examples/vector_databases/Vector_db_introduction.ipynb index 9eebfe0..54c3c2b 100644 --- a/examples/vector_databases/Vector_db_introduction.ipynb +++ b/examples/vector_databases/Vector_db_introduction.ipynb @@ -9,6 +9,16 @@ "\n", "This notebook takes you through a simple flow to download some data, embed it, and then index and search it using a selection of vector databases. This is a common requirement for customers who want to store and search our embeddings with their own data in a secure environment to support production use cases such as chatbots, topic modelling and more.\n", "\n", + "### What is a Vector Database\n", + "\n", + "A vector database is a database made to store, manage and search embedding vectors. The use of embeddings to encode unstructured data (text, audio, video and more) as vectors for consumption by machine-learning models has exploded in recent years, due to the increasing effectiveness of AI in solving use cases involving natural language, image recognition and other unstructured forms of data. Vector databases have emerged as an effective solution for enterprises to deliver and scale these use cases.\n", + "\n", + "### Why use a Vector Database\n", + "\n", + "Vector databases enable enterprises to take many of the embeddings use cases we've shared in this repo (question and answering, chatbot and recommendation services, for example), and make use of them in a secure, scalable environment. Many of our customers make embeddings solve their problems at small scale but performance and security hold them back from going into production - we see vector databases as a key component in solving that, and in this guide we'll walk through the basics of embedding text data, storing it in a vector database and using it for semantic search.\n", + "\n", + "\n", + "### Demo Flow\n", "The demo flow is:\n", "- **Setup**: Import packages and set any required variables\n", "- **Load data**: Load a dataset and embed it using OpenAI embeddings\n", @@ -21,7 +31,7 @@ " - *Index Data*: We'll create an index with __title__ search vectors in it\n", " - *Search Data*: We'll run a few searches to confirm it works\n", "\n", - "Once you've run through this notebook you should have a basic understanding of how to setup and use vector databases, and can move on to more complex use cases making use of our embeddings" + "Once you've run through this notebook you should have a basic understanding of how to setup and use vector databases, and can move on to more complex use cases making use of our embeddings." ] }, { @@ -31,12 +41,12 @@ "source": [ "## Setup\n", "\n", - "Here we import the required libraries and set the embedding model that we'd like to use" + "Import the required libraries and set the embedding model that we'd like to use." ] }, { "cell_type": "code", - "execution_count": 98, + "execution_count": 1, "id": "5be94df6", "metadata": {}, "outputs": [], @@ -60,7 +70,7 @@ "import weaviate\n", "\n", "# I've set this to our new embeddings model, this can be changed to the embedding model of your choice\n", - "MODEL = \"text-embedding-ada-002\"\n", + "EMBEDDING_MODEL = \"text-embedding-ada-002\"\n", "\n", "# Ignore unclosed SSL socket warnings - optional in case you get these errors\n", "import warnings\n", @@ -76,14 +86,12 @@ "source": [ "## Load data\n", "\n", - "In this section we'll source the data for this task, embed it and format it for insertion into a vector database\n", - "\n", - "*Thanks to Ryan Greene for the template used for the batch ingestion" + "In this section we'll source the data for this task, embed it and format it for insertion into a vector database" ] }, { "cell_type": "code", - "execution_count": 116, + "execution_count": 6, "id": "bd99e08e", "metadata": {}, "outputs": [], @@ -92,7 +100,7 @@ "def get_embeddings(input: List):\n", " response = openai.Embedding.create(\n", " input=input,\n", - " model=MODEL,\n", + " model=EMBEDDING_MODEL,\n", " )[\"data\"]\n", " return [data[\"embedding\"] for data in response]\n", "\n", @@ -102,7 +110,6 @@ " yield iterable[ndx : min(ndx + n, l)]\n", "\n", "# Function for batching and parallel processing the embeddings\n", - "@retry(wait=wait_random_exponential(min=1, max=20), stop=stop_after_attempt(6))\n", "def embed_corpus(\n", " corpus: List[str],\n", " batch_size=64,\n", @@ -126,28 +133,21 @@ " # Embed the corpus\n", " with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:\n", " \n", - " try:\n", - " futures = [\n", - " executor.submit(get_embeddings, text_batch)\n", - " for text_batch in batchify(encoded_corpus, batch_size)\n", - " ]\n", + " futures = [\n", + " executor.submit(get_embeddings, text_batch)\n", + " for text_batch in batchify(encoded_corpus, batch_size)\n", + " ]\n", "\n", - " with tqdm(total=len(encoded_corpus)) as pbar:\n", - " for _ in concurrent.futures.as_completed(futures):\n", - " pbar.update(batch_size)\n", + " with tqdm(total=len(encoded_corpus)) as pbar:\n", + " for _ in concurrent.futures.as_completed(futures):\n", + " pbar.update(batch_size)\n", "\n", - " embeddings = []\n", - " for future in futures:\n", - " data = future.result()\n", - " embeddings.extend(data)\n", - " \n", - " return embeddings\n", - " \n", - " except Exception as e:\n", - " print('Get embeddings failed, returning exception')\n", - " \n", - " return e\n", - " " + " embeddings = []\n", + " for future in futures:\n", + " data = future.result()\n", + " embeddings.extend(data)\n", + "\n", + " return embeddings" ] }, { @@ -159,13 +159,13 @@ "source": [ "# We'll use the datasets library to pull the Simple Wikipedia dataset for embedding\n", "dataset = list(load_dataset(\"wikipedia\", \"20220301.simple\")[\"train\"])\n", - "# Limited to 50k articles for demo purposes\n", - "dataset = dataset[:50_000] " + "# Limited to 25k articles for demo purposes\n", + "dataset = dataset[:25_000] " ] }, { "cell_type": "code", - "execution_count": 118, + "execution_count": 15, "id": "e6ee90ce", "metadata": {}, "outputs": [ @@ -173,36 +173,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "num_articles=50000, num_tokens=18272526, est_embedding_cost=7.31 USD\n" + "num_articles=25000, num_tokens=12896881, est_embedding_cost=5.16 USD\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "50048it [02:30, 332.26it/s] \n" + "25024it [01:11, 348.92it/s] " ] }, { "name": "stdout", "output_type": "stream", "text": [ - "num_articles=50000, num_tokens=202363, est_embedding_cost=0.08 USD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "50048it [00:53, 942.94it/s] " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 48.7 s, sys: 1min 19s, total: 2min 7s\n", - "Wall time: 5min 53s\n" + "CPU times: user 15.8 s, sys: 1.96 s, total: 17.8 s\n", + "Wall time: 1min 14s\n" ] }, { @@ -216,14 +202,38 @@ "source": [ "%%time\n", "# Embed the article text\n", - "dataset_embeddings = embed_corpus([article[\"text\"] for article in dataset])\n", + "dataset_embeddings = embed_corpus([article[\"text\"] for article in dataset])" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "850c7215", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "num_articles=25000, num_tokens=88300, est_embedding_cost=0.04 USD\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "25024it [00:21, 1164.97it/s] \n" + ] + } + ], + "source": [ "# Embed the article titles separately\n", "title_embeddings = embed_corpus([article[\"title\"] for article in dataset])" ] }, { "cell_type": "code", - "execution_count": 119, + "execution_count": 17, "id": "1410daaa", "metadata": {}, "outputs": [ @@ -264,7 +274,7 @@ " https://simple.wikipedia.org/wiki/April\n", " April\n", " April is the fourth month of the year in the J...\n", - " [0.00107035250402987, -0.02077057771384716, -0...\n", + " [0.0010547508718445897, -0.020757636055350304,...\n", " [-0.011253940872848034, -0.013491976074874401,...\n", " 0\n", " \n", @@ -274,7 +284,7 @@ " https://simple.wikipedia.org/wiki/August\n", " August\n", " August (Aug.) is the eighth month of the year ...\n", - " [0.0010461278725415468, 0.0008924593566916883,...\n", + " [0.0009623901569284499, 0.0008108559413813055,...\n", " [0.0003609954728744924, 0.007262262050062418, ...\n", " 1\n", " \n", @@ -284,7 +294,7 @@ " https://simple.wikipedia.org/wiki/Art\n", " Art\n", " Art is a creative activity that expresses imag...\n", - " [0.0033627033699303865, 0.006122018210589886, ...\n", + " [0.0033528385683894157, 0.006173426751047373, ...\n", " [-0.004959689453244209, 0.015772193670272827, ...\n", " 2\n", " \n", @@ -294,7 +304,7 @@ " https://simple.wikipedia.org/wiki/A\n", " A\n", " A or a is the first letter of the English alph...\n", - " [0.015406121499836445, -0.013689860701560974, ...\n", + " [0.015449387952685356, -0.013746200129389763, ...\n", " [0.024894846603274345, -0.022186409682035446, ...\n", " 3\n", " \n", @@ -304,7 +314,7 @@ " https://simple.wikipedia.org/wiki/Air\n", " Air\n", " Air refers to the Earth's atmosphere. Air is a...\n", - " [0.022219523787498474, -0.020443666726350784, ...\n", + " [0.0222249086946249, -0.020463958382606506, -0...\n", " [0.021524671465158463, 0.018522677943110466, -...\n", " 4\n", " \n", @@ -328,11 +338,11 @@ "4 Air refers to the Earth's atmosphere. Air is a... \n", "\n", " title_vector \\\n", - "0 [0.00107035250402987, -0.02077057771384716, -0... \n", - "1 [0.0010461278725415468, 0.0008924593566916883,... \n", - "2 [0.0033627033699303865, 0.006122018210589886, ... \n", - "3 [0.015406121499836445, -0.013689860701560974, ... \n", - "4 [0.022219523787498474, -0.020443666726350784, ... \n", + "0 [0.0010547508718445897, -0.020757636055350304,... \n", + "1 [0.0009623901569284499, 0.0008108559413813055,... \n", + "2 [0.0033528385683894157, 0.006173426751047373, ... \n", + "3 [0.015449387952685356, -0.013746200129389763, ... \n", + "4 [0.0222249086946249, -0.020463958382606506, -0... \n", "\n", " content_vector vector_id \n", "0 [-0.011253940872848034, -0.013491976074874401,... 0 \n", @@ -342,7 +352,7 @@ "4 [0.021524671465158463, 0.018522677943110466, -... 4 " ] }, - "execution_count": 119, + "execution_count": 17, "metadata": {}, "output_type": "execute_result" } @@ -376,7 +386,7 @@ }, { "cell_type": "code", - "execution_count": 93, + "execution_count": 18, "id": "92e6152a", "metadata": {}, "outputs": [], @@ -392,12 +402,14 @@ "source": [ "### Create Index\n", "\n", - "First we need to create an index, which we'll call `wikipedia-articles`. Once we have an index, we can create multiple namespaces, which can make a single index searchable for various use cases. For more details, consult [this article](https://docs.pinecone.io/docs/namespaces#:~:text=Pinecone%20allows%20you%20to%20partition,different%20subsets%20of%20your%20index.)." + "First we need to create an index, which we'll call `wikipedia-articles`. Once we have an index, we can create multiple namespaces, which can make a single index searchable for various use cases. For more details, consult [Pinecone documentation](https://docs.pinecone.io/docs/namespaces#:~:text=Pinecone%20allows%20you%20to%20partition,different%20subsets%20of%20your%20index.).\n", + "\n", + "If you want to batch insert to your index in parallel to increase insertion speed then there is a great guide in the Pinecone documentation on [batch inserts in parallel](https://docs.pinecone.io/docs/insert-data#sending-upserts-in-parallel)." ] }, { "cell_type": "code", - "execution_count": 94, + "execution_count": 19, "id": "0a71c575", "metadata": {}, "outputs": [], @@ -429,7 +441,7 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": 20, "id": "7ea9ad46", "metadata": {}, "outputs": [ @@ -439,7 +451,7 @@ "['wikipedia-articles']" ] }, - "execution_count": 99, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -462,7 +474,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": 22, "id": "5daeba00", "metadata": {}, "outputs": [ @@ -476,7 +488,6 @@ ], "source": [ "# Upsert content vectors in content namespace\n", - "# NOTE: Using a thread pool here can accelerate this upsert operation\n", "print(\"Uploading vectors to content namespace..\")\n", "for batch_df in df_batcher(article_df):\n", " index.upsert(vectors=zip(batch_df.vector_id, batch_df.content_vector), namespace='content')" @@ -484,7 +495,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": 23, "id": "5fc1b083", "metadata": {}, "outputs": [ @@ -498,7 +509,6 @@ ], "source": [ "# Upsert title vectors in title namespace\n", - "# NOTE: Using a thread pool here can accelerate this upsert operation\n", "print(\"Uploading vectors to title namespace..\")\n", "for batch_df in df_batcher(article_df):\n", " index.upsert(vectors=zip(batch_df.vector_id, batch_df.title_vector), namespace='title')" @@ -506,7 +516,7 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": 24, "id": "f90c7fba", "metadata": {}, "outputs": [ @@ -514,19 +524,19 @@ "data": { "text/plain": [ "{'dimension': 1536,\n", - " 'index_fullness': 0.2,\n", - " 'namespaces': {'content': {'vector_count': 50000},\n", - " 'title': {'vector_count': 50000}},\n", - " 'total_vector_count': 100000}" + " 'index_fullness': 0.1,\n", + " 'namespaces': {'content': {'vector_count': 25000},\n", + " 'title': {'vector_count': 25000}},\n", + " 'total_vector_count': 50000}" ] }, - "execution_count": 102, + "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# Check index size for each namespace\n", + "# Check index size for each namespace to confirm all of our docs have loaded\n", "index.describe_index_stats()" ] }, @@ -542,7 +552,7 @@ }, { "cell_type": "code", - "execution_count": 103, + "execution_count": 25, "id": "d701b3c7", "metadata": {}, "outputs": [], @@ -554,7 +564,7 @@ }, { "cell_type": "code", - "execution_count": 104, + "execution_count": 28, "id": "3c8c2aa1", "metadata": {}, "outputs": [], @@ -566,16 +576,16 @@ " # Create vector embeddings based on the title column\n", " embedded_query = openai.Embedding.create(\n", " input=query,\n", - " model=MODEL,\n", + " model=EMBEDDING_MODEL,\n", " )[\"data\"][0]['embedding']\n", "\n", " # Query namespace passed as parameter using title vector\n", " query_result = index.query(embedded_query, \n", - " namespace=namespace, \n", - " top_k=top_k)\n", + " namespace=namespace, \n", + " top_k=top_k)\n", "\n", " # Print query results \n", - " print(f'\\nMost similar results querying {query} in \"{namespace}\" namespace:\\n')\n", + " print(f'\\nMost similar results to {query} in \"{namespace}\" namespace:\\n')\n", " if not query_result.matches:\n", " print('no query result')\n", " \n", @@ -591,7 +601,7 @@ " counter = 0\n", " for k,v in df.iterrows():\n", " counter += 1\n", - " print(f'Result {counter} with a score of {v.score} is {v.title}')\n", + " print(f'{v.title} (score = {v.score})')\n", " \n", " print('\\n')\n", "\n", @@ -600,7 +610,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": 29, "id": "67b3584d", "metadata": {}, "outputs": [ @@ -609,13 +619,13 @@ "output_type": "stream", "text": [ "\n", - "Most similar results querying modern art in Europe in \"title\" namespace:\n", + "Most similar results to modern art in Europe in \"title\" namespace:\n", "\n", - "Result 1 with a score of 0.890994787 is Early modern Europe\n", - "Result 2 with a score of 0.875286043 is Museum of Modern Art\n", - "Result 3 with a score of 0.867404044 is Western Europe\n", - "Result 4 with a score of 0.864250064 is Renaissance art\n", - "Result 5 with a score of 0.860506058 is Pop art\n", + "Museum of Modern Art (score = 0.875286043)\n", + "Western Europe (score = 0.867383599)\n", + "Renaissance art (score = 0.864250064)\n", + "Pop art (score = 0.860506058)\n", + "Northern Europe (score = 0.854678154)\n", "\n", "\n" ] @@ -627,7 +637,7 @@ }, { "cell_type": "code", - "execution_count": 106, + "execution_count": 30, "id": "3e7ac79b", "metadata": {}, "outputs": [ @@ -636,13 +646,13 @@ "output_type": "stream", "text": [ "\n", - "Most similar results querying Famous battles in Scottish history in \"content\" namespace:\n", + "Most similar results to Famous battles in Scottish history in \"content\" namespace:\n", "\n", - "Result 1 with a score of 0.869324744 is Battle of Bannockburn\n", - "Result 2 with a score of 0.861479 is Wars of Scottish Independence\n", - "Result 3 with a score of 0.852555931 is 1651\n", - "Result 4 with a score of 0.84969604 is First War of Scottish Independence\n", - "Result 5 with a score of 0.846192539 is Robert I of Scotland\n", + "Battle of Bannockburn (score = 0.869324744)\n", + "Wars of Scottish Independence (score = 0.861479)\n", + "1651 (score = 0.852555931)\n", + "First War of Scottish Independence (score = 0.84969604)\n", + "Robert I of Scotland (score = 0.846192539)\n", "\n", "\n" ] @@ -685,7 +695,7 @@ }, { "cell_type": "code", - "execution_count": 107, + "execution_count": 33, "id": "b9ea472d", "metadata": {}, "outputs": [], @@ -695,7 +705,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": 34, "id": "13be220d", "metadata": {}, "outputs": [ @@ -705,7 +715,7 @@ "{'classes': []}" ] }, - "execution_count": 108, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -717,7 +727,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": 35, "id": "73d33184", "metadata": {}, "outputs": [ @@ -727,7 +737,7 @@ "True" ] }, - "execution_count": 109, + "execution_count": 35, "metadata": {}, "output_type": "execute_result" } @@ -752,7 +762,7 @@ }, { "cell_type": "code", - "execution_count": 110, + "execution_count": 36, "id": "e868d143", "metadata": {}, "outputs": [ @@ -794,7 +804,7 @@ " 'vectorizer': 'none'}]}" ] }, - "execution_count": 110, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } @@ -802,7 +812,7 @@ "source": [ "class_obj = {\n", " \"class\": \"Article\",\n", - " \"vectorizer\": \"none\", # explicitly tell Weaviate not to vectorize anything, we are providing the vectors ourselves through our BERT model\n", + " \"vectorizer\": \"none\", # explicitly tell Weaviate not to vectorize anything, we are providing the vectors ourselves\n", " \"properties\": [{\n", " \"name\": \"title\",\n", " \"description\": \"Title of the article\",\n", @@ -824,7 +834,7 @@ }, { "cell_type": "code", - "execution_count": 111, + "execution_count": 39, "id": "786d437f", "metadata": {}, "outputs": [ @@ -843,41 +853,79 @@ " data_objects.append((v['title'],v['text'],v['title_vector'],v['vector_id']))\n", "\n", "# Upsert into article schema\n", - "# NOTE: Using a thread pool here can accelerate this upsert operation\n", "print(\"Uploading vectors to article schema..\")\n", + "\n", + "# Store a list of UUIDs in case we want to use to refer back to the initial dataframe\n", "uuids = []\n", - "for articles in data_objects:\n", - " uuid = client.data_object.create(\n", + "\n", + "# Reuse our batcher from the Pinecone ingestion\n", + "for batch_df in df_batcher(article_df):\n", + " for k,v in batch_df.iterrows():\n", + " #print(articles)\n", + " uuid = client.data_object.create(\n", " {\n", - " \"title\": articles[0],\n", - " \"content\": articles[1]\n", + " \"title\": v['title'],\n", + " \"content\": v['text']\n", " },\n", " \"Article\",\n", - " vector=articles[2]\n", + " vector=v['title_vector']\n", " )\n", - " uuids.append(uuid)" + " uuids.append(uuid)" ] }, { "cell_type": "code", - "execution_count": 112, + "execution_count": 47, "id": "3658693c", "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cave Story\n", + "is a freeware video game released in 2004 for PC. It was thought of and created over five years by Daisuke Amaya, known by his pseudonym, or art name, Pixel. The game is an action-adventure game, and is similar to the Castlevania and Metroid games. It was first made in Japanese, and was translated to English by the fan translating group, Aeon Genesis.\n", + "\n", + "References \n", + "\n", + "Notes\n", + "\n", + "2004 video games\n", + "Amiga games\n", + "Dreamcast games\n", + "Freeware games\n", + "Indie video games\n", + "Nintendo 3DS games\n", + "Nintendo Switch games\n", + "MacOS games\n", + "Platform games\n", + "Sega Genesis games\n", + "Video games developed in Japan\n", + "Wii games\n", + "Windows games\n" + ] + }, { "data": { "text/plain": [ - "{'content': 'Eddie Cantor (January 31, 1892 - October 10, 1964) was an American comedian, singer, actor, songwriter. Familiar to Broadway, radio and early television audiences, this \"Apostle of Pep\" was regarded almost as a family member by millions because his top-rated radio shows revealed intimate stories and amusing anecdotes about his wife Ida and five daughters. His eye-rolling song-and-dance routines eventually led to his nickname, Banjo Eyes, and in 1933, the artist Frederick J. Garner caricatured Cantor with large round and white eyes resembling the drum-like pot of a banjo. Cantor\\'s eyes became his trademark, often exaggerated in illustrations, and leading to his appearance on Broadway in the musical Banjo Eyes (1941). He was the original singer of 1929 hit song \"Makin\\' Whoopie\".\\n\\nReferences\\n\\nPresidents of the Screen Actors Guild\\nAmerican stage actors\\nComedians from New York City\\nAmerican Jews\\nActors from New York City\\nSingers from New York City\\nAmerican television actors\\nAmerican radio actors\\n1892 births\\n1964 deaths',\n", - " 'title': 'Eddie Cantor'}" + "{'Aggregate': {'Article': [{'meta': {'count': 25000}}]}}" ] }, - "execution_count": 112, + "execution_count": 47, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "client.data_object.get()['objects'][0]['properties']" + "# Test our insert has worked by checking one object\n", + "print(client.data_object.get()['objects'][0]['properties']['title'])\n", + "print(client.data_object.get()['objects'][0]['properties']['content'])\n", + "\n", + "# Test that all data has loaded\n", + "result = client.query.aggregate(\"Article\") \\\n", + " .with_fields('meta { count }') \\\n", + " .do()\n", + "result['data']" ] }, { @@ -892,7 +940,7 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": 48, "id": "5acd5437", "metadata": {}, "outputs": [], @@ -902,7 +950,7 @@ " # Creates embedding vector from user query\n", " embedded_query = openai.Embedding.create(\n", " input=query,\n", - " model=MODEL,\n", + " model=EMBEDDING_MODEL,\n", " )[\"data\"][0]['embedding']\n", " \n", " near_vector = {\"vector\": embedded_query}\n", @@ -918,7 +966,7 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": 49, "id": "15def653", "metadata": {}, "outputs": [ @@ -926,26 +974,26 @@ "name": "stdout", "output_type": "stream", "text": [ - "1. Title: Early modern Europe Certainty: 0.9454971551895142\n", - "2. Title: Museum of Modern Art Certainty: 0.9376430511474609\n", - "3. Title: Western Europe Certainty: 0.9337018430233002\n", - "4. Title: Renaissance art Certainty: 0.932124525308609\n", - "5. Title: Pop art Certainty: 0.9302527010440826\n", - "6. Title: Art exhibition Certainty: 0.9282020926475525\n", - "7. Title: History of Europe Certainty: 0.927833616733551\n", - "8. Title: Northern Europe Certainty: 0.9273514151573181\n", - "9. Title: Concert of Europe Certainty: 0.9268475472927094\n", - "10. Title: Hellenistic art Certainty: 0.9264959394931793\n", - "11. Title: Piet Mondrian Certainty: 0.9235787093639374\n", - "12. Title: Modernist literature Certainty: 0.9235587120056152\n", - "13. Title: European Capital of Culture Certainty: 0.9227772951126099\n", - "14. Title: Art film Certainty: 0.9217384457588196\n", - "15. Title: Europa Certainty: 0.9216940104961395\n", - "16. Title: Art rock Certainty: 0.9212885200977325\n", - "17. Title: Central Europe Certainty: 0.9212715923786163\n", - "18. Title: Art Certainty: 0.9207542240619659\n", - "19. Title: European Certainty: 0.9207191467285156\n", - "20. Title: Byzantine art Certainty: 0.9204496443271637\n" + "1. Museum of Modern Art (Score: 0.938)\n", + "2. Western Europe (Score: 0.934)\n", + "3. Renaissance art (Score: 0.932)\n", + "4. Pop art (Score: 0.93)\n", + "5. Northern Europe (Score: 0.927)\n", + "6. Hellenistic art (Score: 0.926)\n", + "7. Modernist literature (Score: 0.924)\n", + "8. Art film (Score: 0.922)\n", + "9. Central Europe (Score: 0.921)\n", + "10. Art (Score: 0.921)\n", + "11. European (Score: 0.921)\n", + "12. Byzantine art (Score: 0.92)\n", + "13. Postmodernism (Score: 0.92)\n", + "14. Eastern Europe (Score: 0.92)\n", + "15. Cubism (Score: 0.92)\n", + "16. Europe (Score: 0.919)\n", + "17. Impressionism (Score: 0.919)\n", + "18. Bauhaus (Score: 0.919)\n", + "19. Surrealism (Score: 0.919)\n", + "20. Expressionism (Score: 0.918)\n" ] } ], @@ -954,12 +1002,12 @@ "counter = 0\n", "for article in query_result['data']['Get']['Article']:\n", " counter += 1\n", - " print(f\"{counter}. Title: {article['title']} Certainty: {article['_additional']['certainty']}\")" + " print(f\"{counter}. { article['title']} (Score: {round(article['_additional']['certainty'],3) })\")" ] }, { "cell_type": "code", - "execution_count": 115, + "execution_count": 50, "id": "93c4a696", "metadata": {}, "outputs": [ @@ -967,26 +1015,26 @@ "name": "stdout", "output_type": "stream", "text": [ - "1. Title: Historic Scotland Certainty: 0.9465253949165344\n", - "2. Title: First War of Scottish Independence Certainty: 0.9461104869842529\n", - "3. Title: Battle of Bannockburn Certainty: 0.9455604553222656\n", - "4. Title: Wars of Scottish Independence Certainty: 0.944368839263916\n", - "5. Title: Second War of Scottish Independence Certainty: 0.9394940435886383\n", - "6. Title: List of Scottish monarchs Certainty: 0.9366503059864044\n", - "7. Title: Kingdom of Scotland Certainty: 0.9353288412094116\n", - "8. Title: Scottish Borders Certainty: 0.9317235946655273\n", - "9. Title: List of rivers of Scotland Certainty: 0.9296278059482574\n", - "10. Title: Braveheart Certainty: 0.9294214248657227\n", - "11. Title: John of Scotland Certainty: 0.9292325675487518\n", - "12. Title: Duncan II of Scotland Certainty: 0.9291643798351288\n", - "13. Title: Bannockburn Certainty: 0.929103285074234\n", - "14. Title: The Scotsman Certainty: 0.9280981719493866\n", - "15. Title: Flag of Scotland Certainty: 0.9270428121089935\n", - "16. Title: Banff and Macduff Certainty: 0.9267247915267944\n", - "17. Title: Guardians of Scotland Certainty: 0.9260668158531189\n", - "18. Title: Scottish Parliament Certainty: 0.9251855313777924\n", - "19. Title: Holyrood Abbey Certainty: 0.925055593252182\n", - "20. Title: Scottish Certainty: 0.9249534606933594\n" + "1. Historic Scotland (Score: 0.947)\n", + "2. First War of Scottish Independence (Score: 0.946)\n", + "3. Battle of Bannockburn (Score: 0.946)\n", + "4. Wars of Scottish Independence (Score: 0.944)\n", + "5. Second War of Scottish Independence (Score: 0.94)\n", + "6. List of Scottish monarchs (Score: 0.937)\n", + "7. Scottish Borders (Score: 0.932)\n", + "8. Braveheart (Score: 0.929)\n", + "9. John of Scotland (Score: 0.929)\n", + "10. Guardians of Scotland (Score: 0.926)\n", + "11. Holyrood Abbey (Score: 0.925)\n", + "12. Scottish (Score: 0.925)\n", + "13. Scots (Score: 0.925)\n", + "14. Robert I of Scotland (Score: 0.924)\n", + "15. Scottish people (Score: 0.924)\n", + "16. Alexander I of Scotland (Score: 0.924)\n", + "17. Edinburgh Castle (Score: 0.924)\n", + "18. Robert Burns (Score: 0.923)\n", + "19. Battle of Bosworth Field (Score: 0.922)\n", + "20. David II of Scotland (Score: 0.922)\n" ] } ], @@ -995,7 +1043,7 @@ "counter = 0\n", "for article in query_result['data']['Get']['Article']:\n", " counter += 1\n", - " print(f\"{counter}. Title: {article['title']} Certainty: {article['_additional']['certainty']}\")" + " print(f\"{counter}. {article['title']} (Score: {round(article['_additional']['certainty'],3) })\")" ] }, { @@ -1003,7 +1051,7 @@ "id": "ad74202e", "metadata": {}, "source": [ - "Thanks for following along, you're now equipped to set up your own vector databases and use embeddings to do all kinds of cool things - enjoy! For more complex use cases please continue to work through other cookbook examples in this repo" + "Thanks for following along, you're now equipped to set up your own vector databases and use embeddings to do all kinds of cool things - enjoy! For more complex use cases please continue to work through other cookbook examples in this repo." ] } ], From f20f6acfd1dd4041c96f2c7a59f4b106be2c61fa Mon Sep 17 00:00:00 2001 From: Colin Jarvis Date: Mon, 16 Jan 2023 07:41:42 -0800 Subject: [PATCH 06/13] Updated document name to be more descriptive --- ...n.ipynb => Using_vector_databases_for_embeddings_search.ipynb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/vector_databases/{Vector_db_introduction.ipynb => Using_vector_databases_for_embeddings_search.ipynb} (100%) diff --git a/examples/vector_databases/Vector_db_introduction.ipynb b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb similarity index 100% rename from examples/vector_databases/Vector_db_introduction.ipynb rename to examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb From a367e11e39d71e4f31a49fc365d9e9088c6173b3 Mon Sep 17 00:00:00 2001 From: Colin Jarvis Date: Mon, 16 Jan 2023 07:45:17 -0800 Subject: [PATCH 07/13] Updated title --- .../Using_vector_databases_for_embeddings_search.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb index 54c3c2b..fe88385 100644 --- a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb +++ b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb @@ -5,7 +5,7 @@ "id": "cb1537e6", "metadata": {}, "source": [ - "# Vector Database Introduction\n", + "# Using Vector Databases for Embeddings Search\n", "\n", "This notebook takes you through a simple flow to download some data, embed it, and then index and search it using a selection of vector databases. This is a common requirement for customers who want to store and search our embeddings with their own data in a secure environment to support production use cases such as chatbots, topic modelling and more.\n", "\n", From 225b9177c834c71aac9e82456b540286cc89f1b1 Mon Sep 17 00:00:00 2001 From: Colin Jarvis Date: Mon, 16 Jan 2023 07:47:58 -0800 Subject: [PATCH 08/13] Updated wordings of headers --- .../Using_vector_databases_for_embeddings_search.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb index fe88385..f048d97 100644 --- a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb +++ b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb @@ -374,7 +374,7 @@ "source": [ "## Pinecone\n", "\n", - "Now we'll look to index these embedded documents in a vector database and search them. The first option we'll look at is **Pinecone**, a managed vector database which offers a cloud-native option.\n", + "We'll index these embedded documents in a vector database and search them. The first option we'll look at is **Pinecone**, a managed vector database which offers a cloud-native option.\n", "\n", "Before you proceed with this step you'll need to navigate to [Pinecone](pinecone.io), sign up and then save your API key as an environment variable titled ```PINECONE_API_KEY```.\n", "\n", @@ -686,7 +686,7 @@ "source": [ "### Setup\n", "\n", - "To get Weaviate running locally we used Docker and followed the instructions contained in this article: https://weaviate.io/developers/weaviate/current/installation/docker-compose.html\n", + "To get Weaviate running locally we used Docker and followed the instructions contained the Weaviate documentation here: https://weaviate.io/developers/weaviate/current/installation/docker-compose.html\n", "\n", "For an example docker-compose.yaml file please refer to `./weaviate/docker-compose.yaml` in this repo\n", "\n", @@ -757,7 +757,7 @@ "\n", "In this case we'll create a schema called **Article** with the **title** vector from above included for us to search by.\n", "\n", - "The next few steps closely follow the documents Weaviate provides [here](https://weaviate.io/developers/weaviate/current/tutorials/how-to-use-weaviate-without-modules.htm)" + "The next few steps closely follow the documentation Weaviate provides [here](https://weaviate.io/developers/weaviate/current/tutorials/how-to-use-weaviate-without-modules.htm)" ] }, { From 5ee5fecb76c5a25274869de76adb8485d55869a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kacper=20=C5=81ukawski?= Date: Wed, 18 Jan 2023 11:15:36 +0100 Subject: [PATCH 09/13] Add Qdrant as another example of vector database --- ...ctor_databases_for_embeddings_search.ipynb | 311 +++++++++++++++++- .../qdrant/docker-compose.yaml | 8 + 2 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 examples/vector_databases/qdrant/docker-compose.yaml diff --git a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb index f048d97..55b8ceb 100644 --- a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb +++ b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb @@ -69,6 +69,9 @@ "# Weaviate's client library for Python\n", "import weaviate\n", "\n", + "# Qdrant's client library for Python\n", + "import qdrant_client\n", + "\n", "# I've set this to our new embeddings model, this can be changed to the embedding model of your choice\n", "EMBEDDING_MODEL = \"text-embedding-ada-002\"\n", "\n", @@ -1048,7 +1051,313 @@ }, { "cell_type": "markdown", - "id": "ad74202e", + "metadata": {}, + "source": [ + "## Qdrant\n", + "\n", + "The last vector database we'll consider in **[Qdrant](https://qdrant.tech/)**. This is a high-performant vector search database written in Rust. It offers both on-premise and cloud version, but for the purposes of that example we're going to use the local deployment mode.\n", + "\n", + "Setting everything up will require:\n", + "- Spinning up a local instance of Qdrant\n", + "- Configuring the collection and storing the data in it\n", + "- Trying out with some queries" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setup\n", + "\n", + "For the local deployment, we are going to use Docker, according to the Qdrant documentation: https://qdrant.tech/documentation/quick_start/. Qdrant requires just a single container, but an example of the docker-compose.yaml file is available at `./qdrant/docker-compose.yaml` in this repo.\n", + "\n", + "You can start Qdrant instance locally by navigating to this directory and running `docker-compose up -d `" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "ExecuteTime": { + "end_time": "2023-01-18T09:28:38.928205Z", + "start_time": "2023-01-18T09:28:38.913987Z" + } + }, + "outputs": [], + "source": [ + "qdrant = qdrant_client.QdrantClient(host='localhost', prefer_grpc=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "ExecuteTime": { + "end_time": "2023-01-18T09:29:19.806639Z", + "start_time": "2023-01-18T09:29:19.727897Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "CollectionsResponse(collections=[])" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "qdrant.get_collections()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Index data\n", + "\n", + "Qdrant stores data in __collections__ where each object is described by at least one vector and may contain an additional metadata called __payload__. Our collection will be called **Articles** and each object will be described by both **title** and **content** vectors.\n", + "\n", + "We're going to be using an official [qdrant-client](https://github.com/qdrant/qdrant_client) package that has all the utility methods already built-in." + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "ExecuteTime": { + "end_time": "2023-01-18T09:29:22.530121Z", + "start_time": "2023-01-18T09:29:22.524604Z" + } + }, + "outputs": [], + "source": [ + "from qdrant_client.http import models as rest" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "ExecuteTime": { + "end_time": "2023-01-18T09:31:14.413334Z", + "start_time": "2023-01-18T09:31:13.619079Z" + } + }, + "outputs": [], + "source": [ + "vector_size = len(article_df['content_vector'][0])\n", + "\n", + "qdrant.recreate_collection(\n", + " collection_name='Articles',\n", + " vectors_config={\n", + " 'title': rest.VectorParams(\n", + " distance=rest.Distance.COSINE,\n", + " size=vector_size,\n", + " ),\n", + " 'content': rest.VectorParams(\n", + " distance=rest.Distance.COSINE,\n", + " size=vector_size,\n", + " ),\n", + " }\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "ExecuteTime": { + "end_time": "2023-01-18T09:36:28.597535Z", + "start_time": "2023-01-18T09:36:24.108867Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "UpdateResult(operation_id=0, status=)" + ] + }, + "execution_count": 37, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "qdrant.upsert(\n", + " collection_name='Articles',\n", + " points=[\n", + " rest.PointStruct(\n", + " id=k,\n", + " vector={\n", + " 'title': v['title_vector'],\n", + " 'content': v['content_vector'],\n", + " },\n", + " payload=v.to_dict(),\n", + " )\n", + " for k, v in article_df.iterrows()\n", + " ],\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 52, + "metadata": { + "ExecuteTime": { + "end_time": "2023-01-18T09:58:13.825886Z", + "start_time": "2023-01-18T09:58:13.816248Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "CountResult(count=250)" + ] + }, + "execution_count": 52, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check the collection size to make sure all the points have been stored\n", + "qdrant.count(collection_name='Articles')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Search Data\n", + "\n", + "Once the data is put into Qdrant we can start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search." + ] + }, + { + "cell_type": "code", + "execution_count": 49, + "metadata": { + "ExecuteTime": { + "end_time": "2023-01-18T09:50:35.265647Z", + "start_time": "2023-01-18T09:50:35.256065Z" + } + }, + "outputs": [], + "source": [ + "def query_qdrant(query, collection_name, vector_name='title', top_k=20):\n", + "\n", + " # Creates embedding vector from user query\n", + " embedded_query = openai.Embedding.create(\n", + " input=query,\n", + " model=EMBEDDING_MODEL,\n", + " )['data'][0]['embedding']\n", + " \n", + " query_results = qdrant.search(\n", + " collection_name=collection_name,\n", + " query_vector=(\n", + " vector_name, embedded_query\n", + " ),\n", + " limit=top_k,\n", + " )\n", + " \n", + " return query_results" + ] + }, + { + "cell_type": "code", + "execution_count": 50, + "metadata": { + "ExecuteTime": { + "end_time": "2023-01-18T09:50:46.545145Z", + "start_time": "2023-01-18T09:50:35.711020Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0. Art (Score: 0.841)\n", + "1. Europe (Score: 0.839)\n", + "2. Italy (Score: 0.816)\n", + "3. Architecture (Score: 0.815)\n", + "4. Madrid (Score: 0.815)\n", + "5. France (Score: 0.812)\n", + "6. Belgium (Score: 0.808)\n", + "7. Austria (Score: 0.802)\n", + "8. London (Score: 0.799)\n", + "9. History (Score: 0.797)\n", + "10. Creativity (Score: 0.796)\n", + "11. Archaeology (Score: 0.795)\n", + "12. Cartography (Score: 0.794)\n", + "13. Denmark (Score: 0.793)\n", + "14. Finland (Score: 0.79)\n", + "15. English (Score: 0.789)\n", + "16. Catharism (Score: 0.788)\n", + "17. Dublin (Score: 0.787)\n", + "18. Ireland (Score: 0.787)\n", + "19. Japan (Score: 0.787)\n" + ] + } + ], + "source": [ + "query_results = query_qdrant('modern art in Europe', 'Articles')\n", + "for i, article in enumerate(query_results):\n", + " print(f'{i + 1}. {article.payload[\"title\"]} (Score: {round(article.score, 3)})')" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "metadata": { + "ExecuteTime": { + "end_time": "2023-01-18T09:53:11.038910Z", + "start_time": "2023-01-18T09:52:55.248029Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1. History (Score: 0.797)\n", + "2. Dublin (Score: 0.787)\n", + "3. Ireland (Score: 0.786)\n", + "4. History of Australia (Score: 0.782)\n", + "5. Historian (Score: 0.778)\n", + "6. Belgium (Score: 0.776)\n", + "7. Black pudding (Score: 0.773)\n", + "8. London (Score: 0.769)\n", + "9. History of Spain (Score: 0.768)\n", + "10. Cartography (Score: 0.763)\n", + "11. March (Score: 0.762)\n", + "12. France (Score: 0.761)\n", + "13. Bubonic plague (Score: 0.76)\n", + "14. Great Lakes (Score: 0.759)\n", + "15. Inch (Score: 0.758)\n", + "16. Dissolution of the monasteries (Score: 0.758)\n", + "17. Austria (Score: 0.757)\n", + "18. English (Score: 0.757)\n", + "19. British English (Score: 0.757)\n", + "20. Armenia (Score: 0.756)\n" + ] + } + ], + "source": [ + "# This time we're going to query using content vector\n", + "query_results = query_qdrant('Famous battles in Scottish history', 'Articles', 'content')\n", + "for i, article in enumerate(query_results):\n", + " print(f'{i + 1}. {article.payload[\"title\"]} (Score: {round(article.score, 3)})')" + ] + }, + { + "cell_type": "markdown", "metadata": {}, "source": [ "Thanks for following along, you're now equipped to set up your own vector databases and use embeddings to do all kinds of cool things - enjoy! For more complex use cases please continue to work through other cookbook examples in this repo." diff --git a/examples/vector_databases/qdrant/docker-compose.yaml b/examples/vector_databases/qdrant/docker-compose.yaml new file mode 100644 index 0000000..d924aff --- /dev/null +++ b/examples/vector_databases/qdrant/docker-compose.yaml @@ -0,0 +1,8 @@ +version: '3.4' +services: + qdrant: + image: qdrant/qdrant:v0.11.7 + restart: on-failure + ports: + - "6333:6333" + - "6334:6334" \ No newline at end of file From ea8a52d00bfe8a438f23a47923f5a0a2b7314744 Mon Sep 17 00:00:00 2001 From: colin-openai Date: Wed, 25 Jan 2023 16:42:33 -0800 Subject: [PATCH 10/13] Updated text to include Qdrant in guide --- ...ctor_databases_for_embeddings_search.ipynb | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb index 55b8ceb..9347621 100644 --- a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb +++ b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb @@ -30,6 +30,10 @@ " - *Setup*: Here we setup the Python client for Weaviate. For more details go [here](https://weaviate.io/developers/weaviate/current/client-libraries/python.html)\n", " - *Index Data*: We'll create an index with __title__ search vectors in it\n", " - *Search Data*: We'll run a few searches to confirm it works\n", + "- **Qdrant**\n", + " - *Setup*: Here we setup the Python client for Qdrant. For more details go [here](https://github.com/qdrant/qdrant_client)\n", + " - *Index Data*: We'll create a collection with vectors for __titles__ and __content__\n", + " - *Search Data*: We'll run a few searches to confirm it works\n", "\n", "Once you've run through this notebook you should have a basic understanding of how to setup and use vector databases, and can move on to more complex use cases making use of our embeddings." ] @@ -46,7 +50,20 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, + "id": "8d8810f9", + "metadata": {}, + "outputs": [], + "source": [ + "# Here we install the clients for all vector databases\n", + "!pip install pinecone-client\n", + "!pip install weaviate-client\n", + "!pip install qdrant-client" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "5be94df6", "metadata": {}, "outputs": [], @@ -1051,11 +1068,12 @@ }, { "cell_type": "markdown", + "id": "9cfaed9d", "metadata": {}, "source": [ "## Qdrant\n", "\n", - "The last vector database we'll consider in **[Qdrant](https://qdrant.tech/)**. This is a high-performant vector search database written in Rust. It offers both on-premise and cloud version, but for the purposes of that example we're going to use the local deployment mode.\n", + "The last vector database we'll consider is **[Qdrant](https://qdrant.tech/)**. This is a high-performant vector search database written in Rust. It offers both on-premise and cloud version, but for the purposes of that example we're going to use the local deployment mode.\n", "\n", "Setting everything up will require:\n", "- Spinning up a local instance of Qdrant\n", @@ -1065,6 +1083,7 @@ }, { "cell_type": "markdown", + "id": "38774565", "metadata": {}, "source": [ "### Setup\n", @@ -1077,6 +1096,7 @@ { "cell_type": "code", "execution_count": 27, + "id": "76d697e9", "metadata": { "ExecuteTime": { "end_time": "2023-01-18T09:28:38.928205Z", @@ -1091,6 +1111,7 @@ { "cell_type": "code", "execution_count": 29, + "id": "1deeb539", "metadata": { "ExecuteTime": { "end_time": "2023-01-18T09:29:19.806639Z", @@ -1115,6 +1136,7 @@ }, { "cell_type": "markdown", + "id": "bc006b6f", "metadata": {}, "source": [ "### Index data\n", @@ -1127,6 +1149,7 @@ { "cell_type": "code", "execution_count": 30, + "id": "1a84ee1d", "metadata": { "ExecuteTime": { "end_time": "2023-01-18T09:29:22.530121Z", @@ -1141,6 +1164,7 @@ { "cell_type": "code", "execution_count": 34, + "id": "00876f92", "metadata": { "ExecuteTime": { "end_time": "2023-01-18T09:31:14.413334Z", @@ -1169,6 +1193,7 @@ { "cell_type": "code", "execution_count": 37, + "id": "f24e76ab", "metadata": { "ExecuteTime": { "end_time": "2023-01-18T09:36:28.597535Z", @@ -1207,6 +1232,7 @@ { "cell_type": "code", "execution_count": 52, + "id": "d1188a12", "metadata": { "ExecuteTime": { "end_time": "2023-01-18T09:58:13.825886Z", @@ -1232,6 +1258,7 @@ }, { "cell_type": "markdown", + "id": "06ed119b", "metadata": {}, "source": [ "### Search Data\n", @@ -1242,6 +1269,7 @@ { "cell_type": "code", "execution_count": 49, + "id": "f1bac4ef", "metadata": { "ExecuteTime": { "end_time": "2023-01-18T09:50:35.265647Z", @@ -1272,6 +1300,7 @@ { "cell_type": "code", "execution_count": 50, + "id": "aa92f3d3", "metadata": { "ExecuteTime": { "end_time": "2023-01-18T09:50:46.545145Z", @@ -1315,6 +1344,7 @@ { "cell_type": "code", "execution_count": 51, + "id": "7ed116b8", "metadata": { "ExecuteTime": { "end_time": "2023-01-18T09:53:11.038910Z", @@ -1358,6 +1388,7 @@ }, { "cell_type": "markdown", + "id": "55afccbf", "metadata": {}, "source": [ "Thanks for following along, you're now equipped to set up your own vector databases and use embeddings to do all kinds of cool things - enjoy! For more complex use cases please continue to work through other cookbook examples in this repo." From a37477f9bce1b069c878adef0cc847df0cfbc458 Mon Sep 17 00:00:00 2001 From: colin-openai Date: Thu, 26 Jan 2023 11:11:23 -0800 Subject: [PATCH 11/13] Tested notebook end to end --- ...ctor_databases_for_embeddings_search.ipynb | 86 +++++++++---------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb index 9347621..3227045 100644 --- a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb +++ b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb @@ -63,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "id": "5be94df6", "metadata": {}, "outputs": [], @@ -111,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 2, "id": "bd99e08e", "metadata": {}, "outputs": [], @@ -172,10 +172,32 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "0c1c73cb", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Found cached dataset wikipedia (/Users/colin.jarvis/.cache/huggingface/datasets/wikipedia/20220301.simple/2.0.0/aa542ed919df55cc5d3347f42dd4521d05ca68751f50dbc32bae2a7f1e167559)\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "6aa3a11d70424916915334e267f4964b", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + " 0%| | 0/1 [00:00 Date: Tue, 31 Jan 2023 14:37:22 -0800 Subject: [PATCH 12/13] Updated to align wording --- ...ctor_databases_for_embeddings_search.ipynb | 348 ++++++------------ 1 file changed, 120 insertions(+), 228 deletions(-) diff --git a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb index 3227045..964cf35 100644 --- a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb +++ b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb @@ -23,15 +23,15 @@ "- **Setup**: Import packages and set any required variables\n", "- **Load data**: Load a dataset and embed it using OpenAI embeddings\n", "- **Pinecone**\n", - " - *Setup*: Here we setup the Python client for Pinecone. For more details go [here](https://docs.pinecone.io/docs/quickstart)\n", + " - *Setup*: Here we'll set up the Python client for Pinecone. For more details go [here](https://docs.pinecone.io/docs/quickstart)\n", " - *Index Data*: We'll create an index with namespaces for __titles__ and __content__\n", " - *Search Data*: We'll test out both namespaces with search queries to confirm it works\n", "- **Weaviate**\n", - " - *Setup*: Here we setup the Python client for Weaviate. For more details go [here](https://weaviate.io/developers/weaviate/current/client-libraries/python.html)\n", + " - *Setup*: Here we'll set up the Python client for Weaviate. For more details go [here](https://weaviate.io/developers/weaviate/current/client-libraries/python.html)\n", " - *Index Data*: We'll create an index with __title__ search vectors in it\n", " - *Search Data*: We'll run a few searches to confirm it works\n", "- **Qdrant**\n", - " - *Setup*: Here we setup the Python client for Qdrant. For more details go [here](https://github.com/qdrant/qdrant_client)\n", + " - *Setup*: Here we'll set up the Python client for Qdrant. For more details go [here](https://github.com/qdrant/qdrant_client)\n", " - *Index Data*: We'll create a collection with vectors for __titles__ and __content__\n", " - *Search Data*: We'll run a few searches to confirm it works\n", "\n", @@ -55,7 +55,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Here we install the clients for all vector databases\n", + "# We'll need to install the clients for all vector databases\n", "!pip install pinecone-client\n", "!pip install weaviate-client\n", "!pip install qdrant-client" @@ -63,7 +63,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 3, "id": "5be94df6", "metadata": {}, "outputs": [], @@ -111,7 +111,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "id": "bd99e08e", "metadata": {}, "outputs": [], @@ -172,32 +172,10 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "0c1c73cb", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Found cached dataset wikipedia (/Users/colin.jarvis/.cache/huggingface/datasets/wikipedia/20220301.simple/2.0.0/aa542ed919df55cc5d3347f42dd4521d05ca68751f50dbc32bae2a7f1e167559)\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "6aa3a11d70424916915334e267f4964b", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - " 0%| | 0/1 [00:00https://simple.wikipedia.org/wiki/April\n", " April\n", " April is the fourth month of the year in the J...\n", - " [0.0010547508718445897, -0.020757636055350304,...\n", + " [0.001009464613161981, -0.020700545981526375, ...\n", " [-0.011253940872848034, -0.013491976074874401,...\n", " 0\n", " \n", @@ -326,7 +304,7 @@ " https://simple.wikipedia.org/wiki/August\n", " August\n", " August (Aug.) is the eighth month of the year ...\n", - " [0.0009623901569284499, 0.0008108559413813055,...\n", + " [0.0009286514250561595, 0.000820168002974242, ...\n", " [0.0003609954728744924, 0.007262262050062418, ...\n", " 1\n", " \n", @@ -336,7 +314,7 @@ " https://simple.wikipedia.org/wiki/Art\n", " Art\n", " Art is a creative activity that expresses imag...\n", - " [0.0033528385683894157, 0.006173426751047373, ...\n", + " [0.003393713850528002, 0.0061537534929811954, ...\n", " [-0.004959689453244209, 0.015772193670272827, ...\n", " 2\n", " \n", @@ -346,7 +324,7 @@ " https://simple.wikipedia.org/wiki/A\n", " A\n", " A or a is the first letter of the English alph...\n", - " [0.015449387952685356, -0.013746200129389763, ...\n", + " [0.0153952119871974, -0.013759135268628597, 0....\n", " [0.024894846603274345, -0.022186409682035446, ...\n", " 3\n", " \n", @@ -356,7 +334,7 @@ " https://simple.wikipedia.org/wiki/Air\n", " Air\n", " Air refers to the Earth's atmosphere. Air is a...\n", - " [0.0222249086946249, -0.020463958382606506, -0...\n", + " [0.02224554680287838, -0.02044147066771984, -0...\n", " [0.021524671465158463, 0.018522677943110466, -...\n", " 4\n", " \n", @@ -380,11 +358,11 @@ "4 Air refers to the Earth's atmosphere. Air is a... \n", "\n", " title_vector \\\n", - "0 [0.0010547508718445897, -0.020757636055350304,... \n", - "1 [0.0009623901569284499, 0.0008108559413813055,... \n", - "2 [0.0033528385683894157, 0.006173426751047373, ... \n", - "3 [0.015449387952685356, -0.013746200129389763, ... \n", - "4 [0.0222249086946249, -0.020463958382606506, -0... \n", + "0 [0.001009464613161981, -0.020700545981526375, ... \n", + "1 [0.0009286514250561595, 0.000820168002974242, ... \n", + "2 [0.003393713850528002, 0.0061537534929811954, ... \n", + "3 [0.0153952119871974, -0.013759135268628597, 0.... \n", + "4 [0.02224554680287838, -0.02044147066771984, -0... \n", "\n", " content_vector vector_id \n", "0 [-0.011253940872848034, -0.013491976074874401,... 0 \n", @@ -394,13 +372,13 @@ "4 [0.021524671465158463, 0.018522677943110466, -... 4 " ] }, - "execution_count": 13, + "execution_count": 122, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "# We then store the result in another dataframe, and prep the data for insertion into a vector DB\n", + "# We will then store the result in another dataframe, and prep the data for insertion into a vector DB\n", "article_df = pd.DataFrame(dataset)\n", "article_df['title_vector'] = title_embeddings\n", "article_df['content_vector'] = dataset_embeddings\n", @@ -444,14 +422,14 @@ "source": [ "### Create Index\n", "\n", - "First we need to create an index, which we'll call `wikipedia-articles`. Once we have an index, we can create multiple namespaces, which can make a single index searchable for various use cases. For more details, consult [Pinecone documentation](https://docs.pinecone.io/docs/namespaces#:~:text=Pinecone%20allows%20you%20to%20partition,different%20subsets%20of%20your%20index.).\n", + "First we will need to create an index, which we'll call `wikipedia-articles`. Once we have an index, we can create multiple namespaces, which can make a single index searchable for various use cases. For more details, consult [Pinecone documentation](https://docs.pinecone.io/docs/namespaces#:~:text=Pinecone%20allows%20you%20to%20partition,different%20subsets%20of%20your%20index.).\n", "\n", "If you want to batch insert to your index in parallel to increase insertion speed then there is a great guide in the Pinecone documentation on [batch inserts in parallel](https://docs.pinecone.io/docs/insert-data#sending-upserts-in-parallel)." ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 108, "id": "0a71c575", "metadata": {}, "outputs": [], @@ -483,7 +461,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 124, "id": "7ea9ad46", "metadata": {}, "outputs": [ @@ -493,7 +471,7 @@ "['wikipedia-articles']" ] }, - "execution_count": 20, + "execution_count": 124, "metadata": {}, "output_type": "execute_result" } @@ -516,7 +494,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 126, "id": "5daeba00", "metadata": {}, "outputs": [ @@ -529,7 +507,7 @@ } ], "source": [ - "# Upsert content vectors in content namespace\n", + "# Upsert content vectors in content namespace - this can take a few minutes\n", "print(\"Uploading vectors to content namespace..\")\n", "for batch_df in df_batcher(article_df):\n", " index.upsert(vectors=zip(batch_df.vector_id, batch_df.content_vector), namespace='content')" @@ -537,7 +515,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 127, "id": "5fc1b083", "metadata": {}, "outputs": [ @@ -550,7 +528,7 @@ } ], "source": [ - "# Upsert title vectors in title namespace\n", + "# Upsert title vectors in title namespace - this can also take a few minutes\n", "print(\"Uploading vectors to title namespace..\")\n", "for batch_df in df_batcher(article_df):\n", " index.upsert(vectors=zip(batch_df.vector_id, batch_df.title_vector), namespace='title')" @@ -558,7 +536,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 128, "id": "f90c7fba", "metadata": {}, "outputs": [ @@ -572,7 +550,7 @@ " 'total_vector_count': 50000}" ] }, - "execution_count": 24, + "execution_count": 128, "metadata": {}, "output_type": "execute_result" } @@ -594,7 +572,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": null, "id": "d701b3c7", "metadata": {}, "outputs": [], @@ -606,7 +584,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 72, "id": "3c8c2aa1", "metadata": {}, "outputs": [], @@ -652,54 +630,20 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "id": "67b3584d", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Most similar results to modern art in Europe in \"title\" namespace:\n", - "\n", - "Museum of Modern Art (score = 0.875286043)\n", - "Western Europe (score = 0.867383599)\n", - "Renaissance art (score = 0.864250064)\n", - "Pop art (score = 0.860506058)\n", - "Northern Europe (score = 0.854678154)\n", - "\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "query_output = query_article('modern art in Europe','title')" ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": null, "id": "3e7ac79b", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Most similar results to Famous battles in Scottish history in \"content\" namespace:\n", - "\n", - "Battle of Bannockburn (score = 0.869324744)\n", - "Wars of Scottish Independence (score = 0.861479)\n", - "1651 (score = 0.852555931)\n", - "First War of Scottish Independence (score = 0.84969604)\n", - "Robert I of Scotland (score = 0.846192539)\n", - "\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "content_query_output = query_article(\"Famous battles in Scottish history\",'content')" ] @@ -728,7 +672,7 @@ "source": [ "### Setup\n", "\n", - "To get Weaviate running locally we used Docker and followed the instructions contained the Weaviate documentation here: https://weaviate.io/developers/weaviate/current/installation/docker-compose.html\n", + "To get Weaviate running locally we will use Docker and follow the instructions contained in the Weaviate documentation here: https://weaviate.io/developers/weaviate/current/installation/docker-compose.html\n", "\n", "For an example docker-compose.yaml file please refer to `./weaviate/docker-compose.yaml` in this repo\n", "\n", @@ -737,7 +681,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 113, "id": "b9ea472d", "metadata": {}, "outputs": [], @@ -747,10 +691,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 114, "id": "13be220d", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "{'classes': []}" + ] + }, + "execution_count": 114, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "client.schema.delete_all()\n", "client.schema.get()" @@ -758,10 +713,21 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 115, "id": "73d33184", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "True" + ] + }, + "execution_count": 115, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "client.is_ready()" ] @@ -782,7 +748,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 116, "id": "e868d143", "metadata": {}, "outputs": [ @@ -824,7 +790,7 @@ " 'vectorizer': 'none'}]}" ] }, - "execution_count": 36, + "execution_count": 116, "metadata": {}, "output_type": "execute_result" } @@ -854,7 +820,7 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 117, "id": "786d437f", "metadata": {}, "outputs": [ @@ -895,7 +861,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 118, "id": "3658693c", "metadata": {}, "outputs": [ @@ -903,26 +869,28 @@ "name": "stdout", "output_type": "stream", "text": [ - "Cave Story\n", - "is a freeware video game released in 2004 for PC. It was thought of and created over five years by Daisuke Amaya, known by his pseudonym, or art name, Pixel. The game is an action-adventure game, and is similar to the Castlevania and Metroid games. It was first made in Japanese, and was translated to English by the fan translating group, Aeon Genesis.\n", + "Kim Jong-nam\n", + "Kim Jong-nam (May 10, 1971 - February 13, 2017) was the eldest son of Kim Jong-il, the former leader of North Korea.\n", "\n", - "References \n", + "He tried to enter Japan using a fake passport in May 2001. This was to visit Disneyland. This caused his father to not approve of him. Kim Jong-nam's younger half-brother Kim Jong-un was made the heir in September 2010.\n", "\n", - "Notes\n", + "In June 2010, Kim Jong-nam gave a brief interview to the Associated Press in Macau. He told the reporter that he had \"no plans\" to defect to Europe. The press had recently said this. Kim Jong-nam lived in an apartment on the southern tip of Macau's Coloane Island until 2007. An anonymous South Korean official reported in October 2010 that Jong-nam had not lived in Macau for \"months\", and now goes between China and \"another country.\"\n", "\n", - "2004 video games\n", - "Amiga games\n", - "Dreamcast games\n", - "Freeware games\n", - "Indie video games\n", - "Nintendo 3DS games\n", - "Nintendo Switch games\n", - "MacOS games\n", - "Platform games\n", - "Sega Genesis games\n", - "Video games developed in Japan\n", - "Wii games\n", - "Windows games\n" + "When his father died, Kim Jong-nam did not attend the funeral. This was to avoid rumours on the succession.\n", + "\n", + "He was assassinated in Malaysia on February 13, 2017, which is believed to be ordered by his half-brother Kim Jong-un.\n", + "\n", + "Personal life\n", + "The South Korean newspaper The Chosun Ilbo said that Kim Jong-nam has two wives, at least one mistress, and several children. His first wife Shin Jong-hui (born c. 1980) and their son Kum-sol (born c. 1996) live at a home called Dragon Villa on the northern outskirts of Beijing. His second wife Lee Hye-kyong (born c. 1970), their son Han-sol (born c. 1995) and their daughter Sol-hui (born c. 1998) live in an apartment building in Macau. Jong-nam's mistress, former Air Koryo flight attendant So Yong-la (born c. 1980), also lives in Macau. \n", + "\n", + "Jong-nam is often given attention by the media for his gambling and extravagant spending.\n", + "\n", + "References\n", + "\n", + "1971 births\n", + "2017 deaths\n", + "Assassinated people\n", + "North Korean politicians\n" ] }, { @@ -931,7 +899,7 @@ "{'Aggregate': {'Article': [{'meta': {'count': 25000}}]}}" ] }, - "execution_count": 47, + "execution_count": 118, "metadata": {}, "output_type": "execute_result" } @@ -960,7 +928,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 119, "id": "5acd5437", "metadata": {}, "outputs": [], @@ -986,7 +954,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 120, "id": "15def653", "metadata": {}, "outputs": [ @@ -1013,7 +981,7 @@ "17. Impressionism (Score: 0.919)\n", "18. Bauhaus (Score: 0.919)\n", "19. Surrealism (Score: 0.919)\n", - "20. Expressionism (Score: 0.918)\n" + "20. Expressionism (Score: 0.919)\n" ] } ], @@ -1027,7 +995,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 85, "id": "93c4a696", "metadata": {}, "outputs": [ @@ -1035,11 +1003,11 @@ "name": "stdout", "output_type": "stream", "text": [ - "1. Historic Scotland (Score: 0.947)\n", + "1. Historic Scotland (Score: 0.946)\n", "2. First War of Scottish Independence (Score: 0.946)\n", "3. Battle of Bannockburn (Score: 0.946)\n", "4. Wars of Scottish Independence (Score: 0.944)\n", - "5. Second War of Scottish Independence (Score: 0.94)\n", + "5. Second War of Scottish Independence (Score: 0.939)\n", "6. List of Scottish monarchs (Score: 0.937)\n", "7. Scottish Borders (Score: 0.932)\n", "8. Braveheart (Score: 0.929)\n", @@ -1095,7 +1063,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 99, "id": "76d697e9", "metadata": { "ExecuteTime": { @@ -1110,7 +1078,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 100, "id": "1deeb539", "metadata": { "ExecuteTime": { @@ -1125,7 +1093,7 @@ "CollectionsResponse(collections=[])" ] }, - "execution_count": 8, + "execution_count": 100, "metadata": {}, "output_type": "execute_result" } @@ -1143,12 +1111,12 @@ "\n", "Qdrant stores data in __collections__ where each object is described by at least one vector and may contain an additional metadata called __payload__. Our collection will be called **Articles** and each object will be described by both **title** and **content** vectors.\n", "\n", - "We're going to be using an official [qdrant-client](https://github.com/qdrant/qdrant_client) package that has all the utility methods already built-in." + "We'll be using an official [qdrant-client](https://github.com/qdrant/qdrant_client) package that has all the utility methods already built-in." ] }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 101, "id": "1a84ee1d", "metadata": { "ExecuteTime": { @@ -1163,7 +1131,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 102, "id": "00876f92", "metadata": { "ExecuteTime": { @@ -1192,7 +1160,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": null, "id": "f24e76ab", "metadata": { "ExecuteTime": { @@ -1200,18 +1168,7 @@ "start_time": "2023-01-18T09:36:24.108867Z" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "UpdateResult(operation_id=0, status=)" - ] - }, - "execution_count": 37, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "qdrant.upsert(\n", " collection_name='Articles',\n", @@ -1231,7 +1188,7 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": null, "id": "d1188a12", "metadata": { "ExecuteTime": { @@ -1239,18 +1196,7 @@ "start_time": "2023-01-18T09:58:13.816248Z" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "CountResult(count=250)" - ] - }, - "execution_count": 52, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Check the collection size to make sure all the points have been stored\n", "qdrant.count(collection_name='Articles')" @@ -1263,12 +1209,12 @@ "source": [ "### Search Data\n", "\n", - "Once the data is put into Qdrant we can start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search." + "Once the data is put into Qdrant we will start querying the collection for the closest vectors. We may provide an additional parameter `vector_name` to switch from title to content based search." ] }, { "cell_type": "code", - "execution_count": 49, + "execution_count": null, "id": "f1bac4ef", "metadata": { "ExecuteTime": { @@ -1299,7 +1245,7 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": null, "id": "aa92f3d3", "metadata": { "ExecuteTime": { @@ -1307,34 +1253,7 @@ "start_time": "2023-01-18T09:50:35.711020Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "0. Art (Score: 0.841)\n", - "1. Europe (Score: 0.839)\n", - "2. Italy (Score: 0.816)\n", - "3. Architecture (Score: 0.815)\n", - "4. Madrid (Score: 0.815)\n", - "5. France (Score: 0.812)\n", - "6. Belgium (Score: 0.808)\n", - "7. Austria (Score: 0.802)\n", - "8. London (Score: 0.799)\n", - "9. History (Score: 0.797)\n", - "10. Creativity (Score: 0.796)\n", - "11. Archaeology (Score: 0.795)\n", - "12. Cartography (Score: 0.794)\n", - "13. Denmark (Score: 0.793)\n", - "14. Finland (Score: 0.79)\n", - "15. English (Score: 0.789)\n", - "16. Catharism (Score: 0.788)\n", - "17. Dublin (Score: 0.787)\n", - "18. Ireland (Score: 0.787)\n", - "19. Japan (Score: 0.787)\n" - ] - } - ], + "outputs": [], "source": [ "query_results = query_qdrant('modern art in Europe', 'Articles')\n", "for i, article in enumerate(query_results):\n", @@ -1343,7 +1262,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": null, "id": "7ed116b8", "metadata": { "ExecuteTime": { @@ -1351,36 +1270,9 @@ "start_time": "2023-01-18T09:52:55.248029Z" } }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1. History (Score: 0.797)\n", - "2. Dublin (Score: 0.787)\n", - "3. Ireland (Score: 0.786)\n", - "4. History of Australia (Score: 0.782)\n", - "5. Historian (Score: 0.778)\n", - "6. Belgium (Score: 0.776)\n", - "7. Black pudding (Score: 0.773)\n", - "8. London (Score: 0.769)\n", - "9. History of Spain (Score: 0.768)\n", - "10. Cartography (Score: 0.763)\n", - "11. March (Score: 0.762)\n", - "12. France (Score: 0.761)\n", - "13. Bubonic plague (Score: 0.76)\n", - "14. Great Lakes (Score: 0.759)\n", - "15. Inch (Score: 0.758)\n", - "16. Dissolution of the monasteries (Score: 0.758)\n", - "17. Austria (Score: 0.757)\n", - "18. English (Score: 0.757)\n", - "19. British English (Score: 0.757)\n", - "20. Armenia (Score: 0.756)\n" - ] - } - ], + "outputs": [], "source": [ - "# This time we're going to query using content vector\n", + "# This time we'll query using content vector\n", "query_results = query_qdrant('Famous battles in Scottish history', 'Articles', 'content')\n", "for i, article in enumerate(query_results):\n", " print(f'{i + 1}. {article.payload[\"title\"]} (Score: {round(article.score, 3)})')" @@ -1397,9 +1289,9 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "vectordb", "language": "python", - "name": "python3" + "name": "vectordb" }, "language_info": { "codemirror_mode": { From 3ad0e718cb44c15e59385836e316d9988308543c Mon Sep 17 00:00:00 2001 From: colin-openai Date: Mon, 6 Feb 2023 03:48:31 -0800 Subject: [PATCH 13/13] Pushing update to remove data loading --- ...ctor_databases_for_embeddings_search.ipynb | 596 +++--------------- 1 file changed, 81 insertions(+), 515 deletions(-) diff --git a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb index 964cf35..23b8b0e 100644 --- a/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb +++ b/examples/vector_databases/Using_vector_databases_for_embeddings_search.ipynb @@ -58,12 +58,15 @@ "# We'll need to install the clients for all vector databases\n", "!pip install pinecone-client\n", "!pip install weaviate-client\n", - "!pip install qdrant-client" + "!pip install qdrant-client\n", + "\n", + "#Install wget to pull zip file\n", + "!pip install wget" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "5be94df6", "metadata": {}, "outputs": [], @@ -71,14 +74,12 @@ "import openai\n", "\n", "import tiktoken\n", - "from tenacity import retry, wait_random_exponential, stop_after_attempt\n", "from typing import List, Iterator\n", - "import concurrent\n", - "from tqdm import tqdm\n", "import pandas as pd\n", - "from datasets import load_dataset\n", "import numpy as np\n", "import os\n", + "import wget\n", + "from ast import literal_eval\n", "\n", "# Pinecone's client library for Python\n", "import pinecone\n", @@ -106,287 +107,71 @@ "source": [ "## Load data\n", "\n", - "In this section we'll source the data for this task, embed it and format it for insertion into a vector database" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "bd99e08e", - "metadata": {}, - "outputs": [], - "source": [ - "# Simple function to take in a list of text objects and return them as a list of embeddings\n", - "def get_embeddings(input: List):\n", - " response = openai.Embedding.create(\n", - " input=input,\n", - " model=EMBEDDING_MODEL,\n", - " )[\"data\"]\n", - " return [data[\"embedding\"] for data in response]\n", - "\n", - "def batchify(iterable, n=1):\n", - " l = len(iterable)\n", - " for ndx in range(0, l, n):\n", - " yield iterable[ndx : min(ndx + n, l)]\n", - "\n", - "# Function for batching and parallel processing the embeddings\n", - "def embed_corpus(\n", - " corpus: List[str],\n", - " batch_size=64,\n", - " num_workers=8,\n", - " max_context_len=8191,\n", - "):\n", - "\n", - " # Encode the corpus, truncating to max_context_len\n", - " encoding = tiktoken.get_encoding(\"cl100k_base\")\n", - " encoded_corpus = [\n", - " encoded_article[:max_context_len] for encoded_article in encoding.encode_batch(corpus)\n", - " ]\n", - "\n", - " # Calculate corpus statistics: the number of inputs, the total number of tokens, and the estimated cost to embed\n", - " num_tokens = sum(len(article) for article in encoded_corpus)\n", - " cost_to_embed_tokens = num_tokens / 1_000 * 0.0004\n", - " print(\n", - " f\"num_articles={len(encoded_corpus)}, num_tokens={num_tokens}, est_embedding_cost={cost_to_embed_tokens:.2f} USD\"\n", - " )\n", - "\n", - " # Embed the corpus\n", - " with concurrent.futures.ThreadPoolExecutor(max_workers=num_workers) as executor:\n", - " \n", - " futures = [\n", - " executor.submit(get_embeddings, text_batch)\n", - " for text_batch in batchify(encoded_corpus, batch_size)\n", - " ]\n", - "\n", - " with tqdm(total=len(encoded_corpus)) as pbar:\n", - " for _ in concurrent.futures.as_completed(futures):\n", - " pbar.update(batch_size)\n", - "\n", - " embeddings = []\n", - " for future in futures:\n", - " data = future.result()\n", - " embeddings.extend(data)\n", - "\n", - " return embeddings" + "In this section we'll load embedded data that we've prepared previous to this session." ] }, { "cell_type": "code", "execution_count": null, - "id": "0c1c73cb", + "id": "5dff8b55", "metadata": {}, "outputs": [], "source": [ - "# We'll use the datasets library to pull the Simple Wikipedia dataset for embedding\n", - "dataset = list(load_dataset(\"wikipedia\", \"20220301.simple\")[\"train\"])\n", - "# Limited to 25k articles for demo purposes\n", - "dataset = dataset[:25_000] " + "embeddings_url = 'https://cdn.openai.com/API/examples/data/vector_database_wikipedia_articles_embedded.zip'\n", + "\n", + "# Warning, the file is pretty big so this will take some time\n", + "wget.download(embeddings_url)" ] }, { "cell_type": "code", - "execution_count": 6, - "id": "e6ee90ce", + "execution_count": null, + "id": "21097972", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "num_articles=25000, num_tokens=12896881, est_embedding_cost=5.16 USD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "25024it [01:06, 377.31it/s] " - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "CPU times: user 16.3 s, sys: 2.24 s, total: 18.5 s\n", - "Wall time: 1min 8s\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], + "outputs": [], "source": [ - "%%time\n", - "# Embed the article text\n", - "dataset_embeddings = embed_corpus([article[\"text\"] for article in dataset])" + "import zipfile\n", + "with zipfile.ZipFile(\"vector_database_wikipedia_articles_embedded.zip\",\"r\") as zip_ref:\n", + " zip_ref.extractall(\"../data\")\n", + " \n", + "article_df = pd.read_csv('../data/vector_database_wikipedia_articles_embedded.csv')" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "850c7215", + "execution_count": null, + "id": "1721e45d", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "num_articles=25000, num_tokens=88300, est_embedding_cost=0.04 USD\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "25024it [00:36, 683.22it/s] \n" - ] - } - ], + "outputs": [], "source": [ - "# Embed the article titles separately\n", - "title_embeddings = embed_corpus([article[\"title\"] for article in dataset])" - ] - }, - { - "cell_type": "code", - "execution_count": 122, - "id": "1410daaa", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idurltitletexttitle_vectorcontent_vectorvector_id
01https://simple.wikipedia.org/wiki/AprilAprilApril is the fourth month of the year in the J...[0.001009464613161981, -0.020700545981526375, ...[-0.011253940872848034, -0.013491976074874401,...0
12https://simple.wikipedia.org/wiki/AugustAugustAugust (Aug.) is the eighth month of the year ...[0.0009286514250561595, 0.000820168002974242, ...[0.0003609954728744924, 0.007262262050062418, ...1
26https://simple.wikipedia.org/wiki/ArtArtArt is a creative activity that expresses imag...[0.003393713850528002, 0.0061537534929811954, ...[-0.004959689453244209, 0.015772193670272827, ...2
38https://simple.wikipedia.org/wiki/AAA or a is the first letter of the English alph...[0.0153952119871974, -0.013759135268628597, 0....[0.024894846603274345, -0.022186409682035446, ...3
49https://simple.wikipedia.org/wiki/AirAirAir refers to the Earth's atmosphere. Air is a...[0.02224554680287838, -0.02044147066771984, -0...[0.021524671465158463, 0.018522677943110466, -...4
\n", - "
" - ], - "text/plain": [ - " id url title \\\n", - "0 1 https://simple.wikipedia.org/wiki/April April \n", - "1 2 https://simple.wikipedia.org/wiki/August August \n", - "2 6 https://simple.wikipedia.org/wiki/Art Art \n", - "3 8 https://simple.wikipedia.org/wiki/A A \n", - "4 9 https://simple.wikipedia.org/wiki/Air Air \n", - "\n", - " text \\\n", - "0 April is the fourth month of the year in the J... \n", - "1 August (Aug.) is the eighth month of the year ... \n", - "2 Art is a creative activity that expresses imag... \n", - "3 A or a is the first letter of the English alph... \n", - "4 Air refers to the Earth's atmosphere. Air is a... \n", - "\n", - " title_vector \\\n", - "0 [0.001009464613161981, -0.020700545981526375, ... \n", - "1 [0.0009286514250561595, 0.000820168002974242, ... \n", - "2 [0.003393713850528002, 0.0061537534929811954, ... \n", - "3 [0.0153952119871974, -0.013759135268628597, 0.... \n", - "4 [0.02224554680287838, -0.02044147066771984, -0... \n", - "\n", - " content_vector vector_id \n", - "0 [-0.011253940872848034, -0.013491976074874401,... 0 \n", - "1 [0.0003609954728744924, 0.007262262050062418, ... 1 \n", - "2 [-0.004959689453244209, 0.015772193670272827, ... 2 \n", - "3 [0.024894846603274345, -0.022186409682035446, ... 3 \n", - "4 [0.021524671465158463, 0.018522677943110466, -... 4 " - ] - }, - "execution_count": 122, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# We will then store the result in another dataframe, and prep the data for insertion into a vector DB\n", - "article_df = pd.DataFrame(dataset)\n", - "article_df['title_vector'] = title_embeddings\n", - "article_df['content_vector'] = dataset_embeddings\n", - "article_df['vector_id'] = article_df.index\n", - "article_df['vector_id'] = article_df['vector_id'].apply(str)\n", "article_df.head()" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "960b82af", + "metadata": {}, + "outputs": [], + "source": [ + "# Read vectors from strings back into a list\n", + "#article_df['title_vector'] = article_df.title_vector.apply(literal_eval)\n", + "article_df['content_vector'] = article_df.content_vector.apply(literal_eval)\n", + "\n", + "# Set vector_id to be a string\n", + "article_df['vector_id'] = article_df['vector_id'].apply(str)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a334ab8b", + "metadata": {}, + "outputs": [], + "source": [ + "len(article_df['title_vector'][0])" + ] + }, { "cell_type": "markdown", "id": "ed32fc87", @@ -406,7 +191,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "92e6152a", "metadata": {}, "outputs": [], @@ -429,7 +214,7 @@ }, { "cell_type": "code", - "execution_count": 108, + "execution_count": null, "id": "0a71c575", "metadata": {}, "outputs": [], @@ -461,21 +246,10 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": null, "id": "7ea9ad46", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['wikipedia-articles']" - ] - }, - "execution_count": 124, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Pick a name for the new index\n", "index_name = 'wikipedia-articles'\n", @@ -494,18 +268,10 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": null, "id": "5daeba00", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Uploading vectors to content namespace..\n" - ] - } - ], + "outputs": [], "source": [ "# Upsert content vectors in content namespace - this can take a few minutes\n", "print(\"Uploading vectors to content namespace..\")\n", @@ -515,18 +281,10 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": null, "id": "5fc1b083", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Uploading vectors to title namespace..\n" - ] - } - ], + "outputs": [], "source": [ "# Upsert title vectors in title namespace - this can also take a few minutes\n", "print(\"Uploading vectors to title namespace..\")\n", @@ -536,25 +294,10 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": null, "id": "f90c7fba", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'dimension': 1536,\n", - " 'index_fullness': 0.1,\n", - " 'namespaces': {'content': {'vector_count': 25000},\n", - " 'title': {'vector_count': 25000}},\n", - " 'total_vector_count': 50000}" - ] - }, - "execution_count": 128, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Check index size for each namespace to confirm all of our docs have loaded\n", "index.describe_index_stats()" @@ -584,7 +327,7 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "id": "3c8c2aa1", "metadata": {}, "outputs": [], @@ -681,7 +424,7 @@ }, { "cell_type": "code", - "execution_count": 113, + "execution_count": null, "id": "b9ea472d", "metadata": {}, "outputs": [], @@ -691,21 +434,10 @@ }, { "cell_type": "code", - "execution_count": 114, + "execution_count": null, "id": "13be220d", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'classes': []}" - ] - }, - "execution_count": 114, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "client.schema.delete_all()\n", "client.schema.get()" @@ -713,21 +445,10 @@ }, { "cell_type": "code", - "execution_count": 115, + "execution_count": null, "id": "73d33184", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "True" - ] - }, - "execution_count": 115, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "client.is_ready()" ] @@ -748,53 +469,10 @@ }, { "cell_type": "code", - "execution_count": 116, + "execution_count": null, "id": "e868d143", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'classes': [{'class': 'Article',\n", - " 'invertedIndexConfig': {'bm25': {'b': 0.75, 'k1': 1.2},\n", - " 'cleanupIntervalSeconds': 60,\n", - " 'stopwords': {'additions': None, 'preset': 'en', 'removals': None}},\n", - " 'properties': [{'dataType': ['text'],\n", - " 'description': 'Title of the article',\n", - " 'name': 'title',\n", - " 'tokenization': 'word'},\n", - " {'dataType': ['text'],\n", - " 'description': 'Contents of the article',\n", - " 'name': 'content',\n", - " 'tokenization': 'word'}],\n", - " 'shardingConfig': {'virtualPerPhysical': 128,\n", - " 'desiredCount': 1,\n", - " 'actualCount': 1,\n", - " 'desiredVirtualCount': 128,\n", - " 'actualVirtualCount': 128,\n", - " 'key': '_id',\n", - " 'strategy': 'hash',\n", - " 'function': 'murmur3'},\n", - " 'vectorIndexConfig': {'skip': False,\n", - " 'cleanupIntervalSeconds': 300,\n", - " 'maxConnections': 64,\n", - " 'efConstruction': 128,\n", - " 'ef': -1,\n", - " 'dynamicEfMin': 100,\n", - " 'dynamicEfMax': 500,\n", - " 'dynamicEfFactor': 8,\n", - " 'vectorCacheMaxObjects': 2000000,\n", - " 'flatSearchCutoff': 40000,\n", - " 'distance': 'cosine'},\n", - " 'vectorIndexType': 'hnsw',\n", - " 'vectorizer': 'none'}]}" - ] - }, - "execution_count": 116, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "class_obj = {\n", " \"class\": \"Article\",\n", @@ -820,18 +498,10 @@ }, { "cell_type": "code", - "execution_count": 117, + "execution_count": null, "id": "786d437f", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Uploading vectors to article schema..\n" - ] - } - ], + "outputs": [], "source": [ "# Convert DF into a list of tuples\n", "data_objects = []\n", @@ -861,49 +531,10 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": null, "id": "3658693c", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Kim Jong-nam\n", - "Kim Jong-nam (May 10, 1971 - February 13, 2017) was the eldest son of Kim Jong-il, the former leader of North Korea.\n", - "\n", - "He tried to enter Japan using a fake passport in May 2001. This was to visit Disneyland. This caused his father to not approve of him. Kim Jong-nam's younger half-brother Kim Jong-un was made the heir in September 2010.\n", - "\n", - "In June 2010, Kim Jong-nam gave a brief interview to the Associated Press in Macau. He told the reporter that he had \"no plans\" to defect to Europe. The press had recently said this. Kim Jong-nam lived in an apartment on the southern tip of Macau's Coloane Island until 2007. An anonymous South Korean official reported in October 2010 that Jong-nam had not lived in Macau for \"months\", and now goes between China and \"another country.\"\n", - "\n", - "When his father died, Kim Jong-nam did not attend the funeral. This was to avoid rumours on the succession.\n", - "\n", - "He was assassinated in Malaysia on February 13, 2017, which is believed to be ordered by his half-brother Kim Jong-un.\n", - "\n", - "Personal life\n", - "The South Korean newspaper The Chosun Ilbo said that Kim Jong-nam has two wives, at least one mistress, and several children. His first wife Shin Jong-hui (born c. 1980) and their son Kum-sol (born c. 1996) live at a home called Dragon Villa on the northern outskirts of Beijing. His second wife Lee Hye-kyong (born c. 1970), their son Han-sol (born c. 1995) and their daughter Sol-hui (born c. 1998) live in an apartment building in Macau. Jong-nam's mistress, former Air Koryo flight attendant So Yong-la (born c. 1980), also lives in Macau. \n", - "\n", - "Jong-nam is often given attention by the media for his gambling and extravagant spending.\n", - "\n", - "References\n", - "\n", - "1971 births\n", - "2017 deaths\n", - "Assassinated people\n", - "North Korean politicians\n" - ] - }, - { - "data": { - "text/plain": [ - "{'Aggregate': {'Article': [{'meta': {'count': 25000}}]}}" - ] - }, - "execution_count": 118, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# Test our insert has worked by checking one object\n", "print(client.data_object.get()['objects'][0]['properties']['title'])\n", @@ -928,7 +559,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": null, "id": "5acd5437", "metadata": {}, "outputs": [], @@ -954,37 +585,10 @@ }, { "cell_type": "code", - "execution_count": 120, + "execution_count": null, "id": "15def653", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1. Museum of Modern Art (Score: 0.938)\n", - "2. Western Europe (Score: 0.934)\n", - "3. Renaissance art (Score: 0.932)\n", - "4. Pop art (Score: 0.93)\n", - "5. Northern Europe (Score: 0.927)\n", - "6. Hellenistic art (Score: 0.926)\n", - "7. Modernist literature (Score: 0.924)\n", - "8. Art film (Score: 0.922)\n", - "9. Central Europe (Score: 0.921)\n", - "10. Art (Score: 0.921)\n", - "11. European (Score: 0.921)\n", - "12. Byzantine art (Score: 0.92)\n", - "13. Postmodernism (Score: 0.92)\n", - "14. Eastern Europe (Score: 0.92)\n", - "15. Cubism (Score: 0.92)\n", - "16. Europe (Score: 0.919)\n", - "17. Impressionism (Score: 0.919)\n", - "18. Bauhaus (Score: 0.919)\n", - "19. Surrealism (Score: 0.919)\n", - "20. Expressionism (Score: 0.919)\n" - ] - } - ], + "outputs": [], "source": [ "query_result = query_weaviate('modern art in Europe','Article')\n", "counter = 0\n", @@ -995,37 +599,10 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": null, "id": "93c4a696", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1. Historic Scotland (Score: 0.946)\n", - "2. First War of Scottish Independence (Score: 0.946)\n", - "3. Battle of Bannockburn (Score: 0.946)\n", - "4. Wars of Scottish Independence (Score: 0.944)\n", - "5. Second War of Scottish Independence (Score: 0.939)\n", - "6. List of Scottish monarchs (Score: 0.937)\n", - "7. Scottish Borders (Score: 0.932)\n", - "8. Braveheart (Score: 0.929)\n", - "9. John of Scotland (Score: 0.929)\n", - "10. Guardians of Scotland (Score: 0.926)\n", - "11. Holyrood Abbey (Score: 0.925)\n", - "12. Scottish (Score: 0.925)\n", - "13. Scots (Score: 0.925)\n", - "14. Robert I of Scotland (Score: 0.924)\n", - "15. Scottish people (Score: 0.924)\n", - "16. Alexander I of Scotland (Score: 0.924)\n", - "17. Edinburgh Castle (Score: 0.924)\n", - "18. Robert Burns (Score: 0.923)\n", - "19. Battle of Bosworth Field (Score: 0.922)\n", - "20. David II of Scotland (Score: 0.922)\n" - ] - } - ], + "outputs": [], "source": [ "query_result = query_weaviate('Famous battles in Scottish history','Article')\n", "counter = 0\n", @@ -1063,7 +640,7 @@ }, { "cell_type": "code", - "execution_count": 99, + "execution_count": null, "id": "76d697e9", "metadata": { "ExecuteTime": { @@ -1078,7 +655,7 @@ }, { "cell_type": "code", - "execution_count": 100, + "execution_count": null, "id": "1deeb539", "metadata": { "ExecuteTime": { @@ -1086,18 +663,7 @@ "start_time": "2023-01-18T09:29:19.727897Z" } }, - "outputs": [ - { - "data": { - "text/plain": [ - "CollectionsResponse(collections=[])" - ] - }, - "execution_count": 100, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "qdrant.get_collections()" ] @@ -1116,7 +682,7 @@ }, { "cell_type": "code", - "execution_count": 101, + "execution_count": null, "id": "1a84ee1d", "metadata": { "ExecuteTime": { @@ -1131,7 +697,7 @@ }, { "cell_type": "code", - "execution_count": 102, + "execution_count": null, "id": "00876f92", "metadata": { "ExecuteTime": {