From f15257ae914295a678391fcb52ae1ce0bfaa5f6f Mon Sep 17 00:00:00 2001 From: Sounak Chakraborty Date: Wed, 25 Oct 2023 17:35:52 -0700 Subject: [PATCH 1/7] Initial benchmark Changes --- ann_benchmarks/algorithms/deeplake/Dockerfile | 8 ++ ann_benchmarks/algorithms/deeplake/config.yml | 45 ++++++++++ ann_benchmarks/algorithms/deeplake/module.py | 84 +++++++++++++++++++ ann_benchmarks/runner.py | 2 + 4 files changed, 139 insertions(+) create mode 100644 ann_benchmarks/algorithms/deeplake/Dockerfile create mode 100644 ann_benchmarks/algorithms/deeplake/config.yml create mode 100644 ann_benchmarks/algorithms/deeplake/module.py diff --git a/ann_benchmarks/algorithms/deeplake/Dockerfile b/ann_benchmarks/algorithms/deeplake/Dockerfile new file mode 100644 index 000000000..098d0c0e3 --- /dev/null +++ b/ann_benchmarks/algorithms/deeplake/Dockerfile @@ -0,0 +1,8 @@ +#WORKDIR /home/app +FROM ann-benchmarks +ENV BUGGER_OFF=true +#COPY requirements.txt run_algorithm.py ./ + +RUN python3.10 -m pip install deeplake +RUN python3.10 -c 'import deeplake' +RUN python3.10 -c 'pip install data/libdeeplake-0.0.84-cp310-cp310-manylinux2010_x86_64.whl' \ No newline at end of file diff --git a/ann_benchmarks/algorithms/deeplake/config.yml b/ann_benchmarks/algorithms/deeplake/config.yml new file mode 100644 index 000000000..a554b308f --- /dev/null +++ b/ann_benchmarks/algorithms/deeplake/config.yml @@ -0,0 +1,45 @@ +float: + any: + - base_args: ['@metric'] + constructor: ActiveloopHnsw + disabled: false + docker_tag: ann-benchmarks-activeloop + module: ann_benchmarks.algorithms.activeloop + name: Activeloop + run_groups: + M-12: + arg_groups: [{M: 12, efConstruction: 500}] + args: {} + query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] + M-16: + arg_groups: [{M: 16, efConstruction: 500}] + args: {} + query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] + M-24: + arg_groups: [{M: 24, efConstruction: 500}] + args: {} + query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] + M-36: + arg_groups: [{M: 36, efConstruction: 500}] + args: {} + query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] + M-4: + arg_groups: [{M: 4, efConstruction: 500}] + args: {} + query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] + M-48: + arg_groups: [{M: 48, efConstruction: 500}] + args: {} + query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] + M-64: + arg_groups: [{M: 64, efConstruction: 500}] + args: {} + query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] + M-8: + arg_groups: [{M: 8, efConstruction: 500}] + args: {} + query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] + M-96: + arg_groups: [{M: 96, efConstruction: 500}] + args: {} + query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] \ No newline at end of file diff --git a/ann_benchmarks/algorithms/deeplake/module.py b/ann_benchmarks/algorithms/deeplake/module.py new file mode 100644 index 000000000..ca271da5c --- /dev/null +++ b/ann_benchmarks/algorithms/deeplake/module.py @@ -0,0 +1,84 @@ +import deeplake +from deeplake.core.vectorstore.vector_search.indra.index import METRIC_TO_INDEX_METRIC +from deeplake.core.vectorstore.deeplake_vectorstore import DeepLakeVectorStore +import os +import threading +import subprocess +import numpy as np +from ..base.module import BaseANN + + +# Class using the Deeplake implementation of an HNSW index for nearest neighbor +# search over data points in a high dimensional vector space. + +class DeeplakeHnsw(BaseANN): + def __init__(self, metric, param, enable_normalize = True, dimension = None): + + if metric not in ("angular", "euclidean"): + raise NotImplementedError("Deeplake doesn't support metric %s" % metric) + #self.metric = {"angular": METRIC_TO_INDEX_METRIC["COS"], "euclidean": METRIC_TO_INDEX_METRIC["L2"]}[metric] + if metric == "angular": + self.metric = "COS" + else: + self.metric = "L2" + self.param = param + self._ef_construction = param.get("efConstruction", 200) + self._m = param.get("M", 16) + self.dimension = dimension + self.neighbors_to_explore = 200 + self.local_path = "ANN_benchmarks-embeddings" + self.name = "Deeplake" + self.my_token = os.environ.get('ACTIVELOOP_TOKEN') + + def list_open_files(self): + pid = os.getpid() + result = subprocess.check_output(['lsof', '-p', str(pid)]) + return result.decode('utf-8') + + + def __del__(self): + # Cleanup code here + if hasattr(self, 'index'): + del self.index + + def fit(self, X): + self.index = DeepLakeVectorStore(path=self.local_path, + overwrite=True, + verbose=True, + exec_option="compute_engine", + index_params={"threshold": 5, "distance_metric": self.metric, + "additional_params": + {"efConstruction": self._ef_construction, "M": self._m, }}, + token=self.my_token) + + num_of_items, embedding_dim = X.shape + ids = [f"{i}" for i in range(num_of_items)] + # Creating embeddings of float32 as dtype of embedding tensor is float32. + embedding = X + text = ["aadfv" for i in range(num_of_items)] + metadata = [{"key": i} for i in range(num_of_items)] + + self.index.add( + id=ids, + text=text, + embedding=embedding, + metadata=metadata, + ) + ds2 = self.index.dataset + emb_tensor = ds2.embedding + self.index_al = emb_tensor.load_vdb_index("hnsw_1") + + def set_query_arguments(self, ef): + return + + def query(self, v, n): + v_float = np.array(v).astype(np.float32) + view = self.index_al.search_knn(v_float, n) + return view.indices + + def __str__(self): + return f"Deeplake(m={self._m}, ef_construction={self._ef_construction})" + + def freeIndex(self): + print("Clearing out self.index") + del self.index \ No newline at end of file diff --git a/ann_benchmarks/runner.py b/ann_benchmarks/runner.py index 1f31e41eb..bd112cfda 100644 --- a/ann_benchmarks/runner.py +++ b/ann_benchmarks/runner.py @@ -319,6 +319,7 @@ def run_docker( if mem_limit is None: mem_limit = psutil.virtual_memory().available + active_loop_token = os.environ.get('ACTIVELOOP_TOKEN') container = client.containers.run( definition.docker_tag, cmd, @@ -327,6 +328,7 @@ def run_docker( os.path.abspath("data"): {"bind": "/home/app/data", "mode": "ro"}, os.path.abspath("results"): {"bind": "/home/app/results", "mode": "rw"}, }, + environment={"ACTIVELOOP_TOKEN": active_loop_token, "BUGGER_OFF": "true"}, cpuset_cpus=cpu_limit, mem_limit=mem_limit, detach=True, From 29b5890d0eacabec348b8f9a3227238a9568dccf Mon Sep 17 00:00:00 2001 From: Sasun Hambardzumyan Date: Thu, 26 Oct 2023 17:21:15 +0000 Subject: [PATCH 2/7] Renamed to deeplake. --- ann_benchmarks/algorithms/deeplake/config.yml | 10 +++++----- ann_benchmarks/algorithms/deeplake/module.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ann_benchmarks/algorithms/deeplake/config.yml b/ann_benchmarks/algorithms/deeplake/config.yml index a554b308f..057baafa8 100644 --- a/ann_benchmarks/algorithms/deeplake/config.yml +++ b/ann_benchmarks/algorithms/deeplake/config.yml @@ -1,11 +1,11 @@ float: any: - base_args: ['@metric'] - constructor: ActiveloopHnsw + constructor: DeeplakeHnsw disabled: false - docker_tag: ann-benchmarks-activeloop - module: ann_benchmarks.algorithms.activeloop - name: Activeloop + docker_tag: ann-benchmarks-deeplake + module: ann_benchmarks.algorithms.deeplake + name: Deeplake run_groups: M-12: arg_groups: [{M: 12, efConstruction: 500}] @@ -42,4 +42,4 @@ float: M-96: arg_groups: [{M: 96, efConstruction: 500}] args: {} - query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] \ No newline at end of file + query_args: [[10, 20, 40, 80, 120, 200, 400, 600, 800]] diff --git a/ann_benchmarks/algorithms/deeplake/module.py b/ann_benchmarks/algorithms/deeplake/module.py index ca271da5c..46050660f 100644 --- a/ann_benchmarks/algorithms/deeplake/module.py +++ b/ann_benchmarks/algorithms/deeplake/module.py @@ -81,4 +81,4 @@ def __str__(self): def freeIndex(self): print("Clearing out self.index") - del self.index \ No newline at end of file + del self.index From 5c19c88aabaeaea5755463596df9692b4551637e Mon Sep 17 00:00:00 2001 From: Sasun Hambardzumyan Date: Fri, 27 Oct 2023 08:46:10 +0000 Subject: [PATCH 3/7] Finalize. --- ann_benchmarks/algorithms/deeplake/Dockerfile | 8 +-- ann_benchmarks/algorithms/deeplake/config.yml | 2 +- ann_benchmarks/algorithms/deeplake/module.py | 64 ++++++------------- 3 files changed, 24 insertions(+), 50 deletions(-) diff --git a/ann_benchmarks/algorithms/deeplake/Dockerfile b/ann_benchmarks/algorithms/deeplake/Dockerfile index 098d0c0e3..9a12d7b3a 100644 --- a/ann_benchmarks/algorithms/deeplake/Dockerfile +++ b/ann_benchmarks/algorithms/deeplake/Dockerfile @@ -1,8 +1,6 @@ -#WORKDIR /home/app FROM ann-benchmarks ENV BUGGER_OFF=true -#COPY requirements.txt run_algorithm.py ./ -RUN python3.10 -m pip install deeplake -RUN python3.10 -c 'import deeplake' -RUN python3.10 -c 'pip install data/libdeeplake-0.0.84-cp310-cp310-manylinux2010_x86_64.whl' \ No newline at end of file +RUN apt-get install -y python-setuptools python-pip +RUN pip3 install deeplake +RUN python3 -c 'import deeplake' diff --git a/ann_benchmarks/algorithms/deeplake/config.yml b/ann_benchmarks/algorithms/deeplake/config.yml index 057baafa8..6ccf2eea1 100644 --- a/ann_benchmarks/algorithms/deeplake/config.yml +++ b/ann_benchmarks/algorithms/deeplake/config.yml @@ -5,7 +5,7 @@ float: disabled: false docker_tag: ann-benchmarks-deeplake module: ann_benchmarks.algorithms.deeplake - name: Deeplake + name: deeplake run_groups: M-12: arg_groups: [{M: 12, efConstruction: 500}] diff --git a/ann_benchmarks/algorithms/deeplake/module.py b/ann_benchmarks/algorithms/deeplake/module.py index 46050660f..1d6703cc6 100644 --- a/ann_benchmarks/algorithms/deeplake/module.py +++ b/ann_benchmarks/algorithms/deeplake/module.py @@ -13,72 +13,48 @@ class DeeplakeHnsw(BaseANN): def __init__(self, metric, param, enable_normalize = True, dimension = None): - if metric not in ("angular", "euclidean"): - raise NotImplementedError("Deeplake doesn't support metric %s" % metric) - #self.metric = {"angular": METRIC_TO_INDEX_METRIC["COS"], "euclidean": METRIC_TO_INDEX_METRIC["L2"]}[metric] + raise NotImplementedError(f"Deeplake doesn't support metric {metric}") if metric == "angular": - self.metric = "COS" + self.metric = "cosine_similarity" else: - self.metric = "L2" + self.metric = "l2_norm" self.param = param self._ef_construction = param.get("efConstruction", 200) self._m = param.get("M", 16) self.dimension = dimension - self.neighbors_to_explore = 200 self.local_path = "ANN_benchmarks-embeddings" - self.name = "Deeplake" - self.my_token = os.environ.get('ACTIVELOOP_TOKEN') - - def list_open_files(self): - pid = os.getpid() - result = subprocess.check_output(['lsof', '-p', str(pid)]) - return result.decode('utf-8') - + self.name = "deeplake" + self.token = os.environ.get('ACTIVELOOP_TOKEN') def __del__(self): - # Cleanup code here + if hasattr(self, 'ds'): + del self.ds if hasattr(self, 'index'): del self.index def fit(self, X): - self.index = DeepLakeVectorStore(path=self.local_path, - overwrite=True, - verbose=True, - exec_option="compute_engine", - index_params={"threshold": 5, "distance_metric": self.metric, - "additional_params": - {"efConstruction": self._ef_construction, "M": self._m, }}, - token=self.my_token) - - num_of_items, embedding_dim = X.shape - ids = [f"{i}" for i in range(num_of_items)] - # Creating embeddings of float32 as dtype of embedding tensor is float32. - embedding = X - text = ["aadfv" for i in range(num_of_items)] - metadata = [{"key": i} for i in range(num_of_items)] - - self.index.add( - id=ids, - text=text, - embedding=embedding, - metadata=metadata, - ) - ds2 = self.index.dataset - emb_tensor = ds2.embedding - self.index_al = emb_tensor.load_vdb_index("hnsw_1") + self.ds = deeplake.dataset(self.local_path, overwrite=True, token=self.token) + self.ds.create_tensor("embedding", htype="embedding", dtype="float32") + self.ds.embedding.extend(X) + self.ds.embedding.create_vdb_index("hnsw_1", distance=self.metric, additional_params={ + "efConstruction": self._ef_construction, "M": self._m + }) + self.index = self.ds.embedding.load_vdb_index("hnsw_1") def set_query_arguments(self, ef): - return + self.index.set_search_params(ef=ef) def query(self, v, n): v_float = np.array(v).astype(np.float32) - view = self.index_al.search_knn(v_float, n) + view = self.index.search_knn(v_float, n) return view.indices def __str__(self): return f"Deeplake(m={self._m}, ef_construction={self._ef_construction})" def freeIndex(self): - print("Clearing out self.index") - del self.index + if hasattr(self, 'ds'): + del self.ds + if hasattr(self, 'index'): + del self.index From a32e26205d264d90027ca19538629383e0b06ab3 Mon Sep 17 00:00:00 2001 From: Sasun Hambardzumyan Date: Fri, 27 Oct 2023 09:51:05 +0000 Subject: [PATCH 4/7] Cleanup and finalization. --- ann_benchmarks/algorithms/deeplake/module.py | 23 ++++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ann_benchmarks/algorithms/deeplake/module.py b/ann_benchmarks/algorithms/deeplake/module.py index 1d6703cc6..a7f8404db 100644 --- a/ann_benchmarks/algorithms/deeplake/module.py +++ b/ann_benchmarks/algorithms/deeplake/module.py @@ -1,10 +1,8 @@ import deeplake -from deeplake.core.vectorstore.vector_search.indra.index import METRIC_TO_INDEX_METRIC -from deeplake.core.vectorstore.deeplake_vectorstore import DeepLakeVectorStore -import os -import threading -import subprocess import numpy as np +import os +import random +import string from ..base.module import BaseANN @@ -23,15 +21,13 @@ def __init__(self, metric, param, enable_normalize = True, dimension = None): self._ef_construction = param.get("efConstruction", 200) self._m = param.get("M", 16) self.dimension = dimension - self.local_path = "ANN_benchmarks-embeddings" + suffix = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(8)) + self.local_path = f"ANN_benchmarks-embeddings_{suffix}" self.name = "deeplake" self.token = os.environ.get('ACTIVELOOP_TOKEN') def __del__(self): - if hasattr(self, 'ds'): - del self.ds - if hasattr(self, 'index'): - del self.index + self.freeIndex() def fit(self, X): self.ds = deeplake.dataset(self.local_path, overwrite=True, token=self.token) @@ -54,7 +50,10 @@ def __str__(self): return f"Deeplake(m={self._m}, ef_construction={self._ef_construction})" def freeIndex(self): - if hasattr(self, 'ds'): - del self.ds if hasattr(self, 'index'): del self.index + if hasattr(self, 'ds'): + del self.ds + deeplake.delete(self.local_path) + if os.path.isfile(f"/tmp/{self.local_path}/embedding"): + os.remove(f"/tmp/{self.local_path}/embedding") From 2bcc831a4c6dbec1cc12e44ea90ab7454b622088 Mon Sep 17 00:00:00 2001 From: Sasun Hambardzumyan Date: Sat, 11 Nov 2023 12:14:57 +0000 Subject: [PATCH 5/7] Added org and token. --- ann_benchmarks/algorithms/deeplake/Dockerfile | 1 + ann_benchmarks/algorithms/deeplake/module.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ann_benchmarks/algorithms/deeplake/Dockerfile b/ann_benchmarks/algorithms/deeplake/Dockerfile index 9a12d7b3a..e0fe78527 100644 --- a/ann_benchmarks/algorithms/deeplake/Dockerfile +++ b/ann_benchmarks/algorithms/deeplake/Dockerfile @@ -1,5 +1,6 @@ FROM ann-benchmarks ENV BUGGER_OFF=true +ENV ACTIVELOOP_TOKEN=eyJhbGciOiJIUzUxMiIsImlhdCI6MTY4MTI4Mzc3NywiZXhwIjoxNzEyOTA2MDk5fQ.eyJpZCI6Im5vdGlmeSJ9.C3zTjQODfq0TUhkdRye639aKJ0FOanZuCwL2ks2NiKXJ6YecYVMBrdFu3AabGsk7iuS-ELtQYp7WxITv76hcSg RUN apt-get install -y python-setuptools python-pip RUN pip3 install deeplake diff --git a/ann_benchmarks/algorithms/deeplake/module.py b/ann_benchmarks/algorithms/deeplake/module.py index a7f8404db..6a8bb0039 100644 --- a/ann_benchmarks/algorithms/deeplake/module.py +++ b/ann_benchmarks/algorithms/deeplake/module.py @@ -25,13 +25,14 @@ def __init__(self, metric, param, enable_normalize = True, dimension = None): self.local_path = f"ANN_benchmarks-embeddings_{suffix}" self.name = "deeplake" self.token = os.environ.get('ACTIVELOOP_TOKEN') + self.org = os.environ.get('ACTIVELOOP_ORG') def __del__(self): self.freeIndex() def fit(self, X): - self.ds = deeplake.dataset(self.local_path, overwrite=True, token=self.token) - self.ds.create_tensor("embedding", htype="embedding", dtype="float32") + self.ds = deeplake.dataset(self.local_path, overwrite=True, token=self.token, org_id=self.org) + self.ds.create_tensor("embedding", htype="embedding", dtype="float32", create_shape_tensor=False, create_id_tensor=False) self.ds.embedding.extend(X) self.ds.embedding.create_vdb_index("hnsw_1", distance=self.metric, additional_params={ "efConstruction": self._ef_construction, "M": self._m From 1d25f8294deaebc473b878646d4de079a989643e Mon Sep 17 00:00:00 2001 From: Sasun Hambardzumyan Date: Sat, 11 Nov 2023 12:18:07 +0000 Subject: [PATCH 6/7] Remove unneeded code. --- ann_benchmarks/runner.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/ann_benchmarks/runner.py b/ann_benchmarks/runner.py index bd112cfda..1f31e41eb 100644 --- a/ann_benchmarks/runner.py +++ b/ann_benchmarks/runner.py @@ -319,7 +319,6 @@ def run_docker( if mem_limit is None: mem_limit = psutil.virtual_memory().available - active_loop_token = os.environ.get('ACTIVELOOP_TOKEN') container = client.containers.run( definition.docker_tag, cmd, @@ -328,7 +327,6 @@ def run_docker( os.path.abspath("data"): {"bind": "/home/app/data", "mode": "ro"}, os.path.abspath("results"): {"bind": "/home/app/results", "mode": "rw"}, }, - environment={"ACTIVELOOP_TOKEN": active_loop_token, "BUGGER_OFF": "true"}, cpuset_cpus=cpu_limit, mem_limit=mem_limit, detach=True, From 1f9d02a1880737440e532aa32e5838c6fe960951 Mon Sep 17 00:00:00 2001 From: Sasun Hambardzumyan Date: Sat, 11 Nov 2023 16:19:09 +0000 Subject: [PATCH 7/7] Added org to env. --- ann_benchmarks/algorithms/deeplake/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/ann_benchmarks/algorithms/deeplake/Dockerfile b/ann_benchmarks/algorithms/deeplake/Dockerfile index e0fe78527..224022dee 100644 --- a/ann_benchmarks/algorithms/deeplake/Dockerfile +++ b/ann_benchmarks/algorithms/deeplake/Dockerfile @@ -1,6 +1,7 @@ FROM ann-benchmarks ENV BUGGER_OFF=true ENV ACTIVELOOP_TOKEN=eyJhbGciOiJIUzUxMiIsImlhdCI6MTY4MTI4Mzc3NywiZXhwIjoxNzEyOTA2MDk5fQ.eyJpZCI6Im5vdGlmeSJ9.C3zTjQODfq0TUhkdRye639aKJ0FOanZuCwL2ks2NiKXJ6YecYVMBrdFu3AabGsk7iuS-ELtQYp7WxITv76hcSg +ENV ACTIVELOOP_ORG=notify RUN apt-get install -y python-setuptools python-pip RUN pip3 install deeplake