Adding config files for remaining (relevant) ann-benchmarks million-s…

…cale datasets (rapidsai#1761) Also exposing new `dataset_memtype` option for the RAFT index types that allows us to specify (in the config files) for each index whether we want the dataset to be fed in from "host", "device" or "mmap" (host memory-mapped). Authors: - Corey J. Nolet (https://github.com/cjnolet) Approvers: - Divye Gala (https://github.com/divyegala) URL: rapidsai#1761
benfred · Aug 24, 2023 · 08a1fad · 08a1fad
1 parent d5bd840
commit 08a1fad
Show file tree

Hide file tree

Showing 23 changed files with 10,950 additions and 30 deletions.
diff --git a/bench/ann/conf/bigann-100M.json b/bench/ann/conf/bigann-100M.json
@@ -17,6 +17,7 @@
     {
       "name": "raft_ivf_pq.dimpq64-cluster5K-float-float",
       "algo": "raft_ivf_pq",
+      "dataset_memtype": "host",
       "build_param": {
         "niter": 25,
         "nlist": 5000,
@@ -149,6 +150,7 @@
     {
       "name" : "raft_ivf_flat.nlist100K",
       "algo" : "raft_ivf_flat",
+      "dataset_memtype": "host",
       "build_param": {
         "nlist" : 100000,
         "niter" : 25,
@@ -171,6 +173,7 @@
     {
       "name" : "raft_cagra.dim32",
       "algo" : "raft_cagra",
+      "dataset_memtype": "host",
       "build_param": {
         "index_dim" : 32
       },
@@ -187,6 +190,7 @@
     {
       "name" : "raft_cagra.dim64",
       "algo" : "raft_cagra",
+      "dataset_memtype": "host",
       "build_param": {
         "index_dim" : 64
       },

diff --git a/bench/ann/conf/deep-100M.json b/bench/ann/conf/deep-100M.json
@@ -199,6 +199,7 @@
     {
       "name" : "raft_ivf_flat.nlist10K",
       "algo" : "raft_ivf_flat",
+      "dataset_memtype": "host",
       "build_param": {
         "nlist" : 10000,
         "niter" : 25,
@@ -222,6 +223,7 @@
     {
       "name" : "raft_ivf_flat.nlist100K",
       "algo" : "raft_ivf_flat",
+      "dataset_memtype": "host",
       "build_param": {
         "nlist" : 100000,
         "niter" : 25,
@@ -246,6 +248,7 @@
     {
       "name" : "raft_ivf_pq.nlist10K",
       "algo" : "raft_ivf_pq",
+      "dataset_memtype": "host",
       "build_param": {
         "nlist" : 10000,
         "niter" : 25,
@@ -269,6 +272,7 @@
     {
       "name" : "raft_ivf_pq.nlist10Kdim64",
       "algo" : "raft_ivf_pq",
+      "dataset_memtype": "host",
       "build_param": {
         "nlist" : 10000,
         "niter" : 25,
@@ -293,6 +297,7 @@
     {
       "name" : "raft_ivf_pq.nlist10Kdim32",
       "algo" : "raft_ivf_pq",
+      "dataset_memtype": "host",
       "build_param": {
         "nlist" : 10000,
         "niter" : 25,
@@ -317,6 +322,7 @@
     {
       "name" : "raft_ivf_pq.nlist100K",
       "algo" : "raft_ivf_pq",
+      "dataset_memtype": "host",
       "build_param": {
         "nlist" : 100000,
         "niter" : 25,
@@ -340,6 +346,7 @@
     {
       "name" : "raft_ivf_pq.nlist100Kdim64",
       "algo" : "raft_ivf_pq",
+      "dataset_memtype": "host",
       "build_param": {
         "nlist" : 100000,
         "niter" : 25,
@@ -364,6 +371,7 @@
     {
       "name" : "raft_ivf_pq.nlist100Kdim32",
       "algo" : "raft_ivf_pq",
+      "dataset_memtype": "host",
       "build_param": {
         "nlist" : 100000,
         "niter" : 25,
@@ -389,6 +397,7 @@
     {
       "name" : "raft_cagra.dim32",
       "algo" : "raft_cagra",
+      "dataset_memtype": "host",
       "build_param": {
         "index_dim": 32,
         "intermediate_graph_degree": 48
@@ -619,6 +628,7 @@
     {
       "name" : "raft_cagra.dim64",
       "algo" : "raft_cagra",
+      "dataset_memtype": "host",
       "build_param": {
         "index_dim": 64
       },