From 23eb1a986c5c6fdb956a09704bf079c8f2d0a244 Mon Sep 17 00:00:00 2001 From: George Williams Date: Fri, 13 Sep 2024 13:30:20 -0700 Subject: [PATCH] removing algos in IGNORE_ALGOS config; rerender markdown --- neurips23/latitude-m4-metal-medium.md | 289 +++++----- neurips23/latitude/analysis.ipynb | 787 ++++++++++++-------------- 2 files changed, 510 insertions(+), 566 deletions(-) diff --git a/neurips23/latitude-m4-metal-medium.md b/neurips23/latitude-m4-metal-medium.md index 864cf3cf..dde7af52 100644 --- a/neurips23/latitude-m4-metal-medium.md +++ b/neurips23/latitude-m4-metal-medium.md @@ -1,190 +1,179 @@ # Eval On AMD 3GHz/16-Core + 125GB RAM + NVMe SSD (Bare Metal) - +
- - - + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + +
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K
2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K
3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.3K
hwtl_sdu_anns_filter
12
diskann
 6.3K
13
cufe
 5.4K13
cufe
 5.4K
hanns
puck-fizz
puck-fizz
diff --git a/neurips23/latitude/analysis.ipynb b/neurips23/latitude/analysis.ipynb index 26ce6266..ac38aa31 100644 --- a/neurips23/latitude/analysis.ipynb +++ b/neurips23/latitude/analysis.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 15, + "execution_count": 31, "id": "21ac7846-3ebf-4f2a-b545-be471d0ce5a8", "metadata": {}, "outputs": [], @@ -27,7 +27,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 32, "id": "936c9c86-f611-4b3f-ae6b-d9d02347a4e2", "metadata": {}, "outputs": [], @@ -46,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 33, "id": "071746d6-1cdc-404e-beef-d643596218fd", "metadata": {}, "outputs": [], @@ -58,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 34, "id": "7193be1a-a6de-43df-9f60-9243f5ecf473", "metadata": {}, "outputs": [], @@ -70,7 +70,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 35, "id": "5c83a9db-fc3f-4813-91c8-a7a1275b086d", "metadata": {}, "outputs": [], @@ -82,7 +82,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 37, "id": "303b19d1-d0a6-4f0c-9dd0-08879df9cdd6", "metadata": {}, "outputs": [], @@ -116,6 +116,9 @@ " # retrieve all participating track algorithm names via track algo subdirectory \n", " track_dir = os.path.join( COMPETITION_DIR, \"%s/*/Dockerfile\" % track )\n", " algos_participating = [os.path.basename(os.path.dirname(p)) for p in glob.glob( track_dir ) ]\n", + "\n", + " # remove any temp ignores\n", + " algos_participating = list( set(algos_participating ) - set( IGNORE_ALGOS ) ) \n", " \n", " # compute difference of algo lists of track subdirs and algos in results - these did not qualify\n", " algos_did_not_qualify = list( set(algos_participating) - set(list(ranking_df[\"algorithm\"])) )\n", @@ -141,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 38, "id": "51cf0a75-1acf-44de-b7f1-2cf030cadd94", "metadata": {}, "outputs": [ @@ -428,21 +431,11 @@ " \n", " \n", " 13\n", - " hanns\n", - " NaN\n", - " NaN\n", - " error\n", - " 14\n", - " ood\n", - " text2image-10M\n", - " \n", - " \n", - " 14\n", " puck-fizz\n", " NaN\n", " NaN\n", " error\n", - " 15\n", + " 14\n", " ood\n", " text2image-10M\n", " \n", @@ -567,8 +560,7 @@ "10 epsearch 7.66 0.92 ok 11 ood \n", "11 diskann 6.30 0.91 ok 12 ood \n", "12 cufe 5.40 0.91 ok 13 ood \n", - "13 hanns NaN NaN error 14 ood \n", - "14 puck-fizz NaN NaN error 15 ood \n", + "13 puck-fizz NaN NaN error 14 ood \n", "0 zilliz 34.76 0.93 ok 1 sparse \n", "1 pyanns 26.86 0.92 ok 2 sparse \n", "2 pinecone_smips 12.00 0.90 ok 3 sparse \n", @@ -606,7 +598,6 @@ "11 text2image-10M \n", "12 text2image-10M \n", "13 text2image-10M \n", - "14 text2image-10M \n", "0 sparse-full \n", "1 sparse-full \n", "2 sparse-full \n", @@ -618,7 +609,7 @@ "8 sparse-full " ] }, - "execution_count": 21, + "execution_count": 38, "metadata": {}, "output_type": "execute_result" } @@ -631,7 +622,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 39, "id": "fbc47e1b-a441-413a-8166-bb834a386cc3", "metadata": {}, "outputs": [ @@ -888,21 +879,11 @@ " \n", " \n", " 13\n", - " hanns\n", - " NaN\n", - " NaN\n", - " error\n", - " 14\n", - " ood\n", - " text2image-10M\n", - " \n", - " \n", - " 14\n", " puck-fizz\n", " NaN\n", " NaN\n", " error\n", - " 15\n", + " 14\n", " ood\n", " text2image-10M\n", " \n", @@ -1054,8 +1035,7 @@ "10 epsearch 7.66 0.92 ok 11 ood \n", "11 diskann 6.30 0.91 ok 12 ood \n", "12 cufe 5.40 0.91 ok 13 ood \n", - "13 hanns NaN NaN error 14 ood \n", - "14 puck-fizz NaN NaN error 15 ood \n", + "13 puck-fizz NaN NaN error 14 ood \n", "0 zilliz 213.29 0.93 ok 1 filter \n", "1 pinecone 146.72 0.92 ok 2 filter \n", "2 puck 62.26 0.94 ok 3 filter \n", @@ -1093,7 +1073,6 @@ "11 text2image-10M \n", "12 text2image-10M \n", "13 text2image-10M \n", - "14 text2image-10M \n", "0 yfcc-10M \n", "1 yfcc-10M \n", "2 yfcc-10M \n", @@ -1108,7 +1087,7 @@ "11 yfcc-10M " ] }, - "execution_count": 22, + "execution_count": 39, "metadata": {}, "output_type": "execute_result" } @@ -1122,7 +1101,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 40, "id": "c90fba89-728a-4f54-bb80-95834bd33005", "metadata": {}, "outputs": [], @@ -1138,7 +1117,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 41, "id": "f4e617b4-112b-45e1-bc3b-2692d980d19f", "metadata": {}, "outputs": [], @@ -1153,7 +1132,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 42, "id": "8d2653b4-f070-4761-8b5e-9a9fa03f19cc", "metadata": {}, "outputs": [], @@ -1166,7 +1145,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 43, "id": "d0accb53-c34b-418e-85a4-253c46cdb90b", "metadata": {}, "outputs": [], @@ -1189,7 +1168,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 44, "id": "5cee1395-0e41-4ba4-b586-f9c4b56a9024", "metadata": {}, "outputs": [ @@ -1198,25 +1177,25 @@ "text/html": [ "\n", - "\n", + "
\n", " \n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", @@ -1233,193 +1212,181 @@ " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
trackfiltersparseoodfiltersparseood
 rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
rank
11zilliz213.2850591zilliz34.7599761scann107.413009
22pinecone146.7212422pyanns26.8585192pinecone-ood76.865423
33puck62.2576953pinecone_smips12.0031823zilliz73.509610
44parlayivf55.0331984shnsw8.2472774pyanns55.504352
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855spmatnan8vamana19.965517
99cufe6.318462sustech-whunan9puck18.960224
10dhqnannannan10ngt11.921399
11fdufilterdiskannnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.295839
13nannannannan13cufe5.404492
14nannannannanhannsnan
15nannannannanpuck-fizznan11zilliz213.2850591zilliz34.7599761scann107.413009
22pinecone146.7212422pyanns26.8585192pinecone-ood76.865423
33puck62.2576953pinecone_smips12.0031823zilliz73.509610
44parlayivf55.0331984shnsw8.2472774pyanns55.504352
55wm_filter20.8748575nle2.9456265sustech-ood28.458263
66pyanns8.9880226cufe0.0849766mysteryann-dif27.946097
77faissplus8.4932647linscan0.0607567mysteryann26.560867
88faiss7.327855spmatnan8vamana19.965517
99cufe6.318462sustech-whunan9puck18.960224
10dhqnannannan10ngt11.921399
11fdufilterdiskannnannannan11epsearch7.663219
12hwtl_sdu_anns_filternannannan12diskann6.295839
13nannannannan13cufe5.404492
14nannannannanpuck-fizznan
\n" ], "text/plain": [ - "" + "" ] }, - "execution_count": 27, + "execution_count": 44, "metadata": {}, "output_type": "execute_result" } @@ -1433,7 +1400,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 45, "id": "884f5113-01a9-42a7-9262-5cea8e9cd751", "metadata": {}, "outputs": [ @@ -1446,197 +1413,185 @@ "WARNING: Using error file path for filter/dhq\n", "WARNING: Using error file path for filter/fdufilterdiskann\n", "WARNING: Using error file path for filter/hwtl_sdu_anns_filter\n", - "WARNING: Ignoring hanns\n", "WARNING: Using error file path for ood/puck-fizz\n" ] }, { "data": { "text/html": [ - "\n", + "
\n", " \n", " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", " \n", " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " \n", "
filtersparseoodfiltersparseood
rankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqpsrankalgorithmqps
1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K
2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K
3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.3K
13
cufe
 5.4K
hanns
puck-fizz
1
zilliz
 213.3K1
zilliz
 34.8K1
scann
 107.4K
2
pinecone
 146.7K2
pyanns
 26.9K2
pinecone-ood
 76.9K
3
puck
 62.3K3
pinecone_smips
 12.0K3
zilliz
 73.5K
4
parlayivf
 55.0K4
shnsw
 8.2K4
pyanns
 55.5K
5
wm_filter
 20.9K5
nle
 2.9K5
sustech-ood
 28.5K
6
pyanns
 9.0K6
cufe
 0.1K6
mysteryann-dif
 27.9K
7
faissplus
 8.5K7
linscan
 0.1K7
mysteryann
 26.6K
8
faiss
 7.3K
spmat
8
vamana
 20.0K
9
cufe
 6.3K
sustech-whu
9
puck
 19.0K
dhq
10
ngt
 11.9K
fdufilterdiskann
11
epsearch
 7.7K
hwtl_sdu_anns_filter
12
diskann
 6.3K
13
cufe
 5.4K
puck-fizz
\n" @@ -1645,7 +1600,7 @@ "" ] }, - "execution_count": 28, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -1681,7 +1636,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 46, "id": "25fc371b-1a14-4fca-97de-acd79070c0ad", "metadata": {}, "outputs": [