Skip to content

Commit

Permalink
fix broken faiss save/load functionality (#880)
Browse files Browse the repository at this point in the history
Fixes issues #789 and #853 by saving/loading docstore and
index_to_docstore_id alongside index
  • Loading branch information
ShreyJ1729 authored Feb 7, 2023
1 parent ba5a2f0 commit d9b32ba
Showing 1 changed file with 27 additions and 8 deletions.
35 changes: 27 additions & 8 deletions langchain/vectorstores/faiss.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""Wrapper around FAISS vector database."""
from __future__ import annotations

import os
import pickle
import uuid
from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple

Expand Down Expand Up @@ -201,20 +203,37 @@ def from_texts(
)
return cls(embedding.embed_query, index, docstore, index_to_id)

def save_local(self, path: str) -> None:
"""Save FAISS index to disk.
def save_local(self, folder_path: str) -> None:
"""Save FAISS index, docstore, and index_to_docstore_id to disk.
Args:
path: Path to save FAISS index to.
path: folder path to save index, docstore, and index_to_docstore_id to.
"""
if not os.path.exists(folder_path):
os.mkdir(folder_path)

# save index separately since it is not picklable
faiss = dependable_faiss_import()
faiss.write_index(self.index, path)
faiss.write_index(self.index, os.path.join(folder_path, "index.faiss"))

# save docstore and index_to_docstore_id
pickle.dump(
(self.docstore, self.index_to_docstore_id),
open(os.path.join(folder_path, "index.pkl"), "wb"),
)

def load_local(self, path: str) -> None:
"""Load FAISS index from disk.
def load_local(self, folder_path: str) -> None:
"""Load FAISS index, docstore, and index_to_docstore_id to disk.
Args:
path: Path to load FAISS index from.
path: folder path to load index, docstore, and index_to_docstore_id from.
"""

# load index separately since it is not picklable
faiss = dependable_faiss_import()
self.index = faiss.read_index(path)
self.index = faiss.read_index(os.path.join(folder_path, "index.faiss"))

# load docstore and index_to_docstore_id
self.docstore, self.index_to_docstore_id = pickle.load(
open(os.path.join(folder_path, "index.pkl"), "rb")
)

0 comments on commit d9b32ba

Please sign in to comment.