Skip to content

Commit

Permalink
emptied init file,moved HF example usage and changed print statment t…
Browse files Browse the repository at this point in the history
…o log
  • Loading branch information
abhisomala committed Jul 9, 2024
1 parent 152a99e commit 2f783ca
Show file tree
Hide file tree
Showing 3 changed files with 4 additions and 10 deletions.
7 changes: 0 additions & 7 deletions connectors/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +0,0 @@
from nomic.connectors import huggingface_connecter

atlas_dataset = huggingface_connecter.load('aaa/bbb')

atlas_dataset.create_index(topic_model=True, embedding_model='NomicEmbed')

print("Atlas dataset has been loaded and indexed successfully.")
4 changes: 2 additions & 2 deletions connectors/huggingface_connecter.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
from ulid import ULID


#need to add an init.py


# Function to fetch data from a Hugging Face dataset
def fetch_data_from_huggingface(dataset_identifier, dataset_split=None):
Expand Down Expand Up @@ -53,7 +53,7 @@ def fetch_data_from_huggingface(dataset_identifier, dataset_split=None):
else:
raise e # Re-raise other ValueErrors

# Main load function to be used as a connector
# Load function to be used as a connector
def load(dataset_identifier, dataset_split=None):
data = fetch_data_from_huggingface(dataset_identifier.strip(), dataset_split)

Expand Down
3 changes: 2 additions & 1 deletion connectors/example_usage.py → examples/HF_example_usage.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
from nomic.connectors import huggingface_connecter
import logging

# Example source url: https://huggingface.co/datasets/allenai/quartz
#Takes last two parts of url to get allenai/quartz
atlas_dataset = huggingface_connecter.load('allenai/quartz')

atlas_dataset.create_index(topic_model=True, embedding_model='NomicEmbed')

print("Atlas dataset has been loaded and indexed successfully.")
logging.info("Atlas dataset has been loaded and indexed successfully.")

0 comments on commit 2f783ca

Please sign in to comment.