commit 2

download.py

@@ -0,0 +1,25 @@
+import argparse
+from huggingface_hub import hf_hub_download
+
+parser = argparse.ArgumentParser(description="Download files from a Hugging Face dataset repository.")
+parser.add_argument("--repo_id", type=str, default="PeterJinGo/wiki-18-e5-index", help="Hugging Face repository ID")
+parser.add_argument("--save_path", type=str, required=True, help="Local directory to save files")
+    
+args = parser.parse_args()
+
+repo_id = "PeterJinGo/wiki-18-e5-index"
+for file in ["part_aa", "part_ab"]:
+    hf_hub_download(
+        repo_id=repo_id,
+        filename=file,  # e.g., "e5_Flat.index"
+        repo_type="dataset",
+        local_dir=args.save_path,
+    )
+
+repo_id = "PeterJinGo/wiki-18-corpus"
+hf_hub_download(
+        repo_id=repo_id,
+        filename="wiki-18.jsonl.gz",
+        repo_type="dataset",
+        local_dir=args.save_path,
+)

download_corpus.sh

@@ -0,0 +1,5 @@
+save_path=./data
+python download.py --save_path $save_path || exit -1
+mv $save_path/part_aa $save_path/e5_Flat.index
+cat $save_path/part_ab >> $save_path/e5_Flat.index && rm $save_path/part_ab
+gzip -d $save_path/wiki-18.jsonl.gz

install_env.sh

@@ -1,9 +1,10 @@
-conda create -n faiss_env python=3.10
-conda activate faiss_env
+# Run this code manually
+# conda create -n faiss_env python=3.10
+# conda activate faiss_env
 
-conda install pytorch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 pytorch-cuda=12.1 -c pytorch -c nvidia
+conda install pytorch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 pytorch-cuda=12.1 -c pytorch -c nvidia -y
 pip install transformers datasets pyserini
 
-conda install -c pytorch -c nvidia faiss-gpu=1.8.0
+conda install -c pytorch -c nvidia faiss-gpu=1.8.0 -y
 
-pip install uvicorn fastapi
+pip install uvicorn fastapi huggingface_hub