cb1716pics commited on
Commit
973db40
·
verified ·
1 Parent(s): 9bfc86c

Upload data_processing.py

Browse files
Files changed (1) hide show
  1. data_processing.py +4 -4
data_processing.py CHANGED
@@ -44,10 +44,10 @@ def create_faiss_index_file():
44
  index.add(embeddings_np)
45
 
46
  # Save FAISS index
47
- faiss.write_index(index, f"data_local\rag7_index.faiss")
48
 
49
  # Save documents in JSON (metadata storage)
50
- with open(f"data_local\rag7_docs.json", "w") as f:
51
  json.dump(all_documents, f)
52
 
53
  print(f"data is stored!")
@@ -64,10 +64,10 @@ def load_ragbench():
64
 
65
  def load_faiss():
66
  # Load the correct FAISS index
67
- faiss_index_path = f"data_local\rag7_index.faiss"
68
  index = faiss.read_index(faiss_index_path)
69
 
70
  def load_metatdata():
71
  # Load document metadata
72
- with open(f"data_local\rag7_docs.json", "r") as f:
73
  actual_docs = json.load(f) # Contains all documents for this dataset
 
44
  index.add(embeddings_np)
45
 
46
  # Save FAISS index
47
+ faiss.write_index(index, f"data_local/rag7_index.faiss")
48
 
49
  # Save documents in JSON (metadata storage)
50
+ with open(f"data_local/rag7_docs.json", "w") as f:
51
  json.dump(all_documents, f)
52
 
53
  print(f"data is stored!")
 
64
 
65
  def load_faiss():
66
  # Load the correct FAISS index
67
+ faiss_index_path = f"data_local/rag7_index.faiss"
68
  index = faiss.read_index(faiss_index_path)
69
 
70
  def load_metatdata():
71
  # Load document metadata
72
+ with open(f"data_local/rag7_docs.json", "r") as f:
73
  actual_docs = json.load(f) # Contains all documents for this dataset