Tollef Jørgensen commited on
Commit
4d794c6
1 Parent(s): 8eef9b2

update app to blocks

Browse files
Files changed (2) hide show
  1. __pycache__/app.cpython-39.pyc +0 -0
  2. app.py +48 -28
__pycache__/app.cpython-39.pyc ADDED
Binary file (2.63 kB). View file
 
app.py CHANGED
@@ -5,9 +5,15 @@ import pandas as pd
5
  from datasets import load_dataset
6
  from sentence_transformers import SentenceTransformer
7
 
 
8
 
9
- def build_doc_frame(df, idx=0):
10
- doc = df.iloc[0]
 
 
 
 
 
11
  # as df:
12
  doc_df = pd.DataFrame(doc).T
13
  # keep only sentences + embedding:
@@ -21,17 +27,18 @@ def get_doc_embeddings(doc):
21
  return np.array(doc.embedding.tolist(), dtype="float32")
22
 
23
 
24
- def faiss_search(doc_idx, query_str, K=5):
25
- # doc_idx is a choice option of (idx, text)
26
- idx = doc_idx[0] - 1
27
- newdoc = build_doc_frame(df, idx=idx)
 
 
28
  embeddings = get_doc_embeddings(newdoc)
29
 
30
  faiss.normalize_L2(embeddings)
31
  index = faiss.IndexFlatIP(768)
32
  index.add(embeddings)
33
 
34
- query_str = "Skade mellom kjøretøy"
35
  target_emb = model.encode([query_str])
36
  target_emb = np.array([target_emb.reshape(-1)])
37
  faiss.normalize_L2(target_emb)
@@ -43,30 +50,43 @@ def faiss_search(doc_idx, query_str, K=5):
43
  pretty_results = []
44
  for idx, score in zip(I[0], D[0]):
45
  pretty_results.append((round(float(score), 3), newdoc.iloc[idx].sentences))
46
- pretty_results_str = "\n".join([f"{score}\t{sent}" for score, sent in pretty_results])
47
  top_k_str = f"Top {K} results for: {query_str}"
48
- underlines = "__" * 40
49
 
50
  # return str:
51
- return f"{top_k_str}\n{pretty_results_str}\n{underlines}"
52
 
53
 
54
- dataset = load_dataset("tollefj/rettsavgjoerelser_100samples_embeddings")
55
- model = SentenceTransformer("NbAiLab/nb-sbert-base")
56
- df = dataset["train"].to_pandas()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
 
58
- dropdown_opts = [(idx + 1, f"\t{doc.summary[0][:60]}...") for idx, doc in df.iterrows()]
59
-
60
- iface = gr.Interface(
61
- fn=faiss_search,
62
- inputs=[
63
- gr.Dropdown(label="Select a court case", choices=dropdown_opts),
64
- gr.Textbox(lines=2, placeholder="Your query here..."),
65
- gr.Slider(minimum=1, maximum=10, label="Number of matches", value=5),
66
- ],
67
- outputs="text",
68
- title="Lovdata rettsavgjørelser - semantisk søk",
69
- description="Velg en rettsak og søk for å hente ut lignende setninger i saken",
70
- )
71
-
72
- iface.launch()
 
5
  from datasets import load_dataset
6
  from sentence_transformers import SentenceTransformer
7
 
8
+ idx = 0
9
 
10
+ dataset = load_dataset("tollefj/rettsavgjoerelser_100samples_embeddings")
11
+ model = SentenceTransformer("NbAiLab/nb-sbert-base")
12
+ df = dataset["train"].to_pandas()
13
+
14
+
15
+ def build_doc_frame(df, idx):
16
+ doc = df.iloc[idx]
17
  # as df:
18
  doc_df = pd.DataFrame(doc).T
19
  # keep only sentences + embedding:
 
27
  return np.array(doc.embedding.tolist(), dtype="float32")
28
 
29
 
30
+ def faiss_search(doc_url, query_str, K=5):
31
+ global idx
32
+ # find idx from url:
33
+ doc_idx = df[df.url == doc_url].index[0]
34
+ idx = int(doc_idx)
35
+ newdoc = build_doc_frame(df, idx)
36
  embeddings = get_doc_embeddings(newdoc)
37
 
38
  faiss.normalize_L2(embeddings)
39
  index = faiss.IndexFlatIP(768)
40
  index.add(embeddings)
41
 
 
42
  target_emb = model.encode([query_str])
43
  target_emb = np.array([target_emb.reshape(-1)])
44
  faiss.normalize_L2(target_emb)
 
50
  pretty_results = []
51
  for idx, score in zip(I[0], D[0]):
52
  pretty_results.append((round(float(score), 3), newdoc.iloc[idx].sentences))
53
+ pretty_results_str = "\n".join([f"Score: {score}\t\t{sent}" for score, sent in pretty_results])
54
  top_k_str = f"Top {K} results for: {query_str}"
 
55
 
56
  # return str:
57
+ return f"{top_k_str}\n{pretty_results_str}"
58
 
59
 
60
+ # def DropdownSummary():
61
+ # next_opts = df.iloc[idx].summary.tolist()
62
+ # return gr.Dropdown.update(choices=next_opts, label="Velg fra oppsummeringene")
63
+
64
+
65
+ dropdown_opts = [doc.url for idx, doc in df.iterrows()]
66
+
67
+ with gr.Blocks() as demo:
68
+ gr.Label("Lovdata rettsavgjørelser - semantisk søk")
69
+
70
+ case_dropdown = gr.Dropdown(label="Velg en rettsavgjørelse", choices=dropdown_opts, default=dropdown_opts[0])
71
+ # when case_dropdown changes, update the summary dropdown:
72
+ # idx_label = gr.Label(f"Current index: {idx}")
73
+
74
+ query = gr.Textbox(
75
+ label="Søk etter setninger",
76
+ lines=1,
77
+ placeholder="Kollisjon mellom to kjøretøy.",
78
+ )
79
+ k_slider = gr.Slider(minimum=1, maximum=10, label="Number of matches", value=5)
80
+
81
+ search_btn = gr.Button("Search")
82
+
83
+ output = gr.Textbox(label="Results", lines=10)
84
+
85
+ # from the selected URL, find the index in the df:
86
+ search_btn.click(faiss_search, inputs=[case_dropdown, query, k_slider], outputs=[output])
87
+
88
+ # clear_btn.click(None, inputs=[None, None], outputs=None)
89
+ # search_btn.click(faiss_search, inputs=[None, None, None], outputs=["text"])
90
+ # search_btn.click(faiss_search, inputs=[idx, query, k_slider], outputs=["text"])
91
 
92
+ demo.launch()