Spaces:
Runtime error
Runtime error
Tollef Jørgensen
commited on
Commit
•
4d794c6
1
Parent(s):
8eef9b2
update app to blocks
Browse files- __pycache__/app.cpython-39.pyc +0 -0
- app.py +48 -28
__pycache__/app.cpython-39.pyc
ADDED
Binary file (2.63 kB). View file
|
|
app.py
CHANGED
@@ -5,9 +5,15 @@ import pandas as pd
|
|
5 |
from datasets import load_dataset
|
6 |
from sentence_transformers import SentenceTransformer
|
7 |
|
|
|
8 |
|
9 |
-
|
10 |
-
|
|
|
|
|
|
|
|
|
|
|
11 |
# as df:
|
12 |
doc_df = pd.DataFrame(doc).T
|
13 |
# keep only sentences + embedding:
|
@@ -21,17 +27,18 @@ def get_doc_embeddings(doc):
|
|
21 |
return np.array(doc.embedding.tolist(), dtype="float32")
|
22 |
|
23 |
|
24 |
-
def faiss_search(
|
25 |
-
|
26 |
-
|
27 |
-
|
|
|
|
|
28 |
embeddings = get_doc_embeddings(newdoc)
|
29 |
|
30 |
faiss.normalize_L2(embeddings)
|
31 |
index = faiss.IndexFlatIP(768)
|
32 |
index.add(embeddings)
|
33 |
|
34 |
-
query_str = "Skade mellom kjøretøy"
|
35 |
target_emb = model.encode([query_str])
|
36 |
target_emb = np.array([target_emb.reshape(-1)])
|
37 |
faiss.normalize_L2(target_emb)
|
@@ -43,30 +50,43 @@ def faiss_search(doc_idx, query_str, K=5):
|
|
43 |
pretty_results = []
|
44 |
for idx, score in zip(I[0], D[0]):
|
45 |
pretty_results.append((round(float(score), 3), newdoc.iloc[idx].sentences))
|
46 |
-
pretty_results_str = "\n".join([f"{score}\t{sent}" for score, sent in pretty_results])
|
47 |
top_k_str = f"Top {K} results for: {query_str}"
|
48 |
-
underlines = "__" * 40
|
49 |
|
50 |
# return str:
|
51 |
-
return f"{top_k_str}\n{pretty_results_str}
|
52 |
|
53 |
|
54 |
-
|
55 |
-
|
56 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
57 |
|
58 |
-
|
59 |
-
|
60 |
-
iface = gr.Interface(
|
61 |
-
fn=faiss_search,
|
62 |
-
inputs=[
|
63 |
-
gr.Dropdown(label="Select a court case", choices=dropdown_opts),
|
64 |
-
gr.Textbox(lines=2, placeholder="Your query here..."),
|
65 |
-
gr.Slider(minimum=1, maximum=10, label="Number of matches", value=5),
|
66 |
-
],
|
67 |
-
outputs="text",
|
68 |
-
title="Lovdata rettsavgjørelser - semantisk søk",
|
69 |
-
description="Velg en rettsak og søk for å hente ut lignende setninger i saken",
|
70 |
-
)
|
71 |
-
|
72 |
-
iface.launch()
|
|
|
5 |
from datasets import load_dataset
|
6 |
from sentence_transformers import SentenceTransformer
|
7 |
|
8 |
+
idx = 0
|
9 |
|
10 |
+
dataset = load_dataset("tollefj/rettsavgjoerelser_100samples_embeddings")
|
11 |
+
model = SentenceTransformer("NbAiLab/nb-sbert-base")
|
12 |
+
df = dataset["train"].to_pandas()
|
13 |
+
|
14 |
+
|
15 |
+
def build_doc_frame(df, idx):
|
16 |
+
doc = df.iloc[idx]
|
17 |
# as df:
|
18 |
doc_df = pd.DataFrame(doc).T
|
19 |
# keep only sentences + embedding:
|
|
|
27 |
return np.array(doc.embedding.tolist(), dtype="float32")
|
28 |
|
29 |
|
30 |
+
def faiss_search(doc_url, query_str, K=5):
|
31 |
+
global idx
|
32 |
+
# find idx from url:
|
33 |
+
doc_idx = df[df.url == doc_url].index[0]
|
34 |
+
idx = int(doc_idx)
|
35 |
+
newdoc = build_doc_frame(df, idx)
|
36 |
embeddings = get_doc_embeddings(newdoc)
|
37 |
|
38 |
faiss.normalize_L2(embeddings)
|
39 |
index = faiss.IndexFlatIP(768)
|
40 |
index.add(embeddings)
|
41 |
|
|
|
42 |
target_emb = model.encode([query_str])
|
43 |
target_emb = np.array([target_emb.reshape(-1)])
|
44 |
faiss.normalize_L2(target_emb)
|
|
|
50 |
pretty_results = []
|
51 |
for idx, score in zip(I[0], D[0]):
|
52 |
pretty_results.append((round(float(score), 3), newdoc.iloc[idx].sentences))
|
53 |
+
pretty_results_str = "\n".join([f"Score: {score}\t\t{sent}" for score, sent in pretty_results])
|
54 |
top_k_str = f"Top {K} results for: {query_str}"
|
|
|
55 |
|
56 |
# return str:
|
57 |
+
return f"{top_k_str}\n{pretty_results_str}"
|
58 |
|
59 |
|
60 |
+
# def DropdownSummary():
|
61 |
+
# next_opts = df.iloc[idx].summary.tolist()
|
62 |
+
# return gr.Dropdown.update(choices=next_opts, label="Velg fra oppsummeringene")
|
63 |
+
|
64 |
+
|
65 |
+
dropdown_opts = [doc.url for idx, doc in df.iterrows()]
|
66 |
+
|
67 |
+
with gr.Blocks() as demo:
|
68 |
+
gr.Label("Lovdata rettsavgjørelser - semantisk søk")
|
69 |
+
|
70 |
+
case_dropdown = gr.Dropdown(label="Velg en rettsavgjørelse", choices=dropdown_opts, default=dropdown_opts[0])
|
71 |
+
# when case_dropdown changes, update the summary dropdown:
|
72 |
+
# idx_label = gr.Label(f"Current index: {idx}")
|
73 |
+
|
74 |
+
query = gr.Textbox(
|
75 |
+
label="Søk etter setninger",
|
76 |
+
lines=1,
|
77 |
+
placeholder="Kollisjon mellom to kjøretøy.",
|
78 |
+
)
|
79 |
+
k_slider = gr.Slider(minimum=1, maximum=10, label="Number of matches", value=5)
|
80 |
+
|
81 |
+
search_btn = gr.Button("Search")
|
82 |
+
|
83 |
+
output = gr.Textbox(label="Results", lines=10)
|
84 |
+
|
85 |
+
# from the selected URL, find the index in the df:
|
86 |
+
search_btn.click(faiss_search, inputs=[case_dropdown, query, k_slider], outputs=[output])
|
87 |
+
|
88 |
+
# clear_btn.click(None, inputs=[None, None], outputs=None)
|
89 |
+
# search_btn.click(faiss_search, inputs=[None, None, None], outputs=["text"])
|
90 |
+
# search_btn.click(faiss_search, inputs=[idx, query, k_slider], outputs=["text"])
|
91 |
|
92 |
+
demo.launch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|