app setup

pages/1_Retrieval_App.py

@@ -0,0 +1,145 @@
+import os
+from typing import Dict, List
+
+import datasets
+import pandas as pd
+import sentence_transformers
+import streamlit as st
+from findkit import feature_extractors, indexes, retrieval_pipeline
+from toolz import partial
+
+
+def truncate_description(description, length=50):
+    return " ".join(description.split()[:length])
+
+
+def get_repos_with_descriptions(repos_df, repos):
+    return repos_df.loc[repos]
+
+
+def search_f(
+    retrieval_pipe: retrieval_pipeline.RetrievalPipeline,
+    query: str,
+    k: int,
+    description_length: int,
+    doc_col: List[str],
+):
+    results = retrieval_pipe.find_similar(query, k)
+    # results['repo'] = results.index
+    results["link"] = "https://github.com/" + results["repo"]
+    for col in doc_col:
+        results[col] = results[col].apply(
+            lambda desc: truncate_description(desc, description_length)
+        )
+    shown_cols = ["repo", "tasks", "link", "distance"]
+    shown_cols = shown_cols + doc_col
+    return results.reset_index(drop=True)[shown_cols]
+
+
+def show_retrieval_results(
+    retrieval_pipe: retrieval_pipeline.RetrievalPipeline,
+    query: str,
+    k: int,
+    all_queries: List[str],
+    description_length: int,
+    repos_by_query: Dict[str, pd.DataFrame],
+    doc_col: str,
+):
+    print("started retrieval")
+    if query in all_queries:
+        with st.expander(
+            "query is in gold standard set queries. Toggle viewing gold standard results?"
+        ):
+            st.write("gold standard results")
+            task_repos = repos_by_query.get_group(query)
+            st.table(get_repos_with_descriptions(retrieval_pipe.X_df, task_repos))
+    with st.spinner(text="fetching results"):
+        st.write(
+            search_f(retrieval_pipe, query, k, description_length, doc_col).to_html(
+                escape=False, index=False
+            ),
+            unsafe_allow_html=True,
+        )
+    print("finished retrieval")
+
+
+def setup_pipeline(
+    extractor: feature_extractors.SentenceEncoderFeatureExtractor,
+    documents_df: pd.DataFrame,
+    text_col: str,
+):
+    retrieval_pipeline.RetrievalPipelineFactory.build(
+        documents_df[text_col], metadata=documents_df
+    )
+
+
+@st.cache
+def setup_retrieval_pipeline(
+    query_encoder_path, document_encoder_path, documents, metadata
+):
+    document_encoder = feature_extractors.SentenceEncoderFeatureExtractor(
+        sentence_transformers.SentenceTransformer(document_encoder_path, device="cpu")
+    )
+    query_encoder = feature_extractors.SentenceEncoderFeatureExtractor(
+        sentence_transformers.SentenceTransformer(query_encoder_path, device="cpu")
+    )
+    retrieval_pipe = retrieval_pipeline.RetrievalPipelineFactory(
+        feature_extractor=document_encoder,
+        query_feature_extractor=query_encoder,
+        index_factory=partial(indexes.NMSLIBIndex.build, distance="cosinesimil"),
+    )
+    return retrieval_pipe.build(documents, metadata=metadata)
+
+
+def app(retrieval_pipeline, retrieval_df, doc_col):
+
+    retrieved_results = st.sidebar.number_input("number of results", value=10)
+    description_length = st.sidebar.number_input(
+        "number of used description words", value=10
+    )
+
+    tasks_deduped = (
+        retrieval_df["tasks"].explode().value_counts().reset_index()
+    )  # drop_duplicates().sort_values().reset_index(drop=True)
+    tasks_deduped.columns = ["task", "documents per task"]
+    with st.sidebar.expander("View test set queries"):
+        st.table(tasks_deduped.explode("task"))
+
+    additional_shown_cols = st.sidebar.multiselect(
+        label="additional cols", options=[doc_col], default=doc_col
+    )
+
+    repos_by_query = retrieval_df.explode("tasks").groupby("tasks")
+    query = st.text_input("input query", value="metric learning")
+    show_retrieval_results(
+        retrieval_pipeline,
+        query,
+        retrieved_results,
+        tasks_deduped["task"].to_list(),
+        description_length,
+        repos_by_query,
+        additional_shown_cols,
+    )
+
+
+def app_main(
+    query_encoder_path,
+    document_encoder_path,
+    data_path,
+):
+    print("loading data")
+
+    retrieval_df = datasets.load_dataset(data_path)["train"].to_pandas()
+    print("setting up retrieval_pipe")
+    doc_col = "dependencies"
+    retrieval_pipeline = setup_retrieval_pipeline(
+        query_encoder_path, document_encoder_path, retrieval_df[doc_col], retrieval_df
+    )
+    app(retrieval_pipeline, retrieval_df, doc_col)
+
+
+app_main(
+    query_encoder_path="lambdaofgod/query_nbow_1_2000",
+    document_encoder_path="lambdaofgod/document_nbow_1_2000",
+    data_path="lambdaofgod/pwc_repositories_with_dependencies",
+)

pages/2_Statistics.py

@@ -0,0 +1,39 @@
+import pandas as pd
+import streamlit as st
+
+best_results_df = pd.read_csv("output/best_tasks_with_hits.csv")
+
+
+worst_results_df = pd.read_csv(
+    "output/worst_tasks_with_hits.csv"
+)  # , data_path="output/papers_with_dependencies.csv",
+
+show_worst_best_statistics = st.sidebar.checkbox(
+    label="show worst/best statistics grouped by area"
+)
+
+show_area_aggregated_results = st.sidebar.checkbox(
+    label="show results aggregated by area"
+)
+if show_worst_best_statistics:
+    st.markdown("""
+    ## Worst/best queries
+    The following are top 10 worst/best queries per area by number of hits.
+    There are at least 10 documents per query in the test set, so number of hits/10 is the accuracy.
+    """)
+    sort_key = st.selectbox("sort by", list(best_results_df.columns))
+    st.markdown("## Queries with best results")
+    st.table(best_results_df.sort_values(sort_key, ascending=False))
+    st.markdown("## Queries with worst results")
+    st.table(worst_results_df.sort_values(sort_key, ascending=False))
+
+if show_area_aggregated_results:
+    st.markdown("## Area aggregated results")
+    best_results_agg = best_results_df.groupby("area").agg("mean").reset_index()
+    worst_results_agg = worst_results_df.groupby("area").agg("mean").reset_index()
+    sort_key = st.selectbox("sort by", list(best_results_agg.columns))
+    st.markdown("Best results")
+    st.table(best_results_agg.sort_values(sort_key, ascending=False))
+    st.markdown("Worst results")
+    st.table(worst_results_agg.sort_values(sort_key, ascending=False))
+

project_retrieval_app.py

@@ -0,0 +1,28 @@
+import streamlit as st
+
+st.set_page_config(page_title="Start")
+st.markdown("""
+# Searching Python projects with neural networks
+
+## Authors
+
+- Jakub Bartczuk
+- Paweł Rychlikowski (promotor)
+
+## Motivation
+The following application illustrates neural network based models for searching github.
+
+With over 500 starred repositories searching through them became cumbersome. I did a [small project for retrieval on starred repositories](https://github.com/lambdaofgod/examples-counterexamples/blob/master/notebooks/text_mining/Github_Starred_Repositories.ipynb) which looked promising, but it is hard to gauge how useful such solution would be in practice.
+
+In the thesis I use [PapersWithCode](https://paperswithcode.com/) data for information retrieval.
+
+PapersWithCode contains links between papers and repositories that implement them. Most repositories are tagged with at least one task like "unsupervised segmentation" or "semantic parsing".
+
+Tasks are research topics like "object detection" or "multivariate time series imputation".
+
+## Features
+- [x] Searching using Neural Bag of Words features
+- [ ] Searching using selectable model
+  - [ ] add Word2Vec on READMEs
+
+""")

requirements.txt

@@ -0,0 +1,3 @@
+-e git+https://github.com/lambdaofgod/findkit#egg=findkit
+sentence-transformers==2.2.2
+toolz