Initial Commit

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+__pycache__

README.md

@@ -0,0 +1,13 @@
+---
+title: Text Similarity Prediction and Analysis
+emoji: 🚀
+colorFrom: pink
+colorTo: purple
+sdk: streamlit
+sdk_version: 1.33.0
+app_file: app.py
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,53 @@
+import joblib
+from sentence_transformers import CrossEncoder, SentenceTransformer
+import streamlit as st
+from sklearn.metrics.pairwise import cosine_similarity
+from samples import get_samples
+import textdistance
+from sklearn.feature_extraction.text import TfidfVectorizer
+from encode_sentences import encode_sentences
+
+model_save_path = 'trained_model_stsbenchmark_bert-base-uncased'
+bi_encoder = 'Bi-Encoder'
+cross_encoder = 'Cross-Encoder'
+levenshtein_distance = 'Levenshtein Distance'
+tf_idf = 'TF-IDF'
+random_forest = 'RandomForest'
+title = 'Sentence Similarity with Transformers'
+tfidf_vectorizer = TfidfVectorizer()
+cross_encoder_trasformer = CrossEncoder(model_save_path)
+bi_encoder_trasformer = SentenceTransformer(model_save_path)
+random_forest_model = joblib.load('trained_model_random_forest.joblib')
+@st.cache_data
+def compute_similarity(sentence_1, sentence_2, comparison):
+    if comparison == bi_encoder:
+        return cosine_similarity([bi_encoder_trasformer.encode(sentence_1)], [bi_encoder_trasformer.encode(sentence_2)])[0][0]
+    return cross_encoder_trasformer.predict([sentence_1, sentence_2])
+
+st.set_page_config(page_title=title, layout = 'wide', initial_sidebar_state = 'auto')
+st.title(title)
+st.write("This app takes two sentences and outputs their similarity score using a fine-tuned transformer model.")
+
+# Example sentences section
+test_samples = get_samples()
+st.sidebar.header("Example Sentences")
+example_1 = st.sidebar.radio("Sentence 1", test_samples['sentence1'].values.tolist())
+example_2 = st.sidebar.radio("Sentence 2", test_samples['sentence2'].values.tolist())
+
+# Input fields
+sentence_1 = st.text_input("Enter Sentence 1:", example_1)
+sentence_2 = st.text_input("Enter Sentence 2:", example_2)
+comparison = st.selectbox("Comparicon:", [bi_encoder, cross_encoder, levenshtein_distance, tf_idf, random_forest])
+
+if st.button("Compare"):
+    # Compute similarity
+    if comparison in [bi_encoder, cross_encoder]:
+        similarity = compute_similarity(sentence_1, sentence_2, comparison)
+    elif comparison == levenshtein_distance:
+        similarity = textdistance.levenshtein.normalized_similarity(sentence_1, sentence_2)
+    elif comparison == tf_idf:
+        similarity = cosine_similarity(tfidf_vectorizer.fit_transform([sentence_1, sentence_2]))[0][1]
+    elif comparison == random_forest:
+        similarity = random_forest_model.predict(encode_sentences(bi_encoder_trasformer, sentence_1, sentence_2))[0]
+    st.write(f"Similarity Score: {similarity:.4f}")
+    st.write("A higher score indicates greater similarity. The score ranges from 0 to 1.")

encode_sentences.py

@@ -0,0 +1,12 @@
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+
+def encode_sentences(tokenizer, sentence1, sentence2):
+    # Encode the sentences
+    embedding1 = tokenizer.encode(sentence1, convert_to_tensor=True).cpu()
+    embedding2 = tokenizer.encode(sentence2, convert_to_tensor=True).cpu()
+    # Compute the absolute difference of embeddings as features
+    feature = abs(embedding1 - embedding2).numpy().reshape(1, -1)
+    # Scale features (use the same scaler as used during training)
+    feature_scaled = scaler.fit_transform(feature)# scaler.transform(feature)
+    return feature_scaled

requirements.txt

@@ -0,0 +1,3 @@
+scikit-learn==1.3.2
+accelerate==0.28.0
+sentence-transformers==2.6.1

samples.py

@@ -0,0 +1,20 @@
+import pandas as pd
+from datasets import load_dataset
+
+def get_samples():
+    dataset = load_dataset("mteb/stsbenchmark-sts")
+    get_where = lambda score: dataset['validation'].filter(lambda x: x['score'] == score, load_from_cache_file = False)[0]
+    test_samples = pd.DataFrame([
+        get_where(5),
+        get_where(4.5),
+        get_where(4),
+        get_where(3.5),
+        get_where(3),
+        get_where(2.5),
+        get_where(2),
+        get_where(1.5),
+        get_where(1),
+        get_where(0.5),
+        get_where(0),
+    ], columns=['sentence1', 'sentence2', 'score'])
+    return test_samples

trained_model_random_forest.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69284946a405027ded9d5a6d0589912d42ccbe40506da21aaefcf0634c0c8f9f
+size 27083201

trained_model_stsbenchmark_bert-base-uncased/1_Pooling/config.json

@@ -0,0 +1,10 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false,
+  "pooling_mode_weightedmean_tokens": false,
+  "pooling_mode_lasttoken": false,
+  "include_prompt": true
+}

trained_model_stsbenchmark_bert-base-uncased/README.md

@@ -0,0 +1,127 @@
+---
+library_name: sentence-transformers
+pipeline_tag: sentence-similarity
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+
+---
+
+# {MODEL_NAME}
+
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 768 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+<!--- Describe your model here -->
+
+## Usage (Sentence-Transformers)
+
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('{MODEL_NAME}')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('{MODEL_NAME}')
+model = AutoModel.from_pretrained('{MODEL_NAME}')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling. In this case, mean pooling.
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+
+
+## Evaluation Results
+
+<!--- Describe how your model was evaluated -->
+
+For an automated evaluation of this model, see the *Sentence Embeddings Benchmark*: [https://seb.sbert.net](https://seb.sbert.net?model_name={MODEL_NAME})
+
+
+## Training
+The model was trained with the parameters:
+
+**DataLoader**:
+
+`torch.utils.data.dataloader.DataLoader` of length 180 with parameters:
+```
+{'batch_size': 32, 'sampler': 'torch.utils.data.sampler.RandomSampler', 'batch_sampler': 'torch.utils.data.sampler.BatchSampler'}
+```
+
+**Loss**:
+
+`sentence_transformers.losses.CosineSimilarityLoss.CosineSimilarityLoss` 
+
+Parameters of the fit()-Method:
+```
+{
+    "epochs": 16,
+    "evaluation_steps": 10000,
+    "evaluator": "sentence_transformers.evaluation.EmbeddingSimilarityEvaluator.EmbeddingSimilarityEvaluator",
+    "max_grad_norm": 1,
+    "optimizer_class": "<class 'torch.optim.adamw.AdamW'>",
+    "optimizer_params": {
+        "lr": 2e-05
+    },
+    "scheduler": "WarmupLinear",
+    "steps_per_epoch": null,
+    "warmup_steps": 288,
+    "weight_decay": 0.01
+}
+```
+
+
+## Full Model Architecture
+```
+SentenceTransformer(
+  (0): Transformer({'max_seq_length': 512, 'do_lower_case': False}) with Transformer model: BertModel 
+  (1): Pooling({'word_embedding_dimension': 768, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
+)
+```
+
+## Citing & Authors
+
+<!--- Describe where people can find more information -->

trained_model_stsbenchmark_bert-base-uncased/config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "bert-base-uncased",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "torch_dtype": "float32",
+  "transformers_version": "4.39.1",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

trained_model_stsbenchmark_bert-base-uncased/config_sentence_transformers.json

@@ -0,0 +1,9 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.6.1",
+    "transformers": "4.39.1",
+    "pytorch": "2.1.0"
+  },
+  "prompts": {},
+  "default_prompt_name": null
+}

trained_model_stsbenchmark_bert-base-uncased/eval/similarity_evaluation_sts-dev_results.csv

@@ -0,0 +1,37 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+0,-1,0.8314586112849952,0.8222861463449044,0.7993198330481376,0.7950011735438326,0.7984725573815618,0.7942570025560868,0.6980204859229657,0.691568924812272
+1,-1,0.8407055288130449,0.834337287719971,0.8152096287261316,0.8122204244014548,0.8144975251483847,0.8116885956480807,0.732746004281146,0.730360525368856
+2,-1,0.8439256364593497,0.8359042369327582,0.8262066732022268,0.8221511615595501,0.8259239791965207,0.8216808891288429,0.7487936308349514,0.7378949198846093
+3,-1,0.8464276481190168,0.8394870233934864,0.8281315560034785,0.8252383288066867,0.827827341976526,0.8247387146010591,0.755207305963072,0.7448581382442837
+0,-1,0.7922021805853781,0.7713972176442391,0.7603672763034102,0.7439105700492636,0.7602031813189104,0.7441131583398928,0.6816470416797301,0.6671730658498958
+1,-1,0.8353590575047928,0.8271301553141639,0.8070309131863589,0.8038339361946429,0.8064465524675363,0.8034004299728619,0.724916249478606,0.7157522961344986
+2,-1,0.8383577657525391,0.8303132485677799,0.8194439900889535,0.8157611441363017,0.8193997494383647,0.8159934845337001,0.7427196371940937,0.7310482210576092
+3,-1,0.8423781754611726,0.8376504193628221,0.8220139794418871,0.8205376067862071,0.8219503650163204,0.820750767022779,0.746923157755294,0.7374371616324693
+4,-1,0.8435332150937506,0.8378118737570728,0.8309533350978225,0.8276985663782159,0.8312413640405018,0.8280579248554486,0.7600228119842427,0.7466608648818838
+5,-1,0.8426154400776431,0.8388481909555153,0.827527318476358,0.8270527243509243,0.827468221593408,0.8272636902812877,0.7583694197302585,0.7463647005705575
+6,-1,0.8434839830593363,0.8387893586960224,0.8278000664778432,0.8262394126974679,0.8279299017271557,0.8266215271437661,0.75634507023444,0.7456149356556544
+7,-1,0.8447623486297048,0.8402910393235883,0.8286159386784336,0.8273430910606207,0.8284183482230283,0.8273611189913042,0.7632274591667264,0.7515728597906741
+8,-1,0.8430060550087329,0.8397695352256395,0.8278598137318165,0.8268930065058908,0.8278714418474407,0.8270507201740991,0.765544059776489,0.7535096986556942
+9,-1,0.8463043217945104,0.8428571356192778,0.828913574421771,0.8280654267505405,0.8286295779818086,0.8280033348152811,0.7655773899365361,0.7545359162753758
+10,-1,0.8454911166453738,0.841096177887406,0.8312677636079294,0.8301731025956736,0.831042276187397,0.8299344352421627,0.7665158189322197,0.7543237041125989
+11,-1,0.8450383066521526,0.8415202053543727,0.8300219045475918,0.8293242873398408,0.8298684020017837,0.8291249032692225,0.7646272684826112,0.7525458091130097
+12,-1,0.845263138308006,0.8422066938360778,0.8310570453409322,0.8303017631159286,0.8309400428836656,0.8301016874460738,0.7663755880184066,0.7548354410662842
+13,-1,0.8449131314461613,0.8419114160316352,0.830002123134018,0.8289793811849208,0.8299623591619595,0.8290426390954682,0.7668088075160115,0.7552771459844687
+14,-1,0.8449501341914755,0.8423892317883578,0.8306677759491086,0.829802245120922,0.8305861595344564,0.8297362152477388,0.7668176398821887,0.7552218664389473
+15,-1,0.8451687758660085,0.8425458732023435,0.8305440831498357,0.8297474624645348,0.8304598348669994,0.8296288883562123,0.7670469989815153,0.7557805421607521
+0,-1,0.7899956845018291,0.771507036834188,0.7511551366882981,0.7358397838917168,0.7505945939602987,0.7354465940362954,0.6563770654647433,0.6475599646483539
+1,-1,0.8347231117460785,0.8256291067400063,0.8058966618423721,0.8011948649863158,0.8052061914434191,0.8006856206134588,0.7141478665675217,0.704298398125894
+2,-1,0.8436092176321606,0.8367681848505647,0.8245531713906449,0.82164701173788,0.8240046280750112,0.821232277497679,0.7502878000647448,0.7401174636541575
+3,-1,0.8442901759402588,0.8373153268200199,0.8233687598308753,0.8208385882658366,0.8229073657313626,0.8207712116582747,0.7526192815662727,0.7432087475182582
+4,-1,0.848353412564754,0.8438935423413328,0.829263800730978,0.8287148585262668,0.8290973494834554,0.8288209896643224,0.7396624558596283,0.7298966325807095
+5,-1,0.8469338550018505,0.8413124282991357,0.8287247365357678,0.8271664056872715,0.8283838259367564,0.8270491991390324,0.7624021030507305,0.7519352784652986
+6,-1,0.8455852769710634,0.8414048493223681,0.8298968746305437,0.8288217774732614,0.8297699976985162,0.8287398477439649,0.7642665063372014,0.7535908976466645
+7,-1,0.8453879300462057,0.8408612718889563,0.8312776123581294,0.8296165810948535,0.8311280728049801,0.8296103221073954,0.7677834508191366,0.7561287744345223
+8,-1,0.8469803565050675,0.8436047151916304,0.8317179509745953,0.8307405519533304,0.8315029958649137,0.8304593208160991,0.7688328892882389,0.7577201046864561
+9,-1,0.8467204857153575,0.8436336011136023,0.8298114267477297,0.8295115852024595,0.8296737666691951,0.829267596713595,0.7655896476711741,0.754997451917312
+10,-1,0.8471160424084345,0.8432494435492464,0.8321402716339013,0.8307532470984191,0.831911482208498,0.8307224780133322,0.767914550413006,0.7548514050623367
+11,-1,0.847349492530715,0.8440041197580994,0.8324373880423506,0.8316586340589276,0.8321991304921575,0.8315150976339994,0.769308193094527,0.7582940571896748
+12,-1,0.8475483430525071,0.844448690182024,0.8327238696222913,0.8321645956431722,0.832418112421938,0.8319732981883223,0.7685207480088893,0.7576570458712267
+13,-1,0.8473602432987508,0.8439963794305544,0.832768618687382,0.8318741466665907,0.8324902011429588,0.8316555805352172,0.7688799040213863,0.7573533244895924
+14,-1,0.8481167455381742,0.8449734428778592,0.8328092293062578,0.8320476868822251,0.8325869268980407,0.8319152283577276,0.7698448588934874,0.7584867636278891
+15,-1,0.8480331647871943,0.8449084186287636,0.8327584026598132,0.8319446046864493,0.8325269188372508,0.8319455576240813,0.7694074517084232,0.7580956102185132

trained_model_stsbenchmark_bert-base-uncased/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2da70ebc1789b8d4da14934bcf6f70845b7e0c1e93d3891cc4995fb1e41af7ef
+size 437951328

trained_model_stsbenchmark_bert-base-uncased/modules.json

@@ -0,0 +1,14 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  }
+]

trained_model_stsbenchmark_bert-base-uncased/sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 512,
+  "do_lower_case": false
+}

trained_model_stsbenchmark_bert-base-uncased/similarity_evaluation_sts-test_results.csv

@@ -0,0 +1,5 @@
+epoch,steps,cosine_pearson,cosine_spearman,euclidean_pearson,euclidean_spearman,manhattan_pearson,manhattan_spearman,dot_pearson,dot_spearman
+-1,-1,0.875137977555676,0.8745703963672751,0.8398310472578498,0.8442842374538337,0.8390145445499485,0.8434616141103319,0.7948977053014381,0.7981928188151527
+-1,-1,0.8728396119084929,0.8735078768384611,0.8418952687938658,0.8462348206504858,0.8413260025721658,0.8456096433108078,0.8123532015656593,0.8122313711954551
+-1,-1,0.8746751636534322,0.8759463607276905,0.844710906429047,0.8485861720526876,0.8444537119546713,0.8484802573889416,0.8106640181302358,0.811605998878314
+-1,-1,0.8746751636534322,0.8759463607276905,0.844710906429047,0.8485861720526876,0.8444537119546713,0.8484802573889416,0.8106640181302358,0.811605998878314

trained_model_stsbenchmark_bert-base-uncased/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

trained_model_stsbenchmark_bert-base-uncased/tokenizer_config.json

@@ -0,0 +1,55 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}