Upload 14 files

.env

@@ -0,0 +1 @@
+COHERE_API_KEY = 7rMjNpj7LLTNlAcoR1Sc6cH23aURrBQoMPi9vzam

.gitattributes

@@ -1,34 +1,7 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+# Auto detect text files and perform LF normalization
+* text=auto
+amharic.csv filter=lfs diff=lfs merge=lfs -text
+hausa.csv filter=lfs diff=lfs merge=lfs -text
+news.ann filter=lfs diff=lfs merge=lfs -text
+swahili.csv filter=lfs diff=lfs merge=lfs -text
+yoruba.csv filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,2 @@
----
-title: AfriNews Cluster
-emoji: 🔥
-colorFrom: gray
-colorTo: green
-sdk: streamlit
-sdk_version: 1.19.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# cluster_news
+ 

amharic.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59b8670c19f95f0cff667b8d5f69033e93bcdd2dec5e1cc069f82d93699da894
+size 36144176

app.py

@@ -0,0 +1,51 @@
+import streamlit as st
+
+from process import *
+
+df = import_ds()
+
+st.title('AFri News Multilingual Embedding')
+
+form = st.form(key="user_settings")
+
+textcontainer = st.container()
+
+plotcontainer = st.container()
+
+with form:
+
+    query = st.text_input('Please input your news text here:')
+
+    num_nearest = int(st.slider('Please input the number of news to find: ', value=15, min_value=1, max_value=200))
+    
+    generate_button = form.form_submit_button("Cluster News")
+
+    if generate_button:
+        key = get_key()
+
+        co = cohere.Client(key)
+        
+        embeddings = getEmbeddings(co,df)
+
+        indexfile = 'news.ann'
+
+        semantic_search(embeddings, indexfile)
+
+        query_embed = get_query_embed(co, query)
+
+        nearest_ids = getClosestNeighbours(indexfile, query_embed, num_nearest)
+        
+        nn_embeddings = embeddings[nearest_ids[0]]
+        
+        all_embeddings = np.vstack([nn_embeddings, query_embed])
+        
+        umap_embeds  = getUMAPEmbed(embeddings)
+        
+        text_news = display_news(df,nearest_ids)
+        
+        fig = plot2DChart(df, umap_embeds)
+        
+        textcontainer.write(text_news)
+        
+        plotcontainer.write(fig)
+        

hausa.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5279476f52eded50fa5254c9a6be01abe1393484eb57a8858f90c6d079e520e
+size 14590027

news.ann

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:71443c486fb4dc39f3a600b705642795ad19c8ebca8e495259790e5351610b74
+size 1603680

process.py

@@ -0,0 +1,133 @@
+#from dotenv import load_dotenv
+from annoy import AnnoyIndex
+import pandas as pd
+import numpy as np
+import cohere
+import os
+import plotly.express as px
+import umap
+import plotly.graph_objects as go
+
+
+def get_key():
+    key =  "7rMjNpj7LLTNlAcoR1Sc6cH23aURrBQoMPi9vzam"
+    #load_dotenv()
+    return key
+
+
+def import_ds():
+    newsfiles = ['amharic','hausa','swahili','yoruba','igbo']
+    
+    df_am =  pd.read_csv(f'{newsfiles[0]}.csv')
+    df_am = df_am.sample(frac=0.5)
+    #df_en =  pd.read_csv(f'{newsfiles[1]}.csv')
+    #df_en = df_en.sample(frac=0.3)
+    df_hs =  pd.read_csv(f'{newsfiles[1]}.csv')
+    df_hs = df_hs.sample(frac=0.5)
+    df_sw =  pd.read_csv(f'{newsfiles[2]}.csv')
+    df_sw = df_sw.sample(frac=0.5)
+    df_yr =  pd.read_csv(f'{newsfiles[3]}.csv')
+    df_yr = df_yr.sample(frac=0.5)
+    df_ig =  pd.read_csv(f'{newsfiles[4]}.csv')
+    df_ig = df_ig.sample(frac=0.5)
+    
+    df_news = pd.concat([df_am,df_hs,df_sw,df_yr,df_ig],axis=0)
+    
+    df_news = df_news.sample(frac = 1)
+    
+    df_news = df_news[df_news['title'].notna()]
+    
+    df_news = df_news.drop_duplicates("title")
+        
+    df_news  = df_news.sample(500)
+     
+    return df_news
+
+    
+def getEmbeddings(co,df):
+    
+    df['text'] = df['title'] + df['summary']
+    
+    df = df.drop(['title','id','summary'],axis=1)
+    
+    embeds = co.embed(texts=list(df['text']),model="multilingual-22-12",truncate="RIGHT").embeddings  
+    
+    embeds = np.array(embeds)
+    
+    return embeds
+
+def semantic_search(emb,indexfile):
+    
+    emb = np.array(emb)
+
+    search_index = AnnoyIndex(emb.shape[1], 'angular')
+    print(emb.shape[1])
+
+    for i in range(len(emb)):
+        search_index.add_item(i, emb[i])
+
+    search_index.build(10)
+    search_index.save(indexfile)
+
+def get_query_embed(co, query):
+    query_embed = co.embed(texts=[query],
+                           model='multilingual-22-12',
+                           truncate='right').embeddings
+
+    return np.array(query_embed)
+    
+def getClosestNeighbours(indexfile,query_embed,neighbours=15):
+
+    search_index = AnnoyIndex(768, 'angular')
+    search_index.load(indexfile)
+
+
+    # Retrieve the nearest neighbors
+    similar_item_ids = search_index.get_nns_by_vector(query_embed[0],neighbours,
+                                                        include_distances=True)
+    
+    return similar_item_ids
+
+def display_news(df,similar_item_ids):
+    # Format the results
+    #print(similar_item_ids)
+    
+    results = pd.DataFrame(data={'title': df.iloc[similar_item_ids[0]]['title'],
+                                 'url': df.iloc[similar_item_ids[0]]['url'],
+                                  'summary': df.iloc[similar_item_ids[0]]['summary']})
+                                 #'distance': similar_item_ids[1]})
+    results.reset_index(drop=True, inplace=True)                            
+    
+    return results
+
+def getUMAPEmbed(embeds):
+    # Map the nearest embeddings to 2d
+    reducer = umap.UMAP(n_neighbors=20)
+    
+    return reducer.fit_transform(embeds)
+
+
+def plot2DChart(df, umap_embeds, clusters=None):
+    if clusters is None:
+        clusters = {}
+
+    df_viz = pd.DataFrame(data={'url': df['url'], 'title': df['title']})
+    df_viz['x'] = umap_embeds[:, 0]
+    df_viz['y'] = umap_embeds[:, 1]
+
+    #print(df_explore)
+    # Plot
+    fig = px.scatter(df_viz, x='x', y='y', hover_data=['title'])
+
+
+    fig.data = fig.data[::-1]
+
+    return fig
+
+if __name__ == '__main__':
+    key = get_key()
+    co = cohere.Client(key)
+    df_news = import_ds()
+    embed = process(co,df_news)
+    semantic_search(embed)
+    getClosestNeighbours(df_news)

requirements.txt

@@ -0,0 +1,16 @@
+altair==4.2.2
+annoy==1.17.0
+huggingface-hub==0.14.1
+numpy==1.21.6
+pandas==1.3.5
+plotly==5.14.1
+scipy==1.7.3
+beautifulsoup4==4.11.1
+cohere==2.7.0
+matplotlib==3.5.1
+python-dotenv==0.21.0
+scikit_learn==1.0.2
+streamlit==1.22.0
+streamlit_plotly_events==0.0.6
+umap==0.1.1
+umap_learn==0.5.3

swahili.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bccf0a9aaa7f5399fa51b6d34df9848f5a077a771ce2318f7f6beb58686dee99
+size 20901981

utils.py

@@ -0,0 +1,32 @@
+from datasets import load_dataset_builder, load_dataset
+import logging 
+
+def inspect():
+    langs = ['amharic','english','hausa','swahili','yoruba','igbo']
+
+    for lang in langs:
+        ds_builder = load_dataset_builder("csebuetnlp/xlsum",lang)
+        
+        desc = ds_builder.info.description
+        
+        feat = ds_builder.info.features
+        
+        return desc,feat
+    
+def load():
+    try:
+        langs = ['amharic','hausa','swahili','yoruba','igbo']
+        
+        for lang in langs:
+
+            dataset = load_dataset("csebuetnlp/xlsum", lang ,split="train")
+            #for split, data in dataset.items():
+            dataset.to_csv(f"{lang}.csv", index = None)
+            #dataset.save_to_disk(lang) 
+            #return dataset
+    except Exception as ex:
+        logging.debug(ex)
+
+if __name__ == '__main__':
+    #print(inspect())
+    load()

yoruba.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f5df52e87acfcd2fae999e7108a4f8c5e44345070b3c41380d72c47f8fd1412
+size 16448886