<file_sep>READ ME ----------------------------------------------------


main()
No changes just remember to add os.chdir() if your files are not in the same directory as this script
global totdf will need to reflect the columns in which you plan on merging



mergeData()
Will use logic to merge csv and txt files using a similar format. You can change the individual code logic for if 'txt' or if 'csv' to match what your new files have
mtotdf & df will need to reflect the colums in which you plan on merging

sentimentAnalysis(sentimentcolumn, keycolumn)

**inputs**
sentimentcolumn = whatever dataframe column you would like to run sentiment on
keycolumn = key that is associated with that record

example for this HW sentimentAnalysis('Purpose:','Name:')
can change these variables to output more than one record if needed using last 2 iloc commands

**output**
best sentiment , best key, worst sentiment, worst key(of highest & lowest both)




mostUsedTokens(num, column)

**inputs**
num = how many tokens you want to retrieve
column = which column you would like to get the tokens from

example for this HW mostUsedTokens(10, 'Purpose:')

**output**
an array of value_counts() which shows the token and amount it shows up
<file_sep>#!/usr/bin/env python
# coding: utf-8

# In[192]:


import os
import pandas as pd
import numpy as np
import re
from textblob import TextBlob
import nltk
#corpus
nltk.download('punkt')
#merged DataFrame to be used in each function
totdf = pd.DataFrame(columns = ['Name:','Purpose:'])
files = []


def main():
    return 0 
def mergeData():
    #local dataframe
    mtotdf = pd.DataFrame(columns = ['Name:','Purpose:'])
    #array of files within this directory
    files = os.listdir()
    for f in files:
        #logic for all text files
        if 'txt' in f:
            file = open(f,'r')
            #splitting text file into individual lines that go into an array
            dataframe = file.read().splitlines()
            #specific logic for individual file
            if f == 'hw2a2.txt':
                #use --- as delimeter
                dataframe = [re.split('---',x) for x in dataframe]
                #popout array of array into one array
                dataframe = [item for elem in dataframe for item in elem]
            #delete empty entries
            dataframe = [x for x in dataframe if x]
            #create a dataframe that has 2 columns (reshape) while deleting all special characters and deleting Name and Purpose while creating columns for Name and Purpose 
            df = pd.DataFrame(np.reshape(np.array([re.sub(r"[^a-zA-Z0-9]+", ' ', x.replace('Name','').replace('Purpose','')) for x in dataframe]),(-1,2)), columns = ['Name:','Purpose:'])
        
        
        #csv logic
        elif 'csv' in f:
            df = pd.read_csv(f)
            #logic for csv with index as first column
            if len(df.columns) == 3:
                df = df.drop(columns = 'Unnamed: 0')
            columns = df.columns.values
            #standardize column names
            df = df.rename(columns = {columns[0]:'Name:',columns[1]:'Purpose:'})
        else:
            df = pd.DataFrame(columns = ['Name:','Purpose:'])
        mtotdf = mtotdf.append(df,ignore_index = True)
    
    #make global df equal to local df
    global totdf 
    totdf = mtotdf
    return 'Data has been merged'
       


def sentimentAnalysis(sentimentcolumn, keycolumn):
    global totdf
    #polarity ranges from -1 (negative) to 1 (positive)
    polarity = []
    #subjectivity ranges from 0 (factual in messaging) to 1 (mostly opinion phrases)
    subjectivity = []
    index = 0
    #iterating through global dataframe for specified column
    for x in totdf[sentimentcolumn]:
        phrase = TextBlob(x)
        #Textblob values for polarity and sentiment are put in seperate arrays
        polarity = polarity + [phrase.sentiment.polarity]
        subjectivity = subjectivity  + [phrase.sentiment.subjectivity]
    #after iteration these arrays are added to the dataframe
    totdf['subjectivity'] = subjectivity
    totdf['polarity'] = polarity
    #bestsentiment is based on the highest polarity with the lowest subjectivity (most positive and most factual)
    bestsentiment = totdf[sentimentcolumn].iloc[totdf.sort_values(by = ['polarity','subjectivity'], ascending = [False,True]).index[0]]
    worstsentiment = totdf[sentimentcolumn].iloc[totdf.sort_values(by = ['polarity','subjectivity'], ascending = [False,True]).index[-1]]
    #key is returned as well to say which company in this case has that purpose
    bestkey = totdf[keycolumn].iloc[totdf.sort_values(by = ['polarity','subjectivity'], ascending = [False,True]).index[0]]
    worstkey = totdf[keycolumn].iloc[totdf.sort_values(by = ['polarity','subjectivity'], ascending = [False,True]).index[-1]]
    return bestsentiment, bestkey, worstsentiment, worstkey

def mostUsedTokens(num, column):
    global totdf
    tottokens = []
    #iterate through dataframe for a given column
    for x in totdf[column]:
        #iterate through each purpose in order to retrieve individual words
        for token in TextBlob(x).words:
            tottokens = tottokens + [token]
    #put tokens into dataframe
    tokendf = pd.DataFrame(tottokens,columns = ['Tokens'])
    #use value_counts to find the top num amount of tokens within that column
    topnum = tokendf['Tokens'].value_counts()[:num]
    #return the array of tokens with their values
    return topnum

if __name__ == "__main__":
    main()


