<repo_name>o-b-one/Trendy<file_sep>/processors/estimator/src/classes/entity.py
class Entity:

    def __init__(self, text, label,id, total_entities):
        self.text = text
        self.label = label
        self.id = id
        self.count = 0
        self.total_entities = total_entities
    
    def increment(self):
        self.count += 1
    
    def decrese(self):
        self.count -= 1

    def add_to_total(self, total_to_add):
        self.total_entities += total_to_add

    def get_ratio(self):
        total = self.total_entities if self.total_entities is not None and self.total_entities > 0 else 1
        return self.count / total

    def to_tuple(self):
        
        return (self.text, self.label, self.id, self.count, self.total_entities, self.get_ratio())<file_sep>/processors/estimator/src/processors/estimator.py
import re
import spacy
import pandas as pd
from src.classes import entity
from src.processors import sentiment_estimator



TRASHOLD = .25


FILTERED_LABELS = [
    'ORDINAL',
    'QUANTITY',
    'MONEY',
    'PERCENT',
    'TIME',
    'DATE',
    'CARDINAL',
    'LANGUAGE',
    'LOC'
]

class Estimator:
    languages_dictionary = {
        'en': 'English'
    }
    def __init__(self, lang = 'en', analyze_algo = 'afinn'):
        self.lang = self.languages_dictionary[lang]
        self.analyze_algo = analyze_algo
        self.nlp = spacy.load('en_core_web_sm')
        self.sentiment_estimator = sentiment_estimator.SentimentEstimator()
    
    # def lemmatize_stemming(self,text):
        # token = WordNetLemmatizer().lemmatize(text, pos='v')
        # print(token)
        # return stemmer.stem(token)


    def build_pipeline(self):
        self.nlp.add_pipe(self.nlp.create_pipe('sentencizer'))

    
    def analyze_paragraph(self, paragraph, topics_dictionary):
        text = (paragraph)
        doc = self.nlp(text)
        paragraph_topics = {}
        # Analyze syntax
        #print("Noun phrases:", [chunk.text for chunk in doc.noun_chunks])
        #print("Verbs:", [token.lemma_ for token in doc if token.pos_ == "VERB"])

        # Find named entities, phrases and concepts
        
        filtered_entities = {entity.text.lower() for entity in doc.ents if entity.label_ not in FILTERED_LABELS }
        total_entities = len(filtered_entities)
        
        for doc_entity in doc.ents:
            entity_text = doc_entity.text.lower()
            if entity_text not in topics_dictionary:
                topics_dictionary[entity_text] = entity.Entity(entity_text, doc_entity.label_, doc_entity.label, total_entities)
            if entity_text not in paragraph_topics:
                paragraph_topics[entity_text] = entity.Entity(entity_text, doc_entity.label_, doc_entity.label, total_entities)
                topics_dictionary[entity_text].add_to_total(total_entities)

            if topics_dictionary[entity_text].label not in FILTERED_LABELS:
                topics_dictionary[entity_text].increment()
                paragraph_topics[entity_text].increment()
        
        sentiment, accuracy = self.sentiment_estimator.estimate(paragraph)
        return Estimator.sort_topics(Estimator.normalize_results(paragraph_topics)), sentiment, accuracy


       

    # Return estimation 
    # @param {string} text
    # @return {[topic: String]: Float}
    def estimate(self, text):
        topics_dictionary = {}

        # Get paragraphs from text
        paragraphs = [match.group(0) for match in re.finditer('(.)+\n*|\n(.)+/g', text)]
        print(len(paragraphs))

        paragraphs_results = []
        for paragraph in paragraphs:
            paragraph_topics, sentiment, accuracy = self.analyze_paragraph(paragraph, topics_dictionary)
            paragraphs_results.append((paragraph_topics,sentiment, accuracy))

        topics_list = Estimator.normalize_results(topics_dictionary)
        Estimator.sort_topics(topics_list)
        print(pd.DataFrame(paragraphs_results, columns=['Results', 'Sentiment', 'Accuracy']))
        return topics_list[:5]


    @staticmethod
    def sort_topics(topics_tuple):
        topics_tuple.sort(key = lambda topic: topic[5], reverse = True)
        return topics_tuple

    @staticmethod
    def normalize_results(topics_dictionary):
        return [topics_dictionary[topic].to_tuple() for topic in topics_dictionary if topics_dictionary[topic].get_ratio() >= TRASHOLD]    
   <file_sep>/server/src/app/app.ts
import { defaultErrorHandler, express, HttpServer } from "@sugoi/server";
import * as bodyParser from 'body-parser';
import * as compression from 'compression';
import * as path from "path";
import { paths } from "../config/paths";
import { BootstrapModule } from "./bootstrap.module";
import { Authorization } from "./classes/authorization.class"
import { MongoModel } from "@sugoi/mongodb";
import { services } from "../config/services";

const DEVELOPMENT = process.env['isDev']  as any;
const TESTING = process.env['isTest']  as any;
const PROD = process.env['isProd']  as any;

const setDBs = function (app) {
    MongoModel.setConnection(services.MONGODB).catch(console.error);
};


const server: HttpServer = HttpServer.init(BootstrapModule, "/", Authorization)
    .setStatic(paths.staticDir) // set static file directory path
    .setMiddlewares((app) => {
        app.disable('x-powered-by');
        app.set('etag', 'strong');
        app.set('host', process.env.HOST || '0.0.0.0');
        app.use(express.json());
        app.use(compression());

        setDBs(app);
    })
    .setErrorHandlers((app) => {
        app.use((req, res, next) => {
            // Set fallback to send the web app index file
            return res.sendFile(path.resolve(paths.index))
        });
        // The value which will returns to the client in case of an exception
        app.use(console.error);
        app.use(defaultErrorHandler(DEVELOPMENT || TESTING));
    });

export {server};
<file_sep>/processors/estimator/__main__.py
import sys
from pathlib import Path
from src.processors import estimator

lang = 'en'
file_path = 'mock/article.txt'
text = None
topics = ''

if(len(sys.argv) > 1):
    file_path = sys.argv[1]
if(len(sys.argv) > 2):
    lang = sys.argv[2]


filename = Path(file_path)

with open(filename.absolute(),'r') as reader:
    text = reader.read()

estimator = estimator.Estimator(lang)
print(estimator.estimate(text))<file_sep>/server/src/modules/index/models/data.model.ts
import { ModelName, MongoModel } from '@sugoi/mongodb';

@ModelName("data")
export class DataModel extends MongoModel {
    public get id() {
            return this._id.toString();
    }

    constructor() {
        super();
    }

}<file_sep>/processors/estimator/src/processors/sentiment_estimator.py
import ktrain
from ktrain import text

from sklearn.datasets import fetch_20newsgroups


class SentimentEstimator:

    def __init__(self, language = 'english'):
        self.train(language)

    def train(self, language):
        categories = ['alt.atheism', 'soc.religion.christian','comp.graphics', 'sci.med']
        train_b = fetch_20newsgroups(subset='train',categories=categories, shuffle=True, random_state=42)
        test_b = fetch_20newsgroups(subset='test',categories=categories, shuffle=True, random_state=42)
        print('size of training set: %s' % (len(train_b['data'])))
        print('size of validation set: %s' % (len(test_b['data'])))
        print('classes: %s' % (train_b.target_names))

        x_train = train_b.data
        y_train = train_b.target
        x_test = test_b.data
        y_test = test_b.target
        (x_train,  y_train), (x_test, y_test), preproc = text.texts_from_array(x_train=x_train, y_train=y_train, 
       x_test=x_test, y_test=y_test,class_names=train_b.target_names,preprocess_mode='bert',maxlen=350,max_features=35000)

        model = text.text_classifier('bert', train_data=(x_train, y_train), preproc=preproc)
        learner = ktrain.get_learner(model, train_data=(x_train, y_train), batch_size=6)
        learner.fit_onecycle(2e-5, 4)
        learner.validate(val_data=(x_test, y_test), class_names=train_b.target_names)
        self.predictor = ktrain.get_predictor(learner.model, preproc)
        

    def estimate(self, text):
       print('predict', self.predictor.predict([text]))
       return sentiment, accuracy
<file_sep>/server/src/modules/processor/controllers/processor.controller.ts
import {
    Controller,
    RequestBody,
    HttpPost
 } from "@sugoi/server";
import { ProcessorService } from "../services/processor.service";

@Controller('/processor')
export class ProcessorController {

    constructor(
        private processorService:ProcessorService
    ){
    }


    @HttpPost("/emotion")
    async index(@RequestBody() body:{text: string}){
        const estimation = await this.processorService.estimateEmotion(body.text, 'Apple');
        return {estimation};
    }

}<file_sep>/processors/estimator/requirements.txt
asn1crypto==0.24.0
astor==0.8.0
astroid==2.3.3
attrs==19.3.0
blis==0.4.1
bokeh==1.4.0
boto==2.49.0
boto3==1.10.25
botocore==1.13.25
catalogue==0.0.8
cchardet==2.1.5
certifi==2019.11.28
chardet==3.0.4
cryptography==2.1.4
cycler==0.10.0
cymem==2.0.3
decorator==4.4.1
docutils==0.15.2
eli5==0.10.1
fastprogress==0.1.22
gast==0.3.2
gensim==3.8.1
google-pasta==0.1.8
graphviz==0.13.2
grpcio==1.25.0
guidedlda==2.0.0.dev22
h5py==2.10.0
httplib2==0.9.2
idna==2.8
importlib-metadata==0.23
isort==4.3.21
jieba==0.39
Jinja2==2.10.3
jmespath==0.9.4
joblib==0.14.0
Keras==2.2.4
Keras-Applications==1.0.8
keras-bert==0.80.0
keras-embed-sim==0.7.0
keras-layer-normalization==0.14.0
keras-multi-head==0.22.0
keras-pos-embd==0.11.0
keras-position-wise-feed-forward==0.6.0
Keras-Preprocessing==1.1.0
keras-self-attention==0.41.0
keras-transformer==0.31.0
keyring==10.6.0
keyrings.alt==3.0
kiwisolver==1.1.0
ktrain==0.6.1
langdetect==1.0.7
launchpadlib==1.10.6
lazr.restfulclient==0.13.5
lazr.uri==1.0.3
lazy-object-proxy==1.4.3
macaroonbakery==1.1.3
Mako==1.0.7
Markdown==3.1.1
MarkupSafe==1.1.1
matplotlib==3.1.2
mccabe==0.6.1
more-itertools==7.2.0
murmurhash==1.0.2
netifaces==0.10.4
networkx==2.3
nltk==3.4.5
numpy==1.17.4
oauth==1.0.1
olefile==0.45.1
packaging==19.2
pandas==0.25.3
pexpect==4.2.1
Pillow==6.2.1
plac==1.1.3
preshed==3.0.2
protobuf==3.11.0
pycairo==1.16.2
pycrypto==2.6.1
pycups==1.9.73
pylint==2.4.4
pymacaroons==0.13.0
PyNaCl==1.1.2
pyparsing==2.4.5
python-dateutil==2.8.1
python-debian==0.1.32
pytz==2019.3
pyxdg==0.25
PyYAML==5.1.2
regex==2019.11.1
reportlab==3.4.0
requests==2.22.0
requests-unixsocket==0.1.5
s3transfer==0.2.1
scikit-learn==0.21.3
scipy==1.3.3
SecretStorage==2.3.1
seqeval==0.0.12
simplejson==3.13.2
six==1.13.0
sklearn==0.0
smart-open==1.9.0
spacy==2.2.3
srsly==0.2.0
systemd-python==234
tabulate==0.8.6
tensorboard==1.14.0
tensorflow==1.14.0
tensorflow-estimator==1.14.0
termcolor==1.1.0
thinc==7.3.1
tornado==6.0.3
tqdm==4.39.0
typed-ast==1.4.0
ubuntu-drivers-common==0.0.0
ufw==0.36
unattended-upgrades==0.1
urllib3==1.25.7
usb-creator==0.3.3
virtualenv==16.7.8
wadllib==1.3.2
wasabi==0.4.0
Werkzeug==0.16.0
wrapt==1.11.2
xkit==0.0.0
zipp==0.6.0
zope.interface==4.3.2
<file_sep>/processors/estimator/src/processors/sentiment_estimator copy.py
import collections
import numpy
import random
import re, string, random  
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.corpus import twitter_samples, stopwords
from nltk.tag import pos_tag
from nltk.tokenize import word_tokenize
from nltk import FreqDist, classify, NaiveBayesClassifier
from nltk.metrics.scores import (precision, recall)
import spacy
from pandas import DataFrame, read_csv

class SentimentEstimatorV2:

    def __init__(self, language = 'english'):
        self.train(language)

    def remove_noise(self, tweet_tokens, stop_words = ()):

        cleaned_tokens = []

        for token, tag in pos_tag(tweet_tokens):
            token = re.sub('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+#]|[!*\(\),]|'\
                        '(?:%[0-9a-fA-F][0-9a-fA-F]))+','', token)
            token = re.sub("(@[A-Za-z0-9_]+)","", token)

            if tag.startswith("NN"):
                pos = 'n'
            elif tag.startswith('VB'):
                pos = 'v'
            else:
                pos = 'a'

            lemmatizer = WordNetLemmatizer()
            token = lemmatizer.lemmatize(token, pos)

            if len(token) > 0 and token not in string.punctuation and token.lower() not in stop_words:
                cleaned_tokens.append(token.lower())
        return cleaned_tokens

    def get_all_words(self, cleaned_tokens_list):
        for tokens in cleaned_tokens_list:
            for token in tokens:
                yield token

    def get_tweets_for_model(self, cleaned_tokens_list):
        for tweet_tokens in cleaned_tokens_list:
            yield dict([token, True] for token in tweet_tokens)

    def train(self, language):
        df = read_csv("./dataset/SentiWordNet_3.0.0.tsv", sep="\t", header = 0, index_col='ID')

        labeled = []
        for row in df.iterrows():
            score = 0
            if float(row[1]['NegScore']) > 0 :
                score = float(numpy.tanh(row[1]['PosScore']) / -float(-row[1]['NegScore']))
            else:
                score = float(numpy.tanh(row[1]['PosScore']))
            try:
                tokenized = word_tokenize(row[1]['Gloss'])
            except:
                continue
            
            item = (
                tokenized,
                score
            )
            labeled.append(item)

        stop_words = stopwords.words(language)
        
        # positive_tweet_tokens = twitter_samples.tokenized('positive_tweets.json')
        # negative_tweet_tokens = twitter_samples.tokenized('negative_tweets.json')

        # positive_cleaned_tokens_list = []
        # negative_cleaned_tokens_list = []

        # for tokens in positive_tweet_tokens:
        #     positive_cleaned_tokens_list.append(self.remove_noise(tokens, stop_words))

        # for tokens in negative_tweet_tokens:
        #     negative_cleaned_tokens_list.append(self.remove_noise(tokens, stop_words))

        # labeled_cleaned_tokens_list = self.remove_noise(labeled, stop_words)
        # all_pos_words = self.get_all_words(labeled_cleaned_tokens_list)

        # freq_dist_pos = FreqDist(all_pos_words)

        # positive_tokens_for_model = self.get_tweets_for_model(positive_cleaned_tokens_list)
        # negative_tokens_for_model = self.get_tweets_for_model(negative_cleaned_tokens_list)

        # positive_dataset = [(tweet_dict, "Positive")
        #                         for tweet_dict in positive_tokens_for_model]

        # negative_dataset = [(tweet_dict, "Negative")
                                # for tweet_dict in negative_tokens_for_model]

        dataset = labeled

        random.shuffle(dataset)

        train_data = dataset[:7000]
        test_data = dataset[7000:]
        self.classifier = NaiveBayesClassifier.train(train_data)
        self.total_accuracy = classify.accuracy(self.classifier, test_data)

        self.refsets = collections.defaultdict(set)
        self.testsets = collections.defaultdict(set)
        print('Total accuracy: ',self.total_accuracy)

    def estimate(self, text):
       custom_tokens = self.remove_noise(word_tokenize(text))
       bow = dict([token, True] for token in custom_tokens)
       sentiment = self.classifier.classify(bow)
       accuracy = -1 # precision(self.refsets[sentiment], self.testsets[sentiment])
       return sentiment, accuracy
<file_sep>/server/src/modules/index/controllers/index.controller.ts
import {
    Controller,
    HttpGet,
    RequestParam,
    Authorized
 } from "@sugoi/server";
import { IndexService } from "../services/index.service";

@Controller('/index')
@Authorized()
export class IndexController {

    constructor(
        private indexService:IndexService
    ){
    }


    @HttpGet("/:payload?")
    async index(@RequestParam("payload") payload:string){
        const responseMsg = "index is ready!";
        return payload
                ? `${responseMsg} got payload ${payload}`
                : `${responseMsg}`;
    }

}<file_sep>/server/src/modules/processor/processor.module.ts
import { ServerModule } from "@sugoi/server"
import { ProcessorController } from "./controllers/processor.controller";
import { ProcessorService } from "./services/processor.service";

@ServerModule({
    modules:[],
    controllers:[ProcessorController],
    services:[ProcessorService]
})
export class ProcessorModule{}<file_sep>/server/src/modules/processor/services/processor.service.ts
import { Injectable } from "@sugoi/server";
import * as NLP from "natural";


@Injectable()
export class ProcessorService {
    private analyzer = new NLP.SentimentAnalyzer('English', NLP.ProterStemmer, "afinn");
    private sentenceTokenizer = new NLP.SentenceTokenizer();
    private wordTokenizer = new NLP.WordTokenizer();
    // private sentenceAnalyzer = new NLP.SentenceAnalyzer();

    constructor(){
    }


    async estimateEmotion(text:string, subject?: string): Promise<number>{
        // todo: Check emotion per subject for understand the Subject of the emotion.
        let estimation = 0, unnutral = 0;
        const sentences = this.sentenceTokenizer.tokenize(text);
        if( sentences.length === 0 ){
            return estimation;
        }
        // // const corpus = new NLP.Corpus(text, 1, NLP.Sentence);
        // // console.log(corpus)

        try{
        console.log('text',text);
            console.log("sentences", sentences);
            await Promise.all(
                sentences.map( async sentence => {
                    const tokens: string[] = this.wordTokenizer.tokenize(sentence);
                    // const analyzer = await new Promise(res => new NLP.SentenceAnalyzer(pos,res));
                    // console.log(analyzer);
                    const tokenEstimation = this.analyzer.getSentiment(tokens);
                    console.log(sentence,tokens,tokenEstimation);
                    estimation += tokenEstimation;
                    if(tokenEstimation !== 0){
                        unnutral++;
                    }
                })
            );
        }catch(e){
            console.error(e);
            throw e;
        }
        return estimation / (unnutral || 1);
    }


}<file_sep>/processors/estimator/setup.py
import os
from setuptools import setup, find_packages


# string in below ...
def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(
    name = "sentiment_estimator",
    version = "0.0.1",
    author = "<NAME>",
    author_email = "<EMAIL>",
    description = ("A way to estimate sentiment based on articles "),
    license = "BSD",
    keywords = "",
    packages=[],
    long_description=read('README'),
    classifiers=[
        "Development Status :: 1 - Planning",
        "Topic :: NLP",
        "Environment :: Console",
        "License :: OSI Approved :: BSD License",
    ],
)
