<repo_name>zkatemor/sentiment-analysis<file_sep>/models/dictionary.py
from sqlalchemy import Integer, String, ForeignKey, ARRAY

from db import session


class Dictionary(session.Model):
    __tablename__ = 'dictionary'
    id = session.Column(Integer, primary_key=True, autoincrement=True)
    name = session.Column(String(50), nullable=False)

    def __init__(self, name):
        self.name = name

    def __repr__(self):
        return "<Dictionary(id={}, name={}".format(
            self.id, self.name
        )

    def __str__(self):
        return self.name

    def serialize(self):
        return {
            'id': self.id,
            'name': self.name
        }


class Word(session.Model):
    __tablename__ = 'word'
    id = session.Column(Integer, primary_key=True, autoincrement=True)
    word = session.Column(String(120), nullable=False)
    part_of_speech = session.Column(String(50), nullable=False)
    tone = session.Column(String(50), nullable=False, default='None')

    dictionaryId = session.Column(Integer, ForeignKey('dictionary.id'), nullable=False)
    dictionary = session.relationship('Dictionary', backref=session.backref('words', lazy=True))

    def __init__(self, dictionaryId, word, tone, part_of_speech):
        self.dictionaryId = dictionaryId
        self.word = word
        self.tone = tone
        self.part_of_speech = part_of_speech

    def __repr__(self):
        return "<Word(id={}, word={}, tone={}, part_of_speech={}".format(
            self.id, self.word, self.tone, self.part_of_speech
        )

    def __str__(self):
        return "{}_{}".format(self.word, self.part_of_speech)

    def serialize(self):
        return {
            'id': self.id,
            'word': self.word,
            'tone': self.tone,
            'part_of_speech': self.part_of_speech,
            'dictionary_id': self.dictionaryId
        }
<file_sep>/scripts/get_test_review.py
import requests
import json
import os
from dotenv import load_dotenv

load_dotenv()

url = "https://market-scanner.ru/api/reviews"
key = os.environ.get('key')
reviews = []


def getReviewById(id):
    data = {'key': key,
            'id': id,
            }

    response = requests.post(url=url, data=data)

    js = response.json().get('reviews')
    print(js)
    reviews.append(js)


# 1) леново 201 2) леново 438 3) сяоми 629 4) хонор 421 5) мейзу 330 = 2019
ids = [9323494, 10414225, 13527763, 12423732, 12748971]

for i in ids:
    getReviewById(i)

with open('D:\\GitHub\\sentiment-analyzer\\app\\static\\reviews.json', 'w', encoding='utf-8') as f:
    json.dump(reviews, f, ensure_ascii=False, indent=4)
<file_sep>/README.md
<h1>Классификация текстов по тональности

Lexicon-based approach</h1>

<h2>How to</h2>

Запуск проекта:

`virtualenv venv`

`source venv/bin/activate`

`pip install -r requirements.txt`

`python manage.py runserver`

<h2>Наборы данных</h2>

Источник: https://market-scanner.ru/doc/reviews

* **app/static/review_db.json** – набор данных для построения словаря оценочной лексики
* **app/static/reviews.json** – набор данных для тестирования метода классикации
* **app/dictionaries** - используемые словари оценочной лексики и слова-модификаторы

<h2>Ссылка на проект</h2>

https://sentiment-dictionary-based.herokuapp.com/sentiment

<h2>About</h2>

Забабурина Е.А. РАЗРАБОТКА ПРОГРАММЫ КЛАССИФИКАЦИИ ТЕКСТОВ ПО ТОНАЛЬНОСТИ: Выпускная квалификационная работа/ ВятГУ, каф. ПМиИ; рук. Разова Е.В. – Киров, 2020. – ПЗ 79 с., 13 рис., 40 источников, 5 приложений.
<file_sep>/migrations/versions/78f4d0125f95_.py
"""empty message

Revision ID: 78f4d0125f95
Revises: <KEY>
Create Date: 2020-04-13 19:37:18.613766

"""
from alembic import op
import sqlalchemy as sa


# revision identifiers, used by Alembic.
revision = '<KEY>'
down_revision = '<KEY>'
branch_labels = None
depends_on = None


def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_column('word', 'part_of_speech')
    # ### end Alembic commands ###


def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.add_column('word', sa.Column('part_of_speech', sa.VARCHAR(length=50), autoincrement=False, nullable=False))
    # ### end Alembic commands ###
<file_sep>/app/__init__.py
def create_app():
    import os

    from flask import Flask
    from app.admin_panel import admin
    from db import session

    app = Flask(__name__)
    app.config.from_object(os.environ['APP_SETTINGS'])
    app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
    admin.init_app(app)
    session.init_app(app)

    from models.dictionary import Dictionary, Word
    return app
<file_sep>/app/services/reporter.py
import datetime as dt
import json


class Reporter:
    def __init__(self, text, result):
        self.text = text
        self.result = result

    def get_dict(self):
        report = {
            'text': self.text,
            'positive': list(self.result['words']['positive']),
            'negative': list(self.result['words']['negative']),
            'modifier': list(self.result['words']['modifier']),
            'score': self.result['score'],
            'sentiment': ""
        }

        if self.result['score'] > 0:
            report['sentiment'] = 'Положительная тональность. Чистая положительная оценка: ' + str(
                self.result['positive'])
        else:
            report['sentiment'] = 'Отрицательная тональность. Чистая отрицательная оценка: ' + str(
                self.result['negative'])

        return report

    def get_report(self):
        report = self.get_dict()
        date = str(dt.datetime.now().date()) + str(dt.datetime.now().time()).replace(".", ":").replace(":", "-")
        filename = "app/static/reports/Отчёт-" + date + ".json"

        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(report, f, ensure_ascii=False, indent=4)
        return filename.replace('app/static/', '')

    @staticmethod
    def get_reports(report):
        date = str(dt.datetime.now().date()) + str(dt.datetime.now().time()).replace(".", ":").replace(":", "-")
        filename = "app/static/reports/Отчёт-" + date + ".json"

        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(report, f, ensure_ascii=False, indent=4)
        return filename.replace('app/static/', '')
<file_sep>/scripts/score_rusentilex.py
"""формированиен весов слов для словаря rusentilex"""
import csv
from itertools import zip_longest
import pandas as pd


def get_weights(tones):
    result = []
    for i in tones:
        if i == 'positive':
            result.append(1)
        else:
            result.append(-1)
    return result


df = pd.read_csv('D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\rusentilex.csv',
                 encoding='utf-8')
term = df['term'].tolist()
weight = get_weights(df['tone'].tolist())

d = [term, weight]
export_data = zip_longest(*d, fillvalue='')

with open('D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\rusentilex.csv', 'w',
          newline='', encoding='utf-8') as f:
    wr = csv.writer(f)
    wr.writerow(("term", "weight"))
    wr.writerows(export_data)
f.close()
<file_sep>/requirements.txt
alembic==1.4.2
aniso8601==8.0.0
atomicwrites==1.4.0
attrs==19.3.0
certifi==2020.4.5.2
chardet==3.0.4
Click==7.0
colorama==0.4.3
cycler==0.10.0
DAWG-Python==0.7.2
docopt==0.6.2
enum34==1.1.10
et-xmlfile==1.0.1
Flask==1.1.1
Flask-Admin==1.5.6
Flask-Ext==0.1
Flask-Migrate==2.5.3
Flask-RESTful==0.3.8
Flask-Script==2.0.6
Flask-SQLAlchemy==2.4.1
gunicorn==20.0.4
idna==2.9
importlib-metadata==1.6.1
itsdangerous==1.1.0
jdcal==1.4.1
Jinja2==2.11.1
joblib==0.15.1
kiwisolver==1.2.0
Mako==1.1.2
MarkupSafe==1.1.1
matplotlib==3.2.1
more-itertools==8.3.0
nltk==3.5
numpy==1.18.2
openpyxl==3.0.3
packaging==20.4
pandas==0.23.4
pandas-ml==0.6.1
pluggy==0.13.1
psycopg2-binary==2.8.5
py==1.8.1
pymorphy2==0.8
pymorphy2-dicts==2.4.393442.3710985
pyparsing==2.4.7
pytest==5.4.3
python-dateutil==2.8.1
python-dotenv==0.12.0
python-editor==1.0.4
pytz==2019.3
regex==2020.6.8
requests==2.23.0
scikit-learn==0.23.1
scipy==1.4.1
seaborn==0.10.1
six==1.14.0
sklearn==0.0
SQLAlchemy==1.3.16
threadpoolctl==2.1.0
tqdm==4.46.1
urllib3==1.25.9
wcwidth==0.2.4
Werkzeug==1.0.0
WTForms==2.2.1
xlrd==1.2.0
zipp==3.1.0
<file_sep>/db/__init__.py
from flask_sqlalchemy import SQLAlchemy


session = SQLAlchemy()
<file_sep>/scripts/test.py
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report
import seaborn as sn

from app.services.sentimental import Sentimental


def get_report_test(dictionary, is_unigram):
    # результат классификации
    prediction = []
    # реальный результат
    actual = []

    sent = Sentimental(dictionary=dictionary,
                       negation='D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\negations.csv',
                       modifier='D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\modifier.csv',
                       is_unigram=is_unigram)

    for i in range(0, len(texts)):
        result = sent.analyze(texts[i])

        if ratings[i] in range(4, 6):
            actual.append('positive')
            if result['score'] > 0:
                prediction.append('positive')
            elif result['score'] <= 0:
                prediction.append('negative')
        else:
            actual.append('negative')
            if result['score'] > 0:
                prediction.append('positive')
            elif result['score'] <= 0:
                prediction.append('negative')

        print(i)

    print(classification_report(actual, prediction))
    return prediction, actual


# загрузка тестового набора данных
df = pd.read_excel('D:\\GitHub\\sentiment-analyzer\\app\static\\reviews.xlsx', sheet_name="Sheet1")
texts = df['Text'].tolist()
ratings = df['Rating'].tolist()

dictionary_rusentilex = ['D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\rusentilex.csv']

dictionary_chi = ['D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\chi_minus.csv',
                  'D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\chi_plus.csv']

dictionary_cnn = ['D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\cnn_dict.csv']

dictionary_chi_collocations = ['D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\chi_collocations_minus.csv',
                               'D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\chi_collocations_plus.csv']

prediction, actual = get_report_test(dictionary_chi, True)

data = {'prediction': prediction,
        'actual': actual
        }

df = pd.DataFrame(data, columns=['actual', 'prediction'])
matrix = pd.crosstab(df['actual'], df['prediction'], rownames=['Actual'], colnames=['Predicted'])
print(matrix)
plt.figure(figsize=(10, 7))
sn.heatmap(matrix, annot=True, cmap="Greens")
plt.title("CHI UNIGRAM")
plt.show()
<file_sep>/app/api/views.py
from flask_restful import Resource, reqparse
from models.dictionary import Dictionary, Word
from db import session


class DictionaryView(Resource):
    def get(self):
        parser = reqparse.RequestParser()
        parser.add_argument('id', required=False)
        args = parser.parse_args()

        if args['id']:
            if Dictionary.query.filter(Dictionary.id == args['id']).count():
                dictionaries = Dictionary.query.filter(Dictionary.id == args['id'])
                js = Dictionary.serialize(dictionaries[0])
            else:
                return {'message': 'Dictionary not found', 'result': {}}, 404
        else:
            dictionaries = Dictionary.query.all()
            js = [Dictionary.serialize(d) for d in dictionaries]

        return {'message': 'Success', 'result': js}, 200

    def post(self):
        parser = reqparse.RequestParser()
        parser.add_argument('name', required=True)
        args = parser.parse_args()
        dictionary = Dictionary(args['name'])

        session.session.add(dictionary)
        session.session.commit()
        return {'message': 'Success', 'result': {}}, 201, {'Location': '/dictionary/:' + str(dictionary.id)}


class WordView(Resource):
    def get(self):
        parser = reqparse.RequestParser()
        parser.add_argument('id', required=False)
        args = parser.parse_args()

        if args['id']:
            if Word.query.filter(Word.id == args['id']).count():
                words = Word.query.filter(Word.id == args['id'])
                js = Word.serialize(words[0])
            else:
                return {'message': 'Word not found', 'result': {}}, 404
        else:
            words = Word.query.all()
            js = [Word.serialize(d) for d in words]

        return {'message': 'Success', 'result': js}, 200

    def post(self):
        parser = reqparse.RequestParser()
        parser.add_argument('word', required=True)
        parser.add_argument('part_of_speech', required=True)
        parser.add_argument('tone', required=True)
        parser.add_argument('dictionary_id', required=True)
        args = parser.parse_args()

        word = Word(args['dictionary_id'],
                    args['word'],
                    args['tone'],
                    args['part_of_speech']
                    )

        session.session.add(word)
        session.session.commit()
        return {'message': 'Success', 'result': {}}, 201, {'Location': '/word/:' + str(word.id)}
<file_sep>/scripts/script.py
import json

import pandas as pd

with open('D:\\GitHub\\sentiment-analyzer\\app\static\\reviews.json', 'r', encoding='utf-8') as f:
    js = json.load(f)

texts = []
ratings = []

for model in js:
    for review in model:
        texts.append(str(review['pluses']) + '\n' + str(review['minuses']) + '\n' + str(review['comment']))
        ratings.append(review['rating'])

df = pd.DataFrame()
df['Text'] = texts
df['Rating'] = ratings

df.to_excel('D:\\GitHub\\sentiment-analyzer\\app\\static\\reviews.xlsx', index=False)
<file_sep>/scripts/get_weight_cnn.py
# загружаем неразмеченный словарь из json файла
import csv
import json
from itertools import zip_longest

import pandas as pd
import numpy as np

""" 
Get new csv file with term, frequency, tone for next operations
with open('unallocated_dictionary.json', 'r', encoding='utf-8') as f:
    js = json.load(f)

unallocated_words = [word[0] for word in js]
unallocated_frequency = [word[1] for word in js]

df = pd.read_csv('D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\original\\dict_cnn.csv', encoding='utf-8')
tone = df['tone'].tolist()

d = [unallocated_words, unallocated_frequency, tone]
export_data = zip_longest(*d, fillvalue='')

with open('D:\\GitHub\\sentiment-analyzer\\scripts\\cnn_dict_tmp.csv', 'w', encoding='utf-8',
          newline='') as f:
    wr = csv.writer(f)
    wr.writerow(("term", "frequency", "tone"))
    wr.writerows(export_data)
f.close()
"""


def scale_positive(chi_square):
    return 1 * (chi_square - np.min(chi_square)) / (np.max(chi_square) - np.min(chi_square))


def scale_negative(chi_square):
    return -1 * (chi_square - np.min(chi_square)) / (np.max(chi_square) - np.min(chi_square))


df = pd.read_csv('D:\\GitHub\\sentiment-analyzer\\scripts\\cnn_dict_tmp.csv', encoding='utf-8')

tone = df['tone'].tolist()
term = df['term'].tolist()
frequency = df['frequency'].tolist()

positive_tone = []
positive_term = []
positive_f = []

negative_term = []
negative_f = []

for i in range(0, len(tone)):
    if tone[i] == 'positive':
        positive_term.append(term[i])
        positive_f.append(frequency[i])
    else:
        negative_term.append(term[i])
        negative_f.append(frequency[i])

negative = np.array(negative_f)
negative_scaled = scale_negative(negative)

positive = np.array(positive_f)
positive_scaled = scale_positive(positive)

scale = list(positive_scaled) + list(negative_scaled)

d = [positive_term + negative_term, scale]
export_data = zip_longest(*d, fillvalue='')

with open('D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\cnn_dict.csv', 'w', encoding='utf-8',
          newline='') as f:
    wr = csv.writer(f)
    wr.writerow(("term", "weight"))
    wr.writerows(export_data)
f.close()
<file_sep>/app/services/sentimental.py
""" словарный метод классификации """
import os
import csv
import pymorphy2
import nltk
from collections import defaultdict


class Sentimental(object):
    def __init__(self, dictionary=None, negation=None, modifier=None, is_unigram=True):
        self.morph = pymorphy2.MorphAnalyzer()

        if dictionary is None and negation is None and modifier is None:
            base_dir = os.path.dirname(__file__)
            dictionary = [os.path.join(base_dir, p) for p in ['dictionaries/rusentilex.csv']]
            negation = os.path.join(base_dir, 'dictionaries/negations.csv')
            modifier = os.path.join(base_dir, 'dictionaries/modifier.csv')

        # словарь оценочной лексики
        self.dictionary = {}
        # слова-отрицатели
        self.negations = set()
        # слова-усилители
        self.modifiers = {}
        # является ли словарь - словарем униграм
        self.is_unigram = is_unigram

        # загрузка словаря и модификаторов из назначенных файлов
        for wl_filename in self.__to_arg_list(dictionary):
            self.load_dictionary(wl_filename)
        for negations_filename in self.__to_arg_list(negation):
            self.load_negations(negations_filename)
        for modifier_filename in self.__to_arg_list(modifier):
            self.load_modifiers(modifier_filename)

        self.__negation_skip = {'очень', 'самый', 'достаточно', 'должен'}

    @staticmethod
    def __to_arg_list(obj):
        if obj is not None:
            if not isinstance(obj, list):
                obj = [obj]
        else:
            obj = []
        return obj

    # распознавание слов-отрицателей
    def __is_prefixed_by_negation(self, token_idx, tokens):
        prev_idx = token_idx - 1
        if tokens[prev_idx] in self.__negation_skip:
            prev_idx -= 1

        is_prefixed = False
        if token_idx > 0 and prev_idx >= 0 and tokens[prev_idx] in self.negations:
            is_prefixed = True

        return is_prefixed, tokens[prev_idx]

    # распознавание слов-усилителей
    def __is_prefixed_by_modifier(self, token_idx, tokens):
        prev_idx = token_idx - 1
        percent = 0

        is_prefixed = False
        if token_idx > 0 and prev_idx >= 0 and tokens[prev_idx] in self.modifiers:
            is_prefixed = True
            percent = self.modifiers[tokens[prev_idx]]

        return is_prefixed, tokens[prev_idx], percent

    def load_negations(self, filename):
        with open(filename, 'r', encoding='utf-8') as f:
            reader = csv.DictReader(f)
            negations = set([row['token'] for row in reader])
        self.negations |= negations

    def load_modifiers(self, filename):
        with open(filename, 'r', encoding='utf-8') as f:
            reader = csv.DictReader(f)
            modifiers = {row['token']: row['percent'] for row in reader}
        self.modifiers.update(modifiers)

    def load_dictionary(self, filename):
        with open(filename, 'r', encoding='utf-8') as f:
            reader = csv.DictReader(f)
            if 'app/dictionaries/rusentilex.csv' in filename \
                    or 'D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\rusentilex.csv' in filename:
                dictionary = {row['term']: row['weight'] for row in reader}
            else:
                if self.is_unigram:
                    dictionary = {self.__delete_part_of_speech(row['term']): row['weight'] for row in reader}
                else:
                    dictionary = {self.__delete_part_of_speech(row['term']): row['weight'] for row in reader}
        self.dictionary.update(dictionary)

    # разбивание текста на токены
    def __get_tokens(self, sentence):
        # удаляем все символы, кроме кириллицы
        regex_tokenizer = nltk.tokenize.RegexpTokenizer('[а-яА-ЯЁё]+')
        # приводим к нижнему регистру
        words = regex_tokenizer.tokenize(sentence.lower())
        # приводим каждое слово в нормальную форму и добавляем тег - часть речи
        result = [self.morph.parse(w)[0].normal_form for w in words]
        return result

    # разбивание текста на биграмы
    def __get_bigrams(self, sentence):
        regex_tokenizer = nltk.tokenize.RegexpTokenizer('[а-яА-ЯЁё]+')
        words = regex_tokenizer.tokenize(sentence.lower())
        result = [self.morph.parse(words[i])[0].normal_form + ' ' + self.morph.parse(words[i + 1])[0].normal_form
                  for i in range(0, len(words) - 1)]
        return result

    # удаление части речи в слова или словосочетании
    def __delete_part_of_speech(self, term):
        if self.is_unigram:
            new_term = term[:-5]
        else:
            terms = term.split()
            new_term = terms[0][:-5] + ' ' + terms[1][:-5]
        return new_term

    # словарный метод
    def analyze(self, sentence):
        if self.is_unigram:
            terms = self.__get_tokens(sentence)
        else:
            terms = self.__get_bigrams(sentence)

        scores = defaultdict(float)
        words = defaultdict(set)
        comparative = 0

        for i, term in enumerate(terms):
            is_prefixed_by_negation, negation = self.__is_prefixed_by_negation(i, terms)
            is_prefixed_by_modifier, modifier, mod_percent = self.__is_prefixed_by_modifier(i, terms)
            if term in self.dictionary:
                tone = 'positive' if float(self.dictionary[term]) > 0 else 'negative'
                score = float(self.dictionary[term])
                words[tone].add(term)

                if is_prefixed_by_negation:
                    scores[tone] += score * -1
                    words['modifier'].add(negation)
                else:
                    scores[tone] += score

                if is_prefixed_by_modifier:
                    scores[tone] += score * float(mod_percent)
                    words['modifier'].add(modifier)

        if len(terms) > 0:
            comparative = (scores['positive'] + scores['negative']) / len(terms)

        # результат - словарь с отчетом
        result = {
            'score': scores['positive'] + scores['negative'],
            'positive': scores['positive'],
            'negative': scores['negative'],
            'comparative': comparative,
            'words': words
        }

        return result
<file_sep>/manage.py
"""исходный код программы классификации текста по тональности"""
import csv
import os
import shutil

import pandas as pd
from werkzeug.utils import secure_filename

from dotenv import load_dotenv
from flask import render_template, request
from flask_restful import Api
from flask_script import Manager
from flask_migrate import Migrate, MigrateCommand

from app import create_app
from app.api.views import DictionaryView, WordView
from app.services.random import Random
from app.services.reporter import Reporter
from db import session
from app.services.sentimental import Sentimental

load_dotenv()

UPLOAD_FOLDER = 'app/static/uploads'
ALLOWED_EXTENSIONS = {'xlsx'}

app = create_app()

migrate = Migrate(app, session)
manager = Manager(app)

manager.add_command('db', MigrateCommand)

api = Api(app)
api.add_resource(DictionaryView, '/dictionary')
api.add_resource(WordView, '/word')
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['UPLOAD_FILENAME'] = ''
app.config['DICTIONARY'] = ['app/dictionaries/rusentilex.csv']
app.config['IS_UNIGRAM'] = True


# удаление старых файлов отчетов для экономии места на сервере
def delete_old_files(folder):
    for filename in os.listdir(folder):
        file_path = os.path.join(folder, filename)
        try:
            if os.path.isfile(file_path) or os.path.islink(file_path):
                os.unlink(file_path)
            elif os.path.isdir(file_path):
                shutil.rmtree(file_path)
        except Exception as e:
            print('Failed to delete %s. Reason: %s' % (file_path, e))


@app.route('/sentiment', methods=['GET', 'POST'])
def index():
    try:
        if request.method == 'GET':
            """home page"""
            result = {
                'score': 0,
                'positive': 0,
                'negative': 0,
                'comparative': 0,
                'words': {}
            }
            random = Random('app/static/reviews.json')
            text = random.get_review()
            img = 'icons/hand.png'
            positive = ''
            negative = ''
            modifier = ''
            report = ''
        else:
            delete_old_files('app/static/reports')
            text = request.form['text']
            select = request.form.get('select')
            dictionary = ['app/dictionaries/rusentilex.csv']
            is_unigram = True

            if str(select) == 'CHI unigram':
                dictionary = ['app/dictionaries/chi_minus.csv', 'app/dictionaries/chi_plus.csv']
                is_unigram = True
            elif str(select) == 'CNN unigram':
                dictionary = ['app/dictionaries/cnn_dict.csv']
                is_unigram = True
            elif str(select) == 'CHI bigram':
                dictionary = ['app/dictionaries/chi_collocations_minus.csv',
                              'app/dictionaries/chi_collocations_plus.csv']
                is_unigram = False

            sent = Sentimental(dictionary=dictionary,
                               negation='app/dictionaries/negations.csv',
                               modifier='app/dictionaries/modifier.csv',
                               is_unigram=is_unigram)
            result = sent.analyze(text)
            if result['score'] > 0:
                img = 'icons/positive.png'
            elif result['score'] == 0:
                img = 'icons/hand.png'
            else:
                img = 'icons/negative.png'

            positive_str = [str(s) for s in result['words']['positive']]
            positive = ", ".join(positive_str)

            negative_str = [str(s) for s in result['words']['negative']]
            negative = ", ".join(negative_str)

            mod_str = [str(s) for s in result['words']['modifier']]
            modifier = ", ".join(mod_str)

            report = Reporter(text, result).get_report()

        return render_template("index.html",
                               text=text,
                               result=result,
                               img=img,
                               positive=positive,
                               negative=negative,
                               modifier=modifier,
                               download_file=report)

    except Exception as e:
        print(e)


@app.route('/dictionaries/<name>', methods=['GET'])
def get_dictionaries(name):
    about_dictionary = ""
    name_dictionary = ""
    dictionary = {}
    html_file = ""
    if name == 'chi_unigram':
        name_dictionary = "<NAME>, построенный с помощью критерия согласия Пирсона (Хи-квадрат)"
        about_dictionary = "Распределение тонально-окрашенных слов с помощью критерия анализа – оценки корреляции " \
                           "двух событий: «слово содержится в корпусе» и «слово содержится в подкорпусе " \
                           "достоинств или недостатков». Благодаря методу, основанному на распределении оценочных слов " \
                           "с помощью критерия согласия Пирсона, удалось получить 1260 положительно окрашенных слов и " \
                           "2267 с отрицательной окраской."
        with open('app/dictionaries/chi_plus.csv', encoding='utf-8') as f:
            dictionary = dict(filter(None, csv.reader(f)))
        dictionary.pop('term')
        with open('app/dictionaries/chi_minus.csv', encoding='utf-8') as f:
            dictionary_minus = dict(filter(None, csv.reader(f)))
        dictionary_minus.pop('term')
        dictionary.update(dictionary_minus)
    elif name == 'cnn_unigram':
        name_dictionary = "<NAME>, построенный с помощью свёрточной нейронной сети"
        about_dictionary = "Обученная с помощью сверточной нейросети модель и предобученной модели векторных " \
                           "представлений слов Word2Vec показала следующие результаты: удалось выявить 373 " \
                           "положительно-окрашенных слов, 5216 отрицательно-окрашенных слов и 114 " \
                           "нейтрально-окрашенных слов. Данное распределение связано с тем, что и размеченные данных, " \
                           "взятые из словаря RuSentiLex не были сбалансированными (отрицательных слов 7148, " \
                           "положительных – 2774 и нейтральных – 746)."
        with open('app/dictionaries/cnn_dict.csv', encoding='utf-8') as f:
            dictionary = dict(filter(None, csv.reader(f)))
        dictionary.pop('term')
    elif name == 'chi_bigram':
        name_dictionary = "<NAME>, построенный с помощью критерия согласия Пирсона (Хи-квадрат)"
        about_dictionary = "Распределение тонально-окрашенных словосочетаний с помощью критерия анализа – оценки " \
                           "корреляции " \
                           "двух событий: «слововочетание содержится в корпусе» и «словосочетание содержится в " \
                           "подкорпусе " \
                           "достоинств или недостатков». Благодаря методу, основанному на распределении оценочных слов " \
                           "с помощью критерия согласия Пирсона, удалось получить 20520 положительно окрашенных " \
                           "словосочетаний и " \
                           "19550 с отрицательной окраской."
        with open('app/dictionaries/chi_collocations_plus.csv', encoding='utf-8') as f:
            dictionary = dict(filter(None, csv.reader(f)))
        dictionary.pop('term')
        with open('app/dictionaries/chi_collocations_minus.csv', encoding='utf-8') as f:
            dictionary_minus = dict(filter(None, csv.reader(f)))
        dictionary_minus.pop('term')
        dictionary.update(dictionary_minus)
    elif name == 'all':
        name_dictionary = "Словари оценочной лексики для классификации текстов по тональности"
        html_file = "all_dictionary.html"
        dictionary = dict(
            хороший=1,
            неплохой=0.5,
            нехороший=-0.5,
            плохой=-1
        )
    return render_template("dictionaries.html",
                           about_dictionary=about_dictionary,
                           dictionary=dictionary,
                           name_dictionary=name_dictionary,
                           html_file=html_file)


@app.route('/about', methods=['GET'])
def about_project():
    return render_template("about.html")


def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS


@app.route('/analyze/files', methods=['GET', 'POST'])
def files_classifier():
    delete_old_files('app/static/reports')
    delete_old_files('app/static/uploads')
    if request.method == 'POST':
        file = request.files['file']
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            app.config['UPLOAD_FILENAME'] = filename
            select = request.form.get('select')
            if str(select) == 'CHI unigram':
                dictionary = ['app/dictionaries/chi_minus.csv', 'app/dictionaries/chi_plus.csv']
                is_unigram = True
            elif str(select) == 'CNN unigram':
                dictionary = ['app/dictionaries/cnn_dict.csv']
                is_unigram = True
            elif str(select) == 'RuSentiLex':
                dictionary = ['app/dictionaries/rusentilex.csv']
                is_unigram = True
            elif str(select) == 'CHI bigram':
                dictionary = ['app/dictionaries/chi_collocations_minus.csv',
                              'app/dictionaries/chi_collocations_plus.csv']
                is_unigram = False
            app.config['DICTIONARY'] = dictionary
            app.config['IS_UNIGRAM'] = is_unigram

    return render_template("files_analyze.html")


@app.route('/analyze/files/result', methods=['GET', 'POST'])
def ajax_files():
    if os.listdir(app.config['UPLOAD_FOLDER']):
        filename = 'app/static/uploads/' + app.config['UPLOAD_FILENAME']
        df = pd.read_excel(filename, sheet_name="Лист1")
        texts = df['text'].tolist()

        dictionary = app.config['DICTIONARY']
        is_unigram = app.config['IS_UNIGRAM']

        sent = Sentimental(dictionary=dictionary,
                           negation='app/dictionaries/negations.csv',
                           modifier='app/dictionaries/modifier.csv', is_unigram=is_unigram)
        report = []

        for text in texts:
            result = sent.analyze(text)
            report.append(Reporter(text, result).get_dict())

        download_file = Reporter.get_reports(report)
        return render_template('include_files.html', download_file=download_file)
    else:
        return '<h3>Файл ещё не загружен</h3>'


if __name__ == '__main__':
    manager.run()
<file_sep>/app/services/random.py
import json
import random


class Random:
    def __init__(self, filename):
        self.reviews = None
        self.load_review(filename)

    def load_review(self, filename):
        with open(filename, 'r', encoding='utf-8') as f:
            reviews = json.load(f)
        self.reviews = reviews

    def get_review(self):
        texts = [str(review['pluses']) + '\n' + str(review['minuses']) + '\n' + str(review['comment'])
                 for model in self.reviews for review in model]
        texts = list(map(lambda x: x.replace('None', ''), texts))
        return random.choice(texts)
<file_sep>/app/admin_panel/__init__.py
from flask_admin import Admin
from flask_admin.contrib.sqla import ModelView


from db import session
from models.dictionary import Dictionary, Word

admin = Admin(name='Sentiment dashboard', template_mode='bootstrap3')

admin.add_view(ModelView(Dictionary, session.session))
admin.add_view(ModelView(Word, session.session))

<file_sep>/scripts/get_weight_minus.py
"""формирование весов слов для отлицательного словаря хи-квадрат"""
import csv
from itertools import zip_longest
import numpy as np
import pandas as pd


def scale(chi_square):
    return -1 * (chi_square - np.min(chi_square)) / (np.max(chi_square) - np.min(chi_square))


df = pd.read_csv('D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\original\\dict_collocations_minus.csv', encoding='utf-8')
chi = np.array(df['chi_square'])
chi_scaled = scale(chi)

words = df['term'].tolist()

d = [words, chi_scaled]
export_data = zip_longest(*d, fillvalue='')

with open('D:\\GitHub\\sentiment-analyzer\\app\\dictionaries\\chi_collocations_minus.csv', 'w', encoding='utf-8',
          newline='') as f:
    wr = csv.writer(f)
    wr.writerow(("term", "weight"))
    wr.writerows(export_data)
f.close()
