<file_sep># -*- coding: utf-8 -*-

from vectorizers.bert_vectorizer import BERTVectorizer
from vectorizers import tokenizationK
from models.bert_nlu_model import BertNLUModel
from utils import flatten

import argparse
import os
import pickle
import tensorflow as tf



# read command-line parameters
parser = argparse.ArgumentParser('Evaluating the Joint BERT / ALBERT NLU model')
parser.add_argument('--model', '-m', help = 'Path to BERT / ALBERT NLU model', type = str, required = True)
parser.add_argument('--type', '-tp', help = 'bert   or    albert', type = str, default = 'bert', required = False)


VALID_TYPES = ['bert', 'albert']

args = parser.parse_args()
load_folder_path = args.model
type_ = args.type

# this line is to disable gpu
os.environ["CUDA_VISIBLE_DEVICES"]="-1"

config = tf.ConfigProto(intra_op_parallelism_threads=1,
                        inter_op_parallelism_threads=1,
                        allow_soft_placement=True,
                        device_count = {'CPU': 1})
sess = tf.compat.v1.Session(config=config)

if type_ == 'bert':
    bert_model_hub_path = './bert-module'
    is_bert = True
elif type_ == 'albert':
    bert_model_hub_path = 'https://tfhub.dev/google/albert_base/1'
    is_bert = False
else:
    raise ValueError('type must be one of these values: %s' % str(VALID_TYPES))


bert_vectorizer = BERTVectorizer(is_bert, bert_model_hub_path)

# loading models
print('Loading models ...')
if not os.path.exists(load_folder_path):
    print('Folder `%s` not exist' % load_folder_path)

with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'), 'rb') as handle:
    intents_label_encoder = pickle.load(handle)
    intents_num = len(intents_label_encoder.classes_)


model = BertNLUModel.load(load_folder_path, sess)

vocab_file = "./bert-module/assets/vocab.korean.rawtext.list"
tokenizer = tokenizationK.FullTokenizer(vocab_file=vocab_file)

while True:
    print('\nEnter your sentence: ')
    try:
        input_text = input().strip()
    except:
        continue
        
    if input_text == 'quit':
        break

    tokens = tokenizer.tokenize(input_text)
    tokens = [' '.join(tokens)]

    data_input_ids, data_input_mask, data_segment_ids = bert_vectorizer.transform(tokens)
    
    first_inferred_intent, first_inferred_intent_score, _, _ = model.predict_intent([data_input_ids, data_input_mask, data_segment_ids], intents_label_encoder)

    print(first_inferred_intent)
    print(first_inferred_intent_score)

tf.compat.v1.reset_default_graph()

<file_sep>import re
import os
import sys
import json

import logging
import numpy as np
import pandas as pd
import tensorflow as tf
import tensorflow_hub as hub

from tensorflow import keras
from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint

from sklearn.model_selection import train_test_split

from modeling import BertModel, BertConfig


# get TF logger 
log = logging.getLogger('tensorflow')
log.handlers = []

def build_module_fn(config_path, vocab_path, do_lower_case=True):

    def bert_module_fn(is_training):
        """Spec function for a token embedding module."""

        input_ids = tf.placeholder(shape=[None, None], dtype=tf.int32, name="input_ids")
        input_mask = tf.placeholder(shape=[None, None], dtype=tf.int32, name="input_mask")
        token_type = tf.placeholder(shape=[None, None], dtype=tf.int32, name="segment_ids")

        config = BertConfig.from_json_file(config_path)
        model = BertModel(config=config, is_training=is_training,
                          input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type)

        model.input_to_output()
        seq_output = model.get_all_encoder_layers()[-1]
        pool_output = model.get_pooled_output()

        config_file = tf.constant(value=config_path, dtype=tf.string, name="config_file")
        vocab_file = tf.constant(value=vocab_path, dtype=tf.string, name="vocab_file")
        lower_case = tf.constant(do_lower_case)

        tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, config_file)
        tf.add_to_collection(tf.GraphKeys.ASSET_FILEPATHS, vocab_file)
        
        input_map = {"input_ids": input_ids,
                     "input_mask": input_mask,
                     "segment_ids": token_type}
        
        output_map = {"pooled_output": pool_output,
                      "sequence_output": seq_output}

        output_info_map = {"vocab_file": vocab_file,
                           "do_lower_case": lower_case}
                
        hub.add_signature(name="tokens", inputs=input_map, outputs=output_map)
        hub.add_signature(name="tokenization_info", inputs={}, outputs=output_info_map)

    return bert_module_fn

MODEL_DIR = "004_bert_eojeol_tensorflow"

config_path = "{}/bert_config.json".format(MODEL_DIR)
vocab_path = "{}/vocab.korean.rawtext.list".format(MODEL_DIR)

tags_and_args = []
for is_training in (True, False):
  tags = set()
  if is_training:
    tags.add("train")
  tags_and_args.append((tags, dict(is_training=is_training)))

module_fn = build_module_fn(config_path, vocab_path)
spec = hub.create_module_spec(module_fn, tags_and_args=tags_and_args)
spec.export("bert-module", 
            checkpoint_path="{}/model.ckpt-56000".format(MODEL_DIR))
print('complete')

<file_sep>import os, re
import pandas as pd
import numpy as np
from selenium import webdriver
import time

class WebUtil :
    
    def __init__(self) :
        self.category_dict = {'치킨':5, '피자&양식':6,
                              '중국집':7, '한식':8, '일식&돈까스':9,
                              '족발&보쌈':10, '야식':11,
                              '분식':12, '카페&디저트':13 }
        
    def open_url(self, chromedriver_path, url) :
        self.driver = webdriver.Chrome(chromedriver_path)
        self.driver.maximize_window()
        self.driver.get(url)

    
    def set_location(self, location) :
        print(location +'으로 위치 설정 하는중...')
        self.driver.find_element_by_css_selector('#search > div > form > input').click()
        time.sleep(2)
        self.driver.find_element_by_css_selector('#button_search_address > button.btn-search-location-cancel.btn-search-location.btn.btn-default > span').click()
        time.sleep(2)
        self.driver.find_element_by_css_selector('#search > div > form > input').send_keys(location)
        self.driver.find_element_by_css_selector('#button_search_address > button.btn.btn-default.ico-pick').click()
        time.sleep(2)
        self.driver.find_element_by_css_selector('#search > div > form > ul > li:nth-child(3) > a').click()
        time.sleep(2)
        print(location+'으로 위치 설정 완료!')
        
        
    def go_to_category(self, category) :
        print(category+' 카테고리 페이지 로드중...')
        self.driver.find_element_by_xpath('//*[@id="category"]/ul/li[{}]/span'.format(self.category_dict.get(category))).click()
        time.sleep(2)
        print(category+' 카테고리 페이지 로드 완료!')
        

    def scroll_bottom(self) :
        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")


    def stretch_category_store_page(self) : 
        print("카테고리 가게 펼치는 중...")
        prev_height = self.driver.execute_script("return document.body.scrollHeight")
        
        while True :
            self.scroll_bottom()
            time.sleep(2)
            curr_height = self.driver.execute_script("return document.body.scrollHeight")
            
            if curr_height == prev_height:
                break
        
            prev_height = curr_height
    
        print("카테고리 가게 펼치기 완료!")


    def get_restaurant_list(self) :
        print("가게 리스트 만들기 시작...")
        restaurant_list = []
        restaurants = self.driver.find_elements_by_css_selector('#content > div > div:nth-child(4) > div > div.restaurant-list > div.col-sm-6.contract')
        for restaurant in restaurants:
            review_cnt = restaurant.find_element_by_css_selector('div > table > tbody > tr > td:nth-child(2) > div > div.stars > span:nth-child(2)').text
            try :
                if int(review_cnt.replace('리뷰 ', '')) >= 10 :
                    restaurant_list.append(restaurant.find_element_by_css_selector('div > table > tbody > tr > td:nth-child(2) > div > div.restaurant-name.ng-binding').text)
            except ValueError as e :
                pass        
        self.restaurant_list = list(set(restaurant_list)) 
        print("가게 리스트 만들기 성공!")


    def search_restaurant(self, restaurant_name):
        print("가게 검색중...")
        self.driver.find_element_by_xpath('//*[@id="category"]/ul/li[1]/a').click()
        self.driver.find_element_by_xpath('//*[@id="category"]/ul/li[15]/form/div/input').send_keys(restaurant_name)
        self.driver.find_element_by_xpath('//*[@id="category_search_button"]').click()
        time.sleep(5)
        print("가게 검색 성공!")
        

    def go_to_restaurant(self):
        print("검색한 가게 가는 중...")
        self.driver.find_element_by_xpath('//*[@id="content"]/div/div[5]/div/div/div/div/table/tbody/tr/td[2]/div/div[1]').click()
        time.sleep(5)
        print("검색한 가게 갔음!")


    def go_to_cleanreview(self):
        print("클린리뷰 가는 중")
        self.driver.find_element_by_xpath('//*[@id="content"]/div[2]/div[1]/ul/li[2]/a').click()
        time.sleep(2)
        print("클린리뷰 갔음!")
        

    def click_more_review(self):
        self.driver.find_element_by_class_name('btn-more').click()
        time.sleep(2)


    def stretch_review_page(self):
        review_count = int(self.driver.find_element_by_xpath('//*[@id="content"]/div[2]/div[1]/ul/li[2]/a/span').text)
        #click_count = min(int(review_count/10), 10)
        click_count = int((review_count/10))
        print('모든 리뷰 불러오기 시작...')
        for _ in range(click_count):
            self.scroll_bottom()
            self.click_more_review()
        
        self.scroll_bottom()
        print('모든 리뷰 불러오기 완료!')


    def get_all_review_elements(self):
        print('리뷰 모든 요소 가져오는 중...')
        self.reviews = self.driver.find_elements_by_css_selector('#review > li.list-group-item.star-point.ng-scope')
        print('리뷰 모든 요소 가져옴!')
        
    def go_back_page(self):
        print('페이지 돌아가기중...')
        self.driver.execute_script("window.history.go(-1)")
        time.sleep(5)
        print('페이지 돌아가기 완료!'+'\n')

        
    
    def Crawl_Main(self, gu: str, file_path: str, encoding: str, chromedriver_path: str, url: str) :
        self.open_url(chromedriver_path, url)
        
        dong_list = Location_setting(gu)
        for dong in dong_list :
            location = gu + " " + dong
            self.set_location(location)
            
            for key in self.category_dict :
                self.go_to_category(key)
                self.stretch_category_store_page()
                self.get_restaurant_list()
                
                for restaurant_name in self.restaurant_list :
                    self.search_restaurant(restaurant_name)
                    self.go_to_restaurant()
                    self.go_to_cleanreview()
                    self.stretch_review_page()
                    self.get_all_review_elements()
                    
                    store = Store()
                    self.review_Dataframe = store.make_review_dataframe(self.reviews, self.driver, key, location)
                
                    
                    write_dataframe_to_csv(file_path, restaurant_name, self.review_Dataframe, encoding)
                    
                    self.go_back_page()
                    

                
class Store :
    def __init__(self) :
        self.review_Dataframe = pd.DataFrame(columns=['Category','Location','Restaurant','UserID','Menu','Review',
                                   'Total','Taste','Quantity','Delivery','Date'])
    
     
    def make_review_dataframe(self, reviews, driver, category, location) :
        print("리뷰 데이터프레임 만드는 중...")
        for review in reviews :
            # 가-힣, 공백 이외의 리뷰는 제외
            review_txt = review.find_element_by_css_selector('p').text
            re_review_txt = re.sub('[^가-힣 ]', '', review_txt)            
            
            if len(re_review_txt) >= 12 :
                self.review_Dataframe.loc[len(self.review_Dataframe)] = {
                    'Category': category,
                    'Location': location,
                    'Restaurant': driver.find_element_by_class_name('restaurant-name').text,
                    'UserID': review.find_element_by_css_selector('span.review-id.ng-binding').text,
                    'Menu': review.find_element_by_css_selector('div.order-items.default.ng-binding').text,
                    'Review': re_review_txt,
                    'Total': str(len(review.find_elements_by_css_selector('div > span.total > span.full.ng-scope'))),
                    'Taste': review.find_element_by_css_selector('div:nth-child(2) > div > span.category > span:nth-child(3)').text,
                    'Quantity': review.find_element_by_css_selector('div:nth-child(2) > div > span.category > span:nth-child(6)').text,
                    'Delivery': review.find_element_by_css_selector('div:nth-child(2) > div > span.category > span:nth-child(9)').text,
                    'Date': review.find_element_by_css_selector('div:nth-child(1) > span.review-time.ng-binding').text,
                    }
            
        print("리뷰 데이터프레임 완성!")
        return self.review_Dataframe
                
                
        
        
        
def Location_setting(gu) :
    
    if gu == '송파구' :
        dong_list = ['가락동','거여동','마천동','문정동','방이동','삼전동','석촌동','송파동','오금동','오륜동','잠실동','장지동','풍납동']
        return dong_list
    
    elif gu == '강남구' :
        dong_list = ['논현동','삼성동','청담동','개포동','대치동','신사동','역삼동','압구정동','세곡동','자곡동','율현동','일원동','수서동','도곡동']
        return dong_list
    
    elif gu == '서초구' :
        dong_list = ['방배동','양재동','우면동','원지동','잠원동','반포동','서초동','내곡동','염곡동','신원동']
        return dong_list
        
    
    
def write_dataframe_to_csv(file_path, restaurant_name, review_Dataframe, encoding) :
    print("csv파일 생성중...")
    review_Dataframe.to_csv(f"{file_path}{restaurant_name}.csv", encoding = encoding)
    print("csv파일 생성 완료!")








if __name__ == "__main__" :
    gu = "강남구"
    file_path = "C:/sklee/Academy/미디어젠 프로젝트/Crawl_test/"
    encoding = "UTF-8-sig"
    chromedriver_path = "C:/Users/samsung/Downloads/chromedriver.exe"
    url = "https://www.yogiyo.co.kr/mobile/#/"
    
    Crawl = WebUtil()
    Crawl.Crawl_Main(gu, file_path, encoding, chromedriver_path, url)
    
    
##############################################################################
"""
file_path에는 csv파일이 생성될 폴더 경로
chromedirver_path에는 chromedriver의 위치 경로

Location_setting에는 형식에 맞춰 '구'와 '동'을 기입한 뒤에
gu에 검색할 '구'를 써넣고 실행시키면 됩니다.


리뷰수 10개 이상인 가게만
가-힣과 공백 이외의 텍스트는 제외 

"""













<file_sep>import pandas as pd
from selenium import webdriver

import time

from tqdm import tqdm
from tqdm import trange

# 크롬 드라이버로 크롬 켜기
driver = webdriver.Chrome("./data/chromedriver.exe")

# 창 최소화모드에서는 카테고리가 보이지 않기에 창 최대화
driver.maximize_window()

# 접속할 웹 사이트 주소(요기요 홈페이지)
url = "https://www.yogiyo.co.kr/mobile/#/"
driver.get(url)


# 검색할 주소(나중에 구+동으로 나눠야 함)
location = "화성시 능동"

# 시간을 2초로 줌
interval = 2

# 주소 입력해서 페이지 넘어감
# 입력창 클릭 -> 입력창에 써져있는거 지우기 클릭 -> 로케이션 값 보내기 -> 검색버튼 클릭 -> 첫번째 검색결과 클릭
print(location +'으로 위치 설정 하는중...')
driver.find_element_by_css_selector('#search > div > form > input').click()
driver.find_element_by_css_selector('#button_search_address > button.btn-search-location-cancel.btn-search-location.btn.btn-default > span').click()
driver.find_element_by_css_selector('#search > div > form > input').send_keys(location)
driver.find_element_by_css_selector('#button_search_address > button.btn.btn-default.ico-pick').click()
time.sleep(interval)
driver.find_element_by_css_selector('#search > div > form > ul > li:nth-child(3) > a').click()
time.sleep(interval)
print(location+'으로 위치 설정 완료!')


#요기요 카테고리 페이지의 Element Number Dictionary 정의
#카테고리의 xpath 값이 카테고리 별로 다르기에 숫자를 다르게 넣어주려고 이렇게 정의한 것
food_dict = { '치킨':5, '피자&양식':6,
              '중국집':7, '한식':8, '일식&돈까스':9,
              '족발&보쌈':10, '야식':11,
              '분식':12, '카페&디저트':13 }


for category in food_dict:
    print(category+' 카테고리 페이지 로드중...')
    # food_dict.get(category) -> category라는 key에 대응되는 value를 리턴
    # .format() 이건 문자열 포매팅을 한 것이다.
    # food_dict.get() 이건 괄호 안에 들어가는 key에 대한 value를 리턴하는 것이다.
    # 그리고 문자열 안의 {} 안에 그 value(여기선 숫자)가 들어가게 되는 것이다.
    driver.find_element_by_xpath('//*[@id="category"]/ul/li[{}]/span'.format(food_dict.get(category))).click()
    time.sleep(interval)
    print(category+' 카테고리 페이지 로드 완료!')


    # 현재 문서 높이를 가져와서 저장
    prev_height = driver.execute_script("return document.body.scrollHeight")

    # 반복 수행
    while True:
        # 스크롤을 가장 아래로 내림
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
    
    
        # 페이지 로딩 대기
        time.sleep(interval)


        # 현재 문서 높이를 가져와서 저장
        curr_height = driver.execute_script("return document.body.scrollHeight")
    
        # 전에 저장한 문서 높이와 현재 저장한 문서 높이가 같으면 while문 빠져나감
        if curr_height == prev_height:
            break

        prev_height = curr_height

    print("스크롤 완료")

    # 리스트 객체 만들고
    restaurant_list=[]

    # 우리동네플러스나 슈퍼레드위크추천이 아닌 요기요등록음식점 부분으로 설정하는 것
    restaurants = driver.find_elements_by_css_selector('#content > div > div:nth-child(4) > div > div.restaurant-list > div.col-sm-6.contract')
    
    #  restaurants에 저장된 elements에서 loop를 돌리며 (각각의 가게들을 순회하며)
    for restaurant in restaurants:
        restaurant_list.append(restaurant.find_element_by_css_selector('div > table > tbody > tr > td:nth-child(2) > div > div.restaurant-name.ng-binding').text)
        
    # 일단 실험해보는거니까 리스트에서 첫번째것만 빼서 전달
    for restaurant_name in restaurant_list:
        # 주소 설정창 말고 음식점 검색 버튼 클릭
        driver.find_element_by_xpath('//*[@id="category"]/ul/li[1]/a').click()

        # 음식점 검색 입력창에 가게 이름 전달
        driver.find_element_by_xpath('//*[@id="category"]/ul/li[15]/form/div/input').send_keys(restaurant_name)

        # 음식점 검색 버튼 클릭
        driver.find_element_by_xpath('//*[@id="category_search_button"]').click()

        time.sleep(interval)

        # 첫번째 나오는 음식점 클릭
        driver.find_element_by_css_selector('#content > div > div:nth-child(5) > div > div > div:nth-child(1) > div').click()
        
        time.sleep(interval)

        # 클린리뷰 클릭
        driver.find_element_by_xpath('//*[@id="content"]/div[2]/div[1]/ul/li[2]/a').click()

        time.sleep(interval)

        # 클린리뷰 옆에 있는 리뷰 숫자가 써져있는데 그걸 보여주는 요소의 text를 가져와서 정수형으로 변환
        review_count = int(driver.find_element_by_xpath('//*[@id="content"]/div[2]/div[1]/ul/li[2]/a/span').text)

        # 요기요 리뷰는 더보기를 눌렀을 때 10개를 로드하니까 몇번 더보기를 누를지 정해야하니까 10으로 나누어줌
        click_count = int((review_count/10))

        # 리뷰 페이지를 모두 펼치는 for문 시작
        print('모든 리뷰 불러오는 중...')
        for _ in trange(click_count):
            try:
                # 스크롤을 가장 아래로 내림
                driver.execute_script("window.scrollTo(0,document.body.scrollHeight);")
                
                # 더보기 클릭
                driver.find_element_by_class_name('btn-more').click()
        
                # 2초 쉼
                time.sleep(2)
        
            except Exception as e:
                print(e)
                print('페이지 돌아가기중...')
                driver.execute_script("window.history.go(-1)")
                time.sleep(interval)
                print('페이지 돌아가기 완료!\n')
                continue
    
        # 스크롤을 가장 아래로 내림
        driver.execute_script("window.scrollTo(0,document.body.scrollHeight);")
        print('모든 리뷰 불러오기 완료!')

        ### 리뷰 크롤링

        # 리뷰 가져와서 revies객체에 할당
        reviews = driver.find_elements_by_css_selector('#review > li.list-group-item.star-point.ng-scope')

        # 데이터프레임 객체 생성 및 열에 이름 부여
        print('리뷰 저장하는 중...')
        df = pd.DataFrame(columns=['Category','Location','Restaurant','UserID','Menu','Review','Date','Total','Taste','Quantity','Delivery'])
        
        for review in tqdm(reviews):
            try:
                df.loc[len(df)] = {
                    'Category':category,
                    'Location':location,
                    'Restaurant':driver.find_element_by_class_name('restaurant-name').text,
                    'UserID':review.find_element_by_css_selector('span.review-id.ng-binding').text,
                    'Menu':review.find_element_by_css_selector('div.order-items.default.ng-binding').text,
                    'Review':review.find_element_by_css_selector('p').text,
                    'Date':review.find_element_by_css_selector('div:nth-child(1) > span.review-time.ng-binding').text,
                    'Total':str(len(review.find_elements_by_css_selector('div > span.total > span.full.ng-scope'))),
                    'Taste':review.find_element_by_css_selector('div:nth-child(2) > div > span.category > span:nth-child(3)').text,
                    'Quantity':review.find_element_by_css_selector('div:nth-child(2) > div > span.category > span:nth-child(6)').text,
                    'Delivery':review.find_element_by_css_selector('div:nth-child(2) > div > span.category > span:nth-child(9)').text,
                    }
                
            except Exception as e:
                print('리뷰 페이지 에러')
                print(e)
                print('페이지 돌아가기중...')
                driver.execute_script("window.history.go(-1)")
                time.sleep(interval)
                print('페이지 돌아가기 완료!\n')
                continue
        try:
            df.to_csv('./data/yogiyo/' + df.Restaurant[0] + '.csv')
            print('리뷰 저장하는 완료!')
        except:
            pass
        
        print('페이지 돌아가기중...')
        driver.execute_script("window.history.go(-1)")
        time.sleep(interval)
        print('페이지 돌아가기 완료!\n')
<file_sep># -*- coding: utf-8 -*-

from readers.data_reader import Reader
from vectorizers.bert_vectorizer import BERTVectorizer
from vectorizers import tokenizationK as tk
from models.bert_nlu_model import BertNLUModel
from utils import flatten

import argparse
import os
import pickle
import tensorflow as tf
from sklearn import metrics


# read command-line parameters
parser = argparse.ArgumentParser('Evaluating the BERT / ALBERT NLU model')
parser.add_argument('--model', '-m', help = 'Path to BERT / ALBERT NLU model', type = str, required = True)
parser.add_argument('--data', '-d', help = 'Path to data', type = str, required = True)
parser.add_argument('--type', '-tp', help = 'bert   or    albert', type = str, default = 'bert', required = False)


VALID_TYPES = ['bert', 'albert']

args = parser.parse_args()
load_folder_path = args.model
data_folder_path = args.data
type_ = args.type

# this line is to disable gpu
os.environ['CUDA_VISIBLE_DEVICES']='-1'

config = tf.ConfigProto(intra_op_parallelism_threads=8, 
                        inter_op_parallelism_threads=0,
                        allow_soft_placement=True,
                        device_count = {'CPU': 8})
sess = tf.Session(config=config)

if type_ == 'bert':
    bert_model_hub_path = './bert-module'
    is_bert = True
elif type_ == 'albert':
    bert_model_hub_path = 'https://tfhub.dev/google/albert_base/1'
    is_bert = False
else:
    raise ValueError('type must be one of these values: %s' % str(VALID_TYPES))
    
bert_vectorizer = BERTVectorizer(is_bert, bert_model_hub_path)

# loading models
print('Loading models ...')
if not os.path.exists(load_folder_path):
    print('Folder `%s` not exist' % load_folder_path)

with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'), 'rb') as handle:
    intents_label_encoder = pickle.load(handle)
    intents_num = len(intents_label_encoder.classes_)
    
model = BertNLUModel.load(load_folder_path, sess)

tokenizer = tk.FullTokenizer('./bert-module/assets/vocab.korean.rawtext.list')

data_text_arr, data_intents = Reader.read(data_folder_path)
data_input_ids, data_input_mask, data_segment_ids = bert_vectorizer.transform(data_text_arr)

def get_results(input_ids, input_mask, segment_ids, intents, intents_label_encoder):
    first_inferred_intent, first_inferred_intent_score, _, _ = model.predict_intent([data_input_ids, data_input_mask, data_segment_ids], intents_label_encoder)
    acc = metrics.accuracy_score(intents, first_inferred_intent)

    intent_incorrect = ''
    for i, sent in enumerate(input_ids):

        if intents[i] != first_inferred_intent[i]:
            tokens = tokenizer.convert_ids_to_tokens(input_ids[i])
            intent_incorrect += ('sent {}\n'.format(tokens))
            intent_incorrect += ('pred: {}\n'.format(first_inferred_intent[i].strip()))
            intent_incorrect += ('score: {}\n'.format(first_inferred_intent_score[i]))
            intent_incorrect += ('ansr: {}\n'.format(intents[i].strip()))
            intent_incorrect += '\n'

    return acc, intent_incorrect

print('==== Evaluation ====')
acc, intent_incorrect = get_results(data_input_ids, data_input_mask, data_segment_ids,
                                                            data_intents, intents_label_encoder)

# 테스트 결과를 모델 디렉토리의 하위 디렉토리 'test_results'에 저장해 준다.
result_path = os.path.join(load_folder_path, 'test_results')

if not os.path.isdir(result_path):
    os.mkdir(result_path)

with open(os.path.join(result_path, 'intent_incorrect.txt'), 'w') as f:
    f.write(intent_incorrect)

with open(os.path.join(result_path, 'test_total.txt'), 'w') as f:
    f.write('Intent accuracy = {}\n'.format(acc))

tf.compat.v1.reset_default_graph()
<file_sep># -*- coding: utf-8 -*-

import tensorflow as tf
import os
from tensorflow.keras import backend as K
import tensorflow_hub as hub
import json

class KorBertLayer(tf.keras.layers.Layer):
    def __init__(self, bert_path="./bert-module", n_tune_layers=10, 
                 pooling="cls", tune_embeddings=True, trainable=True, **kwargs):

        self.trainable = trainable
        self.n_tune_layers = n_tune_layers
        self.tune_embeddings = tune_embeddings
        self.pooling = pooling
        self.bert_path = bert_path
        self.output_size = 768

        self.var_per_encoder = 16
        if self.pooling not in ["cls", "mean", None]:
            raise NameError(
                f"Undefined pooling type (must be either 'cls', 'mean', or None, but is {self.pooling}"
            )

        super(KorBertLayer, self).__init__(**kwargs)
        print('init ok')

    def build(self, input_shape):

        self.bert = hub.Module(self.build_abspath(self.bert_path), 
                               trainable=self.trainable, name=f"{self.name}_module")

        # Remove unused layers
        trainable_vars = self.bert.variables
        if self.pooling == 'cls':
            trainable_vars = [var for var in trainable_vars if not "/cls/" in var.name]
            trainable_layers = ["pooler/dense"]

        elif self.pooling == 'mean':
            trainable_vars = [var for var in trainable_vars
                if not "/cls/" in var.name and not "/pooler/" in var.name
            ]
            trainable_layers = []
        else:
            raise NameError(
                f"Undefined pooling type (must be either first or mean, but is {self.pooling}"
            )

        # append word_embedding to trainable_layers
        trainable_layers.append('bert/embeddings/word_embeddings:0')

        # Select how many layers to fine tune
        for i in range(self.n_tune_layers):
            trainable_layers.append(f"encoder/layer_{str(11 - i)}")

        # Update trainable vars to contain only the specified layers
        trainable_vars = [
            var
            for var in trainable_vars
            if any([l in var.name for l in trainable_layers])
        ]

        # Add to trainable weights
        for var in trainable_vars:
            self._trainable_weights.append(var)

        for var in self.bert.variables:
            if var not in self._trainable_weights:
                self._non_trainable_weights.append(var)

        super(KorBertLayer, self).build(input_shape)



    def build_abspath(self, path):
        if path.startswith("https://") or path.startswith("gs://"):
          return path
        else:
          return os.path.abspath(path)

    def call(self, inputs):
        inputs = [K.cast(x, dtype="int32") for x in inputs]
        input_ids, input_mask, segment_ids= inputs
        bert_inputs = dict(
            input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids
        )
        result = self.bert(inputs=bert_inputs, signature='tokens', as_dict=True)

        print('call ok')
        return result['pooled_output']
    
    def compute_output_shape(self, input_shape):
        return (input_shape[0], input_shape[1], self.output_size)


    def get_config(self):
        config = super().get_config().copy()
        config.update({
            'n_tune_layers': self.n_tune_layers,
            'trainable': self.trainable,
            'output_size': self.output_size,
            'pooling': self.pooling,
            'bert_path': self.bert_path,
        })
        return config
<file_sep># -*- coding: utf-8 -*-

import tensorflow as tf
import numpy as np
import sys
from vectorizers import tokenizationK as tk

class BERTVectorizer:
    
    def __init__(self, is_bert,
                 bert_model_hub_path="./bert-module"):

        print('initializing!')
        self.is_bert = is_bert
        print('is_bert :', self.is_bert)
        self.bert_model_hub_path = bert_model_hub_path
        print('bert_model_hub_path :', self.bert_model_hub_path)
        self.tokenizer = tk.FullTokenizer('./bert-module/assets/vocab.korean.rawtext.list')
        print('initialized!')

    
    def transform(self, text_arr):
        print('transform started')

        input_ids = []
        input_mask = []
        segment_ids = []

        # print('text arr :', text_arr)
        for text in text_arr: 

            ids, mask, seg_ids= self.__vectorize(text.strip())

            # print('text :', text)
            # print('ids :', ids)
            # print('mask :', mask)
            # print('seg_ids :', seg_ids)
            # print('valid pos :', valid_pos)

            input_ids.append(ids)
            input_mask.append(mask)
            segment_ids.append(seg_ids)

        input_ids = tf.keras.preprocessing.sequence.pad_sequences(input_ids, padding='post')
        input_mask = tf.keras.preprocessing.sequence.pad_sequences(input_mask, padding='post')
        segment_ids = tf.keras.preprocessing.sequence.pad_sequences(segment_ids, padding='post')
        return input_ids, input_mask, segment_ids
    
    
    def __vectorize(self, text: str):

        # str -> tokens list
        tokens = text.split() # whitespace tokenizer


        # insert "[CLS]"
        tokens.insert(0, '[CLS]')
        # insert "[SEP]"
        tokens.append('[SEP]')
        
        segment_ids = [0] * len(tokens)
        input_ids = self.tokenizer.convert_tokens_to_ids(tokens)

        input_mask = [1] * len(input_ids)
        
        return input_ids, input_mask, segment_ids
        
<file_sep># flask_app/server.py​
import datetime
import re

from flask_ngrok import run_with_ngrok 
from flask import Flask, request, jsonify, render_template, session, url_for, redirect
from flask_dropzone import Dropzone
import time
from urllib.parse import unquote
import os
import uuid
import secrets

from run_intent_classify import sentiment_predict, csv_predict
from settings import *
import requests
import pdb

from input_module import read_file

app = Flask(__name__)

# colab에서의 서빙을 위해 flask_ngrok 모듈의 run_with_ngrok 함수를 사용합니다.
run_with_ngrok(app)

UPLOAD_FOLDER = os.path.join(app.root_path,'upload_files')
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
app.config['DROPZONE_UPLOAD_MULTIPLE'] = True
app.config['DROPZONE_ALLOWED_FILE_TYPE'] = 'text'
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config.update(
    SECRET_KEY=secrets.token_urlsafe(32),
    SESSION_COOKIE_NAME='InteractiveTransformer-WebSession'
)

dropzone = Dropzone(app)

def delay_func(func):
    def inner(*args, **kwargs):
        returned_value = func(*args, **kwargs)
        time.sleep(0)
        return returned_value
    return inner

# 실행 시작
@app.route("/")
def index():
    # 처음 보여지는 화면입니다. 
    return render_template("index.html")

@app.route('/_evaluate_helper')
def evaluate_helper():
    eval_text = request.args.get("evaluate_data", "", type=str).strip()
    eval_text = re.sub("%20", "", eval_text)
    prediction = sentiment_predict(eval_text)
    
    if prediction and eval_text:
        return jsonify(result=render_template('live_eval.html', predict=[prediction[0]], eval_text=[eval_text], intent=[prediction[1]]))
    return jsonify(result="")

@app.route('/_evaluate_helper_file', methods=['POST'])
def evaluate_helper_file():
    if 'file' not in request.files:
        print('No file part')
    else:
        file = request.files.get('file')
        print(file)
    input_df = read_file(file)
    prediction = csv_predict(input_df)
      
    return jsonify(result=render_template("result_file.html", predict=[prediction]))

if __name__ == '__main__':
    #app.run(host="0.0.0.0", debug=True, port=PORT)
    app.run()
<file_sep># -*- coding: utf-8 -*-

from readers.data_reader import Reader
from vectorizers.bert_vectorizer import BERTVectorizer
from models.bert_nlu_model import BertNLUModel

import argparse
from sklearn.preprocessing import LabelEncoder
import numpy as np
import os
import pickle
import tensorflow as tf
from tensorflow import keras


# read command-line parameters
parser = argparse.ArgumentParser('Training the BERT NLU model')
parser.add_argument('--train', '-t', help = 'Path to training data', type = str, required = True)
parser.add_argument('--val', '-v', help = 'Path to validation data', type = str, default = "", required = False)
parser.add_argument('--save', '-s', help = 'Folder path to save the trained model', type = str, required = True)
parser.add_argument('--epochs', '-e', help = 'Number of epochs', type = int, default = 5, required = False)
parser.add_argument('--batch', '-bs', help = 'Batch size', type = int, default = 64, required = False)
parser.add_argument('--type', '-tp', help = 'bert   or    albert', type = str, default = 'bert', required = False)
parser.add_argument('--model', '-mo', help = 'load model', type = str, default = "", required = False)


VALID_TYPES = ['bert', 'albert']

args = parser.parse_args()
train_data_folder_path = args.train
val_data_folder_path = args.val
save_folder_path = args.save
epochs = args.epochs
batch_size = args.batch
type_ = args.type
model_path = args.model

# # this line is to disable gpu
# os.environ["CUDA_VISIBLE_DEVICES"]="-1"

tf.compat.v1.random.set_random_seed(7)

config = tf.ConfigProto(intra_op_parallelism_threads=8, 
                        inter_op_parallelism_threads=0,
                        allow_soft_placement=True,
                        device_count = {'CPU': 8})
sess = tf.compat.v1.Session(config=config)

if type_ == 'bert':
    bert_model_hub_path = "./bert-module" # to use KorBert by Etri
    is_bert = True
elif type_ == 'albert':
    bert_model_hub_path = 'https://tfhub.dev/google/albert_base/1'
    is_bert = False
else:
    raise ValueError('type must be one of these values: %s' % str(VALID_TYPES))

#이 부분을 고쳐서 Database를 이용해서 데이터를 읽어올 수 있음
print('read data ...')
train_text_arr, train_intents = Reader.read(train_data_folder_path)

print('train_text_arr[0:2] :', train_text_arr[0:2])
print('train_intents[0:2] :', train_intents[0:2])

print('vectorize data ...')
bert_vectorizer = BERTVectorizer(is_bert, bert_model_hub_path) 
# now bert model hub path exists --> already tokenized dataset
# bert vectorizer MUST NOT tokenize input !!!

print('bert vectorizer started ...') #valid pos removed
train_input_ids, train_input_mask, train_segment_ids = bert_vectorizer.transform(train_text_arr)


print('encode labels ...')
intents_label_encoder = LabelEncoder()
train_intents = intents_label_encoder.fit_transform(train_intents).astype(np.int32)
intents_num = len(intents_label_encoder.classes_)
print('intents num :', intents_num)

if model_path == "":
    model = BertNLUModel(intents_num, bert_model_hub_path, sess, num_bert_fine_tune_layers=10, is_bert=is_bert)
else:
    model = BertNLUModel.load(load_folder_path, sess)

print('train input shape :', train_input_ids.shape, train_input_ids[0:2])
print('train_input_mask :', train_input_mask.shape, train_input_mask[0:2])
print('train_segment_ids :', train_segment_ids.shape, train_segment_ids[0:2])
print('train_intents :', train_intents.shape, train_intents[0:2])

if not os.path.exists(save_folder_path):
    os.makedirs(save_folder_path)
    print('Folder `%s` created' % save_folder_path)

checkpoint_cb = keras.callbacks.ModelCheckpoint(save_folder_path+"/my_yogiyo_model.h5", save_best_only=True)
early_stopping_cb = keras.callbacks.EarlyStopping(patience=5, restore_best_weights=True)

if val_data_folder_path:
    print('preparing validation data')
    val_text_arr, val_intents = Reader.read(val_data_folder_path)
    val_input_ids, val_input_mask, val_segment_ids = bert_vectorizer.transform(val_text_arr)
    val_intents = intents_label_encoder.transform(val_intents).astype(np.int32)
    
    print('training model ...')
    model.fit([train_input_ids, train_input_mask, train_segment_ids], train_intents,
        validation_data=([val_input_ids, val_input_mask, val_segment_ids], val_intents),
        epochs=epochs, batch_size=batch_size, callbacks=[checkpoint_cb, early_stopping_cb])    
else:
    print('training model ...')
    model.fit([train_input_ids, train_input_mask, train_segment_ids], train_intents,
        validation_data=None, 
        epochs=epochs, batch_size=batch_size, callbacks=[checkpoint_cb, early_stopping_cb])    

### saving
print('Saving ..')
model.save(save_folder_path)
with open(os.path.join(save_folder_path, 'intents_label_encoder.pkl'), 'wb') as handle:
    pickle.dump(intents_label_encoder, handle, protocol=pickle.HIGHEST_PROTOCOL)


tf.compat.v1.reset_default_graph()
<file_sep>Flask==1.1.1
tensorflow_gpu==1.15
tensorflow_hub==0.7.0
flask_dropzone
flask-ngrok
konlpy
h5py==2.10.0
<file_sep>import pandas as pd

def read_file(file):
  input_df = pd.read_csv(file, header=None)
  return input_df<file_sep># -*- coding: utf-8 -*-

from vectorizers.bert_vectorizer import BERTVectorizer
from vectorizers import tokenizationK
from models.bert_nlu_model import BertNLUModel
from utils import flatten

import argparse
import os
import pickle
import numpy as np
import tensorflow as tf
import math


def sentiment_predict(input_sentence):
    # read command-line parameters
    load_folder_path = "yogiyo_model"
    type_ = "bert"

    # this line is to disable gpu
    #os.environ["CUDA_VISIBLE_DEVICES"]="-1"

    config = tf.ConfigProto(intra_op_parallelism_threads=1,
                            inter_op_parallelism_threads=1,
                            allow_soft_placement=True,)
    sess = tf.compat.v1.Session(config=config)

    bert_model_hub_path = './bert-module'
    is_bert = True

    bert_vectorizer = BERTVectorizer(is_bert, bert_model_hub_path)

    with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'), 'rb') as handle:
        intents_label_encoder = pickle.load(handle)
        intents_num = len(intents_label_encoder.classes_)

    model = BertNLUModel.load(load_folder_path, sess)

    vocab_file = "./bert-module/assets/vocab.korean.rawtext.list"
    tokenizer = tokenizationK.FullTokenizer(vocab_file=vocab_file)

    tokens = tokenizer.tokenize(input_sentence)
    tokens = [' '.join(tokens)]

    data_input_ids, data_input_mask, data_segment_ids = bert_vectorizer.transform(tokens)
    
    first_inferred_intent, first_inferred_intent_score, _, _ = model.predict_intent([data_input_ids, data_input_mask, data_segment_ids], intents_label_encoder)

    intent = str(first_inferred_intent[0])
    score = float(first_inferred_intent_score[0]) * 100

    tf.compat.v1.reset_default_graph()

    string = ''
    rating = int(float(intent))

    for i in range(1, 6):
        if i > rating:
            string +='☆'
        else:
            string += '★'

    answer_text = "{:.2f}% 확률로 [{} : {}점] 입니다.".format(score, string, intent)

    return answer_text, intent

def csv_predict(input_df):
    # read command-line parameters
    load_folder_path = "yogiyo_model"
    type_ = "bert"

    # this line is to disable gpu
    os.environ["CUDA_VISIBLE_DEVICES"]="-1"

    config = tf.ConfigProto(intra_op_parallelism_threads=1,
                            inter_op_parallelism_threads=1,
                            allow_soft_placement=True,)
    sess = tf.compat.v1.Session(config=config)

    bert_model_hub_path = './bert-module'
    is_bert = True

    bert_vectorizer = BERTVectorizer(is_bert, bert_model_hub_path)

    with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'), 'rb') as handle:
        intents_label_encoder = pickle.load(handle)
        intents_num = len(intents_label_encoder.classes_)

    model = BertNLUModel.load(load_folder_path, sess)

    vocab_file = "./bert-module/assets/vocab.korean.rawtext.list"
    tokenizer = tokenizationK.FullTokenizer(vocab_file=vocab_file)
    score_list = []
    for i,input_sentence in input_df[0].items():
        print(f'{i}번째 데이터:{input_sentence}')
        tokens = tokenizer.tokenize(input_sentence)
        tokens = [' '.join(tokens)]

        data_input_ids, data_input_mask, data_segment_ids = bert_vectorizer.transform(tokens)
    
        first_inferred_intent, first_inferred_intent_score, _, _ = model.predict_intent([data_input_ids, data_input_mask, data_segment_ids], intents_label_encoder)

        # intent = str(first_inferred_intent[0])
      
        intent = float(str(first_inferred_intent[0]))

        score_list.append(intent)
        score = float(first_inferred_intent_score[0]) * 100

    mean_score = np.mean(score_list)
    tf.compat.v1.reset_default_graph()

    # answer_text = "{:.2f}% 확률로 {}점입니다.".format(score, intent)
    print('평점의 평균:',mean_score)
    return round(mean_score, 1), math.floor(mean_score)

if __name__ == "__main__":
    print(sentiment_predict("최악입니다."))

<file_sep># -*- coding: utf-8 -*-


import os, re
import argparse
from vectorizers.tokenizationK import FullTokenizer

import pdb

# 상황에 따라서 다른 tokenizer를 사용해도 된다.
tokenizer = FullTokenizer(vocab_file="./bert-module/assets/vocab.korean.rawtext.list")


multi_spaces = re.compile(r"\s+")

def process_file(file_path, output_dir):
    """
    단방향 데이터가 있는 file_path을 argument로 주면 가공을 한 이후에
    output_dir 아래에 2개의 파일(seq.in, label)을 저장해 주는 함수.
    output_dir의 경우 만약 존재하지 않는다면 
    """
    if not os.path.isdir(output_dir):
        os.mkdir(output_dir)

    data = open(file_path).read().splitlines()

    # line별로 process를 해준 뒤,
    processed_data = [process_line(line, tokenizer) for line in data]

    intentions = list(map(lambda x: x[0], processed_data))
    tokens = list(map(lambda x: x[1], processed_data))

    # seq_in : 토큰들로만 이루어진 파일
    intention_file = os.path.join(output_dir, "label")
    seq_in = os.path.join(output_dir, "seq.in")

    with open(intention_file, "w") as f:
        f.write("\n".join(intentions) + "\n")

    with open(seq_in, "w") as f:
        f.write("\n".join(tokens)+ "\n")


def process_line(line, tokenizer):
    """
    데이터를 라인별로 처리해 주는 함수이다.

    """
    intention, sentence = line.split("\t")
    tokens = ""

    for word in sentence.split():
        word_tokens = " ".join(tokenizer.tokenize(word))
        tokens += word_tokens + " "

    tokens = multi_spaces.sub(" ", tokens.strip())

    return intention, tokens


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--input', '-i', help = '단방향 데이터 형식의 텍스트 파일', type = str, required = True)
    parser.add_argument('--output', '-o', help = 'Path to data', type = str, required = True)

    args = parser.parse_args()
    file_path = args.input
    output_dir = args.output

    process_file(file_path, output_dir)
<file_sep># -*- coding: utf-8 -*-

import tensorflow as tf
from tensorflow.keras import backend as K
import tensorflow_hub as hub


class AlbertLayer(tf.keras.layers.Layer):
    
    def __init__(self, fine_tune=True, pooling='first',
        albert_path="https://tfhub.dev/google/albert_base/1",
        **kwargs,):
        self.fine_tune = fine_tune
        self.output_size = 768
        self.pooling = pooling
        self.albert_path = albert_path
        if self.pooling not in ['first', 'mean']:
            raise NameError(
                f"Undefined pooling type (must be either first or mean, but is {self.pooling}"
            )

        super(AlbertLayer, self).__init__(**kwargs)
        

    def build(self, input_shape):
        self.albert = hub.Module(
            self.albert_path, trainable=self.fine_tune, name=f"{self.name}_module"
        )
        
        if self.fine_tune:
            # Remove unused layers
            trainable_vars = self.albert.variables
            if self.pooling == 'first':
                trainable_vars = [var for var in trainable_vars if not "/cls/" in var.name]
                trainable_layers = ["pooler/dense"]
    
            elif self.pooling == 'mean':
                trainable_vars = [var for var in trainable_vars
                    if not "/cls/" in var.name and not "/pooler/" in var.name
                ]
                trainable_layers = []
            else:
                raise NameError(
                    f"Undefined pooling type (must be either first or mean, but is {self.pooling}"
                )
    
            # Select how many layers to fine tune
            trainable_layers.append("encoder/transformer/group_0")
    
            # Update trainable vars to contain only the specified layers
            trainable_vars = [
                var
                for var in trainable_vars
                if any([l in var.name for l in trainable_layers])
            ]
    
            # Add to trainable weights
            for var in trainable_vars:
                self._trainable_weights.append(var)
    
            for var in self.albert.variables:
                if var not in self._trainable_weights:
                    self._non_trainable_weights.append(var)
        
        super(AlbertLayer, self).build(input_shape)

    
    def call(self, inputs):
        inputs = [K.cast(x, dtype="int32") for x in inputs]
        input_ids, input_mask, segment_ids, valid_positions = inputs
        bert_inputs = dict(
            input_ids=input_ids, input_mask=input_mask, segment_ids=segment_ids
        )
        result = self.albert(inputs=bert_inputs, signature='tokens', as_dict=True)
        return result['pooled_output']
    

    def compute_output_shape(self, input_shape):
        return (input_shape[0], input_shape[1], self.output_size)
    
    
    def get_config(self):
        config = super().get_config().copy()
        config.update({
            'fine_tune': self.fine_tune,
            'output_size': self.output_size,
            'pooling': self.pooling,
            'albert_path': self.albert_path,
        })
        return config
<file_sep>from readers.data_reader import Reader
from vectorizers import tokenizationK as tk

import argparse
import os
import pdb

# read command-line parameters
parser = argparse.ArgumentParser('Get the vectors from tokens')
parser.add_argument('--data', '-d', help = 'Path to data', type = str, required = True)
parser.add_argument('--tsv', '-t', help = 'Path to {meta,vecs}.tsv', type = str, required = True)

args = parser.parse_args()
data_folder_path = args.data
tsv_folder_path  = args.tsv

# read tokenized data from ${data_folder_path}
with open(os.path.join(data_folder_path, 'seq.in'), encoding='utf-8') as f:
    text_arr = f.read().splitlines()

# read tsvs from ${tsv_folder_path}
with open(os.path.join(tsv_folder_path, 'vecs.tsv'), encoding='utf-8') as f:
    vecs = f.read().splitlines()

with open(os.path.join(tsv_folder_path, 'meta.tsv'), encoding='utf-8') as f:
    meta = f.read().splitlines()

# tokenizer
tokenizer = tk.FullTokenizer('./bert-module/assets/vocab.korean.rawtext.list')

# get all token ids that is used in the data
print('collecting token_ids from tokenized data...')
token_id_set = set()
for text in text_arr:
    token_id_set.update(tokenizer.convert_tokens_to_ids(text.split()))

# token_ids = sorted(map(lambda x: x+1, token_id_set)) # add 1 since id starts from 0..?
token_ids = sorted(token_id_set)

# get vecs and meta from 
vecs_appeared = [vecs[i] for i in token_ids]
meta_appeared = [meta[i] for i in token_ids]

# save 
print('saving vecs and meta that appeared in the dataset')
with open(os.path.join(data_folder_path, 'vecs_appeared.tsv'), 'w') as f:
    f.write('\n'.join(vecs_appeared))

with open(os.path.join(data_folder_path, 'meta_appeared.tsv'), 'w') as f:
    f.write('\n'.join(meta_appeared))
<file_sep># -*- coding: utf-8 -*-

from vectorizers.bert_vectorizer import BERTVectorizer
from vectorizers import tokenizationK
from models.bert_nlu_model import BertNLUModel
from utils import flatten

import argparse
import os
import pickle
import tensorflow as tf

import pdb

# read command-line parameters
parser = argparse.ArgumentParser('Evaluating the Joint BERT / ALBERT NLU model')
parser.add_argument('--model', '-m', help = 'Path to BERT / ALBERT NLU model', type = str, required = True)
parser.add_argument('--type', '-tp', help = 'bert   or    albert', type = str, default = 'bert', required = False)


VALID_TYPES = ['bert', 'albert']

args = parser.parse_args()
load_folder_path = args.model
type_ = args.type

# this line is to disable gpu
os.environ["CUDA_VISIBLE_DEVICES"]="-1"

config = tf.ConfigProto(intra_op_parallelism_threads=1,
                        inter_op_parallelism_threads=1,
                        allow_soft_placement=True,
                        device_count = {'CPU': 1})
sess = tf.compat.v1.Session(config=config)

if type_ == 'bert':
    bert_model_hub_path = './bert-module'
    is_bert = True
elif type_ == 'albert':
    bert_model_hub_path = 'https://tfhub.dev/google/albert_base/1'
    is_bert = False
else:
    raise ValueError('type must be one of these values: %s' % str(VALID_TYPES))


bert_vectorizer = BERTVectorizer(is_bert, bert_model_hub_path)

# loading models
print('Loading models ...')
if not os.path.exists(load_folder_path):
    print('Folder `%s` not exist' % load_folder_path)

with open(os.path.join(load_folder_path, 'intents_label_encoder.pkl'), 'rb') as handle:
    intents_label_encoder = pickle.load(handle)
    intents_num = len(intents_label_encoder.classes_)


model = BertNLUModel.load(load_folder_path, sess)

vocab_file = "./bert-module/assets/vocab.korean.rawtext.list"
tokenizer = tokenizationK.FullTokenizer(vocab_file=vocab_file)

kobert_layer = model.model.layers[3]
# token_embedding = kobert_layer.weights[0]
token_embedding = kobert_layer.weights[0].eval(session=sess)
# pdb.set_trace()

out_m_path = os.path.join(load_folder_path, 'meta.tsv')
out_v_path = os.path.join(load_folder_path, 'vecs.tsv')

# make out_m(meta.tsv) first
print(f'creating {out_m_path} out of {vocab_file}')
os.system(f"sed 1,2d {vocab_file} | cut -f 1 > {out_m_path}")

out_v = open(out_v_path, 'w', encoding='utf-8')

print(f'creating {out_v_path}')
for token_emb in token_embedding:
    # pdb.set_trace()
    out_v.write('\t'.join([str(x) for x in token_emb]) + "\n")
<file_sep># koBERT-yogiyo-reviews-classfication

## 1. 프로젝트 기획
 - 현 별점 시스템의 문제점 의식
    - 리뷰 내용-별점 불일치가 많아 별점만으로는 식당에 대한 긍정/부정 판단이 어려움
    - 쿠폰 및 리뷰 이벤트로 인한 이벤트성 리뷰와 의미 없는 댓글로 매장에 대한 정확한 평가가 어려움
 - 리뷰 데이터에 대한 평점을 KorBERT 모델을 이용하여 계산하는 시스템
 
## 2. ETRI koBERT(pre trained)
 - https://aiopen.etri.re.kr/service_dataset.php 에서 협약서 작성 및 tensorflow 버전 다운로드

## 3. 데이터 수집 및 전처리
 - pre processing
   - Merge_Regex: 크롤링된 파일 다루기
   - Make_Sample: 재라벨링한 파일 다루기

 - 카테고리별 식당 리뷰 무작위 수집 (Selenium을 사용하여 리뷰 크롤링 진행)
 
![KakaoTalk_20210823_170740087](https://user-images.githubusercontent.com/8359931/130415887-f93f4949-19bf-43df-b39b-e86277d07ebe.gif)
 
 - 수집된 데이터에 대해 한글, 공백 제외한 모든 숫자/특수문자/모음과 자음만 있는 글자 제거
 - 맞춤법 교정
 - 리뷰 최소 길이 : 12음절
 - 성능 개선을 위해 문장당 긍부정 비율에 기반한 기준 수립후 재라벨링
 - 수집된 리뷰 코멘트를 30797 vocabs로 토큰화
 - KorBERT 모델을 파인튜닝 할 수 있게 label과 input sequence로 나누어 줌 data.txt -> label, seq.in
 - seq.in은 KorBERT가 pre-train된 대로 tokenization 과정을 거쳐 생성됨
   
 ![label](https://user-images.githubusercontent.com/8359931/130418608-57c32632-98ba-4119-a9de-e7ad8379d231.PNG)
 ![sequence](https://user-images.githubusercontent.com/8359931/130418620-d5982006-9965-403f-b732-7b9a51d65604.PNG)
  
 ## 4. 파인튜닝 및 서비스
  - 생성된 label, seq.in을 통해 파인튜닝 진행
  - Flask를 이용해 웹 서버를 구동하고, 개별 리뷰에 대한 평점과
    csv파일 업로드를 한 후 개선된 평점 확인 가능
  
 ![시연영상](https://user-images.githubusercontent.com/8359931/130537602-b3cba16c-9a00-4b8c-b3c5-a003447d8676.gif)

 ## 5. 결론
  - BERT 모델은 동일한 단어의 문맥적 의미를 파악하여 표현
  - 대용량 코퍼스 기반으로 사전학습된 모델을 가지고 수행하고자 하는 작업에 따라
    미세조정을 하여 다양하게 활용가능하다.
  - 다른 모델들과 성능비교시 정확도 또한 더 우수하다.
  
   ![](https://i.imgur.com/gYlPKr7.png)
  


<file_sep>$(function() {
    $('button#evaluate').on('click', function () {
        $.getJSON($SCRIPT_ROOT + '/_evaluate_helper', {
            evaluate_data: $('#evaluate-data').val()
        }, function (data) {
        $("#senti-result").prepend(data.result);
        $("#evaluate-input").show();
        $("#evaluate-data").val("");
      });
      return false;
    });
});

$('#evaluate_csv').on('click', function () {
    var url = $('#fileForm').attr('action');
    var form_data = new FormData();
    console.log($('#file').prop('files')[0]);
    form_data.append('file', $('#file').prop('files')[0]);

    $.ajax({
        type: 'POST',
        url: url,
        data: form_data,
        contentType: false,
        processData: false,
        success: function (data) {
            $("#csv-result").prepend(data.result);
        }
    });
});

$('textarea').each(function () {
    this.setAttribute('style', 'height:' + (this.scrollHeight) + 'px;overflow-y:hidden;');
}).on('input', function () {
    this.style.height = 'auto';
    this.style.height = (this.scrollHeight) + 'px';
});


$('a#toggle-history').on('click', function () {
    var hist_toggle = $('#toggle-history');
    hist_toggle.text() === "(hide)" ? hist_toggle.text("(show)") : hist_toggle.text("(hide)");
    $("#history").toggle();
});
<file_sep># -*- coding: utf-8 -*-

import tensorflow as tf
from tensorflow.python.keras import backend as K
from tensorflow.python.keras.models import Model
from tensorflow.python.keras.layers import Input, Dense, Multiply, TimeDistributed
from models.base_nlu_model import BaseNLUModel
from layers.korbert_layer import KorBertLayer
import numpy as np
import os
import json


class BertNLUModel(BaseNLUModel):

    def __init__(self, intents_num, bert_hub_path, sess, num_bert_fine_tune_layers=10,
                 is_bert=True, is_training=True):
        self.intents_num = intents_num
        self.bert_hub_path = bert_hub_path
        self.num_bert_fine_tune_layers = num_bert_fine_tune_layers
        self.is_bert = is_bert
        self.is_training = is_training
 
        self.model_params = {
                'intents_num': intents_num,
                'bert_hub_path': bert_hub_path,
                'num_bert_fine_tune_layers': num_bert_fine_tune_layers,
                'is_bert': is_bert
                }
        
        self.build_model()
        self.compile_model()
        
        self.initialize_vars(sess)
        
        
    def compile_model(self):
        
        optimizer = tf.keras.optimizers.Adam(lr=5e-5)#0.001)

        losses = {
        	'intent_classifier': 'sparse_categorical_crossentropy',
        }

        metrics = {'intent_classifier': 'acc'}

        self.model.compile(optimizer=optimizer, loss=losses, metrics=metrics)
        self.model.summary()
        

    def build_model(self):
        in_id = Input(shape=(None,), dtype=tf.int32, name='input_ids')
        in_mask = Input(shape=(None,), dtype=tf.int32, name='input_masks')
        in_segment = Input(shape=(None,), dtype=tf.int32, name='segment_ids')

        bert_inputs = [in_id, in_mask, in_segment] # tf.keras layers
        

        print('bert inputs :', bert_inputs)

        bert_pooled_output = KorBertLayer(
            n_tune_layers=self.num_bert_fine_tune_layers,
            pooling='mean', name='KorBertLayer')(bert_inputs)
    
        
        # fine-tuning layer
        intents_fc = Dense(self.intents_num, activation='softmax', name='intent_classifier')(bert_pooled_output)
        

        self.model =  Model(inputs=bert_inputs, outputs=intents_fc)

        
    def fit(self, X, Y, validation_data=None, epochs=5, batch_size=64, callbacks=None):
        """
        X: batch of [input_ids, input_mask, segment_ids, valid_positions]
        """

        X = (X[0], X[1], X[2])
        if validation_data is not None:
            X_val, Y_val = validation_data
            validation_data = ((X_val[0], X_val[1], X_val[2]), Y_val)

        history = self.model.fit(X, Y, validation_data=validation_data, 
                                 epochs=epochs, batch_size=batch_size, callbacks=callbacks)
        print(history.history)
        self.visualize_metric(history.history, 'loss')
        self.visualize_metric(history.history, 'acc')
        
        
    def initialize_vars(self, sess):
        sess.run(tf.compat.v1.local_variables_initializer())
        sess.run(tf.compat.v1.global_variables_initializer())
        K.set_session(sess)
        
        
    def predict_intent(self, x, intent_vectorizer, remove_start_end=True):

        # x = (x[0], x[1], x[2])
        y_intent = self.predict(x)

#        print('y_intent :', y_intent)

        first_intents_score = np.array([np.max(y_intent[i]) for i in range(y_intent.shape[0])])
        first_intents = np.array([intent_vectorizer.inverse_transform([np.argmax(y_intent[i])])[0] for i in range(y_intent.shape[0])])
        second_intents_score = np.array([np.sort(y_intent[i])[-2] for i in range(y_intent.shape[0])])
        second_intents = np.array([intent_vectorizer.inverse_transform([np.argsort(y_intent[i])[-2]])[0] for i in range(y_intent.shape[0])])
        
        return first_intents, first_intents_score, second_intents, second_intents_score

    def save(self, model_path):
        with open(os.path.join(model_path, 'params.json'), 'w') as json_file:
            json.dump(self.model_params, json_file)
        self.model.save(os.path.join(model_path, 'bert_nlu_model.h5'))
        
    def load(load_folder_path, sess): # load for inference or model evaluation
        with open(os.path.join(load_folder_path, 'params.json'), 'r') as json_file:
            model_params = json.load(json_file)
            
        intents_num = model_params['intents_num']
        bert_hub_path = model_params['bert_hub_path']
        num_bert_fine_tune_layers = model_params['num_bert_fine_tune_layers']
        is_bert = model_params['is_bert']
            
        new_model = BertNLUModel(intents_num, bert_hub_path, sess, num_bert_fine_tune_layers, is_bert, is_training=False)
        new_model.model.load_weights(os.path.join(load_folder_path,'bert_nlu_model.h5'))
        return new_model
<file_sep># -*- coding: utf-8 -*-

from tensorflow.keras.models import load_model
import matplotlib.pyplot as plt
import numpy as np


class BaseNLUModel:
    
    def __init__(self):
        self.model = None
    
    def visualize_metric(self, history_dic, metric_name):
        plt.plot(history_dic[metric_name])
        legend = ['train']
        if 'val_' + metric_name in history_dic:
            plt.plot(history_dic['val_' + metric_name])
            legend.append('test')
        plt.title('model ' + metric_name)
        plt.ylabel(metric_name)
        plt.xlabel('epoch')
        plt.legend(legend, loc='upper left')
        plt.savefig('train_log.png')
        
    def predict(self, x):
        return self.model.predict(x)
    
    def save(self, model_path):
        self.model.save(model_path)
        
    def load(model_path, custom_objects=None):
        new_model = BaseNLUModel()
        new_model.model = load_model(model_path, custom_objects=custom_objects)
        return new_model
    
    def predict_intent(self, x, intents_label_encoder):
        if len(x.shape) == 1:
            x = x[np.newaxis, ...]
        y = self.predict(x)
        intents = np.array([intents_label_encoder.inverse_transform([np.argmax(y[i])])[0] for i in range(y.shape[0])])
        return intents, slots
