The model is built on Leo Tolstoy's collected works and represents his individual semantics
Preparation
All texts are converted from the TEI markup, splitted into sentences and lemmatized. Only modern orthography left in the data.
import html
import os
import re
import shutil
from bs4 import BeautifulSoup
!pip install razdel # for splitting
from razdel import sentenize
from tqdm import tqdm
!git clone https://github.com/tolstoydigital/TEI.git
relevant_dirs = ['diaries', 'letters', 'notes', 'works']
path = 'TEI/reference/bibllist_works.xml' # allows to work with fiction and non fiction separately
xml = open(path).read()
soup = BeautifulSoup(xml, features="xml")
group_texts = {}
for it in soup.find_all("item"):
ref = it.find("ref")
for related in it.find_all("relatedItem"):
for ref_ana in related.find_all("ref"):
group_texts[ref_ana.text] = ref.text
prefix_texts = 'extracted_texts'
os.mkdir(prefix_texts)
if os.path.exists(prefix_texts):
shutil.rmtree(prefix_texts)
os.mkdir(prefix_texts)
# extract texts from XML
complex_texts = {}
for rel_dir in relevant_dirs:
path = os.path.join('TEI/texts', rel_dir)
for file in tqdm(sorted(os.listdir(path))):
fiction = 0
if not file.endswith('.xml'):
continue
xml = open(os.path.join(path, file)).read()
if 'Печатные варианты' in xml:
continue
nameID = file.replace('.xml', '')
soup = BeautifulSoup(xml, features="xml")
if soup.find("catRef", {"ana":"#fiction"}):
fiction = 1
s = soup.find("body")
paragraphs = []
for erase in s.find_all(["orig", "comments", "sic", "note"]):
erase.decompose()
for p in s.find_all(["p", "l"]):
paragraphs.append(html.unescape(p.text.replace('\n', ' ').strip()))
if not fiction:
with open(os.path.join(prefix_texts, rel_dir + '.txt'), 'a') as f:
for par in paragraphs:
par = re.sub(' ([.,;:!?)"»])', '\\1', par)
par = par.replace('\n', ' ')
par = par.strip()
par = re.sub('\s+', ' ', par)
par = re.sub('\[.+?\]', '', par)
for sent in sentenize(par):
f.write(list(sent)[2].strip() + '\n')
else:
if nameID in group_texts:
hyper_name = group_texts[nameID]
if hyper_name not in complex_texts:
complex_texts[hyper_name] = paragraphs
else:
complex_texts[hyper_name].extend(paragraphs)
else:
with open(os.path.join(prefix_texts, nameID + '.txt'), 'w') as f:
f.write('\n'.join(paragraphs))
for hyper_name in complex_texts:
with open(os.path.join(prefix_texts, hyper_name + '.txt'), 'w') as f:
f.write('\n'.join(complex_texts[hyper_name]))
# tagging
from pymystem3 import Mystem
pos = ['S', 'V', 'A', 'ADV']
def tagging():
m = Mystem()
for fl in os.listdir(prefix_texts):
#print(fl)
if 'mystem' in fl:
continue
with open(os.path.join(prefix_texts, fl)) as f:
text = f.read()
lines = text.split('\n')
ana_lines = []
for line in lines:
line = ' '.join(line.split()[1:])
line = line.replace('ò', 'о')
line = line.replace('è', 'е')
line = line.replace('à', 'а')
line = line.replace('ѝ', 'и')
line = line.replace('ỳ', 'у')
line = line.replace('о̀', 'о')
#line = line.replace('Изд.̀', 'издательство')
ana = []
info = m.analyze(line)
for token in info:
if "analysis" in token:
try:
analysis = token["analysis"][0]
except:
#print(token)
continue
# if "lex" in analysis:
lex = analysis["lex"]
#if 'gr' in analysis:
gr = analysis['gr']
#print(gr)
const = gr.split('=')[0]
if ',' in const:
pos = const.split(',')[0]
else:
pos = const
ana.append('{}_{}'.format(lex, pos))
ln = ' '.join(ana)
if re.search('[А-Яа-я]', ln):
ana_lines.append(ln)
with open('{}/mystem-{}'.format(prefix_texts, fl), 'w') as fw:
fw.write('\n'.join(ana_lines))
def mk_input():
inp = []
for fl in os.listdir(prefix_texts):
if not 'mystem' in fl:
continue
#print(fl)
with open(os.path.join(prefix_texts, fl)) as f:
text = f.read()
lines = text.split('\n')
for line in lines:
words = []
for w in line.split():
word = w.split('_')
if word[1] in pos:
words.append(w)
if len(words) > 1:
inp.append(' '.join(words))
with open('input.txt', 'w') as fw:
fw.write('\n'.join(inp))
tagging()
mk_input()
The whole code is in the w2v-prep.ipynb notebook.
Models
There are 2 models in the repository. Their parameters are taen from the general language models to be comparable from rusvectores site.
Here is the code for building models:
import sys
import logging
import gensim
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
pth = './input.txt'
data = gensim.models.word2vec.LineSentence(pth) # train sentence by sentence
modelLNT1 = gensim.models.Word2Vec(data, vector_size=500, window=2, min_count=2, sg=1) # comparable with web_mystem_skipgram_500_2_2015.bin
modelLNT1.save('skipgram_500_2.model') # saving
modelLNT2 = gensim.models.Word2Vec(data, vector_size=300, window=10, min_count=2, sg=0) # comparable with ruwikiruscorpora_upos_cbow_300_10_2021
modelLNT2.save('cbow_300_10.model')
Usage
# load models
modelLNT1 = Word2Vec.load("skipgram_500_2.model")
# most similar words viz
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
sns.set_style("darkgrid")
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
def tsnescatterplot(model, word, list_names): # stolen code
""" Plot in seaborn the results from the t-SNE dimensionality reduction algorithm of the vectors of a query word,
its list of most similar words, and a list of words.
"""
arrays = np.empty((0, 300), dtype='f')
word_labels = [word]
color_list = ['red']
# adds the vector of the query word
arrays = np.append(arrays, model.wv.__getitem__([word]), axis=0)
# gets list of most similar words
close_words = model.wv.most_similar([word])
# adds the vector for each of the closest words to the array
for wrd_score in close_words:
wrd_vector = model.wv.__getitem__([wrd_score[0]])
word_labels.append(wrd_score[0])
color_list.append('blue')
arrays = np.append(arrays, wrd_vector, axis=0)
# adds the vector for each of the words from list_names to the array
for wrd in list_names:
wrd_vector = model.wv.__getitem__([wrd])
word_labels.append(wrd)
color_list.append('green')
arrays = np.append(arrays, wrd_vector, axis=0)
# Reduces the dimensionality from 300 to 50 dimensions with PCA
reduc = PCA(n_components=20).fit_transform(arrays)
# Finds t-SNE coordinates for 2 dimensions
np.set_printoptions(suppress=True)
Y = TSNE(n_components=2, random_state=0, perplexity=15).fit_transform(reduc)
# Sets everything up to plot
df = pd.DataFrame({'x': [x for x in Y[:, 0]],
'y': [y for y in Y[:, 1]],
'words': word_labels,
'color': color_list})
fig, _ = plt.subplots()
fig.set_size_inches(9, 9)
# Basic plot
p1 = sns.regplot(data=df,
x="x",
y="y",
fit_reg=False,
marker="o",
scatter_kws={'s': 40,
'facecolors': df['color']
}
)
# Adds annotations one by one with a loop
for line in range(0, df.shape[0]):
p1.text(df["x"][line],
df['y'][line],
' ' + df["words"][line].title(),
horizontalalignment='left',
verticalalignment='bottom', size='medium',
color=df['color'][line],
weight='normal'
).set_size(15)
plt.xlim(Y[:, 0].min()-50, Y[:, 0].max()+50)
plt.ylim(Y[:, 1].min()-50, Y[:, 1].max()+50)
plt.title('t-SNE visualization for {}'.format(word.title()))
tsnescatterplot(modelLNT2, 'бог_S', [i[0] for i in modelLNT2.wv.most_similar(negative=["бог_S"])])
Train data
Train corpus inclded in this repository as an input.txt file. It contains more than 7 mln words. For detailed explanation see Bonch-Osmolovskaya, A., Skorinkin, D., Pavlova, I., Kolbasov, M., & Orekhov, B. (2019). Tolstoy semanticized: Constructing a digital edition for knowledge discovery. Journal of Web Semantics, 59, 100483.
Publication
Орехов Б. В. Индивидуальная семантика Л. Н. Толстого в свете векторных моделей // Terra Linguistica. 2023. Т. 14. No 4. С. 119–129. DOI: 10.18721/JHSS.14409
@article{орехов2023индивидуальная,
title={Индивидуальная семантика Л. Н. Толстого в свете векторных моделей},
author={Орехов, Б.В.},
journal={Terra Linguistica},
volume={14},
number={4},
pages={119--129},
doi={10.18721/JHSS.14409}
url={https://human.spbstu.ru/userfiles/files/articles/2023/4/119-129.pdf}
year={2023}
}