# process_text.py

from lib.chat.setup import openai_embeddings
from langchain.docstore.document import Document
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import Chroma
from langchain.document_loaders.csv_loader import CSVLoader
import requests
import json
import chardet


def read_input_files(file_paths):
    combined_articles = []
    for file_path in file_paths:
        content = read_file(file_path)
        combined_articles.append(content)
    return combined_articles


def split_texts(all_combined_articles):
    all_texts = []
    text_splitter = RecursiveCharacterTextSplitter(
                                                  chunk_size=4096,
                                                  chunk_overlap=256,
                                                  length_function=len
                                                  )
    for combined_articles in all_combined_articles:
        all_texts.append(text_splitter.split_text(combined_articles))
    return all_texts

def create_documents(all_texts, prefixes):
    all_docs = []
    for texts, prefix in zip(all_texts, prefixes):
        metadatas = [{"SOURCES": f"{prefix}{i}"} for i in range(len(texts))]
        docs = [Document(id=f'{prefix}{i}', page_content=texts[i], metadata=metadatas[i]) for i in range(len(texts))]
        all_docs.extend(docs)
    return all_docs

def get_fieldnames_from_csv(csv_path):
    with open(csv_path, newline="", encoding="utf-8") as csvfile:
        fieldnames = csvfile.readline().strip().split(';')
    return fieldnames

def documents_from_csv(csv_path,fieldnames):
    loader = CSVLoader(file_path=csv_path,
                       csv_args={
                                'delimiter': ';',
                                'quotechar': '"',
                                'fieldnames': fieldnames
})
    docs = loader.load()
    return docs


def create_vectordb_from_docs(docs, persist_directory=None):
    vectordb = Chroma.from_documents(documents=docs, embedding=openai_embeddings, persist_directory=persist_directory)
    if persist_directory:
        vectordb.persist()
    return vectordb


def load_persisted_chromadb(persist_directory):
    vectordb = Chroma(persist_directory=persist_directory, 
                      embedding_function=openai_embeddings)
    return vectordb

def read_file(file_path):
    with open(file_path, 'rb') as file:
        raw_data = file.read()
    detected_encoding = chardet.detect(raw_data)['encoding']
    content = raw_data.decode(detected_encoding, errors='ignore')
    return content


def write_file(file_path, content):
    with open(file_path, 'w', encoding='utf-8') as file:
        file.write(content)

def split_text(text, max_size=16 * 1024):
    words = text.split(' ')
    chunks = []
    current_chunk = []

    for word in words:
        if len(' '.join(current_chunk)) + len(word) < max_size:
            current_chunk.append(word)
        else:
            chunks.append(' '.join(current_chunk))
            current_chunk = [word]

    if current_chunk:
        chunks.append(' '.join(current_chunk))

    return chunks

def translate_text(text, api_key, source_lang='EL', target_lang='EN'):
    url = 'https://api-free.deepl.com/v2/translate'
    headers = {'Authorization': f'DeepL-Auth-Key {api_key}'}
    data = {'text': text, 'source_lang': source_lang, 'target_lang': target_lang}
    response = requests.post(url, headers=headers, data=data)
    
    try:
        translations = json.loads(response.text)['translations']
        return translations[0]['text']
    except json.JSONDecodeError:
        print(f"Error: status code {response.status_code}, response content: {response.text}")
        return ""
    
    
def translate_documents(file_paths, api_key, source_lang, target_lang):
    translated_files = []
    for file_path in file_paths:
        text = read_file(file_path)
        chunks = split_text(text)
        translated_chunks = [translate_text(chunk, api_key, source_lang, target_lang) for chunk in chunks]
        translated_text = ' '.join(translated_chunks)

        translated_file_path = f"{file_path[:-4]}_translated.txt"
        write_file(translated_file_path, translated_text)
        translated_files.append(translated_file_path)

    return translated_files



langchain.text_splitter.RecursiveCharacterTextSplitter langchain.docstore.document.Document langchain.vectorstores.Chroma.from_documents langchain.document_loaders.csv_loader.CSVLoader langchain.vectorstores.Chroma
[((612, 703), 'langchain.text_splitter.RecursiveCharacterTextSplitter', 'RecursiveCharacterTextSplitter', ([], {'chunk_size': '(4096)', 'chunk_overlap': '(256)', 'length_function': 'len'}), '(chunk_size=4096, chunk_overlap=256,\n    length_function=len)\n', (642, 703), False, 'from langchain.text_splitter import RecursiveCharacterTextSplitter\n'), ((1649, 1755), 'langchain.document_loaders.csv_loader.CSVLoader', 'CSVLoader', ([], {'file_path': 'csv_path', 'csv_args': '{\'delimiter\': \';\', \'quotechar\': \'"\', \'fieldnames\': fieldnames}'}), '(file_path=csv_path, csv_args={\'delimiter\': \';\', \'quotechar\': \'"\',\n    \'fieldnames\': fieldnames})\n', (1658, 1755), False, 'from langchain.document_loaders.csv_loader import CSVLoader\n'), ((1992, 2099), 'langchain.vectorstores.Chroma.from_documents', 'Chroma.from_documents', ([], {'documents': 'docs', 'embedding': 'openai_embeddings', 'persist_directory': 'persist_directory'}), '(documents=docs, embedding=openai_embeddings,\n    persist_directory=persist_directory)\n', (2013, 2099), False, 'from langchain.vectorstores import Chroma\n'), ((2234, 2320), 'langchain.vectorstores.Chroma', 'Chroma', ([], {'persist_directory': 'persist_directory', 'embedding_function': 'openai_embeddings'}), '(persist_directory=persist_directory, embedding_function=\n    openai_embeddings)\n', (2240, 2320), False, 'from langchain.vectorstores import Chroma\n'), ((3434, 3480), 'requests.post', 'requests.post', (['url'], {'headers': 'headers', 'data': 'data'}), '(url, headers=headers, data=data)\n', (3447, 3480), False, 'import requests\n'), ((2481, 2505), 'chardet.detect', 'chardet.detect', (['raw_data'], {}), '(raw_data)\n', (2495, 2505), False, 'import chardet\n'), ((1251, 1324), 'langchain.docstore.document.Document', 'Document', ([], {'id': 'f"""{prefix}{i}"""', 'page_content': 'texts[i]', 'metadata': 'metadatas[i]'}), "(id=f'{prefix}{i}', page_content=texts[i], metadata=metadatas[i])\n", (1259, 1324), False, 'from langchain.docstore.document import Document\n'), ((3518, 3543), 'json.loads', 'json.loads', (['response.text'], {}), '(response.text)\n', (3528, 3543), False, 'import json\n')]