|
import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; |
|
import { MemoryVectorStore } from 'langchain/vectorstores/memory'; |
|
import { HuggingFaceTransformersEmbeddings } from "langchain/embeddings/hf_transformers"; |
|
|
|
const DEFAULT_CHUNK_SIZE = 1000; |
|
const VECTOR_STORE_SIZE = 10; |
|
const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize: DEFAULT_CHUNK_SIZE }); |
|
|
|
const model = new HuggingFaceTransformersEmbeddings({ |
|
modelName: "Xenova/all-MiniLM-L6-v2", |
|
}); |
|
|
|
export const similaritySearch = async (input: string, content: string) => { |
|
const documents = await textSplitter.createDocuments([content]); |
|
|
|
const vectorStore = await MemoryVectorStore.fromTexts( |
|
[...documents.map(doc => doc.pageContent)], |
|
[...documents.map((v, k) => k)], |
|
model |
|
) |
|
const queryResult = await vectorStore.similaritySearch(input, VECTOR_STORE_SIZE); |
|
return queryResult; |
|
} |