src/models.py

import json
import os
from pydantic import BaseModel
from typing import Literal


class ChunkConfig(BaseModel):
	input_dataset: str
	input_splits: list[str]
	input_text_col: str
	output_dataset: str
	strategy: Literal["spacy", "sequence", "constant"]
	split_seq: str | list[str]
	chunk_len: int
	private: bool


class EmbedConfig(BaseModel):
	input_dataset: str
	input_splits: list[str]
	input_text_col: str
	output_dataset: str
	private: bool
	semaphore_bound: int


class WebhookPayloadEvent(BaseModel):
	action: Literal["create", "update", "delete"]
	scope: str


class WebhookPayloadRepo(BaseModel):
	type: Literal["dataset", "model", "space"]
	name: str
	id: str
	private: bool
	headSha: str


class WebhookPayload(BaseModel):
	event: WebhookPayloadEvent
	repo: WebhookPayloadRepo


with open(os.path.join(os.getcwd(), "chunk_config.json")) as c:
	data = json.load(c)
	chunk_config = ChunkConfig.model_validate_json(json.dumps(data))

with open(os.path.join(os.getcwd(), "embed_config.json")) as c:
	data = json.load(c)
	embed_config = EmbedConfig.model_validate_json(json.dumps(data))