[tool.poetry]
authors = ["Sylvain Lesage <sylvain.lesage@huggingface.co>"]
description = "Worker that processes jobs and stores the responses in the cache"
name = "worker"
version = "1.0.0"
license = "Apache-2.0"

[tool.poetry.dependencies]
Pillow = "^9.4.0"
PyICU = "^2.10.2"
aiohttp = "^3.8.4"
aiolimiter = "^1.0.0"
bs4 = "^0.0.1"
conllu = "^4.5.2"
duckdb = "^0.8.1"
environs = "^9.5.0"
gdown = "^4.6.3"
huggingface-hub = { git = "https://github.com/huggingface/huggingface_hub", rev = "1055a56b2d2723b55ba4fdf1f3296e04cfd8d6db" }
kenlm = { git = "https://github.com/kpu/kenlm", branch = "master" }
kss = "^2.6.0"
libcommon = {path = "../../libs/libcommon", develop = true}
lm-dataformat = "^0.0.20"
lxml = "^4.9.2"
mirakuru = "^2.4.2"
nlp = "^0.4.0"
nltk = "^3.8.1"
numpy = "~1.22.4"
openpyxl = "^3.1.1"
pdf2image = "^1.16.2"
py7zr = "^0.20.4"
pyarrow = "^11.0.0"
pydub = "^0.25.1"
python = "3.9.15"
rarfile = "^4.0"
scikit-learn = "^1.2.1"
tensorflow-cpu = [
    {version = "^2.11.1", markers = "sys_platform == 'linux' and platform_machine != 'aarch64'"},
    {version = "^2.11.1", markers = "sys_platform == 'darwin' and platform_machine != 'arm64'"}
]
tensorflow-aarch64 = {version = "^2.11.1", markers = "sys_platform == 'linux' and platform_machine == 'aarch64'"}
tensorflow-macos = {version = "^2.11.1", markers = "sys_platform == 'darwin' and platform_machine == 'arm64'"}
tfrecord = "^1.14.1"
torch = { url = "https://download.pytorch.org/whl/cpu/torch-1.13.1%2Bcpu-cp39-cp39-linux_x86_64.whl" }
transformers = "^4.30.0"
trec-car-tools = { path = "vendors/trec-car-tools/python3" }
typer = "^0.4.2"
wget = "^3.2"

[tool.poetry.group.dev.dependencies]
bandit = "^1.7.4"
black = "^22.12.0"
flake8 = "^3.9.2"
isort = "^5.12.0"
mypy = "^1.0.0"
pandas-stubs = "^1.5.3"
pip-audit = "^2.5.4"
pytest = "^7.2.1"
pytest-asyncio = "^0.21.0"
pytest-cov = "^2.12.1"
types-pillow = "^9.5.0.4"
types-psutil = "^5.9.5"
types-requests = "^2.28.11"

[build-system]
build-backend = "poetry.core.masonry.api"
requires = ["poetry-core>=1.0.0"]

[tool.pytest.ini_options]
filterwarnings = ["ignore::DeprecationWarning"]
markers = [
    "wip: tests being developed"
]

[tool.coverage.run]
source = ["worker"]

[tool.isort]
profile = "black"

[tool.black]
line-length = 119
preview = true

[tool.mypy]
exclude = 'vendors'
strict = true

[[tool.mypy.overrides]]
module = [
    "aiolimiter.*",
    "datasets.*",
    "huggingface_hub.*",
    "pyarrow.*",
    "tqdm.*",
    "fsspec.*"
]
ignore_missing_imports = true
