global:
  huggingface:
    imageRegistry: ""
    imagePullSecrets: []
    privateHub:
      enabled: false
    ingress:
      enabled: true
      domain: huggingface.co
      ssl: true
      subdomains:
        datasetsServer: datasets-server
    service:
      type: ClusterIP
      ports:
        datasetsServer:
          proxy: 30020
          admin: 30021
          api: 30022
          rows: 30023

images:
  pullPolicy: IfNotPresent
  pullSecrets: []
  reverseProxy:
    useGlobalRegistry: false
    registry: docker.io
    repository: nginx
    tag: "1.20"
  jobs:
    mongodbMigration:
      registry: huggingface
      useGlobalRegistry: false
      repository: datasets-server-jobs-mongodb_migration
      tag: sha-fb3399a
    cacheMaintenance:
      registry: huggingface
      useGlobalRegistry: false
      repository: datasets-server-jobs-cache_maintenance
      tag: sha-fb3399a
  services:
    admin:
      registry: huggingface
      useGlobalRegistry: false
      repository: datasets-server-services-admin
      tag: sha-fb3399a
    api:
      registry: huggingface
      useGlobalRegistry: false
      repository: datasets-server-services-api
      tag: sha-fb3399a
    rows:
      registry: huggingface
      useGlobalRegistry: false
      repository: datasets-server-services-rows
      tag: sha-fb3399a
    worker:
      registry: huggingface
      useGlobalRegistry: false
      repository: datasets-server-services-worker
      tag: sha-fb3399a


common:
  # URL of the HuggingFace Hub
  hfEndpoint: ""

log:
  # Log level
  level: "INFO"

# --- common parameters ---

secrets:
  externalSecret:
    enabled: false
    secretName: ""
    secretStoreName: ""
    parameters: { }
  mongoUrl:
    fromSecret: false
    secretName: "mongo-url"
    value: mongo://
  appHfToken:
    fromSecret: true
    secretName: ""
    value: ""
  appParquetConverterHfToken:
    fromSecret: true
    secretName: ""
    value: ""
  hfWebhookSecret:
    fromSecret: false
    secretName: "webhook-secret"
    value: ""
  spawningToken:
    fromSecret: true
    secretName: "spawning-token"

uid: 1000
gid: 3000

persistence:
  existingClaim: ""
  storageClass: ""
  size: 20Gi

cachePersistence:
  existingClaim: ""

parquetPersistence:
  existingClaim: ""

duckdbPersistence:
  existingClaim: ""

monitoring:
  enabled: false

mongodb:
  enabled: true
  nameOverride: datasets-server-mongodb
  useStatefulSet: true
  auth:
    enabled: false
  serviceAccount:
    create: false

cache:
  # Maximum number of days to keep the cached API responses
  maxDays: 90
  # Name of the mongo db database used to cache the API responses
  mongoDatabase: "datasets_server_cache"

queue:
  # Name of the mongo db database used to store the jobs queue
  mongoDatabase: "datasets_server_queue"

metrics:
  # Name of the mongo db database used to store metrics
  mongoDatabase: "datasets_server_metrics"

worker:
  # maximum size in bytes of the response content computed by a worker
  contentMaxBytes: "10_000_000"
  # the time interval between two heartbeats. Each heartbeat updates the job "last_heartbeat" field in the queue.
  heartbeatIntervalSeconds: 60
  # the time interval at which the worker looks for zombie jobs to kill them
  killZombiesIntervalSeconds: 600
  # maximum disk usage of every storage disk in the list (in percentage) to allow a job to start. Set to 0 to disable the test.
  maxDiskUsagePct: 90
  # the maximum duration of a job before it gets stopped for exceeded the maximum duration
  maxJobDurationSeconds: 2400
  # Max CPU load (%) - if reached, sleeps until it comes back under the limit. Set to 0 to disable the test.
  maxLoadPct: 0
  # Max memory (RAM + SWAP) (%) - if reached, sleeps until it comes back under the limit. Set to 0 to disable the test.
  maxMemoryPct: 0
  # the number of hearbeats a job must have missed to be considered a zombie job.
  maxMissingHeartbeats: 5
  # Number of seconds a worker will sleep before trying to process a new job
  sleepSeconds: 5

firstRows:
  # Max size of the /first-rows endpoint response in bytes
  maxBytes: "1_000_000"
  # Max number of rows in the /first-rows endpoint response
  maxNumber: 100
  # Min size of a cell in the /first-rows endpoint response in bytes
  minCellBytes: 100
  # Min number of rows in the /first-rows endpoint response
  minNumber: 10
  # Max number of columns in the /first-rows endpoint response
  columnsMaxNumber: 1_000

parquetAndInfo:
  # comma-separated list of the blocked datasets. Defaults to empty.
  blockedDatasets: ""
  # the git commit message when the parquet files are uploaded to the Hub. Defaults to `Update parquet files`.
  commitMessage: "Update parquet files"
  # the maximum size in bytes of the dataset to pre-compute the parquet files. Bigger datasets, or datasets without that information, are partially streamed to get parquet files up to this value.
  maxDatasetSize: "100_000_000"
  # the maximum number of external files of the datasets. Bigger datasets, or datasets without that information, are partially streamed to get parquet files up to `maxDatasetSize` bytes.
  maxExternalDataFiles: "10_000"
  # the maximum size in bytes of the row groups of parquet datasets that are copied to the target revision. Bigger datasets, or datasets without that information, are partially streamed to get parquet files up to `maxDatasetSize` bytes.
  maxRowGroupByteSizeForCopy: "100_000_000"
  # comma-separated list of datasets that are fully converted to parquet (no partial conversion).
  noMaxSizeLimitDatasets: ""
  # the git revision of the dataset to use to prepare the parquet files. Defaults to `main`.
  sourceRevision: "main"
  # comma-separated list of the supported datasets. If empty, all the datasets are processed. Defaults to empty.
  supportedDatasets: ""
  # the git revision of the dataset where to store the parquet files. Make sure the hf_token (see the "Common" section) allows to write there. Defaults to `refs/convert/parquet`.
  targetRevision: "refs/convert/parquet"
  # the URL template to build the parquet file URLs. Defaults to `/datasets/%s/resolve/%s/%s`.
  urlTemplate: "/datasets/%s/resolve/%s/%s"

optInOutUrlsScan:
  columnsMaxNumber: 10
  # the max number of columns to scan
  maxConcurrentRequestsNumber: 10
  # the max concurrent request number 
  maxRequestsPerSecond: 20
  # the max number of request allowed to process in parallel per second
  rowsMaxNumber: 1_000
  # the max number of rows to scan
  urlsNumberPerBatch: 1_000
  # the number of grouped urls to be send in every request to spawning 
  spawningUrl: "https://opts-api.spawningaiapi.com/api/v2/query/urls"
  # the URL for spawning requests

configNames:
  # the max number of configs per dataset
  maxNumber: 3_000

assets:
  # base URL for the assets files. It should be set accordingly to the datasets-server domain, eg https://datasets-server.huggingface.co/assets
  # baseUrl: "not used for now"
  # Directory on the shared storage (audio files and images)
  storageDirectory: "/assets"

cachedAssets:
  # base URL for the cached assets files. It should be set accordingly to the datasets-server domain, eg https://datasets-server.huggingface.co/cached-assets
  # baseUrl: "not used for now"
  # Directory on the cached shared storage (audio files and images)
  storageDirectory: "/cached-assets"
  # Probability of cleaning the cached assets directory at each request.
  cleanCacheProba: 0.05
  # When cleaning the cached assets directory: keep the rows with an index below a certain number.
  keepFirstRowsNumber: 100
  # When cleaning the cached assets directory: keep the most recently accessed rows.
  keepMostRecentRowsNumber: 200
  # When cleaning the cached assets directory: maximum number of rows to discard.
  maxCleanedRowsNumber: 10000


parquetMetadata:
  # Directory on the shared storage (parquet metadata files used for random access in /rows)
  storageDirectory: "/parquet-metadata"

duckDBIndex:
  # Directory on the shared storage (used temporarily to prepare the duckdb indexes before sending to the Hub)
  storageDirectory: "/duckdb-index"
  # the git commit message when the duckdb index file is uploaded to the Hub. Defaults to `Update duckdb index files`.
  commitMessage: "Update duckdb index files"
  # the git revision of the dataset where to store the duckdb index file. Defaults to `refs/convert/parquet`.
  targetRevision: "refs/convert/parquet"
  # the URL template to build the duckdb index file URL. Defaults to `/datasets/%s/resolve/%s/%s`.
  urlTemplate: "/datasets/%s/resolve/%s/%s"
  # the maximum size of the split parquets.
  maxParquetSizeBytes: "100_000_000"

# Directory where the cache data will be stored
cacheDirectory: "/datasets-server-cache"

# --- jobs (pre-install/upgrade hooks) ---

mongodbMigration:
  # Name of the mongo db database used for storing the migrations history
  mongoDatabase: "datasets_server_maintenance"

  nodeSelector: {}
  resources:
    requests:
      cpu: 0
    limits:
      cpu: 0
  tolerations: []

cacheMaintenance:
  action: "skip"
  # ^ allowed values are {skip,backfill,upgrade,collect-metrics}
  log:
    level: "info"
  backfill:
    error_codes_to_retry: ""
  nodeSelector: {}
  resources:
    requests:
      cpu: 0
    limits:
      cpu: 0
  tolerations: []

# --- cron jobs  ---
backfill:
  enabled: false
  log:
    level: "info"
  action: "backfill"
  error_codes_to_retry: ""
  schedule: "0 */3 * * *"
  # every 3 hours
  nodeSelector: {}
  resources:
    requests:
      cpu: 0
    limits:
      cpu: 0
  tolerations: []


metricsCollector:
  action: "collect-metrics"
  schedule: "*/5 * * * *"
  # every five minutes
  nodeSelector: {}
  resources:
    requests:
      cpu: 0
    limits:
      cpu: 0
  tolerations: []

# --- storage admin (to manually inspect the storage, in /data) ---

storageAdmin:
  nodeSelector: {}
  replicas: 1
  resources:
    requests:
      cpu: 0
    limits:
      cpu: 0
  service:
    type: ""
    annotations: {}
  tolerations: []

# --- reverse proxy ---

reverseProxy:
  nameOverride: datasets-server-mongodb
  host: localhost
  port: 8080
  nginxTemplateFile: "nginx-templates/default.conf.template"
  openapiFile: "static-files/openapi.json"
  error404File: "nginx-templates/404.html"

  nodeSelector: {}
  replicas: 1
  resources:
    requests:
      cpu: 0
    limits:
      cpu: 0
  service:
    type: ""
    annotations: {}
  tolerations: []

ingress:
  enabled: true
  tls: []
  annotations: {}

# --- services ---

admin:
  # HF organization that is allowed to request the report
  hfOrganization: "huggingface"
  # Number of reports in /cache-reports/... endpoints
  cacheReportsNumResults: 100
  # Number of reports in /cache-reports-with-content/... endpoints
  cacheReportsWithContentNumResults: 100
  # the timeout in seconds for the requests to the Hugging Face Hub.
  hfTimeoutSeconds: "0.2"
  # The path of the whoami service on the hub.
  hfWhoamiPath: "/api/whoami-v2"
  # Number of seconds to set in the `max-age` header on technical endpoints
  maxAge: "10"
  # Directory where the uvicorn workers share their prometheus metrics
  # see https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn
  prometheusMultiprocDirectory: "/tmp"
  # hostname - it must not be set to localhost to work in Kube!
  uvicornHostname: "0.0.0.0"
  # Number of uvicorn workers for running the application
  uvicornNumWorkers: "1"
  # Application endpoint port
  uvicornPort: 8080

  nodeSelector: {}
  replicas: 1
  resources:
    requests:
      cpu: 0
    limits:
      cpu: 0
  service:
    type: ""
    annotations: {}
  tolerations: []

api:
  # the path of the external authentication service on the hub.
  # The string must contain `%s` which will be replaced with the dataset name.
  hfAuthPath: "/api/datasets/%s/auth-check"
  # the URL where the "Hub JWT public key" is published. The "Hub JWT public key" must be in JWK format.
  # It helps to decode a JWT sent by the Hugging Face Hub, for example, to bypass the external authentication
  # check (JWT in the 'X-Api-Key' header). If not set, the JWT are ignored.
  hfJwtPublicKeyUrl: "https://huggingface.co/api/keys/jwt"
  # the algorithm used to encode the JWT.
  hfJwtAlgorithm: "EdDSA"
  # the timeout in seconds for the requests to the Hugging Face Hub.
  hfTimeoutSeconds: "0.2"
  # Number of seconds to set in the `max-age` header on data endpoints
  maxAgeLong: "120"
  # Number of seconds to set in the `max-age` header on technical endpoints
  maxAgeShort: "10"
  # Directory where the uvicorn workers will write the prometheus metrics
  # see https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn
  prometheusMultiprocDirectory: "/tmp"
  # Hostname - it must not be set to localhost to work in Kube!
  uvicornHostname: "0.0.0.0"
  # Number of uvicorn workers for running the application
  uvicornNumWorkers: "1"
  # Application endpoint port
  uvicornPort: 8080

  nodeSelector: {}
  replicas: 1
  resources:
    requests:
      cpu: 0
    limits:
      cpu: 0
  service:
    type: ""
    annotations: {}
  tolerations: []

rows:
  # the path of the external authentication service on the hub.
  # The string must contain `%s` which will be replaced with the dataset name.
  hfAuthPath: "/api/datasets/%s/auth-check"
  # the URL where the "Hub JWT public key" is published. The "Hub JWT public key" must be in JWK format.
  # It helps to decode a JWT sent by the Hugging Face Hub, for example, to bypass the external authentication
  # check (JWT in the 'X-Api-Key' header). If not set, the JWT are ignored.
  hfJwtPublicKeyUrl: "https://huggingface.co/api/keys/jwt"
  # the algorithm used to encode the JWT.
  hfJwtAlgorithm: "EdDSA"
  # the timeout in seconds for the requests to the Hugging Face Hub.
  hfTimeoutSeconds: "0.2"
  # Number of seconds to set in the `max-age` header on data endpoints
  maxAgeLong: "120"
  # Number of seconds to set in the `max-age` header on technical endpoints
  maxAgeShort: "10"
  # Directory where the uvicorn workers will write the prometheus metrics
  # see https://github.com/prometheus/client_python#multiprocess-mode-eg-gunicorn
  prometheusMultiprocDirectory: "/tmp"
  # Hostname - it must not be set to localhost to work in Kube!
  uvicornHostname: "0.0.0.0"
  # Number of uvicorn workers for running the application
  uvicornNumWorkers: "1"
  # Application endpoint port
  uvicornPort: 8082

  nodeSelector: {}
  replicas: 1
  resources:
    requests:
      cpu: 0
    limits:
      cpu: 0
  service:
    type: ""
    annotations: {}
  tolerations: []

workers:
  -
    # name of the deployment
    deployName: "all"
    # max difficulty of the jobs that this worker will process
    workerDifficultyMax: 100
    # min difficulty of the jobs that this worker will process
    workerDifficultyMin: 0
    # job types that this worker will not process
    workerJobTypesBlocked: ""
    # job types that this worker can process
    workerJobTypesOnly: ""
    nodeSelector: {}
    replicas: 1
    resources:
      requests:
        cpu: 0
      limits:
        cpu: 0
    tolerations: []
