readme: ---
  language: 
    - en
  tags:
  - QA
  license: cc-by-4.0
  datasets:
  - BoolQ
  - CommonSenseQA
  - DROP
  - DuoRC
  - HellaSWAG
  - HotpotQA
  - HybridQA
  - NarrativeQA
  - NaturalQuestionsShort
  - NewsQA
  - QAMR
  - RACE
  - SearchQA
  - SIQA
  - SQuAD
  - TriviaQA-web
  metrics:
  - Accuracy
  - Precision
  - Recall
  - F1
  - MRR
  - R@3
  - R@5
  
  
  ---
  
  BERT for Sequence Classification trained on QA Dataset prediction task. 
  - Input: question. 
  - Output: dataset from where that question comes from.
  Original paper: TWEAC: Transformer with Extendable QA Agent Classifiers
  https://arxiv.org/abs/2104.07081
  
  Datasets used for training:
  ```
  list_datasets = ['BoolQ','CommonSenseQA','DROP','DuoRC','HellaSWAG','HotpotQA','HybridQA','NarrativeQA','NaturalQuestionsShort','NewsQA','QAMR','RACE','SearchQA','SIQA','SQuAD','TriviaQA-web']
  ```
  
  Results for all datasets:
  - Accuracy: 0.7919096825783123
  - Precision: 0.731586272892176
  - Recall: 0.7919096825783123
  - F1: 0.7494425609552463
  - MRR: 0.8720871733637521
  - R@3: 0.9438690810655046
  - R@5: 0.9745318608004427
  - Queries/second: 6052.33538824659
  
  
  Results per dataset:
  ```
  "BoolQ": {
              "accuracy": 0.998776758409786,
              "mrr": 0.999388379204893,
              "r@3": 1.0,
              "r@5": 1.0,
              "query_per_second": 6978.947907596168,
              "precision": 0.8649364406779662,
              "recall": 0.998776758409786,
              "f1": 0.9270508089696281
          },
          "CommonSenseQA": {
              "accuracy": 0.9247135842880524,
              "mrr": 0.9476358338878795,
              "r@3": 0.9705400981996727,
              "r@5": 0.9705400981996727,
              "query_per_second": 5823.984138936813,
              "precision": 0.442443226311668,
              "recall": 0.9247135842880524,
              "f1": 0.5985169491525425
          },
          "DROP": {
              "accuracy": 0.9075083892617449,
              "mrr": 0.9378200367399193,
              "r@3": 0.9609899328859061,
              "r@5": 0.9786073825503355,
              "query_per_second": 6440.988897129248,
              "precision": 0.8636726546906187,
              "recall": 0.9075083892617449,
              "f1": 0.8850480670893842
          },
          "DuoRC": {
              "accuracy": 0.5555803405457654,
              "mrr": 0.7368963429107307,
              "r@3": 0.9092125808610305,
              "r@5": 0.9596996059186557,
              "query_per_second": 6853.643198794893,
              "precision": 0.646814404432133,
              "recall": 0.5555803405457654,
              "f1": 0.5977360905563778
          },
          "HellaSWAG": {
              "accuracy": 0.998406691894045,
              "mrr": 0.9990705702715262,
              "r@3": 1.0,
              "r@5": 1.0,
              "query_per_second": 3091.5012960785157,
              "precision": 0.9974134500596896,
              "recall": 0.998406691894045,
              "f1": 0.9979098238280083
          },
          "HotpotQA": {
              "accuracy": 0.7414435784479837,
              "mrr": 0.8435804344945315,
              "r@3": 0.9325652321247034,
              "r@5": 0.973568281938326,
              "query_per_second": 4972.668019223381,
              "precision": 0.7352150537634409,
              "recall": 0.7414435784479837,
              "f1": 0.7383161801923401
          },
          "HybridQA": {
              "accuracy": 0.7934218118869013,
              "mrr": 0.8806947764680021,
              "r@3": 0.964800923254472,
              "r@5": 0.9930755914598961,
              "query_per_second": 4886.494046259562,
              "precision": 0.7198952879581152,
              "recall": 0.7934218118869013,
              "f1": 0.7548723579467472
          },
          "NarrativeQA": {
              "accuracy": 0.5623756749076442,
              "mrr": 0.7416681781060867,
              "r@3": 0.9011082693947144,
              "r@5": 0.9580373212086767,
              "query_per_second": 7081.067049796865,
              "precision": 0.5623224095472628,
              "recall": 0.5623756749076442,
              "f1": 0.5623490409661377
          },
          "NaturalQuestionsShort": {
              "accuracy": 0.7985353692739171,
              "mrr": 0.8743599435345307,
              "r@3": 0.9439077594266126,
              "r@5": 0.9774072919912745,
              "query_per_second": 7136.590426649795,
              "precision": 0.7963020509633313,
              "recall": 0.7985353692739171,
              "f1": 0.7974171464135678
          },
          "NewsQA": {
              "accuracy": 0.5375118708452041,
              "mrr": 0.71192075967717,
              "r@3": 0.855650522317189,
              "r@5": 0.939696106362773,
              "query_per_second": 7193.851409052092,
              "precision": 0.18757249378624688,
              "recall": 0.5375118708452041,
              "f1": 0.2780985136961061
          },
          "QAMR": {
              "accuracy": 0.6658497602557272,
              "mrr": 0.7969741223377345,
              "r@3": 0.9207778369738945,
              "r@5": 0.973361747469366,
              "query_per_second": 7321.775044800525,
              "precision": 0.8654525309881587,
              "recall": 0.6658497