dataset: ambig_qa
subset: light
templates:
  5f79fa25-3804-4e32-9493-a12c1c2ddff0: !Template
    answer_choices: null
    id: 5f79fa25-3804-4e32-9493-a12c1c2ddff0
    jinja: "{# Assignement in if clause breaks test, we need to declare variables\
      \ in global scope first: https://github.com/pallets/jinja/issues/1314 #}\n{%\
      \ set selected_question = \"\" %}\n{% set selected_answer = \"\" %}\n{% set\
      \ random_question_id = -1 %}\n{% if annotations.type[0] == \"multipleQAs\" %}\n\
      \   {% set random_question_id = range(0, annotations.qaPairs[0].question | length)\
      \ | choice%}\n   {% set selected_question = annotations.qaPairs[0].question[random_question_id]\
      \ %}\n   {% set selected_answer = annotations.qaPairs[0].answer[random_question_id]\
      \ | choice %}\n{% else %}\n    {% set selected_question = question %}\n    {%\
      \ set selected_answer = annotations.answer | choice %}\n{% endif %}\n\n{{selected_question}}\n\
      |||\n{{selected_answer}}"
    metadata: !TemplateMetadata
      choices_in_prompt: null
      metrics: []
      original_task: false
    name: ambig_qa_light3
    reference: Randomly choose an annotated question and answer it using one of its
      answers.
  72bf511b-44ce-4b9f-a2d0-5ed6334f0e07: !Template
    answer_choices: null
    id: 72bf511b-44ce-4b9f-a2d0-5ed6334f0e07
    jinja: "{# Assignement in if clause breaks test, we need to declare variables\
      \ in global scope first: https://github.com/pallets/jinja/issues/1314 #}\n{%\
      \ set random_question_id = -1 %}\n{% set random_answer_id = -1 %}\n{% set selected_question\
      \ = \"\" %}\n{% set selected_answer = \"\" %}\n{% if annotations.type[0] ==\
      \ \"multipleQAs\" %}\n   {% set random_question_id = range(0, annotations.qaPairs[0].question\
      \ | length) | choice%}\n   {% set random_answer_id = range(0, annotations.qaPairs[0].answer\
      \ | length) | choice%}\n   {% set selected_question = annotations.qaPairs[0].question[random_question_id]\
      \ %}\n   {% set selected_answer = annotations.qaPairs[0].answer[random_answer_id]\
      \ | choice%}\n{% else %}\n   {% set random_question_id = 0 %}\n   {% set random_answer_id\
      \ = 0 %}\n   {% set selected_question = question %}\n   {% set selected_answer\
      \ = annotations.answer[0] | choice %}\n{% endif %}\n\nIs \"{{selected_answer}}\"\
      \ the answer to \"{{selected_question}}\"?\n\n|||\n\n{% if random_answer_id\
      \ == random_question_id %} Yes {% else %} No {% endif %}"
    metadata: !TemplateMetadata
      choices_in_prompt: null
      metrics: []
      original_task: false
    name: ambig_qa_light4
    reference: Classify if the given answer if correct compared to the chosen question
  7655d2aa-70df-42cf-9bfa-80484521f856: !Template
    answer_choices: null
    id: 7655d2aa-70df-42cf-9bfa-80484521f856
    jinja: "{{question}}\n\n|||\n\n{# Assignement in if clause breaks test, we need\
      \ to declare variables in global scope first: https://github.com/pallets/jinja/issues/1314\
      \ #}\n{% set random_answer = \"\" %}\n{% set random_answer_form = \"\" %}\n\
      {% if annotations.type[0] == \"singleAnswer\" %}\n    {% set random_answer_form\
      \ = annotations.answer[0] | choice %}\n{% else %}\n    {% set random_answer\
      \ = annotations.qaPairs[0].answer | choice %}\n    {% set random_answer_form\
      \ = random_answer | choice %}\n{% endif %}\n\n{{random_answer_form}}"
    metadata: !TemplateMetadata
      choices_in_prompt: null
      metrics: []
      original_task: false
    name: ambig_qa_light1
    reference: Given the question, we choose the answer in single QA and randomly
      choose when in multipleQA.
  bb089312-23cb-475d-93b5-952781bc6be4: !Template
    answer_choices: null
    id: bb089312-23cb-475d-93b5-952781bc6be4
    jinja: "{# Assignement in if clause breaks test, we need to declare variables\
      \ in global scope first: https://github.com/pallets/jinja/issues/1314 #}\n{%\
      \ set selected_question = \"\" %}\n{% set selected_answer = \"\" %}\n{% set\
      \ random_question_id = -1 %}\n{% if annotations.type[0] == \"multipleQAs\" %}\n\
      \   {% set random_question_id = range(0, annotations.qaPairs[0].question | length)\
      \ | choice%}\n   {% set selected_question = annotations.qaPairs[0].question[random_question_id]%}\n\
      \   {% set selected_answer = annotations.qaPairs[0].answer[random_question_id]\
      \ | choice%}\n{% else %}\n   {% set selected_question = question %}\n   {% set\
      \ selected_answer = annotations.answer | choice %}\n{% endif %}\nKnowing that\
      \ \"{{selected_answer}}\" is the answer, what could have been the question?\n\
      |||\n{{selected_question}}"
    metadata: !TemplateMetadata
      choices_in_prompt: null
      metrics: []
      original_task: false
    name: ambig_qa_light5
    reference: Generate the answer from the question
  f53d00ea-98a8-45d3-92f6-93a8909aef2a: !Template
    answer_choices: null
    id: f53d00ea-98a8-45d3-92f6-93a8909aef2a
    jinja: "{{question}}\n\n|||\n\n{% if annotations.type[0] == \"singleAnswer\" %}\n\
      \    {{annotations.answer[0] | choice}}\n{% else %}\n    The questions was ambiguous.\
      \ Did you mean \"{{annotations.qaPairs[0].question |choice}}\"?\n{% endif %}\n"
    metadata: !TemplateMetadata
      choices_in_prompt: null
      metrics: []
      original_task: false
    name: ambig_qa_light2
    reference: If a question is ambiguous, ask another question, otherwise answer.