0.49 check pad_token

app.py

@@ -112,8 +112,8 @@ def generate_both(system_prompt, input_text, chatbot_a, chatbot_b, max_new_token
             tokenized = tokenizer_a(formatted_conversation, return_tensors="pt").to(device)
             logging.debug(tokenized) #attention_mask
             input_ids_a = tokenized.input_ids
-            logging.debug(f'tokenizer_a.eos_token was: {tokenizer_a.eos_token}')
-            tokenizer_a.eos_token = "<|endoftext|>"
+            logging.debug(f'tokenizer_a.pad_token is {tokenizer_a.pad_token}')
+            tokenizer_a.eos_token = "<|endoftext|>" # not set für Pharia
         else: 
             input_ids_a = tokenizer_a.apply_chat_template(
                 new_messages_a,
@@ -127,8 +127,8 @@ def generate_both(system_prompt, input_text, chatbot_a, chatbot_b, max_new_token
             tokenized = tokenizer_b(formatted_conversation, return_tensors="pt").to(device)
             logging.debug(tokenized)
             input_ids_b = tokenized.input_ids
-            logging.debug(f'tokenizer_b.eos_token was: {tokenizer_b.eos_token}')
-            tokenizer_b.eos_token = "<|endoftext|>"
+            logging.debug(f'tokenizer_b.pad_token is {tokenizer_b.pad_token}')
+            tokenizer_b.eos_token = "<|endoftext|>" # not set für Pharia
         else:
             input_ids_b = tokenizer_b.apply_chat_template(
                 new_messages_b,