<h3 dir="auto">System Info</h3>
<p dir="auto">Each run of the trainer two tfevent files are being generated, it looks like this:<br>
/runs<br>
--Feb27_09-46-42_...<br>
----events.out.tfevents....0<br>
----/1677491207.0429652<br>
------events.out.tfevents....1</p>
<p dir="auto">When reading these files with TensorBoard I don't get any output from the .1 file, how can I get rid of it (because these clutter my TensorBoard) or get actual data from it?</p>
<p dir="auto">Thanks in advance</p>
<p dir="auto"><a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/sgugger/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/sgugger">@sgugger</a></p>
<h3 dir="auto">Who can help?</h3>
<p dir="auto"><em>No response</em></p>
<h3 dir="auto">Information</h3>
<ul class="contains-task-list">
<li class="task-list-item"><input type="checkbox" id="" disabled="" class="task-list-item-checkbox"> The official example scripts</li>
<li class="task-list-item"><input type="checkbox" id="" disabled="" class="task-list-item-checkbox"> My own modified scripts</li>
</ul>
<h3 dir="auto">Tasks</h3>
<ul class="contains-task-list">
<li class="task-list-item"><input type="checkbox" id="" disabled="" class="task-list-item-checkbox"> An officially supported task in the <code class="notranslate">examples</code> folder (such as GLUE/SQuAD, ...)</li>
<li class="task-list-item"><input type="checkbox" id="" disabled="" class="task-list-item-checkbox"> My own task or dataset (give details below)</li>
</ul>
<h3 dir="auto">Reproduction</h3>
<p dir="auto">Simply starting any training.</p>
<h3 dir="auto">Expected behavior</h3>
<p dir="auto">One tfevents file or a valid output from the .1 file.</p>
<p dir="auto">Hello,</p>
<p dir="auto">I wanted to use Roberta for Sentence classification on protein sequences which I have converted into sentences. So first I train a tokenizer for my custom vocabulary.</p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="tokenizer.train(&quot;bert_vocab.txt&quot;,vocab_size=8000,special_tokens=[
    &quot;[CLS]&quot;,
    &quot;[SEP]&quot;,
    &quot;[UNK]&quot;,
    &quot;[MASK]&quot;,
])

tokenizer.save_model(&quot;EsperBERTo&quot;)

from tokenizers.implementations import CharBPETokenizer
from tokenizers.processors import BertProcessing

tokenizer._tokenizer.post_processor = BertProcessing((&quot;[SEP]&quot;, tokenizer.token_to_id(&quot;[SEP]&quot;)), (&quot;[CLS]&quot;, tokenizer.token_to_id(&quot;[CLS]&quot;)),)
print(tokenizer.encode(&quot;AAA ATA AKA&quot;))
print(tokenizer.encode(&quot;AAA ATA AKA&quot;).tokens)

tokenizer.enable_truncation(max_length=512)


import torch
torch.cuda.is_available()

from transformers import RobertaConfig

config = RobertaConfig(
    vocab_size=8000,
    max_position_embeddings=514,
    num_attention_heads=12,
    num_hidden_layers=6,
    type_vocab_size=1,
)

from transformers import RobertaTokenizerFast

tokenizer = RobertaTokenizerFast.from_pretrained(&quot;./EsperBERTo&quot;, max_len=512)
"><pre lang="tokenizer" class="notranslate"><code class="notranslate">tokenizer.train("bert_vocab.txt",vocab_size=8000,special_tokens=[
    "[CLS]",
    "[SEP]",
    "[UNK]",
    "[MASK]",
])

tokenizer.save_model("EsperBERTo")

from tokenizers.implementations import CharBPETokenizer
from tokenizers.processors import BertProcessing

tokenizer._tokenizer.post_processor = BertProcessing(("[SEP]", tokenizer.token_to_id("[SEP]")), ("[CLS]", tokenizer.token_to_id("[CLS]")),)
print(tokenizer.encode("AAA ATA AKA"))
print(tokenizer.encode("AAA ATA AKA").tokens)

tokenizer.enable_truncation(max_length=512)


import torch
torch.cuda.is_available()

from transformers import RobertaConfig

config = RobertaConfig(
    vocab_size=8000,
    max_position_embeddings=514,
    num_attention_heads=12,
    num_hidden_layers=6,
    type_vocab_size=1,
)

from transformers import RobertaTokenizerFast

tokenizer = RobertaTokenizerFast.from_pretrained("./EsperBERTo", max_len=512)

</code></pre></div>
<p dir="auto">When i try to train the model, i get the error <code class="notranslate">ValueError: Expected input batch_size (64) to match target batch_size (8192).</code></p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="model = RobertaForSequenceClassification(config=config)


model.num_parameters()


from transformers import DataCollatorForLanguageModeling

from transformers import LineByLineTextDataset

dataset = LineByLineTextDataset(
    tokenizer=tokenizer,
    file_path=&quot;bert_data.txt&quot;,
    block_size=64,
)

data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer)

from transformers import Trainer, TrainingArguments

training_args = TrainingArguments(
    output_dir=&quot;./EsperBERTo&quot;,
    overwrite_output_dir=True,
    num_train_epochs=100,
    per_device_train_batch_size=64,
    save_steps=10_000,
    save_total_limit=2,
)

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=dataset,
    prediction_loss_only=True,
)
trainer.train()

trainer.save_model(&quot;./EsperBERTo&quot;)"><pre class="notranslate"><code class="notranslate">model = RobertaForSequenceClassification(config=config)


model.num_parameters()


from transformers import DataCollatorForLanguageModeling

from transformers import LineByLineTextDataset

dataset = LineByLineTextDataset(
    tokenizer=tokenizer,
    file_path="bert_data.txt",
    block_size=64,
)

data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer)

from transformers import Trainer, TrainingArguments

training_args = TrainingArguments(
    output_dir="./EsperBERTo",
    overwrite_output_dir=True,
    num_train_epochs=100,
    per_device_train_batch_size=64,
    save_steps=10_000,
    save_total_limit=2,
)

trainer = Trainer(
    model=model,
    args=training_args,
    data_collator=data_collator,
    train_dataset=dataset,
    prediction_loss_only=True,
)
trainer.train()

trainer.save_model("./EsperBERTo")
</code></pre></div>
<p dir="auto">I understand i do not specify the output labels anywhere in the code but I am yet to find any example which I could follow to figure this.</p>