tensorflow training problem
import numpy as np
import tensorflow as tf
from tensorflow.keras.preprocessing.sequence import pad_sequences
from transformers import TFBartModel, BartConfig
from tensorflow import keras

npfile = np.load("train_dataset.npz")
inp_ids = npfile["arr_0"][:1000]
out_ids = npfile["arr_1"][:1000]
out_ids = pad_sequences(out_ids, padding="post", truncating="post", value=1, maxlen=inp_ids.shape[1])

config = BartConfig.from_json_file("config.json")

b_model = TFBartModel(config=config)

def models():
    inp = keras.layers.Input(shape=[inp_ids.shape[1]], dtype="int32")
    outputs = b_model(inp, training = True, use_cache = False)
    logits = keras.layers.Dense(config.vocab_size, activation="softmax")(outputs[0])
    return keras.models.Model(inp, logits)

model = models()

model.summary()

dataset = tf.data.Dataset.from_tensor_slices((tf.constant(inp_ids), tf.constant(out_ids))).shuffle(2000).batch(4)

model.compile(optimizer=tf.keras.optimizers.Adam(), loss=tf.keras.losses.SparseCategoricalCrossentropy(), metrics=["acc"])

model.fit(dataset, epochs=100)

model.save_weights("tf_model.h5")


I don’t think there is a problem with the code, but the loss just doesn’t go down. Therefore, I also specially reduced the training sample and changed it to 1000, but it still has no effect. I wonder if the bart network has no weight update, but just updated the self-defined self. What about the definition layer?