RuntimeError: Error while creating shape using tf-xlm-roberta-large
I get the following runtime error after the 2nd fold.

this is my model:
maxlen = 50
`with strategy.scope():
        #bert_encoder = TFBertModel.from_pretrained(model_name)
        base_model = TFAutoModel.from_pretrained(model_name)
        input_word_ids = tf.keras.Input(shape = (maxlen, ), dtype = tf.int32, name = "input_word_ids")
        input_mask = tf.keras.Input(shape = (maxlen, ), dtype = tf.int32, name = "input_mask")
        input_type_ids = tf.keras.Input(shape = (maxlen, ), dtype = tf.int32, name = "input_type_ids")

        embedding = base_model([input_word_ids, input_mask, input_type_ids])[0]
        output = tf.keras.layers.Dense(3, activation = 'softmax')(embedding[:, 0, :])

        model = tf.keras.Model(inputs = [input_word_ids, input_mask, input_type_ids], outputs = output)
        model.compile(tf.keras.optimizers.Adam(lr = 1e-5), loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])

`
And the traceback below:

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-27-a45ce90453f2> in <module>
     22 
     23     K.clear_session()
---> 24     model = build_model(maxlen, model_name)
     25     checkpoint = tf.keras.callbacks.ModelCheckpoint(
     26                 'XLMRoberta_fold-%i.h5'%fold, monitor = 'val_loss', verbose = 1, save_best_only = True,

<ipython-input-23-9faa2e5f1d9b> in build_model(maxlen, model_name)
      2     with strategy.scope():
      3         #bert_encoder = TFBertModel.from_pretrained(model_name)
----> 4         base_model = TFAutoModel.from_pretrained(model_name)
      5         input_word_ids = tf.keras.Input(shape = (maxlen, ), dtype = tf.int32, name = "input_word_ids")
      6         input_mask = tf.keras.Input(shape = (maxlen, ), dtype = tf.int32, name = "input_mask")

/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_auto.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    421         for config_class, model_class in TF_MODEL_MAPPING.items():
    422             if isinstance(config, config_class):
--> 423                 return model_class.from_pretrained(pretrained_model_name_or_path, *model_args, config=config, **kwargs)
    424         raise ValueError(
    425             "Unrecognized configuration class {} for this kind of TFAutoModel: {}.\n"

/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_utils.py in from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs)
    482             return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file, allow_missing_keys=True)
    483 
--> 484         model(model.dummy_inputs, training=False)  # build the network with dummy inputs
    485 
    486         assert os.path.isfile(resolved_archive_file), "Error retrieving file {}".format(resolved_archive_file)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
    966           with base_layer_utils.autocast_context_manager(
    967               self._compute_dtype):
--> 968             outputs = self.call(cast_inputs, *args, **kwargs)
    969           self._handle_activity_regularization(inputs, outputs)
    970           self._set_mask_metadata(inputs, outputs, input_masks)

/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_roberta.py in call(self, inputs, **kwargs)
    229             heads.
    230         """
--> 231         outputs = self.roberta(inputs, **kwargs)
    232         return outputs
    233 

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
    966           with base_layer_utils.autocast_context_manager(
    967               self._compute_dtype):
--> 968             outputs = self.call(cast_inputs, *args, **kwargs)
    969           self._handle_activity_regularization(inputs, outputs)
    970           self._set_mask_metadata(inputs, outputs, input_masks)

/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_bert.py in call(self, inputs, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, output_attentions, output_hidden_states, training)
    604             # head_mask = tf.constant([0] * self.num_hidden_layers)
    605 
--> 606         embedding_output = self.embeddings([input_ids, position_ids, token_type_ids, inputs_embeds], training=training)
    607         encoder_outputs = self.encoder(
    608             [embedding_output, extended_attention_mask, head_mask, output_attentions, output_hidden_states],

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in __call__(self, *args, **kwargs)
    962         # Eager execution on data tensors.
    963         with backend.name_scope(self._name_scope()):
--> 964           self._maybe_build(inputs)
    965           cast_inputs = self._maybe_cast_inputs(inputs)
    966           with base_layer_utils.autocast_context_manager(

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in _maybe_build(self, inputs)
   2414         # operations.
   2415         with tf_utils.maybe_init_scope(self):
-> 2416           self.build(input_shapes)  # pylint:disable=not-callable
   2417       # We must set also ensure that the layer is marked as built, and the build
   2418       # shape is stored since user defined build functions may not be calling

/opt/conda/lib/python3.7/site-packages/transformers/modeling_tf_bert.py in build(self, input_shape)
    144                 "weight",
    145                 shape=[self.vocab_size, self.hidden_size],
--> 146                 initializer=get_initializer(self.initializer_range),
    147             )
    148         super().build(input_shape)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer.py in add_weight(self, name, shape, dtype, initializer, regularizer, trainable, constraint, partitioner, use_resource, synchronization, aggregation, **kwargs)
    575         synchronization=synchronization,
    576         aggregation=aggregation,
--> 577         caching_device=caching_device)
    578     if regularizer is not None:
    579       # TODO(fchollet): in the future, this should be handled at the

/opt/conda/lib/python3.7/site-packages/tensorflow/python/training/tracking/base.py in _add_variable_with_custom_getter(self, name, shape, dtype, initializer, getter, overwrite, **kwargs_for_getter)
    741         dtype=dtype,
    742         initializer=initializer,
--> 743         **kwargs_for_getter)
    744 
    745     # If we set an initializer and the variable processed it, tracking will not

/opt/conda/lib/python3.7/site-packages/tensorflow/python/keras/engine/base_layer_utils.py in make_variable(name, shape, dtype, initializer, trainable, caching_device, validate_shape, constraint, use_resource, collections, synchronization, aggregation, partitioner)
    139       synchronization=synchronization,
    140       aggregation=aggregation,
--> 141       shape=variable_shape if variable_shape else None)
    142 
    143 

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variables.py in __call__(cls, *args, **kwargs)
    257   def __call__(cls, *args, **kwargs):
    258     if cls is VariableV1:
--> 259       return cls._variable_v1_call(*args, **kwargs)
    260     elif cls is Variable:
    261       return cls._variable_v2_call(*args, **kwargs)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variables.py in _variable_v1_call(cls, initial_value, trainable, collections, validate_shape, caching_device, name, variable_def, dtype, expected_shape, import_scope, constraint, use_resource, synchronization, aggregation, shape)
    218         synchronization=synchronization,
    219         aggregation=aggregation,
--> 220         shape=shape)
    221 
    222   def _variable_v2_call(cls,

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variables.py in getter(**kwargs)
     64 
     65   def getter(**kwargs):
---> 66     return captured_getter(captured_previous, **kwargs)
     67 
     68   return getter

/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py in creator_with_resource_vars(next_creator, **kwargs)
   1765         kwargs["initial_value"] = kwargs["initial_value"].wrapped_value
   1766 
-> 1767       return self._create_variable(next_creator, **kwargs)
   1768 
   1769     def distributed_getter(getter, *args, **kwargs):

/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py in _create_variable(self, next_creator, **kwargs)
    670                                            tpu_values.TPUMirroredVariable,
    671                                            tpu_values.TPUSyncOnReadVariable,
--> 672                                            **kwargs)
    673 
    674   def _reduce_to(self, reduce_op, value, destinations, experimental_hints):

/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/values.py in create_mirrored_variable(strategy, real_mirrored_creator, mirrored_cls, sync_on_read_cls, **kwargs)
    692   # here.
    693   with tape.stop_recording():
--> 694     value_list = real_mirrored_creator(**kwargs)
    695     var_cls = sync_on_read_cls if is_sync_on_read else mirrored_cls
    696     result = var_cls(strategy, value_list, aggregation)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/tpu_strategy.py in _real_mirrored_creator(**kwargs)
    660 
    661           with context.device_policy(context.DEVICE_PLACEMENT_SILENT):
--> 662             v = next_creator(**kwargs)
    663 
    664           assert not isinstance(v, tpu_values.TPUMirroredVariable)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variables.py in <lambda>(**kwargs)
    196                         shape=None):
    197     """Call on Variable class. Useful to force the signature."""
--> 198     previous_getter = lambda **kwargs: default_variable_creator(None, **kwargs)
    199     for _, getter in ops.get_default_graph()._variable_creator_stack:  # pylint: disable=protected-access
    200       previous_getter = _make_getter(getter, previous_getter)

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variable_scope.py in default_variable_creator(next_creator, **kwargs)
   2596         synchronization=synchronization,
   2597         aggregation=aggregation,
-> 2598         shape=shape)
   2599   else:
   2600     return variables.RefVariable(

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/variables.py in __call__(cls, *args, **kwargs)
    261       return cls._variable_v2_call(*args, **kwargs)
    262     else:
--> 263       return super(VariableMetaclass, cls).__call__(*args, **kwargs)
    264 
    265 

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py in __init__(self, initial_value, trainable, collections, validate_shape, caching_device, name, dtype, variable_def, import_scope, constraint, distribute_strategy, synchronization, aggregation, shape)
   1432           aggregation=aggregation,
   1433           shape=shape,
-> 1434           distribute_strategy=distribute_strategy)
   1435 
   1436   def _init_from_args(self,

/opt/conda/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py in _init_from_args(self, initial_value, trainable, collections, caching_device, name, dtype, constraint, synchronization, aggregation, distribute_strategy, shape)
   1568                 name="initial_value", dtype=dtype)
   1569           if shape is not None:
-> 1570             if not initial_value.shape.is_compatible_with(shape):
   1571               raise ValueError(
1572                   "The initial value's shape (%s) is not compatible with "

/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/ops.py in shape(self)
   1063         # `_tensor_shape` is declared and defined in the definition of
   1064         # `EagerTensor`, in C.
-> 1065         self._tensor_shape = tensor_shape.TensorShape(self._shape_tuple())
   1066       except core._NotOkStatusException as e:
   1067         six.raise_from(core._status_to_exception(e.code, e.message), None)

RuntimeError: Error while creating shape
Facing the same issue on on GCP + TPU, same TF and TPU versions. <|||||>This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.
