title: How to create a confusion matrix for VGG16 image calssification (2 options) when using preprocessing.image_dataset_from_directory
body: <p>I have been trying to create a confusion matrix to test my data on from my VGG16 classification model (python 3.8, using Keras).</p>
  <p>I found a boiler plate based off of datagenerator. I tried leaving the test data as datagenerator, however then it always picks the first option. I tried converting my data coming in to data gen and ran into all sorts of problems.</p>
  <p>(edit 02AUG2021)
  Now I am trying to fit the confusion matrix to my preprocessing.image_dataset_from_directory and I get</p>
  <pre><code>ValueError: zero-dimensional arrays cannot be concatenated
  </code></pre>
  <p>Here is my code (the directory has been changed as I don't want it on the internet)
  (new code)</p>
  <pre><code>import tensorflow as tf
  import sys
  import os
  import numpy as np
  import pandas as pd
  import seaborn as sns
  import itertools
  import sklearn
  from tensorflow import keras
  from tensorflow.keras.preprocessing import image
  from tensorflow.keras.applications.vgg16 import preprocess_input, decode_predictions
  from tensorflow.keras import layers
  
  
  #Importing images
  train_ds = tf.keras.preprocessing.image_dataset_from_directory(
      'directory\\classification', 
      labels=&quot;inferred&quot;, 
      label_mode=&quot;binary&quot;, 
      class_names=None, 
      color_mode=&quot;rgb&quot;, 
      batch_size=16, 
      image_size=(227, 227), 
      shuffle=True, 
      seed=50, 
      validation_split=0.2, 
      subset=&quot;training&quot;, 
      interpolation=&quot;bilinear&quot;, 
      follow_links=False,  
  )
  
  val_ds = tf.keras.preprocessing.image_dataset_from_directory(
      'directory\\classification', 
      labels=&quot;inferred&quot;, 
      label_mode=&quot;binary&quot;, 
      class_names=None, 
      color_mode=&quot;rgb&quot;, 
      batch_size=16, 
      image_size=(227, 227), 
      shuffle=True, 
      seed=50, 
      validation_split=0.2, 
      subset=&quot;validation&quot;, 
      interpolation=&quot;bilinear&quot;, 
      follow_links=False,  
  )
  
  test_ds = tf.keras.preprocessing.image_dataset_from_directory(
      'directory\\test', 
      labels=&quot;inferred&quot;, 
      label_mode=&quot;binary&quot;, 
      class_names=None, 
      color_mode=&quot;rgb&quot;, 
      batch_size=16, 
      image_size=(227, 227), 
      shuffle=False, 
      seed=50, 
      validation_split=None, 
      subset=None, 
      interpolation=&quot;bilinear&quot;, 
      follow_links=False,  
  )
  
  print(&quot;Number of training samples: %d&quot; % tf.data.experimental.cardinality(train_ds))
  print(&quot;Number of validation samples: %d&quot; % tf.data.experimental.cardinality(val_ds))
  print(&quot;Number of test samples: %d&quot; % tf.data.experimental.cardinality(test_ds))
  
  # Visualise the data
  import matplotlib.pyplot as plt 
  
  plt.figure(figsize=(10, 10))
  for images, labels in train_ds.take(1):
      for i in range(9): 
          ax = plt.subplot(3, 3, i + 1) 
          plt.imshow(images[i].numpy().astype(&quot;uint8&quot;)) 
          plt.title(int(labels[i])) 
          plt.axis(&quot;off&quot;)
  
  # data augmentation
  data_augmentation = keras.Sequential(
      [layers.experimental.preprocessing.RandomFlip(&quot;horizontal_and_vertical&quot;), 
       layers.experimental.preprocessing.RandomRotation(0.2),])
  
  # Visualise
  plt.figure(figsize=(10, 10))
  for images, _ in train_ds.take(1):
      for i in range(9):
          augmented_images = data_augmentation(images)
          ax = plt.subplot(3, 3, i + 1)
          plt.imshow(augmented_images[0].numpy().astype(&quot;uint8&quot;))
          plt.axis(&quot;off&quot;)
  
  # buffered prefetching - last op on dataset, suggested 2xbatch, default batch =32
  train_ds = train_ds.prefetch(buffer_size=32)
  val_ds = val_ds.prefetch(buffer_size=32)
  test_ds = test_ds.prefetch(buffer_size=32)
  
  # build the model
  base_model = tf.keras.applications.VGG16(
      include_top=False,  # Do not include the ImageNet classifier at the top.
      weights=&quot;imagenet&quot;,   # Load weights pre-trained on ImageNet.
      input_tensor=None,
      input_shape=(227, 227, 3),
      pooling=max,
      classes=2,
      classifier_activation=&quot;softmax&quot;) 
  
  # Freeze the base_model
  base_model.trainable = False
  
  # Create new model on top
  inputs = keras.Input(shape=(227, 227, 3))
  x = data_augmentation(inputs)  # Apply random data augmentation (GPU method)
  
  # Pre-trained Imagenet weights requires that input be normalized
  # from (0, 255) to a range (-1., +1.), the normalization layer
  # does the following, outputs = (inputs - mean) / sqrt(var)
  norm_layer = keras.layers.experimental.preprocessing.Normalization()
  mean = np.array([127.5] * 3)
  var = mean ** 2
  # Scale inputs to [-1, +1]
  x = norm_layer(x)
  norm_layer.set_weights([mean, var])
  
  # The base model contains batchnorm layers. We want to keep them in inference mode
  # when we unfreeze the base model for fine-tuning, so we make sure that the
  # base_model is running in inference mode here.
  x = base_model(x, training=False)
  x = keras.layers.GlobalMaxPooling2D()(x)
  x = keras.layers.Dropout(0.2)(x)  # Regularize with dropout
  outputs = keras.layers.Dense(1)(x)
  model = keras.Model(inputs, outputs)
  
  model.summary()
  
  #train the top layer
  model.compile(
      optimizer=keras.optimizers.Adam(),
      loss=keras.losses.BinaryCrossentropy(from_logits=True),
      metrics=[keras.metrics.BinaryAccuracy()],
  )
  
  history = model.fit(train_ds, epochs=2, validation_data=val_ds)
  
  #Evaluate model
  validation_steps = 20
  
  loss0,accuracy0 = model.evaluate(val_ds, steps = validation_steps)
  
  print(&quot;loss: {:.2f}&quot;.format(loss0))
  print(&quot;accuracy: {:.2f}&quot;.format(accuracy0))
  
  #learning curves
  # accuracy
  plt.plot(history.history['binary_accuracy'])
  plt.plot(history.history['val_binary_accuracy'])
  plt.title('model accuracy')
  plt.ylabel('accuracy')
  plt.xlabel('epoch')
  plt.legend(['train', 'validation'], loc='upper left')
  plt.show()
  
  #loss
  plt.plot(history.history['loss'])
  plt.plot(history.history['val_loss'])
  plt.title('model loss')
  plt.ylabel('loss')
  plt.xlabel('epoch')
  plt.legend(['train', 'validation'], loc='upper left')
  plt.show()
  
  #confusion matrix
  # You need to iterate over the data, then you can collect predictions and true classes.
  predicted_probs = np.empty([])
  true_classes =  np.empty([])
  
  for images, labels in test_ds:
    predicted_probs = np.concatenate([predicted_probs,
                         model(images)])
    true_classes = np.concatenate([true_classes, labels.numpy()])
  
  # Since they are sigmoid outputs, you need to transform them into classes with a threshold, i.e 0.5 here:
  predicted_classes = [1 * (x[0]&gt;=0.5) for x in predicted_probs]
  
  # confusion matrix etc:
  conf_matrix = tf.math.confusion_matrix(true_classes, predicted_classes)
  print(conf_matrix) 
  </code></pre>
  <p>I am struggling. I know that there is an issue with the prefect dataset, but I don't know how to fix. I tried classes, class_names, labels. I can't find things to help.</p>
  <p>Any help would be greatly appreciated.</p>
  <p>PS: I have been attempting to get this working using data generators since there were examples. I have managed to get a confusion matrix working with this method.
  However, I am unsure about my conversion of predicted values to binary. I tried using argmax and it said everything was 0 (out of 0 or 1). I have found an alternate method to convert the binarized values and it appears to be working, but I don't understand it and I am worried I have cocked up.</p>
  <pre><code>import tensorflow as tf
  import sys
  import os
  import numpy as np
  import pandas as pd
  import seaborn as sns
  import itertools
  import sklearn
  from tensorflow import keras
  from tensorflow.keras.preprocessing import image
  from tensorflow.keras.applications.vgg16 import preprocess_input, decode_predictions
  from tensorflow.keras import layers
  from tensorflow.keras.preprocessing.image import ImageDataGenerator
  
  #Importing images
  datagen = ImageDataGenerator(validation_split=0.2)
  
  train_ds = datagen.flow_from_directory(
      'directory\\classification',
      target_size=(227, 227),
      batch_size=16,
      color_mode=&quot;rgb&quot;,
      subset=&quot;training&quot;,
      seed=50,
      class_mode=&quot;binary&quot;,
      shuffle=True)
  
  val_ds = datagen.flow_from_directory(
      'directory\\classification',
      target_size=(227, 227),
      batch_size=16,
      color_mode=&quot;rgb&quot;,
      subset=&quot;validation&quot;,
      seed=50,
      class_mode=&quot;binary&quot;,
      shuffle=True)
  
  test_generator = ImageDataGenerator()
  test_data_generator = test_generator.flow_from_directory(
      'directory\\test',   # Put your path here
      target_size=(227, 227),
      batch_size=16,
      color_mode=&quot;rgb&quot;,
      shuffle=False)
  
  
  # Visualise the data
  import matplotlib.pyplot as plt 
  
  
  # data augmentation
  data_augmentation = keras.Sequential(
      [layers.experimental.preprocessing.RandomFlip(&quot;horizontal_and_vertical&quot;), 
       layers.experimental.preprocessing.RandomRotation(0.2),])
  
  
  
  # build the model
  base_model = keras.applications.VGG16(
      include_top=False,  # Do not include the ImageNet classifier at the top.
      weights=&quot;imagenet&quot;,   # Load weights pre-trained on ImageNet.
      input_tensor=None,
      input_shape=(227, 227, 3),
      pooling=max,
      classes=2,
      classifier_activation=&quot;softmax&quot;) 
  
  # Freeze the base_model
  base_model.trainable = False
  
  # Create new model on top
  inputs = keras.Input(shape=(227, 227, 3))
  x = data_augmentation(inputs)  # Apply random data augmentation (GPU method)
  
  # Pre-trained Imagenet weights requires that input be normalized
  # from (0, 255) to a range (-1., +1.), the normalization layer
  # does the following, outputs = (inputs - mean) / sqrt(var)
  norm_layer = keras.layers.experimental.preprocessing.Normalization()
  mean = np.array([127.5] * 3)
  var = mean ** 2
  # Scale inputs to [-1, +1]
  x = norm_layer(x)
  norm_layer.set_weights([mean, var])
  
  # The base model contains batchnorm layers. We want to keep them in inference mode
  # when we unfreeze the base model for fine-tuning, so we make sure that the
  # base_model is running in inference mode here.
  x = base_model(x, training=False)
  x = keras.layers.GlobalMaxPooling2D()(x)
  x = keras.layers.Dropout(0.2)(x)  # Regularize with dropout
  outputs = keras.layers.Dense(1)(x)
  model = keras.Model(inputs, outputs)
  
  model.summary()
  
  #train the top layer
  model.compile(
      optimizer=keras.optimizers.Adam(),
      loss=keras.losses.BinaryCrossentropy(from_logits=True),
      metrics=[keras.metrics.BinaryAccuracy()],
  )
  
  history =model.fit_generator(train_ds, 
      steps_per_epoch=train_ds.samples / train_ds.batch_size, 
      epochs=34, 
      validation_data=val_ds,
      validation_steps=val_ds.samples / val_ds.batch_size,
      verbose=1)
  
  # Unfreeze the base_model. Note that it keeps running in inference mode
  # since we passed `training=False` when calling it. This means that
  # the batchnorm layers will not update their batch statistics.
  # This prevents the batchnorm layers from undoing all the training
  # we've done so far.
  
  # Fine tuning
  base_model.trainable = True
  model.summary()
  
  model.compile(
      optimizer=keras.optimizers.Adam(1e-5),  # Low learning rate
      loss=keras.losses.BinaryCrossentropy(from_logits=True),
      metrics=[keras.metrics.BinaryAccuracy()],
  )
  
  # fit_generator() is normally useful if you cannot fit your data into your main → ResourceExhaustedError:
  history = model.fit_generator(train_ds, 
      steps_per_epoch=train_ds.samples / train_ds.batch_size, 
      epochs=17 , 
      validation_data=val_ds,
      validation_steps=val_ds.samples / val_ds.batch_size,
      verbose=1)
  
  
  #Evaluate model
  validation_steps = 20
  
  loss0,accuracy0 = model.evaluate(val_ds, steps = validation_steps)
  
  print(&quot;loss: {:.2f}&quot;.format(loss0))
  print(&quot;accuracy: {:.2f}&quot;.format(accuracy0))
  
  #learning curves
  # accuracy
  plt.plot(history.history['binary_accuracy'])
  plt.plot(history.history['val_binary_accuracy'])
  plt.title('model accuracy')
  plt.ylabel('accuracy')
  plt.xlabel('epoch')
  plt.legend(['train', 'validation'], loc='upper left')
  plt.show()
  
  #loss
  plt.plot(history.history['loss'])
  plt.plot(history.history['val_loss'])
  plt.title('model loss')
  plt.ylabel('loss')
  plt.xlabel('epoch')
  plt.legend(['train', 'validation'], loc='upper left')
  plt.show()
  
  #confusion matrix
  #1.Predict classes
  from tensorflow.keras.preprocessing.image import ImageDataGenerator
  
  test_steps_per_epoch = np.math.ceil(test_data_generator.samples / test_data_generator.batch_size)
  
  predictions = model.predict_generator(test_data_generator, steps=test_steps_per_epoch)
  
  # Get most likely class
  predicted_classes = [1 * (x[0]&gt;=0.5) for x in predictions]
  
  # 2.Get ground-truth classes and class-labels
  true_classes = test_data_generator.classes
  class_labels = list(test_data_generator.class_indices.keys()) 
  
  # 3. Use scikit-learn to get statistics
  from sklearn.metrics import confusion_matrix,classification_report
  
  print(class_labels)
  
  print(confusion_matrix(test_data_generator.classes, predicted_classes))
  
  report = sklearn.metrics.classification_report(true_classes, predicted_classes, target_names=class_labels)
  print(report) 
  </code></pre>
  <p>This returns a matrix of (rows = true, columns = predicted)</p>
  <pre><code>[[3998     2]
   [   5  3995]]
  </code></pre>