We have a SLURM batch file that fails with TF2 and Keras, and also fails when called directly on a node that has a GPU. Here is the Python script contents:
from datetime import date
import numpy as np
import matplotlib.pyplot as plt

import pandas as pd
from sklearn.decomposition import PCA
from sklearn.linear_model import LinearRegression
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense, SimpleRNN
from keras.optimizers import adam
from keras.layers import Dropout
from tensorflow.keras.callbacks import Callback, EarlyStopping
from sklearn.preprocessing import StandardScaler
from datetime import datetime, timedelta
from sklearn.metrics import r2_score, mean_squared_error, accuracy_score
from keras.layers.core import Dense, Dropout, Activation
from keras.layers.recurrent import LSTM
from keras.models import load_model
from keras.callbacks import EarlyStopping, ModelCheckpoint
import warnings
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = "3"
warnings.filterwarnings('ignore')
import tensorflow as tf
import logging
logging.getLogger('tesorflow').setLevel(logging.FATAL)
delay = 252
window = 60
factor = 15
K = 8.4
sbo = 1.25
sso = 1.25
sbc = 0.75
ssc = 0.5
r = 0.02
tran_cost = 0.0002
leverage = 1.0
start_val = 100
bo = 1
so = -1
X_pd=pd.read_pickle('./data/X_pd.pkl')
X = pd.DataFrame(columns=range(0, window))
Y = []
for tag in X_pd.columns[:1]:
    # i=0 ....len(X_pd.index)-window
    for i in range(0, len(X_pd.index) - window):
        X_example = X_pd.loc[i:i + window - 1][tag].values

        X= X.append(pd.Series(X_example), ignore_index=True)
        Y.append(X_pd.loc[i + window][tag])
    print('done %s stocks' % (tag))
Y=pd.DataFrame(Y)
#normalization
SS = StandardScaler()
features = SS.fit_transform(X.values)
X=features
X=pd.DataFrame(X)
#LSTM model
def trainLSTMModel(layers, neurons, d):
    model = Sequential()

    model.add(LSTM(neurons[0], input_shape=(layers[1], layers[2]), return_sequences=False,activation='relu'))
    #model.add(Dropout(d))

    #model.add(LSTM(neurons[1], input_shape=(layers[1], layers[2]), return_sequences=False))
    #model.add(Dropout(d))

    #model.add(Dense(neurons[2], kernel_initializer="uniform", activation='relu'))
    model.add(Dense(neurons[3], kernel_initializer="uniform", activation='relu'))
    optimizer=adam(learning_rate=0.001)
    #adam = Adam(decay=0.2)
    # predict up and down
    # model.compile(optimizer="adam", loss="binary_crossentropy", metrics=['accuracy'])
    model.compile(loss='mse', optimizer=optimizer)
    model.summary()
    return model
length=X.shape[0]
X=np.array(X)
Y=np.array(Y)
time_step = 60
d = 0.3
output=1
shape = [length,time_step, output] # feature, window, output
neurons = [64, 64, 32, 1]
epochs = 100
batch_size=10000
model = trainLSTMModel(shape, neurons, d)
#shape from [samples, timesteps] into [samples, timesteps, features]
n_features = 1
X = X.reshape((X.shape[0], X.shape[1], n_features))
gpu_no = 0
with tf.device('/gpu:' + str(gpu_no)):
#    sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=True))
#    keras.backend.set_session(sess)

    print('model_manager: running tensorflow version: ' + tf.__version__)
    print('model_manager: will attempt to run on ' + '/gpu:' + str(gpu_no))
    model.fit(X, Y, epochs=epochs, verbose=2,batch_size=batch_size)

The log shows this:
Loading requirement: cuda10.1/toolkit/10.1.243
Loading cm-ml-python3deps/3.3.0
  Loading requirement: gcc5/5.5.0 python36
Loading tensorflow2-py36-cuda10.1-gcc/2.0.0
  Loading requirement: ml-pythondeps-py36-cuda10.1-gcc/3.3.0
    openblas/dynamic/0.2.20 hdf5_18/1.8.20 keras-py36-cuda10.1-gcc/2.3.1
    protobuf3-gcc/3.8.0 nccl2-cuda10.1-gcc/2.7.8
Loading openmpi/cuda/64/3.1.4
  Loading requirement: hpcx/2.4.0
2021-08-18 11:11:43.064175: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1

2021-08-18 11:18:08.026219: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1
2021-08-18 11:18:08.031771: E tensorflow/stream_executor/cuda/cuda_driver.cc:318] failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error
2021-08-18 11:18:08.031811: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:169] retrieving CUDA diagnostic information for host: node001
2021-08-18 11:18:08.031819: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:176] hostname: node001
2021-08-18 11:18:08.031921: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:200] libcuda reported version is: 460.73.1
2021-08-18 11:18:08.031958: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:204] kernel reported version is: 460.73.1
2021-08-18 11:18:08.031966: I tensorflow/stream_executor/cuda/cuda_diagnostics.cc:310] kernel version seems to match DSO: 460.73.1
2021-08-18 11:18:08.032266: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F
Using TensorFlow backend.
done A stocks
Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
lstm_1 (LSTM)                (None, 64)                16896
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 65
=================================================================
Total params: 16,961
Trainable params: 16,961
Non-trainable params: 0
_________________________________________________________________
model_manager: running tensorflow version: 2.0.0
model_manager: will attempt to run on /gpu:0
Traceback (most recent call last):
  File "stocks.py", line 99, in <module>
    model.fit(X, Y, epochs=epochs, verbose=2,batch_size=batch_size)
  File "/cm/shared/apps/keras-py36-cuda10.1-gcc/2.3.1/lib/python3.6/site-packages/keras/engine/training.py", line 1213, in fit
    self._make_train_function()
  File "/cm/shared/apps/keras-py36-cuda10.1-gcc/2.3.1/lib/python3.6/site-packages/keras/engine/training.py", line 316, in _make_train_function
    loss=self.total_loss)
  File "/cm/shared/apps/keras-py36-cuda10.1-gcc/2.3.1/lib/python3.6/site-packages/keras/legacy/interfaces.py", line 91, in wrapper
    return func(*args, **kwargs)
  File "/cm/shared/apps/keras-py36-cuda10.1-gcc/2.3.1/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 75, in symbolic_fn_wrapper
    return func(*args, **kwargs)
  File "/cm/shared/apps/keras-py36-cuda10.1-gcc/2.3.1/lib/python3.6/site-packages/keras/optimizers.py", line 519, in get_updates
    for (i, p) in enumerate(params)]
  File "/cm/shared/apps/keras-py36-cuda10.1-gcc/2.3.1/lib/python3.6/site-packages/keras/optimizers.py", line 519, in <listcomp>
    for (i, p) in enumerate(params)]
  File "/cm/shared/apps/keras-py36-cuda10.1-gcc/2.3.1/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 963, in zeros
    v = tf.zeros(shape=shape, dtype=dtype, name=name)
  File "/cm/shared/apps/tensorflow2-py36-cuda10.1-gcc/2.0.0/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py", line 2349, in zeros
    output = _constant_if_small(zero, shape, dtype, name)
  File "/cm/shared/apps/tensorflow2-py36-cuda10.1-gcc/2.0.0/lib/python3.6/site-packages/tensorflow_core/python/ops/array_ops.py", line 2307, in _constant_if_small
    return constant(value, shape=shape, dtype=dtype, name=name)
  File "/cm/shared/apps/tensorflow2-py36-cuda10.1-gcc/2.0.0/lib/python3.6/site-packages/tensorflow_core/python/framework/constant_op.py", line 227, in constant
    allow_broadcast=True)
  File "/cm/shared/apps/tensorflow2-py36-cuda10.1-gcc/2.0.0/lib/python3.6/site-packages/tensorflow_core/python/framework/constant_op.py", line 235, in _constant_impl
    t = convert_to_eager_tensor(value, ctx, dtype)
  File "/cm/shared/apps/tensorflow2-py36-cuda10.1-gcc/2.0.0/lib/python3.6/site-packages/tensorflow_core/python/framework/constant_op.py", line 96, in convert_to_eager_tensor
    return ops.EagerTensor(value, ctx.device_name, dtype)
RuntimeError: /job:localhost/replica:0/task:0/device:GPU:0 unknown device.

Why is the script not seeing the GPU?