dataset = pd.read_csv(‘augmented_data.csv’)
randomize the dataset
dataset = dataset.sample(frac=1)
Only use 20% of the dataset for ease
mask = np.random.rand(len(dataset)) < 0.02
dataset = dataset[mask]
split the dataset into training, verification and validation
mask = np.random.rand(len(dataset)) < 0.8
train_val_data = dataset[mask]
test_data = dataset[~mask]
print(‘test data’, len(test_data))
mask = np.random.rand(len(train_val_data)) < 0.8
train_data = train_val_data[mask]
val_data = train_val_data[~mask]
print(‘train data’, len(train_data))
print(‘validation data’, len(val_data))
Load all the images for each dataset
IMG_DIM = (200,200)
train_imgs = []
files_read = 0
start = time.time()
for index, row in train_data.iterrows():
try:
train_imgs.append(img_to_array(load_img(‘padded_images/’ + row[‘image_name’] +’.jpg’, target_size=IMG_DIM)))
except:
train_imgs.append(img_to_array(load_img(‘augmented_images/’ + row[‘image_name’] +’.jpeg’, target_size=IMG_DIM)))
files_read += 1
so_far = time.time()
print_time_remaining(so_far, start, files_read, len(train_data))
val_imgs = []
files_read = 0
start = time.time()
for index, row in val_data.iterrows():
try:
val_imgs.append(img_to_array(load_img(‘padded_images/’ + row[‘image_name’] +’.jpg’, target_size=IMG_DIM)))
except:
val_imgs.append(img_to_array(load_img(‘augmented_images/’ + row[‘image_name’] +’.jpeg’, target_size=IMG_DIM)))
files_read += 1
so_far = time.time()
print_time_remaining(so_far, start, files_read, len(val_data))
test_imgs = []
files_read = 0
start = time.time()
for index, row in test_data.iterrows():
try:
test_imgs.append(img_to_array(load_img(‘padded_images/’ + row[‘image_name’] +’.jpg’, target_size=IMG_DIM)))
except:
test_imgs.append(img_to_array(load_img(‘augmented_images/’ + row[‘image_name’] +’.jpeg’, target_size=IMG_DIM)))
files_read += 1
so_far = time.time()
print_time_remaining(so_far, start, files_read, len(test_data))
y_train = torch.tensor(train_data[‘target’].values)
y_val = torch.tensor(val_data[‘target’].values)
y_test = torch.tensor(test_data[‘target’].values)
Convert to float
train_imgs = torch.from_numpy(np.array(train_imgs, dtype=np.float32)).float()
print(‘Train images scaled!’)
val_imgs  = torch.from_numpy(np.array(val_imgs, dtype=np.float32)).float()
print(‘Validation images scaled!’)
test_imgs  = torch.from_numpy(np.array(test_imgs, dtype=np.float32)).float()
print(‘Test images scaled!’)
divide the pixels by 255 to scale the pixels between 0 and 1
train_imgs /= 255
val_imgs /= 255
test_imgs /= 255
print(train_imgs.size())
train_imgs = train_imgs.transpose(2,4)
val_imgs = val_imgs.transpose(2,4)
test_imgs = test_imgs.transpose(2,4)
print(train_imgs.size())
Unsqueeze images
train_imgs = train_imgs.unsqueeze(0).to(device)
val_imgs = val_imgs.unsqueeze(0).to(device)
test_imgs = test_imgs.unsqueeze(0).to(device)
This is how i load the images btw. I don’t know if there is something wrong with this