text_plain_output_1.png
from datasets import Dataset

import pandas as pd

df1 = pd.read_csv('../input/newspaper-articles-csv-ocr/ocr/df_valid.csv')
df1['imgdir'] = '../input/newspaper-articles/articles/validation'
df2 = pd.read_csv('../input/newspaper-articles-csv-ocr/ocr/df_train1.csv')
df2['imgdir'] = '../input/newspaper-articles/articles/training'
df3 = pd.read_csv('../input/newspaper-articles-csv-ocr/ocr/df_train2.csv')
df3['imgdir'] = '../input/newspaper-articles/articles/training'
df4 = pd.read_csv('../input/newspaper-articles-csv-ocr/ocr/df_train3.csv')
df4['imgdir'] = '../input/newspaper-articles/articles/training'
df = pd.concat([df1, df2, df3, df4])

df = df[df.text.apply(lambda x: len(str(x)) > 1)]
df = df.reset_index(drop=True)

df.groupby(['label']).size()

df_meta = df
df_meta = df_meta.drop_duplicates(subset=['image_id'])
df_meta = df_meta.reset_index(drop=True)
df_meta['id'] = df_meta.index
df_meta['image_path'] = df_meta['imgdir'] + '/' + df_meta['image_id']
df_meta = df_meta[['image_id', 'image_path']]
df_meta['split'] = 'train'

ratio_split = int(len(df_meta) * 0.2)
df_meta.loc[df_meta.index[-ratio_split:], 'split'] = 'test'
df_meta

from datasets import Dataset
train_meta = df_meta[df_meta.split == 'train']
train_meta = train_meta.drop(columns=['split'])
train_meta = train_meta.reset_index(drop=True)
dataset_train = Dataset.from_pandas(train_meta)
print(dataset_train)
test_meta = df_meta[df_meta.split == 'test']
test_meta = test_meta.drop(columns=['split'])
test_meta = test_meta.reset_index(drop=True)
dataset_test = Dataset.from_pandas(test_meta)
print(dataset_test)