<file_sep># ChunkProcessingContinuation - Easily continue the processing on large files in pandas without starting all over when processing stops halfway!
# 
#
#


With ChunkProcessingContinuation, you can easily continue the processing of large files or dataframes in pandas without having to start the process all over when your system goes off or the process stops unexpectedly. Just specify the path of the original file, the lastly processed one and that where every processed chunks are being stored.
<file_sep>from large_file_chunk_processing_continuation.dataframe_continue_class import Continue<file_sep>import pandas as pd

class Continue:

    def __init__(
        self,
        path_old=None,
        path_new=None,
        path_init=None,
        chunk_path=None,
        chunk_processing=None,
        chunksize=None
    ):
        self.path_old = path_old
        self.path_new = path_new
        self.path_init = path_init
        self.chunk_path = chunk_path
        self.chunk_processing = chunk_processing
        self.chunksize = chunksize

    def reader(self, path):
        if path.endswith(".csv"):
            df = pd.read_csv(path)
        else:
            if path.endswith(".xls") or path.endswith(".xlsx"):
                df = pd.read_excel(path)

        return df

    
    def get_leftover(self):
        df_old = self.reader(self.path_old)
        df_new = self.reader(self.path_new)

        df_iterator = df_old[df_new.shape[0]:]
        df_iterator.to_csv(self.path_init, index=False)
        df_iterator = pd.read_csv(self.path_init, iterator=True, chunksize=self.chunksize)

        return df_iterator, df_new
    
    def odd(self, num):
        if num%2 == 1:
            return 1
        else:
            return 0

    

    def main_process(self):
        df_iterator, df_new = self.get_leftover()

        chunk_list = [df_new]
        i = 1
        for data_chunk in df_iterator:
            
            processed_chunk = self.chunk_processing(data_chunk)
            chunk_list.append(processed_chunk)


            if len(chunk_list)>1:
                processed_chunk = pd.concat(chunk_list)
                processed_chunk.to_excel(self.chunk_path.format("checkpoint_chunk_"+str(self.odd(i))), index=False)
            i += 1
        return "Done"

if __name__ == "__main__":
    main_process()