tokenized df bug

data_measurements/dataset_statistics.py

@@ -455,7 +455,7 @@ class DatasetStatisticsCacheClass:
             self.vocab_counts_filtered_df = filter_vocab(self.vocab_counts_df)
         else:
             logs.info("Calculating vocab afresh")
-            if len(self.tokenized_df) == 0:
+            if self.tokenized_df is None:
                 self.tokenized_df = self.do_tokenization()
                 if save:
                     logs.info("Writing out.")