Modularization and caching of text length widget

app.py

@@ -177,15 +177,7 @@ def show_column(dstats, ds_name_to_dict, show_embeddings, column_id, use_cache=T
     logs.info("showing general stats")
     st_utils.expander_general_stats(dstats, column_id)
     st_utils.expander_label_distribution(dstats.fig_labels, column_id)
-    st_utils.expander_text_lengths(
-        dstats.tokenized_df,
-        dstats.fig_tok_length,
-        dstats.avg_length,
-        dstats.std_length,
-        OUR_TEXT_FIELD,
-        LENGTH_FIELD,
-        column_id,
-    )
+    st_utils.expander_text_lengths(dstats, column_id)
     st_utils.expander_text_duplicates(dstats, column_id)
 
     # We do the loading of these after the others in order to have some time
@@ -197,8 +189,7 @@ def show_column(dstats, ds_name_to_dict, show_embeddings, column_id, use_cache=T
     )
     available_terms = npmi_stats.get_available_terms()
     st_utils.npmi_widget(
-        column_id, available_terms, npmi_stats, _MIN_VOCAB_COUNT, use_cache=use_cache
-    )
+        column_id, available_terms, npmi_stats, _MIN_VOCAB_COUNT)
     logs.info("showing zipf")
     st_utils.expander_zipf(dstats.z, dstats.zipf_fig, column_id)
     if show_embeddings:
@@ -222,7 +213,7 @@ def main():
     compare_mode = st.sidebar.checkbox("Comparison mode")
 
     # When not doing new development, use the cache.
-    use_cache = False
+    use_cache = True
     show_embeddings = st.sidebar.checkbox("Show embeddings")
     # List of datasets for which embeddings are hard to compute:
 

data_measurements/dataset_statistics.py

@@ -197,6 +197,7 @@ class DatasetStatisticsCacheClass:
         # Tokenized text
         self.tokenized_df = None
         # save sentence length histogram in the class so it doesn't ge re-computed
+        self.length_df = None
         self.fig_tok_length = None
         # Data Frame version of self.label_dset
         self.label_df = None
@@ -262,6 +263,8 @@ class DatasetStatisticsCacheClass:
         self.text_dset_fid = pjoin(self.cache_path, "text_dset")
         self.tokenized_df_fid = pjoin(self.cache_path, "tokenized_df.feather")
         self.label_dset_fid = pjoin(self.cache_path, "label_dset")
+        self.length_df_fid = pjoin(self.cache_path, "length_df.feather")
+        self.length_stats_fid = pjoin(self.cache_path, "length_stats.json")
         self.vocab_counts_df_fid = pjoin(self.cache_path, "vocab_counts.feather")
         self.general_stats_fid = pjoin(self.cache_path, "general_stats_dict.json")
         self.dup_counts_df_fid = pjoin(
@@ -317,24 +320,66 @@ class DatasetStatisticsCacheClass:
 
 
     def load_or_prepare_text_lengths(self, save=True):
-        # TODO: Everything here can be read from cache; it's in a transitory
-        # state atm where just the fig is cached.  Clean up.
-        if self.use_cache and exists(self.fig_tok_length_fid):
+        """
+        The text length widget relies on this function, which provides
+        a figure of the text lengths, some text length statistics, and
+        a text length dataframe to peruse.
+        Args:
+            save:
+        Returns:
+
+        """
+        # Text length figure
+        if (self.use_cache and exists(self.fig_tok_length_fid)):
             self.fig_tok_length = read_plotly(self.fig_tok_length_fid)
+        else:
+            self.prepare_fig_text_lengths()
+            if save:
+                write_plotly(self.fig_tok_length, self.fig_tok_length_fid)
+
+        # Text length dataframe
+        if self.use_cache and exists(self.length_df_fid):
+            self.length_df = feather.read_feather(self.length_df_fid)
+        else:
+            self.prepare_length_df()
+            if save:
+                write_df(self.length_df, self.length_df_fid)
+
+        # Text length stats.
+        if self.use_cache and exists(self.length_stats_fid):
+            with open(self.length_stats_fid, "r") as f:
+                self.length_stats_dict = json.load(f)
+            self.avg_length = self.length_stats_dict["avg length"]
+            self.std_length = self.length_stats_dict["std length"]
+        else:
+            self.prepare_text_length_stats()
+            if save:
+                write_json(self.length_stats_dict, self.length_stats_fid)
+
+    def prepare_length_df(self):
         if self.tokenized_df is None:
             self.tokenized_df = self.do_tokenization()
-        self.tokenized_df[LENGTH_FIELD] = self.tokenized_df[TOKENIZED_FIELD].apply(len)
-        self.avg_length = round(
-            sum(self.tokenized_df[self.our_length_field])
-            / len(self.tokenized_df[self.our_length_field]),
-            1,
-        )
-        self.std_length = round(
-            statistics.stdev(self.tokenized_df[self.our_length_field]), 1
+        self.tokenized_df[LENGTH_FIELD] = self.tokenized_df[
+            TOKENIZED_FIELD].apply(len)
+        self.length_df = self.tokenized_df[
+            [LENGTH_FIELD, OUR_TEXT_FIELD]].sort_values(
+            by=[LENGTH_FIELD], ascending=True
         )
-        self.fig_tok_length = make_fig_lengths(self.tokenized_df, self.our_length_field)
-        if save:
-            write_plotly(self.fig_tok_length, self.fig_tok_length_fid)
+
+    def prepare_text_length_stats(self):
+        if self.tokenized_df is None or LENGTH_FIELD not in self.tokenized_df.columns:
+            self.prepare_length_df()
+        avg_length = sum(self.tokenized_df[LENGTH_FIELD])/len(self.tokenized_df[LENGTH_FIELD])
+        self.avg_length = round(avg_length, 1)
+        std_length = statistics.stdev(self.tokenized_df[LENGTH_FIELD])
+        self.std_length = round(std_length, 1)
+        self.length_stats_dict = {"avg length": self.avg_length,
+                                  "std length": self.std_length}
+
+    def prepare_fig_text_lengths(self):
+        if self.tokenized_df is None or LENGTH_FIELD not in self.tokenized_df.columns:
+            self.prepare_length_df()
+        self.fig_tok_length = make_fig_lengths(self.tokenized_df, LENGTH_FIELD)
 
     def load_or_prepare_embeddings(self, save=True):
         if self.use_cache and exists(self.node_list_fid) and exists(self.fig_tree_fid):

data_measurements/streamlit_utils.py

@@ -147,13 +147,7 @@ def expander_label_distribution(fig_labels, column_id):
             st.markdown("No labels were found in the dataset")
 
 
-def expander_text_lengths(
-    tokenized_df,
-    fig_tok_length,
-    avg_length,
-    std_length,
-    text_field_name,
-    length_field_name,
+def expander_text_lengths(dstats,
     column_id,
 ):
     _TEXT_LENGTH_CAPTION = (
@@ -165,31 +159,28 @@ def expander_text_lengths(
             "Below, you can see how the lengths of the text instances in your dataset are distributed."
         )
         st.markdown(
-            "Any unexpected peaks or valleys in the distribution may help to identify data instances you want to remove or augment."
+            "Any unexpected peaks or valleys in the distribution may help to identify instances you want to remove or augment."
         )
         st.markdown(
             "### Here is the relative frequency of different text lengths in your dataset:"
         )
-        st.plotly_chart(fig_tok_length, use_container_width=True)
-        data = tokenized_df[[length_field_name, text_field_name]].sort_values(
-            by=["length"], ascending=True
-        )
+        st.plotly_chart(dstats.fig_tok_length, use_container_width=True)
         st.markdown(
             "The average length of text instances is **"
-            + str(avg_length)
+            + str(dstats.avg_length)
             + " words**, with a standard deviation of **"
-            + str(std_length)
+            + str(dstats.std_length)
             + "**."
         )
 
         start_id_show_lengths = st.slider(
             f"Show the shortest sentences{column_id} starting at:",
             0,
-            len(data["length"].unique()),
+            len(dstats.length_df["length"].unique()),
             value=0,
             step=1,
         )
-        st.dataframe(data[data["length"] == start_id_show_lengths].set_index("length"))
+        st.dataframe(dstats.length_df[dstats.length_df["length"] == start_id_show_lengths].set_index("length"))
 
 
 ### Third, use a sentence embedding model
@@ -404,7 +395,7 @@ with an ideal α value of 1."""
 
 
 ### Finally finally finally, show nPMI stuff.
-def npmi_widget(column_id, available_terms, npmi_stats, min_vocab, use_cache=False):
+def npmi_widget(column_id, available_terms, npmi_stats, min_vocab):
     """
     Part of the main app, but uses a user interaction so pulled out as its own f'n.
     :param use_cache: