feat: Add data cleaning method

.idea/workspace.xml

@@ -0,0 +1,95 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AutoImportSettings">
+    <option name="autoReloadType" value="SELECTIVE" />
+  </component>
+  <component name="ChangeListManager">
+    <list default="true" id="72819ac5-9a78-4a8a-aab7-199c2790b503" name="Changes" comment="feat: Attempt to get time-series model to work with our data">
+      <change beforePath="$PROJECT_DIR$/app.py" beforeDir="false" afterPath="$PROJECT_DIR$/app.py" afterDir="false" />
+    </list>
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="ComposerSettings">
+    <execution />
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="MarkdownSettingsMigration">
+    <option name="stateVersion" value="1" />
+  </component>
+  <component name="ProjectId" id="2TDHetankSPisemgNRZBEqcbHLu" />
+  <component name="ProjectViewState">
+    <option name="autoscrollFromSource" value="true" />
+    <option name="autoscrollToSource" value="true" />
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent">{
+  &quot;keyToString&quot;: {
+    &quot;RunOnceActivity.OpenProjectViewOnStart&quot;: &quot;true&quot;,
+    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
+    &quot;RunOnceActivity.laravel-idea.search-composer-json&quot;: &quot;true&quot;,
+    &quot;WebServerToolWindowFactoryState&quot;: &quot;false&quot;,
+    &quot;last_opened_file_path&quot;: &quot;//wsl$/Ubuntu/home/zack/code/timeseries-anomaly-detection-autoencoders&quot;,
+    &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
+    &quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
+    &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
+    &quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
+    &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
+  }
+}</component>
+  <component name="RecentsManager">
+    <key name="CopyFile.RECENT_KEYS">
+      <recent name="\\wsl$\Ubuntu\home\zack\code\timeseries-anomaly-detection-autoencoders" />
+    </key>
+    <key name="MoveFile.RECENT_KEYS">
+      <recent name="\\wsl$\Ubuntu\home\zack\code\timeseries-anomaly-detection-autoencoders" />
+    </key>
+  </component>
+  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="72819ac5-9a78-4a8a-aab7-199c2790b503" name="Changes" comment="" />
+      <created>1690570831325</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1690570831325</updated>
+      <workItem from="1690570831453" duration="3175000" />
+      <workItem from="1690584980736" duration="631000" />
+      <workItem from="1690595464187" duration="1660000" />
+      <workItem from="1690814760676" duration="5257000" />
+    </task>
+    <task id="LOCAL-00001" summary="feat: Attempt to get time-series model to work with our data">
+      <option name="closed" value="true" />
+      <created>1690572412668</created>
+      <option name="number" value="00001" />
+      <option name="presentableId" value="LOCAL-00001" />
+      <option name="project" value="LOCAL" />
+      <updated>1690572412668</updated>
+    </task>
+    <option name="localTasksCounter" value="2" />
+    <servers />
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+  <component name="Vcs.Log.Tabs.Properties">
+    <option name="TAB_STATES">
+      <map>
+        <entry key="MAIN">
+          <value>
+            <State />
+          </value>
+        </entry>
+      </map>
+    </option>
+  </component>
+  <component name="VcsManagerConfiguration">
+    <MESSAGE value="feat: Attempt to get time-series model to work with our data" />
+    <option name="LAST_COMMIT_MESSAGE" value="feat: Attempt to get time-series model to work with our data" />
+  </component>
+</project>

app.py

@@ -58,10 +58,27 @@ def plot_anomalies(df_test_value, data, anomalies):
     ax.set_ylabel("Value")
     ax.set_title("Anomalous Data Points")
     return fig
-                                      
+
+def clean_data(df):
+    # Convert "Date" and "Hour" columns into datetime format
+    df["timestamp"] = pd.to_datetime(df["Date"] + ' ' + df["Hour"].astype(str) + ":00:00")
+
+    # Keep only necessary columns
+    df = df[["timestamp", "Hourly_Labor_Hours_Total"]]
+
+    # Rename column
+    df.rename(columns={"Hourly_Labor_Hours_Total": "value"}, inplace=True)
+
+    return df
+
 def master(file):
     # read file
-    data = pd.read_csv(file, parse_dates=True, index_col="timestamp")
+    data = pd.read_csv(file)
+
+    # clean data
+    data = clean_data(data)
+    data.set_index("timestamp", inplace=True)
+
     df_test_value = normalize_data(data)
     # plot input test data
     plot1 = plot_test_data(df_test_value)