Spaces:
Sleeping
Sleeping
devjas1
commited on
Commit
·
9d0759c
1
Parent(s):
2fb5cb5
(FEAT): Add batch processing utilities for multi-file uploads
Browse files- Implement `create_batch_uploader` to handle batch file uploads.
- Add `process_multiple_files` for processing multiple files in batch mode.
- Include `display_batch_results` to render batch processing results in the UI.
- Enhance error handling for batch operations with `safe_execute`.
- Improve user experience with streamlined batch file management and result visualization.
- utils/multifile.py +93 -60
utils/multifile.py
CHANGED
@@ -3,32 +3,33 @@ Handles multiple file uploads and iterative processing."""
|
|
3 |
|
4 |
from typing import List, Dict, Any, Tuple, Optional
|
5 |
import time
|
6 |
-
import streamlit as st
|
7 |
-
import numpy as np
|
8 |
|
9 |
from .preprocessing import resample_spectrum
|
10 |
from .errors import ErrorHandler, safe_execute
|
11 |
from .results_manager import ResultsManager
|
12 |
from .confidence import calculate_softmax_confidence
|
13 |
|
|
|
14 |
def parse_spectrum_data(text_content: str, filename: str = "unknown") -> Tuple[np.ndarray, np.ndarray]:
|
15 |
"""
|
16 |
Parse spectrum data from text content
|
17 |
-
|
18 |
Args:
|
19 |
text_content: Raw text content of the spectrum file
|
20 |
filename: Name of the file for error reporting
|
21 |
-
|
22 |
Returns:
|
23 |
Tuple of (x_values, y_values) as numpy arrays
|
24 |
-
|
25 |
Raises:
|
26 |
ValueError: If the data cannot be parsed
|
27 |
"""
|
28 |
try:
|
29 |
lines = text_content.strip().split('\n')
|
30 |
|
31 |
-
|
32 |
data_lines = []
|
33 |
for line in lines:
|
34 |
line = line.strip()
|
@@ -38,39 +39,52 @@ def parse_spectrum_data(text_content: str, filename: str = "unknown") -> Tuple[n
|
|
38 |
if not data_lines:
|
39 |
raise ValueError("No data lines found in file")
|
40 |
|
41 |
-
|
42 |
x_vals, y_vals = [], []
|
43 |
|
44 |
for i, line in enumerate(data_lines):
|
45 |
try:
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
51 |
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
|
56 |
-
|
57 |
-
|
58 |
|
59 |
-
|
60 |
-
|
|
|
61 |
|
62 |
-
|
63 |
-
|
64 |
-
|
|
|
|
|
|
|
|
|
|
|
65 |
|
66 |
-
|
67 |
-
raise ValueError(f"Insufficient data points ({len(x_vals)}). Need at least 10 points.")
|
68 |
|
69 |
-
return np.array(x_vals), np.array(y_vals)
|
70 |
-
|
71 |
except Exception as e:
|
72 |
raise ValueError(f"Failed to parse spectrum data: {str(e)}")
|
73 |
|
|
|
74 |
def process_single_file(
|
75 |
filename: str,
|
76 |
text_content: str,
|
@@ -81,7 +95,7 @@ def process_single_file(
|
|
81 |
) -> Optional[Dict[str, Any]]:
|
82 |
"""
|
83 |
Process a single spectrum file
|
84 |
-
|
85 |
Args:
|
86 |
filename: Name of the file
|
87 |
text_content: Raw text content
|
@@ -89,15 +103,15 @@ def process_single_file(
|
|
89 |
load_model_func: Function to load the model
|
90 |
run_inference_func: Function to run inference
|
91 |
label_file_func: Function to extract ground truth label
|
92 |
-
|
93 |
Returns:
|
94 |
Dictionary with processing results or None if failed
|
95 |
"""
|
96 |
start_time = time.time()
|
97 |
|
98 |
try:
|
99 |
-
|
100 |
-
|
101 |
parse_spectrum_data,
|
102 |
text_content,
|
103 |
filename,
|
@@ -105,11 +119,13 @@ def process_single_file(
|
|
105 |
show_error=False
|
106 |
)
|
107 |
|
108 |
-
if not success:
|
109 |
return None
|
110 |
|
111 |
-
|
112 |
-
|
|
|
|
|
113 |
resample_spectrum,
|
114 |
x_raw,
|
115 |
y_raw,
|
@@ -118,11 +134,13 @@ def process_single_file(
|
|
118 |
show_error=False
|
119 |
)
|
120 |
|
121 |
-
if not success:
|
122 |
return None
|
123 |
|
124 |
-
|
125 |
-
|
|
|
|
|
126 |
run_inference_func,
|
127 |
y_resampled,
|
128 |
model_choice,
|
@@ -130,27 +148,31 @@ def process_single_file(
|
|
130 |
show_error=False
|
131 |
)
|
132 |
|
133 |
-
if not success or
|
134 |
-
ErrorHandler.log_error(
|
|
|
135 |
return None
|
136 |
|
137 |
-
|
|
|
|
|
138 |
if logits is not None:
|
139 |
-
probs_np, max_confidence, confidence_level, confidence_emoji = calculate_softmax_confidence(
|
|
|
140 |
else:
|
141 |
probs_np = np.array([])
|
142 |
max_confidence = 0.0
|
143 |
confidence_level = "LOW"
|
144 |
confidence_emoji = "🔴"
|
145 |
|
146 |
-
|
147 |
try:
|
148 |
ground_truth = label_file_func(filename)
|
149 |
ground_truth = ground_truth if ground_truth >= 0 else None
|
150 |
except Exception:
|
151 |
ground_truth = None
|
152 |
|
153 |
-
|
154 |
label_map = {0: "Stable (Unweathered)", 1: "Weathered (Degraded)"}
|
155 |
predicted_class = label_map.get(prediction, f"Unknown ({prediction})")
|
156 |
|
@@ -183,6 +205,7 @@ def process_single_file(
|
|
183 |
"processing_time": time.time() - start_time
|
184 |
}
|
185 |
|
|
|
186 |
def process_multiple_files(
|
187 |
uploaded_files: List,
|
188 |
model_choice: str,
|
@@ -193,7 +216,7 @@ def process_multiple_files(
|
|
193 |
) -> List[Dict[str, Any]]:
|
194 |
"""
|
195 |
Process multiple uploaded files
|
196 |
-
|
197 |
Args:
|
198 |
uploaded_files: List of uploaded file objects
|
199 |
model_choice: Selected model name
|
@@ -201,7 +224,7 @@ def process_multiple_files(
|
|
201 |
run_inference_func: Function to run inference
|
202 |
label_file_func: Function to extract ground truth label
|
203 |
progress_callback: Optional callback to update progress
|
204 |
-
|
205 |
Returns:
|
206 |
List of processing results
|
207 |
"""
|
@@ -215,11 +238,12 @@ def process_multiple_files(
|
|
215 |
progress_callback(i, total_files, uploaded_file.name)
|
216 |
|
217 |
try:
|
218 |
-
|
219 |
raw = uploaded_file.read()
|
220 |
-
text_content = raw.decode(
|
|
|
221 |
|
222 |
-
|
223 |
result = process_single_file(
|
224 |
uploaded_file.name,
|
225 |
text_content,
|
@@ -232,7 +256,7 @@ def process_multiple_files(
|
|
232 |
if result:
|
233 |
results.append(result)
|
234 |
|
235 |
-
|
236 |
if result.get("success", False):
|
237 |
ResultsManager.add_results(
|
238 |
filename=result["filename"],
|
@@ -260,14 +284,16 @@ def process_multiple_files(
|
|
260 |
if progress_callback:
|
261 |
progress_callback(total_files, total_files, "Complete")
|
262 |
|
263 |
-
ErrorHandler.log_info(
|
|
|
264 |
|
265 |
return results
|
266 |
|
|
|
267 |
def display_batch_results(results: List[Dict[str, Any]]) -> None:
|
268 |
"""
|
269 |
Display batch processing results in the UI
|
270 |
-
|
271 |
Args:
|
272 |
results: List of processing results
|
273 |
"""
|
@@ -278,16 +304,18 @@ def display_batch_results(results: List[Dict[str, Any]]) -> None:
|
|
278 |
successful = [r for r in results if r.get("success", False)]
|
279 |
failed = [r for r in results if not r.get("success", False)]
|
280 |
|
281 |
-
|
282 |
col1, col2, col3 = st.columns(3)
|
283 |
with col1:
|
284 |
st.metric("Total Files", len(results))
|
285 |
with col2:
|
286 |
-
st.metric("Successful", len(successful),
|
|
|
287 |
with col3:
|
288 |
-
st.metric("Failed", len(
|
|
|
289 |
|
290 |
-
|
291 |
tab1, tab2 = st.tabs(["✅Successful", "❌ Failed"])
|
292 |
|
293 |
with tab1:
|
@@ -296,12 +324,16 @@ def display_batch_results(results: List[Dict[str, Any]]) -> None:
|
|
296 |
with st.expander(f"{result['filename']}", expanded=False):
|
297 |
col1, col2 = st.columns(2)
|
298 |
with col1:
|
299 |
-
st.write(
|
300 |
-
|
|
|
|
|
301 |
with col2:
|
302 |
-
st.write(
|
|
|
303 |
if result['ground_truth'] is not None:
|
304 |
-
gt_label = {0: "Stable", 1: "Weathered"}.get(
|
|
|
305 |
correct = "✅" if result['prediction'] == result['ground_truth'] else "❌"
|
306 |
st.write(f"**Ground Truth:** {gt_label} {correct}")
|
307 |
else:
|
@@ -315,10 +347,11 @@ def display_batch_results(results: List[Dict[str, Any]]) -> None:
|
|
315 |
else:
|
316 |
st.success("No failed files!")
|
317 |
|
|
|
318 |
def create_batch_uploader() -> List:
|
319 |
"""
|
320 |
Create multi-file uploader widget
|
321 |
-
|
322 |
Returns:
|
323 |
List of uploaded files
|
324 |
"""
|
@@ -330,4 +363,4 @@ def create_batch_uploader() -> List:
|
|
330 |
key="batch_uploader"
|
331 |
)
|
332 |
|
333 |
-
return uploaded_files if uploaded_files else []
|
|
|
3 |
|
4 |
from typing import List, Dict, Any, Tuple, Optional
|
5 |
import time
|
6 |
+
import streamlit as st
|
7 |
+
import numpy as np
|
8 |
|
9 |
from .preprocessing import resample_spectrum
|
10 |
from .errors import ErrorHandler, safe_execute
|
11 |
from .results_manager import ResultsManager
|
12 |
from .confidence import calculate_softmax_confidence
|
13 |
|
14 |
+
|
15 |
def parse_spectrum_data(text_content: str, filename: str = "unknown") -> Tuple[np.ndarray, np.ndarray]:
|
16 |
"""
|
17 |
Parse spectrum data from text content
|
18 |
+
|
19 |
Args:
|
20 |
text_content: Raw text content of the spectrum file
|
21 |
filename: Name of the file for error reporting
|
22 |
+
|
23 |
Returns:
|
24 |
Tuple of (x_values, y_values) as numpy arrays
|
25 |
+
|
26 |
Raises:
|
27 |
ValueError: If the data cannot be parsed
|
28 |
"""
|
29 |
try:
|
30 |
lines = text_content.strip().split('\n')
|
31 |
|
32 |
+
# ==Remove empty lines and comments==
|
33 |
data_lines = []
|
34 |
for line in lines:
|
35 |
line = line.strip()
|
|
|
39 |
if not data_lines:
|
40 |
raise ValueError("No data lines found in file")
|
41 |
|
42 |
+
# ==Try to parse==
|
43 |
x_vals, y_vals = [], []
|
44 |
|
45 |
for i, line in enumerate(data_lines):
|
46 |
try:
|
47 |
+
# Handle different separators
|
48 |
+
parts = line.replace(",", " ").split()
|
49 |
+
numbers = [p for p in parts if p.replace('.', '', 1).replace(
|
50 |
+
'-', '', 1).replace('+', '', 1).isdigit()]
|
51 |
+
if len(numbers) >= 2:
|
52 |
+
x_val = float(numbers[0])
|
53 |
+
y_val = float(numbers[1])
|
54 |
+
x_vals.append(x_val)
|
55 |
+
y_vals.append(y_val)
|
56 |
+
|
57 |
+
except ValueError:
|
58 |
+
ErrorHandler.log_warning(
|
59 |
+
f"Could not parse line {i+1}: {line}", f"Parsing {filename}")
|
60 |
+
continue
|
61 |
|
62 |
+
if len(x_vals) < 10: # ==Need minimum points for interpolation==
|
63 |
+
raise ValueError(
|
64 |
+
f"Insufficient data points ({len(x_vals)}). Need at least 10 points.")
|
65 |
|
66 |
+
x = np.array(x_vals)
|
67 |
+
y = np.array(y_vals)
|
68 |
|
69 |
+
# Check for NaNs
|
70 |
+
if np.any(np.isnan(x)) or np.any(np.isnan(y)):
|
71 |
+
raise ValueError("Input data contains NaN values")
|
72 |
|
73 |
+
# Check monotonic increasing x
|
74 |
+
if not np.all(np.diff(x) > 0):
|
75 |
+
raise ValueError("Wavenumbers must be strictly increasing")
|
76 |
+
|
77 |
+
# Check reasonable range for Raman spectroscopy
|
78 |
+
if min(x) < 0 or max(x) > 10000 or (max(x) - min(x)) < 100:
|
79 |
+
raise ValueError(
|
80 |
+
f"Invalid wavenumber range: {min(x)} - {max(x)}. Expected ~400-4000 cm⁻¹ with span >100")
|
81 |
|
82 |
+
return x, y
|
|
|
83 |
|
|
|
|
|
84 |
except Exception as e:
|
85 |
raise ValueError(f"Failed to parse spectrum data: {str(e)}")
|
86 |
|
87 |
+
|
88 |
def process_single_file(
|
89 |
filename: str,
|
90 |
text_content: str,
|
|
|
95 |
) -> Optional[Dict[str, Any]]:
|
96 |
"""
|
97 |
Process a single spectrum file
|
98 |
+
|
99 |
Args:
|
100 |
filename: Name of the file
|
101 |
text_content: Raw text content
|
|
|
103 |
load_model_func: Function to load the model
|
104 |
run_inference_func: Function to run inference
|
105 |
label_file_func: Function to extract ground truth label
|
106 |
+
|
107 |
Returns:
|
108 |
Dictionary with processing results or None if failed
|
109 |
"""
|
110 |
start_time = time.time()
|
111 |
|
112 |
try:
|
113 |
+
# ==Parse spectrum data==
|
114 |
+
result, success = safe_execute(
|
115 |
parse_spectrum_data,
|
116 |
text_content,
|
117 |
filename,
|
|
|
119 |
show_error=False
|
120 |
)
|
121 |
|
122 |
+
if not success or result is None:
|
123 |
return None
|
124 |
|
125 |
+
x_raw, y_raw = result
|
126 |
+
|
127 |
+
# ==Resample spectrum==
|
128 |
+
result, success = safe_execute(
|
129 |
resample_spectrum,
|
130 |
x_raw,
|
131 |
y_raw,
|
|
|
134 |
show_error=False
|
135 |
)
|
136 |
|
137 |
+
if not success or result is None:
|
138 |
return None
|
139 |
|
140 |
+
x_resampled, y_resampled = result
|
141 |
+
|
142 |
+
# ==Run inference==
|
143 |
+
result, success = safe_execute(
|
144 |
run_inference_func,
|
145 |
y_resampled,
|
146 |
model_choice,
|
|
|
148 |
show_error=False
|
149 |
)
|
150 |
|
151 |
+
if not success or result is None:
|
152 |
+
ErrorHandler.log_error(
|
153 |
+
Exception("Inference failed"), f"processing {filename}")
|
154 |
return None
|
155 |
|
156 |
+
prediction, logits_list, probs, inference_time, logits = result
|
157 |
+
|
158 |
+
# ==Calculate confidence==
|
159 |
if logits is not None:
|
160 |
+
probs_np, max_confidence, confidence_level, confidence_emoji = calculate_softmax_confidence(
|
161 |
+
logits)
|
162 |
else:
|
163 |
probs_np = np.array([])
|
164 |
max_confidence = 0.0
|
165 |
confidence_level = "LOW"
|
166 |
confidence_emoji = "🔴"
|
167 |
|
168 |
+
# ==Get ground truth==
|
169 |
try:
|
170 |
ground_truth = label_file_func(filename)
|
171 |
ground_truth = ground_truth if ground_truth >= 0 else None
|
172 |
except Exception:
|
173 |
ground_truth = None
|
174 |
|
175 |
+
# ==Get predicted class==
|
176 |
label_map = {0: "Stable (Unweathered)", 1: "Weathered (Degraded)"}
|
177 |
predicted_class = label_map.get(prediction, f"Unknown ({prediction})")
|
178 |
|
|
|
205 |
"processing_time": time.time() - start_time
|
206 |
}
|
207 |
|
208 |
+
|
209 |
def process_multiple_files(
|
210 |
uploaded_files: List,
|
211 |
model_choice: str,
|
|
|
216 |
) -> List[Dict[str, Any]]:
|
217 |
"""
|
218 |
Process multiple uploaded files
|
219 |
+
|
220 |
Args:
|
221 |
uploaded_files: List of uploaded file objects
|
222 |
model_choice: Selected model name
|
|
|
224 |
run_inference_func: Function to run inference
|
225 |
label_file_func: Function to extract ground truth label
|
226 |
progress_callback: Optional callback to update progress
|
227 |
+
|
228 |
Returns:
|
229 |
List of processing results
|
230 |
"""
|
|
|
238 |
progress_callback(i, total_files, uploaded_file.name)
|
239 |
|
240 |
try:
|
241 |
+
# ==Read file content==
|
242 |
raw = uploaded_file.read()
|
243 |
+
text_content = raw.decode(
|
244 |
+
'utf-8') if isinstance(raw, bytes) else raw
|
245 |
|
246 |
+
# ==Process the file==
|
247 |
result = process_single_file(
|
248 |
uploaded_file.name,
|
249 |
text_content,
|
|
|
256 |
if result:
|
257 |
results.append(result)
|
258 |
|
259 |
+
# ==Add successful results to the results manager==
|
260 |
if result.get("success", False):
|
261 |
ResultsManager.add_results(
|
262 |
filename=result["filename"],
|
|
|
284 |
if progress_callback:
|
285 |
progress_callback(total_files, total_files, "Complete")
|
286 |
|
287 |
+
ErrorHandler.log_info(
|
288 |
+
f"Completed batch processing: {sum(1 for r in results if r.get('success', False))}/{total_files} successful")
|
289 |
|
290 |
return results
|
291 |
|
292 |
+
|
293 |
def display_batch_results(results: List[Dict[str, Any]]) -> None:
|
294 |
"""
|
295 |
Display batch processing results in the UI
|
296 |
+
|
297 |
Args:
|
298 |
results: List of processing results
|
299 |
"""
|
|
|
304 |
successful = [r for r in results if r.get("success", False)]
|
305 |
failed = [r for r in results if not r.get("success", False)]
|
306 |
|
307 |
+
# ==Summary==
|
308 |
col1, col2, col3 = st.columns(3)
|
309 |
with col1:
|
310 |
st.metric("Total Files", len(results))
|
311 |
with col2:
|
312 |
+
st.metric("Successful", len(successful),
|
313 |
+
delta=f"{len(successful)/len(results)*100:.1f}%")
|
314 |
with col3:
|
315 |
+
st.metric("Failed", len(
|
316 |
+
failed), delta=f"-{len(failed)/len(results)*100:.1f}%" if failed else "0%")
|
317 |
|
318 |
+
# ==Results tabs==
|
319 |
tab1, tab2 = st.tabs(["✅Successful", "❌ Failed"])
|
320 |
|
321 |
with tab1:
|
|
|
324 |
with st.expander(f"{result['filename']}", expanded=False):
|
325 |
col1, col2 = st.columns(2)
|
326 |
with col1:
|
327 |
+
st.write(
|
328 |
+
f"**Prediction:** {result['predicted_class']}")
|
329 |
+
st.write(
|
330 |
+
f"**Confidence:** {result['confidence_emoji']} {result['confidence_level']} ({result['confidence']:.3f})")
|
331 |
with col2:
|
332 |
+
st.write(
|
333 |
+
f"**Processing Time:** {result['processing_time']:.3f}s")
|
334 |
if result['ground_truth'] is not None:
|
335 |
+
gt_label = {0: "Stable", 1: "Weathered"}.get(
|
336 |
+
result['ground_truth'], "Unknown")
|
337 |
correct = "✅" if result['prediction'] == result['ground_truth'] else "❌"
|
338 |
st.write(f"**Ground Truth:** {gt_label} {correct}")
|
339 |
else:
|
|
|
347 |
else:
|
348 |
st.success("No failed files!")
|
349 |
|
350 |
+
|
351 |
def create_batch_uploader() -> List:
|
352 |
"""
|
353 |
Create multi-file uploader widget
|
354 |
+
|
355 |
Returns:
|
356 |
List of uploaded files
|
357 |
"""
|
|
|
363 |
key="batch_uploader"
|
364 |
)
|
365 |
|
366 |
+
return uploaded_files if uploaded_files else []
|