Spaces:
Running
Running
fixes on init end bug
Browse files- monitoring.py +14 -0
monitoring.py
CHANGED
@@ -261,6 +261,10 @@ class SmolLM3Monitor:
|
|
261 |
def __init__(self, monitor):
|
262 |
self.monitor = monitor
|
263 |
|
|
|
|
|
|
|
|
|
264 |
def on_log(self, args, state, control, logs=None, **kwargs):
|
265 |
"""Called when logs are created"""
|
266 |
if logs:
|
@@ -277,6 +281,16 @@ class SmolLM3Monitor:
|
|
277 |
"""Called when evaluation is performed"""
|
278 |
if metrics:
|
279 |
self.monitor.log_evaluation_results(metrics, state.global_step)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
280 |
|
281 |
return TrackioCallback(self)
|
282 |
|
|
|
261 |
def __init__(self, monitor):
|
262 |
self.monitor = monitor
|
263 |
|
264 |
+
def on_init_end(self, args, state, control, **kwargs):
|
265 |
+
"""Called when training initialization is complete"""
|
266 |
+
logger.info("Training initialization completed")
|
267 |
+
|
268 |
def on_log(self, args, state, control, logs=None, **kwargs):
|
269 |
"""Called when logs are created"""
|
270 |
if logs:
|
|
|
281 |
"""Called when evaluation is performed"""
|
282 |
if metrics:
|
283 |
self.monitor.log_evaluation_results(metrics, state.global_step)
|
284 |
+
|
285 |
+
def on_train_begin(self, args, state, control, **kwargs):
|
286 |
+
"""Called when training begins"""
|
287 |
+
logger.info("Training started")
|
288 |
+
|
289 |
+
def on_train_end(self, args, state, control, **kwargs):
|
290 |
+
"""Called when training ends"""
|
291 |
+
logger.info("Training completed")
|
292 |
+
if self.monitor:
|
293 |
+
self.monitor.close()
|
294 |
|
295 |
return TrackioCallback(self)
|
296 |
|