Joseph Pollack commited on
Commit
7ca96a1
·
unverified ·
1 Parent(s): 1de56de

removes trackio for testing

Browse files
interface.py CHANGED
@@ -460,11 +460,20 @@ def start_voxtral_training(
460
  # Collect all logs
461
  all_logs = []
462
 
463
- def collect_logs(generator):
464
- """Helper to collect logs from a generator."""
 
 
465
  for line in generator:
466
  all_logs.append(line)
467
- print(line) # Also print to console for debugging
 
 
 
 
 
 
 
468
 
469
  try:
470
  # 1) Train
@@ -493,7 +502,10 @@ def start_voxtral_training(
493
  args += ["--freeze-audio-tower"]
494
 
495
  all_logs.append("🚀 Starting Voxtral training...")
496
- collect_logs(run_command_stream(args, env))
 
 
 
497
  all_logs.append("✅ Training completed!")
498
 
499
  # 2) Push to Hub
@@ -505,8 +517,11 @@ def start_voxtral_training(
505
  full_repo_name,
506
  ]
507
  all_logs.append(f"📤 Pushing model to Hugging Face Hub: {full_repo_name}")
508
- collect_logs(run_command_stream(push_args, env))
509
- all_logs.append("✅ Model pushed successfully!")
 
 
 
510
 
511
  # 3) Deploy demo Space
512
  if deploy_demo:
@@ -522,8 +537,11 @@ def start_voxtral_training(
522
  "--space-name", demo_space_name,
523
  ]
524
  all_logs.append("🚀 Deploying demo Space...")
525
- collect_logs(run_command_stream(deploy_args, env))
526
- all_logs.append("✅ Demo Space deployed!")
 
 
 
527
 
528
  # Return all collected logs as a single string
529
  return "\n".join(all_logs)
 
460
  # Collect all logs
461
  all_logs = []
462
 
463
+ def collect_logs_with_code(generator):
464
+ """Collect logs and parse the exit code from the final status line."""
465
+ import re
466
+ exit_code = None
467
  for line in generator:
468
  all_logs.append(line)
469
+ print(line)
470
+ m = re.search(r"exit code:\s*(\d+)", line)
471
+ if m:
472
+ try:
473
+ exit_code = int(m.group(1))
474
+ except Exception:
475
+ pass
476
+ return 0 if exit_code == 0 else (exit_code if exit_code is not None else 1)
477
 
478
  try:
479
  # 1) Train
 
502
  args += ["--freeze-audio-tower"]
503
 
504
  all_logs.append("🚀 Starting Voxtral training...")
505
+ train_code = collect_logs_with_code(run_command_stream(args, env))
506
+ if train_code != 0:
507
+ all_logs.append("❌ Training failed. Skipping model push and demo deployment.")
508
+ return "\n".join(all_logs)
509
  all_logs.append("✅ Training completed!")
510
 
511
  # 2) Push to Hub
 
517
  full_repo_name,
518
  ]
519
  all_logs.append(f"📤 Pushing model to Hugging Face Hub: {full_repo_name}")
520
+ push_code = collect_logs_with_code(run_command_stream(push_args, env))
521
+ if push_code != 0:
522
+ all_logs.append("❌ Model push failed.")
523
+ else:
524
+ all_logs.append("✅ Model pushed successfully!")
525
 
526
  # 3) Deploy demo Space
527
  if deploy_demo:
 
537
  "--space-name", demo_space_name,
538
  ]
539
  all_logs.append("🚀 Deploying demo Space...")
540
+ deploy_code = collect_logs_with_code(run_command_stream(deploy_args, env))
541
+ if deploy_code != 0:
542
+ all_logs.append("❌ Demo Space deployment failed.")
543
+ else:
544
+ all_logs.append("✅ Demo Space deployed!")
545
 
546
  # Return all collected logs as a single string
547
  return "\n".join(all_logs)
scripts/deploy_demo_space.py CHANGED
@@ -189,36 +189,37 @@ class DemoSpaceDeployer:
189
  # For GPT-OSS models, we need more sophisticated environment setup
190
  model_name = self.model_id.split("/")[-1] if "/" in self.model_id else self.model_id
191
  import json as _json
192
- env_setup = f"""
193
- # Environment variables for GPT-OSS model configuration
 
 
 
194
  import os
195
- os.environ['HF_MODEL_ID'] = json.dumps(self.model_id)
196
- os.environ['LORA_MODEL_ID'] = json.dumps(self.model_id)
 
197
  os.environ['BASE_MODEL_ID'] = 'openai/gpt-oss-20b'
198
- os.environ['MODEL_SUBFOLDER'] = json.dumps(self.subfolder if self.subfolder else "")
199
- os.environ['MODEL_NAME'] = json.dumps(model_name)
200
- os.environ['MODEL_IDENTITY'] = json.dumps(self.model_identity or "")
201
- os.environ['SYSTEM_MESSAGE'] = json.dumps(self.system_message or (self.model_identity or ""))
202
- os.environ['DEVELOPER_MESSAGE'] = json.dumps(self.developer_message or "")
203
- os.environ['REASONING_EFFORT'] = json.dumps((self.reasoning_effort or "medium"))
204
- {"os.environ['EXAMPLES_TYPE'] = " + _json.dumps(self.examples_type) + "\n" if self.examples_type else ''}
205
- {"os.environ['DISABLE_EXAMPLES'] = 'true'\n" if self.disable_examples else ("os.environ['DISABLE_EXAMPLES'] = 'false'\n" if self.disable_examples is not None else '')}
206
- {"os.environ['EXAMPLES_JSON'] = " + _json.dumps(self.examples_json) + "\n" if self.examples_json else ''}
207
 
208
  # Branding/owner variables
209
- os.environ['HF_USERNAME'] = json.dumps(self.hf_username)
210
- os.environ['BRAND_OWNER_NAME'] = json.dumps(self.brand_owner_name)
211
- os.environ['BRAND_TEAM_NAME'] = json.dumps(self.brand_team_name)
212
- os.environ['BRAND_DISCORD_URL'] = json.dumps(self.brand_discord_url)
213
- os.environ['BRAND_HF_ORG'] = json.dumps(self.brand_hf_org)
214
- os.environ['BRAND_HF_LABEL'] = json.dumps(self.brand_hf_label)
215
- os.environ['BRAND_HF_URL'] = json.dumps(self.brand_hf_url)
216
- os.environ['BRAND_GH_ORG'] = json.dumps(self.brand_gh_org)
217
- os.environ['BRAND_GH_LABEL'] = json.dumps(self.brand_gh_label)
218
- os.environ['BRAND_GH_URL'] = json.dumps(self.brand_gh_url)
219
- os.environ['BRAND_PROJECT_NAME'] = json.dumps(self.brand_project_name)
220
- os.environ['BRAND_PROJECT_URL'] = json.dumps(self.brand_project_url)
221
-
222
  """
223
  elif self.demo_type == "voxtral":
224
  # For Voxtral, we do not inject env setup into app.py.
@@ -227,34 +228,35 @@ os.environ['BRAND_PROJECT_URL'] = json.dumps(self.brand_project_url)
227
  else:
228
  # For SmolLM models, use simpler setup
229
  import json as _json
230
- env_setup = f"""
231
- # Environment variables for model configuration
 
 
 
232
  import os
233
- os.environ['HF_MODEL_ID'] = json.dumps(self.model_id)
234
- os.environ['MODEL_SUBFOLDER'] = json.dumps(self.subfolder if self.subfolder else "")
235
- os.environ['MODEL_NAME'] = json.dumps(self.model_id.split("/")[-1])
236
- os.environ['MODEL_IDENTITY'] = json.dumps(self.model_identity or "")
237
- os.environ['SYSTEM_MESSAGE'] = json.dumps(self.system_message or (self.model_identity or ""))
238
- os.environ['DEVELOPER_MESSAGE'] = json.dumps(self.developer_message or "")
239
- os.environ['REASONING_EFFORT'] = json.dumps((self.reasoning_effort or "medium"))
240
- {"os.environ['EXAMPLES_TYPE'] = " + _json.dumps(self.examples_type) + "\n" if self.examples_type else ''}
241
- {"os.environ['DISABLE_EXAMPLES'] = 'true'\n" if self.disable_examples else ("os.environ['DISABLE_EXAMPLES'] = 'false'\n" if self.disable_examples is not None else '')}
242
- {"os.environ['EXAMPLES_JSON'] = " + _json.dumps(self.examples_json) + "\n" if self.examples_json else ''}
243
 
244
  # Branding/owner variables
245
- os.environ['HF_USERNAME'] = json.dumps(self.hf_username)
246
- os.environ['BRAND_OWNER_NAME'] = json.dumps(self.brand_owner_name)
247
- os.environ['BRAND_TEAM_NAME'] = json.dumps(self.brand_team_name)
248
- os.environ['BRAND_DISCORD_URL'] = json.dumps(self.brand_discord_url)
249
- os.environ['BRAND_HF_ORG'] = json.dumps(self.brand_hf_org)
250
- os.environ['BRAND_HF_LABEL'] = json.dumps(self.brand_hf_label)
251
- os.environ['BRAND_HF_URL'] = json.dumps(self.brand_hf_url)
252
- os.environ['BRAND_GH_ORG'] = json.dumps(self.brand_gh_org)
253
- os.environ['BRAND_GH_LABEL'] = json.dumps(self.brand_gh_label)
254
- os.environ['BRAND_GH_URL'] = json.dumps(self.brand_gh_url)
255
- os.environ['BRAND_PROJECT_NAME'] = json.dumps(self.brand_project_name)
256
- os.environ['BRAND_PROJECT_URL'] = json.dumps(self.brand_project_url)
257
-
258
  """
259
  return env_setup
260
 
 
189
  # For GPT-OSS models, we need more sophisticated environment setup
190
  model_name = self.model_id.split("/")[-1] if "/" in self.model_id else self.model_id
191
  import json as _json
192
+ ex_type_line = f"os.environ['EXAMPLES_TYPE'] = {_json.dumps(self.examples_type)}\n" if self.examples_type else ""
193
+ disable_line = ("os.environ['DISABLE_EXAMPLES'] = 'true'\n" if self.disable_examples
194
+ else ("os.environ['DISABLE_EXAMPLES'] = 'false'\n" if self.disable_examples is not None else ""))
195
+ examples_json_line = f"os.environ['EXAMPLES_JSON'] = {_json.dumps(self.examples_json)}\n" if self.examples_json else ""
196
+ env_setup = f"""# Environment variables for GPT-OSS model configuration
197
  import os
198
+ import json
199
+ os.environ['HF_MODEL_ID'] = json.dumps({_json.dumps(self.model_id)})
200
+ os.environ['LORA_MODEL_ID'] = json.dumps({_json.dumps(self.model_id)})
201
  os.environ['BASE_MODEL_ID'] = 'openai/gpt-oss-20b'
202
+ os.environ['MODEL_SUBFOLDER'] = json.dumps({_json.dumps(self.subfolder if self.subfolder else "")})
203
+ os.environ['MODEL_NAME'] = json.dumps({_json.dumps(model_name)})
204
+ os.environ['MODEL_IDENTITY'] = json.dumps({_json.dumps(self.model_identity or "")})
205
+ os.environ['SYSTEM_MESSAGE'] = json.dumps({_json.dumps(self.system_message or (self.model_identity or ""))})
206
+ os.environ['DEVELOPER_MESSAGE'] = json.dumps({_json.dumps(self.developer_message or "")})
207
+ os.environ['REASONING_EFFORT'] = json.dumps({_json.dumps((self.reasoning_effort or "medium"))})
208
+ {ex_type_line}{disable_line}{examples_json_line}
 
 
209
 
210
  # Branding/owner variables
211
+ os.environ['HF_USERNAME'] = json.dumps({_json.dumps(self.hf_username)})
212
+ os.environ['BRAND_OWNER_NAME'] = json.dumps({_json.dumps(self.brand_owner_name)})
213
+ os.environ['BRAND_TEAM_NAME'] = json.dumps({_json.dumps(self.brand_team_name)})
214
+ os.environ['BRAND_DISCORD_URL'] = json.dumps({_json.dumps(self.brand_discord_url)})
215
+ os.environ['BRAND_HF_ORG'] = json.dumps({_json.dumps(self.brand_hf_org)})
216
+ os.environ['BRAND_HF_LABEL'] = json.dumps({_json.dumps(self.brand_hf_label)})
217
+ os.environ['BRAND_HF_URL'] = json.dumps({_json.dumps(self.brand_hf_url)})
218
+ os.environ['BRAND_GH_ORG'] = json.dumps({_json.dumps(self.brand_gh_org)})
219
+ os.environ['BRAND_GH_LABEL'] = json.dumps({_json.dumps(self.brand_gh_label)})
220
+ os.environ['BRAND_GH_URL'] = json.dumps({_json.dumps(self.brand_gh_url)})
221
+ os.environ['BRAND_PROJECT_NAME'] = json.dumps({_json.dumps(self.brand_project_name)})
222
+ os.environ['BRAND_PROJECT_URL'] = json.dumps({_json.dumps(self.brand_project_url)})
 
223
  """
224
  elif self.demo_type == "voxtral":
225
  # For Voxtral, we do not inject env setup into app.py.
 
228
  else:
229
  # For SmolLM models, use simpler setup
230
  import json as _json
231
+ ex_type_line = f"os.environ['EXAMPLES_TYPE'] = {_json.dumps(self.examples_type)}\n" if self.examples_type else ""
232
+ disable_line = ("os.environ['DISABLE_EXAMPLES'] = 'true'\n" if self.disable_examples
233
+ else ("os.environ['DISABLE_EXAMPLES'] = 'false'\n" if self.disable_examples is not None else ""))
234
+ examples_json_line = f"os.environ['EXAMPLES_JSON'] = {_json.dumps(self.examples_json)}\n" if self.examples_json else ""
235
+ env_setup = f"""# Environment variables for model configuration
236
  import os
237
+ import json
238
+ os.environ['HF_MODEL_ID'] = json.dumps({_json.dumps(self.model_id)})
239
+ os.environ['MODEL_SUBFOLDER'] = json.dumps({_json.dumps(self.subfolder if self.subfolder else "")})
240
+ os.environ['MODEL_NAME'] = json.dumps({_json.dumps(self.model_id.split("/")[-1])})
241
+ os.environ['MODEL_IDENTITY'] = json.dumps({_json.dumps(self.model_identity or "")})
242
+ os.environ['SYSTEM_MESSAGE'] = json.dumps({_json.dumps(self.system_message or (self.model_identity or ""))})
243
+ os.environ['DEVELOPER_MESSAGE'] = json.dumps({_json.dumps(self.developer_message or "")})
244
+ os.environ['REASONING_EFFORT'] = json.dumps({_json.dumps((self.reasoning_effort or "medium"))})
245
+ {ex_type_line}{disable_line}{examples_json_line}
 
246
 
247
  # Branding/owner variables
248
+ os.environ['HF_USERNAME'] = json.dumps({_json.dumps(self.hf_username)})
249
+ os.environ['BRAND_OWNER_NAME'] = json.dumps({_json.dumps(self.brand_owner_name)})
250
+ os.environ['BRAND_TEAM_NAME'] = json.dumps({_json.dumps(self.brand_team_name)})
251
+ os.environ['BRAND_DISCORD_URL'] = json.dumps({_json.dumps(self.brand_discord_url)})
252
+ os.environ['BRAND_HF_ORG'] = json.dumps({_json.dumps(self.brand_hf_org)})
253
+ os.environ['BRAND_HF_LABEL'] = json.dumps({_json.dumps(self.brand_hf_label)})
254
+ os.environ['BRAND_HF_URL'] = json.dumps({_json.dumps(self.brand_hf_url)})
255
+ os.environ['BRAND_GH_ORG'] = json.dumps({_json.dumps(self.brand_gh_org)})
256
+ os.environ['BRAND_GH_LABEL'] = json.dumps({_json.dumps(self.brand_gh_label)})
257
+ os.environ['BRAND_GH_URL'] = json.dumps({_json.dumps(self.brand_gh_url)})
258
+ os.environ['BRAND_PROJECT_NAME'] = json.dumps({_json.dumps(self.brand_project_name)})
259
+ os.environ['BRAND_PROJECT_URL'] = json.dumps({_json.dumps(self.brand_project_url)})
 
260
  """
261
  return env_setup
262
 
scripts/train.py CHANGED
@@ -376,11 +376,9 @@ def main():
376
 
377
  data_collator = VoxtralDataCollator(processor, model_checkpoint)
378
 
379
- # Only report to trackio if it's enabled and working
380
  report_to = []
381
- if wandb_enabled:
382
- report_to = ["trackio"]
383
-
384
  training_args = TrainingArguments(
385
  output_dir=output_dir,
386
  per_device_train_batch_size=args.batch_size,
 
376
 
377
  data_collator = VoxtralDataCollator(processor, model_checkpoint)
378
 
379
+ # Disable Transformers Trackio callback to avoid httpx timeouts; logging is handled via trackio.init()
380
  report_to = []
381
+
 
 
382
  training_args = TrainingArguments(
383
  output_dir=output_dir,
384
  per_device_train_batch_size=args.batch_size,
scripts/train_lora.py CHANGED
@@ -405,11 +405,9 @@ def main():
405
 
406
  data_collator = VoxtralDataCollator(processor, model_checkpoint)
407
 
408
- # Only report to trackio if it's enabled and working
409
  report_to = []
410
- if wandb_enabled:
411
- report_to = ["trackio"]
412
-
413
  training_args = TrainingArguments(
414
  output_dir=output_dir,
415
  per_device_train_batch_size=args.batch_size,
 
405
 
406
  data_collator = VoxtralDataCollator(processor, model_checkpoint)
407
 
408
+ # Disable Transformers Trackio callback to avoid httpx timeouts; logging is handled via trackio.init()
409
  report_to = []
410
+
 
 
411
  training_args = TrainingArguments(
412
  output_dir=output_dir,
413
  per_device_train_batch_size=args.batch_size,