I’m having issues useing Ray Tune. I’m trying to use ‘f1_metric’
my code is
def f1_metric(predictions):
  class_probs = predictions[0]
  true_labels = predictions[1]

  f1s = []
  for i in range(len(class_probs)):
    cp = class_probs[i]
    tl = true_labels[i]
    pred_labels = [np.argmax(x) for x in class_probs[0]]
    f1s.append(f1_score(tl,pred_labels))
  return {'f1_metric':np.mean(f1s)}

trainer = CustomTrainer(
        model_init=model_init,
        args=training_args,
        train_dataset=train_dataset,
        eval_dataset=test_dataset,
        compute_metrics=f1_metric,
        tokenizer=tokenizer,
        data_collator=data_collator)


def hp_space_fn(*args, **kwargs):
    config = {
                "warmup_steps": tune.choice([50, 100, 500]),
                "learning_rate": tune.choice([2e-5, 3e-5, 1e-5]),
                "num_train_epochs": tune.quniform(0.01, .02, 0.01),
                 'label_smoothing_factor': tune.choice([0.01,0.03,.07,.10]),
                 'weight_picks_0' : tune.choice([0.01]),
                 'weight_picks_1' : tune.choice([0.05,.07,.10,.2,.5]),
    }
    wandb_config = {
            "wandb": {
                    "project": os.environ.get(
                        "WANDB_PROJECT",
                        "wandb_project"),
                    "api_key": os.environ.get("API_KEY"),
                    "log_config": True
                    }
    }
    config.update(wandb_config)
    return config



best_run = trainer.hyperparameter_search(
            direction="maximize",
            backend="ray",
            scheduler=PopulationBasedTraining(
                        time_attr="time_total_s",
                        mode="max",
                        metric='f1_metric',
                        perturbation_interval=600.0
                    ),
            hp_space=hp_space_fn,
            loggers=DEFAULT_LOGGERS + (WandbLogger, ),
            time_budget_s=4000
    )

No matter what I name the f1_metric I can’t get Ray Tune to recognize it.
When I remove the metric from best_run
best_run = trainer.hyperparameter_search(
            direction="maximize",
            backend="ray",
            scheduler=PopulationBasedTraining(
                        time_attr="time_total_s",
                        mode="max",
                       #### metric='f1_metric',
                        perturbation_interval=600.0
                    ),
            hp_space=hp_space_fn,
            loggers=DEFAULT_LOGGERS + (WandbLogger, ),
            time_budget_s=4000
    )

RuntimeError: Cannot find metric _metric in trial result {‘objective’: 0.0018931623078665865, ‘eval_loss’: 0.5891237854957581, ‘eval_f1_metric’: 0.0018931623078665865, ‘eval_runtime’: 168.5186, ‘eval_samples_per_second’: 5.204, ‘eval_steps_per_second’: 1.305, ‘epoch’: 0.0, ‘time_this_iter_s’: 219.17230415344238, ‘done’: False, ‘timesteps_total’: None, ‘episodes_total’: None, ‘training_iteration’: 1, ‘trial_id’: ‘da2b7_00000’, ‘experiment_id’: ‘f36eae0544124c1cb87fdbbe9cda499c’, ‘date’: ‘2022-01-10_15-52-59’, ‘timestamp’: 1641829979, ‘time_total_s’: 219.17230415344238, ‘pid’: 2537, ‘hostname’: ‘9996f5f6eb70’, ‘node_ip’: ‘172.28.0.2’, ‘time_since_restore’: 219.17230415344238, ‘timesteps_since_restore’: 0, ‘iterations_since_restore’: 1, ‘config/warmup_steps’: 500, ‘config/learning_rate’: 2e-05, ‘config/num_train_epochs’: 0.02, ‘config/label_smoothing_factor’: 0.03, ‘config/weight_picks_0’: 0.01, ‘config/weight_picks_1’: 0.07}. Make sure that this attribute is returned in the results of your Trainable.If this error is expected, you can change this to a warning message by setting PBT(require_attrs=False)