# %pip install setfit matplotlibdataset_id = "emotion"
model_id = "sentence-transformers/paraphrase-mpnet-base-v2"from datasets import load_dataset

reference_dataset = load_dataset(dataset_id)
reference_dataset# Extract ClassLabel feature from "label" column
label_features = reference_dataset["train"].features["label"]
# Label names to classify with
candidate_labels = label_features.names
candidate_labelsfrom datasets import Dataset
from setfit import get_templated_dataset

# A dummy dataset to fill with synthetic examples
dummy_dataset = Dataset.from_dict({})
train_dataset = get_templated_dataset(dummy_dataset, candidate_labels=candidate_labels, sample_size=8)
train_datasettrain_dataset.shuffle()[:3]from setfit import SetFitModel

model = SetFitModel.from_pretrained(model_id)from setfit import SetFitTrainer

trainer = SetFitTrainer(
    model=model,
    train_dataset=train_dataset,
    eval_dataset=reference_dataset["test"]
)%%time
trainer.train()
zeroshot_metrics = trainer.evaluate()
zeroshot_metricsfrom transformers import pipeline

pipe = pipeline("zero-shot-classification", device=0)%%time
zeroshot_preds = pipe(reference_dataset["test"]["text"], batch_size=16, candidate_labels=candidate_labels)zeroshot_preds[0]preds = [label_features.str2int(pred["labels"][0]) for pred in zeroshot_preds]import evaluate

metric = evaluate.load("accuracy")
transformers_metrics = metric.compute(predictions=preds, references=reference_dataset["test"]["label"])
transformers_metricsfrom setfit import sample_dataset

train_dataset = sample_dataset(reference_dataset["train"])
train_datasetmodel = SetFitModel.from_pretrained(model_id)

trainer = SetFitTrainer(
    model=model,
    train_dataset=train_dataset,
    eval_dataset=reference_dataset["test"]
)
trainer.train()
fewshot_metrics = trainer.evaluate()
fewshot_metricsaugmented_dataset = get_templated_dataset(train_dataset, candidate_labels=candidate_labels, sample_size=8)
augmented_datasetmodel = SetFitModel.from_pretrained(model_id)

trainer = SetFitTrainer(
    model=model,
    train_dataset=augmented_dataset,
    eval_dataset=reference_dataset["test"]
)
trainer.train()
augmented_metrics = trainer.evaluate()
augmented_metricsimport pandas as pd

df = pd.DataFrame.from_dict({"Method":["Transformers (zero-shot)", "SetFit (zero-shot)", "SetFit (augmented)"], "Accuracy": [transformers_metrics["accuracy"], zeroshot_metrics["accuracy"], augmented_metrics["accuracy"]]})
df.plot(kind="barh", x="Method");                                       