model_id = "distilbert/distilbert-base-uncased"

model = AutoModelForSequenceClassification.from_pretrained(
    model_id, num_labels=2
)
tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True)

ds = (
    Dataset.from_dict({"text": ["Hello world!"], "labels": [0]})
    .map(
        lambda b: tokenizer(b["text"], truncation=True),
        batched=True,
    )
)


trainer = Trainer(
    model=model,
    args=TrainingArguments(
        output_dir="temp",
        eval_strategy="steps",
        save_strategy="no",
        eval_on_start=True,  # <-- setting to False avoids the error
    ),
    tokenizer=tokenizer,
    train_dataset=ds,
    eval_dataset=ds,
    compute_metrics=lambda _: {"metric": 1.0},
)
trainer.train()