Hello again @stefan-it ,
Thanks for a quick response, I wrote a big code chunk for it which is below:

# -*- coding: utf-8 -*-
import torch
from torch.optim import Adam
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from transformers import BertTokenizer, BertConfig, AdamW
from transformers import BertForTokenClassification
from preprocess import create_bert_data
import argparse
from seqeval.metrics import f1_score
from tqdm import tqdm, trange
import numpy as np
from transformers import get_linear_schedule_with_warmup

EPOCHS = 10
BATCH_SIZE = 16
MAX_GRAD_NORM = 1.0

parser = argparse.ArgumentParser()
parser.add_argument("-t", "--trainpath", required = True, help = "Path to the train raw data")
parser.add_argument("-v", "--valpath", required = True, help = "Path to the val raw data")
parser.add_argument("-f", "--finetune", required = True, help = "Fine tuning option if false just the classifier layer is trained")
args = vars(parser.parse_args())

if args["finetune"] == "True":
    fine_tune = True
if args["finetune"] == "False":
    fine_tune = False

assert fine_tune is True or fine_tune is False, "Fine-tune must be a boolean value but given {}".format(args["finetune"])

#Tokenizer
tokenizer = BertTokenizer.from_pretrained("dbmdz/bert-base-turkish-cased")

#Accuracy on a token level (balanced accuracy)
def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=2).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)

#Set up if the script will run on cpu or gpu
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()

train_input, train_labels, uniq_ent, ent_idx, train_attention_masks = create_bert_data("train.txt", train = True)
val_input, val_labels, val_attention_masks = create_bert_data(args["valpath"], train = False)
one_training_step = len(train_input) / BATCH_SIZE
train_input = torch.tensor(train_input)
train_labels = torch.tensor(train_labels)
train_attention_masks = torch.tensor(train_attention_masks)
val_input = torch.tensor(val_input)
val_labels = torch.tensor(val_labels)
val_attention_masks = torch.tensor(val_attention_masks)

#Define the DataLoader and shuffle the data at training time and test time pass them to SequentialSampler
train_dataset = TensorDataset(train_input, train_attention_masks, train_labels)
train_sampler = RandomSampler(train_dataset)
train_dataloader = DataLoader(train_dataset, sampler = train_sampler, batch_size = BATCH_SIZE)

val_dataset = TensorDataset(val_input, val_attention_masks, val_labels)
val_sampler = RandomSampler(val_dataset)
val_dataloader = DataLoader(val_dataset, sampler = val_sampler, batch_size = BATCH_SIZE)

#Write unique entities and their ids into a file
with open("entity_idx.txt", "w") as foo:
    foo.write(str(ent_idx))

print(ent_idx)

#Load the model
model = BertForTokenClassification.from_pretrained("dbmdz/bert-base-turkish-cased", num_labels = len(ent_idx))

#For gpu
model.cuda()
num_training_steps = int(one_training_step/BATCH_SIZE)*EPOCHS
num_warmup_steps = int(num_training_steps * 0.4)
#Initilize the optimizer
optimizer = AdamW(model.parameters(), lr = 5e-5, correct_bias=False)  #To reproduce BertAdam specific behavior set correct_bias=False
scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps)  #PyTorch schedule


#For printing under progress bars
description_train = tqdm(total=0, position=1, bar_format='{desc}')
description_val = tqdm(total=0, position=1, bar_format='{desc}')


train_losses = []

# TRAINING LOOP
for epoch in range(EPOCHS):
    print("{:d}/{:d} EPOCH".format(epoch, EPOCHS))
    model.train()
    tr_loss = 0
    nb_tr_examples, nb_tr_steps = 0, 0
    # Training progress bar
    pbar_train = tqdm(total = len(train_input), leave = False)

    #TRAINING STEP
    for step, batch in enumerate(train_dataloader):

        # add batch to gpu
        input_batch, input_mask_batch, labels_batch = [b.to(device) for b in batch]
        # forward pass
        loss,_ = model(input_batch, token_type_ids = None,
                    attention_mask = input_mask_batch, labels = labels_batch)
        # backprob
        loss.backward()
        # track train loss
        tr_loss += loss.item()
        nb_tr_examples += input_batch.size(0)
        nb_tr_steps += 1
        # gradient clipping to avoid exploding gradients
        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), max_norm=MAX_GRAD_NORM)
        # update parameters
        optimizer.step()
        scheduler.step()
        optimizer.zero_grad()
        model.zero_grad()
        # Increment progress bar
        pbar_train.update(BATCH_SIZE)
        train_loss = tr_loss/nb_tr_steps
        description_train.set_description_str(f"Epoch: {epoch}/{EPOCHS} - Loss: {train_loss}")
    #Print train loss per epoch
    print("Train loss: {:.5f}".format(tr_loss/nb_tr_steps))
    train_losses.append(tr_loss/nb_tr_steps)
    pbar_train.close()

    #VALIDATION STEP 
    model.eval()
    eval_loss, eval_accuracy = 0, 0
    nb_eval_steps, nb_eval_examples = 0, 0
    predictions, true_labels = [], []
    # Validation progress bar
    pbar_val = tqdm(total = len(val_input), leave = True)

    for batch in val_dataloader:
        # add batch to gpu
        batch = tuple(t.to(device) for t in batch)
        input_batch, input_mask_batch, labels_batch = batch

        with torch.no_grad():
            outputs = model(input_batch, token_type_ids = None,
                                attention_mask = input_mask_batch, labels = labels_batch)
            temp_eval_loss, logits = outputs[:2]
            logits = logits.detach().cpu().numpy()
            label_ids = labels_batch.to("cpu").numpy()
            predictions.extend([list(p) for p in np.argmax(logits, axis = 2)])
            true_labels.append(label_ids)
            temp_eval_accuracy = flat_accuracy(logits, label_ids)
            eval_loss += temp_eval_loss.mean().item()
            eval_accuracy += temp_eval_accuracy

            nb_eval_examples += input_batch.size(0)
            nb_eval_steps += 1
        pbar_val.update(BATCH_SIZE)
        eval_loss = eval_loss/nb_eval_steps
        val_accuracy = eval_accuracy/nb_eval_steps
        predicted_entities = [uniq_ent[p_i] for p in predictions for p_i in p]
        valid_entities = [uniq_ent[l_ii] for l in true_labels for l_i in l for l_ii in l_i]
        f1_score_val = f1_score(valid_entities, predicted_entities)
        description_val.set_description_str(f"Validation Scores -> Loss : {eval_loss} , Accuracy : 
                                                                  {val_accuracy} , F1-Score : {f1_score_val}")
    pbar_val.close()

The code above didnt converge I stuck at 0.020... loss and 0.40 around for f1 score..
Also I tried with your berturk.json file with farm and I used experiment which is presented below

from farm.experiment import run_experiment, load_experiments

experiments = load_experiments("/home/karaz/Desktop/BERT_NER/FARM_TransferLearning/berturk.json")

run_experiment(experiments[0])

I can go with the FARM also but when I executed that code above for farm the f1 score is %55. What am I missing here :)
Thanks

Cannot find the NER Model #10

Description

Metadata

Metadata

Assignees

Labels

Projects

Milestone

Relationships

Development

Issue actions