Skip to content

Commit

Permalink
GH-474: Model interface for sequence labeling, classification and reg…
Browse files Browse the repository at this point in the history 
…ression
  • Loading branch information
@stefan-it
aakbik authored and stefan-it committed Apr 26, 2019
1 parent 43aa2a0 commit 5e77d62
Show file tree
Hide file tree
Showing 5 changed files with 71 additions and 73 deletions.
2 changes: 2 additions & 0 deletions flair/hyperparameter/param_selection.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,8 @@ def _objective(self, params: dict):
curr_scores = list(
map(lambda s: 1 - s, result["dev_score_history"][-3:])
)
print(result)
print(curr_scores)
score = sum(curr_scores) / float(len(curr_scores))
var = np.var(curr_scores)
scores.append(score)
Expand Down
2 changes: 2 additions & 0 deletions flair/models/sequence_tagger_model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,8 @@ def evaluate(
with open(out_path, "w", encoding="utf-8") as outfile:
outfile.write("".join(lines))

print(metric)

detailed_result = (
f"\nMICRO_AVG: acc {metric.micro_avg_accuracy()} - f1-score {metric.micro_avg_f_score()}"
f"\nMACRO_AVG: acc {metric.macro_avg_accuracy()} - f1-score {metric.macro_avg_f_score()}"
Expand Down
132 changes: 62 additions & 70 deletions flair/trainers/trainer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@ def train(
max_epochs: int = 100,
anneal_factor: float = 0.5,
patience: int = 3,
anneal_against_train_loss: bool = True,
train_with_dev: bool = False,
monitor_train: bool = False,
embeddings_in_memory: bool = True,
Expand Down Expand Up @@ -81,40 +80,39 @@ def train(
log_test = True if (not param_selection_mode and self.corpus.test) else False
log_dev = True if not train_with_dev else False

if not param_selection_mode:
loss_txt = init_output_file(base_path, "loss.tsv")
with open(loss_txt, "a") as f:
f.write(f"EPOCH\tTIMESTAMP\tBAD_EPOCHS\tLEARNING_RATE\tTRAIN_LOSS")
loss_txt = init_output_file(base_path, "loss.tsv")
with open(loss_txt, "a") as f:
f.write(f"EPOCH\tTIMESTAMP\tBAD_EPOCHS\tLEARNING_RATE\tTRAIN_LOSS")

dummy_result, _ = self.model.evaluate(
[Sentence("d", labels=["0.1"])],
eval_mini_batch_size,
embeddings_in_memory,
dummy_result, _ = self.model.evaluate(
[Sentence("d", labels=["0.1"])],
eval_mini_batch_size,
embeddings_in_memory,
)
if log_train:
f.write(
"\tTRAIN_" + "\tTRAIN_".join(dummy_result.log_header.split("\t"))
)
if log_dev:
f.write(
"\tDEV_LOSS\tDEV_"
+ "\tDEV_".join(dummy_result.log_header.split("\t"))
)
if log_test:
f.write(
"\tTEST_LOSS\tTEST_"
+ "\tTEST_".join(dummy_result.log_header.split("\t"))
)
if log_train:
f.write(
"\tTRAIN_"
+ "\tTRAIN_".join(dummy_result.log_header.split("\t"))
)
if log_dev:
f.write(
"\tDEV_LOSS\tDEV_"
+ "\tDEV_".join(dummy_result.log_header.split("\t"))
)
if log_test:
f.write(
"\tTEST_LOSS\tTEST_"
+ "\tTEST_".join(dummy_result.log_header.split("\t"))
)

weight_extractor = WeightExtractor(base_path)

optimizer = self.optimizer(self.model.parameters(), lr=learning_rate, **kwargs)
if self.optimizer_state is not None:
optimizer.load_state_dict(self.optimizer_state)

# annealing scheduler
anneal_mode = "min" if anneal_against_train_loss else "max"
# minimize training loss if training with dev data, else maximize dev score
anneal_mode = "min" if train_with_dev else "max"

if isinstance(optimizer, (AdamW, SGDW)):
scheduler = ReduceLRWDOnPlateau(
optimizer,
Expand Down Expand Up @@ -224,51 +222,47 @@ def train(
f"EPOCH {epoch + 1} done: loss {train_loss:.4f} - lr {learning_rate:.4f} - bad epochs {bad_epochs}"
)

dev_loss = "_"
if not param_selection_mode:
with open(loss_txt, "a") as f:
# anneal against train loss if training with dev, otherwise anneal against dev score
current_score = train_loss

f.write(
f"\n{epoch}\t{datetime.datetime.now():%H:%M:%S}\t{bad_epochs}\t{learning_rate:.4f}\t{train_loss}"
with open(loss_txt, "a") as f:

f.write(
f"\n{epoch}\t{datetime.datetime.now():%H:%M:%S}\t{bad_epochs}\t{learning_rate:.4f}\t{train_loss}"
)

if log_train:
train_eval_result, train_loss = self.model.evaluate(
self.corpus.train,
eval_mini_batch_size,
embeddings_in_memory,
)
f.write(f"\t{train_eval_result.log_line}")

if log_train:
train_eval_result, train_loss = self.model.evaluate(
self.corpus.train,
eval_mini_batch_size,
embeddings_in_memory,
)
f.write(f"\t{train_eval_result.log_line}")
if log_dev:
dev_eval_result, dev_loss = self.model.evaluate(
self.corpus.dev, eval_mini_batch_size, embeddings_in_memory
)
f.write(f"\t{dev_loss}\t{dev_eval_result.log_line}")

if log_dev:
dev_eval_result, dev_loss = self.model.evaluate(
self.corpus.dev,
eval_mini_batch_size,
embeddings_in_memory,
)
f.write(f"\t{dev_loss}\t{dev_eval_result.log_line}")

if log_test:
test_eval_result, test_loss = self.model.evaluate(
self.corpus.test,
eval_mini_batch_size,
embeddings_in_memory,
base_path / "test.tsv",
)
f.write(f"\t{test_loss}\t{test_eval_result.log_line}")
log.info(
f"TEST : loss {test_loss} - score {test_eval_result.main_score}"
)
# calculate scores using dev data if available
# append dev score to score history
dev_score_history.append(dev_eval_result.main_score)
dev_loss_history.append(dev_loss)

# calculate scores using dev data if available
dev_score = 0.0
if not train_with_dev:
# append dev score to score history
dev_score_history.append(dev_score)
dev_loss_history.append(dev_loss.item())
current_score = dev_eval_result.main_score

# anneal against train loss if training with dev, otherwise anneal against dev score
current_score = train_loss if anneal_against_train_loss else dev_score
if log_test:
test_eval_result, test_loss = self.model.evaluate(
self.corpus.test,
eval_mini_batch_size,
embeddings_in_memory,
base_path / "test.tsv",
)
f.write(f"\t{test_loss}\t{test_eval_result.log_line}")
log.info(
f"TEST : loss {test_loss} - score {test_eval_result.main_score}"
)

scheduler.step(current_score)

Expand Down Expand Up @@ -351,13 +345,11 @@ def final_test(
if type(self.corpus) is MultiCorpus:
for subcorpus in self.corpus.corpora:
log_line(log)
self._calculate_evaluation_results_for(
subcorpus.name,
self.model.evaluate(
subcorpus.test,
evaluation_metric,
embeddings_in_memory,
eval_mini_batch_size,
base_path / "test.tsv",
embeddings_in_memory,
base_path / f"{subcorpus.name}-test.tsv",
)

# get and return the final test score of best model
Expand Down
2 changes: 2 additions & 0 deletions flair/training_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ def micro_avg_f_score(self):

def macro_avg_f_score(self):
class_f_scores = [self.f_score(class_name) for class_name in self.get_classes()]
if len(class_f_scores) == 0:
return 0.0
macro_f_score = sum(class_f_scores) / len(class_f_scores)
return macro_f_score

Expand Down
6 changes: 3 additions & 3 deletions tests/test_text_regressor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,15 +10,15 @@
from flair.trainers import ModelTrainer


def init(tasks_base_path) -> Tuple[TaggedCorpus, TextRegressor]:
def init(tasks_base_path) -> Tuple[TaggedCorpus, TextRegressor, ModelTrainer]:
corpus = NLPTaskDataFetcher.load_corpus(NLPTask.REGRESSION, tasks_base_path)

glove_embedding: WordEmbeddings = WordEmbeddings("glove")
document_embeddings: DocumentRNNEmbeddings = DocumentRNNEmbeddings(
[glove_embedding], 128, 1, False, 64, False, False
)

model = TextRegressor(document_embeddings, Dictionary(), False)
model = TextRegressor(document_embeddings)

trainer = ModelTrainer(model, corpus)

Expand All @@ -40,7 +40,7 @@ def test_labels_to_indices(tasks_base_path):
def test_trainer_evaluation(tasks_base_path):
corpus, model, trainer = init(tasks_base_path)

expected = trainer._evaluate_text_regressor(model, corpus.dev)
expected = model.evaluate(corpus.dev)

assert expected is not None

Expand Down

0 comments on commit 5e77d62

Please sign in to comment.