From 9ba0aeb05584d7249f863fc5be0f37f3bcb42e95 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Tue, 27 Sep 2022 17:19:23 -0400 Subject: [PATCH 01/25] Minor changes to make these tests pass if a cuda device is available. --- tests/test_seq2labels.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_seq2labels.py b/tests/test_seq2labels.py index 877786f..9a485dd 100644 --- a/tests/test_seq2labels.py +++ b/tests/test_seq2labels.py @@ -8,6 +8,7 @@ from allennlp.data import Token from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder from allennlp.data.vocabulary import Vocabulary +from allennlp.nn.util import move_to_device from gector.bert_token_embedder import PretrainedBertEmbedder from gector.tokenizer_indexer import PretrainedBertIndexer @@ -49,6 +50,7 @@ def setUp(self): self.device = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu" ) + self.device_index = 0 if torch.cuda.is_available() else -1 sentence1 = "the quickest quick brown fox jumped over the lazy dog" tokens1 = [Token(word) for word in sentence1.split()] @@ -71,7 +73,7 @@ def test_forward_pass_runs_correctly(self): self.batch.index_instances(self.vocab) padding_lengths = self.batch.get_padding_lengths() training_tensors = self.batch.as_tensor_dict(padding_lengths) - + training_tensors = move_to_device(training_tensors, self.device_index) model = Seq2Labels( vocab=self.vocab, text_field_embedder=self.text_field_embedder, @@ -93,6 +95,6 @@ def test_forward_pass_runs_correctly(self): "d_tags", ] ) - probs = output_dict["class_probabilities_labels"][0].data.numpy() + probs = output_dict["class_probabilities_labels"][0].data.cpu().numpy() np.testing.assert_almost_equal(np.sum(probs, -1), np.full((10), 1), 5) From 34386bd936eb97343604b9abb0c53a5376cd6107 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Tue, 27 Sep 2022 17:41:45 -0400 Subject: [PATCH 02/25] Adding registered names for use by predictor --- gector/bert_token_embedder.py | 2 +- gector/tokenizer_indexer.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/gector/bert_token_embedder.py b/gector/bert_token_embedder.py index 327cd44..fcd7b6b 100644 --- a/gector/bert_token_embedder.py +++ b/gector/bert_token_embedder.py @@ -221,7 +221,7 @@ def forward( return util.uncombine_initial_dims(selected_embeddings, offsets.size()) -# @TokenEmbedder.register("bert-pretrained") +@TokenEmbedder.register("gec-bert-pretrained") class PretrainedBertEmbedder(BertEmbedder): """ diff --git a/gector/tokenizer_indexer.py b/gector/tokenizer_indexer.py index 4f086d3..7ed2a5b 100644 --- a/gector/tokenizer_indexer.py +++ b/gector/tokenizer_indexer.py @@ -101,6 +101,7 @@ def get_keys(self, index_name: str) -> List[str]: return [index_name, f"{index_name}-offsets", f"{index_name}-type-ids", "mask"] +@TokenIndexer.register("gec-pretrained-bert-indexer") class PretrainedBertIndexer(TokenizerIndexer): # pylint: disable=line-too-long """ From 36ab3590e3e0899256318e83d0b1b2d436374706 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Tue, 27 Sep 2022 17:48:40 -0400 Subject: [PATCH 03/25] Adding expected test output. --- tests/test_gec_predictor.py | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/test_gec_predictor.py diff --git a/tests/test_gec_predictor.py b/tests/test_gec_predictor.py new file mode 100644 index 0000000..968afe7 --- /dev/null +++ b/tests/test_gec_predictor.py @@ -0,0 +1,53 @@ +from pathlib import Path +import torch +import requests +from tqdm import tqdm + +from allennlp.common.testing import ModelTestCase +from allennlp.predictors import Predictor +from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder +from allennlp.data.vocabulary import Vocabulary +from allennlp.data import Token +from allennlp.data.instance import Instance +from allennlp.data.fields import TextField +from allennlp.data.dataset import Batch + +from gector.gec_predictor import GecPredictor +from gector.gec_model import GecBERTModel +from gector.bert_token_embedder import PretrainedBertEmbedder +from gector.tokenizer_indexer import PretrainedBertIndexer + +# These imports are required so that instantiating the predictor can be done automatically +from gector.datareader import Seq2LabelsDatasetReader +from gector.seq2labels_model import Seq2Labels +from gector.bert_token_embedder import PretrainedBertEmbedder +from gector.tokenizer_indexer import PretrainedBertIndexer + + + + +class TestGecPredictor(ModelTestCase): + """Test class for GecModel""" + + def setUp(self): + super().setUp() + test_fixtures_dir_path = Path(__file__).parent.parent / "test_fixtures" + model_path = test_fixtures_dir_path / "roberta_model" + self.model_path = model_path + sentence1 = "I run to a stores every day." + sentence2 = "the quick brown foxes jumps over a elmo's laziest friend" + # This micmics how batches of requests are constructed in predict.py's predict_for_file function + self.input_data = [sentence1, sentence2] + + def test_gec_predictor_prediction(self): + """Test simple prediction integration test with GecPredictor""" + gec_model = Predictor.from_path(self.model_path, predictor_name="gec-predictor") + prediction = gec_model.predict_batch(self.input_data) + # subject verb agreement is not fixed in the second sentence when predicting using GecModel + # (i.e.) brown foxes jump + assert set(prediction[0].keys()) == {'logits_labels', 'logits_d_tags', 'class_probabilities_labels', + 'class_probabilities_d_tags', 'max_error_probability', 'words', 'labels', + 'd_tags', 'corrected_words'} + assert prediction[0]["corrected_words"] == ['I', 'run', 'to', 'the', 'stores', 'every', 'day.'] + assert prediction[1]["corrected_words"] == ['The', 'quick', 'brown', 'foxes', 'jumps', 'over', "Elmo's", + 'laziest', 'friend'] From 9bd6a7b7051871638d811cbc598a831184e21d26 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Tue, 27 Sep 2022 17:50:21 -0400 Subject: [PATCH 04/25] Added WIP docstring to GecBERTModel --- gector/gec_model.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/gector/gec_model.py b/gector/gec_model.py index 7cc636b..0484611 100644 --- a/gector/gec_model.py +++ b/gector/gec_model.py @@ -39,6 +39,28 @@ def __init__(self, vocab_path=None, model_paths=None, del_confidence=0, resolve_cycles=False, ): + """ + Class used to enable prediction from GECtor model. + + Parameters + ---------- + vocab_path + model_paths: List[Path] + weigths + max_len + min_len + lowercase_tokens + log + iterations + model_name + special_tokens_fix + is_ensemble + min_error_probability + confidence + del_confidence + resolve_cycles: bool + This parameter is unused. + """ self.model_weights = list(map(float, weigths)) if weigths else [1] * len(model_paths) self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") self.max_len = max_len From 6f3f7ae35bcaa4dad45ea4c6f349402de76fe868 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Tue, 27 Sep 2022 17:50:33 -0400 Subject: [PATCH 05/25] WIP Gec Predictor. --- gector/gec_predictor.py | 84 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 gector/gec_predictor.py diff --git a/gector/gec_predictor.py b/gector/gec_predictor.py new file mode 100644 index 0000000..a248393 --- /dev/null +++ b/gector/gec_predictor.py @@ -0,0 +1,84 @@ +from typing import Dict, List + +import numpy +from allennlp.predictors import Predictor +from allennlp.models import Model +from allennlp.common.util import sanitize +from overrides import overrides +from allennlp.common.util import JsonDict +from allennlp.data import DatasetReader, Instance +from allennlp.data.fields import TextField, SequenceLabelField +from allennlp.data.tokenizers.word_splitter import SpacyWordSplitter +from allennlp.models import Model + +from utils.helpers import PAD, UNK + + +@Predictor.register("gec-predictor") +class GecPredictor(Predictor): + """ + A Predictor for generating predictions from GECtor. + + Note that currently, this is unable to handle ensemble predictions. + """ + def __init__(self, model: Model, + dataset_reader: DatasetReader, + language: str = 'en_core_web_sm', + iterations: int = 3) -> None: + super().__init__(model, dataset_reader) + self._tokenizer = SpacyWordSplitter(language=language, pos_tags=True) + self._iterations = iterations + + def predict(self, sentence: str) -> JsonDict: + return self.predict_json({"sentence": sentence}) + + def predict_batch(self, sentences: List[str]) -> JsonDict: + return self.predict_batch_json([{"sentence": sentence} for sentence in sentences]) + + @overrides + def predict_instance(self, instance: Instance) -> JsonDict: + for i in range(self._iterations): + outputs = self._model.forward_on_instance(instance) + # integrate predictions back into instance for next iteration + return sanitize(outputs) + + @overrides + def _json_to_instance(self, json_dict: JsonDict) -> Instance: + """ + Expects JSON that looks like ``{"sentence": "..."}``. + Runs the underlying model, and adds the ``"words"`` to the output. + """ + sentence = json_dict["sentence"] + tokens = self._tokenizer.split_words(sentence) + return self._dataset_reader.text_to_instance(tokens) + + def get_token_action(self, token, index, prob, sugg_token): + """Get list of suggested actions for token.""" + # cases when we don't need to do anything + if prob < self.min_error_probability or sugg_token in [UNK, PAD, '$KEEP']: + return None + + if sugg_token.startswith('$REPLACE_') or sugg_token.startswith('$TRANSFORM_') or sugg_token == '$DELETE': + start_pos = index + end_pos = index + 1 + elif sugg_token.startswith("$APPEND_") or sugg_token.startswith("$MERGE_"): + start_pos = index + 1 + end_pos = index + 1 + + if sugg_token == "$DELETE": + sugg_token_clear = "" + elif sugg_token.startswith('$TRANSFORM_') or sugg_token.startswith("$MERGE_"): + sugg_token_clear = sugg_token[:] + else: + sugg_token_clear = sugg_token[sugg_token.index('_') + 1:] + + return start_pos - 1, end_pos - 1, sugg_token_clear, prob + + @overrides + def predictions_to_labeled_instances(self, + instance: Instance, + outputs: Dict[str, numpy.ndarray]) -> List[Instance]: + """ + This method creates an instance out of the predictions generated by the model. + """ + NotImplemented \ No newline at end of file From 9dd6bf181757c2d7ad23cad1a4ac3b7a6eb5fde8 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 09:35:09 -0400 Subject: [PATCH 06/25] words metadata is getting filled if unspecified when text_to_instance is called. --- gector/datareader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gector/datareader.py b/gector/datareader.py index d511afb..5642f19 100644 --- a/gector/datareader.py +++ b/gector/datareader.py @@ -129,6 +129,9 @@ def text_to_instance(self, tokens: List[Token], tags: List[str] = None, fields: Dict[str, Field] = {} sequence = TextField(tokens, self._token_indexers) fields["tokens"] = sequence + # If words has not been explicitly passed in, create it from tokens. + if words is None: + words = [token.text for token in tokens] fields["metadata"] = MetadataField({"words": words}) if tags is not None: labels, detect_tags, complex_flag_dict = self.extract_tags(tags) From 64cbc8d19d19f03c43396d58c7248b2e6894205b Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 12:44:13 -0400 Subject: [PATCH 07/25] Using JustSpacesWordSplitter so that tokenization matches that used by gec_model, get_token_action removed and placed in model. --- gector/gec_predictor.py | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/gector/gec_predictor.py b/gector/gec_predictor.py index a248393..8a91685 100644 --- a/gector/gec_predictor.py +++ b/gector/gec_predictor.py @@ -9,9 +9,9 @@ from allennlp.data import DatasetReader, Instance from allennlp.data.fields import TextField, SequenceLabelField from allennlp.data.tokenizers.word_splitter import SpacyWordSplitter +from allennlp.data.tokenizers.word_splitter import JustSpacesWordSplitter from allennlp.models import Model -from utils.helpers import PAD, UNK @Predictor.register("gec-predictor") @@ -26,7 +26,8 @@ def __init__(self, model: Model, language: str = 'en_core_web_sm', iterations: int = 3) -> None: super().__init__(model, dataset_reader) - self._tokenizer = SpacyWordSplitter(language=language, pos_tags=True) + #self._tokenizer = SpacyWordSplitter(language=language, pos_tags=True) + self._tokenizer = JustSpacesWordSplitter() self._iterations = iterations def predict(self, sentence: str) -> JsonDict: @@ -52,27 +53,6 @@ def _json_to_instance(self, json_dict: JsonDict) -> Instance: tokens = self._tokenizer.split_words(sentence) return self._dataset_reader.text_to_instance(tokens) - def get_token_action(self, token, index, prob, sugg_token): - """Get list of suggested actions for token.""" - # cases when we don't need to do anything - if prob < self.min_error_probability or sugg_token in [UNK, PAD, '$KEEP']: - return None - - if sugg_token.startswith('$REPLACE_') or sugg_token.startswith('$TRANSFORM_') or sugg_token == '$DELETE': - start_pos = index - end_pos = index + 1 - elif sugg_token.startswith("$APPEND_") or sugg_token.startswith("$MERGE_"): - start_pos = index + 1 - end_pos = index + 1 - - if sugg_token == "$DELETE": - sugg_token_clear = "" - elif sugg_token.startswith('$TRANSFORM_') or sugg_token.startswith("$MERGE_"): - sugg_token_clear = sugg_token[:] - else: - sugg_token_clear = sugg_token[sugg_token.index('_') + 1:] - - return start_pos - 1, end_pos - 1, sugg_token_clear, prob @overrides def predictions_to_labeled_instances(self, From 10e80695a0c4e9994aad800f7ce5a6310f383a1c Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 12:45:47 -0400 Subject: [PATCH 08/25] Decode now adds the corrected sentence to the output dict. There's a bug in here somewhere though because the output isn't exactly the same. I suspect there's an off by one error happening when the indices for edit operations are generated. --- gector/seq2labels_model.py | 45 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/gector/seq2labels_model.py b/gector/seq2labels_model.py index cdd6f59..f625fbb 100644 --- a/gector/seq2labels_model.py +++ b/gector/seq2labels_model.py @@ -12,6 +12,7 @@ from allennlp.training.metrics import CategoricalAccuracy from overrides import overrides from torch.nn.modules.linear import Linear +from utils.helpers import PAD, UNK, get_target_sent_by_edits @Model.register("seq2labels") @@ -60,6 +61,7 @@ def __init__(self, vocab: Vocabulary, label_smoothing: float = 0.0, confidence: float = 0.0, del_confidence: float = 0.0, + min_error_probability: float = 0.0, initializer: InitializerApplicator = InitializerApplicator(), regularizer: Optional[RegularizerApplicator] = None) -> None: super(Seq2Labels, self).__init__(vocab, regularizer) @@ -72,6 +74,7 @@ def __init__(self, vocab: Vocabulary, self.label_smoothing = label_smoothing self.confidence = confidence self.del_conf = del_confidence + self.min_error_probability = min_error_probability self.incorr_index = self.vocab.get_token_index("INCORRECT", namespace=detect_namespace) @@ -185,8 +188,50 @@ def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor for x in argmax_indices] all_tags.append(tags) output_dict[f'{label_namespace}'] = all_tags + batch_size = len(output_dict['labels']) + output_dict['corrected_words'] = [] + for i in range(batch_size): + probs = output_dict['class_probabilities_labels'][i] + max_probs = torch.max(probs, dim=0) + probs = max_probs[0].tolist() + indices = max_probs[1].tolist() + print(f"{indices=}") + suggested_token_operations = output_dict['labels'][i] + actions_per_token = self.get_token_actions(indices=indices, probs=probs, + sugg_tokens=suggested_token_operations) + # Filter out cases where no corrections should be applied + actions_per_token = [action for action in actions_per_token if action] + print(f"{actions_per_token=}") + corrected_sent = get_target_sent_by_edits(output_dict['words'][i], actions_per_token) + output_dict['corrected_words'].append(corrected_sent) return output_dict + def get_token_actions(self, indices, probs, sugg_tokens): + """Get list of suggested actions for token.""" + actions = [] + for index, prob, sugg_token in zip(indices, probs, sugg_tokens): + # cases when we don't need to do anything + if prob < self.min_error_probability or sugg_token in [UNK, PAD, '$KEEP']: + actions.append(None) + continue + + if sugg_token.startswith('$REPLACE_') or sugg_token.startswith('$TRANSFORM_') or sugg_token == '$DELETE': + start_pos = index + end_pos = index + 1 + elif sugg_token.startswith("$APPEND_") or sugg_token.startswith("$MERGE_"): + start_pos = index + 1 + end_pos = index + 1 + + if sugg_token == "$DELETE": + sugg_token_clear = "" + elif sugg_token.startswith('$TRANSFORM_') or sugg_token.startswith("$MERGE_"): + sugg_token_clear = sugg_token[:] + else: + print(f"{sugg_token=}") + sugg_token_clear = sugg_token[sugg_token.index('_') + 1:] + actions.append((start_pos, end_pos, sugg_token_clear, prob)) + return actions + @overrides def get_metrics(self, reset: bool = False) -> Dict[str, float]: metrics_to_return = {metric_name: metric.get_metric(reset) for From f6a91859652052a41aa19189db2dee05eb02a5be Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 13:32:16 -0400 Subject: [PATCH 09/25] Updating gitignore to prevent adding .th files --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index fcf6afd..9f1d7eb 100644 --- a/.gitignore +++ b/.gitignore @@ -132,4 +132,7 @@ dmypy.json # PyCharm .idea -*.sh \ No newline at end of file +*.sh + +# pytorch weights files +*.th From 657f72ba9bbac59f0b6713e958274e0cb693b1d9 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 13:34:47 -0400 Subject: [PATCH 10/25] Adding directory fixture as analogue to model archive --- test_fixtures/roberta_model/config.json | 61 + .../roberta_model/vocabulary/d_tags.txt | 4 + .../roberta_model/vocabulary/labels.txt | 5002 +++++++++++++++++ .../vocabulary/non_padded_namespaces.txt | 2 + 4 files changed, 5069 insertions(+) create mode 100644 test_fixtures/roberta_model/config.json create mode 100644 test_fixtures/roberta_model/vocabulary/d_tags.txt create mode 100644 test_fixtures/roberta_model/vocabulary/labels.txt create mode 100644 test_fixtures/roberta_model/vocabulary/non_padded_namespaces.txt diff --git a/test_fixtures/roberta_model/config.json b/test_fixtures/roberta_model/config.json new file mode 100644 index 0000000..6b63071 --- /dev/null +++ b/test_fixtures/roberta_model/config.json @@ -0,0 +1,61 @@ +{ + "dataset_reader": { + "type": "seq2labels_datareader", + "max_len": 50, + "skip_correct": true, + "skip_complex": 0, + "test_mode": false, + "tag_strategy": "keep_one", + "broken_dot_strategy": "keep", + "lazy": true, + "tn_prob": 0, + "tp_prob": 1, + "token_indexers": { + "bert": { + "type": "gec-pretrained-bert-indexer", + "pretrained_model": "roberta-base", + "max_pieces_per_token": 5, + "do_lowercase": false, + "special_tokens_fix": 1 + } + } + }, + "iterator": { + "type": "bucket", + "batch_size": 32, + "sorting_keys": [ + [ + "tokens", + "num_tokens" + ] + ] + }, + "model": { + "type": "seq2labels", + "text_field_embedder": { + "allow_unmatched_keys": true, + "embedder_to_indexer_map": {"bert": ["bert", "bert-offsets"]}, + "token_embedders": { + "bert": { + "type": "gec-bert-pretrained", + "requires_grad": false, + "top_layer_only": true, + "pretrained_model": "roberta-base", + "special_tokens_fix": 1 + } + } + } + }, + "trainer": { + "cuda_device": 0, + "grad_clipping": 5, + "num_epochs": 50, + "optimizer": { + "type": "adagrad", + "lr": 0.001 + }, + "patience": 10, + "validation_metric": "-loss" + }, + "evaluate_on_test": true +} \ No newline at end of file diff --git a/test_fixtures/roberta_model/vocabulary/d_tags.txt b/test_fixtures/roberta_model/vocabulary/d_tags.txt new file mode 100644 index 0000000..56e0504 --- /dev/null +++ b/test_fixtures/roberta_model/vocabulary/d_tags.txt @@ -0,0 +1,4 @@ +CORRECT +INCORRECT +@@UNKNOWN@@ +@@PADDING@@ diff --git a/test_fixtures/roberta_model/vocabulary/labels.txt b/test_fixtures/roberta_model/vocabulary/labels.txt new file mode 100644 index 0000000..0086d2a --- /dev/null +++ b/test_fixtures/roberta_model/vocabulary/labels.txt @@ -0,0 +1,5002 @@ +$KEEP +$DELETE +$TRANSFORM_CASE_CAPITAL +$APPEND_the +$APPEND_, +$APPEND_a +$TRANSFORM_VERB_VB_VBZ +$TRANSFORM_AGREEMENT_PLURAL +$TRANSFORM_CASE_LOWER +$TRANSFORM_VERB_VB_VBN +$REPLACE_the +$REPLACE_a +$REPLACE_to +$TRANSFORM_VERB_VB_VBG +$REPLACE_. +$APPEND_to +$REPLACE_, +$REPLACE_in +$REPLACE_was +$TRANSFORM_VERB_VBZ_VB +$TRANSFORM_AGREEMENT_SINGULAR +$APPEND_I +$APPEND_. +$REPLACE_for +$REPLACE_I +$APPEND_( +$TRANSFORM_VERB_VBG_VB +$REPLACE_is +$REPLACE_have +$REPLACE_on +$REPLACE_are +$REPLACE_of +$REPLACE_it +$TRANSFORM_VERB_VBN_VB +$REPLACE_that +$APPEND_in +$REPLACE_and +$APPEND_it +$APPEND_that +$REPLACE_at +$APPEND_for +$APPEND_of +$APPEND_and +$REPLACE_an +$REPLACE_my +$REPLACE_had +$APPEND_have +$APPEND_an +$REPLACE_has +$APPEND_my +$APPEND_is +$APPEND_The +$APPEND_will +$REPLACE_with +$REPLACE_were +$REPLACE_be +$TRANSFORM_VERB_VBN_VBG +$APPEND_`` +$REPLACE_do +$TRANSFORM_VERB_VBG_VBN +$REPLACE_this +$REPLACE_will +$TRANSFORM_VERB_VB_VBD +$APPEND_was +$REPLACE_n't +$APPEND_about +$REPLACE_from +$REPLACE_about +$REPLACE_It +$APPEND_on +$REPLACE_would +$MERGE_SPACE +$APPEND_at +$APPEND_'s +$REPLACE_as +$REPLACE_'s +$REPLACE_could +$APPEND_with +$REPLACE_did +$REPLACE_them +$REPLACE_The +$REPLACE_by +$REPLACE_so +$REPLACE_not +$REPLACE_can +$APPEND_am +$APPEND_be +$REPLACE_because +$APPEND_/ +$REPLACE_they +$REPLACE_am +$APPEND_are +$TRANSFORM_VERB_VBZ_VBN +$REPLACE_'m +$REPLACE_their +$TRANSFORM_VERB_VBN_VBZ +$APPEND_had +$APPEND_would +$APPEND_- +$REPLACE_( +$TRANSFORM_VERB_VBN_VBD +$REPLACE_very +$REPLACE_people +$REPLACE_get +$REPLACE_there +$REPLACE_? +$APPEND_do +$REPLACE_; +$REPLACE_me +$REPLACE_one +$REPLACE_been +$APPEND_so +$APPEND_) +$APPEND_'m +$REPLACE_or +$REPLACE_some +$REPLACE_you +$TRANSFORM_VERB_VBD_VBN +$APPEND_as +$REPLACE_like +$TRANSFORM_VERB_VBD_VB +$REPLACE_which +$APPEND_has +$REPLACE_these +$REPLACE_This +$APPEND_from +$REPLACE_when +$APPEND_'ve +$REPLACE_`` +$APPEND_there +$REPLACE_does +$APPEND_also +$APPEND_It +$APPEND_can +$REPLACE_: +$REPLACE_other +$APPEND_more +$REPLACE_want +$REPLACE_we +$REPLACE_'ve +$REPLACE_what +$REPLACE_more +$REPLACE_many +$REPLACE_into +$APPEND_been +$APPEND_by +$APPEND_this +$REPLACE_went +$REPLACE_time +$APPEND_only +$TRANSFORM_VERB_VBG_VBZ +$REPLACE_go +$REPLACE_while +$REPLACE_but +$APPEND_all +$APPEND_if +$REPLACE_should +$REPLACE_out +$APPEND_' +$REPLACE_during +$REPLACE_much +$APPEND_like +$REPLACE_! +$APPEND_but +$REPLACE_if +$REPLACE_since +$APPEND_people +$APPEND_because +$REPLACE_any +$APPEND_A +$REPLACE_another +$REPLACE_They +$APPEND_you +$REPLACE_ca +$REPLACE_our +$REPLACE_who +$APPEND_now +$REPLACE_really +$REPLACE_make +$APPEND_me +$APPEND_who +$REPLACE_In +$REPLACE_her +$REPLACE_English +$APPEND_some +$APPEND_when +$APPEND_still +$APPEND_them +$REPLACE_use +$APPEND_just +$REPLACE_things +$REPLACE_/ +$REPLACE_got +$REPLACE_My +$APPEND_were +$REPLACE_he +$REPLACE_countries +$APPEND_their +$REPLACE_using +$TRANSFORM_VERB_VBZ_VBG +$APPEND_'ll +$REPLACE_being +$REPLACE_too +$APPEND_we +$APPEND_they +$REPLACE_lot +$REPLACE_- +$REPLACE_all +$REPLACE_good +$APPEND_[ +$REPLACE_every +$REPLACE_) +$REPLACE_your +$APPEND_My +$APPEND_even +$APPEND_out +$REPLACE_his +$REPLACE_made +$APPEND_any +$REPLACE_where +$APPEND_which +$REPLACE_work +$REPLACE_used +$APPEND_one +$REPLACE_take +$APPEND_In +$REPLACE_There +$REPLACE_up +$REPLACE_how +$REPLACE_myself +$APPEND_what +$APPEND_very +$APPEND_? +$REPLACE_become +$REPLACE_think +$REPLACE_going +$REPLACE_Japanese +$REPLACE_well +$APPEND_being +$APPEND_or +$REPLACE_just +$REPLACE_write +$REPLACE_those +$REPLACE_feel +$REPLACE_until +$APPEND_However +$APPEND_our +$REPLACE_something +$APPEND_get +$REPLACE_diary +$REPLACE_no +$REPLACE_over +$APPEND_time +$APPEND_then +$REPLACE_see +$REPLACE_writing +$REPLACE_wo +$REPLACE_only +$REPLACE_'ll +$REPLACE_after +$REPLACE_know +$REPLACE_anything +$REPLACE_now +$REPLACE_That +$REPLACE_first +$REPLACE_than +$APPEND_up +$REPLACE_better +$REPLACE_hope +$REPLACE_through +$REPLACE_doing +$APPEND_go +$REPLACE_then +$APPEND_too +$REPLACE_studying +$REPLACE_its +$REPLACE_learn +$REPLACE_lives +$REPLACE_having +$REPLACE_told +$REPLACE_What +$REPLACE_she +$REPLACE_thought +$APPEND_not +$REPLACE_around +$REPLACE_him +$REPLACE_different +$APPEND_could +$APPEND_such +$REPLACE_able +$REPLACE_On +$REPLACE_before +$REPLACE_though +$REPLACE_also +$APPEND_entry +$REPLACE_learned +$TRANSFORM_CASE_UPPER +$APPEND_again +$REPLACE_friends +$APPEND_This +$REPLACE_might +$REPLACE_A +$REPLACE_However +$APPEND_really +$REPLACE_started +$REPLACE_improve +$APPEND_English +$REPLACE_years +$REPLACE_' +$REPLACE_most +$APPEND_how +$REPLACE_day +$APPEND_: +$APPEND_today +$REPLACE_find +$REPLACE_help +$APPEND_should +$REPLACE_We +$REPLACE_even +$REPLACE_may +$REPLACE_left +$REPLACE_called +$APPEND_did +$REPLACE_course +$REPLACE_These +$REPLACE_understand +$REPLACE_So +$REPLACE_said +$REPLACE_took +$REPLACE_person +$REPLACE_school +$REPLACE_such +$APPEND_called +$REPLACE_At +$APPEND_before +$REPLACE_way +$APPEND_he +$REPLACE_everyone +$REPLACE_here +$REPLACE_When +$REPLACE_everything +$REPLACE_need +$APPEND_her +$REPLACE_Because +$TRANSFORM_VERB_VBD_VBG +$REPLACE_say +$REPLACE_study +$APPEND_much +$REPLACE_still +$REPLACE_found +$APPEND_always +$REPLACE_last +$APPEND_other +$TRANSFORM_VERB_VBG_VBD +$REPLACE_learning +$REPLACE_correct +$REPLACE_two +$REPLACE_days +$REPLACE_difficult +$REPLACE_never +$APPEND__ +$REPLACE_'d +$APPEND_your +$REPLACE_us +$REPLACE_foreign +$REPLACE_entry +$APPEND_! +$REPLACE_Japan +$APPEND_; +$REPLACE_tell +$REPLACE_give +$REPLACE_decided +$APPEND_during +$REPLACE_Also +$APPEND_his +$REPLACE_speak +$REPLACE_came +$REPLACE_little +$APPEND_while +$TRANSFORM_VERB_VBZ_VBD +$APPEND_things +$REPLACE_especially +$REPLACE_Recently +$REPLACE_come +$APPEND_especially +$REPLACE_needed +$APPEND_make +$REPLACE_whether +$REPLACE_felt +$REPLACE_Although +$REPLACE_someone +$REPLACE_As +$REPLACE_great +$REPLACE_today +$APPEND_since +$REPLACE_hard +$REPLACE_For +$REPLACE_became +$REPLACE_between +$REPLACE_beautiful +$REPLACE_life +$REPLACE_why +$APPEND_though +$APPEND_There +$APPEND_going +$REPLACE_long +$APPEND_where +$REPLACE_believe +$REPLACE_website +$REPLACE_heard +$REPLACE_job +$REPLACE_home +$REPLACE_'re +$REPLACE_But +$REPLACE_anyone +$REPLACE_again +$REPLACE_bad +$REPLACE_recently +$APPEND_here +$REPLACE_practice +$REPLACE_often +$APPEND_got +$APPEND_feel +$REPLACE_saw +$REPLACE_quickly +$REPLACE_language +$REPLACE_wanted +$APPEND_each +$REPLACE_put +$REPLACE_done +$REPLACE_minutes +$REPLACE_each +$APPEND_she +$REPLACE_grammar +$REPLACE_watch +$REPLACE_happy +$REPLACE_back +$REPLACE_friend +$REPLACE_off +$REPLACE_He +$REPLACE_Since +$APPEND_something +$APPEND_using +$APPEND_At +$REPLACE_university +$REPLACE_country +$REPLACE_watching +$REPLACE_received +$REPLACE_enough +$REPLACE_weather +$REPLACE_usually +$APPEND_back +$REPLACE_happened +$APPEND_having +$REPLACE_always +$APPEND_does +$REPLACE_After +$REPLACE_try +$REPLACE_start +$APPEND_already +$REPLACE_talk +$REPLACE_thing +$APPEND_But +$APPEND_For +$REPLACE_Then +$REPLACE_fun +$REPLACE_soon +$REPLACE_starting +$REPLACE_away +$APPEND_want +$REPLACE_asked +$APPEND_went +$REPLACE_trip +$REPLACE_new +$REPLACE_right +$APPEND_after +$REPLACE_keep +$REPLACE_interesting +$REPLACE_together +$REPLACE_Do +$APPEND_So +$REPLACE_beginning +$APPEND_myself +$REPLACE_getting +$APPEND_On +$REPLACE_restaurant +$REPLACE_looking +$REPLACE_children +$APPEND_last +$REPLACE_college +$APPEND_right +$REPLACE_stay +$REPLACE_year +$REPLACE_live +$REPLACE_travel +$REPLACE_favorite +$REPLACE_read +$APPEND_well +$REPLACE_written +$REPLACE_months +$APPEND_yet +$APPEND_first +$APPEND_most +$REPLACE_look +$REPLACE_tried +$REPLACE_clothes +$REPLACE_[ +$REPLACE_kind +$APPEND_its +$REPLACE_& +$REPLACE_remember +$APPEND_him +$REPLACE_problem +$APPEND_* +$REPLACE_meet +$REPLACE_gave +$REPLACE_either +$REPLACE_makes +$REPLACE_elderly +$REPLACE_hobbies +$REPLACE_easily +$REPLACE_important +$APPEND_take +$APPEND_thing +$REPLACE_vocabulary +$REPLACE_listening +$REPLACE_must +$REPLACE_hours +$REPLACE_place +$REPLACE_While +$REPLACE_without +$REPLACE_end +$REPLACE_Korean +$REPLACE_Therefore +$REPLACE_working +$REPLACE_high +$REPLACE_house +$REPLACE_already +$APPEND_good +$REPLACE_opportunity +$APPEND_many +$REPLACE_family +$REPLACE_During +$REPLACE_First +$APPEND_both +$REPLACE_once +$REPLACE_experience +$REPLACE_tomorrow +$APPEND_these +$REPLACE_true +$APPEND_day +$REPLACE_leave +$APPEND_When +$REPLACE_watched +$APPEND_person +$REPLACE_best +$REPLACE_harder +$REPLACE_Today +$REPLACE_morning +$REPLACE_If +$REPLACE_woke +$APPEND_into +$APPEND_made +$REPLACE_foreigners +$REPLACE_part +$APPEND_ever +$APPEND_probably +$APPEND_way +$APPEND_over +$APPEND_n't +$REPLACE_towards +$REPLACE_three +$REPLACE_One +$REPLACE_studied +$REPLACE_nervous +$REPLACE_forward +$REPLACE_seen +$REPLACE_Chinese +$REPLACE_night +$APPEND_own +$REPLACE_taught +$APPEND_usually +$REPLACE_To +$REPLACE_communicate +$APPEND_Japanese +$REPLACE_entries +$REPLACE_traveling +$REPLACE_site +$REPLACE_difference +$APPEND_those +$TRANSFORM_VERB_VBD_VBZ +$REPLACE_rainy +$REPLACE_play +$REPLACE_comfortable +$REPLACE_recommend +$REPLACE_coming +$REPLACE_Is +$REPLACE_asleep +$REPLACE_realized +$APPEND_recently +$APPEND_around +$REPLACE_men +$REPLACE_Finally +$REPLACE_excited +$REPLACE_near +$APPEND_often +$REPLACE_t +$REPLACE_next +$REPLACE_ever +$APPEND_Today +$REPLACE_taking +$APPEND_started +$REPLACE_please +$APPEND_than +$REPLACE_sentences +$APPEND_What +$REPLACE_She +$APPEND_work +$REPLACE_visit +$REPLACE_surprised +$REPLACE_show +$REPLACE_You +$APPEND_used +$REPLACE_ago +$APPEND_Even +$APPEND_That +$REPLACE_similar +$APPEND_soon +$REPLACE_less +$REPLACE_enjoy +$REPLACE_diaries +$REPLACE_speaking +$REPLACE_past +$APPEND_through +$REPLACE_women +$REPLACE_planned +$REPLACE_later +$REPLACE_looked +$REPLACE_yet +$APPEND_us +$REPLACE_And +$APPEND_'d +$APPEND_As +$REPLACE_healthy +$APPEND_might +$REPLACE_class +$REPLACE_Now +$REPLACE_outside +$REPLACE_tired +$APPEND_else +$REPLACE_Please +$REPLACE_problems +$APPEND_They +$REPLACE_food +$REPLACE_reading +$APPEND_& +$APPEND_think +$REPLACE_finished +$REPLACE_popular +$REPLACE_Are +$APPEND_2 +$APPEND_may +$APPEND_found +$APPEND_whether +$APPEND_We +$REPLACE_How +$REPLACE_continue +$REPLACE_everyday +$REPLACE_daily +$REPLACE_talked +$APPEND_new +$REPLACE_reason +$REPLACE_means +$REPLACE_opportunities +$APPEND_different +$REPLACE_business +$REPLACE_making +$APPEND_ago +$REPLACE_favourite +$REPLACE_bit +$REPLACE_delicious +$APPEND_every +$REPLACE_spend +$APPEND_finally +$APPEND_part +$REPLACE_yesterday +$REPLACE_down +$REPLACE_times +$REPLACE_holiday +$REPLACE_nice +$REPLACE_although +$REPLACE_earlier +$REPLACE_Can +$REPLACE_due +$APPEND_help +$REPLACE_caught +$REPLACE_quite +$APPEND_kind +$REPLACE_words +$REPLACE_movie +$REPLACE_else +$APPEND_together +$REPLACE_advertisement +$APPEND_Is +$APPEND_between +$APPEND_enough +$REPLACE_let +$REPLACE_instead +$REPLACE_disappointed +$REPLACE_Have +$APPEND_After +$APPEND_no +$APPEND_doing +$REPLACE_skills +$APPEND_instead +$REPLACE_Some +$REPLACE_Actually +$APPEND_3 +$REPLACE_choose +$REPLACE_An +$APPEND_away +$REPLACE_Does +$REPLACE_played +$APPEND_Because +$REPLACE_both +$REPLACE_easier +$REPLACE_others +$REPLACE_eat +$REPLACE_onto +$REPLACE_sometimes +$REPLACE_began +$REPLACE_usual +$REPLACE_expensive +$APPEND_To +$APPEND_actually +$REPLACE_old +$APPEND_see +$APPEND_know +$REPLACE_few +$APPEND_why +$APPEND_sometimes +$REPLACE_Unfortunately +$APPEND_use +$REPLACE_older +$REPLACE_joined +$REPLACE_own +$REPLACE_raining +$REPLACE_themselves +$REPLACE_example +$APPEND_able +$REPLACE_arrived +$REPLACE_whom +$REPLACE_nothing +$REPLACE_fluently +$APPEND_getting +$REPLACE_convenient +$REPLACE_met +$REPLACE_becoming +$APPEND_better +$APPEND_become +$REPLACE_lots +$REPLACE_fast +$REPLACE_memories +$REPLACE_worse +$REPLACE_interested +$REPLACE_hear +$REPLACE_Secondly +$REPLACE_thoughts +$REPLACE_journal +$REPLACE_bought +$REPLACE_useful +$REPLACE_teach +$APPEND_learn +$REPLACE_throughout +$REPLACE_money +$REPLACE_change +$REPLACE_imagine +$REPLACE_late +$REPLACE_mine +$REPLACE_same +$REPLACE_future +$REPLACE_sure +$REPLACE_students +$REPLACE_along +$REPLACE_exercise +$REPLACE_opinion +$REPLACE_return +$REPLACE_cause +$REPLACE_month +$REPLACE_stop +$REPLACE_worried +$REPLACE_trying +$REPLACE_health +$REPLACE_American +$APPEND_writing +$REPLACE_enjoyed +$REPLACE_second +$APPEND_look +$APPEND_old +$REPLACE_finally +$REPLACE_wish +$REPLACE_famous +$REPLACE_talking +$REPLACE_abroad +$REPLACE_information +$APPEND_And +$REPLACE_stopped +$REPLACE_lose +$REPLACE_sentence +$REPLACE_pronunciation +$REPLACE_feeling +$REPLACE_younger +$REPLACE_passed +$REPLACE_among +$REPLACE_paid +$REPLACE_playing +$REPLACE_attend +$REPLACE_early +$REPLACE_All +$REPLACE_Maybe +$APPEND_high +$REPLACE_child +$APPEND_anything +$REPLACE_order +$REPLACE_saying +$REPLACE_families +$REPLACE_special +$REPLACE_spent +$REPLACE_appreciate +$REPLACE_successful +$APPEND_If +$REPLACE_turned +$REPLACE_cities +$REPLACE_definitely +$REPLACE_fell +$APPEND_try +$APPEND_skills +$REPLACE_world +$REPLACE_technology +$REPLACE_small +$REPLACE_wrote +$REPLACE_takes +$REPLACE_seems +$REPLACE_various +$REPLACE_actually +$REPLACE_exam +$REPLACE_free +$REPLACE_gone +$REPLACE_strong +$REPLACE_receive +$REPLACE_Yesterday +$REPLACE_probably +$REPLACE_Every +$REPLACE_normal +$REPLACE_ask +$REPLACE_company +$REPLACE_environment +$REPLACE_buy +$REPLACE_shows +$REPLACE_easy +$REPLACE_sincerely +$REPLACE_vacation +$REPLACE_far +$REPLACE_sauce +$REPLACE_teacher +$REPLACE_living +$REPLACE_parties +$REPLACE_goes +$REPLACE_Christmas +$REPLACE_Hello +$APPEND_start +$REPLACE_hurt +$REPLACE_wonder +$REPLACE_mind +$REPLACE_possible +$REPLACE_thinking +$APPEND_Now +$REPLACE_relationship +$REPLACE_plan +$REPLACE_man +$REPLACE_woman +$REPLACE_activities +$APPEND_down +$REPLACE_returned +$REPLACE_pay +$REPLACE_ability +$REPLACE_exciting +$REPLACE_safe +$APPEND_off +$APPEND_until +$REPLACE_goal +$APPEND_either +$REPLACE_mistakes +$REPLACE_within +$REPLACE_etc +$REPLACE_cost +$REPLACE_particular +$REPLACE_sense +$REPLACE_longer +$REPLACE_advice +$REPLACE_several +$APPEND_Do +$APPEND_currently +$REPLACE_answer +$REPLACE_Even +$REPLACE_held +$REPLACE_online +$APPEND_life +$REPLACE_Firstly +$REPLACE_close +$APPEND_study +$REPLACE_wear +$APPEND_language +$REPLACE_number +$REPLACE_young +$APPEND_whole +$APPEND_two +$APPEND_Then +$REPLACE_large +$REPLACE_eating +$REPLACE_given +$REPLACE_video +$REPLACE_spoke +$REPLACE_Another +$APPEND_rather +$APPEND_Of +$APPEND_once +$REPLACE_wonderful +$APPEND_must +$REPLACE_tonight +$REPLACE_Their +$REPLACE_languages +$REPLACE_big +$REPLACE_break +$REPLACE_government +$REPLACE_staff +$REPLACE_prepare +$REPLACE_quit +$REPLACE_completely +$REPLACE_ourselves +$APPEND_He +$REPLACE_nor +$APPEND_someone +$REPLACE_sad +$REPLACE_against +$REPLACE_anymore +$APPEND_give +$REPLACE_stayed +$REPLACE_achieve +$APPEND_An +$APPEND_Right +$REPLACE_maybe +$REPLACE_lost +$APPEND_came +$REPLACE_accommodation +$APPEND_One +$APPEND_" +$REPLACE_daughter +$APPEND_next +$REPLACE_love +$REPLACE_cheap +$REPLACE_poor +$REPLACE_By +$REPLACE_whole +$REPLACE_bring +$REPLACE_real +$REPLACE_worked +$REPLACE_almost +$REPLACE_prefer +$APPEND_find +$REPLACE_everybody +$APPEND_another +$REPLACE_create +$REPLACE_addition +$REPLACE_turn +$REPLACE_situation +$APPEND_due +$REPLACE_boyfriend +$APPEND_home +$REPLACE_finish +$REPLACE_rather +$APPEND_said +$APPEND_'re +$REPLACE_careful +$APPEND_long +$REPLACE_recommended +$REPLACE_customers +$REPLACE_level +$REPLACE_died +$REPLACE_comes +$APPEND_You +$REPLACE_glad +$APPEND_come +$REPLACE_short +$REPLACE_knowledge +$REPLACE_set +$REPLACE_Lang +$REPLACE_planning +$REPLACE_confidence +$REPLACE_gain +$REPLACE_check +$REPLACE_immediately +$REPLACE_scared +$REPLACE_conversation +$REPLACE_native +$REPLACE_His +$REPLACE_full +$REPLACE_express +$REPLACE_married +$REPLACE_shopping +$APPEND_working +$APPEND_food +$REPLACE_research +$REPLACE_whenever +$REPLACE_corrections +$REPLACE_weird +$APPEND_quite +$REPLACE_necessary +$REPLACE_Korea +$REPLACE_whose +$REPLACE_higher +$APPEND_entries +$REPLACE_Starting +$REPLACE_attended +$APPEND_5 +$APPEND_past +$REPLACE_realize +$REPLACE_cold +$APPEND_playing +$REPLACE_ended +$REPLACE_inside +$APPEND_please +$REPLACE_coffee +$REPLACE_enjoyable +$APPEND_took +$REPLACE_economic +$APPEND_member +$REPLACE_natural +$REPLACE_registered +$REPLACE_idea +$APPEND_Recently +$APPEND_< +$REPLACE_caused +$REPLACE_student +$REPLACE_questions +$REPLACE_music +$REPLACE_story +$REPLACE_happiness +$REPLACE_gives +$APPEND_She +$REPLACE_Especially +$REPLACE_energy +$REPLACE_available +$REPLACE_anywhere +$REPLACE_taken +$REPLACE_four +$REPLACE_sleep +$REPLACE_afraid +$REPLACE_Everyone +$APPEND_learning +$REPLACE_ate +$APPEND_few +$APPEND_Since +$REPLACE_helps +$REPLACE_vegetables +$REPLACE_kept +$REPLACE_gets +$REPLACE_explain +$REPLACE_girlfriend +$REPLACE_choice +$REPLACE_waiting +$APPEND_put +$APPEND_yesterday +$APPEND_During +$REPLACE_From +$APPEND_starting +$REPLACE_scary +$REPLACE_program +$REPLACE_fish +$REPLACE_hand +$REPLACE_enter +$APPEND_friends +$REPLACE_decide +$REPLACE_score +$REPLACE_lonely +$APPEND_easily +$REPLACE_discovered +$REPLACE_seeing +$REPLACE_message +$REPLACE_week +$APPEND_studying +$REPLACE_universities +$REPLACE_introduce +$REPLACE_common +$REPLACE_heavily +$REPLACE_People +$REPLACE_care +$APPEND_hard +$REPLACE_hit +$REPLACE_America +$REPLACE_point +$APPEND_need +$REPLACE_funny +$APPEND_almost +$REPLACE_pass +$REPLACE_temperature +$REPLACE_performance +$REPLACE_call +$REPLACE_extremely +$REPLACE_chance +$REPLACE_main +$REPLACE_season +$REPLACE_series +$REPLACE_nearby +$REPLACE_license +$REPLACE_expected +$REPLACE_Last +$REPLACE_picture +$REPLACE_movies +$APPEND_Also +$REPLACE_seriously +$REPLACE_via +$REPLACE_running +$REPLACE_run +$REPLACE_regarding +$REPLACE_chose +$REPLACE_moment +$APPEND_feeling +$APPEND_bit +$REPLACE_occurred +$REPLACE_travelling +$REPLACE_brought +$APPEND_makes +$REPLACE_amount +$REPLACE_speakers +$REPLACE_scenery +$APPEND_year +$APPEND_quickly +$REPLACE_grateful +$REPLACE_character +$REPLACE_sleepy +$REPLACE_bed +$REPLACE_increase +$APPEND_Good +$REPLACE_area +$REPLACE_certain +$REPLACE_ways +$REPLACE_looks +$REPLACE_Nowadays +$REPLACE_lucky +$REPLACE_current +$REPLACE_traditional +$APPEND_write +$APPEND_anymore +$REPLACE_noticed +$REPLACE_Did +$REPLACE_matter +$REPLACE_worry +$REPLACE_angry +$REPLACE_With +$REPLACE_biggest +$REPLACE_alcohol +$APPEND_left +$REPLACE_move +$REPLACE_succeed +$REPLACE_post +$REPLACE_] +$REPLACE_abilities +$REPLACE_earthquake +$REPLACE_visited +$APPEND_] +$REPLACE_speech +$REPLACE_Thank +$REPLACE_fewer +$REPLACE_happen +$APPEND_tomorrow +$REPLACE_dinner +$REPLACE_quiet +$APPEND_type +$REPLACE_previous +$REPLACE_Furthermore +$REPLACE_colleagues +$REPLACE_present +$REPLACE_No +$REPLACE_chicken +$REPLACE_city +$REPLACE_weeks +$REPLACE_develop +$REPLACE_join +$APPEND_Last +$REPLACE_except +$REPLACE_economy +$REPLACE_sang +$REPLACE_phrase +$REPLACE_provide +$REPLACE_lately +$REPLACE_experienced +$REPLACE_won +$REPLACE_Though +$APPEND_Therefore +$APPEND_piece +$REPLACE_including +$REPLACE_husband +$REPLACE_changed +$REPLACE_view +$REPLACE_becomes +$REPLACE_share +$APPEND_place +$REPLACE_test +$APPEND_4 +$APPEND_years +$REPLACE_Our +$REPLACE_wrong +$REPLACE_seemed +$REPLACE_wondering +$REPLACE_computer +$REPLACE_known +$REPLACE_culture +$REPLACE_Hong +$REPLACE_clear +$REPLACE_birthday +$REPLACE_despite +$REPLACE_front +$REPLACE_sound +$REPLACE_thankful +$REPLACE_practise +$REPLACE_Will +$REPLACE_atmosphere +$REPLACE_activity +$APPEND_movie +$REPLACE_China +$REPLACE_reasons +$REPLACE_name +$REPLACE_serious +$REPLACE_2 +$REPLACE_warm +$REPLACE_depressed +$REPLACE_simple +$APPEND_trying +$REPLACE_alone +$APPEND_` +$REPLACE_listen +$REPLACE__ +$REPLACE_faithfully +$REPLACE_Which +$REPLACE_relieved +$APPEND_1 +$REPLACE_price +$REPLACE_store +$REPLACE_lower +$REPLACE_strange +$REPLACE_game +$REPLACE_sick +$REPLACE_focus +$REPLACE_suddenly +$APPEND_Please +$REPLACE_Would +$REPLACE_traveled +$REPLACE_event +$REPLACE_ones +$APPEND_Yesterday +$APPEND_making +$REPLACE_remembered +$REPLACE_s +$REPLACE_Lately +$APPEND_S +$REPLACE_member +$APPEND_decided +$REPLACE_across +$REPLACE_entered +$APPEND_maybe +$REPLACE_University +$REPLACE_difficulties +$REPLACE_terrible +$REPLACE_places +$REPLACE_pretty +$REPLACE_weekend +$REPLACE_decision +$APPEND_later +$REPLACE_anybody +$REPLACE_result +$REPLACE_buses +$REPLACE_Fortunately +$APPEND_suddenly +$REPLACE_slept +$APPEND_school +$REPLACE_group +$REPLACE_electricity +$REPLACE_fan +$REPLACE_supposed +$REPLACE_recent +$REPLACE_wants +$APPEND_10 +$REPLACE_low +$APPEND_continue +$APPEND_keep +$APPEND_words +$APPEND_Sometimes +$REPLACE_type +$REPLACE_Tomorrow +$REPLACE_okay +$APPEND_class +$REPLACE_Her +$APPEND_everything +$APPEND_university +$REPLACE_behind +$REPLACE_clean +$REPLACE_anxious +$REPLACE_follow +$APPEND_amount +$REPLACE_parents +$APPEND_While +$REPLACE_email +$REPLACE_mean +$REPLACE_Most +$APPEND_watching +$REPLACE_taste +$APPEND_taking +$REPLACE_Sometimes +$REPLACE_French +$REPLACE_wearing +$APPEND_weather +$REPLACE_law +$REPLACE_difficulty +$APPEND_job +$REPLACE_training +$REPLACE_crowded +$APPEND_All +$REPLACE_gotten +$REPLACE_catch +$REPLACE_method +$REPLACE_public +$REPLACE_classes +$REPLACE_seem +$APPEND_show +$REPLACE_question +$REPLACE_development +$REPLACE_says +$REPLACE_faster +$REPLACE_mother +$REPLACE_guitar +$REPLACE_teeth +$REPLACE_song +$REPLACE_lesson +$REPLACE_knew +$REPLACE_sent +$REPLACE_unable +$REPLACE_alot +$REPLACE_Those +$REPLACE_concert +$APPEND_speak +$REPLACE_software +$REPLACE_German +$REPLACE_Currently +$REPLACE_yourself +$REPLACE_fact +$REPLACE_major +$REPLACE_snowboarding +$REPLACE_apartment +$REPLACE_none +$REPLACE_Here +$REPLACE_reply +$REPLACE_lived +$APPEND_site +$REPLACE_introduction +$REPLACE_exchange +$APPEND_level +$REPLACE_iPhone +$REPLACE_consider +$REPLACE_leaves +$APPEND_early +$REPLACE_requires +$REPLACE_Saturday +$TRANSFORM_CASE_CAPITAL_1 +$REPLACE_further +$REPLACE_absolutely +$REPLACE_realised +$APPEND_heard +$REPLACE_following +$REPLACE_doctor +$REPLACE_beginner +$APPEND_against +$REPLACE_embarrassed +$REPLACE_correctly +$REPLACE_half +$REPLACE_dangerous +$REPLACE_moved +$REPLACE_complete +$REPLACE_perfect +$REPLACE_Anyway +$REPLACE_hold +$REPLACE_differences +$REPLACE_lunch +$REPLACE_himself +$REPLACE_based +$APPEND_thought +$REPLACE_reach +$REPLACE_cheaper +$REPLACE_loud +$APPEND_By +$APPEND_everyone +$REPLACE_leaving +$REPLACE_released +$REPLACE_fine +$REPLACE_Australia +$REPLACE_style +$REPLACE_deal +$APPEND_along +$REPLACE_satisfied +$REPLACE_Of +$REPLACE_variety +$APPEND_improve +$REPLACE_under +$REPLACE_giving +$REPLACE_party +$APPEND_understand +$REPLACE_everywhere +$REPLACE_confident +$APPEND_play +$REPLACE_slow +$REPLACE_centre +$REPLACE_light +$REPLACE_trouble +$REPLACE_Its +$APPEND_became +$REPLACE_begin +$REPLACE_grade +$REPLACE_exams +$REPLACE_busy +$REPLACE_nbsp +$REPLACE_3 +$REPLACE_control +$REPLACE_characters +$REPLACE_needs +$REPLACE_pictures +$APPEND_New +$APPEND_test +$REPLACE_currently +$REPLACE_describe +$REPLACE_uncomfortable +$REPLACE_affected +$REPLACE_songs +$REPLACE_helped +$REPLACE_head +$APPEND_let +$REPLACE_costs +$REPLACE_five +$REPLACE_slowly +$REPLACE_1 +$REPLACE_causes +$REPLACE_ashamed +$APPEND_coming +$APPEND_everyday +$REPLACE_products +$REPLACE_dishes +$REPLACE_least +$REPLACE_wore +$REPLACE_internet +$REPLACE_mentioned +$APPEND_began +$REPLACE_word +$REPLACE_service +$REPLACE_workers +$REPLACE_continued +$REPLACE_sounds +$REPLACE_hour +$REPLACE_jobs +$REPLACE_career +$REPLACE_personal +$REPLACE_piece +$REPLACE_per +$REPLACE_Regarding +$REPLACE_entrance +$REPLACE_improving +$APPEND_= +$REPLACE_areas +$REPLACE_1st +$REPLACE_mostly +$REPLACE_lessons +$REPLACE_drink +$REPLACE_hair +$APPEND_exactly +$REPLACE_e +$REPLACE_luck +$REPLACE_members +$APPEND_means +$REPLACE_mistake +$REPLACE_somewhere +$APPEND_pair +$REPLACE_tomatoes +$APPEND_definitely +$REPLACE_swimming +$REPLACE_perform +$REPLACE_compared +$REPLACE_unfortunately +$REPLACE_however +$REPLACE_twice +$REPLACE_society +$APPEND_20 +$REPLACE_preparing +$REPLACE_Two +$APPEND_Japan +$REPLACE_nobody +$REPLACE_environmental +$REPLACE_till +$REPLACE_fall +$REPLACE_spoken +$REPLACE_forget +$REPLACE_form +$APPEND_number +$APPEND_watch +$APPEND_live +$REPLACE_include +$REPLACE_related +$REPLACE_wait +$APPEND_These +$REPLACE_European +$APPEND_tell +$REPLACE_meeting +$REPLACE_evening +$REPLACE_nowadays +$REPLACE_northern +$REPLACE_convenience +$REPLACE_performed +$REPLACE_plans +$REPLACE_competition +$REPLACE_open +$REPLACE_confused +$REPLACE_practicing +$REPLACE_quality +$REPLACE_professional +$REPLACE_maintain +$REPLACE_pain +$REPLACE_familiar +$REPLACE_classical +$REPLACE_shop +$REPLACE_filled +$REPLACE_improved +$REPLACE_meant +$APPEND_listening +$REPLACE_ceremony +$REPLACE_increasing +$REPLACE_drove +$APPEND_completely +$REPLACE_account +$REPLACE_developed +$REPLACE_lack +$REPLACE_purpose +$REPLACE_upon +$REPLACE_tasted +$REPLACE_crazy +$REPLACE_summer +$REPLACE_regret +$REPLACE_born +$REPLACE_rain +$REPLACE_weight +$REPLACE_required +$REPLACE_accept +$REPLACE_cut +$REPLACE_flew +$REPLACE_waste +$APPEND_ca +$APPEND_trip +$REPLACE_Going +$REPLACE_excellent +$REPLACE_created +$REPLACE_reality +$REPLACE_cultural +$REPLACE_save +$REPLACE_programs +$REPLACE_painful +$REPLACE_Many +$REPLACE_dish +$REPLACE_teaching +$REPLACE_Studying +$REPLACE_water +$REPLACE_happens +$REPLACE_pleased +$REPLACE_ordinary +$APPEND_practice +$REPLACE_train +$REPLACE_results +$REPLACE_Italian +$REPLACE_weak +$REPLACE_period +$REPLACE_above +$REPLACE_hot +$REPLACE_Not +$REPLACE_feelings +$REPLACE_mobile +$REPLACE_walk +$APPEND_game +$REPLACE_impressed +$APPEND_same +$REPLACE_Germany +$REPLACE_girl +$REPLACE_closer +$REPLACE_communication +$REPLACE_worst +$APPEND_No +$REPLACE_located +$REPLACE_phone +$REPLACE_sit +$REPLACE_Lastly +$REPLACE_feels +$APPEND_listen +$APPEND_done +$REPLACE_subtitles +$REPLACE_Whenever +$REPLACE_potatoes +$REPLACE_fluent +$REPLACE_amazing +$REPLACE_neither +$APPEND_With +$APPEND_never +$REPLACE_stressed +$REPLACE_prevent +$REPLACE_photos +$APPEND_$ +$REPLACE_non +$REPLACE_agree +$REPLACE_Moreover +$REPLACE_restaurants +$REPLACE_types +$REPLACE_office +$REPLACE_studies +$REPLACE_history +$REPLACE_calm +$REPLACE_walked +$REPLACE_modern +$APPEND_three +$REPLACE_clothing +$REPLACE_private +$APPEND_little +$APPEND_outside +$APPEND_OR +$REPLACE_simply +$REPLACE_particularly +$REPLACE_notice +$REPLACE_side +$APPEND_looked +$REPLACE_YouTube +$APPEND_students +$REPLACE_afterwards +$APPEND_reading +$REPLACE_graduate +$REPLACE_library +$REPLACE_gained +$REPLACE_bicycle +$REPLACE_son +$APPEND_compared +$REPLACE_events +$APPEND_Although +$REPLACE_US +$REPLACE_properly +$APPEND_Maybe +$APPEND_Can +$APPEND_best +$REPLACE_wondered +$REPLACE_arrive +$APPEND_say +$REPLACE_considered +$REPLACE_dream +$REPLACE_feet +$REPLACE_broke +$APPEND_From +$REPLACE_southern +$REPLACE_hometown +$APPEND_journal +$REPLACE_Everything +$APPEND_money +$REPLACE_concentrate +$REPLACE_stories +$REPLACE_teachers +$APPEND_happened +$REPLACE_New +$REPLACE_transport +$REPLACE_stronger +$REPLACE_heart +$REPLACE_staying +$REPLACE_honest +$REPLACE_sold +$APPEND_wrong +$APPEND_Or +$REPLACE_relax +$REPLACE_heavy +$REPLACE_* +$REPLACE_speaker +$REPLACE_limited +$APPEND_speaking +$APPEND_e +$REPLACE_countryside +$REPLACE_heat +$REPLACE_prepared +$REPLACE_truth +$REPLACE_books +$REPLACE_drank +$REPLACE_nuclear +$REPLACE_title +$REPLACE_6 +$REPLACE_boring +$REPLACE_totally +$REPLACE_practiced +$REPLACE_therefore +$REPLACE_book +$REPLACE_regularly +$REPLACE_safety +$REPLACE_normally +$REPLACE_visiting +$APPEND_kinds +$REPLACE_impressive +$REPLACE_final +$REPLACE_driving +$REPLACE_stuff +$REPLACE_guess +$REPLACE_avoid +$REPLACE_answered +$REPLACE_pleasant +$APPEND_times +$APPEND_without +$REPLACE_focused +$REPLACE_badly +$REPLACE_solve +$REPLACE_grow +$REPLACE_drive +$APPEND_although +$REPLACE_news +$REPLACE_Afterwards +$APPEND_6 +$REPLACE_Learning +$REPLACE_Thanks +$REPLACE_flight +$REPLACE_building +$REPLACE_opened +$REPLACE_shocked +$REPLACE_volleyball +$REPLACE_accepted +$APPEND_exam +$REPLACE_team +$REPLACE_system +$APPEND_ones +$REPLACE_goals +$REPLACE_Before +$REPLACE_meat +$APPEND_Does +$REPLACE_schedule +$REPLACE_cream +$REPLACE_listened +$REPLACE_Why +$REPLACE_worth +$APPEND_members +$REPLACE_strength +$REPLACE_works +$APPEND_m +$REPLACE_surprise +$REPLACE_holidays +$REPLACE_7 +$APPEND_written +$REPLACE_medicine +$REPLACE_contact +$REPLACE_position +$APPEND_tried +$REPLACE_highly +$REPLACE_missed +$REPLACE_typhoon +$REPLACE_celebrate +$REPLACE_February +$REPLACE_greater +$REPLACE_support +$REPLACE_allow +$REPLACE_appeared +$REPLACE_naturally +$REPLACE_breakfast +$REPLACE_afternoon +$REPLACE_dead +$REPLACE_proud +$REPLACE_stuck +$APPEND_half +$REPLACE_lyrics +$APPEND_based +$REPLACE_sing +$REPLACE_process +$REPLACE_search +$REPLACE_sell +$REPLACE_learnt +$REPLACE_responsibility +$REPLACE_field +$REPLACE_lifestyle +$REPLACE_helpful +$REPLACE_Koreans +$REPLACE_awake +$REPLACE_success +$APPEND_living +$REPLACE_latest +$REPLACE_corrected +$REPLACE_communicating +$REPLACE_raise +$REPLACE_showed +$REPLACE_father +$REPLACE_marriage +$REPLACE_elementary +$REPLACE_allows +$APPEND_lot +$REPLACE_eventually +$REPLACE_customer +$REPLACE_unusual +$REPLACE_advise +$REPLACE_letter +$REPLACE_clearly +$REPLACE_essay +$REPLACE_bigger +$REPLACE_habit +$APPEND_system +$REPLACE_ran +$APPEND_speakers +$REPLACE_bored +$REPLACE_whatever +$REPLACE_fourth +$REPLACE_chosen +$REPLACE_room +$REPLACE_30 +$REPLACE_carefully +$REPLACE_loss +$REPLACE_ingredients +$REPLACE_singing +$REPLACE_ride +$REPLACE_build +$REPLACE_cooking +$REPLACE_add +$REPLACE_mom +$REPLACE_sign +$REPLACE_chatting +$REPLACE_happier +$REPLACE_seat +$REPLACE_affect +$REPLACE_appropriate +$REPLACE_named +$APPEND_30 +$REPLACE_female +$REPLACE_fashion +$REPLACE_attending +$REPLACE_Tonight +$REPLACE_role +$REPLACE_somebody +$APPEND_Unfortunately +$REPLACE_employees +$REPLACE_face +$REPLACE_middle +$REPLACE_junior +$REPLACE_lovely +$REPLACE_reduce +$REPLACE_positive +$REPLACE_concerned +$REPLACE_overseas +$REPLACE_" +$REPLACE_Second +$APPEND_Our +$APPEND_named +$REPLACE_mountain +$APPEND_eating +$REPLACE_warmer +$REPLACE_death +$REPLACE_electronic +$REPLACE_figure +$REPLACE_frequently +$REPLACE_pair +$REPLACE_Americans +$REPLACE_rest +$REPLACE_TV +$APPEND_themselves +$APPEND_however +$REPLACE_subject +$APPEND_music +$REPLACE_dormitory +$APPEND_forward +$REPLACE_department +$REPLACE_pronounce +$REPLACE_wake +$REPLACE_cook +$APPEND_visit +$REPLACE_raised +$REPLACE_smaller +$REPLACE_stressful +$APPEND_lately +$REPLACE_completed +$REPLACE_photography +$REPLACE_10 +$APPEND_saying +$REPLACE_dropped +$REPLACE_laughed +$APPEND_read +$REPLACE_complain +$REPLACE_Usually +$APPEND_felt +$REPLACE_Thus +$REPLACE_foreigner +$REPLACE_theatre +$APPEND_website +$APPEND_days +$REPLACE_slightly +$REPLACE_incorrect +$REPLACE_frustrated +$REPLACE_grandmother +$REPLACE_forty +$REPLACE_signed +$APPEND_book +$REPLACE_sore +$REPLACE_classmates +$REPLACE_equipment +$REPLACE_memory +$REPLACE_ordered +$APPEND_stay +$REPLACE_expect +$REPLACE_drunk +$APPEND_gave +$REPLACE_midnight +$APPEND_seem +$APPEND_cut +$REPLACE_address +$REPLACE_couple +$REPLACE_Compared +$REPLACE_friendly +$REPLACE_rode +$REPLACE_losing +$REPLACE_nearly +$REPLACE_six +$REPLACE_speeches +$REPLACE_international +$REPLACE_understood +$REPLACE_thank +$REPLACE_rarely +$REPLACE_match +$REPLACE_uploaded +$REPLACE_Luckily +$REPLACE_failed +$REPLACE_hamburger +$REPLACE_sleeping +$REPLACE_tongue +$REPLACE_colleague +$REPLACE_require +$REPLACE_terribly +$REPLACE_case +$APPEND_traditional +$REPLACE_graduation +$REPLACE_offer +$REPLACE_respond +$REPLACE_perfectly +$REPLACE_businesses +$REPLACE_8 +$APPEND_s +$REPLACE_understanding +$REPLACE_hungry +$REPLACE_conclusion +$REPLACE_homework +$REPLACE_design +$REPLACE_British +$REPLACE_peaceful +$REPLACE_forgot +$REPLACE_suitable +$REPLACE_soccer +$REPLACE_tells +$REPLACE_third +$REPLACE_exactly +$REPLACE_term +$REPLACE_drinking +$REPLACE_searching +$REPLACE_hung +$REPLACE_air +$REPLACE_strongly +$APPEND_looking +$REPLACE_band +$REPLACE_checked +$REPLACE_send +$REPLACE_Zealand +$REPLACE_draw +$REPLACE_educational +$REPLACE_incident +$APPEND_Some +$APPEND_friend +$APPEND_free +$REPLACE_toward +$REPLACE_interview +$APPEND_> +$REPLACE_tough +$REPLACE_canceled +$REPLACE_memorize +$REPLACE_historical +$REPLACE_slang +$REPLACE_replied +$REPLACE_considering +$REPLACE_skill +$REPLACE_musical +$REPLACE_improvement +$REPLACE_carry +$REPLACE_education +$APPEND_great +$REPLACE_companies +$REPLACE_cool +$APPEND_comes +$REPLACE_employee +$REPLACE_age +$APPEND_Yes +$REPLACE_Could +$REPLACE_relaxed +$REPLACE_greatest +$REPLACE_total +$REPLACE_ready +$REPLACE_guy +$REPLACE_chocolate +$APPEND_tense +$REPLACE_earn +$REPLACE_topic +$REPLACE_beat +$REPLACE_date +$REPLACE_illnesses +$REPLACE_conditioner +$APPEND_inside +$REPLACE_suggested +$REPLACE_drama +$REPLACE_pick +$REPLACE_starts +$REPLACE_manage +$APPEND_anyway +$REPLACE_Thailand +$REPLACE_McDonald +$REPLACE_Writing +$APPEND_Are +$REPLACE_2nd +$APPEND_fall +$REPLACE_flu +$REPLACE_websites +$REPLACE_snowy +$APPEND_diary +$REPLACE_road +$REPLACE_professor +$REPLACE_exhausted +$APPEND_held +$REPLACE_colored +$REPLACE_sitting +$REPLACE_wanna +$REPLACE_according +$REPLACE_lead +$REPLACE_scene +$REPLACE_hardly +$REPLACE_ticket +$REPLACE_remain +$REPLACE_worrying +$REPLACE_patience +$REPLACE_Having +$REPLACE_allowed +$REPLACE_whilst +$REPLACE_entire +$REPLACE_promised +$REPLACE_photo +$REPLACE_motivated +$REPLACE_dairy +$APPEND_full +$REPLACE_points +$REPLACE_Soon +$REPLACE_messages +$APPEND_alone +$REPLACE_alive +$APPEND_Every +$APPEND_entire +$REPLACE_programme +$REPLACE_fully +$REPLACE_cloudy +$REPLACE_occur +$REPLACE_meaning +$APPEND_area +$REPLACE_liked +$REPLACE_sweet +$REPLACE_act +$REPLACE_graduated +$REPLACE_childhood +$APPEND_available +$REPLACE_believed +$REPLACE_newspaper +$REPLACE_enjoying +$REPLACE_riding +$APPEND_Not +$REPLACE_body +$REPLACE_beneficial +$REPLACE_recognize +$APPEND_native +$REPLACE_attention +$REPLACE_Until +$REPLACE_struck +$REPLACE_Just +$REPLACE_correcting +$REPLACE_interest +$REPLACE_changing +$REPLACE_pollution +$APPEND_pieces +$REPLACE_According +$REPLACE_autumn +$APPEND_problem +$REPLACE_gym +$REPLACE_basic +$REPLACE_includes +$REPLACE_games +$APPEND_seeing +$REPLACE_sunny +$REPLACE_5 +$APPEND_learned +$REPLACE_stage +$REPLACE_touch +$REPLACE_discuss +$REPLACE_airplane +$REPLACE_Has +$REPLACE_die +$REPLACE_relationships +$REPLACE_effects +$REPLACE_sat +$REPLACE_parts +$REPLACE_tsunami +$REPLACE_response +$REPLACE_teaches +$REPLACE_self +$REPLACE_thanks +$REPLACE_rained +$REPLACE_laundry +$REPLACE_dependent +$APPEND_near +$REPLACE_below +$REPLACE_custom +$REPLACE_inconvenient +$REPLACE_relaxing +$REPLACE_wedding +$REPLACE_challenge +$APPEND_set +$REPLACE_chatted +$APPEND_immediately +$REPLACE_attractive +$REPLACE_translate +$APPEND_Just +$APPEND_TV +$REPLACE_win +$REPLACE_museum +$REPLACE_neighborhood +$REPLACE_Right +$REPLACE_regular +$REPLACE_experiences +$APPEND_word +$APPEND_played +$REPLACE_hobby +$REPLACE_developing +$REPLACE_truly +$APPEND_ended +$REPLACE_issue +$APPEND_correct +$REPLACE_impossible +$REPLACE_concerning +$REPLACE_realise +$REPLACE_brings +$APPEND_room +$REPLACE_advised +$REPLACE_workplace +$REPLACE_surfing +$APPEND_Let +$APPEND_daily +$REPLACE_stomach +$APPEND_night +$REPLACE_meal +$REPLACE_disadvantages +$REPLACE_loudly +$REPLACE_prize +$REPLACE_besides +$APPEND_experience +$REPLACE_Despite +$REPLACE_4 +$APPEND_concert +$REPLACE_3rd +$REPLACE_power +$REPLACE_` +$APPEND_lots +$REPLACE_changes +$REPLACE_kindergarten +$REPLACE_sweat +$REPLACE_ten +$APPEND_wo +$REPLACE_overcome +$REPLACE_effective +$REPLACE_terms +$REPLACE_shown +$REPLACE_chat +$APPEND_team +$REPLACE_sorry +$APPEND_7 +$REPLACE_station +$APPEND_man +$REPLACE_produce +$REPLACE_technological +$REPLACE_differently +$REPLACE_transferred +$APPEND_told +$APPEND_late +$REPLACE_laugh +$REPLACE_worker +$REPLACE_space +$REPLACE_introduced +$REPLACE_single +$REPLACE_cancelled +$REPLACE_methods +$REPLACE_transportation +$REPLACE_Philippines +$REPLACE_possibility +$REPLACE_tasty +$REPLACE_location +$REPLACE_male +$APPEND_simply +$REPLACE_tastes +$REPLACE_ease +$REPLACE_straight +$REPLACE_uses +$REPLACE_participate +$REPLACE_discover +$APPEND_co +$REPLACE_details +$REPLACE_logged +$REPLACE_bright +$REPLACE_Once +$REPLACE_walking +$APPEND_spent +$MERGE_HYPHEN +$REPLACE_growing +$REPLACE_slight +$APPEND_current +$REPLACE_moving +$REPLACE_spring +$REPLACE_August +$REPLACE_fans +$REPLACE_Well +$APPEND_nervous +$REPLACE_version +$REPLACE_upset +$REPLACE_stress +$REPLACE_appointment +$REPLACE_tasks +$REPLACE_Being +$REPLACE_encouraged +$REPLACE_town +$REPLACE_eight +$REPLACE_mood +$REPLACE_forecast +$APPEND_lessons +$APPEND_finished +$REPLACE_increased +$REPLACE_blossoms +$REPLACE_aware +$REPLACE_Besides +$REPLACE_Taiwanese +$REPLACE_someday +$REPLACE_happening +$REPLACE_volunteer +$REPLACE_fireworks +$REPLACE_ideas +$REPLACE_curious +$REPLACE_responsible +$REPLACE_voice +$REPLACE_covered +$APPEND_ice +$REPLACE_rang +$REPLACE_items +$REPLACE_apart +$APPEND_program +$REPLACE_bye +$REPLACE_Next +$REPLACE_complicated +$REPLACE_Someone +$APPEND_earlier +$APPEND_difficult +$REPLACE_invited +$REPLACE_applied +$APPEND_anyone +$REPLACE_gaining +$REPLACE_cute +$REPLACE_line +$REPLACE_partner +$REPLACE_regretted +$REPLACE_clock +$APPEND_according +$REPLACE_greatly +$REPLACE_appear +$REPLACE_opposite +$REPLACE_Like +$REPLACE_patient +$REPLACE_spread +$REPLACE_dollars +$REPLACE_relieve +$REPLACE_article +$REPLACE_benefits +$APPEND_American +$REPLACE_Looking +$REPLACE_Who +$REPLACE_fix +$REPLACE_human +$REPLACE_technologies +$REPLACE_breathe +$REPLACE_strict +$REPLACE_opinions +$APPEND_possibly +$REPLACE_appearance +$REPLACE_explanation +$REPLACE_herself +$APPEND_student +$REPLACE_plane +$REPLACE_hearing +$REPLACE_personality +$REPLACE_attitude +$REPLACE_journey +$REPLACE_recover +$REPLACE_magazine +$REPLACE_disappeared +$APPEND_taken +$REPLACE_Me +$REPLACE_efficiently +$REPLACE_strawberries +$APPEND_becoming +$REPLACE_October +$REPLACE_social +$REPLACE_suicide +$REPLACE_reached +$REPLACE_damaged +$REPLACE_personalities +$REPLACE_valuable +$REPLACE_height +$REPLACE_Asian +$REPLACE_sight +$REPLACE_issues +$REPLACE_titled +$REPLACE_science +$REPLACE_cell +$REPLACE_amongst +$APPEND_movies +$REPLACE_June +$REPLACE_policies +$REPLACE_silent +$REPLACE_girls +$APPEND_company +$APPEND_second +$APPEND_ability +$APPEND_hope +$REPLACE_former +$APPEND_GOOD +$REPLACE_fashionable +$REPLACE_club +$APPEND_end +$REPLACE_path +$APPEND_+ +$REPLACE_top +$APPEND_happy +$REPLACE_lay +$REPLACE_accident +$REPLACE_festival +$REPLACE_Later +$REPLACE_destroyed +$APPEND_plan +$APPEND_famous +$REPLACE_safely +$APPEND_related +$REPLACE_suit +$REPLACE_stand +$REPLACE_contrast +$APPEND_period +$REPLACE_highest +$REPLACE_habits +$APPEND_First +$REPLACE_January +$REPLACE_putting +$REPLACE_grew +$REPLACE_degrees +$REPLACE_latter +$REPLACE_extent +$REPLACE_lang +$REPLACE_episode +$REPLACE_physically +$APPEND_types +$REPLACE_cooked +$REPLACE_original +$REPLACE_fresh +$APPEND_world +$REPLACE_l +$REPLACE_Year +$APPEND_wanted +$REPLACE_Wednesday +$REPLACE_unique +$REPLACE_active +$REPLACE_center +$APPEND_problems +$REPLACE_encourage +$APPEND_8 +$REPLACE_individual +$REPLACE_included +$REPLACE_suggestions +$REPLACE_sea +$REPLACE_smoothly +$REPLACE_headache +$REPLACE_Was +$REPLACE_Internet +$REPLACE_pleasure +$REPLACE_Thursday +$REPLACE_board +$REPLACE_phrases +$REPLACE_built +$APPEND_caused +$REPLACE_subjects +$APPEND_places +$REPLACE_grammatical +$REPLACE_suggest +$APPEND_big +$REPLACE_bath +$APPEND_train +$REPLACE_hesitant +$APPEND_seriously +$REPLACE_deep +$APPEND_children +$REPLACE_refreshed +$APPEND_Correct +$APPEND_yourself +$APPEND_THE +$REPLACE_reasonable +$APPEND_spend +$APPEND_skill +$REPLACE_obvious +$REPLACE_Friday +$REPLACE_soup +$REPLACE_basketball +$REPLACE_Your +$REPLACE_drawing +$REPLACE_m +$APPEND_sentences +$REPLACE_english +$APPEND_fell +$REPLACE_colder +$REPLACE_car +$APPEND_group +$REPLACE_receiving +$REPLACE_sun +$APPEND_15 +$APPEND_hot +$APPEND_verb +$REPLACE_technical +$REPLACE_Through +$APPEND_buy +$REPLACE_route +$REPLACE_Vietnamese +$REPLACE_grandfather +$REPLACE_April +$REPLACE_lasts +$REPLACE_environmentally +$REPLACE_progress +$REPLACE_telling +$REPLACE_preparation +$REPLACE_supermarket +$REPLACE_Perhaps +$REPLACE_plays +$REPLACE_driver +$REPLACE_anyway +$APPEND_within +$REPLACE_Vietnam +$REPLACE_green +$REPLACE_access +$APPEND_t +$REPLACE_concerns +$REPLACE_laptop +$APPEND_eventually +$REPLACE_fried +$REPLACE_pieces +$REPLACE_security +$REPLACE_condition +$REPLACE_dreams +$REPLACE_reminded +$REPLACE_December +$REPLACE_finding +$REPLACE_produced +$REPLACE_broken +$REPLACE_raising +$REPLACE_specific +$REPLACE_humid +$APPEND_reason +$REPLACE_programming +$REPLACE_brush +$REPLACE_powerful +$REPLACE_shape +$REPLACE_involves +$APPEND_summer +$REPLACE_kinds +$APPEND_eat +$REPLACE_market +$REPLACE_Introducing +$APPEND_kept +$APPEND_information +$REPLACE_Filipino +$REPLACE_hang +$REPLACE_nature +$REPLACE_stood +$REPLACE_oldest +$APPEND_books +$APPEND_top +$REPLACE_physical +$REPLACE_Thai +$REPLACE_effort +$REPLACE_U +$APPEND_phone +$REPLACE_author +$REPLACE_imagined +$REPLACE_request +$REPLACE_Australian +$REPLACE_didn +$REPLACE_Something +$REPLACE_translator +$REPLACE_text +$APPEND_account +$REPLACE_protect +$REPLACE_resources +$REPLACE_Additionally +$APPEND_afterwards +$APPEND_Should +$REPLACE_awhile +$REPLACE_meanings +$APPEND_pictures +$REPLACE_benefit +$REPLACE_exist +$REPLACE_connection +$REPLACE_impression +$APPEND_meeting +$REPLACE_electrical +$APPEND_style +$REPLACE_larger +$REPLACE_hotter +$REPLACE_foot +$APPEND_further +$REPLACE_described +$REPLACE_note +$REPLACE_football +$APPEND_ourselves +$REPLACE_searched +$REPLACE_temporary +$REPLACE_semester +$REPLACE_announced +$REPLACE_Suddenly +$APPEND_others +$APPEND_goes +$REPLACE_sort +$REPLACE_itself +$REPLACE_rich +$APPEND_song +$REPLACE_memorable +$REPLACE_Europe +$REPLACE_features +$REPLACE_apply +$REPLACE_celebrated +$REPLACE_delivery +$REPLACE_winter +$REPLACE_miss +$REPLACE_application +$APPEND_onwards +$REPLACE_population +$REPLACE_failure +$REPLACE_lazy +$REPLACE_scored +$REPLACE_November +$APPEND_travel +$REPLACE_Let +$REPLACE_alcoholic +$REPLACE_disappointment +$REPLACE_severe +$REPLACE_effect +$REPLACE_speed +$APPEND_How +$APPEND_sounds +$REPLACE_cooler +$REPLACE_'cause +$APPEND_mean +$REPLACE_divided +$REPLACE_ha +$REPLACE_9 +$REPLACE_advantages +$APPEND_call +$REPLACE_21st +$REPLACE_fit +$REPLACE_lit +$REPLACE_directly +$REPLACE_videos +$REPLACE_pressure +$REPLACE_pursue +$REPLACE_forgotten +$REPLACE_industry +$REPLACE_Speaking +$APPEND_enjoy +$REPLACE_Should +$REPLACE_grown +$REPLACE_participated +$REPLACE_treat +$REPLACE_expression +$REPLACE_fly +$REPLACE_tall +$REPLACE_situations +$REPLACE_host +$REPLACE_visitors +$APPEND_hear +$REPLACE_Instead +$REPLACE_agreed +$REPLACE_affects +$REPLACE_drew +$REPLACE_spending +$REPLACE_huge +$REPLACE_ill +$REPLACE_tradition +$REPLACE_argue +$REPLACE_turns +$REPLACE_ground +$REPLACE_sometime +$REPLACE_Italy +$APPEND_works +$REPLACE_likely +$REPLACE_Madam +$APPEND_questions +$REPLACE_ceremonies +$APPEND_turn +$APPEND_Korean +$REPLACE_gradually +$REPLACE_financial +$REPLACE_involved +$REPLACE_throw +$REPLACE_advertising +$REPLACE_tend +$REPLACE_characteristics +$APPEND_among +$REPLACE_electric +$REPLACE_sister +$APPEND_car +$REPLACE_fantastic +$REPLACE_examination +$APPEND_city +$REPLACE_eaten +$REPLACE_film +$APPEND_small +$REPLACE_players +$REPLACE_stores +$REPLACE_machine +$REPLACE_managed +$REPLACE_tour +$APPEND_video +$REPLACE_journals +$REPLACE_guys +$APPEND_meet +$REPLACE_deeply +$REPLACE_floor +$REPLACE_keeps +$REPLACE_talks +$REPLACE_focusing +$REPLACE_mysterious +$APPEND_less +$REPLACE_rice +$REPLACE_recovered +$REPLACE_injured +$REPLACE_poorly +$REPLACE_comedy +$REPLACE_cigarettes +$REPLACE_anime +$REPLACE_influence +$REPLACE_Eventually +$REPLACE_offered +$REPLACE_sale +$REPLACE_effectively +$REPLACE_disappointing +$REPLACE_illness +$REPLACE_comments +$APPEND_talk +$REPLACE_contains +$APPEND_People +$APPEND_power +$REPLACE_31st +$REPLACE_distance +$REPLACE_appears +$REPLACE_importance +$REPLACE_choosing +$APPEND_interesting +$REPLACE_snow +$APPEND_o +$REPLACE_tennis +$REPLACE_continues +$REPLACE_dress +$REPLACE_percent +$REPLACE_size +$REPLACE_dictionaries +$APPEND_seems +$REPLACE_fever +$APPEND_etc +$APPEND_Though +$REPLACE_whereas +$APPEND_several +$APPEND_far +$APPEND_classes +$APPEND_public +$REPLACE_traffic +$REPLACE_damage +$APPEND_nothing +$REPLACE_worthwhile +$REPLACE_appreciated +$REPLACE_articles +$APPEND_begin +$APPEND_needed +$REPLACE_recommendations +$REPLACE_don +$REPLACE_buildings +$APPEND_four +$REPLACE_jealous +$REPLACE_seminar +$APPEND_gradually +$REPLACE_complaints +$REPLACE_Nothing +$REPLACE_advance +$REPLACE_flowers +$APPEND_Starting +$REPLACE_beyond +$REPLACE_advertised +$APPEND_mainly +$APPEND_possible +$REPLACE_suffering +$APPEND_12 +$REPLACE_Hopefully +$APPEND_countries +$APPEND_similar +$REPLACE_quick +$REPLACE_general +$REPLACE_successfully +$REPLACE_dark +$REPLACE_unbelievable +$REPLACE_causing +$REPLACE_13th +$REPLACE_unexpected +$REPLACE_begins +$REPLACE_tea +$REPLACE_Sunday +$APPEND_somewhere +$REPLACE_digital +$APPEND_stories +$APPEND_idea +$APPEND_tired +$APPEND_family +$REPLACE_animation +$REPLACE_shot +$REPLACE_Or +$APPEND_managed +$REPLACE_bus +$APPEND_close +$REPLACE_disease +$REPLACE_desire +$REPLACE_carried +$REPLACE_disappear +$REPLACE_essential +$APPEND_news +$REPLACE_forced +$REPLACE_fault +$REPLACE_translation +$REPLACE_television +$REPLACE_cried +$REPLACE_freely +$REPLACE_Valentine +$REPLACE_somewhat +$REPLACE_operation +$REPLACE_conversational +$APPEND_absolutely +$APPEND_properly +$REPLACE_sites +$REPLACE_allergies +$REPLACE_salary +$REPLACE_rise +$REPLACE_entertainment +$REPLACE_kitchen +$REPLACE_emotional +$REPLACE_McDonalds +$REPLACE_extra +$APPEND_nearby +$REPLACE_mention +$APPEND_Here +$APPEND_nice +$APPEND_college +$APPEND_Before +$APPEND_form +$REPLACE_likes +$APPEND_turned +$REPLACE_rent +$REPLACE_tourists +$REPLACE_unknown +$REPLACE_actors +$APPEND_longer +$REPLACE_fill +$REPLACE_Nobody +$REPLACE_Singapore +$REPLACE_helping +$REPLACE_exercises +$APPEND_real +$APPEND_located +$APPEND_received +$APPEND_gets +$APPEND_bad +$REPLACE_doubt +$REPLACE_sweaty +$REPLACE_prefecture +$REPLACE_audience +$REPLACE_sports +$REPLACE_minute +$REPLACE_product +$REPLACE_buying +$REPLACE_exact +$REPLACE_temporarily +$REPLACE_Avatar +$REPLACE_Skype +$REPLACE_discussion +$REPLACE_item +$REPLACE_gon +$REPLACE_accessories +$REPLACE_incredibly +$REPLACE_Where +$REPLACE_World +$REPLACE_advantage +$REPLACE_ridiculous +$REPLACE_wherever +$REPLACE_shook +$REPLACE_global +$REPLACE_entitled +$REPLACE_Working +$APPEND_hours +$REPLACE_Starbucks +$REPLACE_routine +$REPLACE_flavored +$APPEND_item +$REPLACE_techniques +$REPLACE_creates +$REPLACE_peace +$REPLACE_annoyed +$REPLACE_rate +$REPLACE_September +$REPLACE_Russian +$REPLACE_assistant +$REPLACE_plenty +$REPLACE_local +$APPEND_store +$REPLACE_sooner +$REPLACE_overslept +$REPLACE_Everybody +$REPLACE_selling +$REPLACE_negative +$REPLACE_setting +$APPEND_helps +$REPLACE_lecture +$APPEND_happen +$REPLACE_survive +$REPLACE_art +$APPEND_certainly +$APPEND_fully +$APPEND_above +$REPLACE_speaks +$REPLACE_asking +$REPLACE_economical +$REPLACE_salaries +$APPEND_clearly +$REPLACE_mail +$REPLACE_holding +$REPLACE_organise +$REPLACE_efficient +$APPEND_name +$REPLACE_constantly +$REPLACE_overtime +$REPLACE_grandma +$REPLACE_returning +$REPLACE_laziness +$REPLACE_importantly +$APPEND_true +$APPEND_series +$REPLACE_converse +$APPEND_session +$REPLACE_sugar +$APPEND_Currently +$REPLACE_mentally +$APPEND_starts +$REPLACE_theater +$APPEND_tonight +$REPLACE_succeeded +$REPLACE_awful +$REPLACE_political +$APPEND_important +$REPLACE_log +$REPLACE_awesome +$REPLACE_00 +$APPEND_Did +$REPLACE_announcement +$REPLACE_addicted +$REPLACE_disaster +$REPLACE_page +$REPLACE_blossom +$REPLACE_stars +$REPLACE_presentation +$REPLACE_Nevertheless +$APPEND_talking +$APPEND_Instead +$APPEND_Chinese +$REPLACE_Festival +$REPLACE_reasonably +$APPEND_someday +$REPLACE_expressions +$APPEND_Lately +$REPLACE_average +$APPEND_season +$REPLACE_cover +$REPLACE_manager +$REPLACE_wife +$REPLACE_12 +$REPLACE_possibly +$REPLACE_approaching +$REPLACE_keeping +$REPLACE_motorcycle +$REPLACE_happily +$APPEND_items +$REPLACE_cherry +$REPLACE_shall +$REPLACE_determined +$REPLACE_cheerful +$REPLACE_ahead +$REPLACE_solution +$REPLACE_patients +$REPLACE_unforgettable +$REPLACE_decreasing +$REPLACE_laid +$REPLACE_arrange +$REPLACE_content +$REPLACE_starring +$REPLACE_opening +$REPLACE_continuing +$REPLACE_bloom +$REPLACE_concern +$APPEND_towards +$REPLACE_extreme +$APPEND_Will +$REPLACE_tests +$REPLACE_replace +$APPEND_mostly +$REPLACE_inform +$REPLACE_lying +$REPLACE_barely +$REPLACE_unpleasant +$REPLACE_brand +$REPLACE_turning +$REPLACE_added +$APPEND_age +$REPLACE_wide +$REPLACE_passing +$REPLACE_production +$REPLACE_23rd +$REPLACE_ramen +$REPLACE_occasionally +$REPLACE_borrowed +$REPLACE_comparison +$REPLACE_curry +$REPLACE_upcoming +$REPLACE_begun +$APPEND_mistakes +$REPLACE_mouth +$REPLACE_scenes +$REPLACE_accidentally +$REPLACE_gases +$REPLACE_blog +$REPLACE_Disney +$APPEND_straight +$REPLACE_topics +$REPLACE_register +$REPLACE_color +$REPLACE_explained +$APPEND_shopping +$REPLACE_Taiwan +$REPLACE_sales +$REPLACE_dictionary +$REPLACE_inexpensive +$APPEND_directly +$REPLACE_comfortably +$REPLACE_suprised +$APPEND_AM +$REPLACE_dance +$REPLACE_eager +$REPLACE_envious +$REPLACE_lie +$REPLACE_Apart +$REPLACE_closed +$REPLACE_brother +$REPLACE_hopefully +$APPEND_caught +$REPLACE_background +$REPLACE_conditions +$REPLACE_attracted +$REPLACE_aim +$REPLACE_twenty +$REPLACE_Each +$APPEND_air +$REPLACE_technique +$REPLACE_umbrella +$REPLACE_Buddhist +$REPLACE_yen +$APPEND_clothes +$APPEND_open +$REPLACE_originally +$APPEND_OK +$REPLACE_complex +$APPEND_upon +$REPLACE_< +$REPLACE_showing +$REPLACE_weaknesses +$REPLACE_OR +$REPLACE_potato +$APPEND_photo +$REPLACE_flavor +$REPLACE_Tuesday +$REPLACE_organized +$REPLACE_preferred +$REPLACE_state +$APPEND_normally +$APPEND_areas +$REPLACE_arranged +$REPLACE_embarrassing +$REPLACE_positively +$REPLACE_coworkers +$APPEND_host +$REPLACE_influenced +$REPLACE_respect +$REPLACE_separate +$REPLACE_comedies +$APPEND_listened +$REPLACE_report +$REPLACE_Using +$REPLACE_performing +$REPLACE_construction +$REPLACE_trees +$REPLACE_conversations +$REPLACE_western +$APPEND_drinking +$APPEND_Next +$APPEND_points +$APPEND_young +$REPLACE_provides +$REPLACE_motivation +$REPLACE_muscle +$REPLACE_diet +$APPEND_fluently +$REPLACE_Such +$REPLACE_task +$REPLACE_sounded +$REPLACE_schools +$REPLACE_park +$APPEND_various +$APPEND_five +$REPLACE_unhappy +$REPLACE_Due +$REPLACE_alright +$REPLACE_campus +$APPEND_foreign +$APPEND_studies +$REPLACE_handle +$REPLACE_continuous +$REPLACE_drug +$REPLACE_expenses +$REPLACE_aged +$REPLACE_surrounded +$REPLACE_thus +$REPLACE_noise +$REPLACE_healthier +$REPLACE_potential +$REPLACE_Potter +$APPEND_self +$APPEND_picture +$REPLACE_None +$REPLACE_sudden +$REPLACE_lifestyles +$APPEND_given +$REPLACE_aspects +$REPLACE_specifically +$REPLACE_destination +$REPLACE_followed +$REPLACE_Other +$REPLACE_horrible +$REPLACE_radiation +$REPLACE_essays +$REPLACE_apologize +$REPLACE_placed +$APPEND_future +$REPLACE_awkward +$REPLACE_thirty +$REPLACE_kids +$REPLACE_responsibilities +$REPLACE_Generally +$REPLACE_relatives +$APPEND_More +$REPLACE_safer +$REPLACE_hoping +$REPLACE_heroes +$REPLACE_psychological +$REPLACE_posted +$REPLACE_treatment +$REPLACE_glasses +$REPLACE_souvenirs +$REPLACE_entertaining +$APPEND_Tomorrow +$APPEND_activities +$REPLACE_serve +$REPLACE_actions +$APPEND_teacher +$REPLACE_o +$REPLACE_forever +$REPLACE_colour +$APPEND_change +$REPLACE_plants +$REPLACE_fulfill +$REPLACE_animated +$REPLACE_textbook +$REPLACE_mathematics +$REPLACE_figured +$APPEND_running +$REPLACE_computers +$REPLACE_Singaporean +$REPLACE_imagination +$REPLACE_runny +$REPLACE_bill +$REPLACE_meals +$APPEND_perhaps +$REPLACE_stupid +$REPLACE_worries +$APPEND_bought +$APPEND_article +$REPLACE_wasted +$REPLACE_falling +$REPLACE_necessity +$APPEND_common +$REPLACE_Tokyo +$REPLACE_fascinating +$REPLACE_Only +$REPLACE_tense +$APPEND_Ever +$REPLACE_behaviour +$REPLACE_magazines +$REPLACE_cultures +$REPLACE_rid +$REPLACE_choices +$REPLACE_track +$REPLACE_complaint +$REPLACE_white +$REPLACE_approximately +$REPLACE_largest +$REPLACE_smart +$APPEND_finish +$REPLACE_acting +$REPLACE_foolish +$REPLACE_prices +$REPLACE_r +$REPLACE_swim +$REPLACE_review +$REPLACE_shameful +$APPEND_Anyway +$REPLACE_senior +$REPLACE_proper +$REPLACE_provided +$REPLACE_troublesome +$APPEND_known +$REPLACE_homes +$REPLACE_spirit +$REPLACE_ga +$REPLACE_Michael +$APPEND_wish +$APPEND_performance +$REPLACE_typical +$APPEND_Well +$REPLACE_previously +$REPLACE_fail +$APPEND_itself +$REPLACE_sung +$REPLACE_citizens +$REPLACE_rapidly +$REPLACE_stadium +$APPEND_page +$APPEND_unfortunately +$REPLACE_surprising +$REPLACE_unfamiliar +$REPLACE_repair +$REPLACE_escape +$REPLACE_actor +$REPLACE_Almost +$REPLACE_shoes +$REPLACE_disagree +$REPLACE_co +$REPLACE_attempt +$REPLACE_instance +$REPLACE_lasted +$APPEND_connect +$APPEND_San +$REPLACE_hairstyle +$REPLACE_internship +$REPLACE_Bye +$REPLACE_tourist +$REPLACE_5th +$REPLACE_cousin +$REPLACE_beside +$REPLACE_facilities +$REPLACE_yummy +$REPLACE_prove +$APPEND_certain +$REPLACE_beginners +$REPLACE_worn +$REPLACE_wont +$APPEND_wearing +$REPLACE_improves +$REPLACE_electronics +$REPLACE_realistic +$REPLACE_annoying +$REPLACE_dreamed +$APPEND_results +$REPLACE_certainly +$REPLACE_classroom +$REPLACE_argument +$REPLACE_warmth +$REPLACE_achieved +$APPEND_meaning +$REPLACE_photographs +$REPLACE_animals +$REPLACE_community +$REPLACE_interests +$REPLACE_medium +$REPLACE_beer +$REPLACE_engineer +$REPLACE_Good +$APPEND_looks +$REPLACE_beauty +$APPEND_speaker +$REPLACE_connect +$APPEND_driving +$APPEND_Have +$REPLACE_reminds +$REPLACE_apologized +$REPLACE_obtain +$REPLACE_Probably +$REPLACE_strangers +$APPEND_bring +$REPLACE_smile +$REPLACE_exhibition +$REPLACE_pot +$REPLACE_encounter +$APPEND_degrees +$REPLACE_lights +$APPEND_bus +$REPLACE_movement +$REPLACE_cancel +$REPLACE_y +$REPLACE_black +$REPLACE_concentration +$REPLACE_graduating +$REPLACE_usage +$REPLACE_handsome +$APPEND_ride +$REPLACE_degree +$APPEND_point +$APPEND_conversation +$REPLACE_menu +$REPLACE_assistance +$REPLACE_Summer +$APPEND_behind +$REPLACE_police +$REPLACE_15th +$REPLACE_separated +$REPLACE_regardless +$REPLACE_significant +$REPLACE_transfer +$REPLACE_religious +$REPLACE_tempura +$REPLACE_academic +$REPLACE_otherwise +$REPLACE_useless +$REPLACE_celebrities +$REPLACE_dislike +$REPLACE_11 +$APPEND_sound +$APPEND_^ +$REPLACE_replaced +$REPLACE_sushi +$REPLACE_wind +$REPLACE_web +$REPLACE_Britain +$REPLACE_complained +$REPLACE_model +$REPLACE_de +$REPLACE_depends +$REPLACE_pm +$REPLACE_cafe +$REPLACE_congratulate +$REPLACE_ending +$APPEND_building +$REPLACE_presented +$REPLACE_shut +$APPEND_restaurant +$APPEND_March +$REPLACE_freedom +$APPEND_story +$REPLACE_creating +$REPLACE_concept +$REPLACE_conduct +$REPLACE_France +$REPLACE_paper +$REPLACE_offers +$REPLACE_Oh +$REPLACE_occured +$REPLACE_touched +$REPLACE_travelled +$APPEND_Thus +$REPLACE_sickness +$REPLACE_neighbor +$REPLACE_paying +$REPLACE_national +$APPEND_needs +$REPLACE_climb +$REPLACE_Take +$APPEND_Everyone +$REPLACE_aftershocks +$REPLACE_committed +$REPLACE_textbooks +$REPLACE_waited +$REPLACE_round +$REPLACE_Okay +$REPLACE_eldest +$APPEND_allow +$REPLACE_Spanish +$REPLACE_Spring +$REPLACE_absence +$REPLACE_actresses +$REPLACE_majority +$REPLACE_growth +$APPEND_requires +$REPLACE_About +$REPLACE_intend +$APPEND_deep +$REPLACE_enjoyment +$APPEND_raining +$REPLACE_Am +$REPLACE_eyes +$REPLACE_Afterward +$REPLACE_drugs +$REPLACE_cram +$REPLACE_dancing +$APPEND_M +$REPLACE_nationalities +$REPLACE_throat +$APPEND_shows +$REPLACE_Facebook +$APPEND_TO +$REPLACE_brilliant +$REPLACE_drop +$REPLACE_owner +$APPEND_side +$REPLACE_struggling +$REPLACE_100 +$REPLACE_surely +$REPLACE_devices +$APPEND_takes +$REPLACE_TO +$REPLACE_neighbors +$REPLACE_youth +$REPLACE_connected +$REPLACE_passes +$REPLACE_kilometers +$APPEND_fun +$REPLACE_viewing +$REPLACE_behavior +$REPLACE_chores +$REPLACE_mystery +$APPEND_shall +$APPEND_taught +$REPLACE_display +$REPLACE_ensure +$APPEND_online +$REPLACE_assignment +$REPLACE_compare +$APPEND_Still +$REPLACE_conditioning +$REPLACE_suffered +$REPLACE_haven't +$REPLACE_muscles +$APPEND_grammar +$APPEND_Two +$REPLACE_chemistry +$REPLACE_consideration +$REPLACE_smoking +$REPLACE_Harry +$APPEND_seemed +$REPLACE_marry +$REPLACE_hunting +$REPLACE_recommendation +$APPEND_previously +$REPLACE_dramas +$REPLACE_passionate +$APPEND_ways +$REPLACE_hurts +$APPEND_sense +$APPEND_drink +$REPLACE_refrigerator +$REPLACE_organised +$REPLACE_cleaning +$REPLACE_courage +$APPEND_arrived +$REPLACE_housework +$REPLACE_charge +$REPLACE_violin +$APPEND_offer +$APPEND_water +$REPLACE_injuries +$REPLACE_perspective +$REPLACE_hoped +$REPLACE_challenging +$REPLACE_THE +$APPEND_regarding +$APPEND_Their +$REPLACE_upload +$REPLACE_luxurious +$REPLACE_unnecessary +$APPEND_harder +$APPEND_twice +$REPLACE_rules +$APPEND_rest +$REPLACE_afford +$APPEND_says +$REPLACE_project +$REPLACE_bear +$REPLACE_mainly +$REPLACE_Yet +$REPLACE_diligently +$REPLACE_led +$REPLACE_architecture +$REPLACE_accurate +$REPLACE_mindset +$REPLACE_fought +$REPLACE_mid +$REPLACE_vocalist +$REPLACE_flexible +$APPEND_girl +$REPLACE_tiring +$REPLACE_broadcast +$REPLACE_July +$APPEND_version +$REPLACE_seven +$REPLACE_Nice +$REPLACE_alarm +$APPEND_dish +$REPLACE_jewelry +$REPLACE_studing +$REPLACE_cuisine +$APPEND_According +$APPEND_delicious +$REPLACE_ladies +$REPLACE_hospital +$REPLACE_sweating +$REPLACE_obviously +$APPEND_interested +$REPLACE_College +$REPLACE_Autumn +$REPLACE_Hawaii +$REPLACE_scheduled +$REPLACE_crying +$REPLACE_climbing +$APPEND_giving +$REPLACE_smoke +$APPEND_9 +$REPLACE_limit +$REPLACE_flying +$APPEND_knowledge +$REPLACE_4th +$REPLACE_Francisco +$REPLACE_tournament +$APPEND_sleep +$REPLACE_participants +$REPLACE_snacks +$REPLACE_energetic +$REPLACE_allergic +$APPEND_fast +$APPEND_score +$REPLACE_clearer +$APPEND_source +$REPLACE_lottery +$APPEND_service +$REPLACE_acquire +$REPLACE_arrival +$APPEND_situation +$REPLACE_polite +$REPLACE_laughter +$REPLACE_Thirdly +$APPEND_particular +$REPLACE_standard +$REPLACE_suppose +$REPLACE_emails +$REPLACE_Disneyland +$REPLACE_nine +$REPLACE_rising +$REPLACE_cartoon +$REPLACE_refreshing +$REPLACE_factories +$REPLACE_20th +$APPEND_single +$APPEND_sometime +$REPLACE_cleaner +$APPEND_Such +$APPEND_particularly +$REPLACE_fruit +$REPLACE_beforehand +$REPLACE_11th +$REPLACE_Halloween +$REPLACE_attract +$APPEND_forms +$APPEND_under +$REPLACE_guests +$REPLACE_classmate +$APPEND_Yours +$REPLACE_learners +$REPLACE_red +$REPLACE_critical +$REPLACE_pitiful +$REPLACE_groups +$REPLACE_grandparents +$REPLACE_primary +$REPLACE_Both +$REPLACE_aside +$REPLACE_youngest +$REPLACE_practising +$APPEND_Am +$REPLACE_summary +$REPLACE_telephone +$APPEND_nowadays +$REPLACE_20 +$REPLACE_tons +$REPLACE_Listening +$REPLACE_guilt +$REPLACE_occurs +$REPLACE_Anyways +$REPLACE_rush +$REPLACE_intermediate +$REPLACE_theirs +$APPEND_business +$REPLACE_neighboring +$REPLACE_independence +$APPEND_cost +$APPEND_country +$REPLACE_beef +$REPLACE_formal +$APPEND_worked +$REPLACE_Hence +$REPLACE_Mother +$REPLACE_picked +$REPLACE_star +$REPLACE_fishing +$REPLACE_planted +$REPLACE_fear +$APPEND_100 +$APPEND_onto +$REPLACE_choir +$REPLACE_spot +$REPLACE_correction +$REPLACE_suits +$REPLACE_Day +$REPLACE_supported +$REPLACE_comfort +$REPLACE_newspapers +$REPLACE_friendship +$REPLACE_May +$REPLACE_freezing +$REPLACE_discussed +$APPEND_{ +$APPEND_whom +$REPLACE_trust +$REPLACE_industries +$REPLACE_decisions +$APPEND_poor +$APPEND_correctly +$REPLACE_hundred +$REPLACE_recipe +$REPLACE_competitive +$REPLACE_burden +$REPLACE_abandoned +$APPEND_walking +$REPLACE_individuals +$APPEND_travelling +$REPLACE_theme +$REPLACE_runs +$REPLACE_threw +$REPLACE_rock +$APPEND_thinking +$REPLACE_Taking +$REPLACE_ideal +$REPLACE_practical +$APPEND_re +$APPEND_station +$REPLACE_collect +$REPLACE_perhaps +$REPLACE_advanced +$REPLACE_humans +$APPEND_realized +$REPLACE_remove +$REPLACE_notebook +$REPLACE_continuously +$REPLACE_beach +$REPLACE_ends +$REPLACE_secret +$REPLACE_skilled +$REPLACE_jump +$REPLACE_episodes +$REPLACE_cup +$REPLACE_consists +$REPLACE_release +$REPLACE_notes +$REPLACE_22nd +$REPLACE_fallen +$APPEND_Which +$APPEND_saw +$REPLACE_libraries +$REPLACE_consecutive +$REPLACE_March +$REPLACE_closely +$REPLACE_century +$APPEND_per +$REPLACE_circumstances +$REPLACE_whoever +$REPLACE_rented +$REPLACE_aging +$APPEND_regularly +$REPLACE_cycling +$REPLACE_depression +$REPLACE_row +$APPEND_constantly +$APPEND_feelings +$REPLACE_Angeles +$REPLACE_talented +$REPLACE_00am +$REPLACE_shower +$REPLACE_functions +$APPEND_love +$APPEND_believe +$REPLACE_basis +$REPLACE_follows +$APPEND_hardly +$REPLACE_teenager +$REPLACE_diverse +$REPLACE_Sir +$REPLACE_decrease +$REPLACE_goodbye +$REPLACE_behave +$APPEND_everywhere +$REPLACE_users +$REPLACE_analysis +$REPLACE_translating +$REPLACE_relaxation +$REPLACE_unexpectedly +$REPLACE_Russia +$REPLACE_championship +$APPEND_lives +$REPLACE_hate +$APPEND_somehow +$REPLACE_joining +$APPEND_stop +$APPEND_enjoyed +$APPEND_cup +$REPLACE_flies +$REPLACE_Talking +$REPLACE_painting +$REPLACE_letters +$REPLACE_master +$REPLACE_stated +$REPLACE_aggressive +$REPLACE_shy +$APPEND_care +$APPEND_wear +$REPLACE_served +$REPLACE_stops +$APPEND_house +$REPLACE_diligent +$REPLACE_IN +$REPLACE_deciding +$REPLACE_sweets +$REPLACE_argued +$REPLACE_bookstore +$APPEND_pretty +$REPLACE_range +$REPLACE_vegetable +$REPLACE_appreciation +$REPLACE_pity +$REPLACE_update +$REPLACE_More +$REPLACE_laughing +$REPLACE_economics +$REPLACE_cellphone +$REPLACE_OK +$REPLACE_pregnant +$REPLACE_spite +$REPLACE_karaoke +$REPLACE_tutor +$REPLACE_cockroaches +$APPEND_Most +$REPLACE_additional +$APPEND_energy +$REPLACE_contain +$REPLACE_actual +$REPLACE_shining +$APPEND_feels +$REPLACE_lesser +$REPLACE_pages +$REPLACE_cartoons +$REPLACE_arise +$REPLACE_f +$REPLACE_luckily +$REPLACE_airport +$REPLACE_windy +$REPLACE_instructor +$APPEND_Why +$REPLACE_weighed +$REPLACE_river +$APPEND_frequently +$APPEND_method +$REPLACE_shrine +$APPEND_short +$REPLACE_suffer +$REPLACE_6th +$REPLACE_fight +$APPEND_worth +$REPLACE_absent +$REPLACE_United +$REPLACE_chef +$REPLACE_anytime +$REPLACE_Three +$REPLACE_noisy +$APPEND_therefore +$REPLACE_iPod +$APPEND_French +$REPLACE_wishes +$REPLACE_Yours +$APPEND_Being +$APPEND_Its +$APPEND_field +$APPEND_photos +$REPLACE_definition +$APPEND_gives +$REPLACE_scores +$APPEND_Having +$REPLACE_statement +$APPEND_spoken +$APPEND_price +$REPLACE_cleaned +$REPLACE_varied +$APPEND_Oh +$REPLACE_wash +$REPLACE_satisfactory +$REPLACE_ceiling +$APPEND_including +$APPEND_special +$APPEND_popular +$REPLACE_invention +$REPLACE_materials +$REPLACE_media +$REPLACE_= +$REPLACE_dialogue +$REPLACE_designed +$REPLACE_popularity +$REPLACE_York +$REPLACE_Getting +$APPEND_shown +$REPLACE_carrying +$REPLACE_00pm +$REPLACE_stations +$REPLACE_puts +$REPLACE_screen +$REPLACE_appreciative +$REPLACE_cruel +$APPEND_main +$REPLACE_action +$REPLACE_unlucky +$REPLACE_God +$APPEND_basically +$REPLACE_d +$REPLACE_climbed +$REPLACE_thoroughly +$REPLACE_Canada +$REPLACE_hesitate +$APPEND_developed +$APPEND_post +$REPLACE_represent +$REPLACE_comment +$REPLACE_controlled +$REPLACE_source +$REPLACE_customs +$REPLACE_drawn +$REPLACE_mature +$REPLACE_commute +$APPEND_Once +$APPEND_letter +$REPLACE_attached +$REPLACE_gift +$REPLACE_nap +$APPEND_asked +$REPLACE_inspired +$APPEND_event +$REPLACE_seafood +$APPEND_watched +$REPLACE_errors +$APPEND_passed +$APPEND_english +$REPLACE_complaining +$REPLACE_roommate +$REPLACE_Life +$REPLACE_mental +$REPLACE_grades +$APPEND_parts +$REPLACE_pronounciation +$REPLACE_strengthen +$REPLACE_priority +$APPEND_abroad +$APPEND_ticket +$REPLACE_insurance +$REPLACE_hesitation +$REPLACE_researched +$REPLACE_unlike +$REPLACE_exercising +$REPLACE_exchanged +$REPLACE_knows +$REPLACE_founded +$REPLACE_messy +$REPLACE_dying +$APPEND_plans +$APPEND_match +$REPLACE_Fourth +$REPLACE_answers +$REPLACE_assignments +$REPLACE_Whether +$REPLACE_elder +$REPLACE_gas +$REPLACE_heading +$REPLACE_laws +$REPLACE_kindly +$REPLACE_wine +$REPLACE_household +$REPLACE_dining +$REPLACE_sensitive +$REPLACE_wet +$REPLACE_Personally +$APPEND_middle +$REPLACE_busier +$REPLACE_dirty +$REPLACE_religion +$REPLACE_facing +$APPEND_totally +$REPLACE_repeatedly +$REPLACE_tries +$REPLACE_organising +$REPLACE_operating +$REPLACE_ex +$APPEND_languages +$REPLACE_services +$REPLACE_remaining +$REPLACE_killed +$REPLACE_fair +$REPLACE_bike +$REPLACE_'t +$APPEND_titled +$REPLACE_exception +$APPEND_carefully +$REPLACE_salon +$REPLACE_translated +$REPLACE_welcome +$REPLACE_gratitude +$REPLACE_Watching +$REPLACE_adults +$APPEND_large +$REPLACE_untill +$REPLACE_coach +$REPLACE_mountains +$REPLACE_sandwich +$REPLACE_examples +$APPEND_gone +$REPLACE_multiple +$APPEND_meant +$REPLACE_delivered +$REPLACE_entering +$APPEND_Hello +$REPLACE_option +$REPLACE_cigarette +$REPLACE_acted +$REPLACE_bathroom +$REPLACE_accustomed +$REPLACE_literature +$REPLACE_bottom +$APPEND_course +$APPEND_choose +$REPLACE_resume +$APPEND_web +$REPLACE_aloud +$REPLACE_material +$REPLACE_struggle +$REPLACE_trains +$REPLACE_dog +$APPEND_Both +$REPLACE_leisure +$REPLACE_climate +$REPLACE_japanese +$REPLACE_reduced +$APPEND_break +$APPEND_grow +$REPLACE_Thinking +$REPLACE_dessert +$REPLACE_Yeah +$REPLACE_salt +$REPLACE_rare +$REPLACE_fairly +$REPLACE_knowing +$REPLACE_varieties +$APPEND_festival +$REPLACE_kitten +$APPEND_changes +$REPLACE_Introduction +$REPLACE_viruses +$APPEND_gotten +$REPLACE_h +$REPLACE_experiencing +$APPEND_rain +$APPEND_weight +$REPLACE_brown +$REPLACE_Everyday +$APPEND_Tokyo +$REPLACE_split +$REPLACE_section +$APPEND_dinner +$REPLACE_Making +$REPLACE_courses +$REPLACE_remains +$REPLACE_Dragon +$REPLACE_soft +$REPLACE_independent +$REPLACE_conducted +$APPEND_mode +$APPEND_tickets +$APPEND_leave +$APPEND_culture +$REPLACE_Iam +$REPLACE_joy +$REPLACE_violent +$REPLACE_leaf +$REPLACE_fortune +$APPEND_reasons +$REPLACE_Fukushima +$APPEND_thus +$REPLACE_boss +$REPLACE_player +$REPLACE_closest +$REPLACE_lies +$APPEND_consists +$REPLACE_impolite +$REPLACE_unpredictable +$REPLACE_shared +$REPLACE_7th +$APPEND_Up +$REPLACE_step +$APPEND_football +$REPLACE_central +$REPLACE_symptoms +$REPLACE_funds +$REPLACE_resolve +$REPLACE_Technology +$REPLACE_solutions +$REPLACE_adult +$REPLACE_military +$REPLACE_supermarkets +$APPEND_sites +$REPLACE_levels +$REPLACE_broad +$REPLACE_smiling +$REPLACE_expecting +$REPLACE_shorter +$APPEND_Like +$REPLACE_gloomy +$REPLACE_weekdays +$REPLACE_blew +$REPLACE_determine +$REPLACE_discount +$APPEND_attend +$REPLACE_treated +$REPLACE_length +$REPLACE_raw +$REPLACE_promote +$REPLACE_court +$REPLACE_commercial +$REPLACE_expectations +$APPEND_exercise +$REPLACE_tickets +$REPLACE_status +$REPLACE_retirement +$REPLACE_crowd +$REPLACE_requested +$REPLACE_South +$APPEND_corrected +$REPLACE_aunt +$REPLACE_Traveling +$REPLACE_region +$REPLACE_pulled +$APPEND_14 +$REPLACE_impatient +$REPLACE_roads +$REPLACE_value +$REPLACE_existence +$REPLACE_applications +$REPLACE_boiled +$REPLACE_warming +$REPLACE_15 +$REPLACE_Iwas +$REPLACE_accomplish +$APPEND_Sounds +$APPEND_send +$APPEND_programs +$REPLACE_costume +$APPEND_1st +$REPLACE_ancient +$REPLACE_physics +$REPLACE_record +$REPLACE_published +$REPLACE_cross +$REPLACE_harmful +$REPLACE_description +$APPEND_wrote +$APPEND_pay +$REPLACE_fond +$APPEND_color +$REPLACE_asks +$APPEND_stuff +$REPLACE_specially +$REPLACE_uneasy +$APPEND_riding +$REPLACE_inthe +$REPLACE_nose +$REPLACE_scientific +$REPLACE_Among +$REPLACE_danger +$REPLACE_commit +$REPLACE_Particularly +$REPLACE_troubles +$REPLACE_button +$REPLACE_delayed +$REPLACE_Diego +$REPLACE_daytime +$REPLACE_phenomenon +$APPEND_following +$REPLACE_Consequently +$REPLACE_saving +$REPLACE_souvenir +$REPLACE_missing +$REPLACE_unless +$APPEND_office +$REPLACE_anniversary +$REPLACE_anger +$APPEND_himself +$APPEND_happening +$REPLACE_cheer +$REPLACE_animal +$APPEND_subject +$REPLACE_nicer +$REPLACE_sells +$REPLACE_lenses +$REPLACE_OF +$REPLACE_possibilities +$REPLACE_efforts +$REPLACE_Years +$REPLACE_merchandise +$REPLACE_subsidies +$REPLACE_forms +$REPLACE_hotel +$APPEND_non +$REPLACE_appetite +$REPLACE_sport +$REPLACE_expand +$REPLACE_rhythm +$APPEND_Another +$REPLACE_Language +$APPEND_Each +$REPLACE_window +$REPLACE_increases +$REPLACE_states +$REPLACE_excitement +$REPLACE_promise +$APPEND_seen +$REPLACE_luggage +$APPEND_generally +$REPLACE_frustrating +$REPLACE_colors +$REPLACE_mosquitoes +$REPLACE_seats +$REPLACE_woken +$REPLACE_switched +$REPLACE_grammatically +$REPLACE_ON +$REPLACE_kindness +$REPLACE_thieves +$REPLACE_spoiled +$REPLACE_States +$REPLACE_hamburgers +$APPEND_nearly +$REPLACE_situated +$REPLACE_foods +$REPLACE_collecting +$REPLACE_unfortunate +$REPLACE_camera +$REPLACE_dramatic +$REPLACE_noodle +$APPEND_human +$REPLACE_re +$REPLACE_humidity +$APPEND_strongly +$REPLACE_kimchi +$APPEND_difference +$REPLACE_artists +$REPLACE_medical +$REPLACE_incredible +$APPEND_helping +$APPEND_ahead +$REPLACE_lines +$REPLACE_thinks +$REPLACE_thousand +$REPLACE_sixth +$REPLACE_exposed +$REPLACE_colours +$REPLACE_widely +$APPEND_nuclear +$REPLACE_worldwide +$REPLACE_comprehension +$APPEND_hair +$REPLACE_halfway +$APPEND_cause +$REPLACE_cast +$APPEND_coffee +$REPLACE_attractions +$REPLACE_beautifully +$REPLACE_handwritten +$APPEND_band +$APPEND_improving +$APPEND_40 +$REPLACE_shops +$REPLACE_basically +$APPEND_studied +$REPLACE_manufacturer +$REPLACE_Western +$APPEND_throughout +$REPLACE_identify +$APPEND_Would +$REPLACE_Switzerland +$APPEND_everybody +$APPEND_grade +$REPLACE_farewell +$REPLACE_romantic +$REPLACE_Celsius +$REPLACE_bread +$APPEND_favorite +$APPEND_Despite +$REPLACE_downloaded +$REPLACE_balance +$APPEND_carry +$REPLACE_cure +$REPLACE_programmer +$APPEND_considered +$APPEND_slowly +$REPLACE_discovery +$APPEND_stopped +$REPLACE_standing +$REPLACE_earned +$REPLACE_skating +$REPLACE_detail +$REPLACE_apology +$REPLACE_writer +$REPLACE_highway +$REPLACE_Goodbye +$REPLACE_quote +$REPLACE_maintenance +$APPEND_taste +$REPLACE_package +$REPLACE_responded +$REPLACE_criticize +$APPEND_deeply +$REPLACE_jogging +$APPEND_waiting +$REPLACE_fatter +$REPLACE_cycle +$APPEND_Only +$REPLACE_afterward +$REPLACE_specialty +$REPLACE_goodness +$REPLACE_groceries +$APPEND_staff +$REPLACE_somehow +$APPEND_Moreover +$APPEND_training +$REPLACE_clever +$REPLACE_camp +$APPEND_traveling +$APPEND_minutes +$REPLACE_sandwiches +$APPEND_run +$REPLACE_options +$REPLACE_calories +$REPLACE_branch +$REPLACE_barbecue +$APPEND_entrance +$REPLACE_noodles +$APPEND_products +$APPEND_helped +$REPLACE_newly +$APPEND_drank +$REPLACE_precise +$REPLACE_increasingly +$APPEND_Dear +$REPLACE_novels +$REPLACE_mix +$REPLACE_budget +$REPLACE_petrol +$REPLACE_trial +$APPEND_Perhaps +$REPLACE_occasions +$APPEND_Actually +$REPLACE_eastern +$REPLACE_sights +$REPLACE_industrial +$APPEND_result +$REPLACE_generally +$REPLACE_Canadian +$REPLACE_Surprisingly +$APPEND_strong +$REPLACE_memorizing +$REPLACE_irritated +$REPLACE_implemented +$REPLACE_Welcome +$REPLACE_coast +$REPLACE_signs +$REPLACE_leading +$APPEND_PM +$APPEND_access +$REPLACE_fat +$REPLACE_breeze +$REPLACE_India +$APPEND_slept +$REPLACE_weigh +$REPLACE_commonly +$REPLACE_supervisor +$REPLACE_tomato +$REPLACE_agency +$APPEND_till +$REPLACE_couldn +$REPLACE_strangely +$APPEND_stayed +$REPLACE_ni +$APPEND_exams +$REPLACE_School +$REPLACE_blue +$APPEND_allowed +$REPLACE_30th +$REPLACE_kittens +$REPLACE_typing +$REPLACE_headed +$APPEND_present +$REPLACE_Reading +$REPLACE_injury +$REPLACE_Dear +$REPLACE_PM +$REPLACE_minor +$REPLACE_drinks +$REPLACE_enthusiasm +$REPLACE_dilemma +$REPLACE_income +$REPLACE_sadness +$REPLACE_weaker +$REPLACE_Thanksgiving +$REPLACE_documents +$REPLACE_fake +$REPLACE_boy +$REPLACE_regards +$APPEND_Finally +$REPLACE_obstacle +$REPLACE_batteries +$APPEND_talked +$APPEND_becomes +$REPLACE_numerous +$REPLACE_cheese +$REPLACE_judge +$APPEND_busy +$APPEND_reach +$APPEND_Fuji +$REPLACE_intelligent +$REPLACE_reception +$REPLACE_Chinatown +$REPLACE_repeat +$APPEND_June +$REPLACE_reported +$APPEND_required +$REPLACE_cases +$REPLACE_matters +$REPLACE_prepositions +$REPLACE_accidents +$REPLACE_fields +$APPEND_ask +$APPEND_sad +$REPLACE_selected +$REPLACE_skipped +$REPLACE_freshmen +$REPLACE_mode +$REPLACE_calendar +$REPLACE_luxury +$REPLACE_summertime +$REPLACE_device +$APPEND_lesson +$APPEND_surely +$REPLACE_loved +$REPLACE_reflect +$REPLACE_shoulder +$REPLACE_muscular +$APPEND_plenty +$REPLACE_Indian +$REPLACE_pork +$REPLACE_double +$REPLACE_loneliness +$REPLACE_economies +$REPLACE_meaningful +$REPLACE_cooperate +$REPLACE_land +$APPEND_report +$REPLACE_block +$REPLACE_cheapest +$REPLACE_mirror +$REPLACE_wealthy +$APPEND_application +$REPLACE_quarter +$REPLACE_babies +$REPLACE_risk +$REPLACE_discussions +$REPLACE_lightning +$REPLACE_briefly +$REPLACE_congratulated +$REPLACE_breathing +$REPLACE_eagerly +$REPLACE_resolved +$APPEND_staying +$APPEND_history +$APPEND_phones +$REPLACE_involving +$REPLACE_enthusiastic +$REPLACE_cookies +$REPLACE_frightened +$REPLACE_entirely +$REPLACE_enormous +$APPEND_aspects +$REPLACE_stable +$APPEND_section +$APPEND_Thanks +$APPEND_women +$REPLACE_phase +$REPLACE_16th +$REPLACE_spicy +$APPEND_produced +$REPLACE_street +$REPLACE_ignore +$REPLACE_designer +$APPEND_club +$REPLACE_mum +$REPLACE_sincere +$REPLACE_offensive +$REPLACE_memorized +$APPEND_question +$REPLACE_wa +$REPLACE_garbage +$REPLACE_Playing +$REPLACE_castle +$REPLACE_swam +$REPLACE_leader +$REPLACE_earthquakes +$REPLACE_displayed +$REPLACE_marathon +$APPEND_songs +$REPLACE_See +$REPLACE_burn +$APPEND_happily +$REPLACE_salesman +$REPLACE_unhealthy +$REPLACE_base +$REPLACE_crossing +$REPLACE_Honestly +$REPLACE_machines +$REPLACE_freshman +$REPLACE_dry +$APPEND_exact +$APPEND_January +$APPEND_terms +$REPLACE_happiest +$APPEND_tastes +$APPEND_design +$REPLACE_champion +$REPLACE_Diary +$REPLACE_expressing +$REPLACE_hardest +$REPLACE_installed +$REPLACE_Go +$REPLACE_dollar +$REPLACE_wooden +$REPLACE_contrary +$REPLACE_refers +$REPLACE_employment +$REPLACE_removed +$REPLACE_opposing +$REPLACE_actress +$REPLACE_Ever +$APPEND_beginning +$REPLACE_approach +$REPLACE_guide +$REPLACE_blooming +$REPLACE_necessarily +$REPLACE_fed +$REPLACE_stands +$REPLACE_principal +$REPLACE_faced +$APPEND_local +$APPEND_highly +$REPLACE_fiction +$APPEND_finding +$REPLACE_attracts +$REPLACE_2011 +$REPLACE_businessmen +$REPLACE_Friends +$REPLACE_repaired +$REPLACE_bet +$REPLACE_hunger +$REPLACE_dealing +$REPLACE_Except +$APPEND_role +$REPLACE_admitted +$REPLACE_island +$REPLACE_quietly +$REPLACE_lets +$REPLACE_fee +$REPLACE_performances +$REPLACE_bar +$REPLACE_maximum +$REPLACE_escaped +$REPLACE_ours +$APPEND_originally +$REPLACE_surroundings +$REPLACE_golden +$APPEND_technology +$APPEND_research +$REPLACE_borrow +$REPLACE_remind +$REPLACE_Beginning +$REPLACE_passage +$APPEND_drive +$APPEND_teaching +$REPLACE_typhoons +$REPLACE_grabbed +$REPLACE_incidents +$REPLACE_hid +$REPLACE_operate +$REPLACE_19th +$APPEND_sure +$REPLACE_permission +$APPEND_previous +$REPLACE_rental +$REPLACE_tothe +$APPEND_round +$REPLACE_Oops +$REPLACE_survival +$REPLACE_shaped +$APPEND_costs +$REPLACE_conference +$APPEND_move +$REPLACE_dressed +$REPLACE_smells +$REPLACE_artistic +$REPLACE_holds +$REPLACE_introducing +$REPLACE_nursery +$APPEND_May +$REPLACE_troubled +$REPLACE_optimistic +$REPLACE_guarantee +$REPLACE_toothache +$REPLACE_bother +$REPLACE_Congratulations +$REPLACE_purchased +$APPEND_21 +$REPLACE_accurately +$REPLACE_belief +$REPLACE_numbers +$REPLACE_switch +$REPLACE_personally +$REPLACE_negatively +$REPLACE_fireflies +$APPEND_receive +$APPEND_shop +$REPLACE_haircut +$REPLACE_productive +$REPLACE_crisis +$REPLACE_relatively +$REPLACE_celebration +$REPLACE_controversial +$REPLACE_AM +$REPLACE_factors +$REPLACE_snowing +$REPLACE_amusing +$REPLACE_sharing +$REPLACE_Companies +$REPLACE_NYC +$REPLACE_moves +$REPLACE_hanging +$REPLACE_simpler +$APPEND_apart +$REPLACE_race +$REPLACE_hip +$REPLACE_underwear +$REPLACE_official +$REPLACE_shift +$APPEND_week +$REPLACE_analyse +$REPLACE_25th +$REPLACE_teenage +$APPEND_recent +$REPLACE_skin +$REPLACE_enroll +$REPLACE_nickname +$APPEND_accidentally +$REPLACE_inventions +$REPLACE_boys +$APPEND_Afterwards +$REPLACE_gentle +$REPLACE_overnight +$APPEND_explain +$REPLACE_wanting +$REPLACE_encouraging +$REPLACE_contribute +$REPLACE_necessities +$REPLACE_enrolled +$REPLACE_Normally +$REPLACE_balloon +$REPLACE_applying +$APPEND_uses +$REPLACE_recall +$REPLACE_nearest +$REPLACE_cashier +$REPLACE_corner +$APPEND_space +$REPLACE_thatI +$REPLACE_treasure +$REPLACE_International +$REPLACE_forth +$REPLACE_assigned +$APPEND_education +$APPEND_except +$REPLACE_jewellery +$REPLACE_manga +$APPEND_participate +$APPEND_increase +$REPLACE_slippery +$REPLACE_snowboard +$REPLACE_novel +$REPLACE_predict +$REPLACE_remained +$REPLACE_outcome +$APPEND_whose +$APPEND_slightly +$APPEND_serious +$REPLACE_Research +$REPLACE_marvelous +$APPEND_excited +$REPLACE_organization +$REPLACE_list +$REPLACE_automatically +$REPLACE_differ +$REPLACE_Mount +$REPLACE_arrangement +$APPEND_spending +$REPLACE_adopt +$APPEND_Soon +$APPEND_Mr +$REPLACE_irritable +$REPLACE_Wish +$REPLACE_writting +$REPLACE_Sincerely +$APPEND_winter +$REPLACE_rose +$REPLACE_businessman +$REPLACE_flavors +$REPLACE_smell +$REPLACE_fortunate +$APPEND_TOEIC +$APPEND_mentioned +$APPEND_process +$APPEND_amp +$APPEND_neither +$REPLACE_enemies +$REPLACE_acceptance +$REPLACE_drivers +$REPLACE_murderer +$REPLACE_Melbourne +$REPLACE_Specifically +$APPEND_complete +$APPEND_focus +$REPLACE_illegal +$APPEND_hurts +$REPLACE_groom +$APPEND_preposition +$APPEND_com +$APPEND_beautiful +$REPLACE_sightseeing +$REPLACE_bringing +$REPLACE_sources +$APPEND_videos +$APPEND_lunch +$APPEND_11 +$REPLACE_suggestion +$REPLACE_programmes +$APPEND_jobs +$REPLACE_scent +$REPLACE_crime +$REPLACE_desperate +$REPLACE_deliver +$APPEND_performed +$REPLACE_cars +$REPLACE_pet +$REPLACE_dangers +$APPEND_perform +$REPLACE_vehicles +$APPEND_figure +$APPEND_Later +$REPLACE_matches +$REPLACE_spaghetti +$APPEND_light +$REPLACE_corrects +$REPLACE_Unlike +$APPEND_occasionally +$APPEND_truly +$REPLACE_silence +$REPLACE_intense +$REPLACE_substitute +$APPEND_freely +$APPEND_party +$APPEND_His +$REPLACE_bothersome +$REPLACE_pursuing +$REPLACE_Out +$REPLACE_direction +$APPEND_check +$REPLACE_authorities +$APPEND_sort +$REPLACE_challenges +$REPLACE_plural +$REPLACE_refused +$REPLACE_informed +$REPLACE_demand +$REPLACE_mess +$REPLACE_force +$REPLACE_paintings +$APPEND_remember +$REPLACE_sky +$APPEND_practicing +$REPLACE_understandable +$REPLACE_crashed +$APPEND_communicate +$REPLACE_manner +$REPLACE_payment +$REPLACE_artist +$APPEND_tend +$REPLACE_recession +$REPLACE_til +$REPLACE_mixed +$APPEND_bar +$REPLACE_England +$REPLACE_gathered +$REPLACE_combined +$REPLACE_Rome +$APPEND_wet +$REPLACE_network +$REPLACE_steak +$REPLACE_California +$REPLACE_birth +$APPEND_state +$REPLACE_expressed +$REPLACE_haven +$REPLACE_seldom +$APPEND_health +$REPLACE_partners +$REPLACE_finishing +$REPLACE_Monday +$REPLACE_liters +$REPLACE_Hi +$APPEND_August +$REPLACE_gorgeous +$APPEND_seven +$APPEND_remaining +$REPLACE_chances +$APPEND_older +$REPLACE_Eating +$APPEND_Christmas +$REPLACE_dentist +$REPLACE_league +$REPLACE_korean +$APPEND_greatly +$APPEND_return +$REPLACE_genres +$REPLACE_authors +$APPEND_Thank +$REPLACE_diseases +$REPLACE_travels +$REPLACE_sheet +$REPLACE_fastest +$APPEND_surprised +$REPLACE_rushed +$APPEND_attending +$APPEND_Furthermore +$REPLACE_Laden +$REPLACE_creative +$REPLACE_meantime +$REPLACE_Turkey +$REPLACE_presenting +$REPLACE_Christian +$REPLACE_nervousness +$REPLACE_meaningless +$APPEND_player +$REPLACE_motivate +$REPLACE_advertisements +$REPLACE_artwork +$REPLACE_encouragement +$REPLACE_regard +$REPLACE_slower +$REPLACE_dolls +$REPLACE_200 +$REPLACE_unconsciously +$APPEND_happens +$REPLACE_facility +$APPEND_advice +$REPLACE_North +$REPLACE_awareness +$APPEND_planned +$REPLACE_genetic +$REPLACE_management +$REPLACE_refund +$REPLACE_brighter +$REPLACE_confirm +$REPLACE_burning +$REPLACE_composition +$APPEND_answer +$REPLACE_conserve +$REPLACE_destruction +$REPLACE_duties +$REPLACE_creativity +$APPEND_expressions +$APPEND_commit +$REPLACE_East +$REPLACE_milk +$REPLACE_30pm +$REPLACE_belong +$REPLACE_autograph +$REPLACE_caring +$REPLACE_download +$APPEND_development +$REPLACE_compete +$REPLACE_qualities +$APPEND_avoid +$REPLACE_recieved +$APPEND_Perfect +$REPLACE_yours +$REPLACE_breaks +$REPLACE_amusement +$REPLACE_models +$REPLACE_persevere +$REPLACE_emergency +$REPLACE_empty +$REPLACE_rescue +$APPEND_term +$REPLACE_requirements +$REPLACE_sufficient +$APPEND_cooking +$REPLACE_fascinated +$REPLACE_14th +$REPLACE_relevant +$REPLACE_listed +$REPLACE_vision +$REPLACE_g +$REPLACE_leadership +$REPLACE_butI +$APPEND_provide +$REPLACE_organize +$APPEND_created +$REPLACE_12th +$REPLACE_collection +$REPLACE_supply +$APPEND_Besides +$REPLACE_stranger +$REPLACE_combination +$REPLACE_farther +$REPLACE_awaiting +$APPEND_hand +$REPLACE_unsure +$REPLACE_profile +$APPEND_moving +$APPEND_street +$REPLACE_delighted +$REPLACE_pretended +$REPLACE_driven +$REPLACE_maintaining +$REPLACE_liar +$TRANSFORM_SPLIT_HYPHEN +$REPLACE_glass +$REPLACE_stick +$REPLACE_itchy +$REPLACE_ought +$REPLACE_consumption +$REPLACE_quicker +$REPLACE_spare +$REPLACE_governments +$APPEND_view +$REPLACE_P +$REPLACE_colorful +$REPLACE_guitarist +$APPEND_wants +$REPLACE_million +$REPLACE_behalf +$REPLACE_kilometres +$REPLACE_bank +$APPEND_morning +$REPLACE_weekends +$REPLACE_occasion +$APPEND_tour +$REPLACE_object +$REPLACE_Others +$REPLACE_Considering +$REPLACE_species +$REPLACE_session +$APPEND_removed +$REPLACE_hiking +$REPLACE_resolutions +$REPLACE_peak +$REPLACE_consequences +$REPLACE_soaked +$REPLACE_presents +$APPEND_25 +$REPLACE_salad +$REPLACE_filling +$REPLACE_attack +$APPEND_foods +$REPLACE_tendency +$REPLACE_discoveries +$REPLACE_immediate +$REPLACE_submitted +$REPLACE_THAT +$APPEND_develop +$REPLACE_battery +$REPLACE_dont +$REPLACE_feature +$APPEND_opportunity +$REPLACE_bodies +$REPLACE_goldfish +$REPLACE_adapt +$REPLACE_views +$REPLACE_forgetting +$REPLACE_saved +$REPLACE_doesn +$REPLACE_thirst +$APPEND_Me +$REPLACE_distant +$REPLACE_opposition +$REPLACE_breed +$REPLACE_practised +$REPLACE_miserable +$APPEND_sore +$REPLACE_brain +$REPLACE_sessions +$REPLACE_policeman +$REPLACE_favor +$REPLACE_managing +$REPLACE_rains +$REPLACE_baths +$REPLACE_surrounding +$REPLACE_Seoul +$APPEND_regardless +$APPEND_Something +$REPLACE_architectural +$REPLACE_ok +$REPLACE_welfare +$APPEND_share +$REPLACE_daughters +$REPLACE_phones +$REPLACE_downstairs +$REPLACE_arriving +$REPLACE_stepped +$REPLACE_competing +$REPLACE_catching +$REPLACE_conversing +$REPLACE_encourages +$REPLACE_depressing +$REPLACE_begining +$REPLACE_admission +$APPEND_voice +$REPLACE_boredom +$APPEND_alot +$APPEND_familiar +$REPLACE_breaking +$REPLACE_fortunately +$REPLACE_Over +$APPEND_lost +$REPLACE_intended +$REPLACE_neighbourhood +$REPLACE_mysteries +$REPLACE_certificate +$REPLACE_data +$APPEND_personal +$REPLACE_joyful +$REPLACE_immigrants +$REPLACE_emotions +$REPLACE_checkup +$REPLACE_licence +$REPLACE_juice +$APPEND_whenever +$REPLACE_dogs +$REPLACE_thereby +$APPEND_department +$APPEND_assignment +$REPLACE_defend +$REPLACE_approached +$REPLACE_Fireworks +$APPEND_activity +$APPEND_quality +$REPLACE_basics +$REPLACE_costumes +$REPLACE_key +$REPLACE_outdoors +$REPLACE_hay +$APPEND_prepare +$REPLACE_hiding +$REPLACE_curiosity +$APPEND_dealing +$REPLACE_passion +$REPLACE_costed +$REPLACE_fries +$REPLACE_HAVE +$REPLACE_divorced +$APPEND_display +$REPLACE_baby +$APPEND_cherry +$REPLACE_Returning +$APPEND_lack +$APPEND_learnt +$REPLACE_Im +$APPEND_naturally +$REPLACE_router +$APPEND_goals +$REPLACE_seaside +$REPLACE_summarize +$APPEND_appeared +$REPLACE_claim +$APPEND_ate +$REPLACE_exchanging +$APPEND_arrive +$APPEND_art +$REPLACE_participating +$REPLACE_seek +$REPLACE_innocent +$APPEND_express +$REPLACE_lunchtime +$REPLACE_reaction +$REPLACE_consisted +$REPLACE_Eastern +$APPEND_track +$APPEND_baby +$REPLACE_touching +$REPLACE_lively +$REPLACE_bridge +$REPLACE_murderers +$REPLACE_Brazil +$REPLACE_feeding +$REPLACE_honestly +$REPLACE_Piece +$REPLACE_springs +$REPLACE_purchase +$REPLACE_pray +$REPLACE_washed +$APPEND_sentence +$REPLACE_Olympics +$REPLACE_strongest +$REPLACE_leads +$REPLACE_stomachache +$REPLACE_John +$REPLACE_opponent +$REPLACE_contents +$REPLACE_plot +$APPEND_Many +$REPLACE_experiment +$REPLACE_beings +$REPLACE_owns +$REPLACE_airline +$REPLACE_severely +$REPLACE_ages +@@UNKNOWN@@ +@@PADDING@@ diff --git a/test_fixtures/roberta_model/vocabulary/non_padded_namespaces.txt b/test_fixtures/roberta_model/vocabulary/non_padded_namespaces.txt new file mode 100644 index 0000000..d5dee50 --- /dev/null +++ b/test_fixtures/roberta_model/vocabulary/non_padded_namespaces.txt @@ -0,0 +1,2 @@ +*tags +*labels From e3cca599f77917612792266d7af18b6622c1eb91 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 13:52:32 -0400 Subject: [PATCH 11/25] Fixing errors in modeling code now that model.decode adds the original words and corrected words to the output. --- tests/test_seq2labels.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/test_seq2labels.py b/tests/test_seq2labels.py index 9a485dd..60d559c 100644 --- a/tests/test_seq2labels.py +++ b/tests/test_seq2labels.py @@ -4,6 +4,7 @@ from allennlp.common.testing import ModelTestCase from allennlp.data.dataset import Batch from allennlp.data.fields import TextField +from allennlp.data.fields import MetadataField from allennlp.data.instance import Instance from allennlp.data import Token from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder @@ -53,16 +54,20 @@ def setUp(self): self.device_index = 0 if torch.cuda.is_available() else -1 sentence1 = "the quickest quick brown fox jumped over the lazy dog" - tokens1 = [Token(word) for word in sentence1.split()] + sentence1_words = [word for word in sentence1.split()] + tokens1 = [Token(word) for word in sentence1_words] sentence2 = "the quick brown fox jumped over the laziest lazy elmo" - tokens2 = [Token(word) for word in sentence2.split()] + sentence2_words = [word for word in sentence2.split()] + tokens2 = [Token(word) for word in sentence2_words] instance1 = Instance( - {"tokens": TextField(tokens1, {"bert": token_indexer})} + {"tokens": TextField(tokens1, {"bert": token_indexer}), + "metadata": MetadataField({"words": sentence1_words})} ) instance2 = Instance( - {"tokens": TextField(tokens2, {"bert": token_indexer})} + {"tokens": TextField(tokens2, {"bert": token_indexer}), + "metadata": MetadataField({"words": sentence2_words})} ) self.batch = Batch([instance1, instance2]) @@ -93,6 +98,8 @@ def test_forward_pass_runs_correctly(self): "max_error_probability", "labels", "d_tags", + "words", + "corrected_words" ] ) probs = output_dict["class_probabilities_labels"][0].data.cpu().numpy() From 1c0025c96b631038401216ad117eef58474b00b6 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 14:37:21 -0400 Subject: [PATCH 12/25] Adding conditional so that no correction is performed in decode if no errors are identified. Fixing offsets in light of addition of START_TOKEN to front. --- gector/seq2labels_model.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/gector/seq2labels_model.py b/gector/seq2labels_model.py index f625fbb..84f2e89 100644 --- a/gector/seq2labels_model.py +++ b/gector/seq2labels_model.py @@ -195,15 +195,16 @@ def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor max_probs = torch.max(probs, dim=0) probs = max_probs[0].tolist() indices = max_probs[1].tolist() - print(f"{indices=}") - suggested_token_operations = output_dict['labels'][i] - actions_per_token = self.get_token_actions(indices=indices, probs=probs, - sugg_tokens=suggested_token_operations) - # Filter out cases where no corrections should be applied - actions_per_token = [action for action in actions_per_token if action] - print(f"{actions_per_token=}") - corrected_sent = get_target_sent_by_edits(output_dict['words'][i], actions_per_token) - output_dict['corrected_words'].append(corrected_sent) + if max(indices) == 0: # No corrections should be performed + output_dict["corrected_words"] = output_dict["words"] + else: + suggested_token_operations = output_dict['labels'][i] + actions_per_token = self.get_token_actions(indices=indices, probs=probs, + sugg_tokens=suggested_token_operations) + # Filter out cases where no corrections should be applied + actions_per_token = [action for action in actions_per_token if action] + corrected_sent = get_target_sent_by_edits(output_dict['words'][i], actions_per_token) + output_dict['corrected_words'].append(corrected_sent) return output_dict def get_token_actions(self, indices, probs, sugg_tokens): @@ -227,9 +228,8 @@ def get_token_actions(self, indices, probs, sugg_tokens): elif sugg_token.startswith('$TRANSFORM_') or sugg_token.startswith("$MERGE_"): sugg_token_clear = sugg_token[:] else: - print(f"{sugg_token=}") sugg_token_clear = sugg_token[sugg_token.index('_') + 1:] - actions.append((start_pos, end_pos, sugg_token_clear, prob)) + actions.append((start_pos - 1, end_pos - 1, sugg_token_clear, prob)) return actions @overrides From 0f204ea8967ed3bc164182dc825f0a9a803cc6e3 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 14:38:18 -0400 Subject: [PATCH 13/25] Appending start token when creating instances from json or string. --- gector/gec_predictor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gector/gec_predictor.py b/gector/gec_predictor.py index 8a91685..2c58d67 100644 --- a/gector/gec_predictor.py +++ b/gector/gec_predictor.py @@ -6,12 +6,12 @@ from allennlp.common.util import sanitize from overrides import overrides from allennlp.common.util import JsonDict -from allennlp.data import DatasetReader, Instance +from allennlp.data import DatasetReader, Instance, Token from allennlp.data.fields import TextField, SequenceLabelField from allennlp.data.tokenizers.word_splitter import SpacyWordSplitter from allennlp.data.tokenizers.word_splitter import JustSpacesWordSplitter from allennlp.models import Model - +from utils.helpers import START_TOKEN @Predictor.register("gec-predictor") @@ -51,6 +51,8 @@ def _json_to_instance(self, json_dict: JsonDict) -> Instance: """ sentence = json_dict["sentence"] tokens = self._tokenizer.split_words(sentence) + # Add start token to front + tokens = [Token(START_TOKEN)] + tokens return self._dataset_reader.text_to_instance(tokens) From 67ef51133bba6c1e1876a885cf91226a6316aed9 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 14:38:49 -0400 Subject: [PATCH 14/25] Start token is expected in ouptut. Everything works now except that the text is getting lowercased somewhere. --- tests/test_gec_predictor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_gec_predictor.py b/tests/test_gec_predictor.py index 968afe7..c2661db 100644 --- a/tests/test_gec_predictor.py +++ b/tests/test_gec_predictor.py @@ -48,6 +48,6 @@ def test_gec_predictor_prediction(self): assert set(prediction[0].keys()) == {'logits_labels', 'logits_d_tags', 'class_probabilities_labels', 'class_probabilities_d_tags', 'max_error_probability', 'words', 'labels', 'd_tags', 'corrected_words'} - assert prediction[0]["corrected_words"] == ['I', 'run', 'to', 'the', 'stores', 'every', 'day.'] - assert prediction[1]["corrected_words"] == ['The', 'quick', 'brown', 'foxes', 'jumps', 'over', "Elmo's", + assert prediction[0]["corrected_words"] == ['$START', 'I', 'run', 'to', 'the', 'stores', 'every', 'day.'] + assert prediction[1]["corrected_words"] == ['$START', 'The', 'quick', 'brown', 'foxes', 'jumps', 'over', "Elmo's", 'laziest', 'friend'] From 53a94dddcd03f419f69224eddf21013c312e8a5d Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 15:59:11 -0400 Subject: [PATCH 15/25] Drop START_TOKEN from output_dict["words"]. This interferes with the offsets when creating the target (corrected sentence). Copying get_token_action method almost verbatim instead of using something more clever. --- gector/seq2labels_model.py | 77 +++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/gector/seq2labels_model.py b/gector/seq2labels_model.py index 84f2e89..500c9f1 100644 --- a/gector/seq2labels_model.py +++ b/gector/seq2labels_model.py @@ -12,7 +12,7 @@ from allennlp.training.metrics import CategoricalAccuracy from overrides import overrides from torch.nn.modules.linear import Linear -from utils.helpers import PAD, UNK, get_target_sent_by_edits +from utils.helpers import PAD, UNK, get_target_sent_by_edits, START_TOKEN @Model.register("seq2labels") @@ -164,7 +164,9 @@ def forward(self, # type: ignore output_dict["loss"] = loss_labels + loss_d if metadata is not None: - output_dict["words"] = [x["words"] for x in metadata] + output_dict["words"] = [] + for instance in metadata: + output_dict["words"].append([word for word in instance["words"] if word != START_TOKEN]) return output_dict @overrides @@ -191,47 +193,54 @@ def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor batch_size = len(output_dict['labels']) output_dict['corrected_words'] = [] for i in range(batch_size): + words_in_instance = output_dict['words'][i] + batch_len = len(words_in_instance) probs = output_dict['class_probabilities_labels'][i] max_probs = torch.max(probs, dim=0) probs = max_probs[0].tolist() indices = max_probs[1].tolist() - if max(indices) == 0: # No corrections should be performed - output_dict["corrected_words"] = output_dict["words"] + if max(indices) == 0: # No corrections should be performed + output_dict["corrected_words"].append(output_dict["words"][i]) else: - suggested_token_operations = output_dict['labels'][i] - actions_per_token = self.get_token_actions(indices=indices, probs=probs, - sugg_tokens=suggested_token_operations) - # Filter out cases where no corrections should be applied - actions_per_token = [action for action in actions_per_token if action] + actions_per_token = [] + for j in range(batch_len): + if j == 0: + token = START_TOKEN + else: + token = words_in_instance[j] + if indices[j] == 0: + continue + suggested_token_operation = output_dict['labels'][i][j] + action = self.get_token_action(index=j, prob=probs[j], + sugg_token=suggested_token_operation) + if not action: + continue + actions_per_token.append(action) corrected_sent = get_target_sent_by_edits(output_dict['words'][i], actions_per_token) output_dict['corrected_words'].append(corrected_sent) return output_dict - def get_token_actions(self, indices, probs, sugg_tokens): - """Get list of suggested actions for token.""" - actions = [] - for index, prob, sugg_token in zip(indices, probs, sugg_tokens): - # cases when we don't need to do anything - if prob < self.min_error_probability or sugg_token in [UNK, PAD, '$KEEP']: - actions.append(None) - continue - - if sugg_token.startswith('$REPLACE_') or sugg_token.startswith('$TRANSFORM_') or sugg_token == '$DELETE': - start_pos = index - end_pos = index + 1 - elif sugg_token.startswith("$APPEND_") or sugg_token.startswith("$MERGE_"): - start_pos = index + 1 - end_pos = index + 1 - - if sugg_token == "$DELETE": - sugg_token_clear = "" - elif sugg_token.startswith('$TRANSFORM_') or sugg_token.startswith("$MERGE_"): - sugg_token_clear = sugg_token[:] - else: - sugg_token_clear = sugg_token[sugg_token.index('_') + 1:] - actions.append((start_pos - 1, end_pos - 1, sugg_token_clear, prob)) - return actions - + def get_token_action(self, index, prob, sugg_token): + """Get lost of suggested actions for token.""" + # cases when we don't need to do anything + if prob < self.min_error_probability or sugg_token in [UNK, PAD, '$KEEP']: + return None + + if sugg_token.startswith('$REPLACE_') or sugg_token.startswith('$TRANSFORM_') or sugg_token == '$DELETE': + start_pos = index + end_pos = index + 1 + elif sugg_token.startswith("$APPEND_") or sugg_token.startswith("$MERGE_"): + start_pos = index + 1 + end_pos = index + 1 + + if sugg_token == "$DELETE": + sugg_token_clear = "" + elif sugg_token.startswith('$TRANSFORM_') or sugg_token.startswith("$MERGE_"): + sugg_token_clear = sugg_token[:] + else: + sugg_token_clear = sugg_token[sugg_token.index('_') + 1:] + + return start_pos - 1, end_pos - 1, sugg_token_clear, prob @overrides def get_metrics(self, reset: bool = False) -> Dict[str, float]: metrics_to_return = {metric_name: metric.get_metric(reset) for From 86212b76d4951bd1e5a2f33f6bf74c2a560d7719 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 15:59:56 -0400 Subject: [PATCH 16/25] The outputs now no longer have $START_TOKEN in the corrected sentence. This is identical to the output for gec_model! --- tests/test_gec_predictor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_gec_predictor.py b/tests/test_gec_predictor.py index c2661db..968afe7 100644 --- a/tests/test_gec_predictor.py +++ b/tests/test_gec_predictor.py @@ -48,6 +48,6 @@ def test_gec_predictor_prediction(self): assert set(prediction[0].keys()) == {'logits_labels', 'logits_d_tags', 'class_probabilities_labels', 'class_probabilities_d_tags', 'max_error_probability', 'words', 'labels', 'd_tags', 'corrected_words'} - assert prediction[0]["corrected_words"] == ['$START', 'I', 'run', 'to', 'the', 'stores', 'every', 'day.'] - assert prediction[1]["corrected_words"] == ['$START', 'The', 'quick', 'brown', 'foxes', 'jumps', 'over', "Elmo's", + assert prediction[0]["corrected_words"] == ['I', 'run', 'to', 'the', 'stores', 'every', 'day.'] + assert prediction[1]["corrected_words"] == ['The', 'quick', 'brown', 'foxes', 'jumps', 'over', "Elmo's", 'laziest', 'friend'] From 20a06923f36487124612f7a3359f8ed006cc29bd Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 16:14:59 -0400 Subject: [PATCH 17/25] Handling multiple iterations of correction in predictor now. --- gector/gec_predictor.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/gector/gec_predictor.py b/gector/gec_predictor.py index 2c58d67..1ff283b 100644 --- a/gector/gec_predictor.py +++ b/gector/gec_predictor.py @@ -38,12 +38,31 @@ def predict_batch(self, sentences: List[str]) -> JsonDict: @overrides def predict_instance(self, instance: Instance) -> JsonDict: + """ + This special predict_instance method allows for applying the correction model multiple times. + """ for i in range(self._iterations): - outputs = self._model.forward_on_instance(instance) + output = self._model.forward_on_instance(instance) # integrate predictions back into instance for next iteration - return sanitize(outputs) + tokens = [Token(word) for word in output["corrected_words"]] + instance = self._dataset_reader.text_to_instance(tokens) + return sanitize(output) @overrides + def predict_batch_instance(self, instances: List[Instance]) -> List[JsonDict]: + """ + This special predict_batch_instance method allows for applying the correction model multiple times. + """ + for i in range(self._iterations): + outputs = self._model.forward_on_instances(instances) + corrected_instances = [] + for output in outputs: + tokens = [Token(word) for word in output["corrected_words"]] + instance = self._dataset_reader.text_to_instance(tokens) + corrected_instances.append(instance) + instances = corrected_instances + return sanitize(outputs) + @overrides def _json_to_instance(self, json_dict: JsonDict) -> Instance: """ Expects JSON that looks like ``{"sentence": "..."}``. From 0b88028369d06993df3f83fb078b322b60a32a30 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 16:26:22 -0400 Subject: [PATCH 18/25] Changed location of weights file so it can be used by gec_predictor as well. --- tests/test_gec_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_gec_model.py b/tests/test_gec_model.py index 8260c6b..0056c59 100644 --- a/tests/test_gec_model.py +++ b/tests/test_gec_model.py @@ -25,7 +25,7 @@ def setUp(self): self.model_name = "roberta-base" model_url = "https://grammarly-nlp-data-public.s3.amazonaws.com/gector/roberta_1_gectorv2.th" test_fixtures_dir_path = Path(__file__).parent.parent / "test_fixtures" - model_path = test_fixtures_dir_path / "roberta_1_gectorv2.th" + model_path = test_fixtures_dir_path / "roberta_model" / "weights.th" if not model_path.exists(): response = requests.get(model_url) with model_path.open("wb") as out_fp: From 0b6b5085ebc0dbea28b67df5491139893c521545 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 28 Sep 2022 16:26:42 -0400 Subject: [PATCH 19/25] setup is now downloading weights file if it does not already exist. --- tests/test_gec_predictor.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_gec_predictor.py b/tests/test_gec_predictor.py index 968afe7..eca214e 100644 --- a/tests/test_gec_predictor.py +++ b/tests/test_gec_predictor.py @@ -32,6 +32,16 @@ class TestGecPredictor(ModelTestCase): def setUp(self): super().setUp() test_fixtures_dir_path = Path(__file__).parent.parent / "test_fixtures" + # Download weights for model archive + weights_url = "https://grammarly-nlp-data-public.s3.amazonaws.com/gector/roberta_1_gectorv2.th" + test_fixtures_dir_path = Path(__file__).parent.parent / "test_fixtures" + model_path = test_fixtures_dir_path / "roberta_model" / "weights.th" + if not model_path.exists(): + response = requests.get(weights_url) + with model_path.open("wb") as out_fp: + # Write out data with progress bar + for data in tqdm(response.iter_content()): + out_fp.write(data) model_path = test_fixtures_dir_path / "roberta_model" self.model_path = model_path sentence1 = "I run to a stores every day." From 6f8f23cf3433b73ecc3e4ddc4d0820b9fed19f77 Mon Sep 17 00:00:00 2001 From: Kenneth Steimel Date: Wed, 12 Oct 2022 12:42:27 -0400 Subject: [PATCH 20/25] Apply suggestions from code review Co-authored-by: Sanjna Kashyap <20379363+Frost45@users.noreply.github.com> --- gector/gec_model.py | 2 +- gector/gec_predictor.py | 3 ++- gector/seq2labels_model.py | 3 ++- tests/test_gec_predictor.py | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/gector/gec_model.py b/gector/gec_model.py index 0484611..63f30aa 100644 --- a/gector/gec_model.py +++ b/gector/gec_model.py @@ -40,7 +40,7 @@ def __init__(self, vocab_path=None, model_paths=None, resolve_cycles=False, ): """ - Class used to enable prediction from GECtor model. + Class used to enable prediction from GECToR model. Parameters ---------- diff --git a/gector/gec_predictor.py b/gector/gec_predictor.py index 1ff283b..8633e91 100644 --- a/gector/gec_predictor.py +++ b/gector/gec_predictor.py @@ -17,7 +17,7 @@ @Predictor.register("gec-predictor") class GecPredictor(Predictor): """ - A Predictor for generating predictions from GECtor. + A Predictor for generating predictions from GECToR. Note that currently, this is unable to handle ensemble predictions. """ @@ -62,6 +62,7 @@ def predict_batch_instance(self, instances: List[Instance]) -> List[JsonDict]: corrected_instances.append(instance) instances = corrected_instances return sanitize(outputs) + @overrides def _json_to_instance(self, json_dict: JsonDict) -> Instance: """ diff --git a/gector/seq2labels_model.py b/gector/seq2labels_model.py index 500c9f1..a85298a 100644 --- a/gector/seq2labels_model.py +++ b/gector/seq2labels_model.py @@ -221,7 +221,7 @@ def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor return output_dict def get_token_action(self, index, prob, sugg_token): - """Get lost of suggested actions for token.""" + """Get list of suggested actions for token.""" # cases when we don't need to do anything if prob < self.min_error_probability or sugg_token in [UNK, PAD, '$KEEP']: return None @@ -241,6 +241,7 @@ def get_token_action(self, index, prob, sugg_token): sugg_token_clear = sugg_token[sugg_token.index('_') + 1:] return start_pos - 1, end_pos - 1, sugg_token_clear, prob + @overrides def get_metrics(self, reset: bool = False) -> Dict[str, float]: metrics_to_return = {metric_name: metric.get_metric(reset) for diff --git a/tests/test_gec_predictor.py b/tests/test_gec_predictor.py index eca214e..7854c56 100644 --- a/tests/test_gec_predictor.py +++ b/tests/test_gec_predictor.py @@ -46,7 +46,7 @@ def setUp(self): self.model_path = model_path sentence1 = "I run to a stores every day." sentence2 = "the quick brown foxes jumps over a elmo's laziest friend" - # This micmics how batches of requests are constructed in predict.py's predict_for_file function + # This mimics how batches of requests are constructed in predict.py's predict_for_file function self.input_data = [sentence1, sentence2] def test_gec_predictor_prediction(self): From 7427f93fd3af8db5095662e31493f58789301eb6 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 12 Oct 2022 13:53:24 -0400 Subject: [PATCH 21/25] Removing unused imports, adding docstrings. --- gector/gec_predictor.py | 125 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 115 insertions(+), 10 deletions(-) diff --git a/gector/gec_predictor.py b/gector/gec_predictor.py index 8633e91..68a2c66 100644 --- a/gector/gec_predictor.py +++ b/gector/gec_predictor.py @@ -7,8 +7,6 @@ from overrides import overrides from allennlp.common.util import JsonDict from allennlp.data import DatasetReader, Instance, Token -from allennlp.data.fields import TextField, SequenceLabelField -from allennlp.data.tokenizers.word_splitter import SpacyWordSplitter from allennlp.data.tokenizers.word_splitter import JustSpacesWordSplitter from allennlp.models import Model from utils.helpers import START_TOKEN @@ -21,25 +19,99 @@ class GecPredictor(Predictor): Note that currently, this is unable to handle ensemble predictions. """ - def __init__(self, model: Model, + + def __init__(self, + model: Model, dataset_reader: DatasetReader, - language: str = 'en_core_web_sm', iterations: int = 3) -> None: + """ + Parameters + --------- + model: Model + An instantiated `Seq2Labels` model for performing grammatical error correction. + dataset_reader: DatasetReader + An instantiated dataset reader, typically `Seq2LabelsDatasetReader`. + iterations: int + This represents the number of times grammatical error correction is applied to the input. + """ super().__init__(model, dataset_reader) - #self._tokenizer = SpacyWordSplitter(language=language, pos_tags=True) self._tokenizer = JustSpacesWordSplitter() self._iterations = iterations def predict(self, sentence: str) -> JsonDict: + """ + Generate error correction predictions for a single input string. + + Parameters + ---------- + sentence: str + The input text to perform error correction on. + + Returns + ------- + JsonDict + A dictionary containing the following keys: + - logits_labels + - logits_d_tags + - class_probabilities_labels + - class_probabilities_d_tags + - max_error_probability + - words + - labels + - d_tags + - corrected_words + For an explanation of each of these see `Seq2Labels.decode()`. + """ return self.predict_json({"sentence": sentence}) - def predict_batch(self, sentences: List[str]) -> JsonDict: + def predict_batch(self, sentences: List[str]) -> List[JsonDict]: + """ + Generate predictions for a sequence of input strings. + + Parameters + ---------- + sentences: List[str] + A list of strings to correct. + + Returns + ------- + List[JsonDict] + A list of dictionaries, each containing the following keys: + - logits_labels + - logits_d_tags + - class_probabilities_labels + - class_probabilities_d_tags + - max_error_probability + - words + - labels + - d_tags + - corrected_words + For an explanation of each of these see `Seq2Labels.decode()`. + """ return self.predict_batch_json([{"sentence": sentence} for sentence in sentences]) @overrides def predict_instance(self, instance: Instance) -> JsonDict: """ This special predict_instance method allows for applying the correction model multiple times. + + Parameters + --------- + + Returns + ------- + JsonDict + A dictionary containing the following keys: + - logits_labels + - logits_d_tags + - class_probabilities_labels + - class_probabilities_d_tags + - max_error_probability + - words + - labels + - d_tags + - corrected_words + For an explanation of each of these see `Seq2Labels.decode()`. """ for i in range(self._iterations): output = self._model.forward_on_instance(instance) @@ -52,6 +124,24 @@ def predict_instance(self, instance: Instance) -> JsonDict: def predict_batch_instance(self, instances: List[Instance]) -> List[JsonDict]: """ This special predict_batch_instance method allows for applying the correction model multiple times. + + Parameters + ---------- + + Returns + ------- + List[JsonDict] + A list of dictionaries, each containing the following keys: + - logits_labels + - logits_d_tags + - class_probabilities_labels + - class_probabilities_d_tags + - max_error_probability + - words + - labels + - d_tags + - corrected_words + For an explanation of each of these see `Seq2Labels.decode()`. """ for i in range(self._iterations): outputs = self._model.forward_on_instances(instances) @@ -66,8 +156,24 @@ def predict_batch_instance(self, instances: List[Instance]) -> List[JsonDict]: @overrides def _json_to_instance(self, json_dict: JsonDict) -> Instance: """ - Expects JSON that looks like ``{"sentence": "..."}``. - Runs the underlying model, and adds the ``"words"`` to the output. + Convert a JsonDict into an Instance. + + This is used internally by `self.predict_json()`. + + Parameters + ---------- + json_dict: JsonDict + Expects a dict with a single key "sentence" with a value representing the string to correct. + i.e. ``{"sentence": "..."}``. + + Returns + ------ + Instance + An instance with the following fields: + - tokens + - metadata + - labels + - d_tags """ sentence = json_dict["sentence"] tokens = self._tokenizer.split_words(sentence) @@ -75,7 +181,6 @@ def _json_to_instance(self, json_dict: JsonDict) -> Instance: tokens = [Token(START_TOKEN)] + tokens return self._dataset_reader.text_to_instance(tokens) - @overrides def predictions_to_labeled_instances(self, instance: Instance, @@ -83,4 +188,4 @@ def predictions_to_labeled_instances(self, """ This method creates an instance out of the predictions generated by the model. """ - NotImplemented \ No newline at end of file + NotImplemented From 73aef1bf31ea8da9b053d79e67b7dbc0754377ec Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 12 Oct 2022 13:54:05 -0400 Subject: [PATCH 22/25] Removing unused predictions to labeled_instances method. --- gector/gec_predictor.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/gector/gec_predictor.py b/gector/gec_predictor.py index 68a2c66..fa5c5a1 100644 --- a/gector/gec_predictor.py +++ b/gector/gec_predictor.py @@ -180,12 +180,3 @@ def _json_to_instance(self, json_dict: JsonDict) -> Instance: # Add start token to front tokens = [Token(START_TOKEN)] + tokens return self._dataset_reader.text_to_instance(tokens) - - @overrides - def predictions_to_labeled_instances(self, - instance: Instance, - outputs: Dict[str, numpy.ndarray]) -> List[Instance]: - """ - This method creates an instance out of the predictions generated by the model. - """ - NotImplemented From 944993cb058fe3f7144c2edbc2056e6b600235d4 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 12 Oct 2022 15:18:03 -0400 Subject: [PATCH 23/25] Updated docstring for decode() --- gector/seq2labels_model.py | 40 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/gector/seq2labels_model.py b/gector/seq2labels_model.py index a85298a..b5feb9f 100644 --- a/gector/seq2labels_model.py +++ b/gector/seq2labels_model.py @@ -174,6 +174,46 @@ def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor """ Does a simple position-wise argmax over each token, converts indices to string labels, and adds a ``"tags"`` key to the dictionary with the result. + + Parameters + ---------- + output_dict: Dict[str, torch.Tensor] + This is expected to have the following keys: + - logits_labels + - logits_d_tags + - class_probabilities_labels + - class_probabilities_d_tags + - max_error_probability + - words + + Returns + ------ + Dict + A dictionary containing the following keys: + - logits_labels + Logits for labels indicating the types of corrections + to perform. + - logits_d_tags + Logits for labels indicating the presence or absence + of grammatical errors. + - class_probabilities_labels + Class probabilities for labels indicating the types + of corrections to perform. + - class_probabilities_d_tags + Class probabilities for labels indicating the presence + or absence of grammatical errors. + - max_error_probability + A threshold probability that has to be exceeded for an + error to be corrected. + - words + The original tokens used to create the instance. + - labels + Labels indicating the types of corrections to perform. + - d_tags + Labels indicating the presence or absence of grammatical errors. + - corrected_words + `words` after applying the correction operations + specified in `labels` """ for label_namespace in self.label_namespaces: all_predictions = output_dict[f'class_probabilities_{label_namespace}'] From 709ba2817c2fc25ad489eb6907ec9ba6903bdff6 Mon Sep 17 00:00:00 2001 From: ksteimel Date: Wed, 12 Oct 2022 15:18:18 -0400 Subject: [PATCH 24/25] Removed unused imports. --- tests/test_gec_predictor.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/tests/test_gec_predictor.py b/tests/test_gec_predictor.py index 7854c56..f3a751d 100644 --- a/tests/test_gec_predictor.py +++ b/tests/test_gec_predictor.py @@ -5,17 +5,6 @@ from allennlp.common.testing import ModelTestCase from allennlp.predictors import Predictor -from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder -from allennlp.data.vocabulary import Vocabulary -from allennlp.data import Token -from allennlp.data.instance import Instance -from allennlp.data.fields import TextField -from allennlp.data.dataset import Batch - -from gector.gec_predictor import GecPredictor -from gector.gec_model import GecBERTModel -from gector.bert_token_embedder import PretrainedBertEmbedder -from gector.tokenizer_indexer import PretrainedBertIndexer # These imports are required so that instantiating the predictor can be done automatically from gector.datareader import Seq2LabelsDatasetReader @@ -24,14 +13,11 @@ from gector.tokenizer_indexer import PretrainedBertIndexer - - class TestGecPredictor(ModelTestCase): """Test class for GecModel""" def setUp(self): super().setUp() - test_fixtures_dir_path = Path(__file__).parent.parent / "test_fixtures" # Download weights for model archive weights_url = "https://grammarly-nlp-data-public.s3.amazonaws.com/gector/roberta_1_gectorv2.th" test_fixtures_dir_path = Path(__file__).parent.parent / "test_fixtures" From b04376c5d972e6d9eca8add8cb6f9789b8bf9a7c Mon Sep 17 00:00:00 2001 From: Kenneth Steimel Date: Fri, 21 Oct 2022 20:40:05 -0400 Subject: [PATCH 25/25] Adding back import of gec_predictor that shouldn't have been removed --- tests/test_gec_predictor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_gec_predictor.py b/tests/test_gec_predictor.py index f3a751d..537a996 100644 --- a/tests/test_gec_predictor.py +++ b/tests/test_gec_predictor.py @@ -11,6 +11,7 @@ from gector.seq2labels_model import Seq2Labels from gector.bert_token_embedder import PretrainedBertEmbedder from gector.tokenizer_indexer import PretrainedBertIndexer +from gector.gec_predictor import GecPredictor class TestGecPredictor(ModelTestCase):