-
Notifications
You must be signed in to change notification settings - Fork 2.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into t5_lm_adaptation
- Loading branch information
Showing
39 changed files
with
2,906 additions
and
472 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
110 changes: 110 additions & 0 deletions
110
examples/nlp/language_modeling/conf/megatron_ptune_gpt.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| name: megatron_ptune_gpt | ||
|
|
||
| trainer: | ||
| gpus: 2 | ||
| num_nodes: 1 | ||
| precision: 16 | ||
| logger: False # logger provided by exp_manager | ||
| checkpoint_callback: False | ||
| replace_sampler_ddp: False | ||
| max_epochs: 3 | ||
| max_steps: null # consumed_samples = global_step * micro_batch_size * data_parallel_size * accumulate_grad_batches | ||
| log_every_n_steps: 10 | ||
| val_check_interval: 300 | ||
| accumulate_grad_batches: 1 | ||
| gradient_clip_val: 1.0 | ||
| resume_from_checkpoint: null # The path to a checkpoint file to continue the training, restores the whole state including the epoch, step, LR schedulers, apex, etc. | ||
|
|
||
|
|
||
| exp_manager: | ||
| explicit_log_dir: null | ||
| exp_dir: null | ||
| name: megatron_ptune_gpt | ||
| create_wandb_logger: False | ||
| wandb_logger_kwargs: | ||
| project: null | ||
| name: null | ||
| resume_if_exists: True | ||
| resume_ignore_no_checkpoint: True | ||
| create_checkpoint_callback: True | ||
| checkpoint_callback_params: | ||
| monitor: val_acc | ||
| save_top_k: 2 | ||
| mode: max | ||
| always_save_nemo: False # TODO: add support | ||
| filename: 'megatron_gpt--{val_acc:.3f}-{step}' | ||
| model_parallel_size: ${model.tensor_model_parallel_size} | ||
| save_best_model: True | ||
|
|
||
| model: | ||
| tensor_model_parallel_size: 1 | ||
| seed: 1234 | ||
| nemo_path: null # filename to save the model and associated artifacts to .nemo file | ||
| use_lm_finetune: False # whether fine tune the language model | ||
| pseudo_token: '[PROMPT]' # pseudo prompt tokens | ||
| max_decode_length: null # max number of tokens length to decode, if set null, it will be inferred from data | ||
|
|
||
| language_model: | ||
| nemo_file: null | ||
|
|
||
| prompt_encoder: | ||
| template: [3, 3, 3] | ||
| dropout: 0.0 | ||
| num_layers: 2 | ||
| task_dependent: true # whether the prompt encoder conditions on the task name or not, useful for multiple tasks | ||
|
|
||
| task_processors: | ||
| - taskname: qa-task | ||
| template: "{v0} Context: {context}{v1} Question: {question}?{v2} Answer:" | ||
| limit_length_field: 'context' | ||
| - taskname: sentiment-task | ||
| template: "{v0}{v1} Sentence: {sentence}{v2} Sentiment:" | ||
| limit_length_field: 'sentence' | ||
|
|
||
| data: | ||
| train_ds: | ||
| file_path: ??? | ||
| batch_size: 32 | ||
| shuffle: True | ||
| num_workers: 8 | ||
| pin_memory: True | ||
|
|
||
| validation_ds: | ||
| file_path: ??? | ||
| batch_size: 32 | ||
| shuffle: False | ||
| num_workers: 8 | ||
| pin_memory: True | ||
|
|
||
| test_ds: | ||
| file_path: ??? | ||
| batch_size: 32 | ||
| shuffle: False | ||
| num_workers: 8 | ||
| pin_memory: True | ||
|
|
||
| optim: | ||
| name: adam | ||
| lr: 1e-5 | ||
| # optimizer arguments | ||
| betas: [0.9, 0.999] | ||
| weight_decay: 0.0005 | ||
| # scheduler setup | ||
| sched: | ||
| name: WarmupAnnealing | ||
| # Scheduler params | ||
| warmup_steps: null | ||
| warmup_ratio: 0.1 | ||
| last_epoch: -1 | ||
| # pytorch lightning args | ||
| monitor: val_loss | ||
| reduce_on_plateau: false | ||
|
|
||
| # List of some sample queries for inference after training is done | ||
| infer_samples: | ||
| - {"prompt_tag": "boolq-full-text", "sentence": "Slave states and free states -- In the 1770s, blacks throughout New England began sending petitions to northern legislatures demanding freedom. Five of the Northern self-declared states adopted policies to at least gradually abolish slavery: Pennsylvania (1780), New Hampshire and Massachusetts (1783), Connecticut and Rhode Island (1784). Vermont had abolished slavery in 1777, while it was still independent, and when it joined the United States as the 14th state in 1791, it was the first state to join untainted by slavery. These state jurisdictions thus enacted the first abolition laws in the Americas. By 1804 (including, New York (1799), New Jersey (1804)), all of the northern states had abolished slavery or set measures in place to gradually abolish it.", "question": "were the new england states free states in 1854"} | ||
| - {"prompt_tag": "boolq-full-text", "sentence": "Titius\u2013Bode law -- Recent astronomical research suggests that planetary systems around some other stars may follow Titius--Bode-like laws. Bovaird and Lineweaver applied a generalized Titius--Bode relation to 68 exoplanet systems that contain four or more planets. They showed that 96% of these exoplanet systems adhere to a generalized Titius--Bode relation to a similar or greater extent than the Solar System does. The locations of potentially undetected exoplanets are predicted in each system.", "question": "do exoplanetary systems follow the titus bode rule"} | ||
| - {"prompt_tag": "boolq-full-text", "sentence": "Bipolar disorder -- Bipolar disorder, previously known as manic depression, is a mental disorder that causes periods of depression and periods of abnormally elevated mood. The elevated mood is significant and is known as mania or hypomania, depending on its severity, or whether symptoms of psychosis are present. During mania, an individual behaves or feels abnormally energetic, happy, or irritable. Individuals often make poorly thought out decisions with little regard to the consequences. The need for sleep is usually reduced during manic phases. During periods of depression, there may be crying, a negative outlook on life, and poor eye contact with others. The risk of suicide among those with the illness is high at greater than 6 percent over 20 years, while self-harm occurs in 30--40 percent. Other mental health issues such as anxiety disorders and substance use disorder are commonly associated.", "question": "is manic depression the same as bi polar"} | ||
| - {"prompt_tag": "boolq-full-text", "sentence": "SS Politician -- SS Politician was an 8000-ton cargo ship owned by T & J Harrison of Liverpool. It left Liverpool on 3 February 1941, bound for Kingston, Jamaica and New Orleans with a cargo including 28,000 cases of malt whisky. The ship sank off the north coast of Eriskay in the Outer Hebrides, off the west coast of Scotland, and much of the wreck's cargo was salvaged by the island's inhabitants. The story of the wreck and looting was the basis for the book and film Whisky Galore!.", "question": "was whiskey galore based on a true story"} | ||
| - {"prompt_tag": "boolq-full-text", "sentence": "Plants in space -- Plant research continued on the International Space Station. Biomass Production System was used on the ISS Expedition 4. The Vegetable Production System (Veggie) system was later used aboard ISS. Plants tested in Veggie before going into space included lettuce, Swiss chard, radishes, Chinese cabbage and peas. Red Romaine lettuce was grown in space on Expedition 40 which were harvested when mature, frozen and tested back on Earth. Expedition 44 members became the first American astronauts to eat plants grown in space on 10 August 2015, when their crop of Red Romaine was harvested. Since 2003 Russian cosmonauts have been eating half of their crop while the other half goes towards further research. In 2012, a sunflower bloomed aboard the ISS under the care of NASA astronaut Donald Pettit. In January 2016, US astronauts announced that a zinnia had blossomed aboard the ISS.", "question": "are there plants on the international space station"} | ||
| - {"prompt_tag": "boolq-full-text", "sentence": "Goal (ice hockey) -- In ice hockey, a goal is scored when the puck entirely crosses the goal line between the two goal posts and below the goal crossbar. A goal awards one point to the team attacking the goal scored upon, regardless of which team the player who actually deflected the puck into the goal belongs to (see also own goal). Typically, a player on the team attempting to score shoots the puck with his/her stick towards the goal net opening, and a player on the opposing team called a goaltender tries to block the shot to prevent a goal from being scored against his/her team.", "question": "does the hockey puck have to cross the line to be a goal"} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.