Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev checkpt load #63

Merged
merged 8 commits into from
Mar 18, 2021
Merged

Dev checkpt load #63

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3e7a97f
add checkpoint loading
ASpeiser Feb 19, 2021
6ab9bb7
add to reference, fix logdir
ASpeiser Feb 19, 2021
e45e9a7
Deprecate model_init in favour of checkpoint init and rename
Haydnspass Mar 1, 2021
c46f80a
Only open issue on failure && push && master
Mar 12, 2021
9524f08
merge
ASpeiser Mar 16, 2021
56758c6
fix vars
ASpeiser Mar 17, 2021
ea2a391
update file logic
ASpeiser Mar 17, 2021
2262d8d
Merge branch 'master' into dev_checkpt_load
Haydnspass Mar 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 25 additions & 13 deletions decode/neuralfitter/train/live_engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,23 +81,28 @@ def live_engine_setup(param_file: str, cuda_ix: int = None, debug: bool = False,

"""Experiment ID"""
if not debug:
experiment_id = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + '_' + socket.gethostname()

if log_comment:
experiment_id = experiment_id + '_' + log_comment

if param.InOut.checkpoint_init is None:
experiment_id = datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + '_' + socket.gethostname()
Haydnspass marked this conversation as resolved.
Show resolved Hide resolved
from_ckpt = False
if log_comment:
experiment_id = experiment_id + '_' + log_comment
else:
from_ckpt = True
experiment_id = param.InOut.checkpoint.split('/')[-1]
else:
experiment_id = 'debug'
from_ckpt = False

"""Set up unique folder for experiment"""
experiment_path = Path(param.InOut.experiment_out) / Path(experiment_id)
if not experiment_path.parent.exists():
experiment_path.parent.mkdir()

if debug:
experiment_path.mkdir(exist_ok=True)
else:
experiment_path.mkdir(exist_ok=False)
if not from_ckpt:
if debug:
experiment_path.mkdir(exist_ok=True)
else:
experiment_path.mkdir(exist_ok=False)

model_out = experiment_path / Path('model.pt')
ckpt_path = model_out.parent / Path('ckpt.pt')
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ckpt_path should not be hard coded to ckpt.pt relative to the model.
Ckpt_path should be inferred from param.InOut.checkpoint_init (will probably require a bit of logic here).

Expand Down Expand Up @@ -154,8 +159,16 @@ def live_engine_setup(param_file: str, cuda_ix: int = None, debug: bool = False,

dl_train, dl_test = setup_dataloader(param, ds_train, ds_test)

# useful if we restart a training
first_epoch = param.HyperParameter.epoch_0 if param.HyperParameter.epoch_0 is not None else 0
if from_ckpt:
ckpt = decode.utils.checkpoint.CheckPoint.load(ckpt_path)
model.load_state_dict(ckpt.model_state)
optimizer.load_state_dict(ckpt.optimizer_state)
lr_scheduler.load_state_dict(ckpt.lr_sched_state)
first_epoch = ckpt.step + 1
model = model.train()
print(f'Resuming training from checkpoint ' + experiment_id)
else:
first_epoch = 0

for i in range(first_epoch, param.HyperParameter.epochs):
logger.add_scalar('learning/learning_rate', optimizer.param_groups[0]['lr'], i)
Expand Down Expand Up @@ -231,8 +244,7 @@ def setup_trainer(simulator_train, simulator_test, logger, model_out, ckpt_path,
model = model.parse(param)

model_ls = decode.utils.model_io.LoadSaveModel(model,
output_file=model_out,
input_file=param.InOut.model_init)
output_file=model_out)

model = model_ls.load_init()
model = model.to(torch.device(device))
Expand Down
2 changes: 1 addition & 1 deletion decode/utils/reference_files/reference.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ HyperParameter:
InOut:
calibration_file: # spline calib
experiment_out: # main output dir
model_init:
checkpoint_init: # initialise from checkpoint (i.e. resume training)
PostProcessing: NMS # (blank) for no post-processing or LookUp, Consistency
PostProcessingParam:
raw_th: 0.5
Expand Down