diff --git a/.dev_scripts/gen_benchmark_script.py b/.dev_scripts/gen_benchmark_script.py new file mode 100644 index 0000000000..3ae1128029 --- /dev/null +++ b/.dev_scripts/gen_benchmark_script.py @@ -0,0 +1,193 @@ +import argparse +import re +from os import path as osp + + +def parse_args(): + parser = argparse.ArgumentParser( + description='Generate benchmark training/testing scripts') + parser.add_argument( + '--input_file', + required=False, + type=str, + help='Input file containing the paths ' + 'of configs to be trained/tested.') + parser.add_argument( + '--output_file', + required=True, + type=str, + help='Output file containing the ' + 'commands to train/test selected models.') + parser.add_argument( + '--gpus_per_node', + type=int, + default=8, + help='GPUs per node config for slurm, ' + 'should be set according to your slurm environment') + parser.add_argument( + '--cpus_per_task', + type=int, + default=5, + help='CPUs per task config for slurm, ' + 'should be set according to your slurm environment') + parser.add_argument( + '--gpus', + type=int, + default=8, + help='Totally used num of GPUs config for slurm (in testing), ' + 'should be set according to your slurm environment') + parser.add_argument( + '--mode', type=str, default='train', help='Train or test') + parser.add_argument( + '--long_work_dir', + action='store_true', + help='Whether use full relative path of config as work dir') + parser.add_argument( + '--max_keep_ckpts', + type=int, + default=1, + help='The max number of checkpoints saved in training') + parser.add_argument( + '--full_log', + action='store_true', + help='Whether save full log in a file') + + args = parser.parse_args() + return args + + +args = parse_args() +assert args.mode in ['train', 'test'], 'Currently we only support ' \ + 'automatically generating training or testing scripts.' + +config_paths = [] + +if args.input_file is not None: + with open(args.input_file, 'r') as fi: + config_paths = fi.read().strip().split('\n') +else: + while True: + print('Please type a config path and ' + 'press enter (press enter directly to exit):') + config_path = input() + if config_path != '': + config_paths.append(config_path) + else: + break + +script = '''PARTITION=$1 +CHECKPOINT_DIR=$2 + +''' + +if args.mode == 'train': + for i, config_path in enumerate(config_paths): + root_dir = osp.dirname(osp.dirname(osp.abspath(__file__))) + if not osp.exists(osp.join(root_dir, config_path)): + print(f'Invalid config path (does not exist):\n{config_path}') + continue + + config_name = config_path.split('/')[-1][:-3] + match_obj = re.match(r'^.*_[0-9]+x([0-9]+)_.*$', config_name) + if match_obj is None: + print(f'Invalid config path (no GPU num in ' + f'config name):\n{config_path}') + continue + + gpu_num = int(match_obj.group(1)) + work_dir_name = config_path if args.long_work_dir else config_name + + script += f"echo '{config_path}' &\n" + if args.full_log: + script += f'mkdir -p $CHECKPOINT_DIR/{work_dir_name}\n' + + # training commands + script += f'GPUS={gpu_num} GPUS_PER_NODE={args.gpus_per_node} ' \ + f'CPUS_PER_TASK={args.cpus_per_task} ' \ + f'./tools/slurm_train.sh $PARTITION {config_name} ' \ + f'{config_path} \\\n' + script += f'$CHECKPOINT_DIR/{work_dir_name} --cfg-options ' \ + f'checkpoint_config.max_keep_ckpts=' \ + f'{args.max_keep_ckpts} \\\n' \ + + # if output full log, redirect stdout and stderr to + # another log file in work dir + if args.full_log: + script += f'2>&1|tee $CHECKPOINT_DIR/{work_dir_name}' \ + f'/FULL_LOG.txt &\n' + else: + script += '>/dev/null &\n' + + if i != len(config_paths) - 1: + script += '\n' + + print(f'Successfully generated script for {config_name}') + + with open(args.output_file, 'w') as fo: + fo.write(script) + +elif args.mode == 'test': + for i, config_path in enumerate(config_paths): + root_dir = osp.dirname(osp.dirname(osp.abspath(__file__))) + if not osp.exists(osp.join(root_dir, config_path)): + print(f'Invalid config path (does not exist):\n{config_path}') + continue + + config_name = config_path.split('/')[-1][:-3] + + tasks = { + 'scannet_seg', 'scannet', 's3dis_seg', 'sunrgbd', 'kitti', 'nus', + 'lyft', 'waymo' + } + eval_option = None + for task in tasks: + if task in config_name: + eval_option = task + break + if eval_option is None: + print(f'Invalid config path (invalid task):\n{config_path}') + continue + + work_dir_name = config_path if args.long_work_dir else config_name + + script += f"echo '{config_path}' &\n" + if args.full_log: + script += f'mkdir -p $CHECKPOINT_DIR/{work_dir_name}\n' + + # training commands + script += f'GPUS={args.gpus} GPUS_PER_NODE={args.gpus_per_node} ' \ + f'CPUS_PER_TASK={args.cpus_per_task} ' \ + f'./tools/slurm_test.sh $PARTITION {config_name} ' \ + f'{config_path} \\\n' + script += f'$CHECKPOINT_DIR/{work_dir_name}/latest.pth ' \ + + if eval_option in ['scannet_seg', 's3dis_seg']: + script += '--eval mIoU \\\n' + elif eval_option in ['scannet', 'sunrgbd', 'kitti', 'nus']: + script += '--eval map \\\n' + elif eval_option in ['lyft']: + script += f'--format-only --eval-options jsonfile_prefix=' \ + f'$CHECKPOINT_DIR/{work_dir_name}/results_challenge ' \ + f'csv_savepath=$CHECKPOINT_DIR/{work_dir_name}/' \ + f'results_challenge.csv \\\n' + elif eval_option in ['waymo']: + script += f'--eval waymo --eval-options pklfile_prefix=' \ + f'$CHECKPOINT_DIR/{work_dir_name}/kitti_results ' \ + f'submission_prefix=$CHECKPOINT_DIR/{work_dir_name}/' \ + f'kitti_results \\\n' + + # if output full log, redirect stdout and stderr to + # another log file in work dir + if args.full_log: + script += f'2>&1|tee $CHECKPOINT_DIR/{work_dir_name}' \ + f'/FULL_LOG.txt &\n' + else: + script += '>/dev/null &\n' + + if i != len(config_paths) - 1: + script += '\n' + + print(f'Successfully generated script for {config_name}') + + with open(args.output_file, 'w') as fo: + fo.write(script) diff --git a/.dev_scripts/test_benchmark.sh b/.dev_scripts/test_benchmark.sh new file mode 100644 index 0000000000..f020d7ff9e --- /dev/null +++ b/.dev_scripts/test_benchmark.sh @@ -0,0 +1,128 @@ +PARTITION=$1 +CHECKPOINT_DIR=$2 + +echo 'configs/3dssd/3dssd_4x4_kitti-3d-car.py' & +mkdir -p $CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION 3dssd_4x4_kitti-3d-car configs/3dssd/3dssd_4x4_kitti-3d-car.py \ +$CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py/FULL_LOG.txt & + +echo 'configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py' & +mkdir -p $CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py \ +$CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py/FULL_LOG.txt & + +echo 'configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py \ +$CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py/FULL_LOG.txt & + +echo 'configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py \ +$CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py/FULL_LOG.txt & + +echo 'configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py' & +mkdir -p $CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION groupfree3d_8x4_scannet-3d-18class-L6-O256 configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py \ +$CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py/FULL_LOG.txt & + +echo 'configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py' & +mkdir -p $CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION h3dnet_3x8_scannet-3d-18class configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py \ +$CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py/FULL_LOG.txt & + +echo 'configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py \ +$CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py/FULL_LOG.txt & + +echo 'configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION imvotenet_stage2_16x8_sunrgbd-3d-10class configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py \ +$CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py/FULL_LOG.txt & + +echo 'configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION imvoxelnet_4x8_kitti-3d-car configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py \ +$CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py/FULL_LOG.txt & + +echo 'configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py \ +$CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py/latest.pth --eval mIoU \ +2>&1|tee $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py/FULL_LOG.txt & + +echo 'configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py \ +$CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py/FULL_LOG.txt & + +echo 'configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py \ +$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/latest.pth --format-only --eval-options jsonfile_prefix=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/results_challenge csv_savepath=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/results_challenge.csv \ +2>&1|tee $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/FULL_LOG.txt & + +echo 'configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py \ +$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/latest.pth --eval waymo --eval-options pklfile_prefix=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/kitti_results submission_prefix=$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/kitti_results \ +2>&1|tee $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/FULL_LOG.txt & + +echo 'configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py \ +$CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py/FULL_LOG.txt & + +echo 'configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_second_secfpn_6x8_80e_kitti-3d-3class configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py \ +$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/latest.pth --format-only --eval-options jsonfile_prefix=$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/results_challenge csv_savepath=$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/results_challenge.csv \ +2>&1|tee $CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/FULL_LOG.txt & + +echo 'configs/votenet/votenet_8x8_scannet-3d-18class.py' & +mkdir -p $CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_test.sh $PARTITION votenet_8x8_scannet-3d-18class configs/votenet/votenet_8x8_scannet-3d-18class.py \ +$CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py/latest.pth --eval map \ +2>&1|tee $CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py/FULL_LOG.txt & diff --git a/.dev_scripts/train_benchmark.sh b/.dev_scripts/train_benchmark.sh new file mode 100644 index 0000000000..7655ab174e --- /dev/null +++ b/.dev_scripts/train_benchmark.sh @@ -0,0 +1,128 @@ +PARTITION=$1 +CHECKPOINT_DIR=$2 + +echo 'configs/3dssd/3dssd_4x4_kitti-3d-car.py' & +mkdir -p $CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py +GPUS=4 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION 3dssd_4x4_kitti-3d-car configs/3dssd/3dssd_4x4_kitti-3d-car.py \ +$CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/3dssd/3dssd_4x4_kitti-3d-car.py/FULL_LOG.txt & + +echo 'configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py' & +mkdir -p $CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py \ +$CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/centerpoint/centerpoint_02pillar_second_secfpn_dcn_circlenms_4x8_cyclic_20e_nus.py/FULL_LOG.txt & + +echo 'configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/dynamic_voxelization/dv_second_secfpn_2x8_cosine_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py \ +$CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/fcos3d/fcos3d_r101_caffe_fpn_gn-head_dcn_2x8_1x_nus-mono3d.py/FULL_LOG.txt & + +echo 'configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/fp16/hv_second_secfpn_fp16_6x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py \ +$CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/free_anchor/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_free-anchor_strong-aug_4x8_3x_nus-3d.py/FULL_LOG.txt & + +echo 'configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py' & +mkdir -p $CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py +GPUS=4 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION groupfree3d_8x4_scannet-3d-18class-L6-O256 configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py \ +$CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/groupfree3d/groupfree3d_8x4_scannet-3d-18class-L6-O256.py/FULL_LOG.txt & + +echo 'configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py' & +mkdir -p $CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION h3dnet_3x8_scannet-3d-18class configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py \ +$CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/h3dnet/h3dnet_3x8_scannet-3d-18class.py/FULL_LOG.txt & + +echo 'configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py +GPUS=4 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py \ +$CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/imvotenet/imvotenet_faster_rcnn_r50_fpn_2x4_sunrgbd-3d-10class.py/FULL_LOG.txt & + +echo 'configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION imvotenet_stage2_16x8_sunrgbd-3d-10class configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py \ +$CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/imvotenet/imvotenet_stage2_16x8_sunrgbd-3d-10class.py/FULL_LOG.txt & + +echo 'configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py' & +mkdir -p $CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION imvoxelnet_4x8_kitti-3d-car configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py \ +$CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py/FULL_LOG.txt & + +echo 'configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/mvxnet/dv_mvx-fpn_second_secfpn_adamw_2x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/parta2/hv_PartA2_secfpn_2x8_cyclic_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py +GPUS=2 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py \ +$CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_80e_s3dis_seg-3d-13class.py/FULL_LOG.txt & + +echo 'configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py +GPUS=2 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py \ +$CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/pointnet2/pointnet2_msg_16x2_cosine_250e_scannet_seg-3d-20class.py/FULL_LOG.txt & + +echo 'configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py \ +$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_fpn_sbn-all_2x8_2x_lyft-3d.py/FULL_LOG.txt & + +echo 'configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py +GPUS=16 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py \ +$CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/pointpillars/hv_pointpillars_secfpn_sbn_2x16_2x_waymoD5-3d-3class.py/FULL_LOG.txt & + +echo 'configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py \ +$CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/regnet/hv_pointpillars_regnet-1.6gf_fpn_sbn-all_4x8_2x_nus-3d.py/FULL_LOG.txt & + +echo 'configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py' & +mkdir -p $CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_second_secfpn_6x8_80e_kitti-3d-3class configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py \ +$CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/second/hv_second_secfpn_6x8_80e_kitti-3d-3class.py/FULL_LOG.txt & + +echo 'configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py' & +mkdir -p $CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py +GPUS=16 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py \ +$CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/ssn/hv_ssn_secfpn_sbn-all_2x16_2x_lyft-3d.py/FULL_LOG.txt & + +echo 'configs/votenet/votenet_8x8_scannet-3d-18class.py' & +mkdir -p $CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py +GPUS=8 GPUS_PER_NODE=8 CPUS_PER_TASK=5 ./tools/slurm_train.sh $PARTITION votenet_8x8_scannet-3d-18class configs/votenet/votenet_8x8_scannet-3d-18class.py \ +$CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py --cfg-options checkpoint_config.max_keep_ckpts=1 \ +2>&1|tee $CHECKPOINT_DIR/configs/votenet/votenet_8x8_scannet-3d-18class.py/FULL_LOG.txt & diff --git a/configs/imvoxelnet/imvoxelnet_kitti-3d-car.py b/configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py similarity index 100% rename from configs/imvoxelnet/imvoxelnet_kitti-3d-car.py rename to configs/imvoxelnet/imvoxelnet_4x8_kitti-3d-car.py