deit3-h14-pt-126-in21k-240ep-ft-224-in1k-50ep.sh

python3 src/main.py \
    --output-dir $GCS_MODEL_DIR \
    --pretrained-ckpt $GCS_MODEL_DIR/deit3-h14-pt-126-in21k-240ep-last.msgpack \
    --train-dataset-shards "$GCS_DATASET_DIR/imagenet-1k-wds/imagenet1k-train-{0000..1023}.tar" \
    --valid-dataset-shards "$GCS_DATASET_DIR/imagenet-1k-wds/imagenet1k-validation-{00..63}.tar" \
    --train-batch-size 2048 \
    --valid-batch-size 512 \
    --train-loader-workers 40 \
    --valid-loader-workers 10 \
    --random-crop src \
    --color-jitter 0.3 \
    --auto-augment "3a" \
    --random-erasing 0.0 \
    --augment-repeats 1 \
    --test-crop-ratio 1.0 \
    --mixup 0.0 \
    --cutmix 1.0 \
    --criterion ce \
    --label-smoothing 0.1 \
    --layers 32 \
    --dim 1280 \
    --heads 16 \
    --labels 1000 \
    --layerscale \
    --patch-size 14 \
    --image-size 224 \
    --posemb learnable \
    --pooling cls \
    --dropout 0.0 \
    --droppath 0.45 \
    --init-seed 0 \
    --mixup-seed 0 \
    --dropout-seed 0 \
    --shuffle-seed 0 \
    --label-mapping $(dirname $0)/in21k_to_in1k.json \
    --optimizer lamb \
    --learning-rate 0.0003 \
    --weight-decay 0.02 \
    --adam-b1 0.9 \
    --adam-b2 0.999 \
    --adam-eps 1e-8 \
    --lr-decay 1.0 \
    --clip-grad 1.0 \
    --grad-accum 2 \
    --warmup-steps $((1281167 * 5 / 2048)) \
    --training-steps $((1281167 * 50 / 2048)) \
    --log-interval 100 \
    --eval-interval $((1281167 * 1 / 2048)) \
    --project deit3-jax \
    --name $(basename $0 .sh) \
    --ipaddr $(curl -s ifconfig.me) \
    --hostname $(hostname)