hparams.json

{
    "epoch_length": 250000,
    "compression": 100,
    "sequence_length": 2048,
    "tokenizer_name": "togethercomputer/LLaMA-2-7B-32K",
    "num_hidden_layers": 16,
    "hidden_size": 2048,
    "intermediate_size": 8192,
    "num_attention_heads": 8,
    "num_key_value_heads": 8,
    "activation_function": "swiGLU",
    "max_position_embeddings": 2048,
    "mixed_precision_param": "bfloat16",
    "mixed_precision_reduce": "float32",
    "window_length": 2,
    "desired_batch_size": 512,
    "learning_rate": 7.5e-05,
    "optimizer_beta1": 0.9,
    "optimizer_beta2": 0.95,
    "optimizer_weight_decay": 0.1,
    "grad_clip": 1.0,
    "cosine_epoch_length": 5000,
    "eta_min": 1e-05,
    "max_history": 10,
    "window_speed": 100,
    "validator_moving_alpha": 0.999,
    "validator_norm_regularization": 0.01,
    "validator_weights_temperature": 10,
    "validator_window_eval_size": 3,
    "valdiator_sample_rate": 0.01
}