Upload training_config.yml with huggingface_hub

training_config.yml

@@ -5,8 +5,8 @@ model:
   - v_proj
   apply_lora_to_mlp: false
   apply_lora_to_output: false
-  lora_rank: 8
-  lora_alpha: 16
+  lora_rank: 16
+  lora_alpha: 32
   perception_tokens: 2
   use_clip: false
 tokenizer:
@@ -22,11 +22,11 @@ checkpointer:
   output_dir: output_checkpoints/experiment_1
   model_type: LLAMA3
 resume_from_checkpoint: false
-interim_checkpoint_steps: 5000
+interim_checkpoint_steps: 15000
 interim_gen_steps: null
-max_new_tokens: 100
+max_new_tokens: 88
 temperature: 0.6
-top_k: 300
+top_k: 200
 dataset:
   _component_: ds.EvenBatcher
   buffer_size: 36
@@ -46,19 +46,19 @@ dataset:
       train_on_input: false
 seed: null
 shuffle: true
-batch_size: 4
+batch_size: 8
 optimizer:
   _component_: torch.optim.AdamW
-  weight_decay: 0.01
+  weight_decay: 0.99
   lr: 0.0003
 lr_scheduler:
   _component_: torchtune.modules.get_cosine_schedule_with_warmup
-  num_warmup_steps: 100
+  num_warmup_steps: 4
 loss:
   _component_: torch.nn.CrossEntropyLoss
-epochs: 5
+epochs: 6
 max_steps_per_epoch: null
-gradient_accumulation_steps: 16
+gradient_accumulation_steps: 512
 compile: false
 output_dir: /tmp/lora_finetune_output
 metric_logger:
@@ -77,8 +77,8 @@ inference:
     {video}
 
     Caption the previous video.'
-  max_new_tokens: 300
-  temperature: 0.6
-  top_k: 300
+  max_new_tokens: 222
+  temperature: 0.8
+  top_k: 222
   quantizer: null
 gradient-accumulation-steps: 32