training_args = GRPOConfig(
learning_rate=5e-6,
#num_train_epochs=1,
max_steps=3400, # Number of dataset passes. For full trainings, use `num_train_epochs` instead
# num_train_epochs=3,
# Parameters that control the data preprocessing
per_device_train_batch_size=4,
generation_batch_size=8, # If `None`, it defaults to the effective training batch size:`per_device_train_batch_size * num_processes * steps_per_generation`. In other words, there is one generation batch processed per optimization step.
max_completion_length=64, # default: 256 # Max completion length produced during training
num_generations=8, # 2, # default: 8 # Number of generations produced during training for comparison
fp16=False,
bf16=True,
ddp_find_unused_parameters=True,
use_vllm=True,
vllm_mode="colocate",
vllm_gpu_memory_utilization=0.55, # 0.30
vllm_max_model_length=8192,
dataloader_num_workers=8,
# Parameters related to reporting and saving
output_dir=output_dir, # Where to save model checkpoints and logs
logging_steps=10, # Log training metrics every N steps
report_to="trackio", # Experiment tracking tool
# Hub integration
push_to_hub=True,
log_completions=True
)
# You may need to update `target_modules` depending on the architecture of your chosen model.
# For example, different VLMs might have different attention/projection layer names.
peft_config = LoraConfig(
r=64,
lora_alpha=64,
lora_dropout=0.1,
# target_modules=["q_proj", "v_proj"],
target_modules="all-linear",
)
{'loss': 0.0, 'grad_norm': 0.0, 'learning_rate': 1.4705882352941176e-09, 'num_tokens': 6685201.0, 'completions/mean_length': 11.1, 'completions/min_length': 10.8, 'completions/max_length': 11.2, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 11.1, 'completions/min_terminated_length': 10.8, 'completions/max_terminated_length': 11.2, 'rewards/format_reward/mean': 0.0, 'rewards/format_reward/std': 0.0, 'rewards/accuracy_reward/mean': 0.6, 'rewards/accuracy_reward/std': 0.0, 'reward': 0.6, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'sampling/sampling_logp_difference/mean': 0.12829887121915817, 'sampling/sampling_logp_difference/max': 2.0802665531635283, 'sampling/importance_sampling_ratio/min': 0.5048314990475774, 'sampling/importance_sampling_ratio/mean': 0.5824737340211869, 'sampling/importance_sampling_ratio/max': 0.6787947922945022, 'entropy': 0.06522154565900565, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 0.9909579122004288, 'epoch': 0.44}