training_args = GRPOConfig(
    learning_rate=5e-6,
    #num_train_epochs=1,
    max_steps=3400,                                        # Number of dataset passes. For full trainings, use `num_train_epochs` instead
    # num_train_epochs=3,
    # Parameters that control the data preprocessing
    per_device_train_batch_size=4,
    generation_batch_size=8,                           # If `None`, it defaults to the effective training batch size:`per_device_train_batch_size * num_processes * steps_per_generation`. In other words, there is one generation batch processed per optimization step.
    max_completion_length=64, # default: 256            # Max completion length produced during training
    num_generations=8, # 2, # default: 8                  # Number of generations produced during training for comparison

    fp16=False,
    bf16=True,
    ddp_find_unused_parameters=True,

    use_vllm=True,
    vllm_mode="colocate",
    vllm_gpu_memory_utilization=0.55,  # 0.30
    vllm_max_model_length=8192,

    dataloader_num_workers=8,

    # Parameters related to reporting and saving
    output_dir=output_dir,                                # Where to save model checkpoints and logs
    logging_steps=10,                                      # Log training metrics every N steps
    report_to="trackio",                                  # Experiment tracking tool

    # Hub integration
    push_to_hub=True,
    log_completions=True
)


# You may need to update `target_modules` depending on the architecture of your chosen model.
# For example, different VLMs might have different attention/projection layer names.



peft_config = LoraConfig(
    r=64,
    lora_alpha=64,
    lora_dropout=0.1,
    # target_modules=["q_proj", "v_proj"],
    target_modules="all-linear",
)

{'loss': 0.0, 'grad_norm': 0.0, 'learning_rate': 1.4705882352941176e-09, 'num_tokens': 6685201.0, 'completions/mean_length': 11.1, 'completions/min_length': 10.8, 'completions/max_length': 11.2, 'completions/clipped_ratio': 0.0, 'completions/mean_terminated_length': 11.1, 'completions/min_terminated_length': 10.8, 'completions/max_terminated_length': 11.2, 'rewards/format_reward/mean': 0.0, 'rewards/format_reward/std': 0.0, 'rewards/accuracy_reward/mean': 0.6, 'rewards/accuracy_reward/std': 0.0, 'reward': 0.6, 'reward_std': 0.0, 'frac_reward_zero_std': 1.0, 'sampling/sampling_logp_difference/mean': 0.12829887121915817, 'sampling/sampling_logp_difference/max': 2.0802665531635283, 'sampling/importance_sampling_ratio/min': 0.5048314990475774, 'sampling/importance_sampling_ratio/mean': 0.5824737340211869, 'sampling/importance_sampling_ratio/max': 0.6787947922945022, 'entropy': 0.06522154565900565, 'clip_ratio/low_mean': 0.0, 'clip_ratio/low_min': 0.0, 'clip_ratio/high_mean': 0.0, 'clip_ratio/high_max': 0.0, 'clip_ratio/region_mean': 0.0, 'step_time': 0.9909579122004288, 'epoch': 0.44}

Downloads last month

-

Downloads are not tracked for this model. How to track
Inference Providers NEW
This model isn't deployed by any Inference Provider. ๐Ÿ™‹ Ask for provider support