Add adapter index and metadata

README.md

@@ -0,0 +1,52 @@
+---
+library_name: peft
+tags:
+- peft
+- lora
+- qlora
+- prompt-tuning
+- instruction-tuning
+---
+
+# PEFT LLM Study Adapters
+
+This repository stores PEFT adapter weights only. Base model weights are not included.
+
+Load a specific adapter from its subfolder, for example:
+
+```python
+from transformers import AutoModelForCausalLM
+from peft import PeftModel
+
+base = AutoModelForCausalLM.from_pretrained("bigcode/starcoder2-3b")
+model = PeftModel.from_pretrained(base, "REPO_ID", subfolder="adapters/RUN_NAME")
+```
+
+## Adapter Index
+
+| Run | Method | Base Model | Dataset | Steps | Eval Loss | PPL | Peak GPU GB | Path |
+| --- | --- | --- | --- | ---: | ---: | ---: | ---: | --- |
+| lora_starcoder2_alpaca | lora | bigcode/starcoder2-3b | tatsu-lab/alpaca | 500 | 1.4354 | 4.2015 | 6.4372 | adapters/lora_starcoder2_alpaca |
+| mistral7b_lora_alpaca | lora | mistralai/Mistral-7B-v0.1 | tatsu-lab/alpaca | 500 | 1.6779 | 5.3541 | 14.4979 | adapters/mistral7b_lora_alpaca |
+| mistral7b_lora_alpaca_cot | lora | mistralai/Mistral-7B-v0.1 | QingyiSi/Alpaca-CoT/combination/alcapa_plus_cot.json | 500 | 1.5490 | 4.7067 | 14.4976 | adapters/mistral7b_lora_alpaca_cot |
+| mistral7b_lora_guanaco | lora | mistralai/Mistral-7B-v0.1 | fengtc/GuanacoDataset | 500 | 1.4493 | 4.2602 | 14.4984 | adapters/mistral7b_lora_guanaco |
+| mistral7b_prompt_alpaca_cot | prompt_tuning | mistralai/Mistral-7B-v0.1 | QingyiSi/Alpaca-CoT/combination/alcapa_plus_cot.json | 500 | 1.0121 | 2.7515 | 13.8476 | adapters/mistral7b_prompt_alpaca_cot |
+| mistral7b_prompt_guanaco | prompt_tuning | mistralai/Mistral-7B-v0.1 | fengtc/GuanacoDataset | 500 | 1.0030 | 2.7266 | 13.8476 | adapters/mistral7b_prompt_guanaco |
+| mistral7b_prompt_tuning_alpaca | prompt_tuning | mistralai/Mistral-7B-v0.1 | tatsu-lab/alpaca | 500 | 1.0024 | 2.7247 | 13.8487 | adapters/mistral7b_prompt_tuning_alpaca |
+| mistral7b_qlora_alpaca | qlora | mistralai/Mistral-7B-v0.1 | tatsu-lab/alpaca | 500 | 1.7437 | 5.7184 | 5.7184 | adapters/mistral7b_qlora_alpaca |
+| mistral7b_qlora_alpaca_cot | qlora | mistralai/Mistral-7B-v0.1 | QingyiSi/Alpaca-CoT/combination/alcapa_plus_cot.json | 500 | 1.5379 | 4.6548 | 5.7184 | adapters/mistral7b_qlora_alpaca_cot |
+| mistral7b_qlora_guanaco | qlora | mistralai/Mistral-7B-v0.1 | fengtc/GuanacoDataset | 500 | 1.4491 | 4.2591 | 5.7177 | adapters/mistral7b_qlora_guanaco |
+| prompt_tuning_starcoder2_alpaca | prompt_tuning | bigcode/starcoder2-3b | tatsu-lab/alpaca | 500 | 1.5602 | 4.7596 | 6.1643 | adapters/prompt_tuning_starcoder2_alpaca |
+| qlora_starcoder2_alpaca | qlora | bigcode/starcoder2-3b | tatsu-lab/alpaca | 500 | 1.4461 | 4.2465 | 3.0791 | adapters/qlora_starcoder2_alpaca |
+| starcoder2_lora_alpaca_cot | lora | bigcode/starcoder2-3b | QingyiSi/Alpaca-CoT/combination/alcapa_plus_cot.json | 500 | 1.2603 | 3.5264 | 6.4582 | adapters/starcoder2_lora_alpaca_cot |
+| starcoder2_lora_guanaco | lora | bigcode/starcoder2-3b | fengtc/GuanacoDataset | 500 | 1.2887 | 3.6282 | 6.4578 | adapters/starcoder2_lora_guanaco |
+| starcoder2_prompt_alpaca_cot | prompt_tuning | bigcode/starcoder2-3b | QingyiSi/Alpaca-CoT/combination/alcapa_plus_cot.json | 500 | 1.6412 | 5.1614 | 6.1846 | adapters/starcoder2_prompt_alpaca_cot |
+| starcoder2_prompt_guanaco | prompt_tuning | bigcode/starcoder2-3b | fengtc/GuanacoDataset | 500 | 1.3930 | 4.0268 | 6.1854 | adapters/starcoder2_prompt_guanaco |
+| starcoder2_qlora_alpaca_cot | qlora | bigcode/starcoder2-3b | QingyiSi/Alpaca-CoT/combination/alcapa_plus_cot.json | 500 | 1.2595 | 3.5237 | 3.0788 | adapters/starcoder2_qlora_alpaca_cot |
+| starcoder2_qlora_guanaco | qlora | bigcode/starcoder2-3b | fengtc/GuanacoDataset | 500 | 1.2983 | 3.6632 | 3.0791 | adapters/starcoder2_qlora_guanaco |
+
+## Notes
+
+- These are adapters trained for an empirical PEFT comparison project.
+- Guanaco proxy runs use `fengtc/GuanacoDataset` when the originally listed gated dataset is unavailable.
+- See `adapter_index.json` and `metadata/*/metrics.json` for machine-readable details.

adapter_index.json

@@ -0,0 +1,272 @@
+[
+  {
+    "base_model": "bigcode/starcoder2-3b",
+    "dataset": "tatsu-lab/alpaca",
+    "dataset_file": null,
+    "eval_loss": 1.4354339838027954,
+    "max_steps": 500,
+    "method": "lora",
+    "path_in_repo": "adapters/lora_starcoder2_alpaca",
+    "peak_gpu_memory_gb": 6.437230587005615,
+    "perplexity": 4.201467982241092,
+    "run": "lora_starcoder2_alpaca",
+    "train_loss": 0.7681699752807617,
+    "trainable_params": 23838720,
+    "trainable_percent": 0.7805199912694414
+  },
+  {
+    "base_model": "mistralai/Mistral-7B-v0.1",
+    "dataset": "tatsu-lab/alpaca",
+    "dataset_file": null,
+    "eval_loss": 1.6778709888458252,
+    "max_steps": 500,
+    "method": "lora",
+    "path_in_repo": "adapters/mistral7b_lora_alpaca",
+    "peak_gpu_memory_gb": 14.497857093811035,
+    "perplexity": 5.354144794182875,
+    "run": "mistral7b_lora_alpaca",
+    "train_loss": 0.24137365317344667,
+    "trainable_params": 41943040,
+    "trainable_percent": 0.5758499550960753
+  },
+  {
+    "base_model": "mistralai/Mistral-7B-v0.1",
+    "dataset": "QingyiSi/Alpaca-CoT",
+    "dataset_file": "combination/alcapa_plus_cot.json",
+    "eval_loss": 1.5489877462387085,
+    "max_steps": 500,
+    "method": "lora",
+    "path_in_repo": "adapters/mistral7b_lora_alpaca_cot",
+    "peak_gpu_memory_gb": 14.497621536254883,
+    "perplexity": 4.706703392184987,
+    "run": "mistral7b_lora_alpaca_cot",
+    "train_loss": 0.4384129907488823,
+    "trainable_params": 41943040,
+    "trainable_percent": 0.5758499550960753
+  },
+  {
+    "base_model": "mistralai/Mistral-7B-v0.1",
+    "dataset": "fengtc/GuanacoDataset",
+    "dataset_file": null,
+    "eval_loss": 1.4493114948272705,
+    "max_steps": 500,
+    "method": "lora",
+    "path_in_repo": "adapters/mistral7b_lora_guanaco",
+    "peak_gpu_memory_gb": 14.498394012451172,
+    "perplexity": 4.2601803489833925,
+    "run": "mistral7b_lora_guanaco",
+    "train_loss": 0.5458194477558136,
+    "trainable_params": 41943040,
+    "trainable_percent": 0.5758499550960753
+  },
+  {
+    "base_model": "mistralai/Mistral-7B-v0.1",
+    "dataset": "QingyiSi/Alpaca-CoT",
+    "dataset_file": "combination/alcapa_plus_cot.json",
+    "eval_loss": 1.012134313583374,
+    "max_steps": 500,
+    "method": "prompt_tuning",
+    "path_in_repo": "adapters/mistral7b_prompt_alpaca_cot",
+    "peak_gpu_memory_gb": 13.847609996795654,
+    "perplexity": 2.7514672465197147,
+    "run": "mistral7b_prompt_alpaca_cot",
+    "train_loss": 1.0451534767150878,
+    "trainable_params": 131072,
+    "trainable_percent": 0.0018099209686697024
+  },
+  {
+    "base_model": "mistralai/Mistral-7B-v0.1",
+    "dataset": "fengtc/GuanacoDataset",
+    "dataset_file": null,
+    "eval_loss": 1.0030479431152344,
+    "max_steps": 500,
+    "method": "prompt_tuning",
+    "path_in_repo": "adapters/mistral7b_prompt_guanaco",
+    "peak_gpu_memory_gb": 13.847609519958496,
+    "perplexity": 2.7265796360422554,
+    "run": "mistral7b_prompt_guanaco",
+    "train_loss": 1.0741594829559327,
+    "trainable_params": 131072,
+    "trainable_percent": 0.0018099209686697024
+  },
+  {
+    "base_model": "mistralai/Mistral-7B-v0.1",
+    "dataset": "tatsu-lab/alpaca",
+    "dataset_file": null,
+    "eval_loss": 1.0023709535598755,
+    "max_steps": 500,
+    "method": "prompt_tuning",
+    "path_in_repo": "adapters/mistral7b_prompt_tuning_alpaca",
+    "peak_gpu_memory_gb": 13.848701477050781,
+    "perplexity": 2.724734394781806,
+    "run": "mistral7b_prompt_tuning_alpaca",
+    "train_loss": 0.5351699771881103,
+    "trainable_params": 131072,
+    "trainable_percent": 0.0018099209686697024
+  },
+  {
+    "base_model": "mistralai/Mistral-7B-v0.1",
+    "dataset": "tatsu-lab/alpaca",
+    "dataset_file": null,
+    "eval_loss": 1.743696689605713,
+    "max_steps": 500,
+    "method": "qlora",
+    "path_in_repo": "adapters/mistral7b_qlora_alpaca",
+    "peak_gpu_memory_gb": 5.718385696411133,
+    "perplexity": 5.718443709459332,
+    "run": "mistral7b_qlora_alpaca",
+    "train_loss": 0.5023761793375016,
+    "trainable_params": 41943040,
+    "trainable_percent": 1.1055056122762943
+  },
+  {
+    "base_model": "mistralai/Mistral-7B-v0.1",
+    "dataset": "QingyiSi/Alpaca-CoT",
+    "dataset_file": "combination/alcapa_plus_cot.json",
+    "eval_loss": 1.5378942489624023,
+    "max_steps": 500,
+    "method": "qlora",
+    "path_in_repo": "adapters/mistral7b_qlora_alpaca_cot",
+    "peak_gpu_memory_gb": 5.718398094177246,
+    "perplexity": 4.654778139763514,
+    "run": "mistral7b_qlora_alpaca_cot",
+    "train_loss": 0.4474883066415787,
+    "trainable_params": 41943040,
+    "trainable_percent": 1.1055056122762943
+  },
+  {
+    "base_model": "mistralai/Mistral-7B-v0.1",
+    "dataset": "fengtc/GuanacoDataset",
+    "dataset_file": null,
+    "eval_loss": 1.4490643739700317,
+    "max_steps": 500,
+    "method": "qlora",
+    "path_in_repo": "adapters/mistral7b_qlora_guanaco",
+    "peak_gpu_memory_gb": 5.7177019119262695,
+    "perplexity": 4.259127699634722,
+    "run": "mistral7b_qlora_guanaco",
+    "train_loss": 0.5492446699142456,
+    "trainable_params": 41943040,
+    "trainable_percent": 1.1055056122762943
+  },
+  {
+    "base_model": "bigcode/starcoder2-3b",
+    "dataset": "tatsu-lab/alpaca",
+    "dataset_file": null,
+    "eval_loss": 1.5601716041564941,
+    "max_steps": 500,
+    "method": "prompt_tuning",
+    "path_in_repo": "adapters/prompt_tuning_starcoder2_alpaca",
+    "peak_gpu_memory_gb": 6.164307594299316,
+    "perplexity": 4.759637948716392,
+    "run": "prompt_tuning_starcoder2_alpaca",
+    "train_loss": 1.4749931907653808,
+    "trainable_params": 98304,
+    "trainable_percent": 0.0032438536575970546
+  },
+  {
+    "base_model": "bigcode/starcoder2-3b",
+    "dataset": "tatsu-lab/alpaca",
+    "dataset_file": null,
+    "eval_loss": 1.4460986852645874,
+    "max_steps": 500,
+    "method": "qlora",
+    "path_in_repo": "adapters/qlora_starcoder2_alpaca",
+    "peak_gpu_memory_gb": 3.079056739807129,
+    "perplexity": 4.2465151643917896,
+    "run": "qlora_starcoder2_alpaca",
+    "train_loss": 0.7822464866638184,
+    "trainable_params": 23838720,
+    "trainable_percent": 1.4760456432877014
+  },
+  {
+    "base_model": "bigcode/starcoder2-3b",
+    "dataset": "QingyiSi/Alpaca-CoT",
+    "dataset_file": "combination/alcapa_plus_cot.json",
+    "eval_loss": 1.2602890729904175,
+    "max_steps": 500,
+    "method": "lora",
+    "path_in_repo": "adapters/starcoder2_lora_alpaca_cot",
+    "peak_gpu_memory_gb": 6.45817756652832,
+    "perplexity": 3.5264407388091508,
+    "run": "starcoder2_lora_alpaca_cot",
+    "train_loss": 1.4059115772247315,
+    "trainable_params": 23838720,
+    "trainable_percent": 0.7805199912694414
+  },
+  {
+    "base_model": "bigcode/starcoder2-3b",
+    "dataset": "fengtc/GuanacoDataset",
+    "dataset_file": null,
+    "eval_loss": 1.2887365818023682,
+    "max_steps": 500,
+    "method": "lora",
+    "path_in_repo": "adapters/starcoder2_lora_guanaco",
+    "peak_gpu_memory_gb": 6.45782995223999,
+    "perplexity": 3.6281997252628484,
+    "run": "starcoder2_lora_guanaco",
+    "train_loss": 1.6917072792053223,
+    "trainable_params": 23838720,
+    "trainable_percent": 0.7805199912694414
+  },
+  {
+    "base_model": "bigcode/starcoder2-3b",
+    "dataset": "QingyiSi/Alpaca-CoT",
+    "dataset_file": "combination/alcapa_plus_cot.json",
+    "eval_loss": 1.6411982774734497,
+    "max_steps": 500,
+    "method": "prompt_tuning",
+    "path_in_repo": "adapters/starcoder2_prompt_alpaca_cot",
+    "peak_gpu_memory_gb": 6.1845526695251465,
+    "perplexity": 5.161350538285551,
+    "run": "starcoder2_prompt_alpaca_cot",
+    "train_loss": 2.839910514831543,
+    "trainable_params": 98304,
+    "trainable_percent": 0.0032438536575970546
+  },
+  {
+    "base_model": "bigcode/starcoder2-3b",
+    "dataset": "fengtc/GuanacoDataset",
+    "dataset_file": null,
+    "eval_loss": 1.3929762840270996,
+    "max_steps": 500,
+    "method": "prompt_tuning",
+    "path_in_repo": "adapters/starcoder2_prompt_guanaco",
+    "peak_gpu_memory_gb": 6.185425758361816,
+    "perplexity": 4.026817187039079,
+    "run": "starcoder2_prompt_guanaco",
+    "train_loss": 3.4105590629577636,
+    "trainable_params": 98304,
+    "trainable_percent": 0.0032438536575970546
+  },
+  {
+    "base_model": "bigcode/starcoder2-3b",
+    "dataset": "QingyiSi/Alpaca-CoT",
+    "dataset_file": "combination/alcapa_plus_cot.json",
+    "eval_loss": 1.2594980001449585,
+    "max_steps": 500,
+    "method": "qlora",
+    "path_in_repo": "adapters/starcoder2_qlora_alpaca_cot",
+    "peak_gpu_memory_gb": 3.0788111686706543,
+    "perplexity": 3.5236521704253443,
+    "run": "starcoder2_qlora_alpaca_cot",
+    "train_loss": 0.6505789184570312,
+    "trainable_params": 23838720,
+    "trainable_percent": 1.4760456432877014
+  },
+  {
+    "base_model": "bigcode/starcoder2-3b",
+    "dataset": "fengtc/GuanacoDataset",
+    "dataset_file": null,
+    "eval_loss": 1.2983253002166748,
+    "max_steps": 500,
+    "method": "qlora",
+    "path_in_repo": "adapters/starcoder2_qlora_guanaco",
+    "peak_gpu_memory_gb": 3.079078197479248,
+    "perplexity": 3.6631568399040884,
+    "run": "starcoder2_qlora_guanaco",
+    "train_loss": 1.715220178604126,
+    "trainable_params": 23838720,
+    "trainable_percent": 1.4760456432877014
+  }
+]

metadata/lora_starcoder2_alpaca/args.json

@@ -0,0 +1,40 @@
+{
+  "bf16": true,
+  "config": "configs/lora.yaml",
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "bigcode/starcoder2-3b",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/lora_starcoder2_alpaca",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": "outputs/lora_starcoder2_alpaca/checkpoint-250",
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "c_fc",
+    "c_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/lora_starcoder2_alpaca/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.4354339838027954,
+    "eval_runtime": 7.5045,
+    "eval_samples_per_second": 34.113,
+    "eval_steps_per_second": 17.057,
+    "perplexity": 4.201467982241092
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "bigcode/starcoder2-3b",
+  "parameter_counts": {
+    "total_params": 3054210048,
+    "trainable_params": 23838720,
+    "trainable_percent": 0.7805199912694414
+  },
+  "peak_gpu_memory_gb": 6.437230587005615,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 1.7099333093818368e+16,
+    "train_loss": 0.7681699752807617,
+    "train_runtime": 1315.7845,
+    "train_samples_per_second": 6.08,
+    "train_steps_per_second": 0.38
+  },
+  "wall_clock_seconds": 1316.437697172165
+}

metadata/mistral7b_lora_alpaca/args.json

@@ -0,0 +1,41 @@
+{
+  "bf16": true,
+  "config": "configs/mistral_lora_alpaca.yaml",
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/mistral7b_lora_alpaca",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": "outputs/mistral7b_lora_alpaca/checkpoint-125",
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/mistral7b_lora_alpaca/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.6778709888458252,
+    "eval_runtime": 9.7536,
+    "eval_samples_per_second": 26.247,
+    "eval_steps_per_second": 13.123,
+    "perplexity": 5.354144794182875
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "parameter_counts": {
+    "total_params": 7283675136,
+    "trainable_params": 41943040,
+    "trainable_percent": 0.5758499550960753
+  },
+  "peak_gpu_memory_gb": 14.497857093811035,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 4.216442382424474e+16,
+    "train_loss": 0.24137365317344667,
+    "train_runtime": 2464.6935,
+    "train_samples_per_second": 3.246,
+    "train_steps_per_second": 0.203
+  },
+  "wall_clock_seconds": 2465.9031274318695
+}

metadata/mistral7b_lora_alpaca_cot/args.json

@@ -0,0 +1,41 @@
+{
+  "bf16": true,
+  "config": "configs/mistral7b_lora_alpaca_cot.yaml",
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/mistral7b_lora_alpaca_cot",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/mistral7b_lora_alpaca_cot/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.5489877462387085,
+    "eval_runtime": 10.5771,
+    "eval_samples_per_second": 24.109,
+    "eval_steps_per_second": 12.102,
+    "perplexity": 4.706703392184987
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "parameter_counts": {
+    "total_params": 7283675136,
+    "trainable_params": 41943040,
+    "trainable_percent": 0.5758499550960753
+  },
+  "peak_gpu_memory_gb": 14.497621536254883,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 4.803390717847142e+16,
+    "train_loss": 0.4384129907488823,
+    "train_runtime": 3637.731,
+    "train_samples_per_second": 2.199,
+    "train_steps_per_second": 0.137
+  },
+  "wall_clock_seconds": 3637.9769699573517
+}

metadata/mistral7b_lora_guanaco/args.json

@@ -0,0 +1,41 @@
+{
+  "bf16": true,
+  "config": "configs/mistral7b_lora_guanaco.yaml",
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/mistral7b_lora_guanaco",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/mistral7b_lora_guanaco/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.4493114948272705,
+    "eval_runtime": 13.4923,
+    "eval_samples_per_second": 18.974,
+    "eval_steps_per_second": 9.487,
+    "perplexity": 4.2601803489833925
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "parameter_counts": {
+    "total_params": 7283675136,
+    "trainable_params": 41943040,
+    "trainable_percent": 0.5758499550960753
+  },
+  "peak_gpu_memory_gb": 14.498394012451172,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 7.071807330975744e+16,
+    "train_loss": 0.5458194477558136,
+    "train_runtime": 3848.5515,
+    "train_samples_per_second": 2.079,
+    "train_steps_per_second": 0.13
+  },
+  "wall_clock_seconds": 3848.8962047100067
+}

metadata/mistral7b_prompt_alpaca_cot/args.json

@@ -0,0 +1,33 @@
+{
+  "bf16": true,
+  "config": "configs/mistral7b_prompt_alpaca_cot.yaml",
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/mistral7b_prompt_alpaca_cot",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": null,
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/mistral7b_prompt_alpaca_cot/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.012134313583374,
+    "eval_runtime": 8.0602,
+    "eval_samples_per_second": 31.637,
+    "eval_steps_per_second": 15.88,
+    "perplexity": 2.7514672465197147
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "parameter_counts": {
+    "total_params": 7241863168,
+    "trainable_params": 131072,
+    "trainable_percent": 0.0018099209686697024
+  },
+  "peak_gpu_memory_gb": 13.847609996795654,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 4.775223517013606e+16,
+    "train_loss": 1.0451534767150878,
+    "train_runtime": 1522.2622,
+    "train_samples_per_second": 5.255,
+    "train_steps_per_second": 0.328
+  },
+  "wall_clock_seconds": 1522.533498287201
+}

metadata/mistral7b_prompt_guanaco/args.json

@@ -0,0 +1,33 @@
+{
+  "bf16": true,
+  "config": "configs/mistral7b_prompt_guanaco.yaml",
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/mistral7b_prompt_guanaco",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": null,
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/mistral7b_prompt_guanaco/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.0030479431152344,
+    "eval_runtime": 10.7067,
+    "eval_samples_per_second": 23.91,
+    "eval_steps_per_second": 11.955,
+    "perplexity": 2.7265796360422554
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "parameter_counts": {
+    "total_params": 7241863168,
+    "trainable_params": 131072,
+    "trainable_percent": 0.0018099209686697024
+  },
+  "peak_gpu_memory_gb": 13.847609519958496,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 7.030338079555584e+16,
+    "train_loss": 1.0741594829559327,
+    "train_runtime": 1610.509,
+    "train_samples_per_second": 4.967,
+    "train_steps_per_second": 0.31
+  },
+  "wall_clock_seconds": 1610.6813061237335
+}

metadata/mistral7b_prompt_tuning_alpaca/args.json

@@ -0,0 +1,33 @@
+{
+  "bf16": true,
+  "config": "configs/mistral_prompt_tuning_alpaca.yaml",
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/mistral7b_prompt_tuning_alpaca",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": "outputs/mistral7b_prompt_tuning_alpaca/checkpoint-250",
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": null,
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/mistral7b_prompt_tuning_alpaca/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.0023709535598755,
+    "eval_runtime": 6.1148,
+    "eval_samples_per_second": 41.866,
+    "eval_steps_per_second": 20.933,
+    "perplexity": 2.724734394781806
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "parameter_counts": {
+    "total_params": 7241863168,
+    "trainable_params": 131072,
+    "trainable_percent": 0.0018099209686697024
+  },
+  "peak_gpu_memory_gb": 13.848701477050781,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 4.19171706100777e+16,
+    "train_loss": 0.5351699771881103,
+    "train_runtime": 725.0655,
+    "train_samples_per_second": 11.033,
+    "train_steps_per_second": 0.69
+  },
+  "wall_clock_seconds": 725.2680373191833
+}

metadata/mistral7b_qlora_alpaca/args.json

@@ -0,0 +1,41 @@
+{
+  "bf16": true,
+  "config": "configs/mistral_qlora_alpaca.yaml",
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/mistral7b_qlora_alpaca",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/mistral7b_qlora_alpaca/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.743696689605713,
+    "eval_runtime": 15.1588,
+    "eval_samples_per_second": 16.888,
+    "eval_steps_per_second": 8.444,
+    "perplexity": 5.718443709459332
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "parameter_counts": {
+    "total_params": 3794014208,
+    "trainable_params": 41943040,
+    "trainable_percent": 1.1055056122762943
+  },
+  "peak_gpu_memory_gb": 5.718385696411133,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 4.216442382424474e+16,
+    "train_loss": 0.5023761793375016,
+    "train_runtime": 4203.0874,
+    "train_samples_per_second": 1.903,
+    "train_steps_per_second": 0.119
+  },
+  "wall_clock_seconds": 4203.367736339569
+}

metadata/mistral7b_qlora_alpaca_cot/args.json

@@ -0,0 +1,41 @@
+{
+  "bf16": true,
+  "config": "configs/mistral7b_qlora_alpaca_cot.yaml",
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/mistral7b_qlora_alpaca_cot",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/mistral7b_qlora_alpaca_cot/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.5378942489624023,
+    "eval_runtime": 14.9658,
+    "eval_samples_per_second": 17.039,
+    "eval_steps_per_second": 8.553,
+    "perplexity": 4.654778139763514
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "parameter_counts": {
+    "total_params": 3794014208,
+    "trainable_params": 41943040,
+    "trainable_percent": 1.1055056122762943
+  },
+  "peak_gpu_memory_gb": 5.718398094177246,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 4.803390717847142e+16,
+    "train_loss": 0.4474883066415787,
+    "train_runtime": 4813.4527,
+    "train_samples_per_second": 1.662,
+    "train_steps_per_second": 0.104
+  },
+  "wall_clock_seconds": 4813.7332644462585
+}

metadata/mistral7b_qlora_guanaco/args.json

@@ -0,0 +1,41 @@
+{
+  "bf16": true,
+  "config": "configs/mistral7b_qlora_guanaco.yaml",
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/mistral7b_qlora_guanaco",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/mistral7b_qlora_guanaco/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.4490643739700317,
+    "eval_runtime": 16.1985,
+    "eval_samples_per_second": 15.804,
+    "eval_steps_per_second": 7.902,
+    "perplexity": 4.259127699634722
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "mistralai/Mistral-7B-v0.1",
+  "parameter_counts": {
+    "total_params": 3794014208,
+    "trainable_params": 41943040,
+    "trainable_percent": 1.1055056122762943
+  },
+  "peak_gpu_memory_gb": 5.7177019119262695,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 7.071807330975744e+16,
+    "train_loss": 0.5492446699142456,
+    "train_runtime": 4817.4079,
+    "train_samples_per_second": 1.661,
+    "train_steps_per_second": 0.104
+  },
+  "wall_clock_seconds": 4817.6931245327
+}

metadata/prompt_tuning_starcoder2_alpaca/args.json

@@ -0,0 +1,33 @@
+{
+  "bf16": true,
+  "config": "configs/prompt_tuning.yaml",
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "bigcode/starcoder2-3b",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/prompt_tuning_starcoder2_alpaca",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": "outputs/prompt_tuning_starcoder2_alpaca/checkpoint-250",
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": null,
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/prompt_tuning_starcoder2_alpaca/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.5601716041564941,
+    "eval_runtime": 4.7153,
+    "eval_samples_per_second": 54.292,
+    "eval_steps_per_second": 27.146,
+    "perplexity": 4.759637948716392
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "bigcode/starcoder2-3b",
+  "parameter_counts": {
+    "total_params": 3030469632,
+    "trainable_params": 98304,
+    "trainable_percent": 0.0032438536575970546
+  },
+  "peak_gpu_memory_gb": 6.164307594299316,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 1.6958927991472128e+16,
+    "train_loss": 1.4749931907653808,
+    "train_runtime": 599.8836,
+    "train_samples_per_second": 13.336,
+    "train_steps_per_second": 0.833
+  },
+  "wall_clock_seconds": 600.0733258724213
+}

metadata/qlora_starcoder2_alpaca/args.json

@@ -0,0 +1,40 @@
+{
+  "bf16": true,
+  "config": "configs/qlora.yaml",
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "bigcode/starcoder2-3b",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/qlora_starcoder2_alpaca",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": "outputs/qlora_starcoder2_alpaca/checkpoint-250",
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "c_fc",
+    "c_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/qlora_starcoder2_alpaca/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "tatsu-lab/alpaca",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.4460986852645874,
+    "eval_runtime": 12.7595,
+    "eval_samples_per_second": 20.063,
+    "eval_steps_per_second": 10.032,
+    "perplexity": 4.2465151643917896
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "bigcode/starcoder2-3b",
+  "parameter_counts": {
+    "total_params": 1615039488,
+    "trainable_params": 23838720,
+    "trainable_percent": 1.4760456432877014
+  },
+  "peak_gpu_memory_gb": 3.079056739807129,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 1.7099333093818368e+16,
+    "train_loss": 0.7822464866638184,
+    "train_runtime": 1784.5326,
+    "train_samples_per_second": 4.483,
+    "train_steps_per_second": 0.28
+  },
+  "wall_clock_seconds": 1785.0911331176758
+}

metadata/starcoder2_lora_alpaca_cot/args.json

@@ -0,0 +1,40 @@
+{
+  "bf16": true,
+  "config": "configs/starcoder2_lora_alpaca_cot.yaml",
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "bigcode/starcoder2-3b",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/starcoder2_lora_alpaca_cot",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "c_fc",
+    "c_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/starcoder2_lora_alpaca_cot/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.2602890729904175,
+    "eval_runtime": 7.8256,
+    "eval_samples_per_second": 32.586,
+    "eval_steps_per_second": 16.357,
+    "perplexity": 3.5264407388091508
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "bigcode/starcoder2-3b",
+  "parameter_counts": {
+    "total_params": 3054210048,
+    "trainable_params": 23838720,
+    "trainable_percent": 0.7805199912694414
+  },
+  "peak_gpu_memory_gb": 6.45817756652832,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 1.954360795417805e+16,
+    "train_loss": 1.4059115772247315,
+    "train_runtime": 2861.1711,
+    "train_samples_per_second": 2.796,
+    "train_steps_per_second": 0.175
+  },
+  "wall_clock_seconds": 2861.4430129528046
+}

metadata/starcoder2_lora_guanaco/args.json

@@ -0,0 +1,40 @@
+{
+  "bf16": true,
+  "config": "configs/starcoder2_lora_guanaco.yaml",
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "bigcode/starcoder2-3b",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/starcoder2_lora_guanaco",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "c_fc",
+    "c_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/starcoder2_lora_guanaco/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.2887365818023682,
+    "eval_runtime": 8.8385,
+    "eval_samples_per_second": 28.964,
+    "eval_steps_per_second": 14.482,
+    "perplexity": 3.6281997252628484
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "lora",
+  "model_name": "bigcode/starcoder2-3b",
+  "parameter_counts": {
+    "total_params": 3054210048,
+    "trainable_params": 23838720,
+    "trainable_percent": 0.7805199912694414
+  },
+  "peak_gpu_memory_gb": 6.45782995223999,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 2.638869239778509e+16,
+    "train_loss": 1.6917072792053223,
+    "train_runtime": 2982.7018,
+    "train_samples_per_second": 2.682,
+    "train_steps_per_second": 0.168
+  },
+  "wall_clock_seconds": 2982.9298701286316
+}

metadata/starcoder2_prompt_alpaca_cot/args.json

@@ -0,0 +1,33 @@
+{
+  "bf16": true,
+  "config": "configs/starcoder2_prompt_alpaca_cot.yaml",
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "bigcode/starcoder2-3b",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/starcoder2_prompt_alpaca_cot",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": null,
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/starcoder2_prompt_alpaca_cot/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.6411982774734497,
+    "eval_runtime": 3.7453,
+    "eval_samples_per_second": 68.086,
+    "eval_steps_per_second": 34.176,
+    "perplexity": 5.161350538285551
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "bigcode/starcoder2-3b",
+  "parameter_counts": {
+    "total_params": 3030469632,
+    "trainable_params": 98304,
+    "trainable_percent": 0.0032438536575970546
+  },
+  "peak_gpu_memory_gb": 6.1845526695251465,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 1.938313255668941e+16,
+    "train_loss": 2.839910514831543,
+    "train_runtime": 1108.6114,
+    "train_samples_per_second": 7.216,
+    "train_steps_per_second": 0.451
+  },
+  "wall_clock_seconds": 1109.3785409927368
+}

metadata/starcoder2_prompt_guanaco/args.json

@@ -0,0 +1,33 @@
+{
+  "bf16": true,
+  "config": "configs/starcoder2_prompt_guanaco.yaml",
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "bigcode/starcoder2-3b",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/starcoder2_prompt_guanaco",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": null,
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/starcoder2_prompt_guanaco/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.3929762840270996,
+    "eval_runtime": 4.401,
+    "eval_samples_per_second": 58.169,
+    "eval_steps_per_second": 29.085,
+    "perplexity": 4.026817187039079
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "prompt_tuning",
+  "model_name": "bigcode/starcoder2-3b",
+  "parameter_counts": {
+    "total_params": 3030469632,
+    "trainable_params": 98304,
+    "trainable_percent": 0.0032438536575970546
+  },
+  "peak_gpu_memory_gb": 6.185425758361816,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 2.617201101982925e+16,
+    "train_loss": 3.4105590629577636,
+    "train_runtime": 1179.8258,
+    "train_samples_per_second": 6.781,
+    "train_steps_per_second": 0.424
+  },
+  "wall_clock_seconds": 1180.018343448639
+}

metadata/starcoder2_qlora_alpaca_cot/args.json

@@ -0,0 +1,40 @@
+{
+  "bf16": true,
+  "config": "configs/starcoder2_qlora_alpaca_cot.yaml",
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "bigcode/starcoder2-3b",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/starcoder2_qlora_alpaca_cot",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": "outputs/starcoder2_qlora_alpaca_cot/checkpoint-250",
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "c_fc",
+    "c_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/starcoder2_qlora_alpaca_cot/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": "combination/alcapa_plus_cot.json",
+  "dataset_name": "QingyiSi/Alpaca-CoT",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.2594980001449585,
+    "eval_runtime": 12.4767,
+    "eval_samples_per_second": 20.438,
+    "eval_steps_per_second": 10.259,
+    "perplexity": 3.5236521704253443
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "bigcode/starcoder2-3b",
+  "parameter_counts": {
+    "total_params": 1615039488,
+    "trainable_params": 23838720,
+    "trainable_percent": 1.4760456432877014
+  },
+  "peak_gpu_memory_gb": 3.0788111686706543,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 1.954360795417805e+16,
+    "train_loss": 0.6505789184570312,
+    "train_runtime": 1766.2618,
+    "train_samples_per_second": 4.529,
+    "train_steps_per_second": 0.283
+  },
+  "wall_clock_seconds": 1767.02236866951
+}

metadata/starcoder2_qlora_guanaco/args.json

@@ -0,0 +1,40 @@
+{
+  "bf16": true,
+  "config": "configs/starcoder2_qlora_guanaco.yaml",
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "dataset_split": "train",
+  "eval_steps": 50,
+  "gradient_accumulation_steps": 16,
+  "gradient_checkpointing": true,
+  "learning_rate": 0.0002,
+  "logging_steps": 10,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "lora_r": 16,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "bigcode/starcoder2-3b",
+  "num_train_epochs": 1,
+  "num_virtual_tokens": 32,
+  "output_dir": "outputs/starcoder2_qlora_guanaco",
+  "per_device_eval_batch_size": 2,
+  "per_device_train_batch_size": 1,
+  "prompt_tuning_init_text": "Follow the instruction carefully and provide a helpful, accurate response.",
+  "resume_from_checkpoint": null,
+  "save_steps": 250,
+  "seed": 42,
+  "target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "c_fc",
+    "c_proj"
+  ],
+  "warmup_ratio": 0.03,
+  "weight_decay": 0.0
+}

metadata/starcoder2_qlora_guanaco/metrics.json

@@ -0,0 +1,36 @@
+{
+  "dataset_file": null,
+  "dataset_name": "fengtc/GuanacoDataset",
+  "effective_batch_size": 16,
+  "eval_metrics": {
+    "epoch": 4.0,
+    "eval_loss": 1.2983253002166748,
+    "eval_runtime": 16.2141,
+    "eval_samples_per_second": 15.789,
+    "eval_steps_per_second": 7.894,
+    "perplexity": 3.6631568399040884
+  },
+  "learning_rate": 0.0002,
+  "max_eval_samples": 256,
+  "max_seq_length": 512,
+  "max_steps": 500,
+  "max_train_samples": 2000,
+  "method": "qlora",
+  "model_name": "bigcode/starcoder2-3b",
+  "parameter_counts": {
+    "total_params": 1615039488,
+    "trainable_params": 23838720,
+    "trainable_percent": 1.4760456432877014
+  },
+  "peak_gpu_memory_gb": 3.079078197479248,
+  "seed": 42,
+  "train_metrics": {
+    "epoch": 4.0,
+    "total_flos": 2.638869239778509e+16,
+    "train_loss": 1.715220178604126,
+    "train_runtime": 4170.5822,
+    "train_samples_per_second": 1.918,
+    "train_steps_per_second": 0.12
+  },
+  "wall_clock_seconds": 4170.863238573074
+}