Hugging Face's logo

yycsu
/
zephyr-7b-sft-qlora

PEFT
TensorBoard
Safetensors
mistral
alignment-handbook
trl
sft
Generated from Trainer
Model card Files
xet
 Metrics Community
yycsu commited on
Commit
77d808b
·
verified ·
1 Parent(s): b90a301

Model save

Browse files
Files changed (7) hide show
  1. README.md +3 -6
  2. adapter_config.json +5 -5
  3. all_results.json +4 -4
  4. runs/Jul31_16-12-24_VM-146-67-centos/events.out.tfevents.1722413635.VM-146-67-centos.3345190.0 +3 -0
  5. train_results.json +4 -4
  6. trainer_state.json +5 -13
  7. training_args.bin +1 -1
README.md CHANGED
@@ -19,8 +19,6 @@ should probably proofread and complete it, then remove this comment. -->
19
  # zephyr-7b-sft-qlora
20
 
21
  This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the generator dataset.
22
- It achieves the following results on the evaluation set:
23
- - Loss: 0.8693
24
 
25
  ## Model description
26
 
@@ -44,8 +42,10 @@ The following hyperparameters were used during training:
44
  - eval_batch_size: 8
45
  - seed: 42
46
  - distributed_type: multi-GPU
 
47
  - gradient_accumulation_steps: 2
48
- - total_train_batch_size: 8
 
49
  - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
50
  - lr_scheduler_type: cosine
51
  - lr_scheduler_warmup_ratio: 0.1
@@ -53,9 +53,6 @@ The following hyperparameters were used during training:
53
 
54
  ### Training results
55
 
56
- | Training Loss | Epoch | Step | Validation Loss |
57
- |:-------------:|:------:|:----:|:---------------:|
58
- | 0.7609 | 0.9836 | 30 | 0.8693 |
59
 
60
 
61
  ### Framework versions
 
19
  # zephyr-7b-sft-qlora
20
 
21
  This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the generator dataset.
 
 
22
 
23
  ## Model description
24
 
 
42
  - eval_batch_size: 8
43
  - seed: 42
44
  - distributed_type: multi-GPU
45
+ - num_devices: 2
46
  - gradient_accumulation_steps: 2
47
+ - total_train_batch_size: 16
48
+ - total_eval_batch_size: 16
49
  - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
50
  - lr_scheduler_type: cosine
51
  - lr_scheduler_warmup_ratio: 0.1
 
53
 
54
  ### Training results
55
 
 
 
 
56
 
57
 
58
  ### Framework versions
adapter_config.json CHANGED
@@ -20,13 +20,13 @@
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
23
- "up_proj",
24
- "gate_proj",
25
  "o_proj",
26
- "q_proj",
27
- "v_proj",
28
  "k_proj",
29
- "down_proj"
 
 
 
 
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
 
20
  "rank_pattern": {},
21
  "revision": null,
22
  "target_modules": [
 
 
23
  "o_proj",
 
 
24
  "k_proj",
25
+ "v_proj",
26
+ "q_proj",
27
+ "down_proj",
28
+ "gate_proj",
29
+ "up_proj"
30
  ],
31
  "task_type": "CAUSAL_LM",
32
  "use_dora": false,
all_results.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "epoch": 0.9836065573770492,
3
  "total_flos": 2.0918103256858624e+16,
4
- "train_loss": 0.7164839665095012,
5
- "train_runtime": 2352.8338,
6
  "train_samples": 153,
7
- "train_samples_per_second": 0.104,
8
- "train_steps_per_second": 0.013
9
  }
 
1
  {
2
  "epoch": 0.9836065573770492,
3
  "total_flos": 2.0918103256858624e+16,
4
+ "train_loss": 0.0,
5
+ "train_runtime": 0.1537,
6
  "train_samples": 153,
7
+ "train_samples_per_second": 1587.386,
8
+ "train_steps_per_second": 97.585
9
  }
runs/Jul31_16-12-24_VM-146-67-centos/events.out.tfevents.1722413635.VM-146-67-centos.3345190.0 ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:367cd8de149001e0845de742d2affa7df6ae447e56e0fbeecc30b57ae56a8003
3
+ size 5290
train_results.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "epoch": 0.9836065573770492,
3
  "total_flos": 2.0918103256858624e+16,
4
- "train_loss": 0.7164839665095012,
5
- "train_runtime": 2352.8338,
6
  "train_samples": 153,
7
- "train_samples_per_second": 0.104,
8
- "train_steps_per_second": 0.013
9
  }
 
1
  {
2
  "epoch": 0.9836065573770492,
3
  "total_flos": 2.0918103256858624e+16,
4
+ "train_loss": 0.0,
5
+ "train_runtime": 0.1537,
6
  "train_samples": 153,
7
+ "train_samples_per_second": 1587.386,
8
+ "train_steps_per_second": 97.585
9
  }
trainer_state.json CHANGED
@@ -57,26 +57,18 @@
57
  "loss": 0.7609,
58
  "step": 30
59
  },
60
- {
61
- "epoch": 0.9836065573770492,
62
- "eval_loss": 0.8693358898162842,
63
- "eval_runtime": 83.8887,
64
- "eval_samples_per_second": 0.322,
65
- "eval_steps_per_second": 0.048,
66
- "step": 30
67
- },
68
  {
69
  "epoch": 0.9836065573770492,
70
  "step": 30,
71
  "total_flos": 2.0918103256858624e+16,
72
- "train_loss": 0.7164839665095012,
73
- "train_runtime": 2352.8338,
74
- "train_samples_per_second": 0.104,
75
- "train_steps_per_second": 0.013
76
  }
77
  ],
78
  "logging_steps": 5,
79
- "max_steps": 30,
80
  "num_input_tokens_seen": 0,
81
  "num_train_epochs": 1,
82
  "save_steps": 100,
 
57
  "loss": 0.7609,
58
  "step": 30
59
  },
 
 
 
 
 
 
 
 
60
  {
61
  "epoch": 0.9836065573770492,
62
  "step": 30,
63
  "total_flos": 2.0918103256858624e+16,
64
+ "train_loss": 0.0,
65
+ "train_runtime": 0.1537,
66
+ "train_samples_per_second": 1587.386,
67
+ "train_steps_per_second": 97.585
68
  }
69
  ],
70
  "logging_steps": 5,
71
+ "max_steps": 15,
72
  "num_input_tokens_seen": 0,
73
  "num_train_epochs": 1,
74
  "save_steps": 100,
training_args.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:5e3dffbbb2e51788d66314121aab4411a1ba9ce6e1dec87a06fcfbb5016f93a7
3
  size 5240
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ebd7d984fa09a666a5006533c7a63743f2fdeb9cface8072d79678f1c5ed4a9f
3
  size 5240