Update README.md
Browse files
README.md
CHANGED
|
@@ -128,6 +128,8 @@ ecipe = QuantizationModifier(
|
|
| 128 |
|
| 129 |
# Apply quantization.
|
| 130 |
oneshot(model=model, recipe=recipe)
|
|
|
|
|
|
|
| 131 |
|
| 132 |
# Save to disk in compressed-tensors format.
|
| 133 |
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-FP8-block"
|
|
|
|
| 128 |
|
| 129 |
# Apply quantization.
|
| 130 |
oneshot(model=model, recipe=recipe)
|
| 131 |
+
dispatch_for_generation(model)
|
| 132 |
+
|
| 133 |
|
| 134 |
# Save to disk in compressed-tensors format.
|
| 135 |
SAVE_DIR = MODEL_ID.rstrip("/").split("/")[-1] + "-FP8-block"
|