Sumkh commited on
Commit
b99c6a9
·
verified ·
1 Parent(s): fc124f8

Update start.sh

Browse files
Files changed (1) hide show
  1. start.sh +1 -0
start.sh CHANGED
@@ -1,6 +1,7 @@
1
  #!/bin/bash
2
  # Start vLLM server in the background with your custom flags
3
  vllm \
 
4
  --model unsloth/llama-3-8b-Instruct-bnb-4bit \
5
  --enable-auto-tool-choice \
6
  --tool-call-parser llama3_json \
 
1
  #!/bin/bash
2
  # Start vLLM server in the background with your custom flags
3
  vllm \
4
+ --runtime nvidia --gpus all --ipc=host \
5
  --model unsloth/llama-3-8b-Instruct-bnb-4bit \
6
  --enable-auto-tool-choice \
7
  --tool-call-parser llama3_json \