Dockerfile

# Use Ubuntu 22.04 as the base image
FROM ubuntu:22.04

# Install Python 3 and pip
RUN apt-get update && \
    apt-get install -y python3 python3-pip && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*

# Upgrade pip and install Python packages
RUN pip3 install --upgrade pip
RUN pip3 install torch torchvision torchaudio
RUN pip install fschat accelerate pandas numpy
RUN pip install xformers --no-deps
RUN pip3 install transformers vllm

# run this command - python -m vllm.entrypoints.openai.api_server --model mistralai/Mistral-7B-Instruct-v0.1
# to start the server

ENTRYPOINT [ "python", "-m", "vllm.entrypoints.openai.api_server", "--host", "0.0.0.0", "--port", "8080", "--model", "mistralai/Mistral-7B-Instruct-v0.1"]