Hermes
3c92d93366
- Add custom llama.cpp Dockerfile with ROCm 6.1 + gfx906 (MI50) build - Add llama-cpp-hermes service serving Hermes 4.3 on dual MI50 GPUs - Strip GPU devices/ROCm env from ollama service (CPU-only for embeddings) Hermes 4.3 runs at ~19 t/s on dual MI50s with 160K context.
31 lines
1.5 KiB
Docker
31 lines
1.5 KiB
Docker
# llama-cpp-rocm6/Dockerfile
|
|
# Custom llama.cpp server with ROCm 6.1 + gfx906 (MI50) support.
|
|
# Build: docker build -t llama-cpp:rocm-gfx906 .
|
|
|
|
FROM rocm/dev-ubuntu-22.04:6.1.2-complete AS builder
|
|
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl git build-essential pkg-config cmake make && rm -rf /var/lib/apt/lists/*
|
|
ARG LLAMACPP_VERSION=b9596
|
|
RUN git clone --depth 1 --branch ${LLAMACPP_VERSION} https://github.com/ggml-org/llama.cpp.git /build
|
|
WORKDIR /build
|
|
ENV HIP_PATH=/opt/rocm ROCM_PATH=/opt/rocm PATH=/opt/rocm/bin:/opt/rocm/llvm/bin:${PATH} CMAKE_PREFIX_PATH=/opt/rocm
|
|
RUN mkdir build && cd build && \
|
|
cmake .. -DGGML_HIP=ON -DCMAKE_BUILD_TYPE=Release \
|
|
-DAMDGPU_TARGETS="gfx906:xnack-" \
|
|
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
|
|
-DGGML_CUDA=OFF -DGGML_VULKAN=OFF -DGGML_METAL=OFF \
|
|
-DBUILD_SHARED_LIBS=OFF && \
|
|
cmake --build . --target llama-server -- -j $(nproc)
|
|
|
|
FROM ubuntu:24.04
|
|
RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
|
|
ca-certificates curl libstdc++6 libgomp1 libopenblas0 \
|
|
libnuma1 libelf1 libdrm2 libdrm-amdgpu1 \
|
|
&& rm -rf /var/lib/apt/lists/*
|
|
COPY --from=builder /opt/rocm/lib/ /opt/rocm/lib/
|
|
COPY --from=builder /opt/rocm/share/ /opt/rocm/share/
|
|
COPY --from=builder /build/build/bin/llama-server /usr/local/bin/llama-server
|
|
RUN echo /opt/rocm/lib > /etc/ld.so.conf.d/rocm.conf && ldconfig
|
|
ENV HSA_OVERRIDE_GFX_VERSION=9.0.6 HCC_AMDGPU_TARGET=gfx906 HSA_ENABLE_SDMA=0
|
|
EXPOSE 8080
|
|
ENTRYPOINT ["/usr/local/bin/llama-server"]
|