iankoulski/nixl-tests repository overview

# Production-optimized slim builds (~17GB each)
./build-all-slim.sh

# Standard runtime builds (~25GB each)
./build-all-runtime.sh

# H100 (default)
./build.sh

# A100
CUDA_ARCH=80 CUDA_ARCH_NAME=A100 ./build.sh

# A10G
CUDA_ARCH=86 CUDA_ARCH_NAME=A10G ./build.sh

# H100 slim (recommended for production)
BUILD_TARGET=slim CUDA_ARCH=90 ./build_vllm.sh

# A100 runtime
CUDA_ARCH=80 CUDA_ARCH_NAME=A100 ./build_vllm.sh

# A10G slim
BUILD_TARGET=slim CUDA_ARCH=86 CUDA_ARCH_NAME=A10G ./build_vllm.sh

# H100 slim (recommended for production)
BUILD_TARGET=slim CUDA_ARCH=90 ./build_trtllm.sh

# A100 runtime
CUDA_ARCH=80 CUDA_ARCH_NAME=A100 ./build_trtllm.sh

# A10G slim
BUILD_TARGET=slim CUDA_ARCH=86 CUDA_ARCH_NAME=A10G ./build_trtllm.sh

./build_vllm.sh  # 10-15 minutes

USE_SOURCE_BUILD=true MAX_JOBS=8 ./build_vllm.sh  # 60-90 minutes

CUDA_ARCH=90              # GPU architecture (90=H100, 80=A100, 86=A10G)
CUDA_ARCH_NAME=H100       # GPU name for environment variables
INSTALL_NCCL=1            # Install NCCL (1=yes, 0=no)
INSTALL_NVSHMEM=0         # Install NVSHMEM (1=yes, 0=no)
NPROC=12                  # Parallel build jobs (12 recommended for 16-core)
TAG=production            # Docker tag

CUDA_ARCH=90              # GPU architecture
CUDA_ARCH_NAME=H100       # GPU name
BUILD_TARGET=runtime      # Container target (runtime/slim/dev)
USE_SOURCE_BUILD=false    # Set to "true" to build vLLM from source
MAX_JOBS=12               # Parallel jobs (for source builds only)
TAG=dynamo-vllm:latest    # Docker tag

CUDA_ARCH=90              # GPU architecture
CUDA_ARCH_NAME=H100       # GPU name
BUILD_TARGET=runtime      # Container target (runtime/slim/dev)
TAG=dynamo-trtllm:latest  # Docker tag

# Run vLLM container
docker run -it --rm --gpus all dynamo-vllm:slim

# Run TensorRT-LLM container
docker run -it --rm --gpus all dynamo-trtllm:slim

# Start vLLM server
docker run -it --gpus all -p 8000:8000 dynamo-vllm:slim \
  vllm serve meta-llama/Llama-2-7b-hf \
  --host 0.0.0.0 --port 8000

# Test the server
curl http://localhost:8000/v1/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta-llama/Llama-2-7b-hf",
    "prompt": "San Francisco is",
    "max_tokens": 50
  }'

# Full EFA setup with networking
docker run --gpus all --net=host --privileged \
  -v /dev/infiniband:/dev/infiniband \
  -it dynamo-vllm:slim

# With ETCD coordination
docker run -it --rm --gpus all \
  -e NIXL_ETCD_ENDPOINTS=http://etcd-service:2379 \
  dynamo-vllm:slim

# Tag for ECR
docker tag dynamo-vllm:slim <account-id>.dkr.ecr.<region>.amazonaws.com/dynamo-vllm:slim

# Login to ECR
aws ecr get-login-password --region <region> | docker login --username AWS --password-stdin <account-id>.dkr.ecr.<region>.amazonaws.com

# Push
docker push <account-id>.dkr.ecr.<region>.amazonaws.com/dynamo-vllm:slim

apiVersion: v1
kind: Pod
metadata:
  name: vllm-server
spec:
  containers:
  - name: vllm
    image: <account-id>.dkr.ecr.<region>.amazonaws.com/dynamo-vllm:slim
    resources:
      limits:
        nvidia.com/gpu: 1
    command: ["vllm", "serve", "meta-llama/Llama-2-7b-hf", "--host", "0.0.0.0"]

CUDAARCHS=90                        # Set by build (90/80/86)
CUDA_ARCH_NAME=H100                 # Set by build (H100/A100/A10G)
CMAKE_CUDA_ARCHITECTURES=90         # For CMake builds
TORCH_CUDA_ARCH_LIST=9.0+PTX        # For PyTorch (9.0/8.0/8.6)
CUDA_HOME=/usr/local/cuda

NIXL_PREFIX=/usr/local/nixl
NIXL_PLUGIN_DIR=/usr/local/nixl/lib/x86_64-linux-gnu/plugins
NIXL_ETCD_NAMESPACE=/nixl/agents
NIXL_ETCD_ENDPOINTS=http://etcd-service:2379  # Optional

FI_PROVIDER=efa
NCCL_DEBUG=INFO
NCCL_SOCKET_IFNAME=^docker,lo,veth
NVSHMEM_REMOTE_TRANSPORT=libfabric
NVSHMEM_LIBFABRIC_PROVIDER=efa

# Environment information
env-info

# EFA connectivity tests (requires EFA hardware)
efa-test

# NIXL performance benchmarks
nixlbench-test

# Container debloating (for custom optimization)
debloat-container.sh

# Test vLLM installation
python3 -c "import vllm; print(vllm.__version__)"

# Test EFA detection
fi_info -p efa

# Test NCCL with EFA
all_reduce_perf -b 8 -e 128M -f 2 -g 1

# Test NIXL Python bindings
python3 -c "import nixl; print(dir(nixl))"

# Check EFA hardware
fi_info -p efa

# Verify EFA driver
cat /sys/class/infiniband/*/device/vendor

# Check device mounting
ls -la /dev/infiniband/

# Ensure container has EFA access
docker run --rm --privileged -v /dev/infiniband:/dev/infiniband \
  dynamo-vllm:slim fi_info -p efa

# Run NCCL tests
all_reduce_perf -b 8 -e 128M -f 2 -g 1

# Enable debug output
export NCCL_DEBUG=INFO

# Check EFA plugin
ls -la /usr/local/lib/libnccl-net.so

# Reduce parallel jobs
MAX_JOBS=8 ./build_vllm.sh

# Or for base builds
NPROC=8 ./build.sh

# Use slim target
BUILD_TARGET=slim ./build_vllm.sh  # Saves 8GB

# Ensure using pip install (default)
./build_vllm.sh  # Should be fast (10-15 min)

# If accidentally using source build:
USE_SOURCE_BUILD=false ./build_vllm.sh

# Check GPU
nvidia-smi

# Test Docker GPU support
docker run --rm --gpus all nvidia/cuda:12.8-runtime-ubuntu24.04 nvidia-smi

# Ensure NVIDIA Container Toolkit is installed
dpkg -l | grep nvidia-container-toolkit

# Direct docker build for A100 slim vLLM
docker build \
  --build-arg CUDA_ARCH=80 \
  --build-arg CUDA_ARCH_NAME=A100 \
  --target slim \
  -f Dockerfile.dynamo-vllm \
  -t dynamo-vllm:a100-slim \
  .

# Build with custom tag
BUILD_TARGET=slim TAG=vllm:prod-v1 ./build_vllm.sh

# Build all architectures
for arch in "80:A100" "86:A10G" "90:H100"; do
  IFS=':' read -r cuda_arch name <<< "$arch"
  BUILD_TARGET=slim \
    CUDA_ARCH=$cuda_arch \
    CUDA_ARCH_NAME=$name \
    TAG=dynamo-vllm:$name-slim \
    ./build_vllm.sh
done

dynamo-workshop/
├── Dockerfile.production              # Base NIXL+EFA container
├── Dockerfile.dynamo-vllm            # vLLM container
├── Dockerfile.dynamo-trtllm          # TensorRT-LLM container
├── build.sh                          # Build production base
├── build_vllm.sh                     # Build vLLM container
├── build_trtllm.sh                   # Build TensorRT-LLM container
├── build-all-slim.sh                 # Build all (optimized)
├── build-all-runtime.sh              # Build all (standard)
├── README.md                         # This file
├── DEBLOAT_GUIDE.md                  # Size optimization guide
├── LICENSE                           # Apache 2.0
├── ATTRIBUTION.md                    # Credits
├── nvidia_entrypoint.sh              # Container entrypoint
├── benchmarks/                       # Performance benchmarks
├── container/                        # Container dependencies
│   ├── deps/                         # Python requirements
│   └── nvidia_entrypoint.sh          # Entrypoint script
├── pkg-config-files/                 # Build dependencies
│   ├── efa.pc                        # EFA pkg-config
│   └── gdrcopy.pc                    # GDRCopy pkg-config
└── scripts/                          # Utility scripts
    ├── debloat-container.sh          # Size optimization
    ├── efa-test.sh                   # EFA testing
    ├── env-info.sh                   # Environment info
    └── nixlbench-test.sh             # Benchmarking