NIM 快速开始

# Set HF_TOKEN for downloading HuggingFace repository
export HF_TOKEN=hf_xxxxxx

# Choose a HuggingFace model 
export NIM_MODEL_NAME=hf://google/gemma-3-27b-it

# Choose a served model name 
export NIM_SERVED_MODEL_NAME=google/gemma-3-27b-it

# Choose a path on your system to cache the downloaded models
export LOCAL_NIM_CACHE=~/.cache/nim
mkdir -p "$LOCAL_NIM_CACHE"

# Add write permissions to the NIM cache for downloading model assets
chmod -R a+w "$LOCAL_NIM_CACHE"

# Start the LLM NIM
docker run -it --rm --name=LLM-NIM \
  --runtime=nvidia \
  --gpus all \
  --shm-size=16GB \
  -e HF_TOKEN=$HF_TOKEN \
  -e NIM_MODEL_NAME=$NIM_MODEL_NAME \
  -e NIM_SERVED_MODEL_NAME=$NIM_SERVED_MODEL_NAME \
  -v "$LOCAL_NIM_CACHE:/opt/nim/.cache" \
  -u $(id -u) \
  -p 8000:8000 \
  nvcr.io/nim/nvidia/llm-nim:latest
   
# docker run -it --rm --name=LLM-NIM   --runtime=nvidia   --gpus '"device=1"'   --shm-size=16GB   -e HF_TOKEN=$HF_TOKEN   -e NIM_MODEL_NAME=/mnt/models   -e NIM_SERVED_MODEL_NAME=qwen3   -e NIM_MAX_IMAGES_PER_PROMPT=0   -v "/root/Models/NIM-cache:/opt/nim/.cache" -v "/root/Models/LLMs/Qwen3-1.7B:/mnt/models"  -u $(id -u)   -p 8000:8000   nvcr.io/nim/nvidia/llm-nim:latest

# 选择镜像（以官方列出的支持版本为准）
export NIM_IMAGE=nvcr.io/nim/meta/llama-3.1-8b-instruct:1.13.1

# 启动容器（仅文本；关闭图像输入）
docker run -it --rm --name=llm-nim-llama31-8b \
  --runtime=nvidia --gpus all \
  --shm-size=16GB \
  -e NGC_API_KEY=$NGC_API_KEY \
  -u $(id -u) \
  -p 8000:8000 \
  nvcr.io/nim/meta/llama-3.1-8b-instruct:1.13.1

export NGC_API_KEY=<PASTE_API_KEY_HERE>
export LOCAL_NIM_CACHE=~/.cache/nim
mkdir -p "$LOCAL_NIM_CACHE"
docker run -it --rm \
    --gpus all \
    --shm-size=16GB \
    -e NGC_API_KEY \
    -v "$LOCAL_NIM_CACHE:/opt/nim/.cache" \
    -u $(id -u) \
    -p 8000:8000 \
    nvcr.io/nim/snowflake/arctic-embed-l:1.0.1

export NGC_API_KEY=<PASTE_API_KEY_HERE>
export LOCAL_NIM_CACHE=~/.cache/nim
mkdir -p "$LOCAL_NIM_CACHE"
docker run -it --rm \
    --gpus all \
    --shm-size=16GB \
    -e NGC_API_KEY \
    -v "$LOCAL_NIM_CACHE:/opt/nim/.cache" \
    -u $(id -u) \
    -p 8000:8000 \
    nvcr.io/nim/baai/bge-m3:latest

curl -s http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta/llama-3.1-8b-instruct",
    "messages": [
      {"role": "system", "content": "You are a helpful assistant."},
      {"role": "user", "content": "用三句话解释什么是NIM。"}
    ],
    "temperature": 0.3,
    "stream": false
  }'

curl -s http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "meta/llama-3.1-8b-instruct",
    "messages": [
      {"role": "user", "content": "查询上海当前天气并给出穿衣建议。"}
    ],
    "tools": [
      {
        "type": "function",
        "function": {
          "name": "get_weather",
          "description": "Get current weather by city name.",
          "parameters": {
            "type": "object",
            "properties": {
              "city": {"type": "string"}
            },
            "required": ["city"]
          }
        }
      }
    ],
    "tool_choice": "auto"
  }'

#!/bin/bash

# Gather parts in alpha order
# 启用 Bash 的 **扩展通配符** 功能（`extglob`）和 **空匹配返回空数组** 功能
shopt -s nullglob extglob
# 获取当前脚本所在目录的绝对路径。
_SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
# 收集 `entrypoint.d/` 下的所有 `.txt` 或 `.sh` 文件，按字母顺序排序存入数组。
declare -a _PARTS=( "${_SCRIPT_DIR}/entrypoint.d"/*@(.txt|.sh) )
shopt -u nullglob extglob

# 辅助函数：打印指定字符若干次（常用于边框线）。
print_repeats() {
  local -r char="$1" count="$2"
  local i
  for ((i=1; i<=$count; i++)); do echo -n "$char"; done
  echo
}
# 辅助函数：以指定字符生成一个包围文字的横幅样式输出，用于美观的日志输出。
print_banner_text() {
  # $1: Banner char
  # $2: Text
  local banner_char=$1
  local -r text="$2"
  local pad="${banner_char}${banner_char}"
  print_repeats "${banner_char}" $((${#text} + 6))
  echo "${pad} ${text} ${pad}"
  print_repeats "${banner_char}" $((${#text} + 6))
}

# Execute the entrypoint 
# 遍历 `_PARTS` 数组中的文件；若为 `.txt` 文件：直接输出其内容；若为 `.sh` 文件：使用 `source` 执行（在当前进程中执行，而非子进程）；这样能共享变量、函数、环境配置。
for _file in "${_PARTS[@]}"; do
  case "${_file}" in
    *.txt) cat "${_file}";;
    *.sh)  source "${_file}";;
  esac
done

echo

# This script can either be a wrapper around arbitrary command lines,
# or it will simply exec bash if no arguments were given
# 无参数启动时：进入交互式 Bash；
# 有参数启动时：用 `exec` 替换当前进程执行指定命令。
if [[ $# -eq 0 ]]; then
  exec "/bin/bash"
else
  exec "$@"
fi

#!/bin/bash

# Gather parts in alpha order
shopt -s nullglob extglob
_SCRIPT_DIR="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")"
declare -a _PARTS=( "${_SCRIPT_DIR}/entrypoint.d"/*@(.txt|.sh) )
shopt -u nullglob extglob

print_repeats() {
  local -r char="$1" count="$2"
  local i
  for ((i=1; i<=$count; i++)); do echo -n "$char"; done
  echo
}

print_banner_text() {
  # $1: Banner char
  # $2: Text
  local banner_char=$1
  local -r text="$2"
  local pad="${banner_char}${banner_char}"
  print_repeats "${banner_char}" $((${#text} + 6))
  echo "${pad} ${text} ${pad}"
  print_repeats "${banner_char}" $((${#text} + 6))
}

# Execute the entrypoint parts
for _file in "${_PARTS[@]}"; do
  case "${_file}" in
    *.txt) cat "${_file}";;
    *.sh)  source "${_file}";;
  esac
done

echo

# This script can either be a wrapper around arbitrary command lines,
# or it will simply exec bash if no arguments were given
if [[ $# -eq 0 ]]; then
  exec "/bin/bash"
else
  exec "$@"
fi
root@c7a38a1d075a:/# cat /opt/nim/start_server.sh
#!/bin/bash
set -u
export VLLM_CONFIGURE_LOGGING=1

NIM_LOG_LEVEL="${NIM_LOG_LEVEL:-DEFAULT}"

case "${NIM_LOG_LEVEL}" in
  "TRACE")
    export TLLM_LOG_LEVEL="TRACE"
    export VLLM_NVEXT_LOG_LEVEL="DEBUG"
    export UVICORN_LOG_LEVEL="trace"
    ;;
  "DEBUG")
    export TLLM_LOG_LEVEL="DEBUG"
    export VLLM_NVEXT_LOG_LEVEL="DEBUG"
    export UVICORN_LOG_LEVEL="debug"
    ;;
  "INFO")
    export TLLM_LOG_LEVEL="INFO"
    export VLLM_NVEXT_LOG_LEVEL="INFO"
    export UVICORN_LOG_LEVEL="info"
    ;;
  "DEFAULT")
    export TLLM_LOG_LEVEL="ERROR"
    export VLLM_NVEXT_LOG_LEVEL="INFO"
    export UVICORN_LOG_LEVEL="info"
    ;;
  "WARNING")
    export TLLM_LOG_LEVEL="WARNING"
    export VLLM_NVEXT_LOG_LEVEL="WARNING"
    export UVICORN_LOG_LEVEL="warning"
    ;;
  "ERROR")
    export TLLM_LOG_LEVEL="ERROR"
    export VLLM_NVEXT_LOG_LEVEL="ERROR"
    export UVICORN_LOG_LEVEL="error"
    ;;
  "CRITICAL")
    export TLLM_LOG_LEVEL="ERROR"
    export VLLM_NVEXT_LOG_LEVEL="CRITICAL"
    export UVICORN_LOG_LEVEL="critical"
    ;;
  *)
    echo "Unsupported value ('${NIM_LOG_LEVEL}') of NIM_LOG_LEVEL. Supported values are: TRACE, DEBUG, INFO, DEFAULT, WARNING, ERROR, CRITICAL." >&2
    exit 1
    ;;
esac

# 优雅关闭（Graceful Shutdown）机制
# 使用 `pgrep -x "pt_main_thread"` 查找主推理线程进程 PID；
#  1. 向子进程发送 `SIGTERM`；
#  2. 等待 5 秒；
#  3. 若仍存活，则发送 `SIGKILL` 强制终止；
#  4. 使用 `wait` 等待清理；
#  最后 `exit 0` 确保容器正常退出。
function terminate() {
  NIM_LLM_PID=$(pgrep -x "pt_main_thread")
  echo "Captured NIM_LLM_PID: ${NIM_LLM_PID:-0}"
  echo "$0 received SIGTERM, attempting graceful shutdown..."
  if [ -z "$NIM_LLM_PID" ]
  then
    echo "$0 has no NIM_LLM process to terminate"
    exit 0
  fi
  # Attempt graceful shutdown.
  echo "Attempting to shutdown process ${NIM_LLM_PID:-0}"
  kill -TERM "$NIM_LLM_PID"
  sleep 5 # Wait 5 seconds for graceful shutdown
  # Check if the child process is still running by sending signal 0.
  if kill -0 "$NIM_LLM_PID" 2>/dev/null; then
    echo "$0 graceful shutdown failed, forcefully killing process..."
    kill -9 "$NIM_LLM_PID"
  fi
  wait "$NIM_LLM_PID"
  exit 0
}
# Trap SIGINT or SIGTERM and call the terminate function.
trap terminate SIGINT SIGTERM

jsonl_config="/etc/nim/config/python_jsonl_logging_config.json"
readable_config="/etc/nim/config/python_readable_logging_config.json"
NIM_JSONL_LOGGING="${NIM_JSONL_LOGGING:-0}"
if [ "${NIM_JSONL_LOGGING}" = 1 ]; then
  export VLLM_LOGGING_CONFIG_PATH="${jsonl_config}"
  export VLLM_NVEXT_LOGGING_CONFIG_PATH="${jsonl_config}"
  python3 -m nim_llm_sdk.entrypoints.launch |& python3 -m nim_llm_sdk.logging.pack_all_logs_into_json
elif [ "${NIM_JSONL_LOGGING}" = 0 ]; then
  export VLLM_LOGGING_CONFIG_PATH="${readable_config}"
  export VLLM_NVEXT_LOGGING_CONFIG_PATH="${readable_config}"
  python3 -m nim_llm_sdk.entrypoints.launch
else
  echo "ERROR: Unsupported value ('${NIM_JSONL_LOGGING}') of NIM_JSONL_LOGGING env variable. Supported values are 0 and 1" >&2
  exit 1
fi

root@c7a38a1d075a:/opt/nim/llm/nim_llm_sdk# tree .
.
|-- __init__.py
|-- constants.py
|-- engine
|   |-- __init__.py
|   |-- async_trtllm_engine.py
|   |-- async_trtllm_engine_factory.py
|   |-- metrics.py
|   |-- trtllm_errors.py
|   `-- trtllm_model_runner.py
|-- entrypoints
|   |-- __init__.py
|   |-- args.py
|   |-- launch.py
|   |-- llamastack
|   |   |-- protocol.py
|   |   |-- serving_chat.py
|   |   |-- serving_completion.py
|   |   `-- serving_engine.py
|   `-- openai
|       |-- __init__.py
|       |-- api_extensions.py
|       `-- api_server.py
|-- env_utils.py
|-- envs.py
|-- hub
|   |-- __init__.py
|   |-- hardware_inspect.py
|   |-- info.py
|   |-- local_cache_manager
|   |   `-- local_cache.py
|   |-- ngc_download.py
|   |-- ngc_injector.py
|   |-- ngc_profile.py
|   |-- pre_download.py
|   |-- utils.py
|   `-- workspace.py
|-- logger.py
|-- logging
|   |-- __init__.py
|   |-- const.py
|   |-- json_formatter.py
|   `-- pack_all_logs_into_json.py
|-- lora
|   |-- __init__.py
|   `-- source.py
|-- mem_estimators
|   |-- __init__.py
|   |-- base.py
|   |-- mem_estimator.py
|   `-- utils.py
|-- model_specific_modules
|   |-- README.md
|   |-- __init__.py
|   `-- dynamic_module_loader.py
|-- perf
|   |-- __init__.py
|   |-- benchmark.py
|   |-- llm_inputs
|   |   |-- __init__.py
|   |   |-- farewell.txt
|   |   |-- genai_default_synthetic_prompt_generator.py
|   |   |-- repeated_context_prompt_generator.py
|   |   |-- table.txt
|   |   |-- table_questions.txt
|   |   |-- tokenizer.py
|   |   `-- utils.py
|   |-- stats.py
|   `-- timer.py
|-- trtllm
|   |-- __init__.py
|   |-- configs.py
|   |-- parse_config.py
|   |-- request.py
|   |-- utils.py
|   `-- weight_manager
|       |-- __init__.py
|       |-- build_utils.py
|       `-- nim_optimize.py
`-- utils
    |-- __init__.py
    `-- caches.py