diff --git a/.github/workflows/code-scans.yaml b/.github/workflows/code-scans.yaml index a94b5dd0..14046c9d 100644 --- a/.github/workflows/code-scans.yaml +++ b/.github/workflows/code-scans.yaml @@ -52,7 +52,7 @@ jobs: continue-on-error: true with: scan-type: 'image' - image-ref: 'public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.10.2' + image-ref: 'vllm/vllm-openai-cpu:v0.24.0' severity: 'HIGH,CRITICAL' exit-code: 0 format: 'table' diff --git a/core/helm-charts/vllm/values.yaml b/core/helm-charts/vllm/values.yaml index 0a3d5df1..264806d7 100644 --- a/core/helm-charts/vllm/values.yaml +++ b/core/helm-charts/vllm/values.yaml @@ -21,11 +21,11 @@ accelDevice: "" port: 2080 shmSize: 1Gi image: - repository: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo + repository: vllm/vllm-openai-cpu # Uncomment the following line to set desired image pull policy if needed, as one of Always, IfNotPresent, Never. # pullPolicy: "" # Overrides the image tag whose default is the chart appVersion. - tag: "v0.10.2" + tag: "v0.24.0" imagePullSecrets: [] nameOverride: "" diff --git a/core/helm-charts/vllm/xeon-values.yaml b/core/helm-charts/vllm/xeon-values.yaml index 77d92bf1..61944121 100644 --- a/core/helm-charts/vllm/xeon-values.yaml +++ b/core/helm-charts/vllm/xeon-values.yaml @@ -26,8 +26,8 @@ max_model_len: 8192 # vLLM CPU image configuration image: - repository: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo - tag: "v0.10.2" + repository: vllm/vllm-openai-cpu + tag: "v0.24.0" pullPolicy: IfNotPresent # Node affinity for Xeon inference nodes @@ -58,6 +58,8 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" HF_HUB_DISABLE_XET: "1" VLLM_CPU_SGL_KERNEL: "1" + USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -74,7 +76,6 @@ modelConfigs: "2048", "--max-num-seqs", "256", - "--disable-log-requests", "--enable-auto-tool-choice", "--tool-call-parser", "llama3_json", ] @@ -90,6 +91,8 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -121,6 +124,8 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -137,7 +142,6 @@ modelConfigs: "2048", "--max-num-seqs", "256", - "--disable-log-requests", ] tensor_parallel_size: "{{ .Values.tensor_parallel_size }}" pipeline_parallel_size: "{{ .Values.pipeline_parallel_size }}" @@ -151,6 +155,8 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -167,7 +173,6 @@ modelConfigs: "2048", "--max-num-seqs", "256", - "--disable-log-requests", ] tensor_parallel_size: "{{ .Values.tensor_parallel_size }}" pipeline_parallel_size: "{{ .Values.pipeline_parallel_size }}" @@ -181,6 +186,8 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -212,6 +219,8 @@ modelConfigs: VLLM_CPU_NUM_OF_RESERVED_CPU: "0" VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -245,6 +254,8 @@ modelConfigs: VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" LOGNAME: "vllm" + USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", @@ -274,8 +285,9 @@ defaultModelConfigs: VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" VLLM_ENGINE_ITERATION_TIMEOUT_S: "120" VLLM_CPU_NUM_OF_RESERVED_CPU: "0" - VLLM_CPU_SGL_KERNEL: "1" HF_HUB_DISABLE_XET: "1" + USER: "vllm" + TRITON_CACHE_DIR: "/tmp/.triton" extraCmdArgs: [ "--block-size", diff --git a/core/roles/inference-tools/tasks/main.yml b/core/roles/inference-tools/tasks/main.yml index 12801161..51ae0eec 100644 --- a/core/roles/inference-tools/tasks/main.yml +++ b/core/roles/inference-tools/tasks/main.yml @@ -19,7 +19,7 @@ ansible.builtin.pip: name: kubernetes state: present - extra_args: "--break-system-packages" + extra_args: "--break-system-packages --ignore-installed" become: true when: pip_install_result is failed tags: always diff --git a/core/scripts/vllm-quickstart/README.md b/core/scripts/vllm-quickstart/README.md index 7dba3473..3a0977c3 100644 --- a/core/scripts/vllm-quickstart/README.md +++ b/core/scripts/vllm-quickstart/README.md @@ -116,7 +116,7 @@ The `models.json` file contains all configuration: ```json { "docker": { - "image": "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.11.2", + "image": "vllm/vllm-openai-cpu:v0.24.0", "port": "8000:8000", "environment": { ... }, "volumes": [ ... ] diff --git a/core/scripts/vllm-quickstart/models.json b/core/scripts/vllm-quickstart/models.json index e7e24eff..330bbe3f 100644 --- a/core/scripts/vllm-quickstart/models.json +++ b/core/scripts/vllm-quickstart/models.json @@ -1,6 +1,6 @@ { "docker": { - "image": "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.11.2", + "image": "vllm/vllm-openai-cpu:v0.24.0", "port": "8000:8000", "environment": { "VLLM_CPU_SGL_KERNEL": "1", diff --git a/docs/deploy-llm-model-from-hugging-face.md b/docs/deploy-llm-model-from-hugging-face.md index 7c104bdd..545f0c06 100644 --- a/docs/deploy-llm-model-from-hugging-face.md +++ b/docs/deploy-llm-model-from-hugging-face.md @@ -20,3 +20,26 @@ This option allows you to deploy any Hugging Face-hosted LLM on the Inference Cl - **Tensor Parallel Size** (based on available IntelĀ® AI Accelerator cards) > **Note**: This deploys a model that has **not** been pre-validated. Make sure the tensor parallel size is configured correctly. An incorrect value can result in the model being stuck in a "not ready" state. + +### Customizing Environment Variables and vLLM Arguments + +Models deployed from Hugging Face use the `defaultModelConfigs` settings in `core/helm-charts/vllm/xeon-values.yaml`. You can customize environment variables and vLLM command-line arguments by editing the `defaultModelConfigs` section of that file before deploying. + +**Example: enabling the SGL kernel, configuring KV cache, and other environment variables** + +```yaml +defaultModelConfigs: + configMapValues: + VLLM_CPU_SGL_KERNEL: "1" # Enable the SGL kernel for improved CPU performance + VLLM_CPU_KVCACHE_SPACE: "40" # KV cache memory allocation in GB + # Add or override any other environment variables here + extraCmdArgs: + [ + "--block-size", "128", + "--dtype", "bfloat16", + "--max-model-len", "8192", + # Add or override any vLLM CLI arguments here + ] +``` + +You can set any vLLM-supported environment variable under `configMapValues` and any vLLM CLI flag under `extraCmdArgs`.