From bd3726a189efc32de55900aba7328e6d850d7a68 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Tue, 12 May 2026 21:30:13 +0000 Subject: [PATCH 1/3] $Update kimik2.5-fp4-b200-vllm vLLM image to v0.20.2\n\nRef #1154\n\nCo-authored-by: Klaud Cold --- .github/configs/nvidia-master.yaml | 2 +- perf-changelog.yaml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index f1cec3c52..f0f5e7ed6 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -2500,7 +2500,7 @@ kimik2.5-int4-h200-vllm: - { tp: 8, conc-start: 4, conc-end: 64 } kimik2.5-fp4-b200-vllm: - image: vllm/vllm-openai:v0.17.0 + image: vllm/vllm-openai:v0.20.2 model: nvidia/Kimi-K2.5-NVFP4 model-prefix: kimik2.5 runner: b200 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ac8269ef2..1f974dd9b 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -2343,3 +2343,9 @@ description: - "Add Qwen3.5-397B-A17B FP8 MI355X ATOM benchmark configs with and without MTP" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1310 + +- config-keys: + - kimik2.5-fp4-b200-vllm + description: + - "Update vLLM image from v0.17.0 to v0.20.2" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/XXX From 74f16dc1ca7fa3087b6b3c35af5974155f7d2827 Mon Sep 17 00:00:00 2001 From: claude-fix-bot Date: Sun, 17 May 2026 03:48:29 -0400 Subject: [PATCH 2/3] fix(kimik2.5_fp4_b200.sh): raise --gpu-memory-utilization 0.90 -> 0.98 vLLM v0.20.2's CUDA-graph memory profiling subtracts an aggressive chunk from the requested utilization, leaving negative space for the KV cache (-39.49 GiB observed). Raising to 0.98 gives the profiler enough headroom to land KV cache positive while still keeping ~2% as hard buffer. Alternative would have been setting VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0, but raising the cap is the minimum-blast-radius fix and matches what similar B200 recipes use. Co-Authored-By: Claude Opus 4.7 (1M context) --- benchmarks/single_node/kimik2.5_fp4_b200.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/single_node/kimik2.5_fp4_b200.sh b/benchmarks/single_node/kimik2.5_fp4_b200.sh index 9c82d2865..437c1507d 100644 --- a/benchmarks/single_node/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/kimik2.5_fp4_b200.sh @@ -36,7 +36,7 @@ start_gpu_monitor set -x vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.90 \ +--gpu-memory-utilization 0.98 \ --max-model-len $MAX_MODEL_LEN \ --max-num-seqs $CONC \ --reasoning-parser kimi_k2 \ From 056017903452c2aa3fda344fdde38b9c934389db Mon Sep 17 00:00:00 2001 From: claude-fix-bot Date: Sun, 17 May 2026 21:29:48 -0400 Subject: [PATCH 3/3] fix(kimik2.5_fp4_b200.sh): disable CUDA-graph memory estimator + restore 0.90 mem-util MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raising --gpu-memory-utilization to 0.98 wasn't enough — vLLM v0.20.2's CUDA-graph memory profiler still pre-reserves ~57 GB/GPU upfront, leaving the effective utilization at ~0.66 and the KV cache at -25 GiB (engine won't start). Disable the estimator with VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 (same pattern as benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh:65) and revert --gpu-memory-utilization back to 0.90. The 0.90 reservation already leaves ~18 GB/GPU as the same safety net the estimator was trying to enforce. --- benchmarks/single_node/kimik2.5_fp4_b200.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/benchmarks/single_node/kimik2.5_fp4_b200.sh b/benchmarks/single_node/kimik2.5_fp4_b200.sh index 437c1507d..b4e85c14f 100644 --- a/benchmarks/single_node/kimik2.5_fp4_b200.sh +++ b/benchmarks/single_node/kimik2.5_fp4_b200.sh @@ -33,10 +33,17 @@ fi # Start GPU monitoring (power, temperature, clocks every second) start_gpu_monitor +# vLLM v0.20.2+'s CUDA-graph memory profiler pre-reserves ~57 GB/GPU upfront +# (~32% of total), which collides with --gpu-memory-utilization=0.90 and +# leaves negative space for the KV cache. Disable the profiler — our 0.90 +# already leaves ~18 GB/GPU as safety net (same pattern as +# benchmarks/single_node/agentic/kimik2.5_fp4_b200.sh). +export VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 + set -x vllm serve $MODEL --host 0.0.0.0 --port $PORT \ --tensor-parallel-size=$TP \ ---gpu-memory-utilization 0.98 \ +--gpu-memory-utilization 0.90 \ --max-model-len $MAX_MODEL_LEN \ --max-num-seqs $CONC \ --reasoning-parser kimi_k2 \