diff --git a/.github/configs/nvidia-master.yaml b/.github/configs/nvidia-master.yaml index cfc8e87ab..43b8be880 100644 --- a/.github/configs/nvidia-master.yaml +++ b/.github/configs/nvidia-master.yaml @@ -2454,7 +2454,7 @@ qwen3.5-fp8-b300-sglang: - { tp: 4, ep: 1, conc-start: 4, conc-end: 256 } qwen3.5-fp4-b300-sglang: - image: lmsysorg/sglang:v0.5.11-cu130 + image: lmsysorg/sglang:v0.5.12-cu130 model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: b300 @@ -2475,7 +2475,7 @@ qwen3.5-fp4-b300-sglang: - { tp: 2, ep: 2, conc-start: 4, conc-end: 128 } qwen3.5-fp4-b300-sglang-mtp: - image: lmsysorg/sglang:v0.5.11-cu130 + image: lmsysorg/sglang:v0.5.12-cu130 model: nvidia/Qwen3.5-397B-A17B-NVFP4 model-prefix: qwen3.5 runner: b300 diff --git a/benchmarks/single_node/qwen3.5_fp4_b300.sh b/benchmarks/single_node/qwen3.5_fp4_b300.sh index e3ae6a6e4..18b6cda09 100755 --- a/benchmarks/single_node/qwen3.5_fp4_b300.sh +++ b/benchmarks/single_node/qwen3.5_fp4_b300.sh @@ -73,7 +73,7 @@ PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0. --cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ --mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ --context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --moe-runner-backend flashinfer_trtllm \ +--attention-backend trtllm_mha --mm-attention-backend triton_attn --moe-runner-backend flashinfer_trtllm \ $EXTRA_ARGS --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ --tokenizer-worker-num 6 --stream-interval 30 > $SERVER_LOG 2>&1 & diff --git a/benchmarks/single_node/qwen3.5_fp4_b300_mtp.sh b/benchmarks/single_node/qwen3.5_fp4_b300_mtp.sh index 033c0408a..9cb5d5464 100755 --- a/benchmarks/single_node/qwen3.5_fp4_b300_mtp.sh +++ b/benchmarks/single_node/qwen3.5_fp4_b300_mtp.sh @@ -73,7 +73,7 @@ PYTHONNOUSERSITE=1 python3 -m sglang.launch_server --model-path=$MODEL --host=0. --cuda-graph-max-bs $CUDA_GRAPH_MAX_BATCH_SIZE --max-running-requests $MAX_RUNNING_REQUESTS \ --mem-fraction-static $MEM_FRAC_STATIC --chunked-prefill-size $CHUNKED_PREFILL_SIZE --max-prefill-tokens $MAX_PREFILL_TOKENS \ --context-length $CONTEXT_LENGTH --disable-radix-cache \ ---attention-backend trtllm_mha --moe-runner-backend flashinfer_trtllm \ +--attention-backend trtllm_mha --mm-attention-backend triton_attn --moe-runner-backend flashinfer_trtllm \ $EXTRA_ARGS --scheduler-recv-interval $SCHEDULER_RECV_INTERVAL \ --tokenizer-worker-num 6 --stream-interval 30 \ --speculative-algorithm EAGLE \ diff --git a/perf-changelog.yaml b/perf-changelog.yaml index 45d8ba110..c62d94781 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -3036,3 +3036,10 @@ - "TP=4 shows +3.2% to +16.3% throughput improvement across 1k1k and 8k1k workloads (concurrency 4-256)" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1411 + +- config-keys: + - qwen3.5-fp4-b300-sglang + - qwen3.5-fp4-b300-sglang-mtp + description: + - "Update SGLang image from v0.5.11-cu130 (5d old) to v0.5.12-cu130" + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/1475