diff --git a/src/dstack/_internal/core/backends/kubernetes/resources.py b/src/dstack/_internal/core/backends/kubernetes/resources.py index 61c24dc81..c60c0c2c3 100644 --- a/src/dstack/_internal/core/backends/kubernetes/resources.py +++ b/src/dstack/_internal/core/backends/kubernetes/resources.py @@ -625,7 +625,8 @@ def get_nvidia_gpu_from_node_labels( gpu_product, _, mig_profile = gpu_product.partition(NVIDIA_MIG_PRODUCT_SEPARATOR) if mig_resource is not None: mig_profile = mig_resource.removeprefix(NVIDIA_MIG_RESOURCE_PREFIX) - gpu_product = gpu_product.replace("RTX-", "RTX") + # Join "RTX" with the model, e.g. "RTX-4090" -> "RTX4090", "RTX-PRO-6000" -> "RTXPRO6000" + gpu_product = gpu_product.replace("RTX-PRO-", "RTXPRO").replace("RTX-", "RTX") for gpu_name in NVIDIA_GPU_NAMES: if gpu_name.lower() in gpu_product.lower().split("-"): break @@ -636,6 +637,9 @@ def get_nvidia_gpu_from_node_labels( # A100 may come in two variants if "40GB" in gpu_product: gpu_memory = 40 * 1024 + # RTX PRO 5000 may come in two variants + if gpu_name == "RTXPRO5000": + gpu_memory = (72 if "72GB" in gpu_product else 48) * 1024 if mig_profile: mig_memory = _get_mig_memory_mib(mig_profile) if mig_memory is None: diff --git a/src/tests/_internal/core/backends/kubernetes/test_resources.py b/src/tests/_internal/core/backends/kubernetes/test_resources.py index 8bb81c5e3..5e4e8c912 100644 --- a/src/tests/_internal/core/backends/kubernetes/test_resources.py +++ b/src/tests/_internal/core/backends/kubernetes/test_resources.py @@ -62,6 +62,39 @@ def test_returns_correct_memory_for_different_A100(self): {"nvidia.com/gpu.product": "A100-SXM4-80GB"} ) == Gpu(vendor=AcceleratorVendor.NVIDIA, name="A100", memory_mib=80 * 1024) + @pytest.mark.parametrize( + ["product", "expected_name", "expected_memory_gb"], + [ + # The label set by GPU Feature Discovery v0.20.1 + pytest.param( + "NVIDIA-RTX-PRO-6000-Blackwell-Max-Q-Workstation-Edition", + "RTXPRO6000", + 96, + id="rtx-pro-6000-max-q", + ), + pytest.param( + "NVIDIA-RTX-PRO-6000-Blackwell-Server-Edition", + "RTXPRO6000", + 96, + id="rtx-pro-6000-server", + ), + # The two RTX PRO 5000 variants share the name; only the 72GB one has its memory + # in the product name + pytest.param("NVIDIA-RTX-PRO-5000-Blackwell", "RTXPRO5000", 48, id="rtx-pro-5000"), + pytest.param( + "NVIDIA-RTX-PRO-5000-72GB-Blackwell", "RTXPRO5000", 72, id="rtx-pro-5000-72gb" + ), + pytest.param("NVIDIA-RTX-PRO-4500-Blackwell", "RTXPRO4500", 32, id="rtx-pro-4500"), + pytest.param("NVIDIA-GeForce-RTX-4090", "RTX4090", 24, id="rtx-4090"), + ], + ) + def test_returns_rtx_gpu(self, product: str, expected_name: str, expected_memory_gb: int): + assert get_nvidia_gpu_from_node_labels({"nvidia.com/gpu.product": product}) == Gpu( + vendor=AcceleratorVendor.NVIDIA, + name=expected_name, + memory_mib=expected_memory_gb * 1024, + ) + @pytest.mark.parametrize( ["product", "expected_name", "expected_memory_gb"], [