diff --git a/docs/reference/supported-models.md b/docs/reference/supported-models.md index 0cd536b7e..6946100a2 100644 --- a/docs/reference/supported-models.md +++ b/docs/reference/supported-models.md @@ -191,6 +191,7 @@ testing. Use `winml catalog` to browse the full list interactively. | `mattmdjaga/segformer_b2_clothes` | Segformer | | `nvidia/segformer-b1-finetuned-ade-512-512` | Segformer | | `nvidia/segformer-b2-finetuned-ade-512-512` | Segformer | +| `nvidia/segformer-b4-finetuned-ade-512-512` | Segformer | | `nvidia/segformer-b5-finetuned-ade-640-640` | Segformer | ### Image-to-Text diff --git a/examples/recipes/README.md b/examples/recipes/README.md index 86aa01869..dbb7e460c 100644 --- a/examples/recipes/README.md +++ b/examples/recipes/README.md @@ -14,7 +14,7 @@ Each *(model, task)* includes: ## Models -Total: **75** (model, task) tuples that pass fp16 eval on all 10 (EP, device) buckets. +Total: **77** (model, task) tuples that pass fp16 eval on all 10 (EP, device) buckets. | Model | Task | |---|---| @@ -77,6 +77,7 @@ Total: **75** (model, task) tuples that pass fp16 eval on all 10 (EP, device) bu | monologg/koelectra-small-v2-distilled-korquad-384 | question-answering | | nvidia/segformer-b1-finetuned-ade-512-512 | image-segmentation | | nvidia/segformer-b2-finetuned-ade-512-512 | image-segmentation | +| nvidia/segformer-b4-finetuned-ade-512-512 | image-segmentation | | nvidia/segformer-b5-finetuned-ade-640-640 | image-segmentation | | openai/clip-vit-base-patch16 | feature-extraction | | openai/clip-vit-base-patch16 | zero-shot-image-classification | diff --git a/examples/recipes/nvidia_segformer-b4-finetuned-ade-512-512/image-segmentation_fp16_config.json b/examples/recipes/nvidia_segformer-b4-finetuned-ade-512-512/image-segmentation_fp16_config.json new file mode 100644 index 000000000..2c6552143 --- /dev/null +++ b/examples/recipes/nvidia_segformer-b4-finetuned-ade-512-512/image-segmentation_fp16_config.json @@ -0,0 +1,54 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "pixel_values", + "dtype": "float32", + "shape": [ + 1, + 3, + 512, + 512 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "logits" + } + ] + }, + "optim": {}, + "quant": null, + "compile": null, + "loader": { + "task": "image-segmentation", + "model_class": "AutoModelForSemanticSegmentation", + "model_type": "segformer" + }, + "eval": { + "task": "image-segmentation", + "dataset": { + "path": "danjacobellis/scene_parse_150", + "split": "validation", + "samples": 100, + "label_mapping_file": "scripts/e2e_eval/datasets/ade20k_gt_to_model_label.json", + "columns_mapping": { + "annotation_column": "annotation" + } + } + } +} diff --git a/examples/recipes/nvidia_segformer-b4-finetuned-ade-512-512/image-segmentation_w8a16_config.json b/examples/recipes/nvidia_segformer-b4-finetuned-ade-512-512/image-segmentation_w8a16_config.json new file mode 100644 index 000000000..caa990d0e --- /dev/null +++ b/examples/recipes/nvidia_segformer-b4-finetuned-ade-512-512/image-segmentation_w8a16_config.json @@ -0,0 +1,71 @@ +{ + "export": { + "opset_version": 17, + "batch_size": 1, + "export_params": true, + "do_constant_folding": true, + "verbose": false, + "dynamo": false, + "enable_hierarchy_tags": true, + "clean_onnx": false, + "hierarchy_tag_format": "full", + "input_tensors": [ + { + "name": "pixel_values", + "dtype": "float32", + "shape": [ + 1, + 3, + 512, + 512 + ], + "value_range": [ + 0, + 1 + ] + } + ], + "output_tensors": [ + { + "name": "logits" + } + ] + }, + "optim": {}, + "quant": { + "mode": "qdq", + "samples": 10, + "calibration_method": "minmax", + "weight_type": "uint8", + "activation_type": "uint16", + "per_channel": false, + "symmetric": false, + "save_calibration": false, + "distribution": "uniform", + "seed": null, + "calibration_load_path": null, + "calibration_save_path": null, + "op_types_to_quantize": null, + "nodes_to_exclude": null, + "task": "image-segmentation", + "model_id": "nvidia/segformer-b4-finetuned-ade-512-512" + }, + "compile": null, + "loader": { + "task": "image-segmentation", + "model_class": "AutoModelForSemanticSegmentation", + "model_type": "segformer" + }, + "eval": { + "task": "image-segmentation", + "dataset": { + "path": "danjacobellis/scene_parse_150", + "split": "validation", + "samples": 1000, + "label_mapping_file": "scripts/e2e_eval/datasets/ade20k_gt_to_model_label.json", + "columns_mapping": { + "annotation_column": "annotation" + } + } + } +} diff --git a/scripts/e2e_eval/testsets/models_with_acc.json b/scripts/e2e_eval/testsets/models_with_acc.json index e21423161..af8dc1b9b 100644 --- a/scripts/e2e_eval/testsets/models_with_acc.json +++ b/scripts/e2e_eval/testsets/models_with_acc.json @@ -516,6 +516,23 @@ } } }, + { + "hf_id": "nvidia/segformer-b4-finetuned-ade-512-512", + "task": "image-segmentation", + "model_type": "segformer", + "group": "Top200", + "priority": "P1", + "dataset_config": { + "path": "danjacobellis/scene_parse_150", + "split": "validation", + "metric": "mean_iou", + "winml_metric_key": "mean_iou", + "label_mapping_file": "scripts/e2e_eval/datasets/ade20k_gt_to_model_label.json", + "columns_mapping": { + "annotation_column": "annotation" + } + } + }, { "hf_id": "nvidia/segformer-b5-finetuned-ade-640-640", "task": "image-segmentation", diff --git a/src/winml/modelkit/data/hub_models.json b/src/winml/modelkit/data/hub_models.json index 4be17ce63..94c0d53b6 100644 --- a/src/winml/modelkit/data/hub_models.json +++ b/src/winml/modelkit/data/hub_models.json @@ -1569,6 +1569,38 @@ }, "size_mb": 27.5 }, + { + "model_id": "nvidia/segformer-b4-finetuned-ade-512-512", + "task": "image-segmentation", + "model_type": "segformer", + "supported_eps": { + "cpu": [ + "CPU" + ], + "dml": [ + "GPU" + ], + "qnn": [ + "GPU", + "NPU" + ], + "openvino": [ + "CPU", + "GPU", + "NPU" + ], + "vitisai": [ + "NPU" + ], + "trtrtx": [ + "GPU" + ], + "migraphx": [ + "GPU" + ] + }, + "size_mb": 64.1 + }, { "model_id": "nvidia/segformer-b5-finetuned-ade-640-640", "task": "image-segmentation",