Skip to content

Add LayoutLM v1 document-question-answering support (impira/layoutlm-invoices) #1096

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
ssss141414 wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Add LayoutLM v1 document-question-answering support (impira/layoutlm-invoices) #1096

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4c0ec9a
recipe(layoutlm): keep invoice support recipe-only
ssss141414 Jul 22, 2026
6fe061c
Add LayoutLM document QA support
ssss141414 Jul 22, 2026
19177bc
fix(layoutlm): repair bbox input generation
ssss141414 Jul 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
99 changes: 99 additions & 0 deletions examples/recipes/impira_layoutlm-invoices/cpu/cpu/question-answering_fp16_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
{
"export": {
"opset_version": 17,
"batch_size": 1,
"export_params": true,
"do_constant_folding": true,
"verbose": false,
"dynamo": false,
"enable_hierarchy_tags": true,
"clean_onnx": false,
"hierarchy_tag_format": "full",
"input_tensors": [
{
"name": "input_ids",
"dtype": "int32",
"shape": [
1,
512
],
"value_range": [
0,
50265
]
},
{
"name": "bbox",
"dtype": "int32",
"shape": [
1,
512,
4
],
"value_range": [
0,
1001
]
},
{
"name": "attention_mask",
"dtype": "int32",
"shape": [
1,
512
],
"value_range": [
0,
2
]
},
{
"name": "token_type_ids",
"dtype": "int32",
"shape": [
1,
512
],
"value_range": [
0,
1
]
}
],
"output_tensors": [
{
"name": "start_logits"
},
{
"name": "end_logits"
}
]
},
"optim": {},
"quant": {
"mode": "fp16",
"samples": 10,
"calibration_method": "minmax",
"weight_type": "uint8",
"activation_type": "uint8",
"per_channel": false,
"symmetric": false,
"weight_symmetric": null,
"activation_symmetric": null,
"save_calibration": false,
"distribution": "uniform",
"seed": null,
"calibration_load_path": null,
"calibration_save_path": null,
"op_types_to_quantize": null,
"nodes_to_exclude": null,
"fp16_keep_io_types": true,
"fp16_op_block_list": null
},
"compile": null,
"loader": {
"task": "question-answering",
"model_class": "LayoutLMForQuestionAnswering",
"model_type": "layoutlm"
}
}
82 changes: 82 additions & 0 deletions examples/recipes/impira_layoutlm-invoices/cpu/cpu/question-answering_fp32_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
{
"export": {
"opset_version": 17,
"batch_size": 1,
"export_params": true,
"do_constant_folding": true,
"verbose": false,
"dynamo": false,
"enable_hierarchy_tags": true,
"clean_onnx": false,
"hierarchy_tag_format": "full",
"input_tensors": [
{
"name": "input_ids",
"dtype": "int32",
"shape": [
1,
512
],
"value_range": [
0,
50265
]
},
{
"name": "bbox",
"dtype": "int32",
"shape": [
1,
512,
4
],
"value_range": [
0,
1001
]
},
{
"name": "attention_mask",
"dtype": "int32",
"shape": [
1,
512
],
"value_range": [
0,
2
]
},
{
"name": "token_type_ids",
"dtype": "int32",
"shape": [
1,
512
],
"value_range": [
0,
1
]
}
],
"output_tensors": [
{
"name": "start_logits"
},
{
"name": "end_logits"
}
]
},
"optim": {

},
"quant": null,
"compile": null,
"loader": {
"task": "question-answering",
"model_class": "LayoutLMForQuestionAnswering",
"model_type": "layoutlm"
}
}
7 changes: 7 additions & 0 deletions src/winml/modelkit/commands/perf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,7 @@ def generate_random_inputs(
symbolic_shapes = io_config.get("input_symbolic_shapes") or [
[None] * len(s) for s in io_config["input_shapes"]
]
value_ranges = io_config.get("input_value_ranges") or {}
overrides = shape_config or {}

specs: dict[str, dict[str, Any]] = {}
Expand Down Expand Up @@ -290,6 +291,12 @@ def generate_random_inputs(
"dtype": gen_dtype,
"shape": list(resolved_shape),
}
if name in value_ranges:
lo, hi = value_ranges[name]
# Build-config ranges are high-exclusive. The legacy NumPy
# generator accepts an inclusive integer maximum, so adapt only
# that boundary; float generation already uses [lo, hi).
specs[name]["range"] = [lo, hi - 1] if gen_dtype == "int" else [lo, hi]

return generate_dummy_inputs_from_specs(specs)

Expand Down
31 changes: 30 additions & 1 deletion src/winml/modelkit/core/model_input_generator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,28 @@
logger = logging.getLogger(__name__)


def _generate_ordered_bounding_boxes(
shape: list[int], min_value: int, max_value: int
) -> np.ndarray:
"""Generate positive-area ``[x1, y1, x2, y2]`` boxes within inclusive bounds."""
if max_value <= min_value:
raise ValueError("Bounding-box range must contain at least two coordinates")

coordinates = np.random.randint(min_value, max_value + 1, size=shape)
x1 = np.minimum(coordinates[..., 0], coordinates[..., 2])
y1 = np.minimum(coordinates[..., 1], coordinates[..., 3])
x2 = np.maximum(coordinates[..., 0], coordinates[..., 2])
y2 = np.maximum(coordinates[..., 1], coordinates[..., 3])

x_equal = x1 == x2
y_equal = y1 == y2
x1 = np.where(x_equal & (x2 == max_value), x1 - 1, x1)
y1 = np.where(y_equal & (y2 == max_value), y1 - 1, y1)
x2 = np.where(x_equal & (x2 < max_value), x2 + 1, x2)
y2 = np.where(y_equal & (y2 < max_value), y2 + 1, y2)
return np.stack((x1, y1, x2, y2), axis=-1).astype(np.int64)


def generate_dummy_inputs_from_specs(
input_specs: dict[str, dict[str, Any]],
) -> dict[str, np.ndarray]:
Expand Down Expand Up @@ -106,7 +128,14 @@ def generate_dummy_inputs_from_specs(
min_val, max_val = spec["range"]

if dtype == np.int64:
inputs[name] = np.random.randint(min_val, max_val + 1, size=shape).astype(dtype)
if name == "bbox" and shape and shape[-1] == 4:
inputs[name] = _generate_ordered_bounding_boxes(
shape, int(min_val), int(max_val)
)
else:
inputs[name] = np.random.randint(min_val, max_val + 1, size=shape).astype(
dtype
)
else:
inputs[name] = np.random.rand(*shape).astype(dtype) * (max_val - min_val) + min_val
else:
Expand Down
51 changes: 44 additions & 7 deletions src/winml/modelkit/loader/resolution.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,6 +132,35 @@ class from the ``transformers`` package.
) from e


def _resolve_architecture_mapping(
config: PretrainedConfig, model_type_normalized: str
) -> tuple[str, type] | None:
"""Resolve an unambiguous registered task from the checkpoint architecture.

Per-architecture ``MODEL_CLASS_MAPPING`` entries normally override the loader
class after task detection. They also provide stronger task evidence when the
concrete class in ``config.architectures`` exactly matches a registered class.
This corrects incomplete upstream task inference without a model-ID default.
"""
try:
architecture_class = _resolve_model_class_from_config(config)
except ValueError:
return None

from ..models.hf import MODEL_CLASS_MAPPING

matches = {
task
for (model_type, task), model_class in MODEL_CLASS_MAPPING.items()
if model_type == model_type_normalized
and task is not None
and model_class is architecture_class
}
if len(matches) != 1:
return None
return matches.pop(), architecture_class


def _detect_task_from_model_class(model_class: type) -> str:
"""Detect task from a model class via TasksManager.

Expand Down Expand Up @@ -251,6 +280,7 @@ class TaskSource(str, Enum):
USER_CLASS = "user-class" # user passed --model-class; task inferred
MODEL_ID_DEFAULT = "model-id-default" # MODEL_TASK_MAPPING model-id default
SENTINEL_DEFAULT = "sentinel-default" # (model_type, None) sentinel
ARCHITECTURE_MAPPING = "architecture-mapping" # exact registered architecture class
TASKS_MANAGER = "tasks-manager" # Optimum inference (incl. fill-mask upgrade)
WRAPPED_LIBRARY = "wrapped-library" # no architectures -> first supported task
PIPELINE_TAG = "pipeline-tag" # Hub pipeline_tag fallback
Expand Down Expand Up @@ -589,7 +619,14 @@ def resolve_task(
else TaskSource.SENTINEL_DEFAULT
)

# 1b. no architectures -> first ONNX-exportable task
# 1b. exact architecture class -> registered task/class mapping
if opt_task is None and model_type_norm:
architecture_mapping = _resolve_architecture_mapping(config, model_type_norm)
if architecture_mapping is not None:
opt_task, resolved = architecture_mapping
source = TaskSource.ARCHITECTURE_MAPPING

# 1c. no architectures -> first ONNX-exportable task
# (merges the old timm wrapped-library stage AND the --model-type fallback)
if opt_task is None and not getattr(config, "architectures", None) and model_type:
# Populate Optimum's ONNX export-config registry before querying it;
Expand All @@ -604,15 +641,15 @@ def resolve_task(
# lookup failure here — e.g. a wrapped library whose classes aren't registered
# under framework="pt" — can't escape as a raw KeyError.

# 1c. TasksManager (reads config.architectures)
# 1d. TasksManager (reads config.architectures)
if opt_task is None:
try:
opt_task = _infer_task_from_architecture(config)
source = TaskSource.TASKS_MANAGER
except ValueError:
opt_task = None

# 1d. Hub pipeline_tag fallback
# 1e. Hub pipeline_tag fallback
if opt_task is None and model_id and model_type:
from ..utils.hub_utils import get_pipeline_tag

Expand All @@ -625,7 +662,7 @@ def resolve_task(
# reinforcement-learning, time-series-forecasting). Admitting one would flow a
# non-exportable task into Stage 2 (model-class) / Stage 3 instead of degrading
# to the last-resort default. Populate Optimum's ONNX export-config registry
# first (as Stage 1b does) so get_supported_tasks doesn't return [].
# first (as Stage 1c does) so get_supported_tasks doesn't return [].
import optimum.exporters.onnx.model_configs # noqa: F401

if normalized_tag in get_supported_tasks(
Expand All @@ -634,12 +671,12 @@ def resolve_task(
opt_task = normalized_tag
source = TaskSource.PIPELINE_TAG

# 1e. last-resort default
# 1f. last-resort default
if opt_task is None:
opt_task = next(iter(HF_TASK_DEFAULTS))
source = TaskSource.HF_TASK_DEFAULT

# --- Stage 2: model class (if not already resolved in 1b) -------------
# --- Stage 2: model class (if not already resolved during detection) --
if resolved is None:
resolved = _get_custom_model_class(model_type_norm, opt_task)
if resolved is None:
Expand All @@ -654,6 +691,6 @@ def resolve_task(
# --- Stage 4: composite tag (detection path) --------------------------
composite = _composite_components_for_task(model_type, opt_task) if model_type else None

if source is None: # structural invariant: Stage 1d always sets a source
if source is None: # structural invariant: Stage 1 always sets a source
raise RuntimeError("resolve_task: internal invariant violated — source was not set")
return TaskResolution(surfaced, to_optimum_task(surfaced), resolved, source, composite)
2 changes: 2 additions & 0 deletions src/winml/modelkit/models/hf/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
from .depth_anything import DepthAnythingIOConfig as _DepthAnythingIOConfig # triggers registration
from .depth_pro import DepthProIOConfig as _DepthProIOConfig # triggers registration
from .detr import DETR_CONFIG
from .layoutlm import MODEL_CLASS_MAPPING as _LAYOUTLM_CLASS_MAPPING
from .layoutlm import LayoutLMQAIOConfig as _LayoutLMQAIOConfig # triggers registration
from .marian import MARIAN_CONFIG
from .marian import MODEL_CLASS_MAPPING as _MARIAN_CLASS_MAPPING
Expand Down Expand Up @@ -121,6 +122,7 @@
_BART_CLASS_MAPPING,
_BLIP_CLASS_MAPPING,
_CLIP_CLASS_MAPPING,
_LAYOUTLM_CLASS_MAPPING,
_MARIAN_CLASS_MAPPING,
_MU2_CLASS_MAPPING,
_QWEN_CLASS_MAPPING,
Expand Down
Loading
Loading