From 5038c104563ce60bd5b326a410a183170e69191c Mon Sep 17 00:00:00 2001 From: Jason Dai Date: Fri, 2 Oct 2026 12:25:08 -0700 Subject: [PATCH] fix: GenAI Client(evals) - Default missing rubric verdicts to False PiperOrigin-RevId: 992471781 --- agentplatform/_genai/types/evals.py | 2 +- tests/unit/agentplatform/genai/test_evals.py | 44 ++++++++++++++++++++ vertexai/_genai/types/evals.py | 2 +- 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/agentplatform/_genai/types/evals.py b/agentplatform/_genai/types/evals.py index b50253ce6c..23c2a22d78 100644 --- a/agentplatform/_genai/types/evals.py +++ b/agentplatform/_genai/types/evals.py @@ -948,7 +948,7 @@ class RubricVerdict(_common.BaseModel): generated.""", ) verdict: Optional[bool] = Field( - default=None, + default=False, description="""Required. Outcome of the evaluation against the rubric, represented as a boolean. `true` indicates a "Pass", `false` indicates a "Fail".""", ) diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 492d9cb429..98ea91ba0a 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -45,6 +45,7 @@ types as agentplatform_genai_types, ) from agentplatform._genai.types import common as common_types +from vertexai._genai import evals as vertexai_evals from google.genai import client from google.genai import errors as genai_errors from google.genai import types as genai_types @@ -2311,6 +2312,49 @@ def test_eval_evaluate_with_agent_info(self, mock_execute_evaluation): assert kwargs["agent_info"] == agent_info +class TestRubricVerdict: + """Unit tests for parsing RubricVerdict from API responses.""" + + @pytest.mark.parametrize( + "evals_module", [evals, vertexai_evals], ids=["agent_platform", "vertexai"] + ) + def test_evaluate_instances_missing_verdict_is_false(self, evals_module): + api_client = mock.MagicMock() + api_client.vertexai = True + api_client.request.return_value.body = json.dumps( + { + "metricResults": [ + { + "score": 0.5, + "rubricVerdicts": [ + { + "evaluatedRubric": { + "content": { + "property": {"description": "In English."} + } + }, + "verdict": True, + }, + { + "evaluatedRubric": { + "content": { + "property": {"description": "One sentence."} + } + }, + "reasoning": "The response has two sentences.", + }, + ], + } + ] + } + ) + + response = evals_module.Evals(api_client_=api_client)._evaluate_instances() + + verdicts = response.metric_results[0].rubric_verdicts + assert [verdict.verdict for verdict in verdicts] == [True, False] + + class TestEvalsVisualization: # fmt: off @mock.patch( diff --git a/vertexai/_genai/types/evals.py b/vertexai/_genai/types/evals.py index 080e46f5ec..4e90bb05ad 100644 --- a/vertexai/_genai/types/evals.py +++ b/vertexai/_genai/types/evals.py @@ -948,7 +948,7 @@ class RubricVerdict(_common.BaseModel): generated.""", ) verdict: Optional[bool] = Field( - default=None, + default=False, description="""Required. Outcome of the evaluation against the rubric, represented as a boolean. `true` indicates a "Pass", `false` indicates a "Fail".""", )