googleapis · scotthart · Jul 7, 2026 · Jul 7, 2026 · Jul 7, 2026 · Jul 8, 2026
@@ -26,17 +26,205 @@ export CC=clang
 export CXX=clang++
 
 mapfile -t args < <(bazel::common_args)
-args+=("--//google/cloud/bigtable:enable_metrics=true")
+args+=(
+  "--//google/cloud/bigtable:enable_metrics=true"
+  "--dynamic_mode=off"
+)
+
 mapfile -t integration_args < <(integration::bazel_args)
 
+observability_key_base="observability-key-$(date +"%Y-%m")"
+readonly KEY_DIR="/dev/shm"
+readonly SECRETS_BUCKET="gs://cloud-cpp-testing-resources-secrets"
+gcloud storage cp --quiet "${SECRETS_BUCKET}/${observability_key_base}.json" "${KEY_DIR}/${observability_key_base}.json" >/dev/null 2>&1 || true
+if [[ -r "${KEY_DIR}/${observability_key_base}.json" ]]; then
+  GOOGLE_CLOUD_CPP_TEST_OBSERVABILITY_KEY_FILE_JSON="${KEY_DIR}/${observability_key_base}.json"
+fi
+
+ORIGINAL_ACCOUNT="$(gcloud config get-value account 2>/dev/null || true)"
+
+# Activate dedicated observability service account credentials for GCE VM & GCS management
+KEY_FILE=""
+for candidate in \
+  "${GOOGLE_CLOUD_CPP_TEST_OBSERVABILITY_KEY_FILE_JSON:-}" \
+  "${GOOGLE_APPLICATION_CREDENTIALS:-}"; do
+  if [[ -n "${candidate}" && -r "${candidate}" ]]; then
+    KEY_FILE="${candidate}"
+    break
+  fi
+done
+
+if [[ -n "${KEY_FILE}" ]]; then
+  io::log_h2 "Activating gcloud service account from ${KEY_FILE}"
+  gcloud auth activate-service-account --key-file="${KEY_FILE}" --quiet || true
+else
+  io::log_yellow "No dedicated observability service account keyfile found; using default gcloud auth."
+fi
+
 io::log_h2 "Building OtelCollector targets and Bigtable Observability integration tests"
 io::run bazel build "${args[@]}" \
   //ci/otel_collector:otel_collector_main \
   //google/cloud/bigtable/tests:observability_integration_test-default \
   //google/cloud/bigtable/tests:observability_integration_test-dynamic-pool
 
-io::log_h2 "Running Bigtable Observability Integration Tests against production endpoint"
-io::run bazel test "${args[@]}" "${integration_args[@]}" \
-  --cache_test_results="auto" \
-  -- //google/cloud/bigtable/tests:observability_integration_test-default \
-  //google/cloud/bigtable/tests:observability_integration_test-dynamic-pool
+BAZEL_BIN="$(bazel info "${args[@]}" bazel-bin)"
+TEST_DEFAULT_BIN="${BAZEL_BIN}/google/cloud/bigtable/tests/observability_integration_test-default"
+TEST_DYNAMIC_BIN="${BAZEL_BIN}/google/cloud/bigtable/tests/observability_integration_test-dynamic-pool"
+
+ZONE="us-central1-f"
+PROJECT_ID="${GOOGLE_CLOUD_PROJECT:-cloud-cpp-testing-resources}"
+BIGTABLE_INSTANCE_ID="${GOOGLE_CLOUD_CPP_BIGTABLE_TEST_INSTANCE_ID:-test-instance}"
+BIGTABLE_CLUSTER_ID="${GOOGLE_CLOUD_CPP_BIGTABLE_TEST_CLUSTER_ID:-test-cluster}"
+BIGTABLE_ZONE_A="${GOOGLE_CLOUD_CPP_BIGTABLE_TEST_ZONE_A:-us-central1-f}"
+BIGTABLE_ZONE_B="${GOOGLE_CLOUD_CPP_BIGTABLE_TEST_ZONE_B:-us-central1-b}"
+BUILD_SUFFIX="${BUILD_ID:-local}-${RANDOM}"
+VM_NAME="dp-obs-test-${BUILD_SUFFIX:0:15}"
+GCS_BUCKET="cloud-cpp-testing-resources-observability-logs"
+GCS_PATH="gs://${GCS_BUCKET}/builds/${VM_NAME}"
+VM_CREATED="false"
+
+cleanup() {
+  local exit_code=$?
+  if [[ "${VM_CREATED:-false}" == "true" ]]; then
+    io::log_h2 "Deleting ephemeral DirectPath VM: ${VM_NAME}"
+    gcloud compute instances delete "${VM_NAME}" \
+      --project="${PROJECT_ID}" \
+      --zone="${ZONE}" \
+      --quiet >/dev/null 2>&1 &
+  fi
+
+  if [[ -n "${ORIGINAL_ACCOUNT:-}" ]]; then
+    io::log_h2 "Restoring original gcloud account: ${ORIGINAL_ACCOUNT}"
+    gcloud config set account "${ORIGINAL_ACCOUNT}" --quiet >/dev/null 2>&1 || true
+  fi
+}
+trap cleanup EXIT SIGINT SIGTERM
+
+# Ensure GCS bucket and 90-day lifecycle policy exist
+if ! gcloud storage buckets describe "gs://${GCS_BUCKET}" --project="${PROJECT_ID}" >/dev/null 2>&1; then
+  io::log_h2 "Creating GCS bucket gs://${GCS_BUCKET} with 90-day lifecycle policy"
+  gcloud storage buckets create "gs://${GCS_BUCKET}" \
+    --project="${PROJECT_ID}" \
+    --location="us-central1" \
+    --uniform-bucket-level-access || true
+
+  cat <<'EOF' >/tmp/observability_lifecycle.json
+{
+  "rule": [
+    {
+      "action": {
+        "type": "Delete"
+      },
+      "condition": {
+        "age": 90
+      }
+    }
+  ]
+}
+EOF
+  gcloud storage buckets update "gs://${GCS_BUCKET}" \
+    --lifecycle-file=/tmp/observability_lifecycle.json \
+    --project="${PROJECT_ID}" || true
+fi
+
+io::log_h2 "Uploading test binaries to GCS: ${GCS_PATH}/binaries/"
+gcloud storage cp --quiet "${TEST_DEFAULT_BIN}" "${TEST_DYNAMIC_BIN}" "${GCS_PATH}/binaries/" >/dev/null 2>&1 || true
+
+# Prepare Startup Script to run on the ephemeral GCE VM
+STARTUP_SCRIPT=$(
+  cat <<EOF
+#!/bin/bash
+exec > /tmp/startup.log 2>&1
+set -x
+
+export HOME="/tmp"
+export GOOGLE_CLOUD_PROJECT="${PROJECT_ID}"
+export GOOGLE_CLOUD_CPP_BIGTABLE_TEST_INSTANCE_ID="${BIGTABLE_INSTANCE_ID}"
+export GOOGLE_CLOUD_CPP_BIGTABLE_TEST_CLUSTER_ID="${BIGTABLE_CLUSTER_ID}"
+export GOOGLE_CLOUD_CPP_BIGTABLE_TEST_ZONE_A="${BIGTABLE_ZONE_A}"
+export GOOGLE_CLOUD_CPP_BIGTABLE_TEST_ZONE_B="${BIGTABLE_ZONE_B}"
+export CBT_ENABLE_DIRECTPATH=true
+export GOOGLE_CLOUD_CPP_TEST_EXPECTED_CLIENT_PROJECT="${PROJECT_ID}"
+export GOOGLE_CLOUD_CPP_TEST_EXPECTED_LOCATION="${ZONE}"
+export GOOGLE_CLOUD_CPP_TEST_EXPECTED_CLOUD_PLATFORM="gcp_compute_engine"
+export GOOGLE_CLOUD_CPP_TEST_EXPECTED_HOSTNAME="\$(hostname)"
+
+echo "Downloading test binaries from GCS..."
+gcloud storage cp --quiet "${GCS_PATH}/binaries/*" /tmp/
+chmod +x /tmp/observability_integration_test-default
+chmod +x /tmp/observability_integration_test-dynamic-pool
+
+TEST_EXIT_CODE=0
+
+echo "Running observability_integration_test-default..."
+/tmp/observability_integration_test-default \
+  --gtest_output=xml:/tmp/test-default.xml > /tmp/test-default.log 2>&1 || TEST_EXIT_CODE=\$?
+
+echo "Running observability_integration_test-dynamic-pool..."
+/tmp/observability_integration_test-dynamic-pool \
+  --gtest_output=xml:/tmp/test-dynamic-pool.xml > /tmp/test-dynamic-pool.log 2>&1 || TEST_EXIT_CODE=\$?
+
+echo "\${TEST_EXIT_CODE}" > /tmp/exit_code.txt
+
+echo "Uploading logs and test results back to GCS..."
+gcloud storage cp --quiet /tmp/startup.log /tmp/*.log /tmp/*.xml /tmp/exit_code.txt "${GCS_PATH}/results/"
+EOF
+)
+
+io::log_h2 "Creating ephemeral DirectPath VM instance: ${VM_NAME}"
+if ! gcloud compute instances create "${VM_NAME}" \
+  --project="${PROJECT_ID}" \
+  --zone="${ZONE}" \
+  --machine-type="e2-standard-4" \
+  --subnet="projects/${PROJECT_ID}/regions/us-central1/subnetworks/directpath-subnet-ipv6" \
+  --stack-type="IPV4_IPV6" \
+  --image-family="ubuntu-2404-lts-amd64" \
+  --image-project="ubuntu-os-cloud" \
+  --metadata="startup-script=${STARTUP_SCRIPT}" \
+  --scopes="cloud-platform" \
+  --quiet >/tmp/vm_create.log 2>&1; then
+  io::log_red "Failed to create ephemeral VM instance: ${VM_NAME}"
+  cat /tmp/vm_create.log || true
+  exit 1
+fi
+VM_CREATED="true"
+
+io::log_h2 "Waiting for test execution to complete on VM ${VM_NAME}..."
+TEST_COMPLETED="false"
+for i in {1..90}; do
+  if gcloud storage cp --quiet "${GCS_PATH}/results/exit_code.txt" /tmp/exit_code.txt >/dev/null 2>&1; then
+    TEST_COMPLETED="true"
+    io::log_yellow "Test execution on VM ${VM_NAME} finished."
+    break
+  fi
+  sleep 5
+done
+
+if [[ "${TEST_COMPLETED}" != "true" ]]; then
+  io::log_red "Timed out waiting for test execution on VM ${VM_NAME}."
+  exit 1
+fi
+
+TEST_RESULT_CODE="$(cat /tmp/exit_code.txt 2>/dev/null || echo "1")"
+
+io::log_h2 "Fetching test logs from GCS: ${GCS_PATH}/results/"
+gcloud storage cp --quiet "${GCS_PATH}/results/*.log" /tmp/ 2>/dev/null || true
+
+if [[ "${TEST_RESULT_CODE}" -ne 0 ]]; then
+  for log_file in /tmp/startup.log /tmp/test-default.log /tmp/test-dynamic-pool.log; do
+    if [[ -f "${log_file}" ]]; then
+      io::log_h2 "=== Content of $(basename "${log_file}") ==="
+      cat "${log_file}" || true
+    fi
+  done
+  io::log_red "Observability integration tests failed with exit code: ${TEST_RESULT_CODE}"
+  exit "${TEST_RESULT_CODE}"
+else
+  for log_file in /tmp/test-default.log /tmp/test-dynamic-pool.log; do
+    if [[ -f "${log_file}" ]]; then
+      io::log_h2 "=== Test Summary: $(basename "${log_file}" .log) (Execution: Live Ephemeral VM - Uncached) ==="
+      grep -E '^\[(==========|----------| RUN      |       OK |  PASSED  |  FAILED  |  SKIPPED )\]' "${log_file}" || cat "${log_file}"
+    fi
+  done
+  io::log_green "Observability integration tests PASSED successfully!"
+fi
@@ -21,6 +21,7 @@ cc_library(
     srcs = ["otel_collector.cc"],
     hdrs = ["otel_collector.h"],
     deps = [
+        "//:common",
         "//protos/google/cloud/opentelemetry/testing:observability_verification_cc_grpc",
         "//protos/google/cloud/opentelemetry/testing:observability_verification_cc_proto",
         "@googleapis//google/monitoring/v3:monitoring_cc_grpc",

@@ -20,10 +20,14 @@ add_library(otel_collector otel_collector.cc otel_collector.h)
 target_link_libraries(
     otel_collector PUBLIC google_cloud_cpp_opentelemetry_testing_protos
                           google-cloud-cpp::monitoring_protos gRPC::grpc++)
-target_include_directories(otel_collector PUBLIC ${PROJECT_SOURCE_DIR})
+add_dependencies(otel_collector google_cloud_cpp_opentelemetry_testing_protos)
+target_include_directories(otel_collector PUBLIC ${PROJECT_SOURCE_DIR}
+                                                 ${PROJECT_BINARY_DIR})
 google_cloud_cpp_add_common_options(otel_collector NO_WARNINGS)
+set_target_properties(otel_collector PROPERTIES CXX_CLANG_TIDY "")
 
 google_cloud_cpp_add_executable(target "otel_collector_main"
                                 "otel_collector_main.cc")
-target_link_libraries(otel_collector_main PRIVATE otel_collector gRPC::grpc++)
-google_cloud_cpp_add_common_options(otel_collector_main NO_WARNINGS)
+target_link_libraries(${target} PRIVATE otel_collector gRPC::grpc++)
+google_cloud_cpp_add_common_options(${target} NO_WARNINGS)
+set_target_properties(${target} PROPERTIES CXX_CLANG_TIDY "")
@@ -16,6 +16,7 @@
 
 namespace google {
 namespace cloud {
+GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
 namespace testing_util {
 
 grpc::Status OtelCollectorServer::CreateTimeSeries(
@@ -68,5 +69,6 @@ void OtelCollectorServer::Clear() {
 }
 
 }  // namespace testing_util
+GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace cloud
 }  // namespace google
@@ -15,6 +15,7 @@
 #ifndef GOOGLE_CLOUD_CPP_CI_OTEL_COLLECTOR_OTEL_COLLECTOR_H
 #define GOOGLE_CLOUD_CPP_CI_OTEL_COLLECTOR_OTEL_COLLECTOR_H
 
+#include "google/cloud/version.h"
 #include "protos/google/cloud/opentelemetry/testing/observability_verification.grpc.pb.h"
 #include <google/monitoring/v3/metric_service.grpc.pb.h>
 #include <grpcpp/grpcpp.h>
@@ -23,6 +24,7 @@
 
 namespace google {
 namespace cloud {
+GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
 namespace testing_util {
 
 /**
@@ -71,6 +73,7 @@ class OtelCollectorServer final
 };
 
 }  // namespace testing_util
+GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
 }  // namespace cloud
 }  // namespace google
 

@@ -43,7 +43,10 @@ cc_library(
     srcs = google_cloud_cpp_bigtable_srcs,
     hdrs = google_cloud_cpp_bigtable_hdrs,
     local_defines = select({
-        ":metrics_enabled": ["GOOGLE_CLOUD_CPP_BIGTABLE_WITH_OTEL_METRICS"],
+        ":metrics_enabled": [
+            "GOOGLE_CLOUD_CPP_BIGTABLE_WITH_OTEL_METRICS",
+            "GOOGLE_CLOUD_CPP_BIGTABLE_WITH_GRPC_OTEL_METRICS",
+        ],
         "//conditions:default": [],
     }),
     visibility = [
@@ -65,6 +68,7 @@ cc_library(
     ] + select({
         ":metrics_enabled": [
             "//:opentelemetry",
+            "@grpc//:grpcpp_otel_plugin",
             "@opentelemetry-cpp//api",
             "@opentelemetry-cpp//sdk/src/metrics",
         ],
@@ -121,6 +125,7 @@ cc_library(
     local_defines = select({
         ":metrics_enabled": [
             "GOOGLE_CLOUD_CPP_BIGTABLE_WITH_OTEL_METRICS",
+            "GOOGLE_CLOUD_CPP_BIGTABLE_WITH_GRPC_OTEL_METRICS",
             "GOOGLE_CLOUD_CPP_BIGTABLE_QUERY_PLAN_REFRESH_ASSERT",
         ],
         "//conditions:default": ["GOOGLE_CLOUD_CPP_BIGTABLE_QUERY_PLAN_REFRESH_ASSERT"],

@@ -190,6 +190,8 @@ add_library(
     internal/endpoint_options.h
     internal/google_bytes_traits.cc
     internal/google_bytes_traits.h
+    internal/grpc_metrics_exporter.cc
+    internal/grpc_metrics_exporter.h
     internal/logging_result_set_reader.cc
     internal/logging_result_set_reader.h
     internal/metrics.cc
@@ -285,7 +287,17 @@ target_compile_definitions(google_cloud_cpp_bigtable
                            PRIVATE GOOGLE_CLOUD_CPP_BIGTABLE_WITH_OTEL_METRICS)
 target_link_libraries(google_cloud_cpp_bigtable
                       PUBLIC google-cloud-cpp::opentelemetry)
-set(EXTRA_MODULES "google_cloud_cpp_opentelemetry" "opentelemetry_metrics")
+if (TARGET gRPC::grpcpp_otel_plugin)
+    target_compile_definitions(
+        google_cloud_cpp_bigtable
+        PRIVATE GOOGLE_CLOUD_CPP_BIGTABLE_WITH_GRPC_OTEL_METRICS)
+    target_link_libraries(google_cloud_cpp_bigtable
+                          PUBLIC gRPC::grpcpp_otel_plugin)
+    set(EXTRA_MODULES "google_cloud_cpp_opentelemetry" "grpcpp_otel_plugin"
+                      "opentelemetry_metrics")
+else ()
+    set(EXTRA_MODULES "google_cloud_cpp_opentelemetry" "opentelemetry_metrics")
+endif ()
 google_cloud_cpp_add_common_options(google_cloud_cpp_bigtable)
 target_include_directories(
     google_cloud_cpp_bigtable
@@ -460,6 +472,7 @@ if (BUILD_TESTING)
         internal/defaults_test.cc
         internal/dynamic_channel_pool_test.cc
         internal/google_bytes_traits_test.cc
+        internal/grpc_metrics_exporter_test.cc
         internal/logging_result_set_reader_test.cc
         internal/metrics_test.cc
         internal/mutate_rows_limiter_test.cc
@@ -528,6 +541,11 @@ if (BUILD_TESTING)
             target_compile_definitions(
                 ${target} PRIVATE GOOGLE_CLOUD_CPP_BIGTABLE_WITH_OTEL_METRICS)
         endif ()
+        if (TARGET gRPC::grpcpp_otel_plugin)
+            target_compile_definitions(
+                ${target}
+                PRIVATE GOOGLE_CLOUD_CPP_BIGTABLE_WITH_GRPC_OTEL_METRICS)
+        endif ()
         google_cloud_cpp_add_common_options(${target})
         add_test(NAME ${target} COMMAND ${target})
     endforeach ()

@@ -55,6 +55,7 @@ bigtable_client_unit_tests = [
     "internal/defaults_test.cc",
     "internal/dynamic_channel_pool_test.cc",
     "internal/google_bytes_traits_test.cc",
+    "internal/grpc_metrics_exporter_test.cc",
     "internal/logging_result_set_reader_test.cc",
     "internal/metrics_test.cc",
     "internal/mutate_rows_limiter_test.cc",