[Observability Testing] register prometheus exporter (#34380)

Working towards testing against CSM Observability. Added ability to
register a prometheus exporter with our Opentelemetry plugin. This will
allow our metrics to be available at the standard prometheus port
`:9464`.

bazel/grpc_deps.bzl

@@ -489,6 +489,23 @@ def grpc_deps():
             ],
         )
 
+    # TODO(stanleycheung): remove this when prometheus-cpp AND
+    #   opentelemetry-cpp cut a new release
+    # This override is needed because this fix
+    #   https://github.com/jupp0r/prometheus-cpp/pull/626
+    #   has not been included in the latest prometheus-cpp release yet.
+    # We also need opentelemetry-cpp to update their dependency on
+    #   prometheus-cpp after that fix is released.
+    # Without the fix, we cannot build the prometheus exporter with bazel 6
+    if "com_github_jupp0r_prometheus_cpp" not in native.existing_rules():
+        http_archive(
+            name = "com_github_jupp0r_prometheus_cpp",
+            strip_prefix = "prometheus-cpp-b1234816facfdda29845c46696a02998a4af115a",
+            urls = [
+                "https://github.com/jupp0r/prometheus-cpp/archive/b123481.zip",
+            ],
+        )
+
     if "io_opentelemetry_cpp" not in native.existing_rules():
         http_archive(
             name = "io_opentelemetry_cpp",

test/cpp/interop/BUILD

@@ -460,12 +460,15 @@ grpc_cc_library(
     ],
     external_deps = [
         "absl/flags:flag",
+        "otel/exporters/prometheus:prometheus_exporter",
+        "otel/sdk/src/metrics",
     ],
     language = "C++",
     tags = ["nobuilder"],
     deps = [
         ":client_helper_lib",
         "//:grpcpp_gcp_observability",
+        "//src/cpp/ext/otel:otel_plugin",
     ],
 )
 

test/cpp/interop/observability_client.cc

@@ -20,6 +20,9 @@
 #include <unordered_map>
 
 #include "absl/flags/flag.h"
+#include "opentelemetry/exporters/prometheus/exporter_factory.h"
+#include "opentelemetry/exporters/prometheus/exporter_options.h"
+#include "opentelemetry/sdk/metrics/meter_provider.h"
 
 #include <grpc/grpc.h>
 #include <grpc/support/alloc.h>
@@ -30,6 +33,7 @@
 
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/crash.h"
+#include "src/cpp/ext/otel/otel_plugin.h"
 #include "test/core/util/test_config.h"
 #include "test/cpp/interop/client_helper.h"
 #include "test/cpp/interop/interop_client.h"
@@ -136,6 +140,8 @@ ABSL_FLAG(
     "grpc-status and error message to the console, in a stable format.");
 ABSL_FLAG(bool, enable_observability, false,
           "Whether to enable GCP Observability");
+ABSL_FLAG(bool, enable_otel_plugin, false,
+          "Whether to enable OpenTelemetry Plugin");
 
 using grpc::testing::CreateChannelForTestCase;
 using grpc::testing::GetServiceAccountJsonKey;
@@ -214,6 +220,25 @@ int main(int argc, char** argv) {
     }
   }
 
+  // TODO(stanleycheung): switch to CsmObservabilityBuilder once xds setup is
+  // ready
+  if (absl::GetFlag(FLAGS_enable_otel_plugin)) {
+    gpr_log(GPR_DEBUG, "Registering Prometheus exporter");
+    opentelemetry::exporter::metrics::PrometheusExporterOptions opts;
+    // default was "localhost:9464" which causes connection issue across GKE
+    // pods
+    opts.url = "0.0.0.0:9464";
+    auto prometheus_exporter =
+        opentelemetry::exporter::metrics::PrometheusExporterFactory::Create(
+            opts);
+    auto meter_provider =
+        std::make_shared<opentelemetry::sdk::metrics::MeterProvider>();
+    meter_provider->AddMetricReader(std::move(prometheus_exporter));
+    grpc::internal::OpenTelemetryPluginBuilder otel_builder;
+    otel_builder.SetMeterProvider(std::move(meter_provider));
+    otel_builder.BuildAndRegisterGlobal();
+  }
+
   grpc::testing::ChannelCreationFunc channel_creation_func;
   std::string test_case = absl::GetFlag(FLAGS_test_case);
   if (absl::GetFlag(FLAGS_additional_metadata).empty()) {

tools/run_tests/sanity/check_bazel_workspace.py

@@ -59,6 +59,8 @@ _GRPC_DEP_NAMES = [
     "com_google_fuzztest",
     "io_opencensus_cpp",
     "io_opentelemetry_cpp",
+    # TODO(stanleycheung): remove when prometheus-cpp has new release
+    "com_github_jupp0r_prometheus_cpp",
     "envoy_api",
     _BAZEL_SKYLIB_DEP_NAME,
     _BAZEL_TOOLCHAINS_DEP_NAME,
@@ -88,6 +90,8 @@ _GRPC_BAZEL_ONLY_DEPS = [
     "com_google_fuzztest",
     "io_opencensus_cpp",
     "io_opentelemetry_cpp",
+    # TODO(stanleycheung): remove when prometheus-cpp has new release
+    "com_github_jupp0r_prometheus_cpp",
     _BAZEL_SKYLIB_DEP_NAME,
     _BAZEL_TOOLCHAINS_DEP_NAME,
     _BAZEL_COMPDB_DEP_NAME,