Add Backend Service Custom Metrics. (#13205)

efimki · web-flow · commit 42f65f7727c8 · 2025-03-05T16:19:52.000Z
Signed-off-by: Misha Efimov &lt;mef@google.com&gt;
diff --git a/mmv1/products/compute/BackendService.yaml b/mmv1/products/compute/BackendService.yaml
@@ -120,6 +120,14 @@ examples:
     primary_resource_id: 'default'
     vars:
       backend_service_name: 'backend-service'
+  - name: 'backend_service_custom_metrics'
+    primary_resource_id: 'default'
+    min_version: 'beta'
+    vars:
+      backend_service_name: 'backend-service'
+      default_neg_name: 'network-endpoint'
+      health_check_name: 'health-check'
+      network_name: 'network'
 parameters:
 properties:
   - name: 'affinityCookieTtlSec'
@@ -147,8 +155,8 @@ properties:
             Specifies the balancing mode for this backend.
 
             For global HTTP(S) or TCP/SSL load balancing, the default is
-            UTILIZATION. Valid values are UTILIZATION, RATE (for HTTP(S))
-            and CONNECTION (for TCP/SSL).
+            UTILIZATION. Valid values are UTILIZATION, RATE (for HTTP(S)),
+            CUSTOM_METRICS (for HTTP(s)) and CONNECTION (for TCP/SSL).
 
             See the [Backend Services Overview](https://cloud.google.com/load-balancing/docs/backend-service#balancing-mode)
             for an explanation of load balancing modes.
@@ -157,6 +165,7 @@ properties:
             - 'UTILIZATION'
             - 'RATE'
             - 'CONNECTION'
+            - 'CUSTOM_METRICS'
         - name: 'capacityScaler'
           type: Double
           description: |
@@ -261,6 +270,38 @@ properties:
             Used when balancingMode is UTILIZATION. This ratio defines the
             CPU utilization target for the group. Valid range is [0.0, 1.0].
           default_from_api: true
+        - name: 'customMetrics'
+          type: Array
+          description: |
+            The set of custom metrics that are used for <code>CUSTOM_METRICS</code> BalancingMode.
+          item_type:
+            type: NestedObject
+            properties:
+              - name: 'name'
+                type: String
+                required: true
+                description: |
+                  Name of a custom utilization signal. The name must be 1-64 characters
+                  long and match the regular expression [a-z]([-_.a-z0-9]*[a-z0-9])? which
+                  means the first character must be a lowercase letter, and all following
+                  characters must be a dash, period, underscore, lowercase letter, or
+                  digit, except the last character, which cannot be a dash, period, or
+                  underscore. For usage guidelines, see Custom Metrics balancing mode. This
+                  field can only be used for a global or regional backend service with the
+                  loadBalancingScheme set to <code>EXTERNAL_MANAGED</code>,
+                  <code>INTERNAL_MANAGED</code> <code>INTERNAL_SELF_MANAGED</code>.
+              - name: 'dryRun'
+                type: Boolean
+                required: true
+                description: |
+                  If true, the metric data is collected and reported to Cloud
+                  Monitoring, but is not used for load balancing.
+              - name: 'maxUtilization'
+                type: Double
+                description: |
+                  Optional parameter to define a target utilization for the Custom Metrics
+                  balancing mode. The valid range is <code>[0.0, 1.0]</code>.
+                default_value: 0.8
   - name: 'circuitBreakers'
     type: NestedObject
     description: |
@@ -840,6 +881,13 @@ properties:
                            UNAVAILABLE_WEIGHT. Otherwise, Load Balancing remains
                            equal-weight.
 
+      * `WEIGHTED_ROUND_ROBIN`: Per-endpoint weighted round-robin Load Balancing using weights computed
+                                from Backend reported Custom Metrics. If set, the Backend Service
+                                responses are expected to contain non-standard HTTP response header field
+                                X-Endpoint-Load-Metrics. The reported metrics
+                                to use for computing the weights are specified via the
+                                backends[].customMetrics fields.
+
       locality_lb_policy is applicable to either:
 
       * A regional backend service with the service_protocol set to HTTP, HTTPS, or HTTP2,
@@ -863,6 +911,7 @@ properties:
       - 'ORIGINAL_DESTINATION'
       - 'MAGLEV'
       - 'WEIGHTED_MAGLEV'
+      - 'WEIGHTED_ROUND_ROBIN'
   - name: 'localityLbPolicies'
     type: Array
     description: |
@@ -960,6 +1009,31 @@ properties:
               description: |
                 An optional, arbitrary JSON object with configuration data, understood
                 by a locally installed custom policy implementation.
+  - name: 'customMetrics'
+    type: Array
+    description: |
+      List of custom metrics that are used for the WEIGHTED_ROUND_ROBIN locality_lb_policy.
+    item_type:
+      type: NestedObject
+      properties:
+        - name: 'name'
+          type: String
+          required: true
+          description: |
+            Name of a custom utilization signal. The name must be 1-64 characters
+            long and match the regular expression [a-z]([-_.a-z0-9]*[a-z0-9])? which
+            means the first character must be a lowercase letter, and all following
+            characters must be a dash, period, underscore, lowercase letter, or
+            digit, except the last character, which cannot be a dash, period, or
+            underscore. For usage guidelines, see Custom Metrics balancing mode. This
+            field can only be used for a global or regional backend service with the
+            loadBalancingScheme set to <code>EXTERNAL_MANAGED</code>,
+            <code>INTERNAL_MANAGED</code> <code>INTERNAL_SELF_MANAGED</code>.
+        - name: 'dryRun'
+          type: Boolean
+          required: true
+          description: |
+            If true, the metric data is not used for load balancing.
   - name: 'name'
     type: String
     description: |
diff --git a/mmv1/products/compute/RegionBackendService.yaml b/mmv1/products/compute/RegionBackendService.yaml
@@ -121,6 +121,14 @@ examples:
     vars:
       region_backend_service_name: 'region-service'
       health_check_name: 'rbs-health-check'
+  - name: 'region_backend_service_ilb_custom_metrics'
+    primary_resource_id: 'default'
+    min_version: 'beta'
+    vars:
+      region_backend_service_name: 'region-service'
+      default_neg_name: 'network-endpoint'
+      health_check_name: 'rbs-health-check'
+      network_name: 'network'
 parameters:
   - name: 'region'
     type: ResourceRef
@@ -165,6 +173,7 @@ properties:
             - 'UTILIZATION'
             - 'RATE'
             - 'CONNECTION'
+            - 'CUSTOM_METRICS'
         - name: 'capacityScaler'
           type: Double
           description: |
@@ -280,6 +289,38 @@ properties:
             Used when balancingMode is UTILIZATION. This ratio defines the
             CPU utilization target for the group. Valid range is [0.0, 1.0].
             Cannot be set for INTERNAL backend services.
+        - name: 'customMetrics'
+          type: Array
+          description: |
+            The set of custom metrics that are used for <code>CUSTOM_METRICS</code> BalancingMode.
+          item_type:
+            type: NestedObject
+            properties:
+              - name: 'name'
+                type: String
+                required: true
+                description: |
+                  Name of a custom utilization signal. The name must be 1-64 characters
+                  long and match the regular expression [a-z]([-_.a-z0-9]*[a-z0-9])? which
+                  means the first character must be a lowercase letter, and all following
+                  characters must be a dash, period, underscore, lowercase letter, or
+                  digit, except the last character, which cannot be a dash, period, or
+                  underscore. For usage guidelines, see Custom Metrics balancing mode. This
+                  field can only be used for a global or regional backend service with the
+                  loadBalancingScheme set to <code>EXTERNAL_MANAGED</code>,
+                  <code>INTERNAL_MANAGED</code> <code>INTERNAL_SELF_MANAGED</code>.
+              - name: 'dryRun'
+                type: Boolean
+                required: true
+                description: |
+                  If true, the metric data is collected and reported to Cloud
+                  Monitoring, but is not used for load balancing.
+              - name: 'maxUtilization'
+                type: Double
+                description: |
+                  Optional parameter to define a target utilization for the Custom Metrics
+                  balancing mode. The valid range is <code>[0.0, 1.0]</code>.
+                default_value: 0.8
   - name: 'circuitBreakers'
     type: NestedObject
     description: |
@@ -851,6 +892,13 @@ properties:
                            UNAVAILABLE_WEIGHT. Otherwise, Load Balancing remains
                            equal-weight.
 
+      * `WEIGHTED_ROUND_ROBIN`: Per-endpoint weighted round-robin Load Balancing using weights computed
+                                from Backend reported Custom Metrics. If set, the Backend Service
+                                responses are expected to contain non-standard HTTP response header field
+                                X-Endpoint-Load-Metrics. The reported metrics
+                                to use for computing the weights are specified via the
+                                backends[].customMetrics fields.
+
       locality_lb_policy is applicable to either:
 
       * A regional backend service with the service_protocol set to HTTP, HTTPS, or HTTP2,
@@ -874,6 +922,7 @@ properties:
       - 'ORIGINAL_DESTINATION'
       - 'MAGLEV'
       - 'WEIGHTED_MAGLEV'
+      - 'WEIGHTED_ROUND_ROBIN'
   - name: 'name'
     type: String
     description: |
@@ -886,6 +935,31 @@ properties:
       character, which cannot be a dash.
     required: true
     immutable: true
+  - name: 'customMetrics'
+    type: Array
+    description: |
+      List of custom metrics that are used for the WEIGHTED_ROUND_ROBIN locality_lb_policy.
+    item_type:
+      type: NestedObject
+      properties:
+        - name: 'name'
+          type: String
+          required: true
+          description: |
+            Name of a custom utilization signal. The name must be 1-64 characters
+            long and match the regular expression [a-z]([-_.a-z0-9]*[a-z0-9])? which
+            means the first character must be a lowercase letter, and all following
+            characters must be a dash, period, underscore, lowercase letter, or
+            digit, except the last character, which cannot be a dash, period, or
+            underscore. For usage guidelines, see Custom Metrics balancing mode. This
+            field can only be used for a global or regional backend service with the
+            loadBalancingScheme set to <code>EXTERNAL_MANAGED</code>,
+            <code>INTERNAL_MANAGED</code> <code>INTERNAL_SELF_MANAGED</code>.
+        - name: 'dryRun'
+          type: Boolean
+          required: true
+          description: |
+            If true, the metric data is not used for load balancing.
   - name: 'outlierDetection'
     type: NestedObject
     description: |
diff --git a/mmv1/templates/terraform/examples/backend_service_custom_metrics.tf.tmpl b/mmv1/templates/terraform/examples/backend_service_custom_metrics.tf.tmpl
@@ -0,0 +1,54 @@
+resource "google_compute_network" "default" {
+  provider = google-beta
+  name                    = "{{index $.Vars "network_name"}}"
+}
+
+// Zonal NEG with GCE_VM_IP_PORT
+resource "google_compute_network_endpoint_group" "default" {
+  provider = google-beta
+  name                  = "{{index $.Vars "default_neg_name"}}"
+  network               = google_compute_network.default.id
+  default_port          = "90"
+  zone                  = "us-central1-a"
+  network_endpoint_type = "GCE_VM_IP_PORT"
+}
+
+resource "google_compute_backend_service" "{{$.PrimaryResourceId}}" {
+  provider = google-beta
+  name                  = "{{index $.Vars "backend_service_name"}}"
+  health_checks = [google_compute_health_check.default.id]
+
+  # WEIGHTED_ROUND_ROBIN and CUSTOM_METRICS require EXTERNAL_MANAGED.
+  load_balancing_scheme = "EXTERNAL_MANAGED"
+  locality_lb_policy    = "WEIGHTED_ROUND_ROBIN"
+  custom_metrics {
+    name    = "orca.application_utilization"
+    # At least one metric should be not dry_run.
+    dry_run = false
+  }
+  backend {
+    group = google_compute_network_endpoint_group.default.id
+    balancing_mode = "CUSTOM_METRICS"
+    custom_metrics {
+      name    = "orca.cpu_utilization"
+      max_utilization = 0.9
+      dry_run = true
+    }
+    custom_metrics {
+      name    = "orca.named_metrics.foo"
+      # At least one metric should be not dry_run.
+      dry_run = false
+    }
+  }
+}
+
+resource "google_compute_health_check" "default" {
+  provider = google-beta
+  name               = "{{index $.Vars "health_check_name"}}"
+  timeout_sec        = 1
+  check_interval_sec = 1
+
+  tcp_health_check {
+    port = "80"
+  }
+}
diff --git a/mmv1/templates/terraform/examples/region_backend_service_ilb_custom_metrics.tf.tmpl b/mmv1/templates/terraform/examples/region_backend_service_ilb_custom_metrics.tf.tmpl
@@ -0,0 +1,51 @@
+resource "google_compute_network" "default" {
+  provider = google-beta
+  name                    = "{{index $.Vars "network_name"}}"
+}
+
+// Zonal NEG with GCE_VM_IP_PORT
+resource "google_compute_network_endpoint_group" "default" {
+  provider = google-beta
+  name                  = "{{index $.Vars "default_neg_name"}}"
+  network               = google_compute_network.default.id
+  default_port          = "90"
+  zone                  = "us-central1-a"
+  network_endpoint_type = "GCE_VM_IP_PORT"
+}
+
+resource "google_compute_region_backend_service" "{{$.PrimaryResourceId}}" {
+  provider = google-beta
+
+  region = "us-central1"
+  name = "{{index $.Vars "region_backend_service_name"}}"
+  health_checks = [google_compute_health_check.health_check.id]
+  load_balancing_scheme = "INTERNAL_MANAGED"
+  locality_lb_policy    = "WEIGHTED_ROUND_ROBIN"
+  custom_metrics {
+    name    = "orca.application_utilization"
+    # At least one metric should be not dry_run.
+    dry_run = false
+  }
+  backend {
+    group = google_compute_network_endpoint_group.default.id
+    balancing_mode = "CUSTOM_METRICS"
+    custom_metrics {
+      name    = "orca.cpu_utilization"
+      max_utilization = 0.9
+      dry_run = true
+    }
+    custom_metrics {
+      name    = "orca.named_metrics.foo"
+      # At least one metric should be not dry_run.
+      dry_run = false
+    }
+  }
+}
+
+resource "google_compute_health_check" "health_check" {
+  provider = google-beta
+  name               = "{{index $.Vars "health_check_name"}}"
+  http_health_check {
+    port = 80
+  }
+}
diff --git a/mmv1/third_party/terraform/services/compute/resource_compute_backend_service_test.go.tmpl b/mmv1/third_party/terraform/services/compute/resource_compute_backend_service_test.go.tmpl
