Add support to user-managed firewall rules for GKE Load Balancer Services

.changelog/13118.txt

@@ -0,0 +1,6 @@
+```release-note:enhancement
+container: added `disable_l4_lb_firewall_reconciliation` field to `google_container_cluster` resource
+```
+```release-note:note
+Upgrade dependency google.golang.org/api to v0.221.0
+```

google-beta/services/container/resource_container_cluster.go

@@ -2130,6 +2130,12 @@ func ResourceContainerCluster() *schema.Resource {
 				Description: `Whether L4ILB Subsetting is enabled for this cluster.`,
 				Default:     false,
 			},
+			"disable_l4_lb_firewall_reconciliation": {
+				Type:        schema.TypeBool,
+				Optional:    true,
+				Description: `Disable L4 load balancer VPC firewalls to enable firewall policies.`,
+				Default:     false,
+			},
 			"enable_multi_networking": {
 				Type:        schema.TypeBool,
 				Optional:    true,
@@ -2544,6 +2550,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er
 			EnableCiliumClusterwideNetworkPolicy: d.Get("enable_cilium_clusterwide_network_policy").(bool),
 			PrivateIpv6GoogleAccess:              d.Get("private_ipv6_google_access").(string),
 			EnableL4ilbSubsetting:                d.Get("enable_l4_ilb_subsetting").(bool),
+			DisableL4LbFirewallReconciliation:    d.Get("disable_l4_lb_firewall_reconciliation").(bool),
 			DnsConfig:                            expandDnsConfig(d.Get("dns_config")),
 			GatewayApiConfig:                     expandGatewayApiConfig(d.Get("gateway_api_config")),
 			EnableMultiNetworking:                d.Get("enable_multi_networking").(bool),
@@ -3090,6 +3097,9 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro
 	if err := d.Set("enable_l4_ilb_subsetting", cluster.NetworkConfig.EnableL4ilbSubsetting); err != nil {
 		return fmt.Errorf("Error setting enable_l4_ilb_subsetting: %s", err)
 	}
+	if err := d.Set("disable_l4_lb_firewall_reconciliation", cluster.NetworkConfig.DisableL4LbFirewallReconciliation); err != nil {
+		return fmt.Errorf("Error setting disable_l4_lb_firewall_reconciliation: %s", err)
+	}
 	if err := d.Set("cost_management_config", flattenManagementConfig(cluster.CostManagementConfig)); err != nil {
 		return fmt.Errorf("Error setting cost_management_config: %s", err)
 	}
@@ -3592,6 +3602,40 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er
 		log.Printf("[INFO] GKE cluster %s L4 ILB Subsetting has been updated to %v", d.Id(), enabled)
 	}
 
+	if d.HasChange("disable_l4_lb_firewall_reconciliation") {
+		enabled := d.Get("disable_l4_lb_firewall_reconciliation").(bool)
+		req := &container.UpdateClusterRequest{
+			Update: &container.ClusterUpdate{
+				DesiredDisableL4LbFirewallReconciliation: enabled,
+				ForceSendFields:                          []string{"DesiredDisableL4LbFirewallReconciliation"},
+			},
+		}
+		updateF := func() error {
+			log.Println("[DEBUG] updating disable_l4_lb_firewall_reconciliation")
+			name := containerClusterFullName(project, location, clusterName)
+			clusterUpdateCall := config.NewContainerClient(userAgent).Projects.Locations.Clusters.Update(name, req)
+			if config.UserProjectOverride {
+				clusterUpdateCall.Header().Add("X-Goog-User-Project", project)
+			}
+			op, err := clusterUpdateCall.Do()
+			if err != nil {
+				return err
+			}
+
+			// Wait until it's updated
+			err = ContainerOperationWait(config, op, project, location, "updating Disable L4 LB Firewall Reconciliation", userAgent, d.Timeout(schema.TimeoutUpdate))
+			log.Println("[DEBUG] done updating disable_l4_lb_firewall_reconciliation")
+			return err
+		}
+
+		// Call update serially.
+		if err := transport_tpg.LockedCall(lockKey, updateF); err != nil {
+			return err
+		}
+
+		log.Printf("[INFO] GKE cluster %s Disable L4 LB Firewall Reconciliation has been updated to %v", d.Id(), enabled)
+	}
+
 	if d.HasChange("enable_fqdn_network_policy") {
 		enabled := d.Get("enable_fqdn_network_policy").(bool)
 		req := &container.UpdateClusterRequest{

google-beta/services/container/resource_container_cluster_meta.yaml

@@ -121,6 +121,8 @@ fields:
   - field: 'enable_kubernetes_alpha'
   - field: 'enable_l4_ilb_subsetting'
     api_field: 'network_config.enable_l4ilb_subsetting'
+  - field: 'disable_l4_lb_firewall_reconciliation'
+    api_field: 'network_config.disable_l4_lb_firewall_reconciliation'
   - field: 'enable_legacy_abac'
     api_field: 'legacy_abac.enabled'
   - field: 'enable_multi_networking'

google-beta/services/container/resource_container_cluster_test.go

@@ -549,6 +549,50 @@ func TestAccContainerCluster_withILBSubsetting(t *testing.T) {
 	})
 }
 
+func TestAccContainerCluster_disableL4LbFirewallReconciliation(t *testing.T) {
+	t.Parallel()
+
+	clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
+	npName := fmt.Sprintf("tf-test-cluster-nodepool-%s", acctest.RandString(t, 10))
+	networkName := acctest.BootstrapSharedTestNetwork(t, "gke-cluster")
+	subnetworkName := acctest.BootstrapSubnet(t, "gke-cluster", networkName)
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckContainerClusterDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccContainerCluster_disableL4LbFirewallReconciliation(clusterName, npName, networkName, subnetworkName),
+			},
+			{
+				ResourceName:            "google_container_cluster.confidential_nodes",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+			},
+			{
+				Config: testAccContainerCluster_enableL4LbFirewallReconciliation(clusterName, npName, networkName, subnetworkName),
+			},
+			{
+				ResourceName:            "google_container_cluster.confidential_nodes",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+			},
+			{
+				Config: testAccContainerCluster_disableL4LbFirewallReconciliation(clusterName, npName, networkName, subnetworkName),
+			},
+			{
+				ResourceName:            "google_container_cluster.confidential_nodes",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"deletion_protection"},
+			},
+		},
+	})
+}
+
 func TestAccContainerCluster_withMultiNetworking(t *testing.T) {
 	t.Parallel()
 
@@ -7027,6 +7071,62 @@ resource "google_container_cluster" "confidential_nodes" {
 `, clusterName, npName, networkName, subnetworkName)
 }
 
+func testAccContainerCluster_disableL4LbFirewallReconciliation(clusterName, npName, networkName, subnetworkName string) string {
+	return fmt.Sprintf(`
+resource "google_container_cluster" "confidential_nodes" {
+  name     = "%s"
+  location = "us-central1-a"
+  release_channel {
+    channel = "RAPID"
+  }
+
+  node_pool {
+    name               = "%s"
+    initial_node_count = 1
+    node_config {
+      machine_type = "e2-medium"
+    }
+  }
+
+  enable_l4_ilb_subsetting = true
+  disable_l4_lb_firewall_reconciliation = true
+
+  network    = "%s"
+  subnetwork = "%s"
+
+  deletion_protection = false
+}
+`, clusterName, npName, networkName, subnetworkName)
+}
+
+func testAccContainerCluster_enableL4LbFirewallReconciliation(clusterName, npName, networkName, subnetworkName string) string {
+	return fmt.Sprintf(`
+resource "google_container_cluster" "confidential_nodes" {
+  name     = "%s"
+  location = "us-central1-a"
+  release_channel {
+    channel = "RAPID"
+  }
+
+  node_pool {
+    name               = "%s"
+    initial_node_count = 1
+    node_config {
+      machine_type = "e2-medium"
+    }
+  }
+
+  enable_l4_ilb_subsetting = true
+  disable_l4_lb_firewall_reconciliation = false
+
+  network    = "%s"
+  subnetwork = "%s"
+
+  deletion_protection = false
+}
+`, clusterName, npName, networkName, subnetworkName)
+}
+
 func testAccContainerCluster_withNetworkPolicyEnabled(clusterName, networkName, subnetworkName string) string {
 	return fmt.Sprintf(`
 resource "google_container_cluster" "with_network_policy_enabled" {

website/docs/r/container_cluster.html.markdown

@@ -372,6 +372,9 @@ subnetwork in which the cluster's instances are launched.
 * `enable_l4_ilb_subsetting` - (Optional)
     Whether L4ILB Subsetting is enabled for this cluster.
 
+* `disable_l4_lb_firewall_reconciliation` - (Optional)
+    Disable L4 load balancer VPC firewalls to enable firewall policies.
+
 * `enable_multi_networking` - (Optional)
     Whether multi-networking is enabled for this cluster.
 
@@ -568,8 +571,8 @@ in addition to node auto-provisioning. Structure is [documented below](#nested_r
 GKE Autopilot clusters.
 Structure is [documented below](#nested_auto_provisioning_defaults).
 
-* `auto_provisioning_locations` - (Optional) The list of Google Compute Engine 
-[zones](https://cloud.google.com/compute/docs/zones#available) in which the 
+* `auto_provisioning_locations` - (Optional) The list of Google Compute Engine
+[zones](https://cloud.google.com/compute/docs/zones#available) in which the
 NodePool's nodes can be created by NAP.
 
 * `autoscaling_profile` - (Optional) Configuration
@@ -670,7 +673,7 @@ This block also contains several computed attributes, documented below.
 
 <a name="auto_monitoring_config"></a>The `auto_monitoring_config` block supports:
 
-* `scope` - (Required) Whether or not to enable GKE Auto-Monitoring. Supported values include: `ALL`, `NONE`.  
+* `scope` - (Required) Whether or not to enable GKE Auto-Monitoring. Supported values include: `ALL`, `NONE`.
 
 <a name="nested_advanced_datapath_observability_config"></a>The `advanced_datapath_observability_config` block supports:
 
@@ -912,7 +915,7 @@ gvnic {
 
 * `guest_accelerator` - (Optional) List of the type and count of accelerator cards attached to the instance.
     Structure [documented below](#nested_guest_accelerator).
-    **Note**: As of 6.0.0, [argument syntax](https://developer.hashicorp.com/terraform/language/syntax/configuration#arguments) 
+    **Note**: As of 6.0.0, [argument syntax](https://developer.hashicorp.com/terraform/language/syntax/configuration#arguments)
     is no longer supported for this field in favor of [block syntax](https://developer.hashicorp.com/terraform/language/syntax/configuration#blocks).
     To dynamically set a list of guest accelerators, use [dynamic blocks](https://developer.hashicorp.com/terraform/language/expressions/dynamic-blocks).
     To set an empty list, use a single `guest_accelerator` block with `count = 0`.
@@ -1359,7 +1362,7 @@ Enables monitoring and attestation of the boot integrity of the instance. The at
 * `cpu_manager_policy` - (Optional) The CPU management policy on the node. See
 [K8S CPU Management Policies](https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/).
 One of `"none"` or `"static"`. If unset (or set to the empty string `""`), the API will treat the field as if set to "none".
-Prior to the 6.4.0 this field was marked as required. The workaround for the required field 
+Prior to the 6.4.0 this field was marked as required. The workaround for the required field
 is setting the empty string `""`, which will function identically to not setting this field.
 
 * `cpu_cfs_quota` - (Optional) If true, enables CPU CFS quota enforcement for
@@ -1428,7 +1431,7 @@ linux_node_config {
 
   * `enabled` (Required) - Enables private registry config. If set to false, all other fields in this object must not be set.
 
-  * `certificate_authority_domain_config` (Optional) - List of configuration objects for CA and domains. Each object identifies a certificate and its assigned domains. See [how to configure for private container registries](https://cloud.google.com/kubernetes-engine/docs/how-to/access-private-registries-private-certificates) for more detail. Example: 
+  * `certificate_authority_domain_config` (Optional) - List of configuration objects for CA and domains. Each object identifies a certificate and its assigned domains. See [how to configure for private container registries](https://cloud.google.com/kubernetes-engine/docs/how-to/access-private-registries-private-certificates) for more detail. Example:
   ```hcl
   certificate_authority_domain_config {
     fqdns = [
@@ -1472,7 +1475,7 @@ linux_node_config {
 
 <a name="nested_security_posture_config"></a>The `security_posture_config` block supports:
 
-**Note:** `ENTERPRISE` and `VULNERABILITY_ENTERPRISE` are only available for [GKE Enterprise](http://cloud/kubernetes-engine/enterprise/docs/concepts/overview) projects.  
+**Note:** `ENTERPRISE` and `VULNERABILITY_ENTERPRISE` are only available for [GKE Enterprise](http://cloud/kubernetes-engine/enterprise/docs/concepts/overview) projects.
 
 * `mode` - (Optional) Sets the mode of the Kubernetes security posture API's off-cluster features. Available options include `DISABLED`, `BASIC`, and `ENTERPRISE`.