Added field network_config to resource google_dataproc_metastore_service (#6893) (#13184)

Signed-off-by: Modular Magician <[email protected]>

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.changelog/6893.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+metastore: added general field `network_config` to `google_dataproc_metastore_service`
+```

google/resource_dataproc_metastore_service.go

@@ -184,6 +184,40 @@ Maintenance window is not needed for services with the 'SPANNER' database type.`
 
 "projects/{projectNumber}/global/networks/{network_id}".`,
 			},
+			"network_config": {
+				Type:        schema.TypeList,
+				Optional:    true,
+				ForceNew:    true,
+				Description: `The configuration specifying the network settings for the Dataproc Metastore service.`,
+				MaxItems:    1,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"consumers": {
+							Type:        schema.TypeList,
+							Required:    true,
+							ForceNew:    true,
+							Description: `The consumer-side network configuration for the Dataproc Metastore instance.`,
+							Elem: &schema.Resource{
+								Schema: map[string]*schema.Schema{
+									"subnetwork": {
+										Type:     schema.TypeString,
+										Required: true,
+										Description: `The subnetwork of the customer project from which an IP address is reserved and used as the Dataproc Metastore service's endpoint.
+It is accessible to hosts in the subnet and to all hosts in a subnet in the same region and same network.
+There must be at least one IP address available in the subnet's primary range. The subnet is specified in the following form:
+'projects/{projectNumber}/regions/{region_id}/subnetworks/{subnetwork_id}`,
+									},
+									"endpoint_uri": {
+										Type:        schema.TypeString,
+										Computed:    true,
+										Description: `The URI of the endpoint used to access the metastore service.`,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
 			"port": {
 				Type:        schema.TypeInt,
 				Computed:    true,
@@ -296,6 +330,12 @@ func resourceDataprocMetastoreServiceCreate(d *schema.ResourceData, meta interfa
 	} else if v, ok := d.GetOkExists("hive_metastore_config"); !isEmptyValue(reflect.ValueOf(hiveMetastoreConfigProp)) && (ok || !reflect.DeepEqual(v, hiveMetastoreConfigProp)) {
 		obj["hiveMetastoreConfig"] = hiveMetastoreConfigProp
 	}
+	networkConfigProp, err := expandDataprocMetastoreServiceNetworkConfig(d.Get("network_config"), d, config)
+	if err != nil {
+		return err
+	} else if v, ok := d.GetOkExists("network_config"); !isEmptyValue(reflect.ValueOf(networkConfigProp)) && (ok || !reflect.DeepEqual(v, networkConfigProp)) {
+		obj["networkConfig"] = networkConfigProp
+	}
 	databaseTypeProp, err := expandDataprocMetastoreServiceDatabaseType(d.Get("database_type"), d, config)
 	if err != nil {
 		return err
@@ -425,6 +465,9 @@ func resourceDataprocMetastoreServiceRead(d *schema.ResourceData, meta interface
 	if err := d.Set("hive_metastore_config", flattenDataprocMetastoreServiceHiveMetastoreConfig(res["hiveMetastoreConfig"], d, config)); err != nil {
 		return fmt.Errorf("Error reading Service: %s", err)
 	}
+	if err := d.Set("network_config", flattenDataprocMetastoreServiceNetworkConfig(res["networkConfig"], d, config)); err != nil {
+		return fmt.Errorf("Error reading Service: %s", err)
+	}
 	if err := d.Set("database_type", flattenDataprocMetastoreServiceDatabaseType(res["databaseType"], d, config)); err != nil {
 		return fmt.Errorf("Error reading Service: %s", err)
 	}
@@ -787,6 +830,46 @@ func flattenDataprocMetastoreServiceHiveMetastoreConfigKerberosConfigKrb5ConfigG
 	return v
 }
 
+func flattenDataprocMetastoreServiceNetworkConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} {
+	if v == nil {
+		return nil
+	}
+	original := v.(map[string]interface{})
+	if len(original) == 0 {
+		return nil
+	}
+	transformed := make(map[string]interface{})
+	transformed["consumers"] =
+		flattenDataprocMetastoreServiceNetworkConfigConsumers(original["consumers"], d, config)
+	return []interface{}{transformed}
+}
+func flattenDataprocMetastoreServiceNetworkConfigConsumers(v interface{}, d *schema.ResourceData, config *Config) interface{} {
+	if v == nil {
+		return v
+	}
+	l := v.([]interface{})
+	transformed := make([]interface{}, 0, len(l))
+	for _, raw := range l {
+		original := raw.(map[string]interface{})
+		if len(original) < 1 {
+			// Do not include empty json objects coming back from the api
+			continue
+		}
+		transformed = append(transformed, map[string]interface{}{
+			"endpoint_uri": flattenDataprocMetastoreServiceNetworkConfigConsumersEndpointUri(original["endpointUri"], d, config),
+			"subnetwork":   flattenDataprocMetastoreServiceNetworkConfigConsumersSubnetwork(original["subnetwork"], d, config),
+		})
+	}
+	return transformed
+}
+func flattenDataprocMetastoreServiceNetworkConfigConsumersEndpointUri(v interface{}, d *schema.ResourceData, config *Config) interface{} {
+	return v
+}
+
+func flattenDataprocMetastoreServiceNetworkConfigConsumersSubnetwork(v interface{}, d *schema.ResourceData, config *Config) interface{} {
+	return v
+}
+
 func flattenDataprocMetastoreServiceDatabaseType(v interface{}, d *schema.ResourceData, config *Config) interface{} {
 	return v
 }
@@ -991,6 +1074,62 @@ func expandDataprocMetastoreServiceHiveMetastoreConfigKerberosConfigKrb5ConfigGc
 	return v, nil
 }
 
+func expandDataprocMetastoreServiceNetworkConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	l := v.([]interface{})
+	if len(l) == 0 || l[0] == nil {
+		return nil, nil
+	}
+	raw := l[0]
+	original := raw.(map[string]interface{})
+	transformed := make(map[string]interface{})
+
+	transformedConsumers, err := expandDataprocMetastoreServiceNetworkConfigConsumers(original["consumers"], d, config)
+	if err != nil {
+		return nil, err
+	} else if val := reflect.ValueOf(transformedConsumers); val.IsValid() && !isEmptyValue(val) {
+		transformed["consumers"] = transformedConsumers
+	}
+
+	return transformed, nil
+}
+
+func expandDataprocMetastoreServiceNetworkConfigConsumers(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	l := v.([]interface{})
+	req := make([]interface{}, 0, len(l))
+	for _, raw := range l {
+		if raw == nil {
+			continue
+		}
+		original := raw.(map[string]interface{})
+		transformed := make(map[string]interface{})
+
+		transformedEndpointUri, err := expandDataprocMetastoreServiceNetworkConfigConsumersEndpointUri(original["endpoint_uri"], d, config)
+		if err != nil {
+			return nil, err
+		} else if val := reflect.ValueOf(transformedEndpointUri); val.IsValid() && !isEmptyValue(val) {
+			transformed["endpointUri"] = transformedEndpointUri
+		}
+
+		transformedSubnetwork, err := expandDataprocMetastoreServiceNetworkConfigConsumersSubnetwork(original["subnetwork"], d, config)
+		if err != nil {
+			return nil, err
+		} else if val := reflect.ValueOf(transformedSubnetwork); val.IsValid() && !isEmptyValue(val) {
+			transformed["subnetwork"] = transformedSubnetwork
+		}
+
+		req = append(req, transformed)
+	}
+	return req, nil
+}
+
+func expandDataprocMetastoreServiceNetworkConfigConsumersEndpointUri(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	return v, nil
+}
+
+func expandDataprocMetastoreServiceNetworkConfigConsumersSubnetwork(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
+	return v, nil
+}
+
 func expandDataprocMetastoreServiceDatabaseType(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
 	return v, nil
 }

google/resource_dataproc_metastore_service_test.go

@@ -51,3 +51,54 @@ resource "google_dataproc_metastore_service" "my_metastore" {
 }
 `, name, tier)
 }
+
+func TestAccDataprocMetastoreService_PrivateServiceConnect(t *testing.T) {
+	t.Parallel()
+
+	context := map[string]interface{}{
+		"random_suffix": randString(t, 10),
+	}
+
+	vcrTest(t, resource.TestCase{
+		PreCheck:     func() { testAccPreCheck(t) },
+		Providers:    testAccProviders,
+		CheckDestroy: testAccCheckDataprocMetastoreServiceDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDataprocMetastoreService_PrivateServiceConnect(context),
+			},
+			{
+				ResourceName:            "google_dataproc_metastore_service.default",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"service_id", "location"},
+			},
+		},
+	})
+}
+
+func testAccDataprocMetastoreService_PrivateServiceConnect(context map[string]interface{}) string {
+	return Nprintf(`
+// Use data source instead of creating a subnetwork due to a bug on API side.
+// With the bug, the new created subnetwork cannot be deleted when deleting the dataproc metastore service.
+data "google_compute_subnetwork" "subnet" {
+  name   = "default"
+  region = "us-central1"
+}
+
+resource "google_dataproc_metastore_service" "default" {
+  service_id = "tf-test-metastore-srv%{random_suffix}"
+  location   = "us-central1"
+
+  hive_metastore_config {
+    version = "3.1.2"
+  }
+
+  network_config {
+    consumers {
+      subnetwork = data.google_compute_subnetwork.subnet.id
+    }
+  }
+}
+`, context)
+}

website/docs/r/dataproc_metastore_service.html.markdown

@@ -23,6 +23,11 @@ description: |-
 A managed metastore service that serves metadata queries.
 
 
+To get more information about Service, see:
+
+* [API documentation](https://cloud.google.com/dataproc-metastore/docs/reference/rest/v1/projects.locations.services)
+* How-to Guides
+    * [Official Documentation](https://cloud.google.com/dataproc-metastore/docs/overview)
 
 <div class = "oics-button" style="float: right; margin: 0 0 -15px">
   <a href="https://console.cloud.google.com/cloudshell/open?cloudshell_git_repo=https%3A%2F%2Fgithub.jpy.wang%2Fterraform-google-modules%2Fdocs-examples.git&cloudshell_working_dir=dataproc_metastore_service_basic&cloudshell_image=gcr.io%2Fgraphite-cloud-shell-images%2Fterraform%3Alatest&open_in_editor=main.tf&cloudshell_print=.%2Fmotd&cloudshell_tutorial=.%2Ftutorial.md" target="_blank">
@@ -80,6 +85,38 @@ resource "google_kms_key_ring" "key_ring" {
   location = "us-central1"
 }
 ```
+## Example Usage - Dataproc Metastore Service Private Service Connect
+
+
+```hcl
+resource "google_compute_network" "net" {
+  name                    = "my-network"
+  auto_create_subnetworks = false
+}
+
+resource "google_compute_subnetwork" "subnet" {
+  name                     = "my-subnetwork"
+  region                   = "us-central1"
+  network                  = google_compute_network.net.id
+  ip_cidr_range            = "10.0.0.0/22"
+  private_ip_google_access = true
+}
+
+resource "google_dataproc_metastore_service" "default" {
+  service_id = "metastore-srv"
+  location   = "us-central1"
+
+  hive_metastore_config {
+    version = "3.1.2"
+  }
+
+  network_config {
+    consumers {
+      subnetwork = google_compute_subnetwork.subnet.id
+    }
+  }
+}
+```
 
 ## Argument Reference
 
@@ -132,6 +169,11 @@ The following arguments are supported:
   Configuration information specific to running Hive metastore software as the metastore service.
   Structure is [documented below](#nested_hive_metastore_config).
 
+* `network_config` -
+  (Optional)
+  The configuration specifying the network settings for the Dataproc Metastore service.
+  Structure is [documented below](#nested_network_config).
+
 * `database_type` -
   (Optional)
   The database type that the Metastore service stores its data.
@@ -244,6 +286,26 @@ The following arguments are supported:
   A mapping of Hive metastore configuration key-value pairs to apply to the auxiliary Hive metastore (configured in hive-site.xml) in addition to the primary version's overrides.
   If keys are present in both the auxiliary version's overrides and the primary version's overrides, the value from the auxiliary version's overrides takes precedence.
 
+<a name="nested_network_config"></a>The `network_config` block supports:
+
+* `consumers` -
+  (Required)
+  The consumer-side network configuration for the Dataproc Metastore instance.
+  Structure is [documented below](#nested_consumers).
+
+
+<a name="nested_consumers"></a>The `consumers` block supports:
+
+* `endpoint_uri` -
+  The URI of the endpoint used to access the metastore service.
+
+* `subnetwork` -
+  (Required)
+  The subnetwork of the customer project from which an IP address is reserved and used as the Dataproc Metastore service's endpoint.
+  It is accessible to hosts in the subnet and to all hosts in a subnet in the same region and same network.
+  There must be at least one IP address available in the subnet's primary range. The subnet is specified in the following form:
+  `projects/{projectNumber}/regions/{region_id}/subnetworks/{subnetwork_id}
+
 <a name="nested_metadata_integration"></a>The `metadata_integration` block supports:
 
 * `data_catalog_config` -