Skip to content

Commit b71a296

Browse files
modular-magicianmodular-magician
authored
Add local_ssd_interface to Dataproc Cluster's disk_config. (#10663) (#18137)
[upstream:87e753c0b86b3e5f00938d90c2080bf2748d62b8] Signed-off-by: Modular Magician <[email protected]>
1 parent 73e539b commit b71a296

File tree

3 files changed

+109
-63
lines changed

3 files changed

+109
-63
lines changed

‎google/services/dataproc/resource_dataproc_cluster.go

+96-63
Original file line numberDiff line numberDiff line change
@@ -78,11 +78,9 @@ var (
7878
"cluster_config.0.gce_cluster_config.0.reservation_affinity.0.values",
7979
}
8080

81-
preemptibleWorkerDiskConfigKeys = []string{
82-
"cluster_config.0.preemptible_worker_config.0.disk_config.0.num_local_ssds",
83-
"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_size_gb",
84-
"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_type",
85-
}
81+
masterDiskConfigKeys = diskConfigKeys("master_config")
82+
workerDiskConfigKeys = diskConfigKeys("worker_config")
83+
preemptibleWorkerDiskConfigKeys = diskConfigKeys("preemptible_worker_config")
8684

8785
clusterSoftwareConfigKeys = []string{
8886
"cluster_config.0.software_config.0.image_version",
@@ -122,6 +120,18 @@ var (
122120
const resourceDataprocGoogleLabelPrefix = "goog-dataproc"
123121
const resourceDataprocGoogleProvidedLabelPrefix = "labels." + resourceDataprocGoogleLabelPrefix
124122

123+
// The keys inside a DiskConfig. configName is the name of the field this disk
124+
// config is inside. E.g. 'master_config', 'worker_config', or
125+
// 'preemptible_worker_config'.
126+
func diskConfigKeys(configName string) []string {
127+
return []string{
128+
"cluster_config.0." + configName + ".0.disk_config.0.num_local_ssds",
129+
"cluster_config.0." + configName + ".0.disk_config.0.boot_disk_size_gb",
130+
"cluster_config.0." + configName + ".0.disk_config.0.boot_disk_type",
131+
"cluster_config.0." + configName + ".0.disk_config.0.local_ssd_interface",
132+
}
133+
}
134+
125135
func resourceDataprocLabelDiffSuppress(k, old, new string, d *schema.ResourceData) bool {
126136
if strings.HasPrefix(k, resourceDataprocGoogleProvidedLabelPrefix) && new == "" {
127137
return true
@@ -821,43 +831,39 @@ func ResourceDataprocCluster() *schema.Resource {
821831
Elem: &schema.Resource{
822832
Schema: map[string]*schema.Schema{
823833
"num_local_ssds": {
824-
Type: schema.TypeInt,
825-
Optional: true,
826-
Computed: true,
827-
Description: `The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0.`,
828-
AtLeastOneOf: []string{
829-
"cluster_config.0.master_config.0.disk_config.0.num_local_ssds",
830-
"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb",
831-
"cluster_config.0.master_config.0.disk_config.0.boot_disk_type",
832-
},
833-
ForceNew: true,
834+
Type: schema.TypeInt,
835+
Optional: true,
836+
Computed: true,
837+
Description: `The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0.`,
838+
AtLeastOneOf: masterDiskConfigKeys,
839+
ForceNew: true,
834840
},
835841

836842
"boot_disk_size_gb": {
837-
Type: schema.TypeInt,
838-
Optional: true,
839-
Computed: true,
840-
Description: `Size of the primary disk attached to each node, specified in GB. The primary disk contains the boot volume and system libraries, and the smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories.`,
841-
AtLeastOneOf: []string{
842-
"cluster_config.0.master_config.0.disk_config.0.num_local_ssds",
843-
"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb",
844-
"cluster_config.0.master_config.0.disk_config.0.boot_disk_type",
845-
},
843+
Type: schema.TypeInt,
844+
Optional: true,
845+
Computed: true,
846+
Description: `Size of the primary disk attached to each node, specified in GB. The primary disk contains the boot volume and system libraries, and the smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories.`,
847+
AtLeastOneOf: masterDiskConfigKeys,
846848
ForceNew: true,
847849
ValidateFunc: validation.IntAtLeast(10),
848850
},
849851

850852
"boot_disk_type": {
851-
Type: schema.TypeString,
852-
Optional: true,
853-
Description: `The disk type of the primary disk attached to each node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
854-
AtLeastOneOf: []string{
855-
"cluster_config.0.master_config.0.disk_config.0.num_local_ssds",
856-
"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb",
857-
"cluster_config.0.master_config.0.disk_config.0.boot_disk_type",
858-
},
859-
ForceNew: true,
860-
Default: "pd-standard",
853+
Type: schema.TypeString,
854+
Optional: true,
855+
Description: `The disk type of the primary disk attached to each node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
856+
AtLeastOneOf: masterDiskConfigKeys,
857+
ForceNew: true,
858+
Default: "pd-standard",
859+
},
860+
861+
"local_ssd_interface": {
862+
Type: schema.TypeString,
863+
Optional: true,
864+
Description: `Interface type of local SSDs (default is "scsi"). Valid values: "scsi" (Small Computer System Interface), "nvme" (Non-Volatile Memory Express).`,
865+
AtLeastOneOf: masterDiskConfigKeys,
866+
ForceNew: true,
861867
},
862868
},
863869
},
@@ -970,43 +976,39 @@ func ResourceDataprocCluster() *schema.Resource {
970976
Elem: &schema.Resource{
971977
Schema: map[string]*schema.Schema{
972978
"num_local_ssds": {
973-
Type: schema.TypeInt,
974-
Optional: true,
975-
Computed: true,
976-
Description: `The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0.`,
977-
AtLeastOneOf: []string{
978-
"cluster_config.0.worker_config.0.disk_config.0.num_local_ssds",
979-
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_size_gb",
980-
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_type",
981-
},
982-
ForceNew: true,
979+
Type: schema.TypeInt,
980+
Optional: true,
981+
Computed: true,
982+
Description: `The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0.`,
983+
AtLeastOneOf: workerDiskConfigKeys,
984+
ForceNew: true,
983985
},
984986

985987
"boot_disk_size_gb": {
986-
Type: schema.TypeInt,
987-
Optional: true,
988-
Computed: true,
989-
Description: `Size of the primary disk attached to each node, specified in GB. The primary disk contains the boot volume and system libraries, and the smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories.`,
990-
AtLeastOneOf: []string{
991-
"cluster_config.0.worker_config.0.disk_config.0.num_local_ssds",
992-
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_size_gb",
993-
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_type",
994-
},
988+
Type: schema.TypeInt,
989+
Optional: true,
990+
Computed: true,
991+
Description: `Size of the primary disk attached to each node, specified in GB. The primary disk contains the boot volume and system libraries, and the smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories.`,
992+
AtLeastOneOf: workerDiskConfigKeys,
995993
ForceNew: true,
996994
ValidateFunc: validation.IntAtLeast(10),
997995
},
998996

999997
"boot_disk_type": {
1000-
Type: schema.TypeString,
1001-
Optional: true,
1002-
Description: `The disk type of the primary disk attached to each node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
1003-
AtLeastOneOf: []string{
1004-
"cluster_config.0.worker_config.0.disk_config.0.num_local_ssds",
1005-
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_size_gb",
1006-
"cluster_config.0.worker_config.0.disk_config.0.boot_disk_type",
1007-
},
1008-
ForceNew: true,
1009-
Default: "pd-standard",
998+
Type: schema.TypeString,
999+
Optional: true,
1000+
Description: `The disk type of the primary disk attached to each node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
1001+
AtLeastOneOf: workerDiskConfigKeys,
1002+
ForceNew: true,
1003+
Default: "pd-standard",
1004+
},
1005+
1006+
"local_ssd_interface": {
1007+
Type: schema.TypeString,
1008+
Optional: true,
1009+
Description: `Interface type of local SSDs (default is "scsi"). Valid values: "scsi" (Small Computer System Interface), "nvme" (Non-Volatile Memory Express).`,
1010+
AtLeastOneOf: workerDiskConfigKeys,
1011+
ForceNew: true,
10101012
},
10111013
},
10121014
},
@@ -1132,6 +1134,14 @@ func ResourceDataprocCluster() *schema.Resource {
11321134
Default: "pd-standard",
11331135
Description: `The disk type of the primary disk attached to each preemptible worker node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
11341136
},
1137+
1138+
"local_ssd_interface": {
1139+
Type: schema.TypeString,
1140+
Optional: true,
1141+
AtLeastOneOf: preemptibleWorkerDiskConfigKeys,
1142+
ForceNew: true,
1143+
Description: `Interface type of local SSDs (default is "scsi"). Valid values: "scsi" (Small Computer System Interface), "nvme" (Non-Volatile Memory Express).`,
1144+
},
11351145
},
11361146
},
11371147
},
@@ -1616,6 +1626,13 @@ by Dataproc`,
16161626
ForceNew: true,
16171627
Default: "pd-standard",
16181628
},
1629+
1630+
"local_ssd_interface": {
1631+
Type: schema.TypeString,
1632+
Optional: true,
1633+
Description: `Interface type of local SSDs (default is "scsi"). Valid values: "scsi" (Small Computer System Interface), "nvme" (Non-Volatile Memory Express).`,
1634+
ForceNew: true,
1635+
},
16191636
},
16201637
},
16211638
},
@@ -2096,6 +2113,9 @@ func expandNodeGroupConfig(cfg map[string]interface{}) *dataproc.InstanceGroupCo
20962113
if v, ok := dcfg["boot_disk_type"]; ok {
20972114
icg.DiskConfig.BootDiskType = v.(string)
20982115
}
2116+
if v, ok := dcfg["local_ssd_interface"]; ok {
2117+
icg.DiskConfig.LocalSsdInterface = v.(string)
2118+
}
20992119
}
21002120
}
21012121

@@ -2368,6 +2388,9 @@ func expandPreemptibleInstanceGroupConfig(cfg map[string]interface{}) *dataproc.
23682388
if v, ok := dcfg["boot_disk_type"]; ok {
23692389
icg.DiskConfig.BootDiskType = v.(string)
23702390
}
2391+
if v, ok := dcfg["local_ssd_interface"]; ok {
2392+
icg.DiskConfig.LocalSsdInterface = v.(string)
2393+
}
23712394
}
23722395
}
23732396

@@ -2441,6 +2464,9 @@ func expandMasterInstanceGroupConfig(cfg map[string]interface{}) *dataproc.Insta
24412464
if v, ok := dcfg["boot_disk_type"]; ok {
24422465
icg.DiskConfig.BootDiskType = v.(string)
24432466
}
2467+
if v, ok := dcfg["local_ssd_interface"]; ok {
2468+
icg.DiskConfig.LocalSsdInterface = v.(string)
2469+
}
24442470
}
24452471
}
24462472

@@ -2482,6 +2508,9 @@ func expandWorkerInstanceGroupConfig(cfg map[string]interface{}) *dataproc.Insta
24822508
if v, ok := dcfg["boot_disk_type"]; ok {
24832509
icg.DiskConfig.BootDiskType = v.(string)
24842510
}
2511+
if v, ok := dcfg["local_ssd_interface"]; ok {
2512+
icg.DiskConfig.LocalSsdInterface = v.(string)
2513+
}
24852514
}
24862515
}
24872516

@@ -3048,6 +3077,7 @@ func flattenNodeGroupConfig(icg *dataproc.InstanceGroupConfig) []map[string]inte
30483077
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
30493078
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
30503079
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
3080+
disk["local_ssd_interface"] = icg.DiskConfig.LocalSsdInterface
30513081
}
30523082
data["accelerators"] = flattenAccelerators(icg.Accelerators)
30533083

@@ -3137,6 +3167,7 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc
31373167
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
31383168
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
31393169
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
3170+
disk["local_ssd_interface"] = icg.DiskConfig.LocalSsdInterface
31403171
}
31413172
if icg.InstanceFlexibilityPolicy != nil {
31423173
instanceFlexibilityPolicy["instance_selection_list"] = flattenInstanceSelectionList(icg.InstanceFlexibilityPolicy.InstanceSelectionList)
@@ -3191,6 +3222,7 @@ func flattenMasterInstanceGroupConfig(d *schema.ResourceData, icg *dataproc.Inst
31913222
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
31923223
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
31933224
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
3225+
disk["local_ssd_interface"] = icg.DiskConfig.LocalSsdInterface
31943226
}
31953227

31963228
data["accelerators"] = flattenAccelerators(icg.Accelerators)
@@ -3215,6 +3247,7 @@ func flattenWorkerInstanceGroupConfig(d *schema.ResourceData, icg *dataproc.Inst
32153247
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
32163248
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
32173249
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
3250+
disk["local_ssd_interface"] = icg.DiskConfig.LocalSsdInterface
32183251
}
32193252

32203253
data["accelerators"] = flattenAccelerators(icg.Accelerators)

‎google/services/dataproc/resource_dataproc_cluster_test.go

+8
Original file line numberDiff line numberDiff line change
@@ -555,6 +555,7 @@ func TestAccDataprocCluster_spotWithAuxiliaryNodeGroups(t *testing.T) {
555555
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.node_group_config.0.disk_config.0.boot_disk_size_gb", "35"),
556556
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.node_group_config.0.disk_config.0.boot_disk_type", "pd-standard"),
557557
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.node_group_config.0.disk_config.0.num_local_ssds", "1"),
558+
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.node_group_config.0.disk_config.0.local_ssd_interface", "nvme"),
558559
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.node_group_config.0.accelerators.0.accelerator_count", "1"),
559560
resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group_id", "node-group-id"),
560561
testAccCheckDataprocAuxiliaryNodeGroupAccelerator(&cluster, project),
@@ -1227,6 +1228,7 @@ func validateDataprocCluster_withConfigOverrides(n string, cluster *dataproc.Clu
12271228
{"cluster_config.0.master_config.0.disk_config.0.boot_disk_size_gb", "35", strconv.Itoa(int(cluster.Config.MasterConfig.DiskConfig.BootDiskSizeGb))},
12281229
{"cluster_config.0.master_config.0.disk_config.0.num_local_ssds", "0", strconv.Itoa(int(cluster.Config.MasterConfig.DiskConfig.NumLocalSsds))},
12291230
{"cluster_config.0.master_config.0.disk_config.0.boot_disk_type", "pd-ssd", cluster.Config.MasterConfig.DiskConfig.BootDiskType},
1231+
{"cluster_config.0.master_config.0.disk_config.0.local_ssd_interface", "nvme", cluster.Config.MasterConfig.DiskConfig.LocalSsdInterface},
12301232
{"cluster_config.0.master_config.0.machine_type", "n1-standard-2", tpgresource.GetResourceNameFromSelfLink(cluster.Config.MasterConfig.MachineTypeUri)},
12311233
{"cluster_config.0.master_config.0.instance_names.#", "3", strconv.Itoa(len(cluster.Config.MasterConfig.InstanceNames))},
12321234
{"cluster_config.0.master_config.0.min_cpu_platform", "Intel Skylake", cluster.Config.MasterConfig.MinCpuPlatform},
@@ -1235,6 +1237,7 @@ func validateDataprocCluster_withConfigOverrides(n string, cluster *dataproc.Clu
12351237
{"cluster_config.0.worker_config.0.disk_config.0.boot_disk_size_gb", "35", strconv.Itoa(int(cluster.Config.WorkerConfig.DiskConfig.BootDiskSizeGb))},
12361238
{"cluster_config.0.worker_config.0.disk_config.0.num_local_ssds", "1", strconv.Itoa(int(cluster.Config.WorkerConfig.DiskConfig.NumLocalSsds))},
12371239
{"cluster_config.0.worker_config.0.disk_config.0.boot_disk_type", "pd-standard", cluster.Config.WorkerConfig.DiskConfig.BootDiskType},
1240+
{"cluster_config.0.worker_config.0.disk_config.0.local_ssd_interface", "scsi", cluster.Config.WorkerConfig.DiskConfig.LocalSsdInterface},
12381241
{"cluster_config.0.worker_config.0.machine_type", "n1-standard-2", tpgresource.GetResourceNameFromSelfLink(cluster.Config.WorkerConfig.MachineTypeUri)},
12391242
{"cluster_config.0.worker_config.0.instance_names.#", "3", strconv.Itoa(len(cluster.Config.WorkerConfig.InstanceNames))},
12401243
{"cluster_config.0.worker_config.0.min_cpu_platform", "Intel Broadwell", cluster.Config.WorkerConfig.MinCpuPlatform},
@@ -1243,6 +1246,7 @@ func validateDataprocCluster_withConfigOverrides(n string, cluster *dataproc.Clu
12431246
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_size_gb", "35", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.DiskConfig.BootDiskSizeGb))},
12441247
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.num_local_ssds", "1", strconv.Itoa(int(cluster.Config.SecondaryWorkerConfig.DiskConfig.NumLocalSsds))},
12451248
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.boot_disk_type", "pd-ssd", cluster.Config.SecondaryWorkerConfig.DiskConfig.BootDiskType},
1249+
{"cluster_config.0.preemptible_worker_config.0.disk_config.0.local_ssd_interface", "nvme", cluster.Config.SecondaryWorkerConfig.DiskConfig.LocalSsdInterface},
12461250
{"cluster_config.0.preemptible_worker_config.0.instance_names.#", "1", strconv.Itoa(len(cluster.Config.SecondaryWorkerConfig.InstanceNames))},
12471251
}
12481252

@@ -1695,6 +1699,7 @@ resource "google_dataproc_cluster" "with_config_overrides" {
16951699
disk_config {
16961700
boot_disk_type = "pd-ssd"
16971701
boot_disk_size_gb = 35
1702+
local_ssd_interface = "nvme"
16981703
}
16991704
min_cpu_platform = "Intel Skylake"
17001705
}
@@ -1706,6 +1711,7 @@ resource "google_dataproc_cluster" "with_config_overrides" {
17061711
boot_disk_type = "pd-standard"
17071712
boot_disk_size_gb = 35
17081713
num_local_ssds = 1
1714+
local_ssd_interface = "scsi"
17091715
}
17101716
17111717
min_cpu_platform = "Intel Broadwell"
@@ -1717,6 +1723,7 @@ resource "google_dataproc_cluster" "with_config_overrides" {
17171723
boot_disk_type = "pd-ssd"
17181724
boot_disk_size_gb = 35
17191725
num_local_ssds = 1
1726+
local_ssd_interface = "nvme"
17201727
}
17211728
}
17221729
}
@@ -1967,6 +1974,7 @@ resource "google_dataproc_cluster" "with_auxiliary_node_groups" {
19671974
boot_disk_size_gb = 35
19681975
boot_disk_type = "pd-standard"
19691976
num_local_ssds = 1
1977+
local_ssd_interface = "nvme"
19701978
}
19711979
accelerators {
19721980
accelerator_count = 1

‎website/docs/r/dataproc_cluster.html.markdown

+5
Original file line numberDiff line numberDiff line change
@@ -522,6 +522,11 @@ cluster_config {
522522
* `num_local_ssds` - (Optional) The amount of local SSD disks that will be
523523
attached to each master cluster node. Defaults to 0.
524524

525+
* `local_ssd_interface` - Optional. Interface type of local SSDs (default is "scsi").
526+
Valid values: "scsi" (Small Computer System Interface), "nvme" (Non-Volatile
527+
Memory Express). See
528+
[local SSD performance](https://cloud.google.com/compute/docs/disks/local-ssd#performance).
529+
525530
* `accelerators` (Optional) The Compute Engine accelerator (GPU) configuration for these instances. Can be specified multiple times.
526531

527532
* `accelerator_type` - (Required) The short name of the accelerator type to expose to this instance. For example, `nvidia-tesla-k80`.

0 commit comments

Comments
 (0)