Skip to content

Commit 311cf8b

Browse files
modular-magicianmodular-magician
authored
add support for auxiliary_node_groups in google_dataproc_cluster (#9484) (#16798)
[upstream:daa26f918c289b2fbf73b36d4028e5d27ac4328e] Signed-off-by: Modular Magician <[email protected]>
1 parent ab9c86f commit 311cf8b

File tree

4 files changed

+443
-0
lines changed

4 files changed

+443
-0
lines changed

‎.changelog/9484.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
dataproc: added `auxiliary_node_groups` field to `google_dataproc_cluster` resource
3+
```

‎google/services/dataproc/resource_dataproc_cluster.go

+270
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,7 @@ var (
115115
"cluster_config.0.lifecycle_config",
116116
"cluster_config.0.endpoint_config",
117117
"cluster_config.0.dataproc_metric_config",
118+
"cluster_config.0.auxiliary_node_groups",
118119
}
119120
)
120121

@@ -1509,6 +1510,7 @@ by Dataproc`,
15091510
"dataproc_metric_config": {
15101511
Type: schema.TypeList,
15111512
Optional: true,
1513+
Computed: true,
15121514
MaxItems: 1,
15131515
Description: `The config for Dataproc metrics.`,
15141516
AtLeastOneOf: clusterConfigKeys,
@@ -1523,6 +1525,129 @@ by Dataproc`,
15231525
},
15241526
},
15251527
},
1528+
"auxiliary_node_groups": {
1529+
Type: schema.TypeList,
1530+
Optional: true,
1531+
Computed: true,
1532+
Description: `The node group settings.`,
1533+
AtLeastOneOf: clusterConfigKeys,
1534+
Elem: &schema.Resource{
1535+
Schema: map[string]*schema.Schema{
1536+
"node_group": {
1537+
Required: true,
1538+
Description: `Node group configuration.`,
1539+
Type: schema.TypeList,
1540+
Elem: &schema.Resource{
1541+
Schema: map[string]*schema.Schema{
1542+
"name": {
1543+
Description: `The Node group resource name.`,
1544+
Type: schema.TypeString,
1545+
Computed: true,
1546+
},
1547+
"roles": {
1548+
Type: schema.TypeList,
1549+
Description: `Node group roles.`,
1550+
Required: true,
1551+
ForceNew: true,
1552+
Elem: &schema.Schema{
1553+
Type: schema.TypeString,
1554+
ValidateFunc: validation.StringInSlice([]string{"ROLE_UNSPECIFIED", "DRIVER"}, false),
1555+
},
1556+
},
1557+
"node_group_config": {
1558+
Description: `The node group instance group configuration.`,
1559+
Optional: true,
1560+
Computed: true,
1561+
MaxItems: 1,
1562+
Type: schema.TypeList,
1563+
Elem: &schema.Resource{
1564+
Schema: map[string]*schema.Schema{
1565+
"num_instances": {
1566+
Type: schema.TypeInt,
1567+
Optional: true,
1568+
ForceNew: true,
1569+
Computed: true,
1570+
Description: `Specifies the number of auxiliary nodes to create. If not specified, GCP will default to a predetermined computed value.`,
1571+
},
1572+
"machine_type": {
1573+
Type: schema.TypeString,
1574+
Optional: true,
1575+
Computed: true,
1576+
ForceNew: true,
1577+
Description: `The name of a Google Compute Engine machine type to create for the master`,
1578+
},
1579+
"min_cpu_platform": {
1580+
Type: schema.TypeString,
1581+
Optional: true,
1582+
Computed: true,
1583+
ForceNew: true,
1584+
Description: `The name of a minimum generation of CPU family for the auxiliary node group. If not specified, GCP will default to a predetermined computed value for each zone.`,
1585+
},
1586+
"disk_config": {
1587+
1588+
Type: schema.TypeList,
1589+
Optional: true,
1590+
Computed: true,
1591+
MaxItems: 1,
1592+
Description: `Disk Config`,
1593+
Elem: &schema.Resource{
1594+
Schema: map[string]*schema.Schema{
1595+
"num_local_ssds": {
1596+
Type: schema.TypeInt,
1597+
Optional: true,
1598+
Computed: true,
1599+
Description: `The amount of local SSD disks that will be attached to each master cluster node. Defaults to 0.`,
1600+
ForceNew: true,
1601+
},
1602+
1603+
"boot_disk_size_gb": {
1604+
Type: schema.TypeInt,
1605+
Optional: true,
1606+
Computed: true,
1607+
Description: `Size of the primary disk attached to each node, specified in GB. The primary disk contains the boot volume and system libraries, and the smallest allowed disk size is 10GB. GCP will default to a predetermined computed value if not set (currently 500GB). Note: If SSDs are not attached, it also contains the HDFS data blocks and Hadoop working directories.`,
1608+
ForceNew: true,
1609+
ValidateFunc: validation.IntAtLeast(10),
1610+
},
1611+
1612+
"boot_disk_type": {
1613+
Type: schema.TypeString,
1614+
Optional: true,
1615+
Description: `The disk type of the primary disk attached to each node. Such as "pd-ssd" or "pd-standard". Defaults to "pd-standard".`,
1616+
ForceNew: true,
1617+
Default: "pd-standard",
1618+
},
1619+
},
1620+
},
1621+
},
1622+
"accelerators": {
1623+
Type: schema.TypeSet,
1624+
Optional: true,
1625+
ForceNew: true,
1626+
Elem: acceleratorsSchema(),
1627+
Description: `The Compute Engine accelerator (GPU) configuration for these instances. Can be specified multiple times.`,
1628+
},
1629+
"instance_names": {
1630+
Type: schema.TypeList,
1631+
Computed: true,
1632+
Elem: &schema.Schema{Type: schema.TypeString},
1633+
Description: `List of auxiliary node group instance names which have been assigned to the cluster.`,
1634+
},
1635+
},
1636+
},
1637+
},
1638+
},
1639+
},
1640+
},
1641+
"node_group_id": {
1642+
Computed: true,
1643+
Optional: true,
1644+
ForceNew: true,
1645+
Type: schema.TypeString,
1646+
Description: `A node group ID. Generated if not specified. The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of from 3 to 33 characters.`,
1647+
},
1648+
},
1649+
},
1650+
},
15261651
},
15271652
},
15281653
},
@@ -1895,9 +2020,89 @@ func expandClusterConfig(d *schema.ResourceData, config *transport_tpg.Config) (
18952020
log.Println("[INFO] got preemptible worker config")
18962021
conf.SecondaryWorkerConfig = expandPreemptibleInstanceGroupConfig(cfg)
18972022
}
2023+
2024+
if v, ok := d.GetOk("cluster_config.0.auxiliary_node_groups"); ok {
2025+
log.Println("[INFO] got auxiliary node group config")
2026+
conf.AuxiliaryNodeGroups = expandAuxiliaryNodeGroupsConfig(v)
2027+
}
18982028
return conf, nil
18992029
}
19002030

2031+
func expandAuxiliaryNodeGroupsConfig(v interface{}) []*dataproc.AuxiliaryNodeGroup {
2032+
auxiliaryNodeGroupsList := v.([]interface{})
2033+
2034+
auxiliaryNodeGroups := []*dataproc.AuxiliaryNodeGroup{}
2035+
for _, v1 := range auxiliaryNodeGroupsList {
2036+
auxiliaryNodeGroupItem := v1.(map[string]interface{})
2037+
auxiliaryNodeGroup := &dataproc.AuxiliaryNodeGroup{
2038+
NodeGroup: expandNodeGroup(auxiliaryNodeGroupItem["node_group"].([]interface{})[0].(map[string]interface{})),
2039+
}
2040+
if x, ok := auxiliaryNodeGroupItem["node_group_id"]; ok {
2041+
auxiliaryNodeGroup.NodeGroupId = x.(string)
2042+
}
2043+
auxiliaryNodeGroups = append(auxiliaryNodeGroups, auxiliaryNodeGroup)
2044+
}
2045+
2046+
return auxiliaryNodeGroups
2047+
}
2048+
2049+
func expandNodeGroup(cfg map[string]interface{}) *dataproc.NodeGroup {
2050+
conf := &dataproc.NodeGroup{}
2051+
roles := []string{}
2052+
roleList := cfg["roles"]
2053+
for _, v1 := range roleList.([]interface{}) {
2054+
roles = append(roles, v1.(string))
2055+
}
2056+
conf.Roles = roles
2057+
2058+
if v, ok := cfg["name"]; ok {
2059+
conf.Name = v.(string)
2060+
}
2061+
2062+
if v, ok := cfg["node_group_config"]; ok {
2063+
ng := v.([]interface{})
2064+
if len(ng) > 0 {
2065+
conf.NodeGroupConfig = expandNodeGroupConfig(v.([]interface{})[0].(map[string]interface{}))
2066+
}
2067+
}
2068+
return conf
2069+
}
2070+
2071+
func expandNodeGroupConfig(cfg map[string]interface{}) *dataproc.InstanceGroupConfig {
2072+
icg := &dataproc.InstanceGroupConfig{}
2073+
2074+
if v, ok := cfg["num_instances"]; ok {
2075+
icg.NumInstances = int64(v.(int))
2076+
}
2077+
if v, ok := cfg["machine_type"]; ok {
2078+
icg.MachineTypeUri = tpgresource.GetResourceNameFromSelfLink(v.(string))
2079+
}
2080+
if v, ok := cfg["min_cpu_platform"]; ok {
2081+
icg.MinCpuPlatform = v.(string)
2082+
}
2083+
2084+
if dc, ok := cfg["disk_config"]; ok {
2085+
d := dc.([]interface{})
2086+
if len(d) > 0 {
2087+
dcfg := d[0].(map[string]interface{})
2088+
icg.DiskConfig = &dataproc.DiskConfig{}
2089+
2090+
if v, ok := dcfg["boot_disk_size_gb"]; ok {
2091+
icg.DiskConfig.BootDiskSizeGb = int64(v.(int))
2092+
}
2093+
if v, ok := dcfg["num_local_ssds"]; ok {
2094+
icg.DiskConfig.NumLocalSsds = int64(v.(int))
2095+
}
2096+
if v, ok := dcfg["boot_disk_type"]; ok {
2097+
icg.DiskConfig.BootDiskType = v.(string)
2098+
}
2099+
}
2100+
}
2101+
2102+
icg.Accelerators = expandAccelerators(cfg["accelerators"].(*schema.Set).List())
2103+
return icg
2104+
}
2105+
19012106
func expandGceClusterConfig(d *schema.ResourceData, config *transport_tpg.Config) (*dataproc.GceClusterConfig, error) {
19022107
conf := &dataproc.GceClusterConfig{}
19032108

@@ -2623,6 +2828,15 @@ func flattenClusterConfig(d *schema.ResourceData, cfg *dataproc.ClusterConfig) (
26232828
}
26242829
data["initialization_action"] = val
26252830
}
2831+
2832+
if len(cfg.AuxiliaryNodeGroups) > 0 {
2833+
val, err := flattenAuxiliaryNodeGroups(cfg.AuxiliaryNodeGroups)
2834+
if err != nil {
2835+
return nil, err
2836+
}
2837+
data["auxiliary_node_groups"] = val
2838+
}
2839+
26262840
return []map[string]interface{}{data}, nil
26272841
}
26282842

@@ -2787,6 +3001,62 @@ func flattenInitializationActions(nia []*dataproc.NodeInitializationAction) ([]m
27873001

27883002
}
27893003

3004+
func flattenAuxiliaryNodeGroups(ang []*dataproc.AuxiliaryNodeGroup) ([]map[string]interface{}, error) {
3005+
3006+
auxiliaryNodeGroups := []map[string]interface{}{}
3007+
for _, v := range ang {
3008+
nodeGroup := map[string]interface{}{
3009+
"node_group": flatternNodeGroup(v.NodeGroup),
3010+
}
3011+
if len(v.NodeGroupId) > 0 {
3012+
nodeGroup["node_group_id"] = v.NodeGroupId
3013+
}
3014+
3015+
auxiliaryNodeGroups = append(auxiliaryNodeGroups, nodeGroup)
3016+
}
3017+
return auxiliaryNodeGroups, nil
3018+
3019+
}
3020+
3021+
func flatternNodeGroup(ng *dataproc.NodeGroup) []map[string]interface{} {
3022+
nodeGroup := map[string]interface{}{
3023+
"roles": ng.Roles,
3024+
}
3025+
3026+
if ng.Name != "" {
3027+
nodeGroup["name"] = ng.Name
3028+
}
3029+
3030+
if ng.NodeGroupConfig != nil {
3031+
nodeGroup["node_group_config"] = flattenNodeGroupConfig(ng.NodeGroupConfig)
3032+
}
3033+
3034+
return []map[string]interface{}{nodeGroup}
3035+
3036+
}
3037+
3038+
func flattenNodeGroupConfig(icg *dataproc.InstanceGroupConfig) []map[string]interface{} {
3039+
disk := map[string]interface{}{}
3040+
data := map[string]interface{}{}
3041+
3042+
if icg != nil {
3043+
data["num_instances"] = icg.NumInstances
3044+
data["machine_type"] = tpgresource.GetResourceNameFromSelfLink(icg.MachineTypeUri)
3045+
data["min_cpu_platform"] = icg.MinCpuPlatform
3046+
data["instance_names"] = icg.InstanceNames
3047+
if icg.DiskConfig != nil {
3048+
disk["boot_disk_size_gb"] = icg.DiskConfig.BootDiskSizeGb
3049+
disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds
3050+
disk["boot_disk_type"] = icg.DiskConfig.BootDiskType
3051+
}
3052+
data["accelerators"] = flattenAccelerators(icg.Accelerators)
3053+
3054+
}
3055+
3056+
data["disk_config"] = []map[string]interface{}{disk}
3057+
return []map[string]interface{}{data}
3058+
}
3059+
27903060
func flattenGceClusterConfig(d *schema.ResourceData, gcc *dataproc.GceClusterConfig) []map[string]interface{} {
27913061
if gcc == nil {
27923062
return []map[string]interface{}{}

0 commit comments

Comments
 (0)