Skip to content

Commit b42214f

Browse files
modular-magicianmodular-magician
authored
feat(vertexai): Add fields to Vertex AI FeatureStore EntityType for Feature Value Monitoring (#6699) (#12983)
* feat: add fields for feature monitoring * fix: use default_value for the fixed default values * fix: update code based on feedback Signed-off-by: Modular Magician <[email protected]> Signed-off-by: Modular Magician <[email protected]>
1 parent e02e98c commit b42214f

5 files changed

+400
-2
lines changed

‎.changelog/6699.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:enhancement
2+
vertexai: add fields to `vertex_ai_featurestore_entitytype` to support feature value monitoring
3+
```

‎google/resource_vertex_ai_featurestore_entitytype.go

+294-1
Original file line numberDiff line numberDiff line change
@@ -64,10 +64,66 @@ If this is populated with [FeaturestoreMonitoringConfig.monitoring_interval] spe
6464
MaxItems: 1,
6565
Elem: &schema.Resource{
6666
Schema: map[string]*schema.Schema{
67+
"categorical_threshold_config": {
68+
Type: schema.TypeList,
69+
Optional: true,
70+
Description: `Threshold for categorical features of anomaly detection. This is shared by all types of Featurestore Monitoring for categorical features (i.e. Features with type (Feature.ValueType) BOOL or STRING).`,
71+
MaxItems: 1,
72+
Elem: &schema.Resource{
73+
Schema: map[string]*schema.Schema{
74+
"value": {
75+
Type: schema.TypeFloat,
76+
Required: true,
77+
Description: `Specify a threshold value that can trigger the alert. For categorical feature, the distribution distance is calculated by L-inifinity norm. Each feature must have a non-zero threshold if they need to be monitored. Otherwise no alert will be triggered for that feature. The default value is 0.3.`,
78+
},
79+
},
80+
},
81+
},
82+
"import_features_analysis": {
83+
Type: schema.TypeList,
84+
Optional: true,
85+
Description: `The config for ImportFeatures Analysis Based Feature Monitoring.`,
86+
MaxItems: 1,
87+
Elem: &schema.Resource{
88+
Schema: map[string]*schema.Schema{
89+
"anomaly_detection_baseline": {
90+
Type: schema.TypeString,
91+
Optional: true,
92+
Description: `Defines the baseline to do anomaly detection for feature values imported by each [entityTypes.importFeatureValues][] operation. The value must be one of the values below:
93+
* LATEST_STATS: Choose the later one statistics generated by either most recent snapshot analysis or previous import features analysis. If non of them exists, skip anomaly detection and only generate a statistics.
94+
* MOST_RECENT_SNAPSHOT_STATS: Use the statistics generated by the most recent snapshot analysis if exists.
95+
* PREVIOUS_IMPORT_FEATURES_STATS: Use the statistics generated by the previous import features analysis if exists.`,
96+
},
97+
"state": {
98+
Type: schema.TypeString,
99+
Optional: true,
100+
Description: `Whether to enable / disable / inherite default hebavior for import features analysis. The value must be one of the values below:
101+
* DEFAULT: The default behavior of whether to enable the monitoring. EntityType-level config: disabled.
102+
* ENABLED: Explicitly enables import features analysis. EntityType-level config: by default enables import features analysis for all Features under it.
103+
* DISABLED: Explicitly disables import features analysis. EntityType-level config: by default disables import features analysis for all Features under it.`,
104+
},
105+
},
106+
},
107+
},
108+
"numerical_threshold_config": {
109+
Type: schema.TypeList,
110+
Optional: true,
111+
Description: `Threshold for numerical features of anomaly detection. This is shared by all objectives of Featurestore Monitoring for numerical features (i.e. Features with type (Feature.ValueType) DOUBLE or INT64).`,
112+
MaxItems: 1,
113+
Elem: &schema.Resource{
114+
Schema: map[string]*schema.Schema{
115+
"value": {
116+
Type: schema.TypeFloat,
117+
Required: true,
118+
Description: `Specify a threshold value that can trigger the alert. For numerical feature, the distribution distance is calculated by Jensen–Shannon divergence. Each feature must have a non-zero threshold if they need to be monitored. Otherwise no alert will be triggered for that feature. The default value is 0.3.`,
119+
},
120+
},
121+
},
122+
},
67123
"snapshot_analysis": {
68124
Type: schema.TypeList,
69125
Optional: true,
70-
Description: `Configuration of how features in Featurestore are monitored.`,
126+
Description: `The config for Snapshot Analysis Based Feature Monitoring.`,
71127
MaxItems: 1,
72128
Elem: &schema.Resource{
73129
Schema: map[string]*schema.Schema{
@@ -77,6 +133,19 @@ If this is populated with [FeaturestoreMonitoringConfig.monitoring_interval] spe
77133
Description: `The monitoring schedule for snapshot analysis. For EntityType-level config: unset / disabled = true indicates disabled by default for Features under it; otherwise by default enable snapshot analysis monitoring with monitoringInterval for Features under it.`,
78134
Default: false,
79135
},
136+
"monitoring_interval_days": {
137+
Type: schema.TypeInt,
138+
Optional: true,
139+
Description: `Configuration of the snapshot analysis based monitoring pipeline running interval. The value indicates number of days. The default value is 1.
140+
If both FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days and [FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval][] are set when creating/updating EntityTypes/Features, FeaturestoreMonitoringConfig.SnapshotAnalysis.monitoring_interval_days will be used.`,
141+
Default: 1,
142+
},
143+
"staleness_days": {
144+
Type: schema.TypeInt,
145+
Optional: true,
146+
Description: `Customized export features time window for snapshot analysis. Unit is one day. The default value is 21 days. Minimum value is 1 day. Maximum value is 4000 days.`,
147+
Default: 21,
148+
},
80149
},
81150
},
82151
},
@@ -403,6 +472,12 @@ func flattenVertexAIFeaturestoreEntitytypeMonitoringConfig(v interface{}, d *sch
403472
transformed := make(map[string]interface{})
404473
transformed["snapshot_analysis"] =
405474
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(original["snapshotAnalysis"], d, config)
475+
transformed["import_features_analysis"] =
476+
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(original["importFeaturesAnalysis"], d, config)
477+
transformed["numerical_threshold_config"] =
478+
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(original["numericalThresholdConfig"], d, config)
479+
transformed["categorical_threshold_config"] =
480+
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(original["categoricalThresholdConfig"], d, config)
406481
return []interface{}{transformed}
407482
}
408483
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(v interface{}, d *schema.ResourceData, config *Config) interface{} {
@@ -416,12 +491,107 @@ func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(v int
416491
transformed := make(map[string]interface{})
417492
transformed["disabled"] =
418493
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisDisabled(original["disabled"], d, config)
494+
transformed["monitoring_interval_days"] =
495+
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(original["monitoringIntervalDays"], d, config)
496+
transformed["staleness_days"] =
497+
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(original["stalenessDays"], d, config)
419498
return []interface{}{transformed}
420499
}
421500
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisDisabled(v interface{}, d *schema.ResourceData, config *Config) interface{} {
422501
return v
423502
}
424503

504+
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(v interface{}, d *schema.ResourceData, config *Config) interface{} {
505+
// Handles the string fixed64 format
506+
if strVal, ok := v.(string); ok {
507+
if intVal, err := stringToFixed64(strVal); err == nil {
508+
return intVal
509+
}
510+
}
511+
512+
// number values are represented as float64
513+
if floatVal, ok := v.(float64); ok {
514+
intVal := int(floatVal)
515+
return intVal
516+
}
517+
518+
return v // let terraform core handle it otherwise
519+
}
520+
521+
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(v interface{}, d *schema.ResourceData, config *Config) interface{} {
522+
// Handles the string fixed64 format
523+
if strVal, ok := v.(string); ok {
524+
if intVal, err := stringToFixed64(strVal); err == nil {
525+
return intVal
526+
}
527+
}
528+
529+
// number values are represented as float64
530+
if floatVal, ok := v.(float64); ok {
531+
intVal := int(floatVal)
532+
return intVal
533+
}
534+
535+
return v // let terraform core handle it otherwise
536+
}
537+
538+
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(v interface{}, d *schema.ResourceData, config *Config) interface{} {
539+
if v == nil {
540+
return nil
541+
}
542+
original := v.(map[string]interface{})
543+
if len(original) == 0 {
544+
return nil
545+
}
546+
transformed := make(map[string]interface{})
547+
transformed["state"] =
548+
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(original["state"], d, config)
549+
transformed["anomaly_detection_baseline"] =
550+
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(original["anomalyDetectionBaseline"], d, config)
551+
return []interface{}{transformed}
552+
}
553+
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(v interface{}, d *schema.ResourceData, config *Config) interface{} {
554+
return v
555+
}
556+
557+
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(v interface{}, d *schema.ResourceData, config *Config) interface{} {
558+
return v
559+
}
560+
561+
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} {
562+
if v == nil {
563+
return nil
564+
}
565+
original := v.(map[string]interface{})
566+
if len(original) == 0 {
567+
return nil
568+
}
569+
transformed := make(map[string]interface{})
570+
transformed["value"] =
571+
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(original["value"], d, config)
572+
return []interface{}{transformed}
573+
}
574+
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(v interface{}, d *schema.ResourceData, config *Config) interface{} {
575+
return v
576+
}
577+
578+
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(v interface{}, d *schema.ResourceData, config *Config) interface{} {
579+
if v == nil {
580+
return nil
581+
}
582+
original := v.(map[string]interface{})
583+
if len(original) == 0 {
584+
return nil
585+
}
586+
transformed := make(map[string]interface{})
587+
transformed["value"] =
588+
flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(original["value"], d, config)
589+
return []interface{}{transformed}
590+
}
591+
func flattenVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(v interface{}, d *schema.ResourceData, config *Config) interface{} {
592+
return v
593+
}
594+
425595
func expandVertexAIFeaturestoreEntitytypeLabels(v interface{}, d TerraformResourceData, config *Config) (map[string]string, error) {
426596
if v == nil {
427597
return map[string]string{}, nil
@@ -449,6 +619,27 @@ func expandVertexAIFeaturestoreEntitytypeMonitoringConfig(v interface{}, d Terra
449619
transformed["snapshotAnalysis"] = transformedSnapshotAnalysis
450620
}
451621

622+
transformedImportFeaturesAnalysis, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(original["import_features_analysis"], d, config)
623+
if err != nil {
624+
return nil, err
625+
} else if val := reflect.ValueOf(transformedImportFeaturesAnalysis); val.IsValid() && !isEmptyValue(val) {
626+
transformed["importFeaturesAnalysis"] = transformedImportFeaturesAnalysis
627+
}
628+
629+
transformedNumericalThresholdConfig, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(original["numerical_threshold_config"], d, config)
630+
if err != nil {
631+
return nil, err
632+
} else if val := reflect.ValueOf(transformedNumericalThresholdConfig); val.IsValid() && !isEmptyValue(val) {
633+
transformed["numericalThresholdConfig"] = transformedNumericalThresholdConfig
634+
}
635+
636+
transformedCategoricalThresholdConfig, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(original["categorical_threshold_config"], d, config)
637+
if err != nil {
638+
return nil, err
639+
} else if val := reflect.ValueOf(transformedCategoricalThresholdConfig); val.IsValid() && !isEmptyValue(val) {
640+
transformed["categoricalThresholdConfig"] = transformedCategoricalThresholdConfig
641+
}
642+
452643
return transformed, nil
453644
}
454645

@@ -468,13 +659,115 @@ func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysis(v inte
468659
transformed["disabled"] = transformedDisabled
469660
}
470661

662+
transformedMonitoringIntervalDays, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(original["monitoring_interval_days"], d, config)
663+
if err != nil {
664+
return nil, err
665+
} else if val := reflect.ValueOf(transformedMonitoringIntervalDays); val.IsValid() && !isEmptyValue(val) {
666+
transformed["monitoringIntervalDays"] = transformedMonitoringIntervalDays
667+
}
668+
669+
transformedStalenessDays, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(original["staleness_days"], d, config)
670+
if err != nil {
671+
return nil, err
672+
} else if val := reflect.ValueOf(transformedStalenessDays); val.IsValid() && !isEmptyValue(val) {
673+
transformed["stalenessDays"] = transformedStalenessDays
674+
}
675+
471676
return transformed, nil
472677
}
473678

474679
func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisDisabled(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
475680
return v, nil
476681
}
477682

683+
func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisMonitoringIntervalDays(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
684+
return v, nil
685+
}
686+
687+
func expandVertexAIFeaturestoreEntitytypeMonitoringConfigSnapshotAnalysisStalenessDays(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
688+
return v, nil
689+
}
690+
691+
func expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysis(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
692+
l := v.([]interface{})
693+
if len(l) == 0 || l[0] == nil {
694+
return nil, nil
695+
}
696+
raw := l[0]
697+
original := raw.(map[string]interface{})
698+
transformed := make(map[string]interface{})
699+
700+
transformedState, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(original["state"], d, config)
701+
if err != nil {
702+
return nil, err
703+
} else if val := reflect.ValueOf(transformedState); val.IsValid() && !isEmptyValue(val) {
704+
transformed["state"] = transformedState
705+
}
706+
707+
transformedAnomalyDetectionBaseline, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(original["anomaly_detection_baseline"], d, config)
708+
if err != nil {
709+
return nil, err
710+
} else if val := reflect.ValueOf(transformedAnomalyDetectionBaseline); val.IsValid() && !isEmptyValue(val) {
711+
transformed["anomalyDetectionBaseline"] = transformedAnomalyDetectionBaseline
712+
}
713+
714+
return transformed, nil
715+
}
716+
717+
func expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisState(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
718+
return v, nil
719+
}
720+
721+
func expandVertexAIFeaturestoreEntitytypeMonitoringConfigImportFeaturesAnalysisAnomalyDetectionBaseline(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
722+
return v, nil
723+
}
724+
725+
func expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
726+
l := v.([]interface{})
727+
if len(l) == 0 || l[0] == nil {
728+
return nil, nil
729+
}
730+
raw := l[0]
731+
original := raw.(map[string]interface{})
732+
transformed := make(map[string]interface{})
733+
734+
transformedValue, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(original["value"], d, config)
735+
if err != nil {
736+
return nil, err
737+
} else if val := reflect.ValueOf(transformedValue); val.IsValid() && !isEmptyValue(val) {
738+
transformed["value"] = transformedValue
739+
}
740+
741+
return transformed, nil
742+
}
743+
744+
func expandVertexAIFeaturestoreEntitytypeMonitoringConfigNumericalThresholdConfigValue(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
745+
return v, nil
746+
}
747+
748+
func expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
749+
l := v.([]interface{})
750+
if len(l) == 0 || l[0] == nil {
751+
return nil, nil
752+
}
753+
raw := l[0]
754+
original := raw.(map[string]interface{})
755+
transformed := make(map[string]interface{})
756+
757+
transformedValue, err := expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(original["value"], d, config)
758+
if err != nil {
759+
return nil, err
760+
} else if val := reflect.ValueOf(transformedValue); val.IsValid() && !isEmptyValue(val) {
761+
transformed["value"] = transformedValue
762+
}
763+
764+
return transformed, nil
765+
}
766+
767+
func expandVertexAIFeaturestoreEntitytypeMonitoringConfigCategoricalThresholdConfigValue(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
768+
return v, nil
769+
}
770+
478771
func resourceVertexAIFeaturestoreEntitytypeEncoder(d *schema.ResourceData, meta interface{}, obj map[string]interface{}) (map[string]interface{}, error) {
479772
if v, ok := d.GetOk("featurestore"); ok {
480773
re := regexp.MustCompile("projects/(.+)/locations/(.+)/featurestores/(.+)$")

‎google/resource_vertex_ai_featurestore_entitytype_generated_test.go

+17
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,23 @@ resource "google_vertex_ai_featurestore_entitytype" "entity" {
7373
foo = "bar"
7474
}
7575
featurestore = google_vertex_ai_featurestore.featurestore.id
76+
monitoring_config {
77+
snapshot_analysis {
78+
disabled = false
79+
monitoring_interval_days = 1
80+
staleness_days = 21
81+
}
82+
numerical_threshold_config {
83+
value = 0.8
84+
}
85+
categorical_threshold_config {
86+
value = 10.0
87+
}
88+
import_features_analysis {
89+
state = "ENABLED"
90+
anomaly_detection_baseline = "PREVIOUS_IMPORT_FEATURES_STATS"
91+
}
92+
}
7693
}
7794
`, context)
7895
}

0 commit comments

Comments
 (0)