Fix bug where -backend-config could not override attributes that weren't at the top level in the backend schema

.changes/v1.13/BUG FIXES-20250423-164150.yaml

@@ -0,0 +1,5 @@
+kind: BUG FIXES
+body: 'backends: Nested attrbiutes can now be overridden during `init` using the `-backend-config` flag'
+time: 2025-04-23T16:41:50.904809+01:00
+custom:
+    Issue: "36911"

internal/backend/remote-state/inmem/backend.go

@@ -6,6 +6,8 @@ package inmem
 import (
 	"errors"
 	"fmt"
+	"maps"
+	"os"
 	"sort"
 	"sync"
 	"time"
@@ -48,19 +50,57 @@ func Reset() {
 
 // New creates a new backend for Inmem remote state.
 func New() backend.Backend {
+	if os.Getenv("TF_INMEM_TEST") != "" {
+		// We use a different schema for testing. This isn't user facing unless they
+		// dig into the code.
+		fmt.Sprintln("TF_INMEM_TEST is set: Using test schema for the inmem backend")
+
+		return &Backend{
+			Base: backendbase.Base{
+				Schema: &configschema.Block{
+					Attributes: testSchemaAttrs(),
+				},
+			},
+		}
+	}
+
+	// Default schema that's user-facing
 	return &Backend{
 		Base: backendbase.Base{
 			Schema: &configschema.Block{
-				Attributes: map[string]*configschema.Attribute{
-					"lock_id": {
-						Type:        cty.String,
-						Optional:    true,
-						Description: "initializes the state in a locked configuration",
-					},
+				Attributes: defaultSchemaAttrs,
+			},
+		},
+	}
+}
+
+var defaultSchemaAttrs = map[string]*configschema.Attribute{
+	"lock_id": {
+		Type:        cty.String,
+		Optional:    true,
+		Description: "initializes the state in a locked configuration",
+	},
+}
+
+func testSchemaAttrs() map[string]*configschema.Attribute {
+	var newSchema = make(map[string]*configschema.Attribute)
+	maps.Copy(newSchema, defaultSchemaAttrs)
+
+	// Append test-specific parts of schema
+	newSchema["test_nesting_single"] = &configschema.Attribute{
+		Description: "An attribute that contains nested attributes, where nesting mode is NestingSingle",
+		NestedType: &configschema.Object{
+			Nesting: configschema.NestingSingle,
+			Attributes: map[string]*configschema.Attribute{
+				"child": {
+					Type:        cty.String,
+					Optional:    true,
+					Description: "A nested attribute inside the parent attribute `test_nesting_single`",
 				},
 			},
 		},
 	}
+	return newSchema
 }
 
 type Backend struct {

internal/command/init.go

@@ -1031,23 +1031,87 @@ func (c *InitCommand) backendConfigOverrideBody(flags arguments.FlagNameValueSli
 			flushVals() // deal with any accumulated individual values first
 			mergeBody(newBody)
 		} else {
+			// The flag value is setting a single attribute's value
 			name := item.Value[:eq]
 			rawValue := item.Value[eq+1:]
-			attrS := schema.Attributes[name]
-			if attrS == nil {
+
+			splitName := strings.Split(name, ".")
+			isNested := len(splitName) > 1
+
+			var value cty.Value
+			var valueDiags tfdiags.Diagnostics
+			switch {
+			case !isNested:
+				// The flag item is overriding a top-level attribute
+				attrS := schema.Attributes[name]
+				if attrS == nil {
+					diags = diags.Append(tfdiags.Sourceless(
+						tfdiags.Error,
+						"Invalid backend configuration argument",
+						fmt.Sprintf("The backend configuration argument %q given on the command line is not expected for the selected backend type.", name),
+					))
+					continue
+				}
+
+				value, valueDiags = configValueFromCLI(item.String(), rawValue, attrS.Type)
+				diags = diags.Append(valueDiags)
+				if valueDiags.HasErrors() {
+					continue
+				}
+
+				// Synthetic values are collected as we parse each flag item
+				synthVals[name] = value
+			case isNested:
+				// The flag item is overriding a nested attribute
+				// e.g. assume_role.role_arn in the s3 backend
+				// We assume a max nesting-depth of 1 as s3 is the only
+				// backend that contains nested fields.
+
+				parentName := splitName[0]
+				nestedName := splitName[1]
+				parentAttr := schema.Attributes[parentName]
+				nestedAttr := parentAttr.NestedType.Attributes[nestedName]
+				if nestedAttr == nil {
+					diags = diags.Append(tfdiags.Sourceless(
+						tfdiags.Error,
+						"Invalid backend configuration argument",
+						fmt.Sprintf("The backend configuration argument %q given on the command line is not expected for the selected backend type.", name),
+					))
+					continue
+				}
+
+				value, valueDiags = configValueFromCLI(item.String(), rawValue, nestedAttr.Type)
+				diags = diags.Append(valueDiags)
+				if valueDiags.HasErrors() {
+					continue
+				}
+
+				// Synthetic values are collected as we parse each flag item
+				// When doing this we need to account for attribute nesting
+				// and multiple nested fields being overridden.
+				synthParent, found := synthVals[parentName]
+				if !found {
+					synthVals[parentName] = cty.ObjectVal(map[string]cty.Value{
+						nestedName: value,
+					})
+				}
+				if found {
+					// add the new attribute override to any existing attributes
+					// also nested under the parent
+					nestedMap := synthParent.AsValueMap()
+					nestedMap[nestedName] = value
+					synthVals[parentName] = cty.ObjectVal(nestedMap)
+				}
+
+			default:
+				// Should not reach here
 				diags = diags.Append(tfdiags.Sourceless(
 					tfdiags.Error,
 					"Invalid backend configuration argument",
 					fmt.Sprintf("The backend configuration argument %q given on the command line is not expected for the selected backend type.", name),
 				))
 				continue
 			}
-			value, valueDiags := configValueFromCLI(item.String(), rawValue, attrS.Type)
-			diags = diags.Append(valueDiags)
-			if valueDiags.HasErrors() {
-				continue
-			}
-			synthVals[name] = value
 		}
 	}
 

internal/command/init_test.go

@@ -792,6 +792,41 @@ func TestInit_backendConfigKV(t *testing.T) {
 	}
 }
 
+func TestInit_backendConfigKVNested(t *testing.T) {
+	// Create a temporary working directory that is empty
+	td := t.TempDir()
+	testCopyDir(t, testFixturePath("init-backend-config-kv-nested"), td)
+	defer testChdir(t, td)()
+	t.Setenv("TF_INMEM_TEST", "1") // Allows use of inmem backend with a more complex schema
+
+	ui := new(cli.MockUi)
+	view, done := testView(t)
+	c := &InitCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(testProvider()),
+			Ui:               ui,
+			View:             view,
+		},
+	}
+
+	// overridden field is nested:
+	// test_nesting_single = {
+	//    child = "..."
+	// }
+	args := []string{
+		"-backend-config=test_nesting_single.child=foobar",
+	}
+	if code := c.Run(args); code != 0 {
+		t.Fatalf("bad: \n%s", done(t).Stderr())
+	}
+
+	// Read our saved backend config and verify we have our settings
+	state := testDataStateRead(t, filepath.Join(DefaultDataDir, DefaultStateFilename))
+	if got, want := normalizeJSON(t, state.Backend.ConfigRaw), `{"lock_id":null,"test_nesting_single":{"child":"foobar"}}`; got != want {
+		t.Errorf("wrong config\ngot:  %s\nwant: %s", got, want)
+	}
+}
+
 func TestInit_backendConfigKVReInit(t *testing.T) {
 	// Create a temporary working directory that is empty
 	td := t.TempDir()

internal/command/testdata/init-backend-config-kv-nested/main.tf

@@ -0,0 +1,7 @@
+terraform {
+  backend "inmem" {
+    test_nesting_single = {
+      child = "" // to be overwritten in test
+    }
+  }
+}