incusd/server/device/nic_routed: Added host_tables

doc/api-extensions.md

@@ -2795,3 +2795,7 @@ This is limited to bridged networks as OVN doesn't support flexible enough SNAT
 ## `memory_hotplug`
 
 This adds memory hotplugging for VMs, allowing them to add memory at runtime without rebooting.
+
+## `instance_nic_routed_host_tables`
+
+This adds support for specifying host-routing tables on `nic` devices that use the routed mode.

doc/config_options.txt

@@ -1164,8 +1164,16 @@ For file systems (shared directories or custom volumes), this is one of:
 ```
 
 ```{config:option} ipv4.host_table devices-nic_routed
-:shortdesc: "The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table)"
+:shortdesc: "Deprecated: Use `ipv4.host_tables` instead"
 :type: "integer"
+The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table)
+
+```
+
+```{config:option} ipv4.host_tables devices-nic_routed
+:default: "254"
+:shortdesc: "Comma-delimited list of routing tables IDs to add IPv4 static routes to"
+:type: "string"
 
 ```
 
@@ -1203,8 +1211,16 @@ For file systems (shared directories or custom volumes), this is one of:
 ```
 
 ```{config:option} ipv6.host_table devices-nic_routed
-:shortdesc: "The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table)"
+:shortdesc: "Deprecated: Use `ipv6.host_tables` instead"
 :type: "integer"
+The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table)
+
+```
+
+```{config:option} ipv6.host_tables devices-nic_routed
+:default: "254"
+:shortdesc: "Comma-delimited list of routing tables IDs to add IPv6 static routes to"
+:type: "string"
 
 ```
 

internal/server/device/nic_routed.go

@@ -139,76 +139,96 @@ func (d *nicRouted) validateConfig(instConf instance.ConfigReader) error {
 		// ---
 		//  type: integer
 		//  shortdesc: The priority for outgoing traffic, to be used by the kernel queuing discipline to prioritize network packets
-
 		"limits.priority",
+
 		// gendoc:generate(entity=devices, group=nic_routed, key=ipv4.gateway)
 		//
 		// ---
 		//  type: string
 		//  default: auto
 		//  shortdesc: Whether to add an automatic default IPv4 gateway (can be `auto` or `none`)
-
 		"ipv4.gateway",
+
 		// gendoc:generate(entity=devices, group=nic_routed, key=ipv6.gateway)
 		//
 		// ---
 		//  type: string
 		//  default: auto
 		//  shortdesc: Whether to add an automatic default IPv6 gateway (can be `auto` or `none`)
-
 		"ipv6.gateway",
+
 		// gendoc:generate(entity=devices, group=nic_routed, key=ipv4.routes)
 		//
 		// ---
 		//  type: string
 		//  shortdesc: Comma-delimited list of IPv4 static routes to add on host to NIC (without L2 ARP/NDP proxy)
-
 		"ipv4.routes",
+
 		// gendoc:generate(entity=devices, group=nic_routed, key=ipv6.routes)
 		//
 		// ---
 		//  type: string
 		//  shortdesc: Comma-delimited list of IPv6 static routes to add on host to NIC (without L2 ARP/NDP proxy)
-
 		"ipv6.routes",
+
 		// gendoc:generate(entity=devices, group=nic_routed, key=ipv4.host_address)
 		//
 		// ---
 		//  type: string
 		//  default: `169.254.0.1`
 		//  shortdesc: The IPv4 address to add to the host-side `veth` interface
-
 		"ipv4.host_address",
+
 		// gendoc:generate(entity=devices, group=nic_routed, key=ipv6.host_address)
 		//
 		// ---
 		//  type: string
 		//  default: `fe80::1`
 		//  shortdesc: The IPv6 address to add to the host-side `veth` interface
-
 		"ipv6.host_address",
+
 		// gendoc:generate(entity=devices, group=nic_routed, key=ipv4.host_table)
 		//
+		// The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table)
+		//
 		// ---
 		//  type: integer
-		//  shortdesc: The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table)
-
+		//  shortdesc: Deprecated: Use `ipv4.host_tables` instead
 		"ipv4.host_table",
+
 		// gendoc:generate(entity=devices, group=nic_routed, key=ipv6.host_table)
 		//
+		// The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table)
+		//
 		// ---
 		//  type: integer
-		//  shortdesc: The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table)
-
+		//  shortdesc: Deprecated: Use `ipv6.host_tables` instead
 		"ipv6.host_table",
+
+		// gendoc:generate(entity=devices, group=nic_routed, key=ipv4.host_tables)
+		//
+		// ---
+		//  type: string
+		//  default: 254
+		//  shortdesc: Comma-delimited list of routing tables IDs to add IPv4 static routes to
+		"ipv4.host_tables",
+
+		// gendoc:generate(entity=devices, group=nic_routed, key=ipv6.host_tables)
+		//
+		// ---
+		//  type: string
+		//  default: 254
+		//  shortdesc: Comma-delimited list of routing tables IDs to add IPv6 static routes to
+		"ipv6.host_tables",
+
 		// gendoc:generate(entity=devices, group=nic_routed, key=gvrp)
 		//
 		// ---
 		//  type: bool
 		//  default: false
 		//  shortdesc: Register VLAN using GARP VLAN Registration Protocol
-
 		"gvrp",
+
 		// gendoc:generate(entity=devices, group=nic_routed, key=vrf)
 		//
 		// ---
@@ -241,8 +261,6 @@ func (d *nicRouted) validateConfig(instConf instance.ConfigReader) error {
 	//  shortdesc: Comma-delimited list of IPv6 static addresses to add to the instance
 	rules["ipv6.address"] = validate.Optional(validate.IsListOf(validate.IsNetworkAddressV6))
 
-	rules["gvrp"] = validate.Optional(validate.IsBool)
-
 	// gendoc:generate(entity=devices, group=nic_routed, key=ipv4.neighbor_probe)
 	//
 	// ---
@@ -259,6 +277,9 @@ func (d *nicRouted) validateConfig(instConf instance.ConfigReader) error {
 	//  shortdesc: Whether to probe the parent network for IP address availability
 	rules["ipv6.neighbor_probe"] = validate.Optional(validate.IsBool)
 
+	rules["ipv4.host_tables"] = validate.Optional(validate.IsListOf(validate.IsInRange(0, 255)))
+	rules["ipv6.host_tables"] = validate.Optional(validate.IsListOf(validate.IsInRange(0, 255)))
+	rules["gvrp"] = validate.Optional(validate.IsBool)
 	rules["vrf"] = validate.Optional(validate.IsAny)
 
 	err = d.config.Validate(rules)
@@ -584,36 +605,55 @@ func (d *nicRouted) Start() (*deviceConfig.RunConfig, error) {
 			}
 		}
 
-		table := "main"
-		if d.config["vrf"] != "" {
-			table = ""
+		getTables := func() []string {
+			// New plural form – honour exactly what the user gives.
+			v := d.config[fmt.Sprintf("%s.host_tables", keyPrefix)]
+			if v != "" {
+				return util.SplitNTrimSpace(v, ",", -1, true)
+			}
+
+			// Legacy – single key: include it plus 254.
+			v = d.config[fmt.Sprintf("%s.host_table", keyPrefix)]
+			if v != "" {
+				if v == "254" {
+					return []string{"254"} // user asked for main only
+				}
+
+				return []string{v, "254"} // custom + main
+			}
+
+			// Default – main only.
+			return []string{"254"}
 		}
 
+		tables := getTables()
+
 		// Perform per-address host-side configuration (static routes and neighbour proxy entries).
 		for _, addrStr := range addresses {
 			// Apply host-side static routes to main routing table or VRF.
-			r := ip.Route{
-				DevName: saveData["host_name"],
-				Route:   fmt.Sprintf("%s/%d", addrStr, subnetSize),
-				Table:   table,
-				Family:  ipFamilyArg,
-				VRF:     d.config["vrf"],
-			}
 
-			err = r.Add()
-			if err != nil {
-				return nil, fmt.Errorf("Failed adding host route %q: %w", r.Route, err)
+			// If a VRF is set we still add a route into the VRF's own table (empty Table value).
+			if d.config["vrf"] != "" {
+				r := ip.Route{
+					DevName: saveData["host_name"],
+					Route:   fmt.Sprintf("%s/%d", addrStr, subnetSize),
+					Table:   "",
+					Family:  ipFamilyArg,
+					VRF:     d.config["vrf"],
+				}
+
+				err = r.Add()
+				if err != nil {
+					return nil, fmt.Errorf("Failed adding host route %q: %w", r.Route, err)
+				}
 			}
 
-			// Add host-side static routes to instance IPs to custom routing table if specified.
-			// This is in addition to the static route added to the main routing table, which is still
-			// critical to ensure that reverse path filtering doesn't kick in blocking traffic from
-			// the instance.
-			if d.config[fmt.Sprintf("%s.host_table", keyPrefix)] != "" {
+			// Add routes to all requested tables.
+			for _, tbl := range tables {
 				r := ip.Route{
 					DevName: saveData["host_name"],
 					Route:   fmt.Sprintf("%s/%d", addrStr, subnetSize),
-					Table:   d.config[fmt.Sprintf("%s.host_table", keyPrefix)],
+					Table:   tbl,
 					Family:  ipFamilyArg,
 				}
 
@@ -645,21 +685,40 @@ func (d *nicRouted) Start() (*deviceConfig.RunConfig, error) {
 			if len(addresses) == 0 {
 				return nil, fmt.Errorf("%s.routes requires %s.address to be set", keyPrefix, keyPrefix)
 			}
+
 			// Add routes
 			for _, routeStr := range routes {
-				// Apply host-side static routes to main routing table or VRF.
-				r := ip.Route{
-					DevName: saveData["host_name"],
-					Route:   routeStr,
-					Table:   table,
-					Family:  ipFamilyArg,
-					Via:     addresses[0],
-					VRF:     d.config["vrf"],
+				// If a VRF is set we still add a route into the VRF's own table (empty Table value).
+				if d.config["vrf"] != "" {
+					r := ip.Route{
+						DevName: saveData["host_name"],
+						Route:   routeStr,
+						Table:   "",
+						Family:  ipFamilyArg,
+						Via:     addresses[0],
+						VRF:     d.config["vrf"],
+					}
+
+					err = r.Add()
+					if err != nil {
+						return nil, fmt.Errorf("Failed adding route %q: %w", r.Route, err)
+					}
 				}
 
-				err = r.Add()
-				if err != nil {
-					return nil, fmt.Errorf("Failed adding route %q: %w", r.Route, err)
+				// Add routes to all requested tables.
+				for _, tbl := range tables {
+					r := ip.Route{
+						DevName: saveData["host_name"],
+						Route:   routeStr,
+						Table:   tbl,
+						Family:  ipFamilyArg,
+						Via:     addresses[0],
+					}
+
+					err = r.Add()
+					if err != nil {
+						return nil, fmt.Errorf("Failed adding route %q to table %q: %w", r.Route, r.Table, err)
+					}
 				}
 			}
 		}

internal/server/metadata/configuration.json

@@ -1326,11 +1326,19 @@
 					},
 					{
 						"ipv4.host_table": {
-							"longdesc": "",
-							"shortdesc": "The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table)",
+							"longdesc": "The custom policy routing table ID to add IPv4 static routes to (in addition to the main routing table)\n",
+							"shortdesc": "Deprecated: Use `ipv4.host_tables` instead",
 							"type": "integer"
 						}
 					},
+					{
+						"ipv4.host_tables": {
+							"default": "254",
+							"longdesc": "",
+							"shortdesc": "Comma-delimited list of routing tables IDs to add IPv4 static routes to",
+							"type": "string"
+						}
+					},
 					{
 						"ipv4.neighbor_probe": {
 							"default": "true",
@@ -1371,11 +1379,19 @@
 					},
 					{
 						"ipv6.host_table": {
-							"longdesc": "",
-							"shortdesc": "The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table)",
+							"longdesc": "The custom policy routing table ID to add IPv6 static routes to (in addition to the main routing table)\n",
+							"shortdesc": "Deprecated: Use `ipv6.host_tables` instead",
 							"type": "integer"
 						}
 					},
+					{
+						"ipv6.host_tables": {
+							"default": "254",
+							"longdesc": "",
+							"shortdesc": "Comma-delimited list of routing tables IDs to add IPv6 static routes to",
+							"type": "string"
+						}
+					},
 					{
 						"ipv6.neighbor_probe": {
 							"default": "true",