feat: opt-in http retrieval client

cmd/ipfs/kubo/daemon.go

@@ -458,6 +458,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 			cfg.Identity.PeerID,
 			cfg.Addresses,
 			cfg.Identity.PrivKey,
+			cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled),
 		)
 	default:
 		return fmt.Errorf("unrecognized routing option: %s", routingOption)

config/bitswap.go

@@ -2,13 +2,14 @@ package config
 
 // Bitswap holds Bitswap configuration options
 type Bitswap struct {
-	// Enabled controls both client and server (enabled by default)
-	Enabled Flag `json:",omitempty"`
-	// ServerEnabled controls if the node responds to WANTs (depends on Enabled, enabled by default)
+	// Libp2pEnabled controls if the node initializes bitswap over libp2p (enabled by default)
+	// (This can be disabled if HTTPRetrieval.Enabled is set to true)
+	Libp2pEnabled Flag `json:",omitempty"`
+	// ServerEnabled controls if the node responds to WANTs (depends on Libp2pEnabled, enabled by default)
 	ServerEnabled Flag `json:",omitempty"`
 }
 
 const (
-	DefaultBitswapEnabled       = true
+	DefaultBitswapLibp2pEnabled = true
 	DefaultBitswapServerEnabled = true
 )

config/config.go

@@ -33,13 +33,14 @@ type Config struct {
 	DNS       DNS
 	Migration Migration
 
-	Provider     Provider
-	Reprovider   Reprovider
-	Experimental Experiments
-	Plugins      Plugins
-	Pinning      Pinning
-	Import       Import
-	Version      Version
+	Provider      Provider
+	Reprovider    Reprovider
+	HTTPRetrieval HTTPRetrieval
+	Experimental  Experiments
+	Plugins       Plugins
+	Pinning       Pinning
+	Import        Import
+	Version       Version
 
 	Internal Internal // experimental/unstable options
 

config/http_retrieval.go

@@ -0,0 +1,19 @@
+package config
+
+// HTTPRetrieval is the configuration object for HTTP Retrieval settings.
+// Implicit defaults can be found in core/node/bitswap.go
+type HTTPRetrieval struct {
+	Enabled               Flag             `json:",omitempty"`
+	Allowlist             []string         `json:",omitempty"`
+	Denylist              []string         `json:",omitempty"`
+	NumWorkers            *OptionalInteger `json:",omitempty"`
+	MaxBlockSize          *OptionalString  `json:",omitempty"`
+	TLSInsecureSkipVerify Flag             `json:",omitempty"`
+}
+
+const (
+	DefaultHTTPRetrievalEnabled               = false // opt-in for now, until we figure out https://github.com/ipfs/specs/issues/496
+	DefaultHTTPRetrievalNumWorkers            = 16
+	DefaultHTTPRetrievalTLSInsecureSkipVerify = false  // only for testing with self-signed HTTPS certs
+	DefaultHTTPRetrievalMaxBlockSize          = "2MiB" // matching bitswap: https://specs.ipfs.tech/bitswap-protocol/#block-sizes
+)

config/init.go

@@ -48,13 +48,6 @@ func InitWithIdentity(identity Identity) (*Config, error) {
 			},
 		},
 
-		Routing: Routing{
-			Type:            nil,
-			Methods:         nil,
-			Routers:         nil,
-			IgnoreProviders: []peer.ID{},
-		},
-
 		// setup the node mount points.
 		Mounts: Mounts{
 			IPFS: "/ipfs",

config/routing.go

@@ -6,27 +6,30 @@ import (
 	"os"
 	"runtime"
 	"strings"
-
-	peer "github.com/libp2p/go-libp2p/core/peer"
 )
 
 const (
 	DefaultAcceleratedDHTClient      = false
 	DefaultLoopbackAddressesOnLanDHT = false
+	CidContactRoutingURL             = "https://cid.contact"
+	PublicGoodDelegatedRoutingURL    = "https://delegated-ipfs.dev" // cid.contact + amino dht (incl. IPNS PUTs)
+	EnvHTTPRouters                   = "IPFS_HTTP_ROUTERS"
+	EnvHTTPRoutersFilterProtocols    = "IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS"
 )
 
 var (
 	// Default HTTP routers used in parallel to DHT when Routing.Type = "auto"
-	DefaultHTTPRouters = getEnvOrDefault("IPFS_HTTP_ROUTERS", []string{
-		"https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
+	DefaultHTTPRouters = getEnvOrDefault(EnvHTTPRouters, []string{
+		CidContactRoutingURL, // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
 	})
 
 	// Default filter-protocols to pass along with delegated routing requests (as defined in IPIP-484)
 	// and also filter out locally
-	DefaultHTTPRoutersFilterProtocols = getEnvOrDefault("IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS", []string{
+	DefaultHTTPRoutersFilterProtocols = getEnvOrDefault(EnvHTTPRoutersFilterProtocols, []string{
 		"unknown", // allow results without protocol list, we can do libp2p identify to test them
 		"transport-bitswap",
-		// TODO: add 'transport-ipfs-gateway-http' once https://github.com/ipfs/rainbow/issues/125 is addressed
+		// http is added dynamically in routing/delegated.go.
+		// 'transport-ipfs-gateway-http'
 	})
 )
 
@@ -43,11 +46,13 @@ type Routing struct {
 
 	LoopbackAddressesOnLanDHT Flag `json:",omitempty"`
 
-	IgnoreProviders []peer.ID
+	IgnoreProviders []string `json:",omitempty"`
+
+	DelegatedRouters []string `json:",omitempty"`
 
-	Routers Routers
+	Routers Routers `json:",omitempty"`
 
-	Methods Methods
+	Methods Methods `json:",omitempty"`
 }
 
 type Router struct {

core/node/bitswap.go

@@ -2,24 +2,28 @@ package node
 
 import (
 	"context"
+	"errors"
 	"io"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/ipfs/boxo/bitswap"
 	"github.com/ipfs/boxo/bitswap/client"
+	"github.com/ipfs/boxo/bitswap/network"
 	bsnet "github.com/ipfs/boxo/bitswap/network/bsnet"
+	"github.com/ipfs/boxo/bitswap/network/httpnet"
 	blockstore "github.com/ipfs/boxo/blockstore"
 	exchange "github.com/ipfs/boxo/exchange"
 	"github.com/ipfs/boxo/exchange/providing"
 	provider "github.com/ipfs/boxo/provider"
 	rpqm "github.com/ipfs/boxo/routing/providerquerymanager"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-datastore"
 	ipld "github.com/ipfs/go-ipld-format"
+	version "github.com/ipfs/kubo"
 	"github.com/ipfs/kubo/config"
 	irouting "github.com/ipfs/kubo/routing"
 	"github.com/libp2p/go-libp2p/core/host"
-	"github.com/libp2p/go-libp2p/core/routing"
+	peer "github.com/libp2p/go-libp2p/core/peer"
 	"go.uber.org/fx"
 
 	blocks "github.com/ipfs/go-block-format"
@@ -79,38 +83,63 @@ type bitswapIn struct {
 // Bitswap creates the BitSwap server/client instance.
 // If Bitswap.ServerEnabled is false, the node will act only as a client
 // using an empty blockstore to prevent serving blocks to other peers.
-func Bitswap(provide bool) interface{} {
+func Bitswap(serverEnabled bool) interface{} {
 	return func(in bitswapIn, lc fx.Lifecycle) (*bitswap.Bitswap, error) {
-		bitswapNetwork := bsnet.NewFromIpfsHost(in.Host)
-		var blockstoree blockstore.Blockstore = in.Bs
-		var provider routing.ContentDiscovery
+		var bitswapNetworks, bitswapLibp2p network.BitSwapNetwork
+		var bitswapBlockstore blockstore.Blockstore = in.Bs
 
-		if provide {
+		libp2pEnabled := in.Cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled)
+		if libp2pEnabled {
+			bitswapLibp2p = bsnet.NewFromIpfsHost(in.Host)
+		}
 
-			var maxProviders int = DefaultMaxProviders
-			if in.Cfg.Internal.Bitswap != nil {
-				maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders))
+		if httpCfg := in.Cfg.HTTPRetrieval; httpCfg.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled) {
+			maxBlockSize, err := humanize.ParseBytes(httpCfg.MaxBlockSize.WithDefault(config.DefaultHTTPRetrievalMaxBlockSize))
+			if err != nil {
+				return nil, err
 			}
-
-			pqm, err := rpqm.New(bitswapNetwork,
-				in.Rt,
-				rpqm.WithMaxProviders(maxProviders),
-				rpqm.WithIgnoreProviders(in.Cfg.Routing.IgnoreProviders...),
+			bitswapHTTP := httpnet.New(in.Host,
+				httpnet.WithHTTPWorkers(int(httpCfg.NumWorkers.WithDefault(config.DefaultHTTPRetrievalNumWorkers))),
+				httpnet.WithAllowlist(httpCfg.Allowlist),
+				httpnet.WithDenylist(httpCfg.Denylist),
+				httpnet.WithInsecureSkipVerify(httpCfg.TLSInsecureSkipVerify.WithDefault(config.DefaultHTTPRetrievalTLSInsecureSkipVerify)),
+				httpnet.WithMaxBlockSize(int64(maxBlockSize)),
+				httpnet.WithUserAgent(version.GetUserAgentVersion()),
 			)
+			bitswapNetworks = network.New(in.Host.Peerstore(), bitswapLibp2p, bitswapHTTP)
+		} else if libp2pEnabled {
+			bitswapNetworks = bitswapLibp2p
+		} else {
+			return nil, errors.New("invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap")
+		}
+
+		// Kubo uses own, customized ProviderQueryManager
+		in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.WithDefaultProviderQueryManager(false)))
+		var maxProviders int = DefaultMaxProviders
+		if in.Cfg.Internal.Bitswap != nil {
+			maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders))
+		}
+		ignoredPeerIDs := make([]peer.ID, 0, len(in.Cfg.Routing.IgnoreProviders))
+		for _, str := range in.Cfg.Routing.IgnoreProviders {
+			pid, err := peer.Decode(str)
 			if err != nil {
 				return nil, err
 			}
-			in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.WithDefaultProviderQueryManager(false)))
-			in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(true))
-			provider = pqm
-		} else {
-			provider = nil
-			// When server is disabled, use an empty blockstore to prevent serving blocks
-			blockstoree = blockstore.NewBlockstore(datastore.NewMapDatastore())
-			in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(false))
+			ignoredPeerIDs = append(ignoredPeerIDs, pid)
+		}
+		providerQueryMgr, err := rpqm.New(bitswapNetworks,
+			in.Rt,
+			rpqm.WithMaxProviders(maxProviders),
+			rpqm.WithIgnoreProviders(ignoredPeerIDs...),
+		)
+		if err != nil {
+			return nil, err
 		}
 
-		bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, provider, blockstoree, in.BitswapOpts...)
+		// Explicitly enable/disable server to ensure desired provide mode
+		in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(serverEnabled))
+
+		bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetworks, providerQueryMgr, bitswapBlockstore, in.BitswapOpts...)
 
 		lc.Append(fx.Hook{
 			OnStop: func(ctx context.Context) error {

core/node/groups.go

@@ -335,17 +335,18 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
 		recordLifetime = d
 	}
 
-	isBitswapEnabled := cfg.Bitswap.Enabled.WithDefault(config.DefaultBitswapEnabled)
+	isBitswapLibp2pEnabled := cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled)
 	isBitswapServerEnabled := cfg.Bitswap.ServerEnabled.WithDefault(config.DefaultBitswapServerEnabled)
-	// Don't provide from bitswap when the strategic provider service is active
-	shouldBitswapProvide := isBitswapEnabled && isBitswapServerEnabled && !cfg.Experimental.StrategicProviding
+
+	// Don't provide from bitswap when the legacy noop experiment "strategic provider service" is active
+	isBitswapServerEnabled = isBitswapServerEnabled && !cfg.Experimental.StrategicProviding
 
 	return fx.Options(
 		fx.Provide(BitswapOptions(cfg)),
-		fx.Provide(Bitswap(shouldBitswapProvide)),
-		fx.Provide(OnlineExchange(isBitswapEnabled)),
+		fx.Provide(Bitswap(isBitswapServerEnabled)),
+		fx.Provide(OnlineExchange(isBitswapLibp2pEnabled)),
 		// Replace our Exchange with a Providing exchange!
-		fx.Decorate(ProvidingExchange(shouldBitswapProvide)),
+		fx.Decorate(ProvidingExchange(isBitswapServerEnabled)),
 		fx.Provide(DNSResolver),
 		fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))),
 		fx.Provide(Peering),

core/node/libp2p/routingopt.go

@@ -2,6 +2,7 @@ package libp2p
 
 import (
 	"context"
+	"os"
 	"time"
 
 	"github.com/ipfs/go-datastore"
@@ -29,21 +30,44 @@ type RoutingOptionArgs struct {
 
 type RoutingOption func(args RoutingOptionArgs) (routing.Routing, error)
 
+var noopRouter = routinghelpers.Null{}
+
 func constructDefaultHTTPRouters(cfg *config.Config) ([]*routinghelpers.ParallelRouter, error) {
 	var routers []*routinghelpers.ParallelRouter
+	httpRetrievalEnabled := cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled)
+
+	// Use config.DefaultHTTPRouters if custom override was sent via config.EnvHTTPRouters
+	// or if user did not set any preference in cfg.Routing.DelegatedRouters
+	var httpRouterEndpoints []string
+	if os.Getenv(config.EnvHTTPRouters) != "" || len(cfg.Routing.DelegatedRouters) == 0 {
+		httpRouterEndpoints = config.DefaultHTTPRouters
+	} else {
+		httpRouterEndpoints = cfg.Routing.DelegatedRouters
+	}
+
 	// Append HTTP routers for additional speed
-	for _, endpoint := range config.DefaultHTTPRouters {
-		httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey)
+	for _, endpoint := range httpRouterEndpoints {
+		httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey, httpRetrievalEnabled)
 		if err != nil {
 			return nil, err
 		}
-
+		// Mapping router to /routing/v1/* endpoints
+		// https://specs.ipfs.tech/routing/http-routing-v1/
 		r := &irouting.Composer{
-			GetValueRouter:      routinghelpers.Null{},
-			PutValueRouter:      routinghelpers.Null{},
-			ProvideRouter:       routinghelpers.Null{}, // modify this when indexers supports provide
-			FindPeersRouter:     routinghelpers.Null{},
-			FindProvidersRouter: httpRouter,
+			GetValueRouter:      httpRouter, // GET /routing/v1/ipns
+			PutValueRouter:      httpRouter, // PUT /routing/v1/ipns
+			ProvideRouter:       noopRouter, // we don't have spec for sending provides to /routing/v1 (revisit once https://github.com/ipfs/specs/pull/378 or similar is ratified)
+			FindPeersRouter:     httpRouter, // /routing/v1/peers
+			FindProvidersRouter: httpRouter, // /routing/v1/providers
+		}
+
+		if endpoint == config.CidContactRoutingURL {
+			// Special-case: cid.contact only supports /routing/v1/providers/cid
+			// we disable other endpoints to avoid sending requests that always fail
+			r.GetValueRouter = noopRouter
+			r.PutValueRouter = noopRouter
+			r.ProvideRouter = noopRouter
+			r.FindPeersRouter = noopRouter
 		}
 
 		routers = append(routers, &routinghelpers.ParallelRouter{
@@ -119,7 +143,7 @@ func constructDHTRouting(mode dht.ModeOpt) RoutingOption {
 }
 
 // ConstructDelegatedRouting is used when Routing.Type = "custom"
-func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs config.Addresses, privKey string) RoutingOption {
+func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs config.Addresses, privKey string, httpRetrieval bool) RoutingOption {
 	return func(args RoutingOptionArgs) (routing.Routing, error) {
 		return irouting.Parse(routers, methods,
 			&irouting.ExtraDHTParams{
@@ -130,9 +154,10 @@ func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, p
 				Context:        args.Ctx,
 			},
 			&irouting.ExtraHTTPParams{
-				PeerID:     peerID,
-				Addrs:      httpAddrsFromConfig(addrs),
-				PrivKeyB64: privKey,
+				PeerID:        peerID,
+				Addrs:         httpAddrsFromConfig(addrs),
+				PrivKeyB64:    privKey,
+				HTTPRetrieval: httpRetrieval,
 			},
 		)
 	}