feat(gateway)!: new trustless mode, and by default

Author: hacdias

examples/gateway/common/handler.go

@@ -10,34 +10,46 @@ import (
 )
 
 func NewHandler(gwAPI gateway.IPFSBackend) http.Handler {
-	// Initialize the headers and gateway configuration. For this example, we do
-	// not add any special headers, but the required ones.
-	headers := map[string][]string{}
-	gateway.AddAccessControlHeaders(headers)
 	conf := gateway.Config{
-		Headers: headers,
-	}
+		// Initialize the headers. For this example, we do not add any special headers,
+		// only the required ones via gateway.AddAccessControlHeaders.
+		Headers: map[string][]string{},
 
-	// Initialize the public gateways that we will want to have available through
-	// Host header rewriting. This step is optional and only required if you're
-	// running multiple public gateways and want different settings and support
-	// for DNSLink and Subdomain Gateways.
-	noDNSLink := false // If you set DNSLink to point at the CID from CAR, you can load it!
-	publicGateways := map[string]*gateway.Specification{
-		// Support public requests with Host: CID.ipfs.example.net and ID.ipns.example.net
-		"example.net": {
-			Paths:         []string{"/ipfs", "/ipns"},
-			NoDNSLink:     noDNSLink,
-			UseSubdomains: true,
-		},
-		// Support local requests
-		"localhost": {
-			Paths:         []string{"/ipfs", "/ipns"},
-			NoDNSLink:     noDNSLink,
-			UseSubdomains: true,
+		// If you set DNSLink to point at the CID from CAR, you can load it!
+		NoDNSLink: false,
+
+		// For these examples we have the trusted mode enabled by default. That is,
+		// all types of requests will be accepted. By default, only Trustless Gateway
+		// requests work: https://specs.ipfs.tech/http-gateways/trustless-gateway/
+		DeserializedResponses: true,
+
+		// Initialize the public gateways that we will want to have available through
+		// Host header rewriting. This step is optional and only required if you're
+		// running multiple public gateways and want different settings and support
+		// for DNSLink and Subdomain Gateways.
+		PublicGateways: map[string]*gateway.Specification{
+			// Support public requests with Host: CID.ipfs.example.net and ID.ipns.example.net
+			"example.net": {
+				Paths:         []string{"/ipfs", "/ipns"},
+				NoDNSLink:     false,
+				UseSubdomains: true,
+				// This gateway is used for testing and therefore we make non-trustless
+				// requests. Thus, we have to manually turn on the trusted mode.
+				DeserializedResponses: true,
+			},
+			// Support local requests
+			"localhost": {
+				Paths:                 []string{"/ipfs", "/ipns"},
+				NoDNSLink:             false,
+				UseSubdomains:         true,
+				DeserializedResponses: true,
+			},
 		},
 	}
 
+	// Add required access control headers to the configuration.
+	gateway.AddAccessControlHeaders(conf.Headers)
+
 	// Creates a mux to serve the gateway paths. This is not strictly necessary
 	// and gwHandler could be used directly. However, on the next step we also want
 	// to add prometheus metrics, hence needing the mux.
@@ -57,7 +69,7 @@ func NewHandler(gwAPI gateway.IPFSBackend) http.Handler {
 	// or example.net. If you want to expose the metrics on such gateways,
 	// you will have to add the path "/debug" to the variable Paths.
 	var handler http.Handler
-	handler = gateway.WithHostname(mux, gwAPI, publicGateways, noDNSLink)
+	handler = gateway.WithHostname(conf, gwAPI, mux)
 
 	// Then, wrap with the withConnect middleware. This is required since we use
 	// http.ServeMux which does not support CONNECT by default.

gateway/gateway.go

@@ -15,7 +15,61 @@ import (
 
 // Config is the configuration used when creating a new gateway handler.
 type Config struct {
+	// Headers is a map containing all the headers that should be sent by default
+	// in all requests. You can define custom headers, as well as add the recommended
+	// headers via AddAccessControlHeaders.
 	Headers map[string][]string
+
+	// DeserializedResponses configures this gateway to support returning data
+	// in deserialized format. By default, the gateway will only provide raw responses,
+	// operating as a trustless gateway, as defined in the specification:
+	// https://specs.ipfs.tech/http-gateways/trustless-gateway/. This flag can be
+	// overridden per FQDN in PublicGateways.
+	DeserializedResponses bool
+
+	// NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in
+	// response to requests with values in `Host` HTTP header. This flag can be
+	// overridden per FQDN in PublicGateways. To be used with WithHostname.
+	NoDNSLink bool
+
+	// PublicGateways configures the behavior of known public gateways. Each key is
+	// a fully qualified domain name (FQDN). To be used with WithHostname.
+	PublicGateways map[string]*Specification
+}
+
+// Specification is the specification of an IPFS Public Gateway.
+type Specification struct {
+	// Paths is explicit list of path prefixes that should be handled by
+	// this gateway. Example: `["/ipfs", "/ipns"]`
+	// Useful if you only want to support immutable `/ipfs`.
+	Paths []string
+
+	// UseSubdomains indicates whether or not this gateway uses subdomains
+	// for IPFS resources instead of paths. That is: http://CID.ipfs.GATEWAY/...
+	//
+	// If this flag is set, any /ipns/$id and/or /ipfs/$id paths in Paths
+	// will be permanently redirected to http://$id.[ipns|ipfs].$gateway/.
+	//
+	// We do not support using both paths and subdomains for a single domain
+	// for security reasons (Origin isolation).
+	UseSubdomains bool
+
+	// NoDNSLink configures this gateway to _not_ resolve DNSLink for the
+	// specific FQDN provided in `Host` HTTP header. Useful when you want to
+	// explicitly allow or refuse hosting a single hostname. To refuse all
+	// DNSLinks in `Host` processing, set NoDNSLink in Config instead. This setting
+	// overrides the global setting.
+	NoDNSLink bool
+
+	// InlineDNSLink configures this gateway to always inline DNSLink names
+	// (FQDN) into a single DNS label in order to interop with wildcard TLS certs
+	// and Origin per CID isolation provided by rules like https://publicsuffix.org
+	// This should be set to true if you use HTTPS.
+	InlineDNSLink bool
+
+	// DeserializedResponses configures this gateway to support returning data
+	// in deserialized format. This setting overrides the global setting.
+	DeserializedResponses bool
 }
 
 // TODO: Is this what we want for ImmutablePath?
@@ -221,7 +275,17 @@ func AddAccessControlHeaders(headers map[string][]string) {
 type RequestContextKey string
 
 const (
-	DNSLinkHostnameKey RequestContextKey = "dnslink-hostname"
+	// GatewayHostnameKey is the key for the hostname at which the gateway is
+	// operating. It may be a DNSLink, Subdomain or Regular gateway.
 	GatewayHostnameKey RequestContextKey = "gw-hostname"
-	ContentPathKey     RequestContextKey = "content-path"
+
+	// DNSLinkHostnameKey is the key for the hostname of a DNSLink Gateway:
+	// https://specs.ipfs.tech/http-gateways/dnslink-gateway/
+	DNSLinkHostnameKey RequestContextKey = "dnslink-hostname"
+
+	// SubdomainHostnameKey is the key for the hostname of a Subdomain Gateway:
+	// https://specs.ipfs.tech/http-gateways/subdomain-gateway/
+	SubdomainHostnameKey RequestContextKey = "subdomain-hostname"
+
+	ContentPathKey RequestContextKey = "content-path"
 )

gateway/gateway_test.go

@@ -198,14 +198,20 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, *mock
 }
 
 func newTestServer(t *testing.T, api IPFSBackend) *httptest.Server {
-	config := Config{Headers: map[string][]string{}}
+	return newTestServerWithConfig(t, api, Config{
+		Headers:               map[string][]string{},
+		DeserializedResponses: true,
+	})
+}
+
+func newTestServerWithConfig(t *testing.T, api IPFSBackend, config Config) *httptest.Server {
 	AddAccessControlHeaders(config.Headers)
 
 	handler := NewHandler(config, api)
 	mux := http.NewServeMux()
 	mux.Handle("/ipfs/", handler)
 	mux.Handle("/ipns/", handler)
-	handler = WithHostname(mux, api, map[string]*Specification{}, false)
+	handler = WithHostname(config, api, mux)
 
 	ts := httptest.NewServer(handler)
 	t.Cleanup(func() { ts.Close() })
@@ -573,3 +579,128 @@ func TestIpnsBase58MultihashRedirect(t *testing.T) {
 		assert.Equal(t, "/ipns/k2k4r8ol4m8kkcqz509c1rcjwunebj02gcnm5excpx842u736nja8ger?keep=query", res.Header.Get("Location"))
 	})
 }
+
+func TestIpfsTrustlessMode(t *testing.T) {
+	api, root := newMockAPI(t)
+
+	ts := newTestServerWithConfig(t, api, Config{
+		Headers:   map[string][]string{},
+		NoDNSLink: false,
+		PublicGateways: map[string]*Specification{
+			"trustless.com": {
+				Paths: []string{"/ipfs", "/ipns"},
+			},
+			"trusted.com": {
+				Paths:                 []string{"/ipfs", "/ipns"},
+				DeserializedResponses: true,
+			},
+		},
+	})
+	t.Logf("test server url: %s", ts.URL)
+
+	trustedFormats := []string{"", "dag-json", "dag-cbor", "tar", "json", "cbor"}
+	trustlessFormats := []string{"raw", "car"}
+
+	doRequest := func(t *testing.T, path, host string, expectedStatus int) {
+		req, err := http.NewRequest(http.MethodGet, ts.URL+path, nil)
+		assert.Nil(t, err)
+
+		if host != "" {
+			req.Host = host
+		}
+
+		res, err := doWithoutRedirect(req)
+		assert.Nil(t, err)
+		defer res.Body.Close()
+		assert.Equal(t, expectedStatus, res.StatusCode)
+	}
+
+	doIpfsCidRequests := func(t *testing.T, formats []string, host string, expectedStatus int) {
+		for _, format := range formats {
+			doRequest(t, "/ipfs/"+root.String()+"/?format="+format, host, expectedStatus)
+		}
+	}
+
+	doIpfsCidPathRequests := func(t *testing.T, formats []string, host string, expectedStatus int) {
+		for _, format := range formats {
+			doRequest(t, "/ipfs/"+root.String()+"/EmptyDir/?format="+format, host, expectedStatus)
+		}
+	}
+
+	trustedTests := func(t *testing.T, host string) {
+		doIpfsCidRequests(t, trustlessFormats, host, http.StatusOK)
+		doIpfsCidRequests(t, trustedFormats, host, http.StatusOK)
+		doIpfsCidPathRequests(t, trustlessFormats, host, http.StatusOK)
+		doIpfsCidPathRequests(t, trustedFormats, host, http.StatusOK)
+	}
+
+	trustlessTests := func(t *testing.T, host string) {
+		doIpfsCidRequests(t, trustlessFormats, host, http.StatusOK)
+		doIpfsCidRequests(t, trustedFormats, host, http.StatusNotImplemented)
+		doIpfsCidPathRequests(t, trustlessFormats, host, http.StatusNotImplemented)
+		doIpfsCidPathRequests(t, trustedFormats, host, http.StatusNotImplemented)
+	}
+
+	t.Run("Explicit Trustless Gateway", func(t *testing.T) {
+		t.Parallel()
+		trustlessTests(t, "trustless.com")
+	})
+
+	t.Run("Explicit Trusted Gateway", func(t *testing.T) {
+		t.Parallel()
+		trustedTests(t, "trusted.com")
+	})
+
+	t.Run("Implicit Default Trustless Gateway", func(t *testing.T) {
+		t.Parallel()
+		trustlessTests(t, "not.configured.com")
+		trustlessTests(t, "localhost")
+		trustlessTests(t, "127.0.0.1")
+		trustlessTests(t, "::1")
+	})
+}
+
+func TestIpnsTrustlessMode(t *testing.T) {
+	api, root := newMockAPI(t)
+	api.namesys["/ipns/trustless.com"] = path.FromCid(root)
+	api.namesys["/ipns/trusted.com"] = path.FromCid(root)
+
+	ts := newTestServerWithConfig(t, api, Config{
+		Headers:   map[string][]string{},
+		NoDNSLink: false,
+		PublicGateways: map[string]*Specification{
+			"trustless.com": {
+				Paths: []string{"/ipfs", "/ipns"},
+			},
+			"trusted.com": {
+				Paths:                 []string{"/ipfs", "/ipns"},
+				DeserializedResponses: true,
+			},
+		},
+	})
+	t.Logf("test server url: %s", ts.URL)
+
+	doRequest := func(t *testing.T, path, host string, expectedStatus int) {
+		req, err := http.NewRequest(http.MethodGet, ts.URL+path, nil)
+		assert.Nil(t, err)
+
+		if host != "" {
+			req.Host = host
+		}
+
+		res, err := doWithoutRedirect(req)
+		assert.Nil(t, err)
+		defer res.Body.Close()
+		assert.Equal(t, expectedStatus, res.StatusCode)
+	}
+
+	// DNSLink only. Not supported for trustless. Supported for trusted, except
+	// format=ipns-record which is unavailable for DNSLink.
+	doRequest(t, "/", "trustless.com", http.StatusNotImplemented)
+	doRequest(t, "/EmptyDir/", "trustless.com", http.StatusNotImplemented)
+	doRequest(t, "/?format=ipns-record", "trustless.com", http.StatusNotImplemented)
+
+	doRequest(t, "/", "trusted.com", http.StatusOK)
+	doRequest(t, "/EmptyDir/", "trusted.com", http.StatusOK)
+	doRequest(t, "/?format=ipns-record", "trusted.com", http.StatusBadRequest)
+}

gateway/handler.go

@@ -17,6 +17,7 @@ import (
 	"time"
 
 	ipath "github.com/ipfs/boxo/coreiface/path"
+	"github.com/ipfs/boxo/ipns"
 	cid "github.com/ipfs/go-cid"
 	logging "github.com/ipfs/go-log"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -238,6 +239,13 @@ func (i *handler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
 	i.addUserHeaders(w) // ok, _now_ write user's headers.
 	w.Header().Set("X-Ipfs-Path", contentPath.String())
 
+	// Trustless gateway.
+	if !i.onlyDeserializedResponses(r) && !i.isSerializedRequest(contentPath, responseFormat) {
+		err := errors.New("only trustless requests are accepted: https://specs.ipfs.tech/http-gateways/trustless-gateway/")
+		webError(w, err, http.StatusNotImplemented)
+		return
+	}
+
 	// TODO: Why did the previous code do path resolution, was that a bug?
 	// TODO: Does If-None-Match apply here?
 	if responseFormat == "application/vnd.ipfs.ipns-record" {
@@ -321,6 +329,62 @@ func (i *handler) addUserHeaders(w http.ResponseWriter) {
 	}
 }
 
+func (i *handler) onlyDeserializedResponses(r *http.Request) bool {
+	// Get the host, by default the request's Host. If this request went through
+	// WithHostname, also check for the key in the context. If that is not present,
+	// also check X-Forwarded-Host to support reverse proxies.
+	host := r.Host
+	if h, ok := r.Context().Value(GatewayHostnameKey).(string); ok {
+		host = h
+	} else if xHost := r.Header.Get("X-Forwarded-Host"); xHost != "" {
+		host = xHost
+	}
+
+	// If the gateway is defined, return whatever is set.
+	if gw, ok := i.config.PublicGateways[host]; ok {
+		return gw.DeserializedResponses
+	}
+
+	// Otherwise, the default.
+	return i.config.DeserializedResponses
+}
+
+func (i *handler) isSerializedRequest(contentPath ipath.Path, responseFormat string) bool {
+	// Only allow "/{#1}/{#2}"-like paths.
+	trimmedPath := strings.Trim(contentPath.String(), "/")
+	pathComponents := strings.Split(trimmedPath, "/")
+	if len(pathComponents) != 2 {
+		return false
+	}
+
+	if contentPath.Namespace() == "ipns" {
+		// Only ipns records allowed until https://github.com/ipfs/specs/issues/369 is resolved
+		if responseFormat != "application/vnd.ipfs.ipns-record" {
+			return false
+		}
+
+		// Only valid IPNS names, no DNSLink.
+		if _, err := ipns.Decode(pathComponents[1]); err != nil {
+			return false
+		}
+
+		return true
+	}
+
+	// Only valid CIDs.
+	if _, err := cid.Decode(pathComponents[1]); err != nil {
+		return false
+	}
+
+	switch responseFormat {
+	case "application/vnd.ipld.raw",
+		"application/vnd.ipld.car":
+		return true
+	default:
+		return false
+	}
+}
+
 func panicHandler(w http.ResponseWriter) {
 	if r := recover(); r != nil {
 		log.Error("A panic occurred in the gateway handler!")