feat(routing/http): IPIP-518: HTTP(S) URLs in Routing V1 API

CHANGELOG.md

@@ -16,6 +16,11 @@ The following emojis are used to highlight certain changes:
 
 ### Added
 
+- `routing/http`: ✨ added `generic` schema support per [IPIP-518](https://github.com/ipfs/specs/pull/518)
+  - new `GenericRecord` type with duck-typed `Addrs` (multiaddrs and URIs) and arbitrary string `ID` (not limited to PeerIDs)
+  - `contentrouter` converts GenericRecord to `peer.AddrInfo` for backward compatibility: HTTP(S) URLs become `/dns/host/tcp/port/https` multiaddrs; PeerID is derived from `did:key:` or generated as a deterministic placeholder when the ID is not a native libp2p PeerID
+  - `filter-addrs` extended to match URI schemes (e.g. `?filter-addrs=https`) in addition to multiaddr protocol names
+  - generic records are capped at 10 KiB per IPIP-518 spec
 - `ipld/unixfs/io`: added `SizeEstimationMode` for configurable HAMT sharding threshold decisions. Supports legacy link-based estimation (`SizeEstimationLinks`), accurate block-based estimation (`SizeEstimationBlock`), or disabling size-based thresholds (`SizeEstimationDisabled`). [#1088](https://github.com/ipfs/boxo/pull/1088), [IPIP-499](https://github.com/ipfs/specs/pull/499)
 - `ipld/unixfs/io`: added `UnixFSProfile` with `UnixFS_v0_2015` and `UnixFS_v1_2025` presets for CID-deterministic file and directory DAG construction. [#1088](https://github.com/ipfs/boxo/pull/1088), [IPIP-499](https://github.com/ipfs/specs/pull/499)
 - `files`: `NewSerialFileWithOptions` now supports controlling whether symlinks are preserved or dereferenced before being added to IPFS. See `SerialFileOptions.DereferenceSymlinks`. [#1088](https://github.com/ipfs/boxo/pull/1088), [IPIP-499](https://github.com/ipfs/specs/pull/499)

go.mod

@@ -51,6 +51,7 @@ require (
 	github.com/multiformats/go-multicodec v0.10.0
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/multiformats/go-multistream v0.6.1
+	github.com/multiformats/go-varint v0.1.0
 	github.com/polydawn/refmt v0.89.0
 	github.com/prometheus/client_golang v1.23.2
 	github.com/prometheus/client_model v0.6.2
@@ -115,7 +116,6 @@ require (
 	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/multiformats/go-base36 v0.2.0 // indirect
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
-	github.com/multiformats/go-varint v0.1.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect

routing/http/client/client.go

@@ -180,8 +180,9 @@ func WithUserAgent(ua string) Option {
 func WithProviderInfo(peerID peer.ID, addrs []multiaddr.Multiaddr) Option {
 	return func(c *Client) error {
 		c.peerID = peerID
-		for _, a := range addrs {
-			c.addrs = append(c.addrs, types.Multiaddr{Multiaddr: a})
+		c.addrs = make([]types.Multiaddr, len(addrs))
+		for i, a := range addrs {
+			c.addrs[i] = types.Multiaddr{Multiaddr: a}
 		}
 		return nil
 	}

routing/http/client/client_test.go

@@ -153,18 +153,19 @@ func makeCID() cid.Cid {
 	return c
 }
 
-func drAddrsToAddrs(drmas []types.Multiaddr) (addrs []multiaddr.Multiaddr) {
-	for _, a := range drmas {
+func drAddrsToAddrs(draddrs []types.Multiaddr) (addrs []multiaddr.Multiaddr) {
+	for _, a := range draddrs {
 		addrs = append(addrs, a.Multiaddr)
 	}
 	return
 }
 
-func addrsToDRAddrs(addrs []multiaddr.Multiaddr) (drmas []types.Multiaddr) {
-	for _, a := range addrs {
-		drmas = append(drmas, types.Multiaddr{Multiaddr: a})
+func addrsToDRAddrs(addrs []multiaddr.Multiaddr) []types.Multiaddr {
+	draddrs := make([]types.Multiaddr, len(addrs))
+	for i, a := range addrs {
+		draddrs[i] = types.Multiaddr{Multiaddr: a}
 	}
-	return
+	return draddrs
 }
 
 func makePeerRecord(protocols []string) types.PeerRecord {
@@ -410,6 +411,94 @@ func TestClient_FindProviders(t *testing.T) {
 	}
 }
 
+// TestClient_FindProvidersWithGenericRecord verifies that a client correctly
+// receives and deserializes GenericRecord from a FindProviders response.
+// This is an end-to-end test through the real server handler and HTTP client.
+func TestClient_FindProvidersWithGenericRecord(t *testing.T) {
+	peerRecord := makePeerRecord([]string{"transport-bitswap"})
+	genericRecord := types.GenericRecord{
+		Schema:    types.SchemaGeneric,
+		ID:        "did:key:z6Mkm1example",
+		Protocols: []string{"transport-ipfs-gateway-http"},
+		Addrs: types.Addresses{
+			mustAddr(t, "https://trustless-gateway.example.com"),
+			mustAddr(t, "/ip4/1.2.3.4/tcp/5000"),
+		},
+	}
+
+	routerResults := []iter.Result[types.Record]{
+		{Val: &peerRecord},
+		{Val: &genericRecord},
+	}
+
+	t.Run("streaming NDJSON response", func(t *testing.T) {
+		deps := makeTestDeps(t, []Option{WithProtocolFilter([]string{})}, nil)
+		client := deps.client
+		router := deps.router
+
+		cid := makeCID()
+		router.On("FindProviders", mock.Anything, cid, 0).
+			Return(iter.FromSlice(routerResults), nil)
+
+		resultIter, err := client.FindProviders(context.Background(), cid)
+		require.NoError(t, err)
+
+		results := iter.ReadAll[iter.Result[types.Record]](resultIter)
+		require.Len(t, results, 2)
+
+		// First result: PeerRecord
+		require.NoError(t, results[0].Err)
+		_, ok := results[0].Val.(*types.PeerRecord)
+		require.True(t, ok, "first result should be PeerRecord")
+
+		// Second result: GenericRecord
+		require.NoError(t, results[1].Err)
+		gr, ok := results[1].Val.(*types.GenericRecord)
+		require.True(t, ok, "second result should be GenericRecord")
+		assert.Equal(t, types.SchemaGeneric, gr.Schema)
+		assert.Equal(t, "did:key:z6Mkm1example", gr.ID)
+		assert.Equal(t, []string{"transport-ipfs-gateway-http"}, gr.Protocols)
+		require.Len(t, gr.Addrs, 2)
+		assert.Equal(t, "https://trustless-gateway.example.com", gr.Addrs[0].String())
+		assert.Equal(t, "/ip4/1.2.3.4/tcp/5000", gr.Addrs[1].String())
+	})
+
+	t.Run("non-streaming JSON response", func(t *testing.T) {
+		deps := makeTestDeps(t,
+			[]Option{WithProtocolFilter([]string{})},
+			[]server.Option{server.WithStreamingResultsDisabled()},
+		)
+		client := deps.client
+		router := deps.router
+
+		cid := makeCID()
+		router.On("FindProviders", mock.Anything, cid, 20).
+			Return(iter.FromSlice(routerResults), nil)
+
+		resultIter, err := client.FindProviders(context.Background(), cid)
+		require.NoError(t, err)
+
+		results := iter.ReadAll[iter.Result[types.Record]](resultIter)
+		require.Len(t, results, 2)
+
+		// Verify GenericRecord survived the JSON (non-streaming) path
+		require.NoError(t, results[1].Err)
+		gr, ok := results[1].Val.(*types.GenericRecord)
+		require.True(t, ok, "second result should be GenericRecord")
+		assert.Equal(t, "did:key:z6Mkm1example", gr.ID)
+		require.Len(t, gr.Addrs, 2)
+		assert.True(t, gr.Addrs[0].IsURL())
+		assert.True(t, gr.Addrs[1].IsMultiaddr())
+	})
+}
+
+func mustAddr(t *testing.T, s string) types.Address {
+	t.Helper()
+	a, err := types.NewAddress(s)
+	require.NoError(t, err)
+	return a
+}
+
 func TestClient_Provide(t *testing.T) {
 	cases := []struct {
 		name            string

routing/http/contentrouter/contentrouter.go

@@ -3,6 +3,7 @@ package contentrouter
 import (
 	"context"
 	"reflect"
+	"slices"
 	"strings"
 	"time"
 
@@ -142,8 +143,21 @@ func (c *contentRouter) Ready() bool {
 	return true
 }
 
-// readProviderResponses reads peer records (and bitswap records for legacy
-// compatibility) from the iterator into the given channel.
+// readProviderResponses reads provider records from the iterator into the given
+// channel. PeerRecord and BitswapRecord are converted directly. GenericRecord
+// is converted on a best-effort basis:
+//   - If the ID is a valid libp2p PeerID, the record is always converted
+//     regardless of Protocols. This supports the legacy pattern where a
+//     PeerID + /https multiaddr was used as a hint to probe for a Trustless
+//     IPFS HTTP Gateway (even without explicit protocol declaration).
+//   - If the ID is not a PeerID but the record advertises
+//     transport-ipfs-gateway-http with HTTP(S) URLs, a PeerID is derived
+//     from did:key: or generated as a placeholder.
+//   - Other records with non-PeerID identifiers are skipped.
+//
+// Addresses are converted via [types.Address.ToMultiaddr]; HTTPS URLs
+// become /dns/host/tcp/443/https multiaddrs. Non-convertible addresses
+// are dropped.
 func readProviderResponses(ctx context.Context, iter iter.ResultIter[types.Record], ch chan<- peer.AddrInfo) {
 	defer close(ch)
 	defer iter.Close()
@@ -175,6 +189,60 @@ func readProviderResponses(ctx context.Context, iter iter.ResultIter[types.Recor
 			}:
 			}
 
+		case types.SchemaGeneric:
+			result, ok := v.(*types.GenericRecord)
+			if !ok {
+				logger.Errorw(
+					"problem casting find providers result",
+					"Schema", v.GetSchema(),
+					"Type", reflect.TypeOf(v).String(),
+				)
+				continue
+			}
+
+			pid, err := peer.Decode(result.ID)
+			if err != nil {
+				// For HTTP gateway providers, try harder to derive a PeerID.
+				// Kubo and Rainbow need a PeerID to pass multiaddr addresses
+				// over legacy routing APIs even when the provider uses
+				// non-PeerID identifiers like did:key:.
+				if slices.Contains(result.Protocols, "transport-ipfs-gateway-http") && hasHTTPURL(result.Addrs) {
+					pid, err = peerIDFromDIDKey(result.ID)
+					if err != nil {
+						pid = peerIDPlaceholderFromArbitraryID(result.ID)
+					}
+				} else {
+					// Records with non-PeerID identifiers and no recognized
+					// protocol are skipped: without a protocol hint we cannot
+					// determine how to use the addresses in legacy routing APIs.
+					logger.Debugw("skipping generic record with non-PeerID identifier", "ID", result.ID)
+					continue
+				}
+			}
+
+			// Convert addresses to multiaddrs. URLs are converted via
+			// ToMultiaddr (e.g. https://host -> /dns/host/tcp/443/https).
+			// Addresses that cannot be converted are dropped.
+			var addrs []multiaddr.Multiaddr
+			for i := range result.Addrs {
+				if ma := result.Addrs[i].ToMultiaddr(); ma != nil {
+					addrs = append(addrs, ma)
+				}
+			}
+			if len(addrs) == 0 {
+				logger.Debugw("skipping generic record with no convertible addresses", "ID", result.ID)
+				continue
+			}
+
+			select {
+			case <-ctx.Done():
+				return
+			case ch <- peer.AddrInfo{
+				ID:    pid,
+				Addrs: addrs,
+			}:
+			}
+
 		//nolint:staticcheck
 		//lint:ignore SA1019 // ignore staticcheck
 		case types.SchemaBitswap: