feat: built-in content blocking based on IPIP-383

README.md

@@ -30,6 +30,7 @@ Featureset
 - [HTTP Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/) (`/api/v0`) to access and control the daemon
 - [Command Line Interface](https://docs.ipfs.tech/reference/kubo/cli/) based on (`/api/v0`) RPC API
 - [WebUI](https:/ipfs/ipfs-webui/#readme) to manage the Kubo node
+- [Content blocking](/docs/content-blocking.md) support for operators of public nodes
 
 ### Other implementations
 

core/commands/dag/export.go

@@ -14,26 +14,32 @@ import (
  cid "github.com/ipfs/go-cid"
  ipld "github.com/ipfs/go-ipld-format"
  "github.com/ipfs/kubo/core/commands/cmdenv"
+ "github.com/ipfs/kubo/core/commands/cmdutils"
 
  cmds "github.com/ipfs/go-ipfs-cmds"
  gocar "github.com/ipld/go-car"
  selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 )
 
 func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
- c, err := cid.Decode(req.Arguments[0])
+ // Accept CID or a content path
+ p, err := cmdutils.PathOrCidPath(req.Arguments[0])
  if err != nil {
- return fmt.Errorf(
- "unable to parse root specification (currently only bare CIDs are supported): %s",
- err,
- )
+ return err
  }
 
  api, err := cmdenv.GetApi(env, req)
  if err != nil {
  return err
  }
 
+ // Resolve path and confirm the root block is available, fail fast if not
+ b, err := api.Block().Stat(req.Context, p)
+ if err != nil {
+ return err
+ }
+ c := b.Path().RootCid()
+
  pipeR, pipeW := io.Pipe()
 
  errCh := make(chan error, 2) // we only report the 1st error

core/corehttp/gateway.go

@@ -132,7 +132,11 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
  }
  }
 
- backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
+ backend, err := gateway.NewBlocksBackend(bserv,
+ gateway.WithValueStore(vsRouting),
+ gateway.WithNameSystem(nsys),
+ gateway.WithResolver(n.UnixFSPathResolver),
+ )
  if err != nil {
  return nil, err
  }

docs/changelogs/v0.24.md

@@ -6,6 +6,7 @@
 
 - [Overview](#overview)
 - [🔦 Highlights](#-highlights)
+ - [Support for content blocking](#support-for-content-blocking)
  - [Gateway: the root of the CARs are no longer meaningful](#gateway-the-root-of-the-cars-are-no-longer-meaningful)
  - [IPNS: improved publishing defaults](#ipns-improved-publishing-defaults)
  - [IPNS: record TTL is used for caching](#ipns-record-ttl-is-used-for-caching)
@@ -16,6 +17,14 @@
 
 ### 🔦 Highlights
 
+#### Support for content blocking
+
+This Kubo release ships with built-in content-blocking subsystem [announced earlier this year](https://blog.ipfs.tech/2023-content-blocking-for-the-ipfs-stack/).
+Content blocking is an opt-in decision made by the operator of `ipfs daemon`.
+The official build does not ship with any denylists.
+
+Learn more at [`/docs/content-blocking.md`](https:/ipfs/kubo/blob/master/docs/content-blocking.md)
+
 #### Gateway: the root of the CARs are no longer meaningful
 
 When requesting a CAR from the gateway, the root of the CAR might no longer be

docs/content-blocking.md

@@ -0,0 +1,62 @@
+<h1 align="center">
+ <br>
+ <a href="#readme"><img src="https:/ipfs-shipyard/nopfs/blob/41484a818e6542314f784da852fc41b76f2d48a6/logo.png?raw=true" alt="content blocking logo" title="content blocking in Kubo" width="200"></a>
+ <br>
+ Content Blocking in Kubo
+ <br>
+</h1>
+
+Kubo ships with built-in support for denylist format from [IPIP-383](https:/ipfs/specs/pull/383).
+
+## Default behavior
+
+Official Kubo build does not ship with any denylists enabled by default.
+
+Content blocking is an opt-in decision made by the operator of `ipfs daemon`.
+
+## How to enable blocking
+
+Place a `*.deny` file in one of directories:
+
+- `$IPFS_PATH/denylists/` (`$HOME/.ipfs/denylists/` if `IPFS_PATH` is not set)
+- `$XDG_CONFIG_HOME/ipfs/denylists/` (`$HOME/.config/ipfs/denylists/` if `XDG_CONFIG_HOME` is not set)
+- `/etc/ipfs/denylists/` (global)
+
+Files need to be present before starting the `ipfs daemon` in order to be watched for updates.
+
+If a new denylist file is added, `ipfs daemon` needs to be restarted.
+
+CLI and Gateway users will receive errors in response to request impacted by a blocklist:
+
+```
+Error: /ipfs/QmQvjk82hPkSaZsyJ8vNER5cmzKW7HyGX5XVusK7EAenCN is blocked and cannot be provided
+```
+
+End user is not informed about the exact reason, see [How to
+debug](#how-to-debug) if you need to find out which line of which denylist
+caused the request to be blocked.
+
+## Denylist file format
+
+[NOpfs](https:/ipfs-shipyard/nopfs) supports the format from [IPIP-383](https:/ipfs/specs/pull/383).
+
+Example: https://badbits.dwebops.pub/badbits.deny
+
+## How to suspend blocking without removing denylists
+
+Set `IPFS_CONTENT_BLOCKING_DISABLE` environment variable to `true` and restart the daemon.
+
+
+## How to debug
+
+Debug logging of `nopfs` subsystem can be enabled with `GOLOG_LOG_LEVEL="nopfs=debug"`
+
+All block events are logged as warnings on a separate level named `nopfs-blocks`.
+
+To only log requests for blocked content set `GOLOG_LOG_LEVEL="nopfs-blocks=warn"`:
+
+```
+WARN (...) QmRFniDxwxoG2n4AcnGhRdjqDjCM5YeUcBE75K8WXmioH3: blocked (test.deny:9)
+```
+
+

docs/environment-variables.md

@@ -131,6 +131,12 @@ The above will replace implicit HTTP routers with single one, allowing for
 inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done`
 or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy).
 
+
+## `IPFS_CONTENT_BLOCKING_DISABLE`
+
+Disables the content-blocking subsystem. No denylists will be watched and no
+content will be blocked.
+
 ## `LIBP2P_TCP_REUSEPORT`
 
 Kubo tries to reuse the same source port for all connections to improve NAT

docs/examples/kubo-as-a-library/go.mod

@@ -7,7 +7,7 @@ go 1.20
 replace github.com/ipfs/kubo => ./../../..
 
 require (
- github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd
+ github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9
  github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
  github.com/libp2p/go-libp2p v0.31.0
  github.com/multiformats/go-multiaddr v0.11.0
@@ -40,6 +40,7 @@ require (
  github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
  github.com/flynn/noise v1.0.0 // indirect
  github.com/francoispqt/gojay v1.2.13 // indirect
+ github.com/fsnotify/fsnotify v1.6.0 // indirect
  github.com/gabriel-vasile/mimetype v1.4.1 // indirect
  github.com/go-logr/logr v1.2.4 // indirect
  github.com/go-logr/stdr v1.2.2 // indirect
@@ -60,6 +61,8 @@ require (
  github.com/hashicorp/golang-lru v0.5.4 // indirect
  github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
  github.com/huin/goupnp v1.2.0 // indirect
+ github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385 // indirect
+ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385 // indirect
  github.com/ipfs/bbloom v0.0.4 // indirect
  github.com/ipfs/go-bitfield v1.1.0 // indirect
  github.com/ipfs/go-block-format v0.2.0 // indirect
@@ -194,5 +197,6 @@ require (
  google.golang.org/grpc v1.55.0 // indirect
  google.golang.org/protobuf v1.31.0 // indirect
  gopkg.in/square/go-jose.v2 v2.5.1 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
  lukechampine.com/blake3 v1.2.1 // indirect
 )

docs/examples/kubo-as-a-library/go.sum

@@ -163,6 +163,7 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
 github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q=
 github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -298,10 +299,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
 github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385 h1:ShsOXLWzQFpqrd/YHmOX+/NU9FbI2BbzwWQjkOvAE9o=
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231019201810-ca629a80a385/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385 h1:jAeuoctl0Bq6Wack8CHfyYQDFaIVpP10HmJoKB1GF7I=
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231019201810-ca629a80a385/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o=
-github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
+github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ=
+github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -1001,6 +1006,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=

go.mod

@@ -15,7 +15,9 @@ require (
  github.com/fsnotify/fsnotify v1.6.0
  github.com/google/uuid v1.3.1
  github.com/hashicorp/go-multierror v1.1.1
- github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd
+ github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214
+ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214
+ github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9
  github.com/ipfs/go-block-format v0.2.0
  github.com/ipfs/go-cid v0.4.1
  github.com/ipfs/go-cidutil v0.1.0

go.sum

@@ -333,10 +333,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
 github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214 h1:awerQVtBHg8ielQL2BOuayV6ne9VNiPgGgx9DrHfkqQ=
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231020165528-0fc78f3af214/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214 h1:RYSU/v4NCf4ifIkhsmyNJqZMEItfLE4A13kjO1m2eXs=
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231020165528-0fc78f3af214/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
-github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o=
-github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
+github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9 h1:dmjRJU0cMxrGIcd9gBgrRATl2DL0xrmJM/jedPKhQbQ=
+github.com/ipfs/boxo v0.13.2-0.20231019090647-a7e134e54ff9/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=

plugin/loader/preload.go

@@ -7,6 +7,7 @@ import (
  pluginfxtest "github.com/ipfs/kubo/plugin/plugins/fxtest"
  pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git"
  pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds"
+ pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs"
  pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog"
 )
 
@@ -22,4 +23,5 @@ func init() {
  Preload(pluginlevelds.Plugins...)
  Preload(pluginpeerlog.Plugins...)
  Preload(pluginfxtest.Plugins...)
+ Preload(pluginnopfs.Plugins...)
 }

plugin/loader/preload_list

@@ -11,3 +11,4 @@ flatfs github.com/ipfs/kubo/plugin/plugins/flatfs *
 levelds github.com/ipfs/kubo/plugin/plugins/levelds *
 peerlog github.com/ipfs/kubo/plugin/plugins/peerlog *
 fxtest github.com/ipfs/kubo/plugin/plugins/fxtest *
+nopfs github.com/ipfs/kubo/plugin/plugins/nopfs *

plugin/plugins/nopfs/nopfs.go

@@ -0,0 +1,83 @@
+package nopfs
+
+import (
+ "os"
+ "path/filepath"
+
+ "github.com/ipfs-shipyard/nopfs"
+ "github.com/ipfs-shipyard/nopfs/ipfs"
+ "github.com/ipfs/kubo/config"
+ "github.com/ipfs/kubo/core"
+ "github.com/ipfs/kubo/core/node"
+ "github.com/ipfs/kubo/plugin"
+ "go.uber.org/fx"
+)
+
+// Plugins sets the list of plugins to be loaded.
+var Plugins = []plugin.Plugin{
+ &nopfsPlugin{},
+}
+
+// fxtestPlugin is used for testing the fx plugin.
+// It merely adds an fx option that logs a debug statement, so we can verify that it works in tests.
+type nopfsPlugin struct{}
+
+var _ plugin.PluginFx = (*nopfsPlugin)(nil)
+
+func (p *nopfsPlugin) Name() string {
+ return "nopfs"
+}
+
+func (p *nopfsPlugin) Version() string {
+ return "0.0.10"
+}
+
+func (p *nopfsPlugin) Init(env *plugin.Environment) error {
+ return nil
+}
+
+// MakeBlocker is a factory for the blocker so that it can be provided with Fx.
+func MakeBlocker() (*nopfs.Blocker, error) {
+ ipfsPath, err := config.PathRoot()
+ if err != nil {
+ return nil, err
+ }
+
+ defaultFiles, err := nopfs.GetDenylistFiles()
+ if err != nil {
+ return nil, err
+ }
+
+ kuboFiles, err := nopfs.GetDenylistFilesInDir(filepath.Join(ipfsPath, "denylists"))
+ if err != nil {
+ return nil, err
+ }
+
+ files := append(defaultFiles, kuboFiles...)
+
+ return nopfs.NewBlocker(files)
+}
+
+// PathResolvers returns wrapped PathResolvers for Kubo.
+func PathResolvers(fetchers node.FetchersIn, blocker *nopfs.Blocker) node.PathResolversOut {
+ res := node.PathResolverConfig(fetchers)
+ return node.PathResolversOut{
+ IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker),
+ UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker),
+ }
+}
+
+func (p *nopfsPlugin) Options(info core.FXNodeInfo) ([]fx.Option, error) {
+ if os.Getenv("IPFS_CONTENT_BLOCKING_DISABLE") != "" {
+ return info.FXOptions, nil
+ }
+
+ opts := append(
+ info.FXOptions,
+ fx.Provide(MakeBlocker),
+ fx.Decorate(ipfs.WrapBlockService),
+ fx.Decorate(ipfs.WrapNameSystem),
+ fx.Decorate(PathResolvers),
+ )
+ return opts, nil
+}