fix: add dag walker for json codec (#247)

Since the json codec is bundled with multiformats we should add a
dag walker for it to allow pinning json blocks.

Fixes #246

packages/helia/package.json

@@ -53,6 +53,8 @@
  "@chainsafe/libp2p-noise": "^13.0.0",
  "@chainsafe/libp2p-yamux": "^5.0.0",
  "@helia/interface": "^2.0.0",
+ "@ipld/dag-cbor": "^9.0.0",
+ "@ipld/dag-json": "^10.0.1",
  "@ipld/dag-pb": "^4.0.3",
  "@libp2p/bootstrap": "^9.0.2",
  "@libp2p/interface": "^0.1.1",
@@ -86,8 +88,6 @@
  "uint8arrays": "^4.0.3"
  },
  "devDependencies": {
- "@ipld/dag-cbor": "^9.0.0",
- "@ipld/dag-json": "^10.0.1",
  "@multiformats/mafmt": "^12.1.5",
  "@types/sinon": "^10.0.14",
  "aegir": "^40.0.8",

packages/helia/src/pins.ts

@@ -6,7 +6,7 @@ import defer from 'p-defer'
 import PQueue from 'p-queue'
 import { CustomProgressEvent, type ProgressOptions } from 'progress-events'
 import { equals as uint8ArrayEquals } from 'uint8arrays/equals'
-import { cborWalker, dagPbWalker, jsonWalker, rawWalker } from './utils/dag-walkers.js'
+import { dagCborWalker, dagJsonWalker, dagPbWalker, jsonWalker, rawWalker } from './utils/dag-walkers.js'
 import type { DAGWalker } from './index.js'
 import type { AddOptions, AddPinEvents, IsPinnedOptions, LsOptions, Pin, Pins, RmOptions } from '@helia/interface/pins'
 import type { GetBlockProgressEvents } from '@helia/interface/src/blocks.js'
@@ -16,7 +16,8 @@ import type { Blockstore } from 'interface-blockstore'
 const DEFAULT_DAG_WALKERS = [
  rawWalker,
  dagPbWalker,
- cborWalker,
+ dagCborWalker,
+ dagJsonWalker,
  jsonWalker
 ]
 

packages/helia/src/utils/dag-walkers.ts

@@ -1,11 +1,14 @@
 /* eslint max-depth: ["error", 7] */
 
+import * as dagCbor from '@ipld/dag-cbor'
+import * as dagJson from '@ipld/dag-json'
 import * as dagPb from '@ipld/dag-pb'
 import * as cborg from 'cborg'
 import { Type, Token } from 'cborg'
 import * as cborgJson from 'cborg/json'
 import { CID } from 'multiformats'
 import { base64 } from 'multiformats/bases/base64'
+import * as json from 'multiformats/codecs/json'
 import * as raw from 'multiformats/codecs/raw'
 import type { DAGWalker } from '../index.js'
 
@@ -39,8 +42,8 @@ const CID_TAG = 42
  * all we are interested in is extracting the the CIDs from the block
  * so we can just use cborg for that.
  */
-export const cborWalker: DAGWalker = {
- codec: 0x71,
+export const dagCborWalker: DAGWalker = {
+ codec: dagCbor.code,
  async * walk (block) {
  const cids: CID[] = []
  const tags: cborg.TagDecoder[] = []
@@ -137,8 +140,8 @@ class DagJsonTokenizer extends cborgJson.Tokenizer {
  * all we are interested in is extracting the the CIDs from the block
  * so we can just use cborg/json for that.
  */
-export const jsonWalker: DAGWalker = {
- codec: 0x0129,
+export const dagJsonWalker: DAGWalker = {
+ codec: dagJson.code,
  async * walk (block) {
  const cids: CID[] = []
  const tags: cborg.TagDecoder[] = []
@@ -167,3 +170,12 @@ export const jsonWalker: DAGWalker = {
  yield * cids
  }
 }
+
+/**
+ * Dag walker for json CIDs. JSON has no facility for linking to
+ * external blocks so the walker is a no-op.
+ */
+export const jsonWalker: DAGWalker = {
+ codec: json.code,
+ async * walk () {}
+}

packages/helia/test/pins.spec.ts

@@ -1,8 +1,12 @@
 /* eslint-env mocha */
 
+import * as dagCbor from '@ipld/dag-cbor'
+import * as dagJson from '@ipld/dag-json'
+import * as dagPb from '@ipld/dag-pb'
 import { expect } from 'aegir/chai'
 import all from 'it-all'
 import { CID } from 'multiformats/cid'
+import * as json from 'multiformats/codecs/json'
 import * as raw from 'multiformats/codecs/raw'
 import { createAndPutBlock } from './fixtures/create-block.js'
 import { createHelia } from './fixtures/create-helia.js'
@@ -156,4 +160,68 @@ describe('pins', () => {
  expect(pins).to.have.nested.property('[0].depth', Infinity)
  expect(pins).to.have.nested.property('[0].metadata').that.eql({})
  })
+
+ it('pins a json block', async () => {
+ const cid1 = await createAndPutBlock(json.code, json.encode({ hello: 'world' }), helia.blockstore)
+
+ await helia.pins.add(cid1)
+
+ const pins = await all(helia.pins.ls())
+
+ expect(pins).to.have.lengthOf(1)
+ expect(pins).to.have.nested.property('[0].cid').that.eql(cid1)
+ expect(pins).to.have.nested.property('[0].depth', Infinity)
+ expect(pins).to.have.nested.property('[0].metadata').that.eql({})
+ })
+
+ it('pins a dag-json block', async () => {
+ const cid1 = await createAndPutBlock(dagJson.code, dagJson.encode({ hello: 'world' }), helia.blockstore)
+ const cid2 = await createAndPutBlock(dagJson.code, dagJson.encode({ hello: 'world', linked: cid1 }), helia.blockstore)
+
+ await helia.pins.add(cid2)
+
+ const pins = await all(helia.pins.ls())
+
+ expect(pins).to.have.lengthOf(1)
+ expect(pins).to.have.nested.property('[0].cid').that.eql(cid2)
+ expect(pins).to.have.nested.property('[0].depth', Infinity)
+ expect(pins).to.have.nested.property('[0].metadata').that.eql({})
+
+ await expect(helia.pins.isPinned(cid1)).to.eventually.be.true()
+ await expect(helia.pins.isPinned(cid2)).to.eventually.be.true()
+ })
+
+ it('pins a dag-cbor block', async () => {
+ const cid1 = await createAndPutBlock(dagCbor.code, dagCbor.encode({ hello: 'world' }), helia.blockstore)
+ const cid2 = await createAndPutBlock(dagCbor.code, dagCbor.encode({ hello: 'world', linked: cid1 }), helia.blockstore)
+
+ await helia.pins.add(cid2)
+
+ const pins = await all(helia.pins.ls())
+
+ expect(pins).to.have.lengthOf(1)
+ expect(pins).to.have.nested.property('[0].cid').that.eql(cid2)
+ expect(pins).to.have.nested.property('[0].depth', Infinity)
+ expect(pins).to.have.nested.property('[0].metadata').that.eql({})
+
+ await expect(helia.pins.isPinned(cid1)).to.eventually.be.true()
+ await expect(helia.pins.isPinned(cid2)).to.eventually.be.true()
+ })
+
+ it('pins a dag-pb block', async () => {
+ const cid1 = await createAndPutBlock(dagPb.code, dagPb.encode({ Data: Uint8Array.from([0, 1, 2, 3, 4]), Links: [] }), helia.blockstore)
+ const cid2 = await createAndPutBlock(dagPb.code, dagPb.encode({ Links: [{ Name: '', Hash: cid1, Tsize: 100 }] }), helia.blockstore)
+
+ await helia.pins.add(cid2)
+
+ const pins = await all(helia.pins.ls())
+
+ expect(pins).to.have.lengthOf(1)
+ expect(pins).to.have.nested.property('[0].cid').that.eql(cid2)
+ expect(pins).to.have.nested.property('[0].depth', Infinity)
+ expect(pins).to.have.nested.property('[0].metadata').that.eql({})
+
+ await expect(helia.pins.isPinned(cid1)).to.eventually.be.true()
+ await expect(helia.pins.isPinned(cid2)).to.eventually.be.true()
+ })
 })