From 689df2c67d5c3d972408fa8f67eb8fd871158d28 Mon Sep 17 00:00:00 2001 From: Volker Mische Date: Fri, 18 Jan 2019 15:33:11 +0100 Subject: [PATCH] feat: use new IPLD API This is part of the Awesome Endeavour: Async Iterators: https://github.com/ipfs/js-ipfs/issues/1670 --- package.json | 4 +- src/core/components/dag.js | 66 +++++++++++++++++++++----- src/core/components/init.js | 11 +++-- src/core/components/object.js | 87 ++++++++++++++++------------------ src/core/components/pin-set.js | 9 ++-- src/core/components/pin.js | 9 ++-- src/core/components/resolve.js | 62 +++++------------------- src/core/index.js | 30 +++++++----- 8 files changed, 145 insertions(+), 133 deletions(-) diff --git a/package.json b/package.json index 02134a2e0c..338c76443b 100644 --- a/package.json +++ b/package.json @@ -82,6 +82,7 @@ "dependencies": { "@nodeutils/defaults-deep": "^1.1.0", "async": "^2.6.1", + "async-iterator-to-pull-stream": "^1.1.0", "bignumber.js": "^8.0.2", "binary-querystring": "~0.1.2", "bl": "^2.1.2", @@ -115,7 +116,7 @@ "ipfs-repo": "~0.26.1", "ipfs-unixfs": "~0.1.16", "ipfs-unixfs-engine": "~0.35.3", - "ipld": "~0.20.1", + "ipld": "git+https://github.com/ipld/js-ipld.git#new-api-impl", "ipld-bitcoin": "~0.1.8", "ipld-dag-pb": "~0.15.0", "ipld-ethereum": "^2.0.1", @@ -148,6 +149,7 @@ "multiaddr": "^6.0.0", "multiaddr-to-uri": "^4.0.0", "multibase": "~0.6.0", + "multicodec": "~0.5.0", "multihashes": "~0.4.14", "multihashing-async": "~0.5.1", "node-fetch": "^2.3.0", diff --git a/src/core/components/dag.js b/src/core/components/dag.js index dc9ef2bdf5..7bccee3149 100644 --- a/src/core/components/dag.js +++ b/src/core/components/dag.js @@ -3,10 +3,12 @@ const promisify = require('promisify-es6') const CID = require('cids') const pull = require('pull-stream') +const iterToPull = require('async-iterator-to-pull-stream') const mapAsync = require('async/map') const setImmediate = require('async/setImmediate') const flattenDeep = require('lodash/flattenDeep') const errCode = require('err-code') +const multicodec = require('multicodec') module.exports = function dag (self) { return { @@ -25,21 +27,39 @@ module.exports = function dag (self) { } const optionDefaults = { - format: 'dag-cbor', - hashAlg: 'sha2-256' + format: multicodec.DAG_CBOR, + hashAlg: multicodec.SHA2_256 } - options = options.cid ? options : Object.assign({}, optionDefaults, options) + // The IPLD expects the format and hashAlg as constants + if (options.format && typeof options.format === 'string') { + const constantName = options.format.toUpperCase().replace(/-/g, '_') + options.format = multicodec[constantName] + } + if (options.hashAlg && typeof options.hashAlg === 'string') { + const constantName = options.hashAlg.toUpperCase().replace(/-/g, '_') + options.hashAlg = multicodec[constantName] + } - self._ipld.put(dagNode, options, (err, cid) => { - if (err) return callback(err) + options = options.cid ? options : Object.assign({}, optionDefaults, options) - if (options.preload !== false) { - self._preload(cid) - } + // js-ipld defaults to verion 1 CIDs. Hence set version 0 explicitly for + // dag-pb nodes + if (options.format === multicodec.DAG_PB && + options.hashAlg === multicodec.SHA2_256 && + options.version === undefined) { + options.version = 0 + } - callback(null, cid) - }) + self._ipld.put([dagNode], options).first().then( + (cid) => { + if (options.preload !== false) { + self._preload(cid) + } + return callback(null, cid) + }, + (error) => callback(error) + ) }), get: promisify((cid, path, options, callback) => { @@ -54,7 +74,7 @@ module.exports = function dag (self) { // Allow options in path position if (typeof path !== 'string') { options = path - path = null + path = undefined } else { options = {} } @@ -90,7 +110,27 @@ module.exports = function dag (self) { self._preload(cid) } - self._ipld.get(cid, path, options, callback) + if (path === undefined || path === '/') { + const result = self._ipld.get([cid]) + result.first().then( + (value) => { + callback(null, { + value, + remainderPath: '' + }) + }, + (error) => callback(error) + ) + } else { + const result = self._ipld.resolve(cid, path) + const promisedValue = options.localResolve ? + result.first() : + result.last() + promisedValue.then( + (value) => callback(null, value), + (error) => callback(error) + ) + } }), tree: promisify((cid, path, options, callback) => { @@ -135,7 +175,7 @@ module.exports = function dag (self) { } pull( - self._ipld.treeStream(cid, path, options), + iterToPull(self._ipld.tree(cid, path, options)), pull.collect(callback) ) }), diff --git a/src/core/components/init.js b/src/core/components/init.js index 29cac89402..b7923349f1 100644 --- a/src/core/components/init.js +++ b/src/core/components/init.js @@ -11,6 +11,7 @@ const { DAGNode } = require('ipld-dag-pb') const UnixFs = require('ipfs-unixfs') +const multicodec = require('multicodec') const IPNS = require('../ipns') const OfflineDatastore = require('../ipns/routing/offline-datastore') @@ -130,11 +131,13 @@ module.exports = function init (self) { (cb) => { waterfall([ (cb) => DAGNode.create(new UnixFs('directory').marshal(), cb), - (node, cb) => self.dag.put(node, { + async (node) => { + return self.dag.put(node, { version: 0, - format: 'dag-pb', - hashAlg: 'sha2-256' - }, cb), + format: multicodec.DAG_PB, + hashAlg: multicodec.SHA2_256 + }) + }, (cid, cb) => self._ipns.initializeKeyspace(privateKey, cid.toBaseEncodedString(), cb) ], cb) } diff --git a/src/core/components/object.js b/src/core/components/object.js index 85a3137522..a6669cbbca 100644 --- a/src/core/components/object.js +++ b/src/core/components/object.js @@ -9,6 +9,7 @@ const DAGNode = dagPB.DAGNode const DAGLink = dagPB.DAGLink const CID = require('cids') const mh = require('multihashes') +const multicodec = require('multicodec') const Unixfs = require('ipfs-unixfs') const errCode = require('err-code') @@ -87,19 +88,20 @@ module.exports = function object (self) { return cb(err) } - self._ipld.put(node, { + self._ipld.put([node], { version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb' - }, (err, cid) => { - if (err) return cb(err) - - if (options.preload !== false) { - self._preload(cid) - } - - cb(null, cid) - }) + hashAlg: multicodec.SHA2_256, + format: multicodec.DAG_PB + }).first().then( + (cid) => { + if (options.preload !== false) { + self._preload(cid) + } + + cb(null, cid) + }, + (error) => cb(error) + ) }) } ], callback) @@ -137,21 +139,20 @@ module.exports = function object (self) { return callback(err) } - self._ipld.put(node, { + self._ipld.put([node], { version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb' - }, (err, cid) => { - if (err) { - return callback(err) - } - - if (options.preload !== false) { - self._preload(cid) - } + hashAlg: multicodec.SHA2_256, + format: multicodec.DAG_PB + }).first().then( + (cid) => { + if (options.preload !== false) { + self._preload(cid) + } - callback(null, cid) - }) + callback(null, cid) + }, + (error) => callback(error) + ) }) }), put: promisify((obj, options, callback) => { @@ -200,21 +201,20 @@ module.exports = function object (self) { } function next () { - self._ipld.put(node, { + self._ipld.put([node], { version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb' - }, (err, cid) => { - if (err) { - return callback(err) - } - - if (options.preload !== false) { - self._preload(cid) - } + hashAlg: multicodec.SHA2_256, + format: multicodec.DAG_PB + }).first().then( + (cid) => { + if (options.preload !== false) { + self._preload(cid) + } - callback(null, cid) - }) + callback(null, cid) + }, + (error) => callback(error) + ) } }), @@ -248,13 +248,10 @@ module.exports = function object (self) { self._preload(cid) } - self._ipld.get(cid, (err, result) => { - if (err) { - return callback(err) - } - - callback(null, result.value) - }) + self._ipld.get([cid]).first().then( + (node) => callback(null, node), + (error) => callback(error) + ) }), data: promisify((multihash, options, callback) => { diff --git a/src/core/components/pin-set.js b/src/core/components/pin-set.js index 91f72bf90b..835aa48d36 100644 --- a/src/core/components/pin-set.js +++ b/src/core/components/pin-set.js @@ -6,6 +6,7 @@ const protobuf = require('protons') const fnv1a = require('fnv1a') const varint = require('varint') const { DAGNode, DAGLink } = require('ipld-dag-pb') +const multicodec = require('multicodec') const some = require('async/some') const eachOf = require('async/eachOf') @@ -110,8 +111,8 @@ exports = module.exports = function (dag) { dag.put(rootNode, { version: 0, - format: 'dag-pb', - hashAlg: 'sha2-256', + format: multicodec.DAG_PB, + hashAlg: multicodec.SHA2_256, preload: false }, (err, cid) => { if (err) { return callback(err, cid) } @@ -194,8 +195,8 @@ exports = module.exports = function (dag) { const opts = { version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb', + format: multicodec.DAG_PB, + hashAlg: multicodec.SHA2_256, preload: false } diff --git a/src/core/components/pin.js b/src/core/components/pin.js index e74427a96a..b6b1b53439 100644 --- a/src/core/components/pin.js +++ b/src/core/components/pin.js @@ -15,6 +15,7 @@ const setImmediate = require('async/setImmediate') const { Key } = require('interface-datastore') const errCode = require('err-code') const multibase = require('multibase') +const multicodec = require('multicodec') const createPinSet = require('./pin-set') const { resolvePath } = require('../utils') @@ -104,8 +105,8 @@ module.exports = (self) => { if (err) { return cb(err) } dag.put(empty, { version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb', + format: multicodec.DAG_PB, + hashAlg: multicodec.SHA2_256, preload: false }, cb) }), @@ -116,8 +117,8 @@ module.exports = (self) => { root = node dag.put(root, { version: 0, - hashAlg: 'sha2-256', - format: 'dag-pb', + format: multicodec.DAG_PB, + hashAlg: multicodec.SHA2_256, preload: false }, (err, cid) => { if (!err) { diff --git a/src/core/components/resolve.js b/src/core/components/resolve.js index bd23a33b20..5501d684fa 100644 --- a/src/core/components/resolve.js +++ b/src/core/components/resolve.js @@ -8,7 +8,7 @@ const CID = require('cids') const { cidToString } = require('../../utils/cid') module.exports = (self) => { - return promisify((name, opts, cb) => { + return promisify(async (name, opts, cb) => { if (typeof opts === 'function') { cb = opts opts = {} @@ -34,54 +34,18 @@ module.exports = (self) => { const path = split.slice(3).join('/') - resolve(cid, path, (err, cid) => { - if (err) return cb(err) - if (!cid) return cb(new Error('found non-link at given path')) - cb(null, `/ipfs/${cidToString(cid, { base: opts.cidBase })}`) - }) - }) - - // Resolve the given CID + path to a CID. - function resolve (cid, path, callback) { + const results = self._ipld.resolve(cid, path) let value - - doUntil( - (cb) => { - self.block.get(cid, (err, block) => { - if (err) return cb(err) - - const r = self._ipld.resolvers[cid.codec] - - if (!r) { - return cb(new Error(`No resolver found for codec "${cid.codec}"`)) - } - - r.resolver.resolve(block.data, path, (err, result) => { - if (err) return cb(err) - value = result.value - path = result.remainderPath - cb() - }) - }) - }, - () => { - const endReached = !path || path === '/' - - if (endReached) { - return true - } - - if (value) { - cid = new CID(value['/']) - } - - return false - }, - (err) => { - if (err) return callback(err) - if (value && value['/']) return callback(null, new CID(value['/'])) - callback() + for await (const result of results) { + if (result.remainderPath === '') { + value = result.value + break } - ) - } + } + if (!CID.isCID(value)) { + return cb(new Error('found non-link at given path')) + } else { + return cb(null, `/ipfs/${cidToString(value, { base: opts.cidBase })}`) + } + }) } diff --git a/src/core/index.js b/src/core/index.js index 71e0b1341f..f183d8e94c 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -10,6 +10,7 @@ const multiaddr = require('multiaddr') const multihash = require('multihashes') const PeerBook = require('peer-book') const multibase = require('multibase') +const multicodec = require('multicodec') const CID = require('cids') const debug = require('debug') const defaultsDeep = require('@nodeutils/defaults-deep') @@ -26,34 +27,34 @@ const mfsPreload = require('./mfs-preload') // All known (non-default) IPLD formats const IpldFormats = { - get 'bitcoin-block' () { + get [multicodec.BITCOIN_BOCK] () { return require('ipld-bitcoin') }, - get 'eth-account-snapshot' () { + get [multicodec.ETH_ACCOUNT_SNAPSHOT] () { return require('ipld-ethereum').ethAccountSnapshot }, - get 'eth-block' () { + get [multicodec.ETH_BLOCK] () { return require('ipld-ethereum').ethBlock }, - get 'eth-block-list' () { + get [multicodec.ETH_BLOCK_LIST] () { return require('ipld-ethereum').ethBlockList }, - get 'eth-state-trie' () { + get [multicodec.ETH_STATE_TRIE] () { return require('ipld-ethereum').ethStateTrie }, - get 'eth-storage-trie' () { + get [multicodec.ETH_STORAGE_TRIE] () { return require('ipld-ethereum').ethStorageTrie }, - get 'eth-tx' () { + get [multicodec.ETH_TX] () { return require('ipld-ethereum').ethTx }, - get 'eth-tx-trie' () { + get [multicodec.ETH_TX_TRIE] () { return require('ipld-ethereum').ethTxTrie }, - get 'git-raw' () { + get [multicodec.GIT_RAW] () { return require('ipld-git') }, - get 'zcash-block' () { + get [multicodec.ZCASH_BLOCK] () { return require('ipld-zcash') } } @@ -117,10 +118,13 @@ class IPFS extends EventEmitter { this._blockService = new BlockService(this._repo) this._ipld = new Ipld({ blockService: this._blockService, - loadFormat: (codec, callback) => { + loadFormat: async (codec) => { this.log('Loading IPLD format', codec) - if (IpldFormats[codec]) return callback(null, IpldFormats[codec]) - callback(new Error(`Missing IPLD format "${codec}"`)) + if (IpldFormats[codec]) { + return IpldFormats[codec] + } else { + throw new Error(`Missing IPLD format "${codec}"`) + } } }) this._preload = preload(this)