Implement ipfs refs

package.json

@@ -72,7 +72,7 @@
  "execa": "^1.0.0",
  "form-data": "^2.3.3",
  "hat": "0.0.3",
- "interface-ipfs-core": "~0.99.1",
+ "interface-ipfs-core": "~0.101.1",
  "ipfsd-ctl": "~0.42.0",
  "libp2p-websocket-star": "~0.10.2",
  "ncp": "^2.0.0",
@@ -86,6 +86,7 @@
  "@hapi/hapi": "^18.3.1",
  "@hapi/joi": "^15.0.1",
  "async": "^2.6.1",
+ "base32.js": "~0.1.0",
  "bignumber.js": "^8.0.2",
  "binary-querystring": "~0.1.2",
  "bl": "^3.0.0",
@@ -112,7 +113,7 @@
  "ipfs-bitswap": "~0.23.0",
  "ipfs-block": "~0.8.0",
  "ipfs-block-service": "~0.15.1",
- "ipfs-http-client": "^31.0.0",
+ "ipfs-http-client": "^31.0.2",
  "ipfs-http-response": "~0.2.1",
  "ipfs-mfs": "~0.10.2",
  "ipfs-multipart": "~0.1.0",
@@ -175,6 +176,7 @@
  "pull-sort": "^1.0.1",
  "pull-stream": "^3.6.9",
  "pull-stream-to-stream": "^1.3.4",
+ "pull-traverse": "^1.0.3",
  "readable-stream": "^3.1.1",
  "receptacle": "^1.3.2",
  "stream-to-pull-stream": "^1.7.3",

src/cli/bin.js

@@ -10,6 +10,7 @@ const mfs = require('ipfs-mfs/cli')
 const debug = require('debug')('ipfs:cli')
 const pkg = require('../../package.json')
 const parser = require('./parser')
+const commandAlias = require('./command-alias')
 
 async function main (args) {
  const oneWeek = 1000 * 60 * 60 * 24 * 7
@@ -22,6 +23,9 @@ async function main (args) {
 
  let getIpfs = null
 
+ // Apply command aliasing (eg `refs local` -> `refs-local`)
+ args = commandAlias(args)
+
  cli
  .parse(args)
  .then(({ data, argv }) => {

src/cli/command-alias.js

@@ -0,0 +1,35 @@
+'use strict'
+
+const aliases = {
+ // We need to be able to show help text for both the `refs` command and the
+ // `refs local` command, but with yargs `refs` cannot be both a command and
+ // a command directory. So alias `refs local` to `refs-local`
+ 'refs-local': ['refs', 'local']
+}
+
+// Replace multi-word command with alias
+// eg replace `refs local` with `refs-local`
+module.exports = function (args) {
+ for (const [alias, original] of Object.entries(aliases)) {
+ if (arrayMatch(args, original)) {
+ return [alias, ...args.slice(original.length)]
+ }
+ }
+
+ return args
+}
+
+// eg arrayMatch([1, 2, 3], [1, 2]) => true
+function arrayMatch (arr, sub) {
+ if (sub.length > arr.length) {
+ return false
+ }
+
+ for (let i = 0; i < sub.length; i++) {
+ if (arr[i] !== sub[i]) {
+ return false
+ }
+ }
+
+ return true
+}

src/cli/commands/refs-local.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const { print } = require('../utils')
+
+module.exports = {
+ command: 'refs-local',
+
+ describe: 'List all local references.',
+
+ handler ({ getIpfs, resolve }) {
+ resolve((async () => {
+ const ipfs = await getIpfs()
+
+ return new Promise((resolve, reject) => {
+ const stream = ipfs.refs.localReadableStream()
+
+ stream.on('error', reject)
+ stream.on('end', resolve)
+
+ stream.on('data', (ref) => {
+ if (ref.err) {
+ print(ref.err, true, true)
+ } else {
+ print(ref.ref)
+ }
+ })
+ })
+ })())
+ }
+}

src/cli/commands/refs.js

@@ -0,0 +1,65 @@
+'use strict'
+
+const { print } = require('../utils')
+
+module.exports = {
+ command: 'refs <key> [keys..]',
+
+ describe: 'List links (references) from an object',
+
+ builder: {
+ recursive: {
+ alias: 'r',
+ desc: 'Recursively list links of child nodes.',
+ type: 'boolean',
+ default: false
+ },
+ format: {
+ desc: 'Output edges with given format. Available tokens: <src> <dst> <linkname>.',
+ type: 'string',
+ default: '<dst>'
+ },
+ edges: {
+ alias: 'e',
+ desc: 'Output edge format: `<from> -> <to>`',
+ type: 'boolean',
+ default: false
+ },
+ unique: {
+ alias: 'u',
+ desc: 'Omit duplicate refs from output.',
+ type: 'boolean',
+ default: false
+ },
+ 'max-depth': {
+ desc: 'Only for recursive refs, limits fetch and listing to the given depth.',
+ type: 'number'
+ }
+ },
+
+ handler ({ getIpfs, key, keys, recursive, format, edges, unique, maxDepth, resolve }) {
+ resolve((async () => {
+ if (maxDepth === 0) {
+ return
+ }
+
+ const ipfs = await getIpfs()
+ const k = [key].concat(keys)
+
+ return new Promise((resolve, reject) => {
+ const stream = ipfs.refsReadableStream(k, { recursive, format, edges, unique, maxDepth })
+
+ stream.on('error', reject)
+ stream.on('end', resolve)
+
+ stream.on('data', (ref) => {
+ if (ref.err) {
+ print(ref.err, true, true)
+ } else {
+ print(ref.ref)
+ }
+ })
+ })
+ })())
+ }
+}

src/cli/utils.js

@@ -81,7 +81,7 @@ exports.getRepoPath = () => {
 let visible = true
 exports.disablePrinting = () => { visible = false }
 
-exports.print = (msg, newline) => {
+exports.print = (msg, newline, isError = false) => {
  if (newline === undefined) {
  newline = true
  }
@@ -91,7 +91,8 @@ exports.print = (msg, newline) => {
  msg = ''
  }
  msg = newline ? msg + '\n' : msg
- process.stdout.write(msg)
+ const outStream = isError ? process.stderr : process.stdout
+ outStream.write(msg)
  }
 }
 

src/core/components/files-regular/index.js

@@ -1,19 +1,28 @@
 'use strict'
 
-module.exports = self => ({
- add: require('./add')(self),
- addFromFs: require('./add-from-fs')(self),
- addFromStream: require('./add-from-stream')(self),
- addFromURL: require('./add-from-url')(self),
- addPullStream: require('./add-pull-stream')(self),
- addReadableStream: require('./add-readable-stream')(self),
- cat: require('./cat')(self),
- catPullStream: require('./cat-pull-stream')(self),
- catReadableStream: require('./cat-readable-stream')(self),
- get: require('./get')(self),
- getPullStream: require('./get-pull-stream')(self),
- getReadableStream: require('./get-readable-stream')(self),
- ls: require('./ls')(self),
- lsPullStream: require('./ls-pull-stream')(self),
- lsReadableStream: require('./ls-readable-stream')(self)
-})
+module.exports = (self) => {
+ const filesRegular = {
+ add: require('./add')(self),
+ addFromFs: require('./add-from-fs')(self),
+ addFromStream: require('./add-from-stream')(self),
+ addFromURL: require('./add-from-url')(self),
+ addPullStream: require('./add-pull-stream')(self),
+ addReadableStream: require('./add-readable-stream')(self),
+ cat: require('./cat')(self),
+ catPullStream: require('./cat-pull-stream')(self),
+ catReadableStream: require('./cat-readable-stream')(self),
+ get: require('./get')(self),
+ getPullStream: require('./get-pull-stream')(self),
+ getReadableStream: require('./get-readable-stream')(self),
+ ls: require('./ls')(self),
+ lsPullStream: require('./ls-pull-stream')(self),
+ lsReadableStream: require('./ls-readable-stream')(self),
+ refs: require('./refs')(self),
+ refsReadableStream: require('./refs-readable-stream')(self),
+ refsPullStream: require('./refs-pull-stream')(self)
+ }
+ filesRegular.refs.local = require('./refs-local')(self)
+ filesRegular.refs.localReadableStream = require('./refs-local-readable-stream')(self)
+ filesRegular.refs.localPullStream = require('./refs-local-pull-stream')(self)
+ return filesRegular
+}

src/core/components/files-regular/refs-local-pull-stream.js

@@ -0,0 +1,34 @@
+'use strict'
+
+const CID = require('cids')
+const base32 = require('base32.js')
+const pull = require('pull-stream')
+const pullDefer = require('pull-defer')
+
+module.exports = function (self) {
+ return () => {
+ const deferred = pullDefer.source()
+
+ self._repo.blocks.query({ keysOnly: true }, (err, blocks) => {
+ if (err) {
+ return deferred.resolve(pull.error(err))
+ }
+
+ const refs = blocks.map(b => dsKeyToRef(b.key))
+ deferred.resolve(pull.values(refs))
+ })
+
+ return deferred
+ }
+}
+
+function dsKeyToRef (key) {
+ try {
+ // Block key is of the form /<base32 encoded string>
+ const decoder = new base32.Decoder()
+ const buff = Buffer.from(decoder.write(key.toString().slice(1)).finalize())
+ return { ref: new CID(buff).toString() }
+ } catch (err) {
+ return { err: `Could not convert block with key '${key}' to CID: ${err.message}` }
+ }
+}

src/core/components/files-regular/refs-local-readable-stream.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const toStream = require('pull-stream-to-stream')
+
+module.exports = function (self) {
+ return (ipfsPath, options) => {
+ return toStream.source(self.refs.localPullStream())
+ }
+}

src/core/components/files-regular/refs-local.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const promisify = require('promisify-es6')
+const pull = require('pull-stream')
+
+module.exports = function (self) {
+ return promisify((callback) => {
+ pull(
+ self.refs.localPullStream(),
+ pull.collect((err, values) => {
+ if (err) {
+ return callback(err)
+ }
+ callback(null, values)
+ })
+ )
+ })
+}