fix: build & export interop with go-ipfs for small file raw leaves

src/builder/reduce.js

@@ -13,51 +13,13 @@ module.exports = function reduce (file, ipld, options) {
  if (leaves.length === 1 && leaves[0].single && options.reduceSingleLeafToSelf) {
  const leaf = leaves[0]
 
- if (options.leafType === 'file' && !options.rawLeaves) {
- return callback(null, {
- path: file.path,
- multihash: leaf.multihash,
- size: leaf.size,
- leafSize: leaf.leafSize,
- name: leaf.name
- })
- }
-
- // we're using raw leaf nodes so we convert the node into a UnixFS `file` node.
- return waterfall([
- (cb) => ipld.get(leaf.cid, cb),
- (result, cb) => {
- // If result.value is a buffer, this is a raw leaf otherwise it's a dag-pb node
- const data = Buffer.isBuffer(result.value) ? result.value : result.value.data
- const fileNode = new UnixFS('file', data)
-
- DAGNode.create(fileNode.marshal(), [], options.hashAlg, (error, node) => {
- cb(error, { DAGNode: node, fileNode: fileNode })
- })
- },
- (result, cb) => {
- if (options.onlyHash) {
- return cb(null, result)
- }
-
- let cid = new CID(result.DAGNode.multihash)
-
- if (options.cidVersion === 1) {
- cid = cid.toV1()
- }
-
- ipld.put(result.DAGNode, { cid }, (error) => cb(error, result))
- },
- (result, cb) => {
- cb(null, {
- path: file.path,
- multihash: result.DAGNode.multihash,
- size: result.DAGNode.size,
- leafSize: result.fileNode.fileSize(),
- name: leaf.name
- })
- }
- ], callback)
+ return callback(null, {
+ path: file.path,
+ multihash: leaf.multihash,
+ size: leaf.size,
+ leafSize: leaf.leafSize,
+ name: leaf.name
+ })
  }
 
  // create a parent node and add all the leaves

src/exporter/dir-flat.js

@@ -6,14 +6,14 @@ const cat = require('pull-cat')
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (node, name, path, pathRest, resolve, size, dag, parent, depth) {
+function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth) {
  const accepts = pathRest[0]
 
  const dir = {
  name: name,
  depth: depth,
  path: path,
- hash: node.multihash,
+ hash: cid,
  size: node.size,
  type: 'dir'
  }

src/exporter/dir-hamt-sharded.js

@@ -2,19 +2,18 @@
 
 const pull = require('pull-stream')
 const cat = require('pull-cat')
-const cleanHash = require('./clean-multihash')
 
 // Logic to export a unixfs directory.
 module.exports = shardedDirExporter
 
-function shardedDirExporter (node, name, path, pathRest, resolve, size, dag, parent, depth) {
+function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth) {
  let dir
  if (!parent || (parent.path !== path)) {
  dir = {
  name: name,
  depth: depth,
  path: path,
- hash: cleanHash(node.multihash),
+ hash: cid,
  size: node.size,
  type: 'dir'
  }

src/exporter/extract-data-from-block.js

@@ -0,0 +1,23 @@
+'use strict'
+
+module.exports = function extractDataFromBlock (block, streamPosition, begin, end) {
+ const blockLength = block.length
+
+ if (begin >= streamPosition + blockLength) {
+ // If begin is after the start of the block, return an empty block
+ // This can happen when internal nodes contain data
+ return Buffer.alloc(0)
+ }
+
+ if (end - streamPosition < blockLength) {
+ // If the end byte is in the current block, truncate the block to the end byte
+ block = block.slice(0, end - streamPosition)
+ }
+
+ if (begin > streamPosition && begin < (streamPosition + blockLength)) {
+ // If the start byte is in the current block, skip to the start byte
+ block = block.slice(begin - streamPosition)
+ }
+
+ return block
+}

src/exporter/file.js

@@ -5,9 +5,10 @@ const UnixFS = require('ipfs-unixfs')
 const CID = require('cids')
 const pull = require('pull-stream')
 const paramap = require('pull-paramap')
+const extractDataFromBlock = require('./extract-data-from-block')
 
 // Logic to export a single (possibly chunked) unixfs file.
-module.exports = (node, name, path, pathRest, resolve, size, dag, parent, depth, offset, length) => {
+module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, offset, length) => {
  const accepts = pathRest[0]
 
  if (accepts !== undefined && accepts !== path) {
@@ -48,7 +49,7 @@ module.exports = (node, name, path, pathRest, resolve, size, dag, parent, depth,
  content: content,
  name: name,
  path: path,
- hash: node.multihash,
+ hash: cid,
  size: fileSize,
  type: 'file'
  }])
@@ -149,25 +150,3 @@ function streamBytes (dag, node, fileSize, offset, length) {
  pull.filter(Boolean)
  )
 }
-
-function extractDataFromBlock (block, streamPosition, begin, end) {
- const blockLength = block.length
-
- if (begin >= streamPosition + blockLength) {
- // If begin is after the start of the block, return an empty block
- // This can happen when internal nodes contain data
- return Buffer.alloc(0)
- }
-
- if (end - streamPosition < blockLength) {
- // If the end byte is in the current block, truncate the block to the end byte
- block = block.slice(0, end - streamPosition)
- }
-
- if (begin > streamPosition && begin < (streamPosition + blockLength)) {
- // If the start byte is in the current block, skip to the start byte
- block = block.slice(begin - streamPosition)
- }
-
- return block
-}

src/exporter/object.js

@@ -3,7 +3,7 @@
 const CID = require('cids')
 const pull = require('pull-stream')
 
-module.exports = (node, name, path, pathRest, resolve, size, dag, parent, depth) => {
+module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth) => {
  let newNode
  if (pathRest.length) {
  const pathElem = pathRest[0]

src/exporter/raw.js

@@ -0,0 +1,57 @@
+'use strict'
+
+const pull = require('pull-stream')
+const extractDataFromBlock = require('./extract-data-from-block')
+
+// Logic to export a single raw block
+module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, offset, length) => {
+ const accepts = pathRest[0]
+
+ if (accepts !== undefined && accepts !== path) {
+ return pull.empty()
+ }
+
+ size = size || node.length
+
+ if (offset < 0) {
+ return pull.error(new Error('Offset must be greater than 0'))
+ }
+
+ if (offset > size) {
+ return pull.error(new Error('Offset must be less than the file size'))
+ }
+
+ if (length < 0) {
+ return pull.error(new Error('Length must be greater than or equal to 0'))
+ }
+
+ if (length === 0) {
+ return pull.once({
+ depth,
+ content: pull.once(Buffer.alloc(0)),
+ hash: cid,
+ name,
+ path,
+ size,
+ type: 'raw'
+ })
+ }
+
+ if (!offset) {
+ offset = 0
+ }
+
+ if (!length || (offset + length > size)) {
+ length = size - offset
+ }
+
+ return pull.once({
+ depth,
+ content: pull.once(extractDataFromBlock(node, 0, offset, offset + length)),
+ hash: cid,
+ name,
+ path,
+ size,
+ type: 'raw'
+ })
+}

src/exporter/resolve.js

@@ -9,7 +9,8 @@ const resolvers = {
  directory: require('./dir-flat'),
  'hamt-sharded-directory': require('./dir-hamt-sharded'),
  file: require('./file'),
- object: require('./object')
+ object: require('./object'),
+ raw: require('./raw')
 }
 
 module.exports = Object.assign({
@@ -31,39 +32,45 @@ function createResolver (dag, options, depth, parent) {
  if ((typeof item.depth) !== 'number') {
  return pull.error(new Error('no depth'))
  }
+
  if (item.object) {
- return cb(null, resolveItem(item.object, item, options.offset, options.length))
+ return cb(null, resolveItem(null, item.object, item, options.offset, options.length))
  }
- dag.get(new CID(item.multihash), (err, node) => {
+
+ const cid = new CID(item.multihash)
+
+ dag.get(cid, (err, node) => {
  if (err) {
  return cb(err)
  }
  // const name = item.fromPathRest ? item.name : item.path
- cb(null, resolveItem(node.value, item, options.offset, options.length))
+ cb(null, resolveItem(cid, node.value, item, options.offset, options.length))
  })
  }),
  pull.flatten(),
  pull.filter(Boolean),
  pull.filter((node) => node.depth <= options.maxDepth)
  )
 
- function resolveItem (node, item, offset, length) {
- return resolve(node, item.name, item.path, item.pathRest, item.size, dag, item.parent || parent, item.depth, offset, length)
+ function resolveItem (cid, node, item, offset, length) {
+ return resolve(cid, node, item.name, item.path, item.pathRest, item.size, dag, item.parent || parent, item.depth, offset, length)
  }
 
- function resolve (node, name, path, pathRest, size, dag, parentNode, depth, offset, length) {
+ function resolve (cid, node, name, path, pathRest, size, dag, parentNode, depth, offset, length) {
  const type = typeOf(node)
  const nodeResolver = resolvers[type]
  if (!nodeResolver) {
  return pull.error(new Error('Unkown node type ' + type))
  }
  const resolveDeep = createResolver(dag, options, depth, node)
- return nodeResolver(node, name, path, pathRest, resolveDeep, size, dag, parentNode, depth, offset, length)
+ return nodeResolver(cid, node, name, path, pathRest, resolveDeep, size, dag, parentNode, depth, offset, length)
  }
 }
 
 function typeOf (node) {
- if (Buffer.isBuffer(node.data)) {
+ if (Buffer.isBuffer(node)) {
+ return 'raw'
+ } else if (Buffer.isBuffer(node.data)) {
  return UnixFS.unmarshal(node.data).type
  } else {
  return 'object'

test/builder-dir-sharding.js

@@ -115,7 +115,7 @@ module.exports = (repo) => {
  expect(nodes.length).to.be.eql(2)
  const expectedHash = new CID(shardedHash).toBaseEncodedString()
  expect(nodes[0].path).to.be.eql(expectedHash)
- expect(nodes[0].hash).to.be.eql(expectedHash)
+ expect(new CID(nodes[0].hash).toBaseEncodedString()).to.be.eql(expectedHash)
  expect(nodes[1].path).to.be.eql(expectedHash + '/b')
  expect(nodes[1].size).to.be.eql(21)
  pull(

test/exporter.js

@@ -30,6 +30,7 @@ const exporter = unixFSEngine.exporter
 const importer = unixFSEngine.importer
 
 const bigFile = loadFixture('test/fixtures/1.2MiB.txt')
+const smallFile = loadFixture('test/fixtures/200Bytes.txt')
 
 module.exports = (repo) => {
  describe('exporter', () => {
@@ -420,12 +421,42 @@ module.exports = (repo) => {
  )
  })
 
- it('exports a large file > 5mb imported with raw leaves', function (done) {
+ it('exports a small file imported with raw leaves', function (done) {
  this.timeout(30 * 1000)
 
  pull(
  pull.values([{
  path: '200Bytes.txt',
+ content: pull.values([smallFile])
+ }]),
+ importer(ipld, {
+ rawLeaves: true
+ }),
+ pull.collect(collected)
+ )
+
+ function collected (err, files) {
+ expect(err).to.not.exist()
+ expect(files.length).to.equal(1)
+
+ pull(
+ exporter(files[0].multihash, ipld),
+ pull.collect((err, files) => {
+ expect(err).to.not.exist()
+ expect(new CID(files[0].hash).toBaseEncodedString()).to.equal('zb2rhXrz1gkCv8p4nUDZRohY6MzBE9C3HVTVDP72g6Du3SD9Q')
+
+ fileEql(files[0], smallFile, done)
+ })
+ )
+ }
+ })
+
+ it('exports a large file > 1mb imported with raw leaves', function (done) {
+ this.timeout(30 * 1000)
+
+ pull(
+ pull.values([{
+ path: '1.2MiB.txt',
  content: pull.values([bigFile])
  }]),
  importer(ipld, {