diff --git a/.travis.yml b/.travis.yml index ad267def..3c3df143 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,7 +46,6 @@ jobs: - stage: check name: linting script: - - npm --version - npm run lint - stage: check diff --git a/package.json b/package.json index 94eed13e..798be93e 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "version": "1.0.0", "description": "JS implementation of the IPFS UnixFS", "scripts": { - "postinstall": "rm -rf package-lock.json packages/*/package-lock.json", "reset": "lerna run clean && rimraf packages/*/node_modules node_modules", "test": "lerna run test", "coverage": "lerna run coverage", @@ -17,7 +16,6 @@ "update-contributors": "aegir release --lint=false --test=false --bump=false --build=false --changelog=false --commit=false --tag=false --push=false --ghrelease=false --docs=false --publish=false" }, "devDependencies": { - "aegir": "^30.3.0", "lerna": "^3.22.1" }, "repository": { diff --git a/packages/ipfs-unixfs-exporter/.aegir.js b/packages/ipfs-unixfs-exporter/.aegir.js index 26267210..e67cd911 100644 --- a/packages/ipfs-unixfs-exporter/.aegir.js +++ b/packages/ipfs-unixfs-exporter/.aegir.js @@ -1,16 +1,35 @@ 'use strict' +/** @type {import('aegir').Options["build"]["config"]} */ +const buildConfig = { + plugins: [ + { + name: 'node built ins', + setup (build) { + build.onResolve({ filter: /^stream$/ }, () => { + return { path: require.resolve('readable-stream') } + }) + build.onResolve({ filter: /^crypto$/ }, () => { + return { path: require.resolve('crypto-browserify') } + }) + build.onResolve({ filter: /^cborg$/ }, () => { + return { path: require.resolve('cborg') } + }) + } + } + ] +} + +/** @type {import('aegir').PartialOptions} */ module.exports = { - karma: { - browserNoActivityTimeout: 1000 * 1000, + build: { + config: buildConfig }, - webpack: { - node: { - // needed by the cbor module - stream: true, - - // needed by the core-util-is module - Buffer: true + test: { + browser: { + config: { + buildConfig + } } } } diff --git a/packages/ipfs-unixfs-exporter/.npmignore b/packages/ipfs-unixfs-exporter/.npmignore deleted file mode 100644 index f9a85ba9..00000000 --- a/packages/ipfs-unixfs-exporter/.npmignore +++ /dev/null @@ -1,3 +0,0 @@ -docs -test -.aegir.js diff --git a/packages/ipfs-unixfs-exporter/README.md b/packages/ipfs-unixfs-exporter/README.md index 1b20481a..7199bcb5 100644 --- a/packages/ipfs-unixfs-exporter/README.md +++ b/packages/ipfs-unixfs-exporter/README.md @@ -26,8 +26,8 @@ - [Raw entries](#raw-entries) - [CBOR entries](#cbor-entries) - [`entry.content({ offset, length })`](#entrycontent-offset-length-) - - [`exporter.path(cid, ipld)`](#exporterpathcid-ipld) - - [`exporter.recursive(cid, ipld)`](#exporterrecursivecid-ipld) + - [`walkPath(cid, ipld)`](#walkpathcid-ipld) + - [`recursive(cid, ipld)`](#recursivecid-ipld) - [Contribute](#contribute) - [License](#license) @@ -43,8 +43,8 @@ ```js // import a file and export it again -const importer = require('ipfs-unixfs-importer') -const exporter = require('ipfs-unixfs-exporter') +const { importer } = require('ipfs-unixfs-importer') +const { exporter } = require('ipfs-unixfs-exporter') const files = [] @@ -80,7 +80,7 @@ console.info(bytes) // 0, 1, 2, 3 #### API ```js -const exporter = require('ipfs-unixfs-exporter') +const { exporter } = require('ipfs-unixfs-exporter') ``` ### `exporter(cid, ipld, options)` @@ -202,28 +202,32 @@ for await (const entry of dir.content({ // `entries` contains the first 5 files/directories in the directory ``` -### `exporter.path(cid, ipld)` +### `walkPath(cid, ipld)` -`exporter.path` will return an async iterator that yields entries for all segments in a path: +`walkPath` will return an async iterator that yields entries for all segments in a path: ```javascript +const { walkPath } = require('ipfs-unixfs-exporter') + const entries = [] -for await (const entry of exporter.path('Qmfoo/foo/bar/baz.txt', ipld)) { +for await (const entry of walkPath('Qmfoo/foo/bar/baz.txt', ipld)) { entries.push(entry) } // entries contains 4x `entry` objects ``` -### `exporter.recursive(cid, ipld)` +### `recursive(cid, ipld)` -`exporter.recursive` will return an async iterator that yields all entries beneath a given CID or IPFS path, as well as the containing directory. +`recursive` will return an async iterator that yields all entries beneath a given CID or IPFS path, as well as the containing directory. ```javascript +const { recursive } = require('ipfs-unixfs-exporter') + const entries = [] -for await (const child of exporter.recursive('Qmfoo/foo/bar', ipld)) { +for await (const child of recursive('Qmfoo/foo/bar', ipld)) { entries.push(entry) } diff --git a/packages/ipfs-unixfs-exporter/package.json b/packages/ipfs-unixfs-exporter/package.json index 560e6761..6b494bfa 100644 --- a/packages/ipfs-unixfs-exporter/package.json +++ b/packages/ipfs-unixfs-exporter/package.json @@ -10,10 +10,11 @@ "scripts": { "prepare": "aegir build --no-bundle", "test": "aegir test", + "build": "aegir build", "clean": "rimraf ./dist", - "lint": "aegir lint", + "lint": "aegir ts --check && aegir lint", "coverage": "nyc -s npm run test -t node && nyc report --reporter=html", - "depcheck": "aegir dep-check -i @types/mocha -i @types/sinon -i nyc -i abort-controller -i rimraf -i ipfs-core-types" + "depcheck": "aegir dep-check -i @types/mocha -i @types/sinon -i nyc -i abort-controller -i rimraf -i ipfs-core-types -i copy -i util -i crypto-browserify -i events -i readable-stream" }, "repository": { "type": "git", @@ -35,23 +36,29 @@ "@types/mocha": "^8.2.1", "@types/sinon": "^9.0.10", "abort-controller": "^3.0.0", - "aegir": "^30.3.0", + "aegir": "^32.0.0", + "copy": "^0.3.2", + "crypto-browserify": "^3.12.0", "detect-node": "^2.0.4", - "ipfs-core-types": "^0.3.0", + "events": "^3.3.0", + "ipfs-core-types": "^0.3.1", "ipfs-unixfs-importer": "^6.0.1", - "ipld": "^0.28.0", - "ipld-dag-pb": "^0.21.0", - "ipld-in-memory": "^7.0.0", + "ipld": "^0.29.0", + "ipld-block": "^0.11.1", + "ipld-dag-pb": "^0.22.1", + "ipld-in-memory": "^8.0.0", "it-all": "^1.0.5", "it-buffer-stream": "^2.0.0", "it-first": "^1.0.6", "merge-options": "^3.0.4", - "multicodec": "^2.1.0", + "multicodec": "^3.0.1", "native-abort-controller": "^1.0.3", "nyc": "^15.0.0", + "readable-stream": "^3.6.0", "rimraf": "^3.0.2", "sinon": "^9.2.4", - "uint8arrays": "^2.1.2" + "uint8arrays": "^2.1.2", + "util": "^0.12.3" }, "dependencies": { "cids": "^1.1.5", diff --git a/packages/ipfs-unixfs-exporter/src/index.js b/packages/ipfs-unixfs-exporter/src/index.js index 6573cf0f..6707446a 100644 --- a/packages/ipfs-unixfs-exporter/src/index.js +++ b/packages/ipfs-unixfs-exporter/src/index.js @@ -6,66 +6,16 @@ const resolve = require('./resolvers') const last = require('it-last') /** - * @typedef {import('ipfs-unixfs')} UnixFS + * @typedef {import('ipfs-unixfs').UnixFS} UnixFS * @typedef {import('ipld-dag-pb').DAGNode} DAGNode - * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD - * - * @typedef {object} UnixFSFile - * @property {'file'} type - * @property {string} name - * @property {string} path - * @property {CID} cid - * @property {number} depth - * @property {UnixFS} unixfs - * @property {DAGNode} node - * @property {(options?: ExporterOptions) => AsyncIterable} content - * - * @typedef {object} UnixFSDirectory - * @property {'directory'} type - * @property {string} name - * @property {string} path - * @property {CID} cid - * @property {number} depth - * @property {UnixFS} unixfs - * @property {DAGNode} node - * @property {(options?: ExporterOptions) => AsyncIterable} content - * - * @typedef {object} ObjectNode - * @property {'object'} type - * @property {string} name - * @property {string} path - * @property {CID} cid - * @property {number} depth - * @property {Uint8Array} node - * @property {(options?: ExporterOptions) => AsyncIterable} content - * - * @typedef {object} RawNode - * @property {'raw'} type - * @property {string} name - * @property {string} path - * @property {CID} cid - * @property {number} depth - * @property {Uint8Array} node - * @property {(options?: ExporterOptions) => AsyncIterable} content - * - * @typedef {object} IdentityNode - * @property {'identity'} type - * @property {string} name - * @property {string} path - * @property {CID} cid - * @property {number} depth - * @property {Uint8Array} node - * @property {(options?: ExporterOptions) => AsyncIterable} content - * - * @typedef {UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode} UnixFSEntry - */ - -/** - * @typedef {object} ExporterOptions - * @property {number} [offset=0] - * @property {number} [length] - * @property {AbortSignal} [signal] - * @property {number} [timeout] + * @typedef {import('ipld')} IPLD + * @typedef {import('./types').ExporterOptions} ExporterOptions + * @typedef {import('./types').UnixFSFile} UnixFSFile + * @typedef {import('./types').UnixFSDirectory} UnixFSDirectory + * @typedef {import('./types').ObjectNode} ObjectNode + * @typedef {import('./types').RawNode} RawNode + * @typedef {import('./types').IdentityNode} IdentityNode + * @typedef {import('./types').UnixFSEntry} UnixFSEntry */ const toPathComponents = (path = '') => { @@ -115,7 +65,7 @@ const cidAndRest = (path) => { * @param {IPLD} ipld * @param {ExporterOptions} [options] */ -const walkPath = async function * (path, ipld, options = {}) { +async function * walkPath (path, ipld, options = {}) { let { cid, toResolve @@ -152,7 +102,7 @@ const walkPath = async function * (path, ipld, options = {}) { * @param {IPLD} ipld * @param {ExporterOptions} [options] */ -const exporter = async (path, ipld, options = {}) => { +async function exporter (path, ipld, options = {}) { const result = await last(walkPath(path, ipld, options)) if (!result) { @@ -167,7 +117,7 @@ const exporter = async (path, ipld, options = {}) => { * @param {IPLD} ipld * @param {ExporterOptions} [options] */ -const recursive = async function * (path, ipld, options = {}) { +async function * recursive (path, ipld, options = {}) { const node = await exporter(path, ipld, options) if (!node) { @@ -202,7 +152,8 @@ const recursive = async function * (path, ipld, options = {}) { } } -exporter.path = walkPath -exporter.recursive = recursive - -module.exports = exporter +module.exports = { + exporter, + walkPath, + recursive +} diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js b/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js index 3bb50af7..79899daa 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js +++ b/packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js @@ -4,7 +4,11 @@ const CID = require('cids') const errCode = require('err-code') /** - * @type {import('./').Resolver} + * @typedef {import('../types').Resolver} Resolver + */ + +/** + * @type {Resolver} */ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => { const object = await ipld.get(cid, options) @@ -29,6 +33,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options cid, node: block, depth, + size: block.length, content: async function * () { yield object } @@ -57,6 +62,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options cid, node: block, depth, + size: block.length, content: async function * () { yield object } diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/identity.js b/packages/ipfs-unixfs-exporter/src/resolvers/identity.js index 83f57db5..06035886 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/identity.js +++ b/packages/ipfs-unixfs-exporter/src/resolvers/identity.js @@ -6,7 +6,8 @@ const validateOffsetAndLength = require('../utils/validate-offset-and-length') const mh = require('multihashing-async').multihash /** - * @typedef {import('../').ExporterOptions} ExporterOptions + * @typedef {import('../types').ExporterOptions} ExporterOptions + * @typedef {import('../types').Resolver} Resolver */ /** @@ -29,7 +30,7 @@ const rawContent = (node) => { } /** - * @type {import('./').Resolver} + * @type {Resolver} */ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => { if (toResolve.length) { @@ -46,6 +47,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options cid, content: rawContent(buf.digest), depth, + size: buf.length, node: buf.digest } } diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/index.js b/packages/ipfs-unixfs-exporter/src/resolvers/index.js index 73d30123..ac304a1c 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/index.js +++ b/packages/ipfs-unixfs-exporter/src/resolvers/index.js @@ -3,30 +3,15 @@ const errCode = require('err-code') /** - * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD - * @typedef {import('../').ExporterOptions} ExporterOptions - * @typedef {import('../').UnixFSEntry} UnixFSEntry * @typedef {import('cids')} CID + * @typedef {import('ipld')} IPLD + * @typedef {import('../types').ExporterOptions} ExporterOptions + * @typedef {import('../types').UnixFSEntry} UnixFSEntry + * @typedef {import('../types').Resolver} Resolver + * @typedef {import('../types').Resolve} Resolve */ /** - * @typedef {object} NextResult - * @property {CID} cid - * @property {string} name - * @property {string} path - * @property {string[]} toResolve - * - * @typedef {object} ResolveResult - * @property {UnixFSEntry} entry - * @property {NextResult} [next] - */ - -/** - * - * @typedef {(cid: CID, name: string, path: string, toResolve: string[], depth: number, ipld: IPLD, options: ExporterOptions) => Promise} Resolve - * - * @typedef {(cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, ipld: IPLD, options: ExporterOptions) => Promise} Resolver - * * @type {{ [ key: string ]: Resolver }} */ const resolvers = { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/raw.js b/packages/ipfs-unixfs-exporter/src/resolvers/raw.js index d85573e5..2e6812eb 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/raw.js +++ b/packages/ipfs-unixfs-exporter/src/resolvers/raw.js @@ -5,7 +5,7 @@ const extractDataFromBlock = require('../utils/extract-data-from-block') const validateOffsetAndLength = require('../utils/validate-offset-and-length') /** - * @typedef {import('../').ExporterOptions} ExporterOptions + * @typedef {import('../types').ExporterOptions} ExporterOptions */ /** @@ -28,7 +28,7 @@ const rawContent = (node) => { } /** - * @type {import('./').Resolver} + * @type {import('../types').Resolver} */ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => { if (toResolve.length) { @@ -45,6 +45,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options cid, content: rawContent(buf), depth, + size: buf.length, node: buf } } diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.js b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.js index 0a7feb10..f30fac00 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.js +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.js @@ -1,10 +1,13 @@ 'use strict' /** - * @typedef {import('../../../').ExporterOptions} ExporterOptions - * @typedef {import('../').UnixfsV1DirectoryContent} UnixfsV1DirectoryContent - * - * @type {import('../').UnixfsV1Resolver} + * @typedef {import('../../../types').ExporterOptions} ExporterOptions + * @typedef {import('../../../types').UnixfsV1DirectoryContent} UnixfsV1DirectoryContent + * @typedef {import('../../../types').UnixfsV1Resolver} UnixfsV1Resolver + */ + +/** + * @type {UnixfsV1Resolver} */ const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => { /** diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js index e06c025d..4bcdba4a 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/file.js @@ -2,14 +2,16 @@ const extractDataFromBlock = require('../../../utils/extract-data-from-block') const validateOffsetAndLength = require('../../../utils/validate-offset-and-length') -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const errCode = require('err-code') /** - * @typedef {import('../../../').ExporterOptions} ExporterOptions - * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD + * @typedef {import('../../../types').ExporterOptions} ExporterOptions + * @typedef {import('ipld')} IPLD * @typedef {import('ipld-dag-pb').DAGNode} DAGNode - * + */ + +/** * @param {IPLD} ipld * @param {DAGNode} node * @param {number} start @@ -62,8 +64,7 @@ async function * emitBytes (ipld, node, start, end, streamPosition = 0, options) (end > childStart && end <= childEnd) || // child has end byte (start < childStart && end > childEnd)) { // child is between offset and end bytes const child = await ipld.get(childLink.Hash, { - signal: options.signal, - timeout: options.timeout + signal: options.signal }) for await (const buf of emitBytes(ipld, child, start, end, streamPosition, options)) { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js index be6f550c..25d784b5 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.js @@ -1,13 +1,16 @@ 'use strict' /** - * @typedef {import('../../../').ExporterOptions} ExporterOptions * @typedef {import('ipld-dag-pb').DAGNode} DAGNode - * @typedef {import('../../').Resolve} Resolve - * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD - * @typedef {import('../').UnixfsV1DirectoryContent} UnixfsV1DirectoryContent - * - * @type {import('../').UnixfsV1Resolver} + * @typedef {import('ipld')} IPLD + * @typedef {import('../../../types').ExporterOptions} ExporterOptions + * @typedef {import('../../../types').Resolve} Resolve + * @typedef {import('../../../types').UnixfsV1DirectoryContent} UnixfsV1DirectoryContent + * @typedef {import('../../../types').UnixfsV1Resolver} UnixfsV1Resolver + */ + +/** + * @type {UnixfsV1Resolver} */ const hamtShardedDirectoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => { /** diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/raw.js b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/raw.js index b385877b..1ac4b17e 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/raw.js +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/raw.js @@ -4,9 +4,12 @@ const extractDataFromBlock = require('../../../utils/extract-data-from-block') const validateOffsetAndLength = require('../../../utils/validate-offset-and-length') /** - * @typedef {import('../../../').ExporterOptions} ExporterOptions - * - * @type {import('../').UnixfsV1Resolver} + * @typedef {import('../../../types').ExporterOptions} ExporterOptions + * @typedef {import('../../../types').UnixfsV1Resolver} UnixfsV1Resolver + */ + +/** + * @type {UnixfsV1Resolver} */ const rawContent = (cid, node, unixfs, path, resolve, depth, ipld) => { /** diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.js b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.js index 9f2d37da..1b74ebf9 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.js +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.js @@ -1,16 +1,18 @@ 'use strict' const errCode = require('err-code') -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const findShardCid = require('../../utils/find-cid-in-shard') /** - * @typedef {import('../../').ExporterOptions} ExporterOptions - * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD - * @typedef {import('../').UnixFSEntry} UnixFSEntry * @typedef {import('cids')} CID + * @typedef {import('ipld')} IPLD * @typedef {import('ipld-dag-pb').DAGNode} DAGNode - * @typedef {import('../').Resolve} Resolve + * @typedef {import('../../types').ExporterOptions} ExporterOptions + * @typedef {import('../../types').UnixFSEntry} UnixFSEntry + * @typedef {import('../../types').Resolve} Resolve + * @typedef {import('../../types').Resolver} Resolver + * @typedef {import('../../types').UnixfsV1Resolver} UnixfsV1Resolver */ /** @@ -24,12 +26,6 @@ const findLinkCid = (node, name) => { } /** - * @typedef {AsyncIterable | Iterable} UnixfsV1FileContent - * @typedef {AsyncIterable | Iterable} UnixfsV1DirectoryContent - * - * @typedef {UnixfsV1FileContent | UnixfsV1DirectoryContent} UnixfsV1Content - * @typedef {(cid: CID, node: DAGNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, ipld: IPLD) => (options: ExporterOptions) => UnixfsV1Content } UnixfsV1Resolver - * * @type {{ [key: string]: UnixfsV1Resolver }} */ const contentExporters = { @@ -46,7 +42,7 @@ const contentExporters = { } /** - * @type {import('../').Resolver} + * @type {Resolver} */ const unixFsResolver = async (cid, name, path, toResolve, resolve, depth, ipld, options) => { const node = await ipld.get(cid, options) @@ -104,7 +100,8 @@ const unixFsResolver = async (cid, name, path, toResolve, resolve, depth, ipld, content: contentExporters[unixfs.type](cid, node, unixfs, path, resolve, depth, ipld), unixfs, depth, - node + node, + size: unixfs.fileSize() }, next } diff --git a/packages/ipfs-unixfs-exporter/src/types.d.ts b/packages/ipfs-unixfs-exporter/src/types.d.ts new file mode 100644 index 00000000..52f3bf1e --- /dev/null +++ b/packages/ipfs-unixfs-exporter/src/types.d.ts @@ -0,0 +1,69 @@ +import CID from 'cids' +import UnixFS from 'ipfs-unixfs' +import DAGNode from 'ipld-dag-pb' + +interface ExporterOptions { + offset?: number + length?: number + signal?: AbortSignal + timeout?: number +} + +interface Exportable { + type: 'file' | 'directory' | 'object' | 'raw' | 'identity', + name: string + path: string + cid: CID + depth: number + size: number + content: (options?: ExporterOptions) => AsyncIterable +} + +interface UnixFSFile extends Exportable { + type: 'file' + unixfs: UnixFS + node: DAGNode +} + +interface UnixFSDirectory extends Exportable { + type: 'directory' + unixfs: UnixFS + node: DAGNode +} + +interface ObjectNode extends Exportable { + type: 'object' + node: Uint8Array +} + +interface RawNode extends Exportable { + type: 'raw' + node: Uint8Array +} + +interface IdentityNode extends Exportable { + type: 'identity' + node: Uint8Array +} + +type UnixFSEntry = UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode + +interface NextResult { + cid: CID + name: string + path: string + toResolve: string[] +} + +interface ResolveResult { + entry: UnixFSEntry + next?: NextResult +} + +type Resolve = (cid: CID, name: string, path: string, toResolve: string[], depth: number, ipld: IPLD, options: ExporterOptions) => Promise +type Resolver = (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, ipld: IPLD, options: ExporterOptions) => Promise + +type UnixfsV1FileContent = AsyncIterable | Iterable +type UnixfsV1DirectoryContent = AsyncIterable | Iterable +type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent +type UnixfsV1Resolver = (cid: CID, node: DAGNode, unixfs: UnixFS, path: string, resolve: Resolve, depth: number, ipld: IPLD) => (options: ExporterOptions) => UnixfsV1Content diff --git a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js index 37cdcbcc..9b48fede 100644 --- a/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js +++ b/packages/ipfs-unixfs-exporter/src/utils/find-cid-in-shard.js @@ -4,8 +4,8 @@ const { Bucket, createHAMT } = require('hamt-sharding') const multihashing = require('multihashing-async') /** - * @typedef {import('../').ExporterOptions} ExporterOptions - * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD + * @typedef {import('../types').ExporterOptions} ExporterOptions + * @typedef {import('ipld')} IPLD * @typedef {import('cids')} CID */ diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.js b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.js index 7960fe82..876591fa 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.js +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.js @@ -6,14 +6,14 @@ const { expect } = require('aegir/utils/chai') const IPLD = require('ipld') // @ts-ignore const inMemory = require('ipld-in-memory') -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const mh = require('multihashing-async').multihash const mc = require('multicodec') const all = require('it-all') const last = require('it-last') const randomBytes = require('it-buffer-stream') -const exporter = require('../src') -const importer = require('ipfs-unixfs-importer') +const { exporter, walkPath } = require('../src') +const { importer } = require('ipfs-unixfs-importer') const { DAGLink, DAGNode @@ -30,9 +30,9 @@ const SHARD_SPLIT_THRESHOLD = 10 describe('exporter sharded', function () { this.timeout(30000) - /** @type {import('ipfs-core-types/src/ipld').IPLD} */ + /** @type {import('ipld')} */ let ipld - /** @type {import('ipfs-unixfs-importer').BlockAPI} */ + /** @type {import('ipfs-unixfs-importer/src/types').BlockAPI} */ let block /** @@ -206,7 +206,7 @@ describe('exporter sharded', function () { it('uses .path to extract all intermediate entries from the sharded directory', async () => { const dirCid = await createShardWithFileNames(31, (index) => `/foo/bar/baz/file-${index}`) - const exported = await all(exporter.path(`/ipfs/${dirCid.toBaseEncodedString()}/foo/bar/baz/file-1`, ipld)) + const exported = await all(walkPath(`/ipfs/${dirCid.toBaseEncodedString()}/foo/bar/baz/file-1`, ipld)) expect(exported.length).to.equal(5) @@ -223,7 +223,7 @@ describe('exporter sharded', function () { it('uses .path to extract all intermediate entries from the sharded directory as well as the contents', async () => { const dirCid = await createShardWithFileNames(31, (index) => `/foo/bar/baz/file-${index}`) - const exported = await all(exporter.path(`/ipfs/${dirCid.toBaseEncodedString()}/foo/bar/baz`, ipld)) + const exported = await all(walkPath(`/ipfs/${dirCid.toBaseEncodedString()}/foo/bar/baz`, ipld)) expect(exported.length).to.equal(4) diff --git a/packages/ipfs-unixfs-exporter/test/exporter-subtree.spec.js b/packages/ipfs-unixfs-exporter/test/exporter-subtree.spec.js index c4aebc87..99633ec9 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-subtree.spec.js +++ b/packages/ipfs-unixfs-exporter/test/exporter-subtree.spec.js @@ -6,7 +6,7 @@ const { expect } = require('aegir/utils/chai') const IPLD = require('ipld') // @ts-ignore const inMemory = require('ipld-in-memory') -const importer = require('ipfs-unixfs-importer') +const { importer } = require('ipfs-unixfs-importer') const all = require('it-all') const last = require('it-last') const blockApi = require('./helpers/block') @@ -15,12 +15,12 @@ const uint8ArrayConcat = require('uint8arrays/concat') const ONE_MEG = Math.pow(1024, 2) -const exporter = require('./../src') +const { exporter, walkPath } = require('./../src') describe('exporter subtree', () => { - /** @type {import('ipfs-core-types/src/ipld').IPLD} */ + /** @type {import('ipld')} */ let ipld - /** @type {import('ipfs-unixfs-importer').BlockAPI} */ + /** @type {import('ipfs-unixfs-importer/src/types').BlockAPI} */ let block before(async () => { @@ -133,7 +133,7 @@ describe('exporter subtree', () => { throw new Error('Nothing imported') } - const exported = await all(exporter.path(`${imported.cid.toBaseEncodedString()}/level-1/level-2/200Bytes.txt`, ipld)) + const exported = await all(walkPath(`${imported.cid.toBaseEncodedString()}/level-1/level-2/200Bytes.txt`, ipld)) expect(exported.length).to.equal(4) expect(exported[0].path).to.equal(imported.cid.toBaseEncodedString()) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.js b/packages/ipfs-unixfs-exporter/test/exporter.spec.js index 7e329f55..47cca47d 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.js +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.js @@ -6,7 +6,7 @@ const { expect } = require('aegir/utils/chai') const IPLD = require('ipld') // @ts-ignore const inMemory = require('ipld-in-memory') -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const CID = require('cids') const { DAGNode, @@ -14,8 +14,8 @@ const { } = require('ipld-dag-pb') const mh = require('multihashing-async').multihash const mc = require('multicodec') -const exporter = require('../src') -const importer = require('ipfs-unixfs-importer') +const { exporter, recursive } = require('../src') +const { importer } = require('ipfs-unixfs-importer') const all = require('it-all') const last = require('it-last') const first = require('it-first') @@ -29,9 +29,9 @@ const uint8ArrayConcat = require('uint8arrays/concat') const ONE_MEG = Math.pow(1024, 2) describe('exporter', () => { - /** @type {import('ipfs-core-types/src/ipld').IPLD} */ + /** @type {import('ipld')} */ let ipld - /** @type {import('ipfs-unixfs-importer').BlockAPI} */ + /** @type {import('ipfs-unixfs-importer/src/types').BlockAPI} */ let block /** @type {Uint8Array} */ let bigFile @@ -134,7 +134,7 @@ describe('exporter', () => { } /** - * @param {import('ipfs-core-types/src/ipld').IPLD} ipld + * @param {import('ipld')} ipld * @param {'file' | 'directory' | 'raw'} type * @param {Uint8Array | ArrayLike | undefined} data * @param {{ node: DAGNode, cid: CID }[]} children @@ -1046,7 +1046,7 @@ describe('exporter', () => { throw new Error('Nothing imported') } - const exported = await all(exporter.recursive(dir.cid, ipld)) + const exported = await all(recursive(dir.cid, ipld)) const dirCid = dir.cid.toBaseEncodedString() expect(exported[0].depth).to.equal(0) diff --git a/packages/ipfs-unixfs-exporter/test/helpers/block.js b/packages/ipfs-unixfs-exporter/test/helpers/block.js index 313fd5d3..64b718fd 100644 --- a/packages/ipfs-unixfs-exporter/test/helpers/block.js +++ b/packages/ipfs-unixfs-exporter/test/helpers/block.js @@ -6,18 +6,30 @@ const { } = require('ipld-dag-pb') const multicodec = require('multicodec') const mh = require('multihashing-async').multihash +const CID = require('cids') +const Block = require('ipld-block') /** - * @param {import('ipfs-core-types/src/ipld').IPLD} ipld + * @param {import('ipld')} ipld */ function createBlockApi (ipld) { // make ipld behave like the block api, some tests need to pull // data from ipld so can't use a simple in-memory cid->block map - /** @type {import('ipfs-unixfs-importer').BlockAPI} */ + /** @type {import('ipfs-unixfs-importer/src/types').BlockAPI} */ const BlockApi = { - put: async (buf, { cid }) => { + put: async (buf, options) => { + if (!options || !options.cid) { + throw new Error('No cid passed') + } + + const cid = new CID(options.cid) + const multihash = mh.decode(cid.multihash) + if (Block.isBlock(buf)) { + buf = buf.data + } + /** @type {any} */ let obj = buf @@ -30,9 +42,11 @@ function createBlockApi (ipld) { hashAlg: multihash.code }) - return { cid, data: buf } + return new Block(buf, cid) }, get: async (cid, options) => { + cid = new CID(cid) + /** @type {Uint8Array} */ let buf = await ipld.get(cid, options) @@ -40,7 +54,7 @@ function createBlockApi (ipld) { buf = buf.serialize() } - return { cid, data: buf } + return new Block(buf, cid) } } diff --git a/packages/ipfs-unixfs-exporter/test/helpers/collect-leaf-cids.js b/packages/ipfs-unixfs-exporter/test/helpers/collect-leaf-cids.js index bb1e5b91..b8f04509 100644 --- a/packages/ipfs-unixfs-exporter/test/helpers/collect-leaf-cids.js +++ b/packages/ipfs-unixfs-exporter/test/helpers/collect-leaf-cids.js @@ -2,7 +2,7 @@ /** * @param {import('cids')} cid - * @param {import('ipfs-core-types/src/ipld').IPLD} ipld + * @param {import('ipld')} ipld */ module.exports = function (cid, ipld) { /** diff --git a/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.js b/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.js index 72f44fd7..64be0cdc 100644 --- a/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.js +++ b/packages/ipfs-unixfs-exporter/test/import-export-dir-sharding.spec.js @@ -1,8 +1,8 @@ /* eslint-env mocha */ 'use strict' -const importer = require('ipfs-unixfs-importer') -const exporter = require('../src') +const { importer } = require('ipfs-unixfs-importer') +const { exporter } = require('../src') const { expect } = require('aegir/utils/chai') // @ts-ignore @@ -22,9 +22,9 @@ const uint8ArrayConcat = require('uint8arrays/concat') */ describe('builder: directory sharding', () => { - /** @type {import('ipfs-core-types/src/ipld').IPLD} */ + /** @type {import('ipld')} */ let ipld - /** @type {import('ipfs-unixfs-importer').BlockAPI} */ + /** @type {import('ipfs-unixfs-importer/src/types').BlockAPI} */ let block before(async () => { diff --git a/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.js b/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.js index ae6fd5d8..8aa1221f 100644 --- a/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.js +++ b/packages/ipfs-unixfs-exporter/test/import-export-nested-dir.spec.js @@ -7,8 +7,8 @@ const IPLD = require('ipld') // @ts-ignore const inMemory = require('ipld-in-memory') const all = require('it-all') -const importer = require('ipfs-unixfs-importer') -const exporter = require('../src') +const { importer } = require('ipfs-unixfs-importer') +const { exporter } = require('../src') const blockApi = require('./helpers/block') const uint8ArrayFromString = require('uint8arrays/from-string') const uint8ArrayToString = require('uint8arrays/to-string') @@ -16,9 +16,9 @@ const uint8ArrayConcat = require('uint8arrays/concat') describe('import and export: directory', () => { const rootHash = 'QmdCrquDwd7RfZ6GCZFEVADwe8uyyw1YmF9mtAB7etDgmK' - /** @type {import('ipfs-core-types/src/ipld').IPLD} */ + /** @type {import('ipld')} */ let ipld - /** @type {import('ipfs-unixfs-importer').BlockAPI} */ + /** @type {import('ipfs-unixfs-importer/src/types').BlockAPI} */ let block before(async () => { diff --git a/packages/ipfs-unixfs-exporter/test/import-export.spec.js b/packages/ipfs-unixfs-exporter/test/import-export.spec.js index 7fff50a7..90d3b2e3 100644 --- a/packages/ipfs-unixfs-exporter/test/import-export.spec.js +++ b/packages/ipfs-unixfs-exporter/test/import-export.spec.js @@ -8,14 +8,14 @@ const IPLD = require('ipld') // @ts-ignore const inMemory = require('ipld-in-memory') // @ts-ignore -const loadFixture = require('aegir/fixtures') +const loadFixture = require('aegir/utils/fixtures') // @ts-ignore const isNode = require('detect-node') const bigFile = loadFixture((isNode ? __dirname : 'test') + '/fixtures/1.2MiB.txt') const blockApi = require('./helpers/block') -const importer = require('ipfs-unixfs-importer') -const exporter = require('../src') +const { importer } = require('ipfs-unixfs-importer') +const { exporter } = require('../src') const strategies = [ 'flat', @@ -30,9 +30,9 @@ describe('import and export', function () { const importerOptions = { strategy: strategy } describe('using builder: ' + strategy, () => { - /** @type {import('ipfs-core-types/src/ipld').IPLD} */ + /** @type {import('ipld')} */ let ipld - /** @type {import('ipfs-unixfs-importer').BlockAPI} */ + /** @type {import('ipfs-unixfs-importer/src/types').BlockAPI} */ let block before(async () => { diff --git a/packages/ipfs-unixfs-exporter/test/importer.spec.js b/packages/ipfs-unixfs-exporter/test/importer.spec.js index 978c9cf7..0f7b2a77 100644 --- a/packages/ipfs-unixfs-exporter/test/importer.spec.js +++ b/packages/ipfs-unixfs-exporter/test/importer.spec.js @@ -1,8 +1,8 @@ /* eslint-env mocha */ 'use strict' -const importer = require('ipfs-unixfs-importer') -const exporter = require('../src') +const { importer } = require('ipfs-unixfs-importer') +const { exporter, recursive } = require('../src') const extend = require('merge-options') const { expect } = require('aegir/utils/chai') const sinon = require('sinon') @@ -10,10 +10,10 @@ const sinon = require('sinon') const IPLD = require('ipld') // @ts-ignore const inMemory = require('ipld-in-memory') -const UnixFs = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const collectLeafCids = require('./helpers/collect-leaf-cids') // @ts-ignore -const loadFixture = require('aegir/fixtures') +const loadFixture = require('aegir/utils/fixtures') // @ts-ignore const isNode = require('detect-node') /** @type {Uint8Array} */ @@ -30,13 +30,13 @@ const last = require('it-last') const CID = require('cids') /** - * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD - * @typedef {import('ipfs-unixfs-importer').BlockAPI} BlockAPI + * @typedef {import('ipld')} IPLD + * @typedef {import('ipfs-unixfs-importer/src/types').BlockAPI} BlockAPI * @typedef {import('ipld-dag-pb').DAGNode} DAGNode */ /** - * @param {{ path?: string, cid: CID, unixfs?: UnixFs }[]} files + * @param {{ path?: string, cid: CID, unixfs?: UnixFS }[]} files */ function stringifyMh (files) { return files.map((file) => { @@ -216,7 +216,7 @@ const checkLeafNodeTypes = async (block, ipld, options, expected) => { /** @type {DAGNode} */ const node = await ipld.get(file.cid) - const meta = UnixFs.unmarshal(node.Data) + const meta = UnixFS.unmarshal(node.Data) expect(meta.type).to.equal('file') expect(node.Links.length).to.equal(2) @@ -226,7 +226,7 @@ const checkLeafNodeTypes = async (block, ipld, options, expected) => { ) linkedNodes.forEach(node => { - const meta = UnixFs.unmarshal(node.Data) + const meta = UnixFS.unmarshal(node.Data) expect(meta.type).to.equal(expected) }) } @@ -243,7 +243,7 @@ const checkNodeLinks = async (block, ipld, options, expected) => { content: new Uint8Array(100).fill(1) }], block, options)) { const node = await ipld.get(file.cid) - const meta = UnixFs.unmarshal(node.Data) + const meta = UnixFS.unmarshal(node.Data) expect(meta.type).to.equal('file') expect(node.Links.length).to.equal(expected) @@ -658,7 +658,7 @@ strategies.forEach((strategy) => { result.forEach(eachFile) /** - * @param {{ path?: string, cid: string, unixfs?: UnixFs }} file + * @param {{ path?: string, cid: string, unixfs?: UnixFS }} file */ function eachFile (file) { if (!file.unixfs) { @@ -883,7 +883,7 @@ strategies.forEach((strategy) => { content: bigFile }], block)) - const nodes = await all(exporter.recursive(entries[entries.length - 1].cid, ipld)) + const nodes = await all(recursive(entries[entries.length - 1].cid, ipld)) const node = nodes.filter(node => node.type === 'directory').pop() if (!node) { @@ -909,7 +909,7 @@ strategies.forEach((strategy) => { content: bigFile }], block)) - const nodes = await all(exporter.recursive(entries[entries.length - 1].cid, ipld)) + const nodes = await all(recursive(entries[entries.length - 1].cid, ipld)) const node = nodes.filter(node => node.type === 'directory').pop() if (!node) { @@ -940,7 +940,7 @@ strategies.forEach((strategy) => { content: bigFile }], block)) - const nodes = await all(exporter.recursive(entries[entries.length - 1].cid, ipld)) + const nodes = await all(recursive(entries[entries.length - 1].cid, ipld)) const node = nodes.filter(node => node.type === 'directory' && node.name === 'bar').pop() if (!node) { @@ -971,7 +971,7 @@ strategies.forEach((strategy) => { shardSplitThreshold: 0 })) - const nodes = await all(exporter.recursive(entries[entries.length - 1].cid, ipld)) + const nodes = await all(recursive(entries[entries.length - 1].cid, ipld)) const node = nodes.filter(node => node.type === 'directory' && node.unixfs.type === 'hamt-sharded-directory').pop() if (!node) { @@ -1089,12 +1089,13 @@ describe('configuration', () => { it('alllows configuring with custom dag and tree builder', async () => { let builtTree = false const cid = new CID('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') - const unixfs = new UnixFs({ type: 'directory' }) + const unixfs = new UnixFS({ type: 'directory' }) const entries = await all(importer([{ path: 'path', content: 'content' }], block, { + /** @type {import('ipfs-unixfs-importer/src/types').DAGBuilder} */ dagBuilder: async function * (source, block, opts) { // eslint-disable-line require-await yield function () { return Promise.resolve({ @@ -1105,6 +1106,7 @@ describe('configuration', () => { }) } }, + /** @type {import('ipfs-unixfs-importer/src/types').TreeBuilder} */ treeBuilder: async function * (source, block, opts) { // eslint-disable-line require-await builtTree = true yield * source @@ -1126,6 +1128,7 @@ describe('configuration', () => { path: 'path', content: uint8ArrayFromString('content') }], block, { + /** @type {import('ipfs-unixfs-importer/src/types').ChunkValidator} */ chunkValidator: async function * (source, opts) { // eslint-disable-line require-await validated = true @@ -1137,6 +1140,7 @@ describe('configuration', () => { } } }, + /** @type {import('ipfs-unixfs-importer/src/types').Chunker} */ chunker: async function * (source, opts) { // eslint-disable-line require-await chunked = true yield * source diff --git a/packages/ipfs-unixfs-exporter/tsconfig.json b/packages/ipfs-unixfs-exporter/tsconfig.json index c6eb8f25..3c4a52f9 100644 --- a/packages/ipfs-unixfs-exporter/tsconfig.json +++ b/packages/ipfs-unixfs-exporter/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "aegir/src/config/tsconfig.aegir.json", + "extends": "../../node_modules/aegir/src/config/tsconfig.aegir.json", "compilerOptions": { "outDir": "dist", "importsNotUsedAsValues": "preserve" diff --git a/packages/ipfs-unixfs-importer/.aegir.js b/packages/ipfs-unixfs-importer/.aegir.js index 40377838..e67cd911 100644 --- a/packages/ipfs-unixfs-importer/.aegir.js +++ b/packages/ipfs-unixfs-importer/.aegir.js @@ -1,16 +1,35 @@ 'use strict' +/** @type {import('aegir').Options["build"]["config"]} */ +const buildConfig = { + plugins: [ + { + name: 'node built ins', + setup (build) { + build.onResolve({ filter: /^stream$/ }, () => { + return { path: require.resolve('readable-stream') } + }) + build.onResolve({ filter: /^crypto$/ }, () => { + return { path: require.resolve('crypto-browserify') } + }) + build.onResolve({ filter: /^cborg$/ }, () => { + return { path: require.resolve('cborg') } + }) + } + } + ] +} + +/** @type {import('aegir').PartialOptions} */ module.exports = { - karma: { - browserNoActivityTimeout: 500 * 1000 + build: { + config: buildConfig }, - webpack: { - node: { - // needed by the cbor module - stream: true, - - // needed by the core-util-is module - Buffer: true + test: { + browser: { + config: { + buildConfig + } } } } diff --git a/packages/ipfs-unixfs-importer/.npmignore b/packages/ipfs-unixfs-importer/.npmignore deleted file mode 100644 index f9a85ba9..00000000 --- a/packages/ipfs-unixfs-importer/.npmignore +++ /dev/null @@ -1,3 +0,0 @@ -docs -test -.aegir.js diff --git a/packages/ipfs-unixfs-importer/README.md b/packages/ipfs-unixfs-importer/README.md index 076d15de..8ea9fd1c 100644 --- a/packages/ipfs-unixfs-importer/README.md +++ b/packages/ipfs-unixfs-importer/README.md @@ -18,7 +18,7 @@ - [Usage](#usage) - [Example](#example) - [API](#api) - - [const import = importer(source, ipld [, options])](#const-import--importersource-ipld--options) + - [const stream = importer(source, ipld [, options])](#const-stream--importersource-ipld--options) - [Overriding internals](#overriding-internals) - [Contribute](#contribute) - [License](#license) @@ -45,7 +45,7 @@ Let's create a little directory to import: And write the importing logic: ```js -const importer = require('ipfs-unixfs-importer') +const { importer } = require('ipfs-unixfs-importer') // Import path /tmp/foo/bar const source = [{ @@ -91,12 +91,12 @@ When run, metadata about DAGNodes in the created tree is printed until the root: #### API ```js -const importer = require('ipfs-unixfs-importer') +const { importer } = require('ipfs-unixfs-importer') ``` -#### const import = importer(source, ipld [, options]) +#### const stream = importer(source, ipld [, options]) -The `import` function returns an async iterator takes a source async iterator that yields objects of the form: +The `importer` function returns an async iterator takes a source async iterator that yields objects of the form: ```js { @@ -107,9 +107,9 @@ The `import` function returns an async iterator takes a source async iterator th } ``` -`import` will output file info objects as files get stored in IPFS. When stats on a node are emitted they are guaranteed to have been written. +`stream` will output file info objects as files get stored in IPFS. When stats on a node are emitted they are guaranteed to have been written. -`ipld` is an instance of the [`IPLD Resolver`](https://github.com/ipld/js-ipld-resolver) or the [`js-ipfs` `dag api`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md) +`ipld` is an instance of the [`IPLD Resolver`](https://github.com/ipld/js-ipld-resolver) The input's file paths and directory structure will be preserved in the [`dag-pb`](https://github.com/ipld/js-ipld-dag-pb) created nodes. diff --git a/packages/ipfs-unixfs-importer/package.json b/packages/ipfs-unixfs-importer/package.json index 4e441d20..50c6e662 100644 --- a/packages/ipfs-unixfs-importer/package.json +++ b/packages/ipfs-unixfs-importer/package.json @@ -10,10 +10,11 @@ "scripts": { "prepare": "aegir build --no-bundle", "test": "aegir test", + "build": "aegir build", "clean": "rimraf ./dist", - "lint": "aegir lint", + "lint": "aegir ts --check && aegir lint", "coverage": "nyc -s npm run test -t node && nyc report --reporter=html", - "depcheck": "aegir dep-check -i @types/mocha -i nyc -i rimraf" + "depcheck": "aegir dep-check -i @types/mocha -i nyc -i rimraf -i copy -i util -i crypto-browserify -i events -i readable-stream -i assert" }, "repository": { "type": "git", @@ -27,19 +28,26 @@ "url": "https://github.com/ipfs/js-ipfs-unixfs/issues" }, "engines": { - "node": ">=10.0.0", - "npm": ">=4.0.0" + "node": ">=15.0.0", + "npm": ">=7.0.0" }, "homepage": "https://github.com/ipfs/js-ipfs-unixfs#readme", "devDependencies": { "@types/mocha": "^8.2.1", - "aegir": "^30.3.0", - "ipld": "^0.28.0", - "ipld-in-memory": "^7.0.0", + "aegir": "^32.0.0", + "assert": "^2.0.0", + "copy": "^0.3.2", + "crypto-browserify": "^3.12.0", + "events": "^3.3.0", + "ipld": "^0.29.0", + "ipld-block": "^0.11.1", + "ipld-in-memory": "^8.0.0", "it-buffer-stream": "^2.0.0", - "multicodec": "^2.1.0", + "multicodec": "^3.0.1", "nyc": "^15.0.0", - "rimraf": "^3.0.2" + "readable-stream": "^3.6.0", + "rimraf": "^3.0.2", + "util": "^0.12.3" }, "dependencies": { "bl": "^4.1.0", diff --git a/packages/ipfs-unixfs-importer/src/chunker/fixed-size.js b/packages/ipfs-unixfs-importer/src/chunker/fixed-size.js index ecbb360a..8ef07361 100644 --- a/packages/ipfs-unixfs-importer/src/chunker/fixed-size.js +++ b/packages/ipfs-unixfs-importer/src/chunker/fixed-size.js @@ -4,7 +4,7 @@ const BufferList = require('bl/BufferList') /** - * @type {import('./').Chunker} + * @type {import('../types').Chunker} */ module.exports = async function * fixedSizeChunker (source, options) { let bl = new BufferList() diff --git a/packages/ipfs-unixfs-importer/src/chunker/index.js b/packages/ipfs-unixfs-importer/src/chunker/index.js index fb371fb9..b7df2f09 100644 --- a/packages/ipfs-unixfs-importer/src/chunker/index.js +++ b/packages/ipfs-unixfs-importer/src/chunker/index.js @@ -3,8 +3,8 @@ const errCode = require('err-code') /** - * @typedef {import('../').ImporterOptions} ImporterOptions - * @typedef {(source: AsyncIterable, options: ImporterOptions) => AsyncIterable} Chunker + * @typedef {import('../types').ImporterOptions} ImporterOptions + * @typedef {import('../types').Chunker} Chunker */ /** @@ -16,10 +16,9 @@ const chunkers = { } /** - * - * @param {import('../').ChunkerType} type + * @param {import('../types').ChunkerType} type * @param {AsyncIterable} source - * @param {import('../').ImporterOptions} options + * @param {import('../types').ImporterOptions} options */ module.exports = (type, source, options) => { const chunker = chunkers[type] diff --git a/packages/ipfs-unixfs-importer/src/chunker/rabin.js b/packages/ipfs-unixfs-importer/src/chunker/rabin.js index 7e9f6475..221659a1 100644 --- a/packages/ipfs-unixfs-importer/src/chunker/rabin.js +++ b/packages/ipfs-unixfs-importer/src/chunker/rabin.js @@ -16,7 +16,7 @@ const errcode = require('err-code') */ /** - * @type {import('./').Chunker} + * @type {import('../types').Chunker} */ module.exports = async function * rabinChunker (source, options) { let min, max, avg diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/dir.js b/packages/ipfs-unixfs-importer/src/dag-builder/dir.js index 9541698e..36106c9e 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/dir.js +++ b/packages/ipfs-unixfs-importer/src/dag-builder/dir.js @@ -1,17 +1,17 @@ 'use strict' -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const persist = require('../utils/persist') const { DAGNode } = require('ipld-dag-pb') /** - * @typedef {import('../').Directory} Directory + * @typedef {import('../types').Directory} Directory */ /** - * @type {import('./').UnixFSV1DagBuilder} + * @type {import('../types').UnixFSV1DagBuilder} */ const dirBuilder = async (item, block, options) => { const unixfs = new UnixFS({ diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file/balanced.js b/packages/ipfs-unixfs-importer/src/dag-builder/file/balanced.js index 31f65446..cb0ace2d 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file/balanced.js +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file/balanced.js @@ -3,23 +3,18 @@ const batch = require('it-batch') /** - * @typedef {import('cids')} CID - * @typedef {import('../..').ImporterOptions} ImporterOptions - * @typedef {import('../..').ImportResult} ImportResult + * @typedef {import('../../types').FileDAGBuilder} FileDAGBuilder */ /** - * @type {import('.').DAGBuilder} + * @type {FileDAGBuilder} */ -async function * balanced (source, reduce, options) { - yield await reduceToParents(source, reduce, options) +function balanced (source, reduce, options) { + return reduceToParents(source, reduce, options) } /** - * @param {AsyncIterable | ImportResult[]} source - * @param {import('.').Reducer} reduce - * @param {ImporterOptions} options - * @returns {Promise} + * @type {FileDAGBuilder} */ async function reduceToParents (source, reduce, options) { const roots = [] diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file/buffer-importer.js b/packages/ipfs-unixfs-importer/src/dag-builder/file/buffer-importer.js index e01df504..b8179e89 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file/buffer-importer.js +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file/buffer-importer.js @@ -1,17 +1,13 @@ 'use strict' -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const persist = require('../../utils/persist') const { DAGNode } = require('ipld-dag-pb') /** - * @typedef {import('../../').BlockAPI} BlockAPI - * @typedef {import('../../').ImporterOptions} ImporterOptions - * @typedef {import('../../').File} File - * @typedef {import('../../').PartialImportResult} PartialImportResult - * @typedef {(file: File, block: BlockAPI, options: ImporterOptions) => AsyncIterable<() => Promise>} BufferImporter + * @typedef {import('../../types').BufferImporter} BufferImporter */ /** @@ -23,8 +19,12 @@ async function * bufferImporter (file, block, options) { options.progress(buffer.length, file.path) let unixfs + /** @type {import('../../types').PersistOptions} */ const opts = { - ...options + codec: 'dag-pb', + cidVersion: options.cidVersion, + hashAlg: options.hashAlg, + onlyHash: options.onlyHash } if (options.rawLeaves) { diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file/flat.js b/packages/ipfs-unixfs-importer/src/dag-builder/file/flat.js index 3b3cedda..549bab0e 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file/flat.js +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file/flat.js @@ -3,8 +3,8 @@ const all = require('it-all') /** - * @type {import('./').DAGBuilder} + * @type {import('../../types').FileDAGBuilder} */ -module.exports = async function * (source, reduce) { - yield await reduce(await all(source)) +module.exports = async function (source, reduce) { + return reduce(await all(source)) } diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file/index.js b/packages/ipfs-unixfs-importer/src/dag-builder/file/index.js index c4ce9892..152dac91 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file/index.js +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file/index.js @@ -1,29 +1,26 @@ 'use strict' const errCode = require('err-code') -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const persist = require('../../utils/persist') const { DAGNode, DAGLink } = require('ipld-dag-pb') -const all = require('it-all') const parallelBatch = require('it-parallel-batch') const mh = require('multihashing-async').multihash /** - * @typedef {import('cids')} CID - * @typedef {import('../../').File} File - * @typedef {import('../../').BlockAPI} BlockAPI - * @typedef {import('../../').ImporterOptions} ImporterOptions - * @typedef {import('../../').PartialImportResult} PartialImportResult - * - * @typedef {(leaves: PartialImportResult[]) => Promise} Reducer - * @typedef {(source: AsyncIterable, reducer: Reducer, options: ImporterOptions) => AsyncIterable} DAGBuilder + * @typedef {import('../../types').BlockAPI} BlockAPI + * @typedef {import('../../types').File} File + * @typedef {import('../../types').ImporterOptions} ImporterOptions + * @typedef {import('../../types').Reducer} Reducer + * @typedef {import('../../types').DAGBuilder} DAGBuilder + * @typedef {import('../../types').FileDAGBuilder} FileDAGBuilder */ /** - * @type {{ [key: string]: DAGBuilder}} + * @type {{ [key: string]: FileDAGBuilder}} */ const dagBuilders = { flat: require('./flat'), @@ -166,22 +163,16 @@ const reduce = (file, block, options) => { } /** - * @type {import('../').UnixFSV1DagBuilder} + * @type {import('../../types').UnixFSV1DagBuilder} */ -const fileBuilder = async (file, block, options) => { +function fileBuilder (file, block, options) { const dagBuilder = dagBuilders[options.strategy] if (!dagBuilder) { throw errCode(new Error(`Unknown importer build strategy name: ${options.strategy}`), 'ERR_BAD_STRATEGY') } - const roots = await all(dagBuilder(buildFileBatch(file, block, options), reduce(file, block, options), options)) - - if (roots.length > 1) { - throw errCode(new Error('expected a maximum of 1 roots and got ' + roots.length), 'ETOOMANYROOTS') - } - - return roots[0] + return dagBuilder(buildFileBatch(file, block, options), reduce(file, block, options), options) } module.exports = fileBuilder diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file/trickle.js b/packages/ipfs-unixfs-importer/src/dag-builder/file/trickle.js index 787537a1..fc1c6aef 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file/trickle.js +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file/trickle.js @@ -4,26 +4,18 @@ const batch = require('it-batch') /** * @typedef {import('cids')} CID - * @typedef {import('../..').ImporterOptions} ImporterOptions - * @typedef {import('../..').PartialImportResult} PartialImportResult - * @typedef {import('.').Reducer} Reducer - * @typedef {import('ipfs-unixfs')} UnixFS + * @typedef {import('ipfs-unixfs').UnixFS} UnixFS + * @typedef {import('../../types').ImporterOptions} ImporterOptions + * @typedef {import('../../types').InProgressImportResult} InProgressImportResult + * @typedef {import('../../types').TrickleDagNode} TrickleDagNode + * @typedef {import('../../types').Reducer} Reducer + * @typedef {import('../../types').FileDAGBuilder} FileDAGBuilder */ /** - * @type {import('./').DAGBuilder} + * @type {FileDAGBuilder} */ -module.exports = function * trickleReduceToRoot (source, reduce, options) { - yield trickleStream(source, reduce, options) -} - -/** - * @param {AsyncIterable | Iterable} source - * @param {Reducer} reduce - * @param {ImporterOptions} options - * @returns {Promise} - */ -async function trickleStream (source, reduce, options) { +module.exports = async function trickleStream (source, reduce, options) { const root = new Root(options.layerRepeat) let iteration = 0 let maxDepth = 1 @@ -56,19 +48,6 @@ async function trickleStream (source, reduce, options) { return root.reduce(reduce) } -/** - * @typedef {object} TrickleDagNode - * @property {PartialImportResult[]} children - * @property {number} depth - * @property {number} maxDepth - * @property {number} maxChildren - * @property {PartialImportResult[]} [data] - * @property {TrickleDagNode} [parent] - * @property {CID} [cid] - * @property {number} [size] - * @property {UnixFS} [unixfs] - */ - class SubTree { /** * @param {number} maxDepth @@ -138,7 +117,7 @@ class SubTree { /** * - * @param {PartialImportResult[]} layer + * @param {InProgressImportResult[]} layer */ append (layer) { this.node.data = layer @@ -154,10 +133,10 @@ class SubTree { /** * @param {TrickleDagNode} node * @param {Reducer} reduce - * @returns {Promise} + * @returns {Promise} */ async _reduce (node, reduce) { - /** @type {PartialImportResult[]} */ + /** @type {InProgressImportResult[]} */ let children = [] if (node.children.length) { @@ -206,7 +185,7 @@ class Root extends SubTree { } /** - * @param {PartialImportResult} child + * @param {InProgressImportResult} child */ addChild (child) { this.root.children.push(child) diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/index.js b/packages/ipfs-unixfs-importer/src/dag-builder/index.js index abb4361d..9a082725 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/index.js +++ b/packages/ipfs-unixfs-importer/src/dag-builder/index.js @@ -5,22 +5,11 @@ const fileBuilder = require('./file') const errCode = require('err-code') /** - * @typedef {import('../').BlockAPI} BlockAPI - * @typedef {import('../').ImporterOptions} ImporterOptions - * @typedef {import('../').ImportResult} ImportResult - * @typedef {import('../').PartialImportResult} PartialImportResult - * @typedef {import('../').ImportCandidate} ImportCandidate - * @typedef {import('../').File} File - * @typedef {import('../').Directory} Directory - */ - -/** - * @template T - * @typedef {(item: T, block: BlockAPI, options: ImporterOptions) => Promise} UnixFSV1DagBuilder - */ - -/** - * @typedef {(source: AsyncIterable | Iterable, block: BlockAPI, options: ImporterOptions) => AsyncIterable<() => Promise>} DAGBuilder + * @typedef {import('../types').File} File + * @typedef {import('../types').Directory} Directory + * @typedef {import('../types').DAGBuilder} DAGBuilder + * @typedef {import('../types').Chunker} Chunker + * @typedef {import('../types').ChunkValidator} ChunkValidator */ /** @@ -74,7 +63,7 @@ async function * dagBuilder (source, block, options) { }()) /** - * @type {import('../chunker').Chunker} + * @type {Chunker} */ let chunker @@ -87,7 +76,7 @@ async function * dagBuilder (source, block, options) { } /** - * @type {import('./validate-chunks').ChunkValidator} + * @type {ChunkValidator} */ let chunkValidator diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/validate-chunks.js b/packages/ipfs-unixfs-importer/src/dag-builder/validate-chunks.js index cfd1f46f..88689a86 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/validate-chunks.js +++ b/packages/ipfs-unixfs-importer/src/dag-builder/validate-chunks.js @@ -4,8 +4,7 @@ const errCode = require('err-code') const uint8ArrayFromString = require('uint8arrays/from-string') /** - * @typedef {import('../').ImporterOptions} ImporterOptions - * @typedef {(source: AsyncIterable>, options: ImporterOptions) => AsyncIterable} ChunkValidator + * @typedef {import('../types').ChunkValidator} ChunkValidator */ /** diff --git a/packages/ipfs-unixfs-importer/src/dir-flat.js b/packages/ipfs-unixfs-importer/src/dir-flat.js index e292f8e8..0a720e4a 100644 --- a/packages/ipfs-unixfs-importer/src/dir-flat.js +++ b/packages/ipfs-unixfs-importer/src/dir-flat.js @@ -4,15 +4,15 @@ const { DAGLink, DAGNode } = require('ipld-dag-pb') -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const Dir = require('./dir') const persist = require('./utils/persist') /** - * @typedef {import('./').ImporterOptions} ImporterOptions - * @typedef {import('./').ImportResult} ImportResult - * @typedef {import('./').PartialImportResult} PartialImportResult - * @typedef {import('./').BlockAPI} BlockAPI + * @typedef {import('./types').ImporterOptions} ImporterOptions + * @typedef {import('./types').ImportResult} ImportResult + * @typedef {import('./types').InProgressImportResult} InProgressImportResult + * @typedef {import('./types').BlockAPI} BlockAPI * @typedef {import('./dir').DirProps} DirProps * @typedef {import('cids')} CID */ @@ -25,13 +25,13 @@ class DirFlat extends Dir { constructor (props, options) { super(props, options) - /** @type {{ [key: string]: PartialImportResult | Dir }} */ + /** @type {{ [key: string]: InProgressImportResult | Dir }} */ this._children = {} } /** * @param {string} name - * @param {PartialImportResult | Dir} value + * @param {InProgressImportResult | Dir} value */ async put (name, value) { this.cid = undefined diff --git a/packages/ipfs-unixfs-importer/src/dir-sharded.js b/packages/ipfs-unixfs-importer/src/dir-sharded.js index 4e9277a1..e0567838 100644 --- a/packages/ipfs-unixfs-importer/src/dir-sharded.js +++ b/packages/ipfs-unixfs-importer/src/dir-sharded.js @@ -4,16 +4,16 @@ const { DAGLink, DAGNode } = require('ipld-dag-pb') -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const Dir = require('./dir') const persist = require('./utils/persist') const { createHAMT, Bucket } = require('hamt-sharding') /** - * @typedef {import('./').ImporterOptions} ImporterOptions - * @typedef {import('./').ImportResult} ImportResult - * @typedef {import('./').PartialImportResult} PartialImportResult - * @typedef {import('./').BlockAPI} BlockAPI + * @typedef {import('./types').ImporterOptions} ImporterOptions + * @typedef {import('./types').ImportResult} ImportResult + * @typedef {import('./types').InProgressImportResult} InProgressImportResult + * @typedef {import('./types').BlockAPI} BlockAPI */ /** @@ -28,7 +28,7 @@ class DirSharded extends Dir { constructor (props, options) { super(props, options) - /** @type {Bucket} */ + /** @type {Bucket} */ this._bucket = createHAMT({ hashFn: options.hamtHashFn, bits: options.hamtBucketBits @@ -37,7 +37,7 @@ class DirSharded extends Dir { /** * @param {string} name - * @param {PartialImportResult | Dir} value + * @param {InProgressImportResult | Dir} value */ async put (name, value) { await this._bucket.put(name, value) diff --git a/packages/ipfs-unixfs-importer/src/dir.js b/packages/ipfs-unixfs-importer/src/dir.js index db7656d7..072d66a6 100644 --- a/packages/ipfs-unixfs-importer/src/dir.js +++ b/packages/ipfs-unixfs-importer/src/dir.js @@ -1,10 +1,10 @@ 'use strict' /** - * @typedef {import('./').ImporterOptions} ImporterOptions - * @typedef {import('./').ImportResult} ImportResult - * @typedef {import('./').PartialImportResult} PartialImportResult - * @typedef {import('./').BlockAPI} BlockAPI + * @typedef {import('./types').ImporterOptions} ImporterOptions + * @typedef {import('./types').ImportResult} ImportResult + * @typedef {import('./types').InProgressImportResult} InProgressImportResult + * @typedef {import('./types').BlockAPI} BlockAPI * @typedef {import('cids')} CID * @typedef {object} DirProps * @property {boolean} root @@ -14,7 +14,7 @@ * @property {boolean} flat * @property {Dir} [parent] * @property {string} [parentKey] - * @property {import('ipfs-unixfs')} [unixfs] + * @property {import('ipfs-unixfs').UnixFS} [unixfs] * @property {number} [mode] * @property {import('ipfs-unixfs').Mtime} [mtime] */ @@ -46,20 +46,20 @@ class Dir { /** * @param {string} name - * @param {PartialImportResult | Dir} value + * @param {InProgressImportResult | Dir} value */ async put (name, value) { } /** * @param {string} name - * @returns {Promise} + * @returns {Promise} */ get (name) { return Promise.resolve(this) } /** - * @returns {AsyncIterable<{ key: string, child: PartialImportResult | Dir}>} + * @returns {AsyncIterable<{ key: string, child: InProgressImportResult | Dir}>} */ async * eachChildSeries () { } diff --git a/packages/ipfs-unixfs-importer/src/flat-to-shard.js b/packages/ipfs-unixfs-importer/src/flat-to-shard.js index 4ef40066..590cd62c 100644 --- a/packages/ipfs-unixfs-importer/src/flat-to-shard.js +++ b/packages/ipfs-unixfs-importer/src/flat-to-shard.js @@ -5,7 +5,7 @@ const DirFlat = require('./dir-flat') /** * @typedef {import('./dir')} Dir - * @typedef {import('./').ImporterOptions} ImporterOptions + * @typedef {import('./types').ImporterOptions} ImporterOptions */ /** diff --git a/packages/ipfs-unixfs-importer/src/index.js b/packages/ipfs-unixfs-importer/src/index.js index 6a993a59..3418f69b 100644 --- a/packages/ipfs-unixfs-importer/src/index.js +++ b/packages/ipfs-unixfs-importer/src/index.js @@ -4,117 +4,13 @@ const parallelBatch = require('it-parallel-batch') const defaultOptions = require('./options') /** - * @typedef {import('cids')} CID - * @typedef {import('ipfs-unixfs')} UnixFS - * - * @typedef {object} Block - * @property {Uint8Array} data - * @property {CID} cid - * - * @typedef {object} BlockAPI - * @property {(cid: CID, options?: any) => Promise} get - * @property {(buffer: Uint8Array, options: any) => Promise} put - * - * @typedef {object} ImportCandidate - * @property {string} [path] - * @property {AsyncIterable | Iterable | Uint8Array | ArrayLike | string} [content] - * @property {import('ipfs-unixfs').Mtime | Date} [mtime] - * @property {number} [mode] - * - * @typedef {object} File - * @property {AsyncIterable} content - * @property {string} [path] - * @property {import('ipfs-unixfs').Mtime | Date} [mtime] - * @property {number} [mode] - * - * @typedef {object} Directory - * @property {string} path - * @property {import('ipfs-unixfs').Mtime | Date} [mtime] - * @property {number} [mode] - * - * @typedef {object} ImportResult - * @property {CID} cid - * @property {number} size - * @property {string} [path] - * @property {UnixFS} [unixfs] - * - * @typedef {object} PartialImportResult - * @property {CID} cid - * @property {number} size - * @property {string} [path] - * @property {UnixFS} [unixfs] - * @property {boolean} [single] - * - * @typedef {'fixed'|'rabin'} ChunkerType - * - * @typedef {object} UserImporterOptions - * @property {ChunkerType | import('./chunker').Chunker} [chunker='fixed'] - * @property {'balanced'|'flat'|'trickle'} [strategy='balanced'] - * @property {boolean} [rawLeaves=false] - * @property {boolean} [onlyHash=false] - * @property {boolean} [reduceSingleLeafToSelf=true] - * @property {import('multicodec').CodecName} [hashAlg='sha2-256'] - * @property {'file'|'raw'} [leafType='file'] - * @property {import('cids').CIDVersion} [cidVersion=0] - * @property {(chunkSize: number, path?: string) => void} [progress=() => {}] - * @property {number} [shardSplitThreshold=1000] - * @property {number} [fileImportConcurrency=50] - * @property {number} [blockWriteConcurrency=10] - * @property {number} [minChunkSize=262144] - * @property {number} [maxChunkSize=262144] - * @property {number} [avgChunkSize=262144] - * @property {number} [window=16] - * @property {number} [polynomial=17437180132763653] - * @property {number} [maxChildrenPerNode=174] - * @property {number} [layerRepeat=4] - * @property {boolean} [wrapWithDirectory=false] - * @property {boolean} [pin=false] - * @property {boolean} [recursive=false] - * @property {boolean} [hidden=false] - * @property {boolean} [preload=false] - * @property {string | number} [timeout] - * @property {(value: Uint8Array) => Promise} [hamtHashFn] - * @property {number} [hamtBucketBits=8] - * @property {number} [hamtHashCode=0x22] - * @property {import('./dag-builder').DAGBuilder} [dagBuilder] - * @property {import('./tree-builder').TreeBuilder} [treeBuilder] - * @property {import('./dag-builder/file/buffer-importer').BufferImporter} [bufferImporter] - * @property {import('./dag-builder/validate-chunks').ChunkValidator} [chunkValidator] - * - * @typedef {object} ImporterOptions - * @property {ChunkerType} chunker - * @property {'balanced'|'flat'|'trickle'} strategy - * @property {boolean} rawLeaves - * @property {boolean} onlyHash - * @property {boolean} reduceSingleLeafToSelf - * @property {'dag-pb'|'raw'} codec - * @property {import('multihashing-async').multihash.HashName} hashAlg - * @property {'file'|'raw'} leafType - * @property {import('cids').CIDVersion} cidVersion - * @property {(chunkSize: number, path?: string) => void} progress - * @property {number} shardSplitThreshold - * @property {number} fileImportConcurrency - * @property {number} blockWriteConcurrency - * @property {number} minChunkSize - * @property {number} maxChunkSize - * @property {number} avgChunkSize - * @property {number} window - * @property {number} polynomial - * @property {number} maxChildrenPerNode - * @property {number} layerRepeat - * @property {boolean} wrapWithDirectory - * @property {boolean} pin - * @property {boolean} recursive - * @property {boolean} hidden - * @property {boolean} preload - * @property {string | number | undefined} timeout - * @property {(value: Uint8Array) => Promise} hamtHashFn - * @property {number} hamtHashCode - * @property {number} hamtBucketBits - * @property {import('./dag-builder').DAGBuilder} [dagBuilder] - * @property {import('./tree-builder').TreeBuilder} [treeBuilder] - * @property {import('./dag-builder/file/buffer-importer').BufferImporter} [bufferImporter] - * @property {import('./dag-builder/validate-chunks').ChunkValidator} [chunkValidator] + * @typedef {import('./types').BlockAPI} BlockAPI + * @typedef {import('./types').ImportCandidate} ImportCandidate + * @typedef {import('./types').UserImporterOptions} UserImporterOptions + * @typedef {import('./types').ImporterOptions} ImporterOptions + * @typedef {import('./types').Directory} Directory + * @typedef {import('./types').File} File + * @typedef {import('./types').ImportResult} ImportResult */ /** @@ -122,7 +18,7 @@ const defaultOptions = require('./options') * @param {BlockAPI} block * @param {UserImporterOptions} options */ -module.exports = async function * (source, block, options = {}) { +async function * importer (source, block, options = {}) { const opts = defaultOptions(options) let dagBuilder @@ -150,3 +46,7 @@ module.exports = async function * (source, block, options = {}) { } } } + +module.exports = { + importer +} diff --git a/packages/ipfs-unixfs-importer/src/options.js b/packages/ipfs-unixfs-importer/src/options.js index 15d02fe6..6e9f4a3c 100644 --- a/packages/ipfs-unixfs-importer/src/options.js +++ b/packages/ipfs-unixfs-importer/src/options.js @@ -25,8 +25,8 @@ async function hamtHashFn (buf) { } /** - * @typedef {import('../').UserImporterOptions} UserImporterOptions - * @typedef {import('../').ImporterOptions} ImporterOptions + * @typedef {import('./types').UserImporterOptions} UserImporterOptions + * @typedef {import('./types').ImporterOptions} ImporterOptions */ /** @@ -38,7 +38,6 @@ const defaultOptions = { rawLeaves: false, onlyHash: false, reduceSingleLeafToSelf: true, - codec: 'dag-pb', hashAlg: 'sha2-256', leafType: 'file', // 'raw' cidVersion: 0, diff --git a/packages/ipfs-unixfs-importer/src/tree-builder.js b/packages/ipfs-unixfs-importer/src/tree-builder.js index d06ee069..b5c63ab8 100644 --- a/packages/ipfs-unixfs-importer/src/tree-builder.js +++ b/packages/ipfs-unixfs-importer/src/tree-builder.js @@ -6,15 +6,15 @@ const Dir = require('./dir') const toPathComponents = require('./utils/to-path-components') /** - * @typedef {import('./').ImportResult} ImportResult - * @typedef {import('./').PartialImportResult} PartialImportResult - * @typedef {import('./').ImporterOptions} ImporterOptions - * @typedef {import('./').BlockAPI} BlockAPI - * @typedef {(source: AsyncIterable, block: BlockAPI, options: ImporterOptions) => AsyncIterable} TreeBuilder + * @typedef {import('./types').ImportResult} ImportResult + * @typedef {import('./types').InProgressImportResult} InProgressImportResult + * @typedef {import('./types').ImporterOptions} ImporterOptions + * @typedef {import('./types').BlockAPI} BlockAPI + * @typedef {(source: AsyncIterable, block: BlockAPI, options: ImporterOptions) => AsyncIterable} TreeBuilder */ /** - * @param {PartialImportResult} elem + * @param {InProgressImportResult} elem * @param {Dir} tree * @param {ImporterOptions} options */ @@ -64,7 +64,7 @@ async function addToTree (elem, tree, options) { } /** - * @param {Dir | PartialImportResult} tree + * @param {Dir | InProgressImportResult} tree * @param {BlockAPI} block */ async function * flushAndYield (tree, block) { diff --git a/packages/ipfs-unixfs-importer/src/types.d.ts b/packages/ipfs-unixfs-importer/src/types.d.ts new file mode 100644 index 00000000..19edd024 --- /dev/null +++ b/packages/ipfs-unixfs-importer/src/types.d.ts @@ -0,0 +1,163 @@ +import { UnixFS, Mtime, MtimeLike } from 'ipfs-unixfs' +import CID, { CIDVersion } from 'cids' +import { HashName } from 'multihashes' +import Block from 'ipld-block' +import { CodecName } from 'multicodec' + +interface ImportCandidate { + path?: string + content?: AsyncIterable | Iterable | Uint8Array | ArrayLike | string + mtime?: MtimeLike + mode?: number +} + +interface File { + content: AsyncIterable + path?: string + mtime?: MtimeLike + mode?: number +} + +interface Directory { + path?: string + mtime?: MtimeLike + mode?: number +} + +interface ImportResult { + cid: CID + size: number + path?: string + unixfs?: UnixFS +} + +interface InProgressImportResult extends ImportResult { + single?: boolean +} + +type ChunkerType = 'fixed' | 'rabin' +type ProgressHandler = (chunkSize: number, path?: string) => void +type HamtHashFn = (value: Uint8Array) => Promise +type Chunker = (source: AsyncIterable, options: ImporterOptions) => AsyncIterable +type DAGBuilder = (source: AsyncIterable | Iterable, block: BlockAPI, options: ImporterOptions) => AsyncIterable<() => Promise> +type TreeBuilder = (source: AsyncIterable, block: BlockAPI, options: ImporterOptions) => AsyncIterable +type BufferImporter = (file: File, block: BlockAPI, options: ImporterOptions) => AsyncIterable<() => Promise> +type ChunkValidator = (source: AsyncIterable>, options: ImporterOptions) => AsyncIterable +type UnixFSV1DagBuilder = (item: T, block: BlockAPI, options: ImporterOptions) => Promise +type Reducer = (leaves: InProgressImportResult[]) => Promise + +type FileDAGBuilder = (source: AsyncIterable | Iterable, reducer: Reducer, options: ImporterOptions) => Promise + +interface UserImporterOptions { + strategy?: 'balanced' | 'flat' | 'trickle' + rawLeaves?: boolean + onlyHash?: boolean + reduceSingleLeafToSelf?: boolean + hashAlg?: HashName + leafType?: 'file' | 'raw' + cidVersion?: CIDVersion + progress?: ProgressHandler + shardSplitThreshold?: number + fileImportConcurrency?: number + blockWriteConcurrency?: number + minChunkSize?: number + maxChunkSize?: number + avgChunkSize?: number + window?: number + polynomial?: number + maxChildrenPerNode?: number + layerRepeat?: number + wrapWithDirectory?: boolean + pin?: boolean + recursive?: boolean + hidden?: boolean + preload?: boolean + timeout?: number + hamtHashFn?: HamtHashFn + hamtBucketBits?: number + hamtHashCode?: number + chunker?: ChunkerType | Chunker + dagBuilder?: DAGBuilder + treeBuilder?: TreeBuilder + bufferImporter?: BufferImporter + chunkValidator?: ChunkValidator +} + +interface ImporterOptions { + strategy: 'balanced' | 'flat' | 'trickle' + rawLeaves: boolean + onlyHash: boolean + reduceSingleLeafToSelf: boolean + hashAlg: HashName + leafType: 'file' | 'raw' + cidVersion: CIDVersion + progress: ProgressHandler + shardSplitThreshold: number + fileImportConcurrency: number + blockWriteConcurrency: number + minChunkSize: number + maxChunkSize: number + avgChunkSize: number + window: number + polynomial: number + maxChildrenPerNode: number + layerRepeat: number + wrapWithDirectory: boolean + pin: boolean + recursive: boolean + hidden: boolean + preload: boolean + timeout?: number + hamtHashFn: HamtHashFn + hamtBucketBits: number + hamtHashCode: number + chunker: ChunkerType | Chunker + dagBuilder?: DAGBuilder + treeBuilder?: TreeBuilder + bufferImporter?: BufferImporter + chunkValidator?: ChunkValidator +} + +export interface TrickleDagNode { + children: InProgressImportResult[], + depth: number, + maxDepth: number, + maxChildren: number, + data?: InProgressImportResult[], + parent?: TrickleDagNode + cid?: CID, + size?: number, + unixfs?: UnixFS +} + +export interface PersistOptions { + codec?: string + cidVersion: CIDVersion + hashAlg: HashName + onlyHash: boolean + preload?: boolean + timeout?: number + signal?: AbortSignal +} + +// TODO: remove this and get from core-ipfs-types +export interface BlockAPI { + get: (cid: CID | string | Uint8Array, options?: BlockOptions) => Promise + put: (block: Block | Uint8Array, options?: PutOptions) => Promise +} + +// TODO: remove this and get from core-ipfs-types +export interface BlockOptions { + signal?: AbortSignal + timeout?: number + preload?: boolean +} + +// TODO: remove this and get from core-ipfs-types +export interface PutOptions extends BlockOptions { + cid?: CID + format?: CodecName + mhtype?: HashName + version?: CIDVersion + pin?: boolean +} diff --git a/packages/ipfs-unixfs-importer/src/utils/persist.js b/packages/ipfs-unixfs-importer/src/utils/persist.js index f00d8832..a4aba3be 100644 --- a/packages/ipfs-unixfs-importer/src/utils/persist.js +++ b/packages/ipfs-unixfs-importer/src/utils/persist.js @@ -5,8 +5,8 @@ const CID = require('cids') /** * @param {Uint8Array} buffer - * @param {import('../').BlockAPI} block - * @param {import('../').ImporterOptions} options + * @param {import('../types').BlockAPI} block + * @param {import('../types').PersistOptions} options */ const persist = async (buffer, block, options) => { if (!options.codec) { @@ -29,7 +29,9 @@ const persist = async (buffer, block, options) => { const cid = new CID(options.cidVersion, options.codec, multihash) if (!options.onlyHash) { + // @ts-ignore block api takes uint8arrays or blocks but is missing from typedefs await block.put(buffer, { + // @ts-ignore pin option is missing from block api typedefs pin: options.pin, preload: options.preload, timeout: options.timeout, diff --git a/packages/ipfs-unixfs-importer/test/benchmark.spec.js b/packages/ipfs-unixfs-importer/test/benchmark.spec.js index a0cdcbca..73079e54 100644 --- a/packages/ipfs-unixfs-importer/test/benchmark.spec.js +++ b/packages/ipfs-unixfs-importer/test/benchmark.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const importer = require('../src') +const { importer } = require('../src') // @ts-ignore const IPLD = require('ipld') @@ -18,7 +18,7 @@ const CHUNK_SIZE = 65536 describe.skip('benchmark', function () { this.timeout(30 * 1000) - /** @type {import('./helpers/block').IPLD} */ + /** @type {import('ipld')} */ let ipld /** @type {import('../src').BlockAPI} */ let block diff --git a/packages/ipfs-unixfs-importer/test/builder-balanced.spec.js b/packages/ipfs-unixfs-importer/test/builder-balanced.spec.js index fe472b41..91d830b2 100644 --- a/packages/ipfs-unixfs-importer/test/builder-balanced.spec.js +++ b/packages/ipfs-unixfs-importer/test/builder-balanced.spec.js @@ -3,15 +3,14 @@ const { expect } = require('aegir/utils/chai') const builder = require('../src/dag-builder/file/balanced') -const all = require('it-all') const CID = require('cids') const defaultOptions = require('../src/options') /** - * @typedef {import('../src').PartialImportResult} PartialImportResult + * @typedef {import('../src/types').InProgressImportResult} InProgressImportResult * - * @param {PartialImportResult[]} leaves - * @returns {Promise} + * @param {InProgressImportResult[]} leaves + * @returns {Promise} */ async function reduce (leaves) { if (leaves.length > 1) { @@ -36,11 +35,11 @@ describe('builder: balanced', () => { size: 0 }] - const result = await all(builder((async function * () { + const result = await builder((async function * () { yield * source - }()), reduce, options)) + }()), reduce, options) - expect(result).to.deep.equal(source) + expect(result).to.deep.equal(source[0]) }) it('reduces 3 values into parent', async () => { @@ -55,41 +54,41 @@ describe('builder: balanced', () => { size: 0 }] - const result = await all(builder((async function * () { + const result = await builder((async function * () { yield * source - }()), reduce, options)) + }()), reduce, options) - expect(result).to.deep.equal([{ + expect(result).to.deep.equal({ children: source - }]) + }) }) it('obeys max children per node', async () => { const source = [1, 2, 3, 4] // @ts-ignore - const result = await all(builder((async function * () { + const result = await builder((async function * () { yield * source - }()), reduce, options)) + }()), reduce, options) - expect(result).to.deep.equal([{ + expect(result).to.deep.equal({ children: [{ children: [1, 2, 3] }, 4 ] - }]) + }) }) it('refolds 2 parent nodes', async () => { const source = [1, 2, 3, 4, 5, 6, 7] // @ts-ignore - const result = await all(builder((async function * () { + const result = await builder((async function * () { yield * source - }()), reduce, options)) + }()), reduce, options) - expect(result).to.deep.equal([{ + expect(result).to.deep.equal({ children: [{ children: [1, 2, 3] }, { @@ -97,6 +96,6 @@ describe('builder: balanced', () => { }, 7 ] - }]) + }) }) }) diff --git a/packages/ipfs-unixfs-importer/test/builder-flat.spec.js b/packages/ipfs-unixfs-importer/test/builder-flat.spec.js index 22f06715..cf0d73dc 100644 --- a/packages/ipfs-unixfs-importer/test/builder-flat.spec.js +++ b/packages/ipfs-unixfs-importer/test/builder-flat.spec.js @@ -3,7 +3,6 @@ const { expect } = require('aegir/utils/chai') const builder = require('../src/dag-builder/file/flat') -const all = require('it-all') /** * @param {*} leaves @@ -20,16 +19,16 @@ describe('builder: flat', () => { it('reduces one value into itself', async () => { const source = [1] // @ts-ignore - const result = await all(builder(source, reduce)) + const result = await builder(source, reduce) - expect(result).to.be.eql([1]) + expect(result).to.be.eql(1) }) it('reduces 2 values into parent', async () => { const source = [1, 2] // @ts-ignore - const result = await all(builder(source, reduce)) + const result = await builder(source, reduce) - expect(result).to.be.eql([{ children: [1, 2] }]) + expect(result).to.be.eql({ children: [1, 2] }) }) }) diff --git a/packages/ipfs-unixfs-importer/test/builder-only-hash.spec.js b/packages/ipfs-unixfs-importer/test/builder-only-hash.spec.js index 8a482e7a..5652ffbd 100644 --- a/packages/ipfs-unixfs-importer/test/builder-only-hash.spec.js +++ b/packages/ipfs-unixfs-importer/test/builder-only-hash.spec.js @@ -2,7 +2,6 @@ 'use strict' const { expect } = require('aegir/utils/chai') -// @ts-ignore const IPLD = require('ipld') // @ts-ignore const inMemory = require('ipld-in-memory') @@ -12,9 +11,9 @@ const blockApi = require('./helpers/block') const defaultOptions = require('../src/options') describe('builder: onlyHash', () => { - /** @type {import('./helpers/block').IPLD} */ + /** @type {IPLD} */ let ipld - /** @type {import('../src').BlockAPI} */ + /** @type {import('../src/types').BlockAPI} */ let block before(async () => { diff --git a/packages/ipfs-unixfs-importer/test/builder-trickle-dag.spec.js b/packages/ipfs-unixfs-importer/test/builder-trickle-dag.spec.js index ed748171..64c27d50 100644 --- a/packages/ipfs-unixfs-importer/test/builder-trickle-dag.spec.js +++ b/packages/ipfs-unixfs-importer/test/builder-trickle-dag.spec.js @@ -3,7 +3,6 @@ const { expect } = require('aegir/utils/chai') const builder = require('../src/dag-builder/file/trickle') -const all = require('it-all') const asAsyncIterable = require('./helpers/as-async-iterable') /** @@ -38,29 +37,29 @@ const options = { describe('builder: trickle', () => { it('reduces one value into itself', async () => { // @ts-ignore - const result = await all(builder(asAsyncIterable([1]), reduce, options)) + const result = await builder(asAsyncIterable([1]), reduce, options) - expect(result).to.deep.equal([1]) + expect(result).to.deep.equal(1) }) it('reduces 3 values into parent', async () => { // @ts-ignore - const result = await all(builder(createValues(3), reduce, options)) + const result = await builder(createValues(3), reduce, options) - expect(result).to.deep.equal([{ + expect(result).to.deep.equal({ children: [ 0, 1, 2 ] - }]) + }) }) it('reduces 6 values correctly', async () => { // @ts-ignore - const result = await all(builder(createValues(6), reduce, options)) + const result = await builder(createValues(6), reduce, options) - expect(result).to.deep.equal([{ + expect(result).to.deep.equal({ children: [ 0, 1, @@ -73,14 +72,14 @@ describe('builder: trickle', () => { ] } ] - }]) + }) }) it('reduces 9 values correctly', async () => { // @ts-ignore - const result = await all(builder(createValues(9), reduce, options)) + const result = await builder(createValues(9), reduce, options) - expect(result).to.deep.equal([{ + expect(result).to.deep.equal({ children: [ 0, 1, @@ -100,14 +99,14 @@ describe('builder: trickle', () => { ] } ] - }]) + }) }) it('reduces 12 values correctly', async () => { // @ts-ignore - const result = await all(builder(createValues(12), reduce, options)) + const result = await builder(createValues(12), reduce, options) - expect(result).to.deep.equal([{ + expect(result).to.deep.equal({ children: [ 0, 1, @@ -134,14 +133,14 @@ describe('builder: trickle', () => { ] } ] - }]) + }) }) it('reduces 21 values correctly', async () => { // @ts-ignore - const result = await all(builder(createValues(21), reduce, options)) + const result = await builder(createValues(21), reduce, options) - expect(result).to.deep.equal([{ + expect(result).to.deep.equal({ children: [ 0, 1, @@ -189,14 +188,14 @@ describe('builder: trickle', () => { ] } ] - }]) + }) }) it('reduces 68 values correctly', async () => { // @ts-ignore - const result = await all(builder(createValues(68), reduce, options)) + const result = await builder(createValues(68), reduce, options) - expect(result).to.deep.equal([ + expect(result).to.deep.equal( { children: [ 0, @@ -357,14 +356,14 @@ describe('builder: trickle', () => { } ] } - ]) + ) }) it('reduces 93 values correctly', async () => { // @ts-ignore - const result = await all(builder(createValues(93), reduce, options)) + const result = await builder(createValues(93), reduce, options) - expect(result).to.deep.equal([ + expect(result).to.deep.equal( { children: [ 0, @@ -582,6 +581,6 @@ describe('builder: trickle', () => { } ] } - ]) + ) }) }) diff --git a/packages/ipfs-unixfs-importer/test/builder.spec.js b/packages/ipfs-unixfs-importer/test/builder.spec.js index b26a94ab..042ba6a5 100644 --- a/packages/ipfs-unixfs-importer/test/builder.spec.js +++ b/packages/ipfs-unixfs-importer/test/builder.spec.js @@ -7,7 +7,7 @@ const mh = require('multihashing-async').multihash const IPLD = require('ipld') // @ts-ignore const inMemory = require('ipld-in-memory') -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const builder = require('../src/dag-builder') const first = require('it-first') const blockApi = require('./helpers/block') @@ -15,7 +15,7 @@ const uint8ArrayFromString = require('uint8arrays/from-string') const defaultOptions = require('../src/options') describe('builder', () => { - /** @type {import('./helpers/block').IPLD} */ + /** @type {import('ipld')} */ let ipld /** @type {import('../src').BlockAPI} */ let block diff --git a/packages/ipfs-unixfs-importer/test/chunker-custom.spec.js b/packages/ipfs-unixfs-importer/test/chunker-custom.spec.js index 78256a39..ebb7a796 100644 --- a/packages/ipfs-unixfs-importer/test/chunker-custom.spec.js +++ b/packages/ipfs-unixfs-importer/test/chunker-custom.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const importer = require('../src') +const { importer } = require('../src') const { expect } = require('aegir/utils/chai') // @ts-ignore const IPLD = require('ipld') @@ -10,7 +10,7 @@ const inMemory = require('ipld-in-memory') const mc = require('multicodec') const blockApi = require('./helpers/block') const uint8ArrayFromString = require('uint8arrays/from-string') -const UnixFS = require('ipfs-unixfs') +const { UnixFS } = require('ipfs-unixfs') const iter = async function * () { yield uint8ArrayFromString('one') @@ -18,7 +18,7 @@ const iter = async function * () { } describe('custom chunker', function () { - /** @type {import('./helpers/block').IPLD} */ + /** @type {import('ipld')} */ let ipld /** @type {import('../src').BlockAPI} */ let block diff --git a/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.js b/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.js index b0e25866..1e996d32 100644 --- a/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.js +++ b/packages/ipfs-unixfs-importer/test/hash-parity-with-go-ipfs.spec.js @@ -1,7 +1,7 @@ /* eslint-env mocha */ 'use strict' -const importer = require('../src') +const { importer } = require('../src') const { expect } = require('aegir/utils/chai') // @ts-ignore @@ -39,7 +39,7 @@ strategies.forEach(strategy => { } describe('go-ipfs interop using importer:' + strategy, () => { - /** @type {import('./helpers/block').IPLD} */ + /** @type {import('ipld')} */ let ipld /** @type {import('../src').BlockAPI} */ let block diff --git a/packages/ipfs-unixfs-importer/test/helpers/block.js b/packages/ipfs-unixfs-importer/test/helpers/block.js index 99bf1b0b..330d8a14 100644 --- a/packages/ipfs-unixfs-importer/test/helpers/block.js +++ b/packages/ipfs-unixfs-importer/test/helpers/block.js @@ -6,26 +6,30 @@ const { } = require('ipld-dag-pb') const multicodec = require('multicodec') const mh = require('multihashing-async').multihash +const CID = require('cids') +const Block = require('ipld-block') /** - * @typedef {import('cids')} CID - * @typedef {object} IPLD - * @property {(cid: CID, options?: any) => Promise} get - * @property {(node: any, codec: number, options?: any) => Promise} put - */ - -/** - * - * @param {IPLD} ipld + * @param {import('ipld')} ipld */ function createBlockApi (ipld) { // make ipld behave like the block api, some tests need to pull // data from ipld so can't use a simple in-memory cid->block map - /** @type {import('../../src').BlockAPI} */ + /** @type {import('../../src/types').BlockAPI} */ const BlockApi = { - put: async (buf, { cid }) => { + put: async (buf, options) => { + if (!options || !options.cid) { + throw new Error('No cid passed') + } + + const cid = new CID(options.cid) + const multihash = mh.decode(cid.multihash) + if (Block.isBlock(buf)) { + buf = buf.data + } + /** @type {any} */ let obj = buf @@ -38,9 +42,11 @@ function createBlockApi (ipld) { hashAlg: multihash.code }) - return { cid, data: buf } + return new Block(buf, cid) }, get: async (cid, options) => { + cid = new CID(cid) + /** @type {Uint8Array} */ let buf = await ipld.get(cid, options) @@ -48,7 +54,7 @@ function createBlockApi (ipld) { buf = buf.serialize() } - return { cid, data: buf } + return new Block(buf, cid) } } diff --git a/packages/ipfs-unixfs-importer/tsconfig.json b/packages/ipfs-unixfs-importer/tsconfig.json index c5c7aae8..3c4a52f9 100644 --- a/packages/ipfs-unixfs-importer/tsconfig.json +++ b/packages/ipfs-unixfs-importer/tsconfig.json @@ -1,7 +1,8 @@ { - "extends": "aegir/src/config/tsconfig.aegir.json", + "extends": "../../node_modules/aegir/src/config/tsconfig.aegir.json", "compilerOptions": { - "outDir": "dist" + "outDir": "dist", + "importsNotUsedAsValues": "preserve" }, "include": [ "src", diff --git a/packages/ipfs-unixfs/.npmignore b/packages/ipfs-unixfs/.npmignore deleted file mode 100644 index f9a85ba9..00000000 --- a/packages/ipfs-unixfs/.npmignore +++ /dev/null @@ -1,3 +0,0 @@ -docs -test -.aegir.js diff --git a/packages/ipfs-unixfs/README.md b/packages/ipfs-unixfs/README.md index 132e4d93..d7ca41e3 100644 --- a/packages/ipfs-unixfs/README.md +++ b/packages/ipfs-unixfs/README.md @@ -47,7 +47,7 @@ The UnixFS spec can be found inside the [ipfs/specs repository](http://github.co ### Use in Node.js ```JavaScript -var UnixFS = require('ipfs-unixfs') +var { UnixFS } = require('ipfs-unixfs') ``` ### Use in a browser with browserify, webpack or any other bundler @@ -55,7 +55,7 @@ var UnixFS = require('ipfs-unixfs') The code published to npm that gets loaded on require is in fact a ES5 transpiled version with the right shims added. This means that you can require it and use with your favourite bundler without having to adjust asset management process. ```JavaScript -var UnixFS = require('ipfs-unixfs') +var { UnixFS } = require('ipfs-unixfs') ``` ### Use in a browser Using a script tag diff --git a/packages/ipfs-unixfs/package.json b/packages/ipfs-unixfs/package.json index 71f6e058..c0ccc90c 100644 --- a/packages/ipfs-unixfs/package.json +++ b/packages/ipfs-unixfs/package.json @@ -10,14 +10,14 @@ "scripts": { "prepare": "run-s prepare:*", "prepare:proto": "pbjs -t static-module -w commonjs --force-number --no-verify --no-delimited --no-create --no-beautify --no-defaults --lint eslint-disable -o src/unixfs.js ./src/unixfs.proto", - "prepare:proto:types": "pbts -o src/unixfs.d.ts src/unixfs.js", + "prepare:proto-types": "pbts -o src/unixfs.d.ts src/unixfs.js", "prepare:types": "aegir build --no-bundle", - "prepare:copy": "copy ./src/*.d.ts ./dist/src", "test": "aegir test", + "build": "aegir build", "clean": "rimraf ./dist", - "lint": "aegir lint", + "lint": "aegir ts --check && aegir lint", "coverage": "nyc -s aegir test -t node && nyc report --reporter=html", - "depcheck": "aegir dep-check -i mkdirp -i @types/mocha -i nyc -i npm-run-all -i copy" + "depcheck": "aegir dep-check -i mkdirp -i @types/mocha -i nyc -i npm-run-all -i copy -i util" }, "repository": { "type": "git", @@ -37,12 +37,13 @@ "homepage": "https://github.com/ipfs/js-ipfs-unixfs#readme", "devDependencies": { "@types/mocha": "^8.2.1", - "aegir": "^30.3.0", + "aegir": "^32.0.0", "copy": "^0.3.2", "mkdirp": "^1.0.4", "npm-run-all": "^4.1.5", "nyc": "^15.0.0", - "uint8arrays": "^2.1.2" + "uint8arrays": "^2.1.2", + "util": "^0.12.3" }, "dependencies": { "err-code": "^3.0.1", diff --git a/packages/ipfs-unixfs/src/index.js b/packages/ipfs-unixfs/src/index.js index fb85f61b..10bd47c9 100644 --- a/packages/ipfs-unixfs/src/index.js +++ b/packages/ipfs-unixfs/src/index.js @@ -11,6 +11,10 @@ const errcode = require('err-code') * @property {number | null} [nsecs] */ +/** + * @typedef {null | undefined | { secs: number, nsecs?: number} | { Seconds: number, FractionalNanoseconds?: number} | Mtime | [number, number] | Date} MtimeLike + */ + const types = [ 'raw', 'directory', @@ -44,7 +48,7 @@ function parseMode (mode) { } /** - * @param {null | undefined | { secs: number, nsecs?: number} | { Seconds: number, FractionalNanoseconds?: number} | Mtime | [number, number] | Date} mtime + * @param {MtimeLike} mtime * @returns {Mtime | undefined} */ function parseMtime (mtime) { @@ -161,7 +165,7 @@ class Data { * @param {number[]} [options.blockSizes] * @param {number} [options.hashType] * @param {number} [options.fanout] - * @param {Mtime | Date | null} [options.mtime] + * @param {MtimeLike | null} [options.mtime] * @param {number | string} [options.mode] */ constructor (options = { @@ -328,4 +332,8 @@ class Data { } } -module.exports = Data +module.exports = { + UnixFS: Data, + parseMode, + parseMtime +} diff --git a/packages/ipfs-unixfs/test/unixfs-format.spec.js b/packages/ipfs-unixfs/test/unixfs-format.spec.js index 5e05432a..deefcfd6 100644 --- a/packages/ipfs-unixfs/test/unixfs-format.spec.js +++ b/packages/ipfs-unixfs/test/unixfs-format.spec.js @@ -5,9 +5,9 @@ const { expect } = require('aegir/utils/chai') /** @type {(path: string) => Uint8Array} */ // @ts-ignore -const loadFixture = require('aegir/fixtures') +const loadFixture = require('aegir/utils/fixtures') -const UnixFS = require('../src') +const { UnixFS } = require('../src') const raw = loadFixture('test/fixtures/raw.unixfs') const directory = loadFixture('test/fixtures/directory.unixfs') diff --git a/packages/ipfs-unixfs/tsconfig.json b/packages/ipfs-unixfs/tsconfig.json index 1f8fd1e4..f121edfe 100644 --- a/packages/ipfs-unixfs/tsconfig.json +++ b/packages/ipfs-unixfs/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "aegir/src/config/tsconfig.aegir.json", + "extends": "../../node_modules/aegir/src/config/tsconfig.aegir.json", "compilerOptions": { "outDir": "dist" }, @@ -10,6 +10,6 @@ "exclude": [ "dist", "node_modules", - "src/unixfs.js" // exclude generated file + "src/unixfs.js" ] }