chore: declare interface types in .d.ts file (#122)

If we don't declare ts interfaces in a .d.ts file they get duplicated in our generated .d.ts files.

We should pull in types from `ipfs-core-types` for the block api but it's yet to be fully typed so this is draft PR that can be finished after the IPFS top level types have improved.

- Moves all types into `.d.ts` files so they do not get duplicated in generated types
- Switches to named exports
- Loosens mtime definition for unixfs
- Exports utility functions for converting mtimes and modes
- Runs ts check during linting

BREAKING CHANGE: switches to named exports

.travis.yml

@@ -46,7 +46,6 @@ jobs:
  - stage: check
  name: linting
  script:
- - npm --version
  - npm run lint
 
  - stage: check

package.json

@@ -3,7 +3,6 @@
  "version": "1.0.0",
  "description": "JS implementation of the IPFS UnixFS",
  "scripts": {
- "postinstall": "rm -rf package-lock.json packages/*/package-lock.json",
  "reset": "lerna run clean && rimraf packages/*/node_modules node_modules",
  "test": "lerna run test",
  "coverage": "lerna run coverage",
@@ -17,7 +16,6 @@
  "update-contributors": "aegir release --lint=false --test=false --bump=false --build=false --changelog=false --commit=false --tag=false --push=false --ghrelease=false --docs=false --publish=false"
  },
  "devDependencies": {
- "aegir": "^30.3.0",
  "lerna": "^3.22.1"
  },
  "repository": {

packages/ipfs-unixfs-exporter/.aegir.js

@@ -1,16 +1,35 @@
 'use strict'
 
+/** @type {import('aegir').Options["build"]["config"]} */
+const buildConfig = {
+ plugins: [
+ {
+ name: 'node built ins',
+ setup (build) {
+ build.onResolve({ filter: /^stream$/ }, () => {
+ return { path: require.resolve('readable-stream') }
+ })
+ build.onResolve({ filter: /^crypto$/ }, () => {
+ return { path: require.resolve('crypto-browserify') }
+ })
+ build.onResolve({ filter: /^cborg$/ }, () => {
+ return { path: require.resolve('cborg') }
+ })
+ }
+ }
+ ]
+}
+
+/** @type {import('aegir').PartialOptions} */
 module.exports = {
- karma: {
- browserNoActivityTimeout: 1000 * 1000,
+ build: {
+ config: buildConfig
  },
- webpack: {
- node: {
- // needed by the cbor module
- stream: true,
-
- // needed by the core-util-is module
- Buffer: true
+ test: {
+ browser: {
+ config: {
+ buildConfig
+ }
  }
  }
 }

packages/ipfs-unixfs-exporter/README.md

@@ -26,8 +26,8 @@
  - [Raw entries](#raw-entries)
  - [CBOR entries](#cbor-entries)
  - [`entry.content({ offset, length })`](#entrycontent-offset-length-)
- - [`exporter.path(cid, ipld)`](#exporterpathcid-ipld)
- - [`exporter.recursive(cid, ipld)`](#exporterrecursivecid-ipld)
+ - [`walkPath(cid, ipld)`](#walkpathcid-ipld)
+ - [`recursive(cid, ipld)`](#recursivecid-ipld)
  - [Contribute](#contribute)
  - [License](#license)
 
@@ -43,8 +43,8 @@
 
 ```js
 // import a file and export it again
-const importer = require('ipfs-unixfs-importer')
-const exporter = require('ipfs-unixfs-exporter')
+const { importer } = require('ipfs-unixfs-importer')
+const { exporter } = require('ipfs-unixfs-exporter')
 
 const files = []
 
@@ -80,7 +80,7 @@ console.info(bytes) // 0, 1, 2, 3
 #### API
 
 ```js
-const exporter = require('ipfs-unixfs-exporter')
+const { exporter } = require('ipfs-unixfs-exporter')
 ```
 
 ### `exporter(cid, ipld, options)`
@@ -202,28 +202,32 @@ for await (const entry of dir.content({
 // `entries` contains the first 5 files/directories in the directory
 ```
 
-### `exporter.path(cid, ipld)`
+### `walkPath(cid, ipld)`
 
-`exporter.path` will return an async iterator that yields entries for all segments in a path:
+`walkPath` will return an async iterator that yields entries for all segments in a path:
 
 ```javascript
+const { walkPath } = require('ipfs-unixfs-exporter')
+
 const entries = []
 
-for await (const entry of exporter.path('Qmfoo/foo/bar/baz.txt', ipld)) {
+for await (const entry of walkPath('Qmfoo/foo/bar/baz.txt', ipld)) {
  entries.push(entry)
 }
 
 // entries contains 4x `entry` objects
 ```
 
-### `exporter.recursive(cid, ipld)`
+### `recursive(cid, ipld)`
 
-`exporter.recursive` will return an async iterator that yields all entries beneath a given CID or IPFS path, as well as the containing directory.
+`recursive` will return an async iterator that yields all entries beneath a given CID or IPFS path, as well as the containing directory.
 
 ```javascript
+const { recursive } = require('ipfs-unixfs-exporter')
+
 const entries = []
 
-for await (const child of exporter.recursive('Qmfoo/foo/bar', ipld)) {
+for await (const child of recursive('Qmfoo/foo/bar', ipld)) {
  entries.push(entry)
 }
 

packages/ipfs-unixfs-exporter/package.json

@@ -10,10 +10,11 @@
  "scripts": {
  "prepare": "aegir build --no-bundle",
  "test": "aegir test",
+ "build": "aegir build",
  "clean": "rimraf ./dist",
- "lint": "aegir lint",
+ "lint": "aegir ts --check && aegir lint",
  "coverage": "nyc -s npm run test -t node && nyc report --reporter=html",
- "depcheck": "aegir dep-check -i @types/mocha -i @types/sinon -i nyc -i abort-controller -i rimraf -i ipfs-core-types"
+ "depcheck": "aegir dep-check -i @types/mocha -i @types/sinon -i nyc -i abort-controller -i rimraf -i ipfs-core-types -i copy -i util -i crypto-browserify -i events -i readable-stream"
  },
  "repository": {
  "type": "git",
@@ -35,23 +36,29 @@
  "@types/mocha": "^8.2.1",
  "@types/sinon": "^9.0.10",
  "abort-controller": "^3.0.0",
- "aegir": "^30.3.0",
+ "aegir": "^32.0.0",
+ "copy": "^0.3.2",
+ "crypto-browserify": "^3.12.0",
  "detect-node": "^2.0.4",
- "ipfs-core-types": "^0.3.0",
+ "events": "^3.3.0",
+ "ipfs-core-types": "^0.3.1",
  "ipfs-unixfs-importer": "^6.0.1",
- "ipld": "^0.28.0",
- "ipld-dag-pb": "^0.21.0",
- "ipld-in-memory": "^7.0.0",
+ "ipld": "^0.29.0",
+ "ipld-block": "^0.11.1",
+ "ipld-dag-pb": "^0.22.1",
+ "ipld-in-memory": "^8.0.0",
  "it-all": "^1.0.5",
  "it-buffer-stream": "^2.0.0",
  "it-first": "^1.0.6",
  "merge-options": "^3.0.4",
- "multicodec": "^2.1.0",
+ "multicodec": "^3.0.1",
  "native-abort-controller": "^1.0.3",
  "nyc": "^15.0.0",
+ "readable-stream": "^3.6.0",
  "rimraf": "^3.0.2",
  "sinon": "^9.2.4",
- "uint8arrays": "^2.1.2"
+ "uint8arrays": "^2.1.2",
+ "util": "^0.12.3"
  },
  "dependencies": {
  "cids": "^1.1.5",

packages/ipfs-unixfs-exporter/src/index.js

@@ -6,66 +6,16 @@ const resolve = require('./resolvers')
 const last = require('it-last')
 
 /**
- * @typedef {import('ipfs-unixfs')} UnixFS
+ * @typedef {import('ipfs-unixfs').UnixFS} UnixFS
  * @typedef {import('ipld-dag-pb').DAGNode} DAGNode
- * @typedef {import('ipfs-core-types/src/ipld').IPLD} IPLD
- *
- * @typedef {object} UnixFSFile
- * @property {'file'} type
- * @property {string} name
- * @property {string} path
- * @property {CID} cid
- * @property {number} depth
- * @property {UnixFS} unixfs
- * @property {DAGNode} node
- * @property {(options?: ExporterOptions) => AsyncIterable<Uint8Array>} content
- *
- * @typedef {object} UnixFSDirectory
- * @property {'directory'} type
- * @property {string} name
- * @property {string} path
- * @property {CID} cid
- * @property {number} depth
- * @property {UnixFS} unixfs
- * @property {DAGNode} node
- * @property {(options?: ExporterOptions) => AsyncIterable<UnixFSEntry>} content
- *
- * @typedef {object} ObjectNode
- * @property {'object'} type
- * @property {string} name
- * @property {string} path
- * @property {CID} cid
- * @property {number} depth
- * @property {Uint8Array} node
- * @property {(options?: ExporterOptions) => AsyncIterable<any>} content
- *
- * @typedef {object} RawNode
- * @property {'raw'} type
- * @property {string} name
- * @property {string} path
- * @property {CID} cid
- * @property {number} depth
- * @property {Uint8Array} node
- * @property {(options?: ExporterOptions) => AsyncIterable<Uint8Array>} content
- *
- * @typedef {object} IdentityNode
- * @property {'identity'} type
- * @property {string} name
- * @property {string} path
- * @property {CID} cid
- * @property {number} depth
- * @property {Uint8Array} node
- * @property {(options?: ExporterOptions) => AsyncIterable<Uint8Array>} content
- *
- * @typedef {UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode} UnixFSEntry
- */
-
-/**
- * @typedef {object} ExporterOptions
- * @property {number} [offset=0]
- * @property {number} [length]
- * @property {AbortSignal} [signal]
- * @property {number} [timeout]
+ * @typedef {import('ipld')} IPLD
+ * @typedef {import('./types').ExporterOptions} ExporterOptions
+ * @typedef {import('./types').UnixFSFile} UnixFSFile
+ * @typedef {import('./types').UnixFSDirectory} UnixFSDirectory
+ * @typedef {import('./types').ObjectNode} ObjectNode
+ * @typedef {import('./types').RawNode} RawNode
+ * @typedef {import('./types').IdentityNode} IdentityNode
+ * @typedef {import('./types').UnixFSEntry} UnixFSEntry
  */
 
 const toPathComponents = (path = '') => {
@@ -115,7 +65,7 @@ const cidAndRest = (path) => {
  * @param {IPLD} ipld
  * @param {ExporterOptions} [options]
  */
-const walkPath = async function * (path, ipld, options = {}) {
+async function * walkPath (path, ipld, options = {}) {
  let {
  cid,
  toResolve
@@ -152,7 +102,7 @@ const walkPath = async function * (path, ipld, options = {}) {
  * @param {IPLD} ipld
  * @param {ExporterOptions} [options]
  */
-const exporter = async (path, ipld, options = {}) => {
+async function exporter (path, ipld, options = {}) {
  const result = await last(walkPath(path, ipld, options))
 
  if (!result) {
@@ -167,7 +117,7 @@ const exporter = async (path, ipld, options = {}) => {
  * @param {IPLD} ipld
  * @param {ExporterOptions} [options]
  */
-const recursive = async function * (path, ipld, options = {}) {
+async function * recursive (path, ipld, options = {}) {
  const node = await exporter(path, ipld, options)
 
  if (!node) {
@@ -202,7 +152,8 @@ const recursive = async function * (path, ipld, options = {}) {
  }
 }
 
-exporter.path = walkPath
-exporter.recursive = recursive
-
-module.exports = exporter
+module.exports = {
+ exporter,
+ walkPath,
+ recursive
+}

packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js

@@ -4,7 +4,11 @@ const CID = require('cids')
 const errCode = require('err-code')
 
 /**
- * @type {import('./').Resolver}
+ * @typedef {import('../types').Resolver} Resolver
+ */
+
+/**
+ * @type {Resolver}
  */
 const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
  const object = await ipld.get(cid, options)
@@ -29,6 +33,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
  cid,
  node: block,
  depth,
+ size: block.length,
  content: async function * () {
  yield object
  }
@@ -57,6 +62,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
  cid,
  node: block,
  depth,
+ size: block.length,
  content: async function * () {
  yield object
  }

packages/ipfs-unixfs-exporter/src/resolvers/identity.js

@@ -6,7 +6,8 @@ const validateOffsetAndLength = require('../utils/validate-offset-and-length')
 const mh = require('multihashing-async').multihash
 
 /**
- * @typedef {import('../').ExporterOptions} ExporterOptions
+ * @typedef {import('../types').ExporterOptions} ExporterOptions
+ * @typedef {import('../types').Resolver} Resolver
  */
 
 /**
@@ -29,7 +30,7 @@ const rawContent = (node) => {
 }
 
 /**
- * @type {import('./').Resolver}
+ * @type {Resolver}
  */
 const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
  if (toResolve.length) {
@@ -46,6 +47,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
  cid,
  content: rawContent(buf.digest),
  depth,
+ size: buf.length,
  node: buf.digest
  }
  }