diff --git a/README.md b/README.md index 21591ba..8c72f5a 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,44 @@ If you desire normalization for keys and values (e.g. to stringify numbers), wra Another reason you might want to use `encoding-down` is that the structured clone algorithm, while rich in types, can be slower than `JSON.stringify`. +### Sort Order + +Unless `level-js` is wrapped with [`encoding-down`][encoding-down], IndexedDB will sort your keys in the following order: + +1. number (numeric) +2. date (numeric, by epoch offset) +3. binary (bitwise) +4. string (lexicographic) +5. array (componentwise). + +You can take advantage of this fact with `levelup` streams. For example, if your keys are dates, you can select everything greater than a specific date (let's be happy and ignore timezones for a moment): + +```js +const db = levelup(leveljs('time-db')) + +db.createReadStream({ gt: new Date('2019-01-01') }) + .pipe(..) +``` + +Or if your keys are arrays, you can do things like: + +```js +const db = levelup(leveljs('books-db')) + +await db.put(['Roald Dahl', 'Charlie and the Chocolate Factory'], {}) +await db.put(['Roald Dahl', 'Fantastic Mr Fox'], {}) + +// Select all books by Roald Dahl +db.createReadStream({ gt: ['Roald Dahl'], lt: ['Roald Dahl', '\xff'] }) + .pipe(..) +``` + +To achieve this on other `abstract-leveldown` implementations, wrap them with [`encoding-down`][encoding-down] and [`charwise`][charwise] (or similar). + +#### Known Browser Issues + +IE11 and Edge yield incorrect results for `{ gte: '' }` if the database contains any key types other than strings. + ### Buffer vs ArrayBuffer For interoperability it is recommended to use `Buffer` as your binary type. While we recognize that Node.js core modules are moving towards supporting `ArrayBuffer` and views thereof, `Buffer` remains the primary binary type in the Level ecosystem. @@ -225,6 +263,8 @@ See the [contribution guide](https://github.com/Level/community/blob/master/CONT [abstract-leveldown]: https://github.com/Level/abstract-leveldown +[charwise]: https://github.com/dominictarr/charwise + [levelup]: https://github.com/Level/levelup [leveldown]: https://github.com/Level/leveldown diff --git a/iterator.js b/iterator.js index 74eb017..0f64738 100644 --- a/iterator.js +++ b/iterator.js @@ -51,12 +51,6 @@ Iterator.prototype.createKeyRange = function (options) { var lowerOpen = ltgt.lowerBoundExclusive(options) var upperOpen = ltgt.upperBoundExclusive(options) - // Temporary workaround for Level/abstract-leveldown#318 - if ((Buffer.isBuffer(lower) || typeof lower === 'string') && lower.length === 0) lower = undefined - if ((Buffer.isBuffer(upper) || typeof upper === 'string') && upper.length === 0) upper = undefined - if ((Buffer.isBuffer(lowerOpen) || typeof lowerOpen === 'string') && lowerOpen.length === 0) lowerOpen = undefined - if ((Buffer.isBuffer(upperOpen) || typeof upperOpen === 'string') && upperOpen.length === 0) upperOpen = undefined - if (lower !== undefined && upper !== undefined) { return IDBKeyRange.bound(lower, upper, lowerOpen, upperOpen) } else if (lower !== undefined) { diff --git a/package.json b/package.json index 561d890..ef72584 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ ] }, "dependencies": { - "abstract-leveldown": "~6.0.0", + "abstract-leveldown": "~6.0.1", "immediate": "~3.2.3", "inherits": "^2.0.3", "ltgt": "^2.1.2", diff --git a/test/index.js b/test/index.js index cb0a6d6..a7ea5dd 100644 --- a/test/index.js +++ b/test/index.js @@ -34,3 +34,4 @@ require('./custom-test')(leveljs, test, testCommon) require('./structured-clone-test')(leveljs, test, testCommon) require('./key-type-test')(leveljs, test, testCommon) require('./key-type-illegal-test')(leveljs, test, testCommon) +require('./native-order-test')(leveljs, test, testCommon) diff --git a/test/native-order-test.js b/test/native-order-test.js new file mode 100644 index 0000000..96446cf --- /dev/null +++ b/test/native-order-test.js @@ -0,0 +1,185 @@ +'use strict' + +var concat = require('level-concat-iterator') + +module.exports = function (leveljs, test, testCommon) { + // Type sort order per IndexedDB Second Edition, excluding + // types that aren't supported by all environments. + var basicKeys = [ + // Should sort naturally + { type: 'number', value: '-Infinity', key: -Infinity }, + { type: 'number', value: '2', key: 2 }, + { type: 'number', value: '10', key: 10 }, + { type: 'number', value: '+Infinity', key: Infinity }, + + // Should sort naturally (by epoch offset) + { type: 'date', value: 'new Date(2)', key: new Date(2) }, + { type: 'date', value: 'new Date(10)', key: new Date(10) }, + + // Should sort lexicographically + { type: 'string', value: '"10"', key: '10' }, + { type: 'string', value: '"2"', key: '2' } + ] + + makeTest('on basic key types', basicKeys, function (verify) { + // Should be ignored + verify({ gt: undefined }) + verify({ gte: undefined }) + verify({ lt: undefined }) + verify({ lte: undefined }) + + verify({ gt: -Infinity }, 1) + verify({ gte: -Infinity }) + verify({ gt: +Infinity }, 4) + verify({ gte: +Infinity }, 3) + + verify({ lt: -Infinity }, 0, 0) + verify({ lte: -Infinity }, 0, 1) + verify({ lt: +Infinity }, 0, 3) + verify({ lte: +Infinity }, 0, 4) + + verify({ gt: 10 }, 3) + verify({ gte: 10 }, 2) + verify({ lt: 10 }, 0, 2) + verify({ lte: 10 }, 0, 3) + + verify({ gt: new Date(10) }, 6) + verify({ gte: new Date(10) }, 5) + verify({ lt: new Date(10) }, 0, 5) + verify({ lte: new Date(10) }, 0, 6) + + // IE 11 and Edge fail this test (yield 0 results), but only when the db + // contains key types other than strings (see strings-only test below). + // verify({ gte: '' }, 6) + + verify({ gt: '' }, 6) + verify({ lt: '' }, 0, 6) + verify({ lte: '' }, 0, 6) + + verify({ gt: '10' }, 7) + verify({ gte: '10' }, 6) + verify({ lt: '10' }, 0, 6) + verify({ lte: '10' }, 0, 7) + + verify({ gt: '2' }, 0, 0) + verify({ gte: '2' }, -1) + verify({ lt: '2' }, 0, -1) + verify({ lte: '2' }) + }) + + makeTest('on string keys only', basicKeys.filter(matchType('string')), function (verify) { + verify({ gt: '' }) + verify({ gte: '' }) + verify({ lt: '' }, 0, 0) + verify({ lte: '' }, 0, 0) + }) + + if (leveljs.binaryKeys) { + var binaryKeys = [ + // Should sort bitwise + { type: 'binary', value: 'Uint8Array.from([0, 2])', key: binary([0, 2]) }, + { type: 'binary', value: 'Uint8Array.from([1, 1])', key: binary([1, 1]) } + ] + + makeTest('on binary keys', basicKeys.concat(binaryKeys), function (verify) { + verify({ gt: binary([]) }, -2) + verify({ gte: binary([]) }, -2) + verify({ lt: binary([]) }, 0, -2) + verify({ lte: binary([]) }, 0, -2) + }) + } + + if (leveljs.arrayKeys) { + var arrayKeys = [ + // Should sort componentwise + { type: 'array', value: '[100]', key: [100] }, + { type: 'array', value: '["10"]', key: ['10'] }, + { type: 'array', value: '["2"]', key: ['2'] } + ] + + makeTest('on array keys', basicKeys.concat(arrayKeys), function (verify) { + verify({ gt: [] }, -3) + verify({ gte: [] }, -3) + verify({ lt: [] }, 0, -3) + verify({ lte: [] }, 0, -3) + }) + } + + if (leveljs.binaryKeys && leveljs.arrayKeys) { + makeTest('on all key types', basicKeys.concat(binaryKeys).concat(arrayKeys)) + } + + function makeTest (name, input, fn) { + var prefix = 'native order (' + name + '): ' + var db + + test(prefix + 'open', function (t) { + db = testCommon.factory() + db.open(t.end.bind(t)) + }) + + test(prefix + 'prepare', function (t) { + db.batch(input.map(function (item) { + return { type: 'put', key: item.key, value: item.value } + }), t.end.bind(t)) + }) + + function verify (options, begin, end) { + test(prefix + humanRange(options), function (t) { + t.plan(2) + + options.valueAsBuffer = false + concat(db.iterator(options), function (err, result) { + t.ifError(err, 'no concat error') + t.same(result.map(getValue), input.slice(begin, end).map(getValue)) + }) + }) + } + + verify({}) + if (fn) fn(verify) + + test(prefix + 'close', function (t) { + db.close(t.end.bind(t)) + }) + } +} + +function matchType (type) { + return function (item) { + return item.type === type + } +} + +function getValue (kv) { + return kv.value +} + +// Replacement for TypedArray.from() +function binary (bytes) { + var arr = new Uint8Array(bytes.length) + for (var i = 0; i < bytes.length; i++) arr[i] = bytes[i] + return arr +} + +function humanRange (options) { + var a = [] + + ;['gt', 'gte', 'lt', 'lte'].forEach(function (opt) { + if (options.hasOwnProperty(opt)) { + var target = options[opt] + + if (typeof target === 'string' || Array.isArray(target)) { + target = JSON.stringify(target) + } else if (Object.prototype.toString.call(target) === '[object Date]') { + target = 'new Date(' + target.valueOf() + ')' + } else if (Object.prototype.toString.call(target) === '[object Uint8Array]') { + target = 'Uint8Array.from([' + target + '])' + } + + a.push(opt + ': ' + target) + } + }) + + return a.length ? a.join(', ') : 'all' +}