feat: Use a proper tag for !!merge << keys (#580)

docs/06_custom_tags.md

@@ -60,15 +60,16 @@ If including more than one custom tag from this set, make sure that the `'float'
 
 These tags are a part of the YAML 1.1 [language-independent types](https://yaml.org/type/), but are not a part of any default YAML 1.2 schema.
 
-| Identifier | YAML Type | JS Type | Description |
-| ------------- | ----------------------------------------------------- | ------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `'binary'` | [`!!binary`](https://yaml.org/type/binary.html) | `Uint8Array` | Binary data, represented in YAML as base64 encoded characters. |
-| `'floatTime'` | [`!!float`](https://yaml.org/type/float.html) | `Number` | Sexagesimal floating-point number format, e.g. `190:20:30.15`. To stringify with this tag, the node `format` must be `'TIME'`. |
-| `'intTime'` | [`!!int`](https://yaml.org/type/int.html) | `Number` | Sexagesimal integer number format, e.g. `190:20:30`. To stringify with this tag, the node `format` must be `'TIME'`. |
-| `'omap'` | [`!!omap`](https://yaml.org/type/omap.html) | `Map` | Ordered sequence of key: value pairs without duplicates. Using `mapAsMap: true` together with this tag is not recommended, as it makes the parse → stringify loop non-idempotent. |
-| `'pairs'` | [`!!pairs`](https://yaml.org/type/pairs.html) | `Array` | Ordered sequence of key: value pairs allowing duplicates. To create from JS, use `doc.createNode(array, { tag: '!!pairs' })`. |
-| `'set'` | [`!!set`](https://yaml.org/type/set.html) | `Set` | Unordered set of non-equal values. |
-| `'timestamp'` | [`!!timestamp`](https://yaml.org/type/timestamp.html) | `Date` | A point in time, e.g. `2001-12-15T02:59:43`. |
+| Identifier | YAML Type | JS Type | Description |
+| ------------- | ----------------------------------------------------- | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `'binary'` | [`!!binary`](https://yaml.org/type/binary.html) | `Uint8Array` | Binary data, represented in YAML as base64 encoded characters. |
+| `'floatTime'` | [`!!float`](https://yaml.org/type/float.html) | `Number` | Sexagesimal floating-point number format, e.g. `190:20:30.15`. To stringify with this tag, the node `format` must be `'TIME'`. |
+| `'intTime'` | [`!!int`](https://yaml.org/type/int.html) | `Number` | Sexagesimal integer number format, e.g. `190:20:30`. To stringify with this tag, the node `format` must be `'TIME'`. |
+| `'merge'` | [`!!merge`](https://yaml.org/type/merge.html) | `Symbol('<<')` | A `<<` merge key which allows one or more mappings to be merged with the current one. |
+| `'omap'` | [`!!omap`](https://yaml.org/type/omap.html) | `Map` | Ordered sequence of key: value pairs without duplicates. Using `mapAsMap: true` together with this tag is not recommended, as it makes the parse → stringify loop non-idempotent. |
+| `'pairs'` | [`!!pairs`](https://yaml.org/type/pairs.html) | `Array` | Ordered sequence of key: value pairs allowing duplicates. To create from JS, use `doc.createNode(array, { tag: '!!pairs' })`. |
+| `'set'` | [`!!set`](https://yaml.org/type/set.html) | `Set` | Unordered set of non-equal values. |
+| `'timestamp'` | [`!!timestamp`](https://yaml.org/type/timestamp.html) | `Date` | A point in time, e.g. `2001-12-15T02:59:43`. |
 
 ## Writing Custom Tags
 

src/compose/compose-doc.ts

@@ -28,6 +28,7 @@ export function composeDoc<
  const opts = Object.assign({ _directives: directives }, options)
  const doc = new Document(undefined, opts) as Document.Parsed<Contents, Strict>
  const ctx: ComposeContext = {
+ atKey: false,
  atRoot: true,
  directives: doc.directives,
  options: doc.options,

src/compose/compose-node.ts

@@ -11,6 +11,7 @@ import { resolveEnd } from './resolve-end.js'
 import { emptyScalarPosition } from './util-empty-scalar-position.js'
 
 export interface ComposeContext {
+ atKey: boolean
  atRoot: boolean
  directives: Directives
  options: Readonly<Required<Omit<ParseOptions, 'lineCounter'>>>

src/compose/compose-scalar.ts

@@ -87,14 +87,16 @@ function findScalarTagByName(
 }
 
 function findScalarTagByTest(
- { directives, schema }: ComposeContext,
+ { atKey, directives, schema }: ComposeContext,
  value: string,
  token: FlowScalar,
  onError: ComposeErrorHandler
 ) {
  const tag =
  (schema.tags.find(
- tag => tag.default && tag.test?.test(value)
+ tag =>
+ (tag.default === true || (atKey && tag.default === 'key')) &&
+ tag.test?.test(value)
  ) as ScalarTag) || schema[SCALAR]
 
  if (schema.compat) {

src/compose/resolve-block-map.ts

@@ -69,11 +69,13 @@ export function resolveBlockMap(
  }
 
  // key value
+ ctx.atKey = true
  const keyStart = keyProps.end
  const keyNode = key
  ? composeNode(ctx, key, keyProps, onError)
  : composeEmptyNode(ctx, keyStart, start, null, keyProps, onError)
  if (ctx.schema.compat) flowIndentCheck(bm.indent, key, onError)
+ ctx.atKey = false
 
  if (mapIncludes(ctx, map.items, keyNode))
  onError(keyStart, 'DUPLICATE_KEY', 'Map keys must be unique')

src/compose/resolve-block-seq.ts

@@ -17,6 +17,7 @@ export function resolveBlockSeq(
  const seq = new NodeClass(ctx.schema) as YAMLSeq
 
  if (ctx.atRoot) ctx.atRoot = false
+ if (ctx.atKey) ctx.atKey = false
  let offset = bs.offset
  let commentEnd: number | null = null
  for (const { start, value } of bs.items) {

src/compose/resolve-flow-collection.ts

@@ -32,6 +32,7 @@ export function resolveFlowCollection(
  coll.flow = true
  const atRoot = ctx.atRoot
  if (atRoot) ctx.atRoot = false
+ if (ctx.atKey) ctx.atKey = false
 
  let offset = fc.offset + fc.start.source.length
  for (let i = 0; i < fc.items.length; ++i) {
@@ -118,11 +119,13 @@ export function resolveFlowCollection(
  // item is a key+value pair
 
  // key value
+ ctx.atKey = true
  const keyStart = props.end
  const keyNode = key
  ? composeNode(ctx, key, props, onError)
  : composeEmptyNode(ctx, keyStart, start, null, props, onError)
  if (isBlock(key)) onError(keyNode.range, 'BLOCK_IN_FLOW', blockMsg)
+ ctx.atKey = false
 
  // value properties
  const valueProps = resolveProps(sep ?? [], {

src/compose/util-map-includes.ts

@@ -14,10 +14,6 @@ export function mapIncludes(
  typeof uniqueKeys === 'function'
  ? uniqueKeys
  : (a: ParsedNode, b: ParsedNode) =>
- a === b ||
- (isScalar(a) &&
- isScalar(b) &&
- a.value === b.value &&
- !(a.value === '<<' && ctx.schema.merge))
+ a === b || (isScalar(a) && isScalar(b) && a.value === b.value)
  return items.some(pair => isEqual(pair.key, search))
 }

src/doc/Document.ts

@@ -383,13 +383,13 @@ export class Document<
  case '1.1':
  if (this.directives) this.directives.yaml.version = '1.1'
  else this.directives = new Directives({ version: '1.1' })
- opt = { merge: true, resolveKnownTags: false, schema: 'yaml-1.1' }
+ opt = { resolveKnownTags: false, schema: 'yaml-1.1' }
  break
  case '1.2':
  case 'next':
  if (this.directives) this.directives.yaml.version = version
  else this.directives = new Directives({ version })
- opt = { merge: false, resolveKnownTags: true, schema: 'core' }
+ opt = { resolveKnownTags: true, schema: 'core' }
  break
  case null:
  if (this.directives) delete this.directives

src/nodes/Node.ts

@@ -7,7 +7,7 @@ import type { Alias } from './Alias.js'
 import { isDocument, NODE_TYPE } from './identity.js'
 import type { Scalar } from './Scalar.js'
 import { toJS, ToJSContext } from './toJS.js'
-import type { YAMLMap } from './YAMLMap.js'
+import type { MapLike, YAMLMap } from './YAMLMap.js'
 import type { YAMLSeq } from './YAMLSeq.js'
 
 export type Node<T = unknown> =
@@ -70,6 +70,16 @@ export abstract class NodeBase {
  /** A fully qualified tag, if required */
  declare tag?: string
 
+ /**
+ * Customize the way that a key-value pair is resolved.
+ * Used for YAML 1.1 !!merge << handling.
+ */
+ declare addToJSMap?: (
+ ctx: ToJSContext | undefined,
+ map: MapLike,
+ value: unknown
+ ) => void
+
  /** A plain JS representation of this node */
  abstract toJSON(): any
 

src/nodes/addPairToJSMap.ts

@@ -1,25 +1,20 @@
 import { warn } from '../log.js'
+import { addMergeToJSMap, isMergeKey } from '../schema/yaml-1.1/merge.js'
 import { createStringifyContext } from '../stringify/stringify.js'
-import { isAlias, isMap, isNode, isScalar, isSeq } from './identity.js'
+import { isNode } from './identity.js'
 import type { Pair } from './Pair.js'
-import { Scalar } from './Scalar.js'
 import { toJS, ToJSContext } from './toJS.js'
 import type { MapLike } from './YAMLMap.js'
 
-const MERGE_KEY = '<<'
-
 export function addPairToJSMap(
  ctx: ToJSContext | undefined,
  map: MapLike,
  { key, value }: Pair
 ) {
- if (ctx?.doc.schema.merge && isMergeKey(key)) {
- value = isAlias(value) ? value.resolve(ctx.doc) : value
- if (isSeq(value)) for (const it of value.items) mergeToJSMap(ctx, map, it)
- else if (Array.isArray(value))
- for (const it of value) mergeToJSMap(ctx, map, it)
- else mergeToJSMap(ctx, map, value)
- } else {
+ if (isNode(key) && key.addToJSMap) key.addToJSMap(ctx, map, value)
+ // TODO: Should drop this special case for bare << handling
+ else if (isMergeKey(ctx, key)) addMergeToJSMap(ctx, map, value)
+ else {
  const jsKey = toJS(key, '', ctx)
  if (map instanceof Map) {
  map.set(jsKey, toJS(value, jsKey, ctx))
@@ -41,45 +36,6 @@ export function addPairToJSMap(
  return map
 }
 
-const isMergeKey = (key: unknown) =>
- key === MERGE_KEY ||
- (isScalar(key) &&
- key.value === MERGE_KEY &&
- (!key.type || key.type === Scalar.PLAIN))
-
-// If the value associated with a merge key is a single mapping node, each of
-// its key/value pairs is inserted into the current mapping, unless the key
-// already exists in it. If the value associated with the merge key is a
-// sequence, then this sequence is expected to contain mapping nodes and each
-// of these nodes is merged in turn according to its order in the sequence.
-// Keys in mapping nodes earlier in the sequence override keys specified in
-// later mapping nodes. -- http://yaml.org/type/merge.html
-function mergeToJSMap(
- ctx: ToJSContext | undefined,
- map: MapLike,
- value: unknown
-) {
- const source = ctx && isAlias(value) ? value.resolve(ctx.doc) : value
- if (!isMap(source))
- throw new Error('Merge sources must be maps or map aliases')
- const srcMap = source.toJSON(null, ctx, Map)
- for (const [key, value] of srcMap) {
- if (map instanceof Map) {
- if (!map.has(key)) map.set(key, value)
- } else if (map instanceof Set) {
- map.add(key)
- } else if (!Object.prototype.hasOwnProperty.call(map, key)) {
- Object.defineProperty(map, key, {
- value,
- writable: true,
- enumerable: true,
- configurable: true
- })
- }
- }
- return map
-}
-
 function stringifyKey(
  key: unknown,
  jsKey: unknown,

src/schema/Schema.ts

@@ -13,7 +13,6 @@ const sortMapEntriesByKey = (a: Pair<any>, b: Pair<any>) =>
 export class Schema {
  compat: Array<CollectionTag | ScalarTag> | null
  knownTags: Record<string, CollectionTag | ScalarTag>
- merge: boolean
  name: string
  sortMapEntries: ((a: Pair, b: Pair) => number) | null
  tags: Array<CollectionTag | ScalarTag>
@@ -38,10 +37,9 @@ export class Schema {
  : compat
  ? getTags(null, compat)
  : null
- this.merge = !!merge
  this.name = (typeof schema === 'string' && schema) || 'core'
  this.knownTags = resolveKnownTags ? coreKnownTags : {}
- this.tags = getTags(customTags, this.name)
+ this.tags = getTags(customTags, this.name, merge)
  this.toStringOptions = toStringDefaults ?? null
 
  Object.defineProperty(this, MAP, { value: map })

src/schema/tags.ts

@@ -9,6 +9,7 @@ import { int, intHex, intOct } from './core/int.js'
 import { schema as core } from './core/schema.js'
 import { schema as json } from './json/schema.js'
 import { binary } from './yaml-1.1/binary.js'
+import { merge } from './yaml-1.1/merge.js'
 import { omap } from './yaml-1.1/omap.js'
 import { pairs } from './yaml-1.1/pairs.js'
 import { schema as yaml11 } from './yaml-1.1/schema.js'
@@ -36,6 +37,7 @@ const tagsByName = {
  intOct,
  intTime,
  map,
+ merge,
  null: nullTag,
  omap,
  pairs,
@@ -50,6 +52,7 @@ export type Tags = Array<ScalarTag | CollectionTag | TagId>
 
 export const coreKnownTags = {
  'tag:yaml.org,2002:binary': binary,
+ 'tag:yaml.org,2002:merge': merge,
  'tag:yaml.org,2002:omap': omap,
  'tag:yaml.org,2002:pairs': pairs,
  'tag:yaml.org,2002:set': set,
@@ -58,9 +61,17 @@ export const coreKnownTags = {
 
 export function getTags(
  customTags: SchemaOptions['customTags'] | undefined,
- schemaName: string
+ schemaName: string,
+ addMergeTag?: boolean
 ) {
- let tags: Tags | undefined = schemas.get(schemaName)
+ const schemaTags = schemas.get(schemaName)
+ if (schemaTags && !customTags) {
+ return addMergeTag && !schemaTags.includes(merge)
+ ? schemaTags.concat(merge)
+ : schemaTags.slice()
+ }
+
+ let tags: Tags | undefined = schemaTags
  if (!tags) {
  if (Array.isArray(customTags)) tags = []
  else {
@@ -79,14 +90,18 @@ export function getTags(
  } else if (typeof customTags === 'function') {
  tags = customTags(tags.slice())
  }
+ if (addMergeTag) tags = tags.concat(merge)
 
- return tags.map(tag => {
- if (typeof tag !== 'string') return tag
- const tagObj = tagsByName[tag]
- if (tagObj) return tagObj
- const keys = Object.keys(tagsByName)
- .map(key => JSON.stringify(key))
- .join(', ')
- throw new Error(`Unknown custom tag "${tag}"; use one of ${keys}`)
- })
+ return tags.reduce<(CollectionTag | ScalarTag)[]>((tags, tag) => {
+ const tagObj = typeof tag === 'string' ? tagsByName[tag] : tag
+ if (!tagObj) {
+ const tagName = JSON.stringify(tag)
+ const keys = Object.keys(tagsByName)
+ .map(key => JSON.stringify(key))
+ .join(', ')
+ throw new Error(`Unknown custom tag ${tagName}; use one of ${keys}`)
+ }
+ if (!tags.includes(tagObj)) tags.push(tagObj)
+ return tags
+ }, [])
 }

src/schema/types.ts

@@ -14,11 +14,13 @@ interface TagBase {
  createNode?: (schema: Schema, value: unknown, ctx: CreateNodeContext) => Node
 
  /**
- * If `true`, together with `test` allows for values to be stringified without
- * an explicit tag. For most cases, it's unlikely that you'll actually want to
- * use this, even if you first think you do.
+ * If `true`, allows for values to be stringified without
+ * an explicit tag together with `test`.
+ * If `'key'`, this only applies if the value is used as a mapping key.
+ * For most cases, it's unlikely that you'll actually want to use this,
+ * even if you first think you do.
  */
- default?: boolean
+ default?: boolean | 'key'
 
  /**
  * If a tag has multiple forms that should be parsed and/or stringified