fix: Improve tab handling (tests DK95 & Y79Y, #553)

eemeli · eemeli · commit 4e66d72c7666 · 2024-06-09T00:35:34.000+03:00
diff --git a/src/compose/compose-doc.ts b/src/compose/compose-doc.ts
@@ -38,6 +38,7 @@ export function composeDoc<
     next: value ?? end?.[0],
     offset,
     onError,
+    parentIndent: 0,
     startOnNewline: true
   })
   if (props.found) {
diff --git a/src/compose/compose-scalar.ts b/src/compose/compose-scalar.ts
@@ -16,7 +16,7 @@ export function composeScalar(
 ) {
   const { value, type, comment, range } =
     token.type === 'block-scalar'
-      ? resolveBlockScalar(token, ctx.options.strict, onError)
+      ? resolveBlockScalar(ctx, token, onError)
       : resolveFlowScalar(token, ctx.options.strict, onError)
 
   const tagName = tagToken
diff --git a/src/compose/resolve-block-map.ts b/src/compose/resolve-block-map.ts
@@ -34,6 +34,7 @@ export function resolveBlockMap(
       next: key ?? sep?.[0],
       offset,
       onError,
+      parentIndent: bm.indent,
       startOnNewline: true
     })
     const implicitKey = !keyProps.found
@@ -83,6 +84,7 @@ export function resolveBlockMap(
       next: value,
       offset: keyNode.range[2],
       onError,
+      parentIndent: bm.indent,
       startOnNewline: !key || key.type === 'block-scalar'
     })
     offset = valueProps.end
diff --git a/src/compose/resolve-block-scalar.ts b/src/compose/resolve-block-scalar.ts
@@ -1,11 +1,12 @@
 import { Range } from '../nodes/Node.js'
 import { Scalar } from '../nodes/Scalar.js'
 import type { BlockScalar } from '../parse/cst.js'
+import type { ComposeContext } from './compose-node.js'
 import type { ComposeErrorHandler } from './composer.js'
 
 export function resolveBlockScalar(
+  ctx: ComposeContext,
   scalar: BlockScalar,
-  strict: boolean,
   onError: ComposeErrorHandler
 ): {
   value: string
@@ -14,7 +15,7 @@ export function resolveBlockScalar(
   range: Range
 } {
   const start = scalar.offset
-  const header = parseBlockScalarHeader(scalar, strict, onError)
+  const header = parseBlockScalarHeader(scalar, ctx.options.strict, onError)
   if (!header)
     return { value: '', type: null, comment: '', range: [start, start, start] }
   const type = header.mode === '>' ? Scalar.BLOCK_FOLDED : Scalar.BLOCK_LITERAL
@@ -56,6 +57,10 @@ export function resolveBlockScalar(
       }
       if (header.indent === 0) trimIndent = indent.length
       contentStart = i
+      if (trimIndent === 0 && !ctx.atRoot) {
+        const message = 'Block scalar values in collections must be indented'
+        onError(offset, 'BAD_INDENT', message)
+      }
       break
     }
     offset += indent.length + content.length + 1
diff --git a/src/compose/resolve-block-seq.ts b/src/compose/resolve-block-seq.ts
@@ -25,6 +25,7 @@ export function resolveBlockSeq(
       next: value,
       offset,
       onError,
+      parentIndent: bs.indent,
       startOnNewline: true
     })
     if (!props.found) {
diff --git a/src/compose/resolve-flow-collection.ts b/src/compose/resolve-flow-collection.ts
@@ -44,6 +44,7 @@ export function resolveFlowCollection(
       next: key ?? sep?.[0],
       offset,
       onError,
+      parentIndent: fc.indent,
       startOnNewline: false
     })
     if (!props.found) {
@@ -130,6 +131,7 @@ export function resolveFlowCollection(
         next: value,
         offset: keyNode.range[2],
         onError,
+        parentIndent: fc.indent,
         startOnNewline: false
       })
 
diff --git a/src/compose/resolve-props.ts b/src/compose/resolve-props.ts
@@ -7,12 +7,21 @@ export interface ResolvePropsArg {
   next: Token | null | undefined
   offset: number
   onError: ComposeErrorHandler
+  parentIndent: number
   startOnNewline: boolean
 }
 
 export function resolveProps(
   tokens: SourceToken[],
-  { flow, indicator, next, offset, onError, startOnNewline }: ResolvePropsArg
+  {
+    flow,
+    indicator,
+    next,
+    offset,
+    onError,
+    parentIndent,
+    startOnNewline
+  }: ResolvePropsArg
 ) {
   let spaceBefore = false
   let atNewline = startOnNewline
@@ -43,7 +52,7 @@ export function resolveProps(
       reqSpace = false
     }
     if (tab) {
-      if (token.type !== 'comment') {
+      if (atNewline && token.type !== 'comment' && token.type !== 'newline') {
         onError(tab, 'TAB_AS_INDENT', 'Tabs are not allowed as indentation')
       }
       tab = null
@@ -55,9 +64,8 @@ export function resolveProps(
         // In a flow collection, only the parser handles indent.
         if (
           !flow &&
-          atNewline &&
           (indicator !== 'doc-start' || next?.type !== 'flow-collection') &&
-          token.source[0] === '\t'
+          token.source.includes('\t')
         ) {
           tab = token
         }
@@ -132,7 +140,8 @@ export function resolveProps(
             `Unexpected ${token.source} in ${flow ?? 'collection'}`
           )
         found = token
-        atNewline = false
+        atNewline =
+          indicator === 'seq-item-ind' || indicator === 'explicit-key-ind'
         hasSpace = false
         break
       case 'comma':
@@ -167,7 +176,13 @@ export function resolveProps(
       'Tags and anchors must be separated from the next token by white space'
     )
   }
-  if (tab) onError(tab, 'TAB_AS_INDENT', 'Tabs are not allowed as indentation')
+  if (
+    tab &&
+    ((atNewline && tab.indent <= parentIndent) ||
+      next?.type === 'block-map' ||
+      next?.type === 'block-seq')
+  )
+    onError(tab, 'TAB_AS_INDENT', 'Tabs are not allowed as indentation')
   return {
     comma,
     found,
diff --git a/src/parse/cst-scalar.ts b/src/parse/cst-scalar.ts
@@ -1,3 +1,4 @@
+import type { ComposeContext } from '../compose/compose-node.js'
 import type { ComposeErrorHandler } from '../compose/composer.js'
 import { resolveBlockScalar } from '../compose/resolve-block-scalar.js'
 import { resolveFlowScalar } from '../compose/resolve-flow-scalar.js'
@@ -55,7 +56,11 @@ export function resolveAsScalar(
       case 'double-quoted-scalar':
         return resolveFlowScalar(token, strict, _onError)
       case 'block-scalar':
-        return resolveBlockScalar(token, strict, _onError)
+        return resolveBlockScalar(
+          { options: { strict } } as ComposeContext,
+          token,
+          _onError
+        )
     }
   }
   return null
diff --git a/src/parse/lexer.ts b/src/parse/lexer.ts
@@ -551,13 +551,23 @@ export class Lexer {
         nl = this.buffer.length
       }
     }
-    if (!this.blockScalarKeep) {
+
+    // Trailing insufficiently indented tabs are invalid.
+    // To catch that during parsing, we include them in the block scalar value.
+    let i = nl + 1
+    ch = this.buffer[i]
+    while (ch === ' ') ch = this.buffer[++i]
+    if (ch === '\t') {
+      while (ch === '\t' || ch === ' ' || ch === '\r' || ch === '\n')
+        ch = this.buffer[++i]
+      nl = i - 1
+    } else if (!this.blockScalarKeep) {
       do {
         let i = nl - 1
         let ch = this.buffer[i]
         if (ch === '\r') ch = this.buffer[--i]
         const lastChar = i // Drop the line if last char not more indented
-        while (ch === ' ' || ch === '\t') ch = this.buffer[--i]
+        while (ch === ' ') ch = this.buffer[--i]
         if (ch === '\n' && i >= this.pos && i + 1 + indent > lastChar) nl = i
         else break
       } while (true)
diff --git a/tests/yaml-test-suite.ts b/tests/yaml-test-suite.ts
@@ -21,15 +21,6 @@ const skip: Record<string, boolean | string[]> = {
   'SF5V/0': ['errors'], // allow duplicate %YAML directives
 
   // FIXME recent upstream additions
-  'DK95/0': true,
-  'DK95/4': true,
-  'DK95/5': true,
-  'Y79Y/4': ['errors'],
-  'Y79Y/5': ['errors'],
-  'Y79Y/6': ['errors'],
-  'Y79Y/7': ['errors'],
-  'Y79Y/8': ['errors'],
-  'Y79Y/9': ['errors'],
   'ZYU8/2': ['errors']
 }
 
