perf(highlight): precompute SGR at compile, sorted span claiming, stream ESC-free fast path

- compileRules precomputes the full SGR open sequence per rule and per band;
  wrap() is now pure concatenation, bandOpen() a table lookup (output bytes
  unchanged, bg keeps its unvalidated white-fallback)
- claim() replaces the linear overlaps() scan: claimed spans stay sorted by
  start, O(1) append on the common left-to-right sweep plus one binary search
  otherwise (match-dense 200KB payload: -19% one-shot, -56% streamed)
- HighlightStream: bufferHasEsc flag skips the O(buffer) escape rescans when
  neither the held text nor the chunk contains ESC, fixing quadratic rescan on
  escape-free floods (plain 200KB streamed: -54%)
- applyHighlights tracks the consumed match budget incrementally instead of
  two budgets.reduce() passes per text token
- ESCAPE_RE now covers DCS/APC/PM/SOS (BEL or ST terminated) and single-char
  Fe escapes (DECSC/DECRC/NEL/RI/RIS, orphan ST); isIncompleteEscape holds cut
  tails of the new families; split-smoke exercises every cut point through them

src/renderer/src/terminal/highlightEngine.ts

@@ -14,14 +14,18 @@ export interface CompiledRule {
   id: string
   /** compiled with the 'g' flag (plus 'i' for case-insensitive rules) */
   regex: RegExp
-  /** hex fg, e.g. "#3fb950" — converted to rgb only at wrap time */
+  /** hex fg, e.g. "#3fb950" — kept for introspection; `open` is precomputed from it */
   fg: string
-  /** optional hex bg */
+  /** optional hex bg (not re-validated; a malformed value folds to white, as before) */
   bg?: string
   /** optional value bands, ascending by `min` — see HighlightRule.bands */
-  bands?: { min: number; fg: string }[]
+  bands?: { min: number; fg: string; open: string }[]
   /** copied from the source rule for stable priority ordering */
   priority: number
+  /** precomputed truecolor fg fragment, e.g. "38;2;63;185;80" */
+  fgSgr: string
+  /** full SGR open sequence `\x1b[38;2;…(;48;2;…)?m` incl. optional bg, built once here */
+  open: string
 }
 
 /** Per-rule counters, only filled when a stats sink is passed in (see `StatsSink`). */
@@ -39,11 +43,26 @@ export interface RuleStat {
 export type StatsSink = Map<string, RuleStat>
 
 type Token = { kind: 'seq' | 'text'; value: string }
-type Span = { start: number; end: number; fg: string; bg?: string }
+/** A claimed span carries its precomputed SGR open sequence, never a hex colour. */
+type Span = { start: number; end: number; open: string }
 
-/** ANSI escape tokens: CSI, OSC, two-char (ESC ( / ) ..), and ESC = / > . */
+/**
+ * ANSI escape tokens:
+ *  - CSI            `ESC [ … final`
+ *  - OSC            `ESC ] … (BEL | ST)`
+ *  - DCS/APC/PM/SOS `ESC P|X|^|_ … (BEL | ST)` — tmux, sixel, kitty graphics.
+ *    The `[^\x1b]*` body means an unterminated one stops at the next ESC
+ *    instead of swallowing the rest of the buffer (it then stays a text run,
+ *    which applyHighlights passes through raw).
+ *  - two-char charset `ESC ( x` / `ESC ) x`, keypad modes `ESC =` / `ESC >`
+ *  - single-char Fe escapes: `ESC` + one 0x30–0x7E byte (DECSC `\x1b7`,
+ *    DECRC `\x1b8`, NEL `\x1bE`, RI `\x1bM`, RIS `\x1bc`, orphan ST `\x1b\`, …).
+ *    Last in the alternation, and the initiators of the longer forms
+ *    (`[ ] ( ) P X ^ _ = >`) are excluded from the class so it can never steal
+ *    the leading bytes of a longer sequence.
+ */
 const ESCAPE_RE =
-  /(\x1b\[[0-9;:?]*[A-Za-z]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[()][0-9A-B]|\x1b[=>])/g
+  /(\x1b\[[0-9;:?]*[A-Za-z]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)|\x1b[PX^_][^\x1b]*(?:\x07|\x1b\\)|\x1b[()][0-9A-B]|\x1b[=>]|\x1b[0-9:;<?@A-OQ-WYZ\\`a-z{|}~])/g
 
 /** Matches a single rule may inject per chunk (bomb guard). */
 const MAX_PER_RULE = 300
@@ -69,6 +88,12 @@ export function hexToRgb(hex: string): { r: number; g: number; b: number } {
   return { r: (n >> 16) & 0xff, g: (n >> 8) & 0xff, b: n & 0xff }
 }
 
+/** "r;g;b" decimal SGR fragment for one hex colour (white fallback via hexToRgb). */
+function rgbFragment(hex: string): string {
+  const { r, g, b } = hexToRgb(hex)
+  return `${r};${g};${b}`
+}
+
 /** Split a chunk into escape-sequence tokens and plain-text runs. */
 export function tokenize(input: string): Token[] {
   const out: Token[] = []
@@ -110,63 +135,94 @@ export function compileRules(rules: HighlightRule[]): CompiledRule[] {
     const fg = typeof rule.color?.fg === 'string' ? rule.color.fg : ''
     if (!/^#[0-9a-fA-F]{3}$|^#[0-9a-fA-F]{6}$/.test(fg)) continue
     const bg = typeof rule.color?.bg === 'string' ? rule.color.bg : undefined
-    out.push({ id: rule.id, regex, fg, bg, bands: compileBands(rule.bands), priority: rule.priority })
+    // bg stays unvalidated on purpose (as before): a malformed hex folds to
+    // white inside hexToRgb, byte-identical to the old wrap-time conversion.
+    const bgPart = bg ? `;48;2;${rgbFragment(bg)}` : ''
+    const fgSgr = rgbFragment(fg)
+    out.push({
+      id: rule.id,
+      regex,
+      fg,
+      bg,
+      bands: compileBands(rule.bands, bgPart),
+      priority: rule.priority,
+      fgSgr,
+      open: `\x1b[38;2;${fgSgr}${bgPart}m`
+    })
   }
   out.sort((a, b) => a.priority - b.priority)
   return out
 }
 
 /** Keep only well-formed value bands, ascending — the engine walks them in order. */
-function compileBands(bands: HighlightRule['bands']): CompiledRule['bands'] {
+function compileBands(bands: HighlightRule['bands'], bgPart: string): CompiledRule['bands'] {
   if (!Array.isArray(bands)) return undefined
-  const kept: { min: number; fg: string }[] = []
+  const kept: NonNullable<CompiledRule['bands']> = []
   for (const band of bands) {
     if (!band || !Number.isFinite(band.min)) continue
     if (typeof band.fg !== 'string' || !/^#[0-9a-fA-F]{3}$|^#[0-9a-fA-F]{6}$/.test(band.fg)) continue
-    kept.push({ min: band.min, fg: band.fg })
+    kept.push({ min: band.min, fg: band.fg, open: `\x1b[38;2;${rgbFragment(band.fg)}${bgPart}m` })
   }
   if (kept.length === 0) return undefined
   kept.sort((a, b) => a.min - b.min)
   return kept
 }
 
-/** Wrap one plain-text span with truecolor SGR. */
+/** Wrap one plain-text span with its precomputed SGR open sequence. */
-function wrap(text: string, fgHex: string, bgHex?: string): string {
+function wrap(text: string, open: string): string {
-  const f = hexToRgb(fgHex)
+  return `${open}${text}\x1b[0m`
-  let s = `\x1b[38;2;${f.r};${f.g};${f.b}`
-  if (bgHex) {
-    const b = hexToRgb(bgHex)
-    s += `;48;2;${b.r};${b.g};${b.b}`
-  }
-  return `${s}m${text}\x1b[0m`
-}
-
-/** True when [s,e) overlaps any claimed span. */
-function overlaps(claimed: Span[], s: number, e: number): boolean {
-  for (const c of claimed) {
-    if (s < c.end && e > c.start) return true
-  }
-  return false
 }
 
 /**
- * Colour for one match: with value bands the match's first number picks the
+ * Try to claim [s,e) for a span. Claimed spans are pairwise disjoint and kept
- * band (last `min` that is <= the value wins); everything else keeps the rule's
+ * sorted by start (ends ascend with starts), which turns the old linear scan
- * own foreground.
+ * over every claimed span (O(spans²) on match-dense runs) into:
+ *  - an O(1) append when the candidate lands past every claim — the common
+ *    case, since rule scans sweep a run left to right; otherwise
+ *  - one binary search: the only possible clash is with the nearest span left
+ *    of the insertion point.
+ * The span colour is only computed on accept. Returns true when claimed.
  */
-function bandColor(rule: CompiledRule, matched: string): string {
+function claim(claimed: Span[], s: number, e: number, rule: CompiledRule, matched: string): boolean {
+  const n = claimed.length
+  if (n === 0 || claimed[n - 1].end <= s) {
+    claimed.push({ start: s, end: e, open: bandOpen(rule, matched) })
+    return true
+  }
+  let lo = 0
+  let hi = n
+  while (lo < hi) {
+    const mid = (lo + hi) >> 1
+    if (claimed[mid].start < e) lo = mid + 1
+    else hi = mid
+  }
+  if (lo > 0 && claimed[lo - 1].end > s) return false
+  claimed.splice(lo, 0, { start: s, end: e, open: bandOpen(rule, matched) })
+  return true
+}
+
+/** First-number finder for band selection; no /g, so the shared literal is stateless. */
+const BAND_NUMBER_RE = /-?\d+(?:\.\d+)?/
+
+/**
+ * SGR open sequence for one match: with value bands the match's first number
+ * picks the band (last `min` that is <= the value wins); everything else keeps
+ * the rule's own precomputed open sequence. Pure table lookup — no colour math
+ * on the hot path.
+ */
+function bandOpen(rule: CompiledRule, matched: string): string {
   const bands = rule.bands
-  if (!bands || bands.length === 0) return rule.fg
+  if (!bands || bands.length === 0) return rule.open
-  const number = /-?\d+(?:\.\d+)?/.exec(matched)
+  const number = BAND_NUMBER_RE.exec(matched)
-  if (number === null) return rule.fg
+  if (number === null) return rule.open
   const value = Number(number[0])
-  if (!Number.isFinite(value)) return rule.fg
+  if (!Number.isFinite(value)) return rule.open
-  let fg = rule.fg
+  let open = rule.open
   for (const band of bands) {
     if (value < band.min) break
-    fg = band.fg
+    open = band.open
   }
-  return fg
+  return open
 }
 
 /** True when `text` holds a newline-free segment longer than MAX_LINE_LEN. */
@@ -189,10 +245,17 @@ function hasLongLine(text: string): boolean {
 /** Millisecond clock for the optional stats sink; `performance` when present. */
 const now = (): number => (typeof performance !== 'undefined' ? performance.now() : Date.now())
 
-function applyRun(text: string, rules: CompiledRule[], budgets: number[], stats?: StatsSink): string {
+function applyRun(text: string, rules: CompiledRule[], budgets: number[], stats?: StatsSink, spent?: { n: number }): string {
-  if (text.length === 0) return ''
+  if (text.length === 0) {
-  if (hasLongLine(text)) return text
+    if (spent !== undefined) spent.n = 0
+    return ''
+  }
+  if (hasLongLine(text)) {
+    if (spent !== undefined) spent.n = 0
+    return text
+  }
   const claimed: Span[] = []
+  let totalMade = 0
   for (let ri = 0; ri < rules.length; ri++) {
     const rule = rules[ri]
     if (budgets[ri] <= 0) continue
@@ -208,9 +271,9 @@ function applyRun(text: string, rules: CompiledRule[], budgets: number[], stats?
         rule.regex.lastIndex++
         continue
       }
-      if (!overlaps(claimed, start, end)) {
+      if (claim(claimed, start, end, rule, m[0])) {
-        claimed.push({ start, end, fg: bandColor(rule, m[0]), bg: rule.bg })
         made++
+        totalMade++
         budgets[ri]--
         if (budgets[ri] <= 0) break
       }
@@ -223,13 +286,14 @@ function applyRun(text: string, rules: CompiledRule[], budgets: number[], stats?
       stats.set(rule.id, stat)
     }
   }
+  if (spent !== undefined) spent.n = totalMade
   if (claimed.length === 0) return text
-  claimed.sort((a, b) => a.start - b.start)
+  // `claim` keeps the spans sorted by start — no final sort needed here.
   let out = ''
   let pos = 0
   for (const c of claimed) {
     if (c.start > pos) out += text.slice(pos, c.start)
-    out += wrap(text.slice(c.start, c.end), c.fg, c.bg)
+    out += wrap(text.slice(c.start, c.end), c.open)
     pos = c.end
   }
   if (pos < text.length) out += text.slice(pos)
@@ -249,6 +313,9 @@ export function applyHighlights(chunk: string, rules: CompiledRule[], stats?: St
   let budgets = rules.map(() => MAX_PER_RULE)
   let usedBudget = false
   let out = ''
+  // applyRun reports how many matches it claimed, replacing the old two
+  // budgets.reduce() passes per text token.
+  const spent = { n: 0 }
   for (const token of tokenize(chunk)) {
     if (token.kind === 'seq') {
       out += token.value
@@ -260,10 +327,9 @@ export function applyHighlights(chunk: string, rules: CompiledRule[], stats?: St
       out += token.value
       continue
     }
-    const before = budgets.reduce((a, b) => a + b, 0)
+    spent.n = 0
-    out += applyRun(token.value, rules, budgets, stats)
+    out += applyRun(token.value, rules, budgets, stats, spent)
-    const after = budgets.reduce((a, b) => a + b, 0)
+    if (spent.n > 0) usedBudget = true
-    if (after < before) usedBudget = true
   }
   return usedBudget ? out : chunk
 }
@@ -308,15 +374,30 @@ export function previewSpans(text: string, rules: CompiledRule[]): PreviewSpan[]
   return spans
 }
 
-/** True when `seg` is a dangling, unfinished escape fragment (must not be emitted). */function isIncompleteEscape(seg: string): boolean {
+/**
+ * True when `seg` is a dangling, unfinished escape fragment (must not be
+ * emitted before its terminator arrives).
+ */
+function isIncompleteEscape(seg: string): boolean {
   // `seg` is expected to start at (or be) a lone ESC.
-  // A lone ESC — the leading half of any CSI/OSC/two-char/ST sequence.
+  // A lone ESC — the leading half of any CSI/OSC/DCS/two-char/ST/Fe sequence.
   if (seg === '\x1b') return true
   // Unfinished CSI: ESC [ plus parameter bytes but no final byte in 0x40..0x7E.
   if (/^\x1b\[[0-9;:?]*$/.test(seg)) return true
-  // Unfinished OSC: ESC ] ... with no BEL (\x07) or ST (\x1b\ ) terminator.
+  // Unfinished OSC / DCS / APC / PM / SOS: opened but no BEL (\x07) or ST
-  // (A trailing half-ST is covered by the lone-ESC case above.)
+  // (\x1b\) terminator yet, so the fragment could still grow into any of them.
-  if (seg.startsWith('\x1b]') && !seg.includes('\x07') && !seg.includes('\x1b\\')) return true
+  // (A trailing half-ST — the ESC of `\x1b\` with the `\` still to come — is
+  // covered by the lone-ESC case above, since findIncompleteIndex slices from
+  // the *last* ESC.)
+  // NB: `[]` would be an *empty* class in JS regex, so `]` is kept in a
+  // startsWith instead of a merged character class.
+  if (
+    (seg.startsWith('\x1b]') || /^\x1b[PX^_]/.test(seg)) &&
+    !seg.includes('\x07') &&
+    !seg.includes('\x1b\\')
+  ) {
+    return true
+  }
   return false
 }
 
@@ -356,6 +437,8 @@ export class HighlightStream {
   private rules: CompiledRule[]
   private stats?: StatsSink
   private buffer = ''
+  /** Whether `buffer` holds any ESC — lets push skip the O(buffer) escape scans on escape-free floods. */
+  private bufferHasEsc = false
 
   constructor(rules: CompiledRule[]) {
     this.rules = rules
@@ -378,6 +461,20 @@ export class HighlightStream {
 
   /** Join held bytes with `chunk`, return content safe to hand to xterm. */
   push(chunk: string): string {
+    // Fast path: neither the held text nor the new chunk contains a single ESC,
+    // so findIncompleteIndex/lastEscapeEnd would both come back -1. Skip their
+    // O(buffer) rescans — they made escape-free floods (cat of a large text
+    // file) quadratic in the held buffer — and go straight to the trailing-text
+    // hold. Decision-identical to the full path below.
+    if (!this.bufferHasEsc && !chunk.includes('\x1b')) {
+      this.buffer += chunk
+      if (this.buffer.length > MAX_CHUNK) {
+        const raw = this.buffer
+        this.buffer = ''
+        return applyHighlights(raw, this.rules, this.stats)
+      }
+      return ''
+    }
     this.buffer += chunk
 
     // (a) trailing incomplete escape -> hold it, emit the closed prefix.
@@ -389,9 +486,11 @@ export class HighlightStream {
         // Give up waiting on a pathologically long unterminated sequence: emit
         // the fragment raw (applyHighlights skips raw \x1b in text runs).
         this.buffer = ''
+        this.bufferHasEsc = false
         return applyHighlights(head, this.rules, this.stats) + tail
       }
       this.buffer = tail
+      this.bufferHasEsc = true // the held tail starts at the dangling ESC
       return applyHighlights(head, this.rules, this.stats)
     }
 
@@ -404,12 +503,16 @@ export class HighlightStream {
       if (this.buffer.length > MAX_CHUNK) {
         const raw = this.buffer
         this.buffer = ''
+        this.bufferHasEsc = false
         return applyHighlights(raw, this.rules, this.stats)
       }
+      // The buffer (or the chunk that just joined it) brought the ESC in.
+      this.bufferHasEsc = true
       return ''
     }
     const head = this.buffer.slice(0, lastEnd)
     this.buffer = this.buffer.slice(lastEnd)
+    this.bufferHasEsc = this.buffer.includes('\x1b')
     return applyHighlights(head, this.rules, this.stats)
   }
 
@@ -417,6 +520,7 @@ export class HighlightStream {
   flush(): string {
     const b = this.buffer
     this.buffer = ''
+    this.bufferHasEsc = false
     return applyHighlights(b, this.rules, this.stats)
   }
 }

tests/hl-rules.mjs

@@ -12,7 +12,7 @@
  *     --outfile=tests/.hl-rules.cjs --alias:@shared=./src/shared
  *   node tests/hl-rules.mjs
  */
-import { compileRules, applyHighlights } from '../src/renderer/src/terminal/highlightEngine.ts'
+import { compileRules, applyHighlights, __testHooks as hooks } from '../src/renderer/src/terminal/highlightEngine.ts'
 import { previewSpans } from '../src/renderer/src/terminal/highlightEngine.ts'
 import { DEFAULT_HIGHLIGHT_RULES } from '../src/shared/settings.ts'
 import { exportHighlightRules, mergeRules, parseHighlightRules } from '../src/shared/highlightIO.ts'
@@ -739,5 +739,52 @@ console.log('[profiles]')
   )
 }
 
+// ---- 19. escape sequences pass through untouched ---------------------------
+console.log('[escape sequences]')
+{
+  // DCS / APC / PM / SOS bodies and single-char Fe escapes must keep their
+  // bytes raw — never wrapped in SGR — while text around them stays highlighted.
+  const families = [
+    ['DCS (ST)', '\x1bP0;1;0q#0;2;0;0;0~..\x1b\\'],
+    ['DCS (BEL)', '\x1bP1;2|9/9\x07'],
+    ['APC (kitty graphics)', '\x1b_Gf=32,s=10,v=10,m=1\x1b\\'],
+    ['PM', '\x1b^weather\x1b\\'],
+    ['SOS', '\x1bXsos-frame\x1b\\']
+  ]
+  // one coloured span, exactly as the engine emits it (ESC-free span text)
+  const SPAN_RE = /\x1b\[38;2;\d+;\d+;\d+(?:;48;2;\d+;\d+;\d+)?m[^\x1b]*\x1b\[0m/g
+  for (const [name, seq] of families) {
+    const out = applyHighlights(`DONE ${seq} ERROR`, rules)
+    ok(out.includes(`${sgr(OK_GREEN)}DONE\x1b[0m`), `${name}: text before it keeps its highlight`)
+    ok(out.includes(`${sgr(BAD_RED)}ERROR\x1b[0m`), `${name}: text after it keeps its highlight`)
+    ok(
+      out.replace(SPAN_RE, '').includes(seq),
+      `${name}: sequence bytes pass through raw, never wrapped in SGR`
+    )
+  }
+
+  const fe = applyHighlights('\x1b7DONE\x1b8 1', rules)
+  ok(fe.includes(`${sgr(OK_GREEN)}DONE\x1b[0m`), 'text between DECSC/DECRC keeps its highlight')
+  ok(fe.includes('\x1b7') && fe.includes('\x1b8'), 'DECSC/DECRC bytes pass through byte-for-byte')
+
+  // backstop: a text run holding a residual (unterminated) ESC passes through
+  // raw byte-for-byte — never partially highlighted, never corrupted
+  const raw = 'DONE \x1bP0;1;0qcut ERROR'
+  ok(applyHighlights(raw, rules) === raw, 'an unterminated DCS keeps its run byte-for-byte raw')
+
+  // the stream treats a cut DCS/APC/PM/SOS tail as incomplete, like OSC
+  ok(hooks.isIncompleteEscape('\x1b') === true, 'a lone trailing ESC stays incomplete')
+  ok(hooks.isIncompleteEscape('\x1bP0;1;0qcut') === true, 'a cut DCS tail counts as incomplete')
+  ok(hooks.isIncompleteEscape('\x1b_Gf=32') === true, 'a cut APC tail counts as incomplete')
+  ok(
+    hooks.isIncompleteEscape('\x1b^w') === true && hooks.isIncompleteEscape('\x1bXs') === true,
+    'cut PM/SOS tails count as incomplete'
+  )
+  ok(hooks.isIncompleteEscape('\x1bP0;1;0qx\x1b\\') === false, 'a terminated DCS is complete')
+  ok(hooks.isIncompleteEscape('\x1b7') === false, 'a complete two-byte Fe escape is complete')
+  ok(hooks.findIncompleteIndex('DONE \x1bP0;1;0qcut') === 5, 'the stream holds from the DCS initiator')
+  ok(hooks.findIncompleteIndex('DONE \x1b7') === -1, 'a trailing DECSC is not held as incomplete')
+}
+
 console.log(failed === 0 ? '\n[hl-rules] ALL CHECKS PASSED' : `\n[hl-rules] ${failed} CHECK(S) FAILED`)
 process.exit(failed === 0 ? 0 : 1)

tests/hl-split-smoke.mjs

@@ -7,12 +7,24 @@ import { DEFAULT_HIGHLIGHT_RULES } from '../src/shared/settings.ts'
 
 const rules = compileRules(DEFAULT_HIGHLIGHT_RULES)
 
-// kimi-like: truecolor SGR, OSC title, box drawing, per-line repaints
+// kimi-like: truecolor SGR, OSC title, box drawing, per-line repaints.
+// The tail line exercises the string-terminated families (DCS/sixel, APC/kitty
+// graphics, PM, SOS), a BEL-terminated DCS, and DECSC/DECRC (`\x1b7`/`\x1b8`)
+// so every cut point through them is checked against the reference.
 const banner =
   '\x1b]0;kimi — session\x07' +
   '\x1b[38;2;79;168;255m╭───────╮\x1b[0m\r\n' +
   '\x1b[38;2;136;136;136mWelcome to Kimi Code!\x1b[0m \x1b[38;2;79;168;255m│\x1b[0m send /help\r\n' +
-  '\x1b[38;2;224;224;224mModel: MiniMax-M3 0.41.038 ERROR 99% https://example.com/x\x1b[0m\r\n'
+  '\x1b[38;2;224;224;224mModel: MiniMax-M3 0.41.038 ERROR 99% https://example.com/x\x1b[0m\r\n' +
+  '\x1b7' +
+  '\x1b[38;2;224;224;224mERROR 85% exit code 1\x1b[0m' +
+  '\x1bP0;1;0q#0;2;0;0;0~-..\x1b\\' +
+  '\x1bP1;2|9/9\x07' +
+  '\x1b_Gf=32,s=10,v=10,m=1\x1b\\' +
+  '\x1b^pm-note\x1b\\' +
+  '\x1bXsos-frame\x1b\\' +
+  '\x1b[38;2;136;136;136mdone \u2713\x1b[0m' +
+  '\x1b8\r\n'
 
 const reference = applyHighlights(banner, rules)