The tokeniser stage

WeftTokeniser is the tokeniser stage: it fills each weft's subtokens by kind. A heading gets its title, its language specifier, and its sink. Arrow and code wefts get their ::[name] anchors; tilde, preamble, and prose wefts get their inline content and their prose anchors, read only where Markdown leaves them live. A frontmatter field gets its part, chapter, and title, or its key and value. After this stage no weft remains incomplete.

The tokeniser service

Prose carries anchors too, so the tokeniser respects Markdown: an anchor inside an inline code span or a fenced block stays inert, the literal text it appears to be. An inline span is a per-line affair the prose handlers mask, but a fence spans many lines, so tokeniseWefts threads its state through the weft stream with Stream.mapAccum. fenceStep advances that state: a prose line of three backticks opens or closes a fence, a code chunk or a heading resets it, and it reports skipAnchors on every line a fence covers. tokeniseWeft then dispatches each weft by kind to the per-weft handlers, telling each whether to skip its anchors.

TypeScript

export class WeftTokeniser extends Context.Service<WeftTokeniser>()('WeftTokeniser', {
  make: Effect.succeed({
    tokeniseWefts:
      (text: string, delims: AnchorDelims = defaultAnchorDelims) =>
      (source: Stream.Stream<LoomWeft>): Stream.Stream<LoomWeft> =>
        Stream.mapAccum(
          source,
          () => false,
          (inFence, weft) => {
            const [nextFence, skipAnchors] = fenceStep(inFence, weft)
            return [nextFence, [tokeniseWeft(text, weft, delims, skipAnchors)]]
          },
        ),
  }),
}) {
  static readonly layer = Layer.effect(this, this.make)
}

const isProseWeft = (weft: LoomWeft): boolean =>
  weft.type === 'PreambleWeft' || weft.type === 'TildeWeft' || weft.type === 'ProseWeft'

const fenceStep = (inFence: boolean, weft: LoomWeft): readonly [boolean, boolean] => {
  if (!isProseWeft(weft)) return [false, false]
  const delimiter = /^\s*```/.test(weft.source)
  return [delimiter ? !inFence : inFence, inFence || delimiter]
}

const tokeniseWeft = (
  text: string,
  weft: LoomWeft,
  delims: AnchorDelims,
  skipAnchors: boolean,
): LoomWeft =>
  pipe(
    Match.value(weft),
    Match.when({ type: 'HeadingWeft' }, (w) => tokeniseHeading(text, w)),
    Match.when({ type: 'ArrowWeft' }, (w) => tokeniseArrow(text, w, delims)),
    Match.when({ type: 'TildeWeft' }, (w) => tokeniseTilde(text, w, delims, skipAnchors)),
    Match.when({ type: 'PreambleWeft' }, (w) => tokenisePreamble(text, w, delims, skipAnchors)),
    Match.when({ type: 'ProseWeft' }, (w) => tokeniseProse(text, w, delims, skipAnchors)),
    Match.when({ type: 'CodeWeft' }, (w) => tokeniseCode(text, w, delims)),
    Match.when({ type: 'FrontmatterWeft' }, (w) => tokeniseFrontmatter(w)),
    Match.when({ type: 'TocWeft' }, (w) => tokeniseToc(w)),
    Match.exhaustive,
  )

Frontmatter fields

A frontmatter field is one key: value line, and tokeniseFrontmatter settles it by its key. It splits on the first colon and classifies what it finds. A Part <Roman> line announces a numbered part, a Chapter <n> line a numbered chapter — each carrying an ordinal past its keyword and a name beside it. Drop the ordinal and the membership is numberless: a bare Part names a part with no number, a bare Chapter a chapter with no number, for a section a reader orders by hand rather than by position. The joined Part <Roman>, Chapter <n> form carries a chapter by the part it belongs to, split at the comma. Target, Package, Language, Status, and Priority are plain keys, the last two the metadata a roadmap reads. A fence weft passes through whole, and a line that is no field settles bare, with no subtokens.

TypeScript

const tokeniseFrontmatter = (weft: FrontmatterWeft): LoomWeft => {
  if (weft.fence !== undefined) return weft
  const lineText = weft.source
  const line = weft.position.start.line
  const base = weft.position.start.offset
  const colon = lineText.indexOf(':')
  if (colon < 0) return bareFrontmatter(weft)
  const key = trimSpan(lineText.slice(0, colon), base)
  const value = trimSpan(lineText.slice(colon + 1), base + colon + 1)
  return buildFrontmatterField(weft, line, key, value)
}

const frontmatterFields = new Set(['Target', 'Package', 'Language', 'Status', 'Priority'])

const buildFrontmatterField = (
  weft: FrontmatterWeft,
  line: number,
  key: TrimmedSpan,
  value: TrimmedSpan,
): FrontmatterWeft => {
  if (key.value.startsWith('Part ') && key.value.includes(', Chapter '))
    return belongingField(weft, line, key, value)
  if (key.value.startsWith('Part '))
    return FrontmatterWeftSchema.make({
      position: weft.position,
      source: weft.source,
      health: okHealth,
      part: ordinalToken(FrontmatterPartTokenSchema, line, key, 'Part '),
      partName: fieldToken(FrontmatterPartNameTokenSchema, line, value),
    })
  if (key.value === 'Part')
    return FrontmatterWeftSchema.make({
      position: weft.position,
      source: weft.source,
      health: okHealth,
      partName: fieldToken(FrontmatterPartNameTokenSchema, line, value),
    })
  if (key.value.startsWith('Chapter '))
    return FrontmatterWeftSchema.make({
      position: weft.position,
      source: weft.source,
      health: okHealth,
      chapter: ordinalToken(FrontmatterChapterTokenSchema, line, key, 'Chapter '),
      title: fieldToken(FrontmatterTitleTokenSchema, line, value),
    })
  if (key.value === 'Chapter')
    return FrontmatterWeftSchema.make({
      position: weft.position,
      source: weft.source,
      health: okHealth,
      title: fieldToken(FrontmatterTitleTokenSchema, line, value),
    })
  if (frontmatterFields.has(key.value))
    return FrontmatterWeftSchema.make({
      position: weft.position,
      source: weft.source,
      health: okHealth,
      key: fieldToken(FrontmatterKeyTokenSchema, line, key),
      value: fieldToken(FrontmatterValueTokenSchema, line, value),
    })
  return bareFrontmatter(weft)
}

const belongingField = (
  weft: FrontmatterWeft,
  line: number,
  key: TrimmedSpan,
  value: TrimmedSpan,
): FrontmatterWeft => {
  const comma = key.value.indexOf(',')
  const partSpan = trimSpan(key.value.slice(0, comma), key.start)
  const chapterSpan = trimSpan(key.value.slice(comma + 1), key.start + comma + 1)
  return FrontmatterWeftSchema.make({
    position: weft.position,
    source: weft.source,
    health: okHealth,
    part: ordinalToken(FrontmatterPartTokenSchema, line, partSpan, 'Part '),
    chapter: ordinalToken(FrontmatterChapterTokenSchema, line, chapterSpan, 'Chapter '),
    title: fieldToken(FrontmatterTitleTokenSchema, line, value),
  })
}

Three builders shape the tokens. ordinalToken reads a Part III or Chapter 1 marker, its value the ordinal past the keyword. fieldToken builds a plain-valued token — a name, a title, a key, or a path. bareFrontmatter is the fence's passthrough and the fallback for a line that is no field: a weft with no subtokens.

TypeScript

type TrimmedSpan = {
  readonly value: string
  readonly start: number
  readonly end: number
}

const ordinalToken = <T>(
  schema: Scannable<T>,
  line: number,
  key: TrimmedSpan,
  keyword: string,
): T =>
  (schema as any).make({
    position: span(line, key.start, key.end),
    source: key.value,
    health: okHealth,
    value: key.value.slice(keyword.length).trim(),
  })

const fieldToken = <T>(schema: Scannable<T>, line: number, seg: TrimmedSpan): T =>
  (schema as any).make({
    position: span(line, seg.start, seg.end),
    source: seg.value,
    health: okHealth,
    value: seg.value,
  })

const bareFrontmatter = (weft: FrontmatterWeft): FrontmatterWeft =>
  FrontmatterWeftSchema.make({
    position: weft.position,
    source: weft.source,
    health: okHealth,
  })

Table-of-contents entries

tokeniseToc settles one line of a contents section, trying two shapes in turn. A #### heading is a part: its text past the marker becomes a TocPart. A numbered item like 5. The multi-file AST is a chapter: the number becomes a TocChapter and the text a TocTitle, so a later pass can resolve the entry to the chapter that number and title name. Any other line — a blank spacer — settles bare, with no subtokens, the way a frontmatter line that is no field does.

TypeScript

const tocPartMarker = /^(#{1,6}\s+)(\S.*)$/
const tocChapterMarker = /^(\s*)(\d+)(\.\s+)(\S.*)$/

const tokeniseToc = (weft: TocWeft): LoomWeft => {
  const line = weft.position.start.line
  const base = weft.position.start.offset
  const content = weft.source.replace(/\r?\n$/, '')

  const part = tocPartMarker.exec(content)
  if (part) {
    const partSpan = trimSpan(part[2], base + part[1].length)
    return TocWeftSchema.make({
      position: weft.position,
      source: weft.source,
      health: okHealth,
      part: fieldToken(TocPartTokenSchema, line, partSpan),
    })
  }

  const chapter = tocChapterMarker.exec(content)
  if (chapter) {
    const numStart = base + chapter[1].length
    const numSpan = trimSpan(chapter[2], numStart)
    const titleStart = numStart + chapter[2].length + chapter[3].length
    const titleSpan = trimSpan(chapter[4], titleStart)
    return TocWeftSchema.make({
      position: weft.position,
      source: weft.source,
      health: okHealth,
      chapter: fieldToken(TocChapterTokenSchema, line, numSpan),
      title: fieldToken(TocTitleTokenSchema, line, titleSpan),
    })
  }

  return TocWeftSchema.make({
    position: weft.position,
    source: weft.source,
    health: okHealth,
  })
}

Body wefts

These handlers fill the body wefts. inlineAfter pulls a line's inline content after its marker — the code after =>, the prose after ~. tokeniseArrow and tokeniseCode scan a code line for its ::[name] anchors. tokeniseTilde, tokenisePreamble, and tokeniseProse scan a prose line for its anchors, and the preamble for its warps too. maskInlineCode blanks an inline code span before the scan, and blankIfSkipped blanks the whole line inside a fence, so a token survives only where Markdown leaves it live. Each aggregates its subtokens' health into the weft.

TypeScript

const codeProbe = Option.getOrThrow(getProbe(CodeTokenSchema))
const proseProbe = Option.getOrThrow(getProbe(ProseTokenSchema))

const inlineAfter = <T>(
  schema: Schema.Schema<T>,
  probe: RegExp,
  text: string,
  linePosition: Position,
): Option.Option<T> => {
  const lineStart = linePosition.start.offset
  const lineText = text.slice(lineStart, linePosition.end.offset).replace(/\r?\n$/, '')
  return pipe(
    Option.fromNullishOr(probe.exec(lineText)),
    Option.filter((m): m is RegExpExecArray & { index: number } => m.index !== undefined),
    Option.map((m) =>
      (schema as any).make({
        position: span(
          linePosition.start.line,
          lineStart + m.index,
          lineStart + m.index + m[0].length,
        ),
        source: m[0],
        health: okHealth,
      }),
    ),
  )
}

const lineSlice = (text: string, position: Position): string =>
  text.slice(position.start.offset, position.end.offset)

const maskInlineCode = (line: string): string =>
  line.replace(/`[^`]*`/g, (m) => ' '.repeat(m.length))

const blankIfSkipped = (line: string, skip: boolean): string =>
  skip ? ' '.repeat(line.length) : maskInlineCode(line)

const tokeniseArrow = (text: string, weft: ArrowWeft, delims: AnchorDelims): LoomWeft => {
  const code = inlineAfter(CodeTokenSchema, codeProbe, text, weft.position)
  const { tokens: anchors, unexpected } = constructAnchors(
    lineSlice(text, weft.position),
    weft.position,
    delims,
  )
  const status = aggregateStatus([
    weft.arrow.health.status,
    ...Option.toArray(code).map((c) => c.health.status),
    ...anchors.map((a) => a.health.status),
    ...unexpected.map(() => 'error' as const),
  ])
  return ArrowWeftSchema.make({
    position: weft.position,
    source: lineSlice(text, weft.position),
    health: strayHealth(text, status, unexpected),
    unexpected: unexpected.length > 0 ? unexpected : undefined,
    arrow: weft.arrow,
    code: Option.getOrUndefined(code),
    anchors,
  })
}

const tokeniseTilde = (
  text: string,
  weft: TildeWeft,
  delims: AnchorDelims,
  skipAnchors: boolean,
): LoomWeft => {
  const lineText = lineSlice(text, weft.position)
  const prose = inlineAfter(ProseTokenSchema, proseProbe, text, weft.position)
  const { tokens: anchors, unexpected } = constructAnchors(
    lineText,
    weft.position,
    delims,
    blankIfSkipped(lineText, skipAnchors),
  )
  const status = aggregateStatus([
    weft.tilde.health.status,
    ...Option.toArray(prose).map((p) => p.health.status),
    ...anchors.map((a) => a.health.status),
    ...unexpected.map(() => 'error' as const),
  ])
  return TildeWeftSchema.make({
    position: weft.position,
    source: lineText,
    health: strayHealth(text, status, unexpected),
    unexpected: unexpected.length > 0 ? unexpected : undefined,
    tilde: weft.tilde,
    prose: Option.getOrUndefined(prose),
    anchors,
  })
}

const tokenisePreamble = (
  text: string,
  weft: PreambleWeft,
  delims: AnchorDelims,
  skipAnchors: boolean,
): LoomWeft => {
  const lineText = lineSlice(text, weft.position)
  const scanText = blankIfSkipped(lineText, skipAnchors)
  const { tokens: warps, unexpected: warpStray } = constructWarps(lineText, weft.position, scanText)
  const { tokens: anchors, unexpected: anchorStray } = constructAnchors(
    lineText,
    weft.position,
    delims,
    scanText,
  )
  const unexpected = [...warpStray, ...anchorStray]
  const status = aggregateStatus([
    ...warps.map((w) => w.health.status),
    ...anchors.map((a) => a.health.status),
    ...unexpected.map(() => 'error' as const),
  ])
  return PreambleWeftSchema.make({
    position: weft.position,
    source: lineText,
    health: strayHealth(text, status, unexpected),
    unexpected: unexpected.length > 0 ? unexpected : undefined,
    warps,
    anchors,
  })
}

const tokeniseProse = (
  text: string,
  weft: ProseWeft,
  delims: AnchorDelims,
  skipAnchors: boolean,
): LoomWeft => {
  const lineText = lineSlice(text, weft.position)
  const { tokens: anchors, unexpected } = constructAnchors(
    lineText,
    weft.position,
    delims,
    blankIfSkipped(lineText, skipAnchors),
  )
  const status = aggregateStatus([
    ...anchors.map((a) => a.health.status),
    ...unexpected.map(() => 'error' as const),
  ])
  return ProseWeftSchema.make({
    position: weft.position,
    source: lineText,
    health: strayHealth(text, status, unexpected),
    unexpected: unexpected.length > 0 ? unexpected : undefined,
    anchors,
  })
}

const tokeniseCode = (text: string, weft: CodeWeft, delims: AnchorDelims): LoomWeft => {
  const { tokens: anchors, unexpected } = constructAnchors(
    lineSlice(text, weft.position),
    weft.position,
    delims,
  )
  const status = aggregateStatus([
    ...anchors.map((a) => a.health.status),
    ...unexpected.map(() => 'error' as const),
  ])
  return CodeWeftSchema.make({
    position: weft.position,
    source: lineSlice(text, weft.position),
    health: strayHealth(text, status, unexpected),
    unexpected: unexpected.length > 0 ? unexpected : undefined,
    anchors,
  })
}

Health and positions

aggregateStatus folds a list of statuses up the lattice ok < incomplete < warning < error, the worst winning. span builds a position; contentEnd is the offset where a line's content ends, before its newline terminator, and is the bound every construct slices and positions against, so the EOL never leaks into a label, an anchor, or a diagnostic; missingClosing is the error health for an unclosed bracket, its diagnostic spanning the open delimiter to that content end so the squiggle lands on the construct rather than the gap after it. brokenLabel is the error health for a label the schema rejected. It raises an EmptyLabel or a MalformedLabel fault through the catalog, keyed by construct, so the same wording reaches the editor wherever a label is built.

A weft that kept stray fragments takes its health from strayHealth. A stray is a fragment the line had no place for: a second specifier on a heading, an orphan bracket. Its status already fails the weft, but a status alone tells nobody anything. strayHealth gives each stray a StrayFragment diagnostic at its own position, so the tangle refuses the file and the editor underlines the stray itself.

Pairing reports a stray by its delimiters, so a second {Tangle} arrives as an orphan { and an orphan }. A diagnostic that named the two braces apart would make a reader assemble the mistake. strayFragments stitches an orphan opener and the closer that matches it back into the one fragment the author wrote, and the diagnostic quotes {Tangle} whole.

TypeScript

const statusRank: Record<Health['status'], number> = {
  ok: 0,
  incomplete: 1,
  warning: 2,
  error: 3,
}

const joinStatus = (a: Health['status'], b: Health['status']): Health['status'] =>
  statusRank[a] >= statusRank[b] ? a : b

const aggregateStatus = (statuses: ReadonlyArray<Health['status']>): Health['status'] =>
  statuses.reduce(joinStatus, 'ok' as Health['status'])

const span = (line: number, start: number, end: number): Position => ({
  start: { line, offset: start },
  end: { line, offset: end },
})

const contentEnd = (lineText: string, lineStart: number): number =>
  lineStart + lineText.replace(/\r?\n$/, '').length

const missingClosing = (line: number, from: number, lineEnd: number, expected: string): Health =>
  faulty(UnclosedDelimiter({ expected }), span(line, from, lineEnd))

type Fragment = { readonly value: string; readonly position: Position }

const closerOf: Readonly<Record<string, string>> = { '{': '}', '[': ']' }

const closes = (opener: Fragment, stray: UnexpectedToken): boolean =>
  closerOf[opener.value] === stray.value && opener.position.start.line === stray.position.start.line

const strayFragments = (
  text: string,
  unexpected: ReadonlyArray<UnexpectedToken>,
): ReadonlyArray<Fragment> =>
  Array.reduce(unexpected, [] as ReadonlyArray<Fragment>, (fragments, stray) =>
    Option.match(
      Option.filter(Array.last(fragments), (last) => closes(last, stray)),
      {
        onNone: () => [...fragments, { value: stray.value, position: stray.position }],
        onSome: (opener) => [
          ...Array.dropRight(fragments, 1),
          {
            value: text.slice(opener.position.start.offset, stray.position.end.offset),
            position: { start: opener.position.start, end: stray.position.end },
          },
        ],
      },
    ),
  )

const strayHealth = (
  text: string,
  status: Health['status'],
  unexpected: ReadonlyArray<UnexpectedToken>,
): Health => ({
  status,
  diagnostics: Array.flatMap(
    strayFragments(text, unexpected),
    (fragment) => faulty(StrayFragment({ value: fragment.value }), fragment.position).diagnostics,
  ),
})

const brokenLabel = (construct: MalformedConstruct, value: string, position: Position): Health =>
  faulty(value === '' ? EmptyLabel({ construct }) : MalformedLabel({ construct, value }), position)

Label builders

A label that fails its strict pattern must still be kept. buildSpecifierLabel and buildSinkFileLabel each try the schema, and on rejection build a valid token with an empty value, the error health brokenLabel renders, and the rejected text stashed in unexpected — strict validation stays on, and bytes are never dropped. The sink builder reuses the path construct for its broken health, since a malformed file name is a malformed path.

TypeScript

const decodeSpecifierLabel = Schema.decodeUnknownResult(SpecifierLabelTokenSchema)
const decodeSinkFileLabel = Schema.decodeUnknownResult(SinkFileLabelTokenSchema)

const buildSpecifierLabel = (value: string, position: Position): SpecifierLabelToken =>
  pipe(
    decodeSpecifierLabel({
      type: 'SpecifierLabel',
      position,
      source: value,
      health: okHealth,
      value,
    }),
    Result.getOrElse(() =>
      SpecifierLabelTokenSchema.make({
        type: 'SpecifierLabel',
        position,
        source: value,
        health: brokenLabel('specifier', value, position),
        value: '',
        unexpected: [UnexpectedTokenSchema.make({ position, value })],
      }),
    ),
  )

const buildSinkFileLabel = (value: string, position: Position): SinkFileLabelToken =>
  pipe(
    decodeSinkFileLabel({
      type: 'SinkFileLabel',
      position,
      source: value,
      health: okHealth,
      value,
    }),
    Result.getOrElse(() =>
      SinkFileLabelTokenSchema.make({
        type: 'SinkFileLabel',
        position,
        source: value,
        health: brokenLabel('path', value, position),
        value: '',
        unexpected: [UnexpectedTokenSchema.make({ position, value })],
      }),
    ),
  )

Scanners

makeScanner reads a token schema's Probe once and returns a per-line scanner that emits a typed token at every match, with real positions and ok health. The scanners for the specifier and sink delimiters are built from it; the schema's literal value guards against probe drift.

TypeScript

type Scannable<T> = Schema.Schema<T>

type Scanner<T> = (lineText: string, linePosition: Position) => ReadonlyArray<T>

const makeScanner = <T>(schema: Scannable<T>): Scanner<T> => {
  const probe = Option.getOrThrowWith(
    getProbe(schema),
    () => new Error('makeScanner: schema has no Probe annotation'),
  )
  return (lineText, linePosition) => {
    const line = linePosition.start.line
    const lineStart = linePosition.start.offset
    return [...lineText.matchAll(probe)]
      .filter((match) => match.index !== undefined)
      .map((match) => {
        const i = match.index!
        return (schema as any).make({
          position: span(line, lineStart + i, lineStart + i + match[0].length),
          source: match[0],
          health: okHealth,
          value: match[0],
        })
      })
  }
}

const scanSpecifierOpen = makeScanner(SpecifierOpenTokenSchema)
const scanSpecifierClose = makeScanner(SpecifierCloseTokenSchema)
const scanSinkOpen = makeScanner(SinkOpenTokenSchema)
const scanSinkClose = makeScanner(SinkCloseTokenSchema)

Pairing specifiers

Each scanner finds opens and closes independently; construction pairs them. partitionFirstClose matches an open with the first close after it. constructSpecifier pairs { and } into a language label. constructSink pairs [ and ] into a sink, its inner text the file a {Tangle} section emits. When no close follows, each synthesises one at the line's content end — the offset before the newline — so a missing bracket reports once, as expected closing, and the captured label never swallows the line's terminator.

TypeScript

type Construction<T> = {
  readonly token: Option.Option<T>
  readonly unexpected: ReadonlyArray<UnexpectedToken>
}

type Positioned = { readonly position: Position; readonly value: string }

const toUnexpected = (t: Positioned): UnexpectedToken =>
  UnexpectedTokenSchema.make({ position: t.position, value: t.value })

const partitionFirstClose = <C extends { position: Position }>(
  open: { position: Position },
  closes: ReadonlyArray<C>,
): { match: C | null; rest: ReadonlyArray<C> } => {
  const matchIdx = closes.findIndex((c) => c.position.start.offset > open.position.start.offset)
  return matchIdx < 0
    ? { match: null, rest: closes }
    : { match: closes[matchIdx], rest: closes.filter((_, i) => i !== matchIdx) }
}

const buildLabelSpecifier = (
  open: SpecifierOpenToken,
  close: SpecifierCloseToken,
  labelText: string,
  labelPos: Position,
  specifierPos: Position,
  specifierSource: string,
): SpecifierToken => {
  const label = buildSpecifierLabel(labelText, labelPos)
  const status = aggregateStatus([open.health.status, label.health.status, close.health.status])
  return SpecifierTokenSchema.make({
    position: specifierPos,
    source: specifierSource,
    health: { status, diagnostics: [] },
    open,
    label,
    close,
  })
}

const constructSpecifier = (
  opens: ReadonlyArray<SpecifierOpenToken>,
  closes: ReadonlyArray<SpecifierCloseToken>,
  lineText: string,
  linePosition: Position,
): Construction<SpecifierToken> => {
  if (opens.length === 0) {
    return {
      token: Option.none(),
      unexpected: closes.map(toUnexpected),
    }
  }

  const line = linePosition.start.line
  const lineStart = linePosition.start.offset
  const lineEnd = contentEnd(lineText, lineStart)

  const [open, ...extraOpens] = opens
  const { match, rest: extraCloses } = partitionFirstClose(open, closes)

  const close: SpecifierCloseToken =
    match ??
    SpecifierCloseTokenSchema.make({
      position: span(line, lineEnd, lineEnd),
      source: '',
      health: missingClosing(line, open.position.start.offset, lineEnd, '}'),
      value: '}',
    })

  const labelStart = open.position.end.offset
  const labelEnd = match ? match.position.start.offset : lineEnd
  const labelText = lineText.slice(labelStart - lineStart, labelEnd - lineStart)
  const labelPos = span(line, labelStart, labelEnd)
  const specifierPos = span(line, open.position.start.offset, close.position.end.offset)
  const specifierSource = lineText.slice(
    specifierPos.start.offset - lineStart,
    specifierPos.end.offset - lineStart,
  )

  return {
    token: Option.some(
      buildLabelSpecifier(open, close, labelText, labelPos, specifierPos, specifierSource),
    ),
    unexpected: [...extraOpens.map(toUnexpected), ...extraCloses.map(toUnexpected)],
  }
}

const constructSink = (
  opens: ReadonlyArray<SinkOpenToken>,
  closes: ReadonlyArray<SinkCloseToken>,
  lineText: string,
  linePosition: Position,
): Construction<SinkToken> => {
  if (opens.length === 0) {
    return {
      token: Option.none(),
      unexpected: closes.map(toUnexpected),
    }
  }

  const line = linePosition.start.line
  const lineStart = linePosition.start.offset
  const lineEnd = contentEnd(lineText, lineStart)

  const [open, ...extraOpens] = opens
  const { match, rest: extraCloses } = partitionFirstClose(open, closes)

  const close: SinkCloseToken =
    match ??
    SinkCloseTokenSchema.make({
      position: span(line, lineEnd, lineEnd),
      source: '',
      health: missingClosing(line, open.position.start.offset, lineEnd, ']'),
      value: ']',
    })

  const innerStart = open.position.end.offset
  const innerEnd = match ? match.position.start.offset : lineEnd
  const inner = lineText.slice(innerStart - lineStart, innerEnd - lineStart)
  const sinkPos = span(line, open.position.start.offset, close.position.end.offset)
  const sinkSource = lineText.slice(
    sinkPos.start.offset - lineStart,
    sinkPos.end.offset - lineStart,
  )

  const fileTrim = trimSpan(inner, innerStart)
  const file = buildSinkFileLabel(fileTrim.value, span(line, fileTrim.start, fileTrim.end))

  const status = aggregateStatus([open.health.status, file.health.status, close.health.status])

  return {
    token: Option.some(
      SinkTokenSchema.make({
        position: sinkPos,
        source: sinkSource,
        health: { status, diagnostics: [] },
        open,
        file,
        close,
      }),
    ),
    unexpected: [...extraOpens.map(toUnexpected), ...extraCloses.map(toUnexpected)],
  }
}

Pairing warp delimiters

Warps and anchors no longer share delimiters: a warp is {{…}} in a preamble line, an anchor is ::[…] in a code line. pairWarpDelims pairs a preamble line's {{ and }} greedily left to right — pairing an unmatched open with a synthetic }} at end-of-line, and surfacing unmatched closes as stray unexpected tokens. Anchors pair differently: scanAnchorOpen finds the ::[ opens, and each closes on the first ] after it, found directly rather than scanned, because ] is ordinary in product code — only the one closing an open is structural, and a bare ] is never a stray.

TypeScript

const scanWarpOpen = makeScanner(WarpOpenTokenSchema)
const scanWarpClose = makeScanner(WarpCloseTokenSchema)

type WarpPair = { readonly open: WarpOpenToken; readonly close: WarpCloseToken }

type PairAcc = {
  readonly pairs: ReadonlyArray<WarpPair>
  readonly remaining: ReadonlyArray<WarpCloseToken>
}

const pairWarpDelims = (
  opens: ReadonlyArray<WarpOpenToken>,
  closes: ReadonlyArray<WarpCloseToken>,
  lineText: string,
  linePosition: Position,
): {
  readonly pairs: ReadonlyArray<WarpPair>
  readonly stray: ReadonlyArray<UnexpectedToken>
} => {
  const line = linePosition.start.line
  const lineEnd = contentEnd(lineText, linePosition.start.offset)
  const synthClose = (open: WarpOpenToken): WarpCloseToken =>
    WarpCloseTokenSchema.make({
      position: span(line, lineEnd, lineEnd),
      source: '',
      health: missingClosing(line, open.position.start.offset, lineEnd, '}}'),
      value: '}}',
    })

  const { pairs, remaining } = opens.reduce<PairAcc>(
    (acc, open) => {
      const idx = acc.remaining.findIndex(
        (c) => c.position.start.offset > open.position.start.offset,
      )
      return idx < 0
        ? {
            pairs: [...acc.pairs, { open, close: synthClose(open) }],
            remaining: acc.remaining,
          }
        : {
            pairs: [...acc.pairs, { open, close: acc.remaining[idx] }],
            remaining: acc.remaining.filter((_, i) => i !== idx),
          }
    },
    { pairs: [], remaining: closes },
  )

  return { pairs, stray: remaining.map(toUnexpected) }
}

Warp content

A warp's content is name [: type] = value. sliceTop cuts at the first top-level , or ;, tracking bracket depth, since a warp declares one binding; trimSpan trims whitespace and reports the trimmed span. buildWarpName mirrors the label builders. buildOpaqueSegment produces the opaque annotation and default tokens, flagging an empty value and returning any trailing content as unexpected.

TypeScript

type SliceAcc = { readonly depth: number; readonly cut: Option.Option<number> }

const stepSlice = (s: SliceAcc, c: string, i: number): SliceAcc => {
  if (Option.isSome(s.cut)) return s
  if (c === '<' || c === '(' || c === '[') return { depth: s.depth + 1, cut: s.cut }
  if (c === '>' || c === ')' || c === ']') return { depth: Math.max(0, s.depth - 1), cut: s.cut }
  if ((c === ',' || c === ';') && s.depth === 0) return { depth: s.depth, cut: Option.some(i) }
  return s
}

const sliceTop = (raw: string): { kept: string; extraStart: number } => {
  const final = [...raw].reduce<SliceAcc>(stepSlice, {
    depth: 0,
    cut: Option.none(),
  })
  return Option.match(final.cut, {
    onNone: () => ({ kept: raw, extraStart: raw.length }),
    onSome: (idx) => ({ kept: raw.slice(0, idx), extraStart: idx }),
  })
}

const trimSpan = (raw: string, rawStart: number): { value: string; start: number; end: number } => {
  const left = raw.match(/^\s*/)?.[0].length ?? 0
  const right = raw.match(/\s*$/)?.[0].length ?? 0
  return {
    value: raw.slice(left, raw.length - right),
    start: rawStart + left,
    end: rawStart + raw.length - right,
  }
}

const decodeWarpName = Schema.decodeUnknownResult(WarpNameTokenSchema)

const buildWarpName = (value: string, position: Position): WarpNameToken =>
  pipe(
    decodeWarpName({
      type: 'WarpName',
      position,
      source: value,
      health: okHealth,
      value,
    }),
    Result.getOrElse(() =>
      WarpNameTokenSchema.make({
        type: 'WarpName',
        position,
        source: value,
        health: brokenLabel('warpName', value, position),
        value: '',
        unexpected: [UnexpectedTokenSchema.make({ position, value })],
      }),
    ),
  )

type OpaqueMake<T> = (args: {
  type: string
  position: Position
  source: string
  health: Health
  value: string
}) => T

const buildOpaqueSegment = <T>(
  make: OpaqueMake<T>,
  typeName: string,
  construct: EmptyConstruct,
  raw: string,
  rawStart: number,
  line: number,
): { token: T; extras: ReadonlyArray<UnexpectedToken> } => {
  const { kept, extraStart } = sliceTop(raw)
  const { value, start, end } = trimSpan(kept, rawStart)

  const tokenPos = span(line, start, end)
  const health: Health = value === '' ? faulty(EmptyLabel({ construct }), tokenPos) : okHealth

  const token = make({
    type: typeName,
    position: tokenPos,
    source: value,
    health,
    value,
  })

  const extras: UnexpectedToken[] =
    extraStart < raw.length
      ? [
          UnexpectedTokenSchema.make({
            position: span(line, rawStart + extraStart, rawStart + raw.length),
            value: raw.slice(extraStart),
          }),
        ]
      : []

  return { token, extras }
}

const buildWarpAnnotation = (raw: string, rawStart: number, line: number) =>
  buildOpaqueSegment<WarpAnnotationToken>(
    (args) => WarpAnnotationTokenSchema.make(args as any),
    'WarpAnnotation',
    'warpAnnotation',
    raw,
    rawStart,
    line,
  )

const buildWarpDefault = (raw: string, rawStart: number, line: number) =>
  buildOpaqueSegment<WarpDefaultToken>(
    (args) => WarpDefaultTokenSchema.make(args as any),
    'WarpDefault',
    'warpDefault',
    raw,
    rawStart,
    line,
  )

Building a warp

buildWarp reads a warp's content as name [: type] = value. It splits on the binding = first — the value is what a Warp binds — then splits the part before it on : to separate the name from an optional type. findAssign finds that binding =: the first one that is a plain assignment, never part of =>, ==, >=, <=, or !=, so a value that holds those operators is read whole. A Warp with no value binds nothing, so buildWarp faults it with MissingWarpValue. splitDecl separates the name from its optional type, and assembleWarp gathers the parts, aggregates their health, and carries the missing-value fault and any extras.

TypeScript

const findAssign = (s: string): Option.Option<number> => {
  const isCompound = (i: number): boolean => {
    const prev = s[i - 1]
    const next = s[i + 1]
    return (
      prev === '=' || prev === '!' || prev === '<' || prev === '>' || next === '=' || next === '>'
    )
  }
  const idx = [...s].findIndex((c, i) => c === '=' && !isCompound(i))
  return idx < 0 ? Option.none() : Option.some(idx)
}

const findChar = (s: string, c: string): Option.Option<number> => {
  const idx = s.indexOf(c)
  return idx < 0 ? Option.none() : Option.some(idx)
}

interface WarpDecl {
  readonly name: WarpNameToken
  readonly annotation: WarpAnnotationToken | undefined
  readonly extras: ReadonlyArray<UnexpectedToken>
}

const splitDecl = (decl: string, declStart: number, line: number): WarpDecl =>
  Option.match(findChar(decl, ':'), {
    onNone: () => {
      const n = trimSpan(decl, declStart)
      return {
        name: buildWarpName(n.value, span(line, n.start, n.end)),
        annotation: undefined,
        extras: [],
      }
    },
    onSome: (colonIdx) => {
      const n = trimSpan(decl.slice(0, colonIdx), declStart)
      const { token: annotation, extras } = buildWarpAnnotation(
        decl.slice(colonIdx + 1),
        declStart + colonIdx + 1,
        line,
      )
      return {
        name: buildWarpName(n.value, span(line, n.start, n.end)),
        annotation,
        extras,
      }
    },
  })

const buildWarp = (
  open: WarpOpenToken,
  close: WarpCloseToken,
  lineText: string,
  linePosition: Position,
): WarpToken => {
  const line = linePosition.start.line
  const lineStart = linePosition.start.offset
  const contentStart = open.position.end.offset
  const contentEnd = close.position.start.offset
  const content = lineText.slice(contentStart - lineStart, contentEnd - lineStart)
  const warpSource = lineText.slice(
    open.position.start.offset - lineStart,
    close.position.end.offset - lineStart,
  )
  const warpPos = span(line, open.position.start.offset, close.position.end.offset)

  return Option.match(findAssign(content), {
    onSome: (eqIdx) => {
      const { name, annotation, extras } = splitDecl(content.slice(0, eqIdx), contentStart, line)
      const { token: defaultToken, extras: defExtras } = buildWarpDefault(
        content.slice(eqIdx + 1),
        contentStart + eqIdx + 1,
        line,
      )
      return assembleWarp(
        open,
        close,
        name,
        annotation,
        defaultToken,
        [...extras, ...defExtras],
        line,
        warpSource,
      )
    },
    onNone: () => {
      const { name, annotation, extras } = splitDecl(content, contentStart, line)
      const own =
        name.health.status === 'ok'
          ? faulty(MissingWarpValue({ name: name.value }), warpPos)
          : okHealth
      return assembleWarp(open, close, name, annotation, undefined, extras, line, warpSource, own)
    },
  })
}

const assembleWarp = (
  open: WarpOpenToken,
  close: WarpCloseToken,
  name: WarpNameToken,
  annotation: WarpAnnotationToken | undefined,
  defaultToken: WarpDefaultToken | undefined,
  extras: ReadonlyArray<UnexpectedToken>,
  line: number,
  source: string,
  own: Health = okHealth,
): WarpToken => {
  const subStatuses = [
    open.health.status,
    name.health.status,
    ...(annotation ? [annotation.health.status] : []),
    ...(defaultToken ? [defaultToken.health.status] : []),
    close.health.status,
  ]
  const status = aggregateStatus([
    ...subStatuses,
    ...extras.map(() => 'error' as const),
    own.status,
  ])
  return WarpTokenSchema.make({
    type: 'Warp',
    position: span(line, open.position.start.offset, close.position.end.offset),
    source,
    health: { status, diagnostics: own.diagnostics },
    unexpected: extras.length > 0 ? extras : undefined,
    open,
    name,
    annotation,
    default: defaultToken,
    close,
  })
}

Building an anchor

buildWarpAnchor reads a ::[…] reference's content as its name — an identifier or a multi-word heading name. Content the anchor pattern rejects (empty) yields an error-health name and surfaces the text as unexpected. anchorTargetAt reads an optional (path) right after the close — the .loom file a directory anchor places. The path is opaque here, for a later pass to resolve; an open paren with no close stays product code. anchorSpecifierAt reads a {…} or […] in that same spot instead — a misplaced language or sink — through constructSpecifier or constructSink, so it reads the same as a heading's; an open with no close stays product code, never a broken specifier. A target and a specifier exclude each other: the one character after the close picks at most one. assembleAnchor gathers the parts and stretches the anchor's span across whichever trails it.

TypeScript

const unexpectedIfNonEmpty = (position: Position, value: string): ReadonlyArray<UnexpectedToken> =>
  value.length > 0 ? [UnexpectedTokenSchema.make({ position, value })] : []

const decodeWarpAnchorName = Schema.decodeUnknownResult(WarpAnchorNameTokenSchema)

type AnchorSpecifier = SpecifierToken | SinkToken

const anchorSpecifierAt = (
  close: AnchorCloseToken,
  lineText: string,
  linePosition: Position,
): Option.Option<{ specifier: AnchorSpecifier; end: number }> => {
  const line = linePosition.start.line
  const lineStart = linePosition.start.offset
  const openAt = close.position.end.offset
  const ch = lineText[openAt - lineStart]
  if (ch === '{') {
    const closeRel = lineText.indexOf('}', openAt - lineStart)
    if (closeRel < 0) return Option.none()
    const closeAt = lineStart + closeRel
    const open = SpecifierOpenTokenSchema.make({
      position: span(line, openAt, openAt + 1),
      source: '{',
      health: okHealth,
      value: '{',
    })
    const closeTok = SpecifierCloseTokenSchema.make({
      position: span(line, closeAt, closeAt + 1),
      source: '}',
      health: okHealth,
      value: '}',
    })
    return Option.map(
      constructSpecifier([open], [closeTok], lineText, linePosition).token,
      (specifier) => ({ specifier, end: closeAt + 1 }),
    )
  }
  if (ch === '[') {
    const closeRel = lineText.indexOf(']', openAt - lineStart)
    if (closeRel < 0) return Option.none()
    const closeAt = lineStart + closeRel
    const open = SinkOpenTokenSchema.make({
      position: span(line, openAt, openAt + 1),
      source: '[',
      health: okHealth,
      value: '[',
    })
    const closeTok = SinkCloseTokenSchema.make({
      position: span(line, closeAt, closeAt + 1),
      source: ']',
      health: okHealth,
      value: ']',
    })
    return Option.map(
      constructSink([open], [closeTok], lineText, linePosition).token,
      (specifier) => ({ specifier, end: closeAt + 1 }),
    )
  }
  return Option.none()
}

const anchorTargetAt = (
  close: AnchorCloseToken,
  lineText: string,
  linePosition: Position,
): Option.Option<{ target: WarpAnchorTargetToken; end: number }> => {
  const line = linePosition.start.line
  const lineStart = linePosition.start.offset
  const openAt = close.position.end.offset
  if (lineText[openAt - lineStart] !== '(') return Option.none()
  const closeRel = lineText.indexOf(')', openAt - lineStart)
  if (closeRel < 0) return Option.none()
  const closeAt = lineStart + closeRel
  const inner = lineText.slice(openAt + 1 - lineStart, closeAt - lineStart)
  return Option.some({
    target: WarpAnchorTargetTokenSchema.make({
      position: span(line, openAt, closeAt + 1),
      source: lineText.slice(openAt - lineStart, closeAt + 1 - lineStart),
      health: okHealth,
      value: inner.trim(),
    }),
    end: closeAt + 1,
  })
}

const buildWarpAnchor = (
  open: AnchorOpenToken,
  close: AnchorCloseToken,
  lineText: string,
  linePosition: Position,
): { anchor: WarpAnchorToken; extras: ReadonlyArray<UnexpectedToken> } => {
  const line = linePosition.start.line
  const lineStart = linePosition.start.offset
  const contentStart = open.position.end.offset
  const contentEnd = close.position.start.offset
  const content = lineText.slice(contentStart - lineStart, contentEnd - lineStart)
  const { value, start, end } = trimSpan(content, contentStart)
  const namePos = span(line, start, end)
  const target = anchorTargetAt(close, lineText, linePosition)
  const spec = anchorSpecifierAt(close, lineText, linePosition)
  const anchorEnd = pipe(
    target,
    Option.map((t) => t.end),
    Option.orElse(() => Option.map(spec, (s) => s.end)),
    Option.getOrElse(() => close.position.end.offset),
  )
  const anchorSource = lineText.slice(open.position.start.offset - lineStart, anchorEnd - lineStart)
  const targetTok = Option.getOrUndefined(Option.map(target, (t) => t.target))
  const specifier = Option.getOrUndefined(Option.map(spec, (s) => s.specifier))

  return pipe(
    decodeWarpAnchorName({
      type: 'WarpAnchorName',
      position: namePos,
      source: value,
      health: okHealth,
      value,
    }),
    Result.match({
      onFailure: () => ({
        anchor: assembleAnchor(
          open,
          close,
          WarpAnchorNameTokenSchema.make({
            type: 'WarpAnchorName',
            position: namePos,
            source: value,
            health: brokenLabel('anchorName', value, namePos),
            value: '',
          }),
          line,
          anchorSource,
          anchorEnd,
          targetTok,
          specifier,
        ),
        extras: unexpectedIfNonEmpty(namePos, value),
      }),
      onSuccess: (name) => ({
        anchor: assembleAnchor(
          open,
          close,
          name,
          line,
          anchorSource,
          anchorEnd,
          targetTok,
          specifier,
        ),
        extras: [],
      }),
    }),
  )
}

const assembleAnchor = (
  open: AnchorOpenToken,
  close: AnchorCloseToken,
  name: WarpAnchorNameToken,
  line: number,
  source: string,
  end: number,
  target: WarpAnchorTargetToken | undefined,
  specifier: AnchorSpecifier | undefined,
): WarpAnchorToken => {
  const status = aggregateStatus([
    open.health.status,
    name.health.status,
    close.health.status,
    ...(target ? [target.health.status] : []),
    ...(specifier ? [specifier.health.status] : []),
  ])
  return WarpAnchorTokenSchema.make({
    type: 'WarpAnchor',
    position: span(line, open.position.start.offset, end),
    source,
    health: { status, diagnostics: [] },
    open,
    name,
    close,
    target,
    specifier,
  })
}

Scanning a line for warps and anchors

constructWarps scans a line, pairs the {{/}} delimiters, and builds a warp per pair. constructAnchors scans a line for ::[ opens and builds an anchor for each, closing it on the first ] that follows. Both take a scan text apart from the line itself, so a prose handler passes the line with its Markdown code spans blanked — a token inside backticks is found nowhere, while the one outside still reads its content from the original line. There is no fallback: a token is always a token. A malformed anchor — unclosed, or empty-named — is kept as an anchor carrying error health, never reinterpreted as literal code; an anchor whose name does not resolve likewise stays an anchor. Each is a diagnostic Loom reports, and a hard failure at tangle. The rare literal ::[ in product code is written through the host language's character escape, so the tokeniser never has to guess whether a ::[ was meant.

TypeScript

const constructWarps = (
  lineText: string,
  linePosition: Position,
  scanText: string = lineText,
): {
  tokens: ReadonlyArray<WarpToken>
  unexpected: ReadonlyArray<UnexpectedToken>
} => {
  const opens = scanWarpOpen(scanText, linePosition)
  const closes = scanWarpClose(scanText, linePosition)
  const { pairs, stray } = pairWarpDelims(opens, closes, scanText, linePosition)
  const tokens = pairs.map(({ open, close }) => buildWarp(open, close, lineText, linePosition))
  return { tokens, unexpected: stray }
}

const indicesOf = (text: string, needle: string): ReadonlyArray<number> => {
  const step = (found: ReadonlyArray<number>, from: number): ReadonlyArray<number> => {
    const at = text.indexOf(needle, from)
    return at < 0 ? found : step([...found, at], at + needle.length)
  }
  return step([], 0)
}

const scanAnchorOpens = (
  open: string,
  lineText: string,
  linePosition: Position,
): ReadonlyArray<AnchorOpenToken> => {
  const line = linePosition.start.line
  const lineStart = linePosition.start.offset
  return indicesOf(lineText, open).map((i) =>
    AnchorOpenTokenSchema.make({
      position: span(line, lineStart + i, lineStart + i + open.length),
      source: open,
      health: okHealth,
      value: open,
    }),
  )
}

const anchorClose = (
  open: AnchorOpenToken,
  close: string,
  lineText: string,
  linePosition: Position,
): AnchorCloseToken => {
  const line = linePosition.start.line
  const lineStart = linePosition.start.offset
  const lineEnd = contentEnd(lineText, lineStart)
  const rel = lineText.indexOf(close, open.position.end.offset - lineStart)
  return rel < 0
    ? AnchorCloseTokenSchema.make({
        position: span(line, lineEnd, lineEnd),
        source: '',
        health: missingClosing(line, open.position.start.offset, lineEnd, close),
        value: close,
      })
    : AnchorCloseTokenSchema.make({
        position: span(line, lineStart + rel, lineStart + rel + close.length),
        source: close,
        health: okHealth,
        value: close,
      })
}

const constructAnchors = (
  lineText: string,
  linePosition: Position,
  delims: AnchorDelims,
  scanText: string = lineText,
): {
  tokens: ReadonlyArray<WarpAnchorToken>
  unexpected: ReadonlyArray<UnexpectedToken>
} => {
  const built = scanAnchorOpens(delims.open, scanText, linePosition).map((open) =>
    buildWarpAnchor(
      open,
      anchorClose(open, delims.close, scanText, linePosition),
      lineText,
      linePosition,
    ),
  )
  return {
    tokens: built.map((b) => b.anchor),
    unexpected: built.flatMap((b) => b.extras),
  }
}

The heading title

headingTitle is the text between the heading marker and the first structural token, trimmed. It returns nothing when that span is empty, and text after the first structural token is dropped.

TypeScript

const headingTitle = (
  text: string,
  position: Position,
  headingStartEnd: number,
  consumed: ReadonlyArray<Position>,
): Option.Option<HeadingTitleToken> => {
  const line = position.start.line
  const lineText = text.slice(position.start.offset, position.end.offset)
  const trailingWs = lineText.match(/\s*$/)?.[0].length ?? 0

  const titleEnd = consumed
    .map((p) => p.start.offset)
    .filter((offset) => offset >= headingStartEnd)
    .reduce((min, offset) => Math.min(min, offset), position.end.offset - trailingWs)

  const raw = text.slice(headingStartEnd, titleEnd)
  const leading = raw.match(/^\s*/)?.[0].length ?? 0
  const trailing = raw.match(/\s*$/)?.[0].length ?? 0
  const start = headingStartEnd + leading
  const end = titleEnd - trailing
  if (start >= end) return Option.none()

  return Option.some(
    HeadingTitleTokenSchema.make({
      position: span(line, start, end),
      source: text.slice(start, end),
      health: okHealth,
    }),
  )
}

Scanning a heading

scanHeading builds a heading's subnodes — the language specifier, the sink, and the title — from one line, recording the spans they consume so the title knows where to stop.

TypeScript

type HeadingTokens = {
  readonly specifier: Option.Option<SpecifierToken>
  readonly sink: Option.Option<SinkToken>
  readonly title: Option.Option<HeadingTitleToken>
  readonly unexpected: ReadonlyArray<UnexpectedToken>
}

const scanHeading = (text: string, position: Position, headingStartEnd: number): HeadingTokens => {
  const lineText = text.slice(position.start.offset, position.end.offset)

  const specResult = constructSpecifier(
    scanSpecifierOpen(lineText, position),
    scanSpecifierClose(lineText, position),
    lineText,
    position,
  )
  const sinkResult = constructSink(
    scanSinkOpen(lineText, position),
    scanSinkClose(lineText, position),
    lineText,
    position,
  )

  const unexpected = [...specResult.unexpected, ...sinkResult.unexpected]

  const consumed: ReadonlyArray<Position> = [
    ...Option.toArray(specResult.token).map((s) => s.position),
    ...Option.toArray(sinkResult.token).map((s) => s.position),
    ...unexpected.map((u) => u.position),
  ]
  const title = headingTitle(text, position, headingStartEnd, consumed)

  return {
    specifier: specResult.token,
    sink: sinkResult.token,
    title,
    unexpected,
  }
}

Tokenising a heading

tokeniseHeading assembles the heading weft from its scanned subnodes — the title, the specifier, and the sink — and aggregates their health. The title names the section: normaliseTitle turns it into an identifier — What the compiler draws on becomes WhatTheCompilerDrawsOn — which the product pass reads to identify the section. Naming makes no name unique; two titles that normalise alike are a duplicate the diagnostics mark.

TypeScript

export const normaliseTitle = (title: string): string => {
  const pascal = (title.match(/[A-Za-z0-9]+/g) ?? [])
    .map((w) => w.charAt(0).toUpperCase() + w.slice(1))
    .join('')
  return /^[A-Za-z]/.test(pascal) ? pascal : `_${pascal}`
}

const tokeniseHeading = (text: string, weft: HeadingWeft): LoomWeft => {
  const { specifier, sink, title, unexpected } = scanHeading(
    text,
    weft.position,
    weft.headingStart.position.end.offset,
  )

  const status = aggregateStatus([
    weft.headingStart.health.status,
    ...Option.toArray(specifier).map((s) => s.health.status),
    ...Option.toArray(sink).map((s) => s.health.status),
    ...Option.toArray(title).map((t) => t.health.status),
    ...unexpected.map(() => 'error' as const),
  ])

  return HeadingWeftSchema.make({
    position: weft.position,
    source: lineSlice(text, weft.position),
    health: strayHealth(text, status, unexpected),
    unexpected: unexpected.length > 0 ? unexpected : undefined,
    headingStart: weft.headingStart,
    title: Option.getOrUndefined(title),
    specifier: Option.getOrUndefined(specifier),
    sink: Option.getOrUndefined(sink),
  })
}

What this module draws on

Effect supplies Stream and Match for the per-weft pass, Schema for decoding, and the rest. The token schemas and getProbe come from LoomTokens, the weft schemas from Weft, and the health and UnexpectedToken types from the foundation. LoomFault gives faulty and the fault constructors each diagnostic here is built from.

TypeScript

import { Array, Context, Effect, Layer, Match, Option, Result, Schema, Stream, pipe } from 'effect'
import {
  okHealth,
  type Health,
  type Position,
  type UnexpectedToken,
  UnexpectedTokenSchema,
} from '@athrio/loom-ast/LoomNode'
import {
  CodeTokenSchema,
  FrontmatterChapterTokenSchema,
  FrontmatterKeyTokenSchema,
  FrontmatterPartTokenSchema,
  FrontmatterPartNameTokenSchema,
  FrontmatterTitleTokenSchema,
  FrontmatterValueTokenSchema,
  HeadingTitleTokenSchema,
  ProseTokenSchema,
  SinkCloseTokenSchema,
  SinkFileLabelTokenSchema,
  SinkOpenTokenSchema,
  SinkTokenSchema,
  SpecifierCloseTokenSchema,
  SpecifierLabelTokenSchema,
  SpecifierOpenTokenSchema,
  SpecifierTokenSchema,
  TocChapterTokenSchema,
  TocPartTokenSchema,
  TocTitleTokenSchema,
  AnchorCloseTokenSchema,
  AnchorOpenTokenSchema,
  defaultAnchorDelims,
  WarpAnchorNameTokenSchema,
  WarpAnchorTargetTokenSchema,
  WarpAnchorTokenSchema,
  WarpAnnotationTokenSchema,
  WarpCloseTokenSchema,
  WarpDefaultTokenSchema,
  WarpNameTokenSchema,
  WarpOpenTokenSchema,
  WarpTokenSchema,
  getProbe,
  type SinkCloseToken,
  type SinkFileLabelToken,
  type SinkOpenToken,
  type SinkToken,
  type SpecifierCloseToken,
  type SpecifierLabelToken,
  type SpecifierOpenToken,
  type SpecifierToken,
  type AnchorCloseToken,
  type AnchorDelims,
  type AnchorOpenToken,
  type HeadingTitleToken,
  type WarpAnchorNameToken,
  type WarpAnchorTargetToken,
  type WarpAnchorToken,
  type WarpAnnotationToken,
  type WarpCloseToken,
  type WarpDefaultToken,
  type WarpNameToken,
  type WarpOpenToken,
  type WarpToken,
} from '@athrio/loom-ast/LoomTokens'
import {
  EmptyLabel,
  faulty,
  MalformedLabel,
  MissingWarpValue,
  StrayFragment,
  UnclosedDelimiter,
  type EmptyConstruct,
  type MalformedConstruct,
} from '#ast/LoomFault'
import {
  type ArrowWeft,
  ArrowWeftSchema,
  type CodeWeft,
  CodeWeftSchema,
  type FrontmatterWeft,
  FrontmatterWeftSchema,
  type HeadingWeft,
  HeadingWeftSchema,
  type LoomWeft,
  type PreambleWeft,
  PreambleWeftSchema,
  type ProseWeft,
  ProseWeftSchema,
  type TildeWeft,
  TildeWeftSchema,
  type TocWeft,
  TocWeftSchema,
} from '@athrio/loom-ast/Weft'

The module

TypeScript

::[What this module draws on]

::[The tokeniser service]

::[Frontmatter fields]

::[Table-of-contents entries]

::[Body wefts]

::[Health and positions]

::[Label builders]

::[Scanners]

::[Pairing specifiers]

::[Pairing warp delimiters]

::[Warp content]

::[Building a warp]

::[Building an anchor]

::[Scanning a line for warps and anchors]

::[The heading title]

::[Scanning a heading]

::[Tokenising a heading]