Optimize consuming grammar

Sources/Bootstrap/Swift/Literal.swift

@@ -8,10 +8,6 @@
 enum Literal {
     enum Wildcard: GrammarLiteral {
         static let literal: Character = "\u{0}" // character never read
-
-        static func consume(stream: inout Stream, context: GrammarContext) -> StreamStateMatch<Output> {
-            stream.next()
-        }
     }
 
     enum LineFeed: GrammarLiteral {

Sources/Grammar/Consume.swift

@@ -0,0 +1,136 @@
+//
+//  Consume.swift
+//  atom
+//
+//  Created by George Elsham on 29/03/2026.
+//
+
+enum Consume {
+    static func consumeMatch(match: any GrammarMatch.Type, stream: inout Stream, context: GrammarContext) -> StreamResult {
+        let source = GrammarPipelineSource(match: match)
+        return consumeSource(source: source, stream: &stream, context: context)
+    }
+
+    static func consumeSource(source: GrammarPipelineSource, stream: inout Stream, context: GrammarContext) -> StreamResult {
+        guard !stream.isEnd() else {
+            return source.canAcceptNothing() ? .doConsume(RawStringIr(string: "")) : .dontConsume
+        }
+
+        headsLoop: for (literal, head) in source.heads {
+            var s = stream
+            switch consumeLiteral(literal: literal.value, stream: &s) {
+            case .dontConsume:
+                continue
+            case .doConsume:
+                let res = consumeHead(head: head, stream: &s, context: context)
+                switch res {
+                case .dontConsume:
+                    break headsLoop
+                case .doConsume, .error:
+                    stream = s
+                    return res
+                }
+            case let .error(diagnostic):
+                return .error(diagnostic)
+            }
+        }
+        if !source.wildcard.bodies.isEmpty {
+            var s = stream
+            switch consumeWildcard(stream: &s) {
+            case .dontConsume:
+                break
+            case .doConsume:
+                let res = consumeHead(head: source.wildcard, stream: &s, context: context)
+                switch res {
+                case .dontConsume:
+                    break
+                case .doConsume, .error:
+                    stream = s
+                    return res
+                }
+            case let .error(diagnostic):
+                return .error(diagnostic)
+            }
+        }
+        return consumeHead(head: source.empty, stream: &stream, context: context)
+    }
+
+    static func consumeHead(head: GrammarPipelineHead, stream: inout Stream, context: GrammarContext) -> StreamResult {
+        var hasSeenEmpty = false
+        var greediest: (stream: Stream, result: StreamResult)?
+        var successfuls = [GrammarPipelineBody]()
+        bodyLoop: for body in head.bodies {
+            for successful in successfuls {
+                if successful.rest.starts(with: body.rest, by: { $0 == $1 }) {
+                    // Isn't doing any better, continue
+                    continue bodyLoop
+                }
+            }
+
+            guard let source = GrammarPipelineSource(parts: body.rest) else {
+                if stream.isEnd() {
+                    // Shortcut because nothing else can be consumed anyways
+                    return .doConsume(RawStringIr(string: ""))
+                }
+                hasSeenEmpty = true
+                continue
+            }
+
+            var s = stream
+            let res = consumeSource(source: source, stream: &s, context: context)
+            switch res {
+            case .dontConsume:
+                continue
+            case .doConsume, .error:
+                successfuls.append(body)
+                if let g = greediest {
+                    if s.isAheadOf(stream: g.stream) {
+                        greediest = (stream: s, result: res)
+                    }
+                } else {
+                    greediest = (stream: s, result: res)
+                }
+            }
+        }
+        if let greediest {
+            stream = greediest.stream
+            return greediest.result
+        }
+
+        return hasSeenEmpty ? .doConsume(RawStringIr(string: "")) : .dontConsume
+    }
+
+    static func consumeLiteral(literal: any GrammarLiteral.Type, stream: inout Stream) -> StreamResult {
+        switch stream.nextIf(char: literal.literal) {
+        case .dontConsume:
+            return .dontConsume
+        case let .doConsume(ir):
+//            print("consume ir = \(ir)")
+            return .doConsume(ir)
+        case .end:
+            fatalError("Unreachable")
+        case let .error(diagnostic):
+            return .error(diagnostic)
+        }
+    }
+
+    static func consumeWildcard(stream: inout Stream) -> StreamResult {
+        switch stream.next() {
+        case .dontConsume:
+            return .dontConsume
+        case let .doConsume(ir):
+//            print("consume wildcard ir = \(ir)")
+            return .doConsume(ir)
+        case .end:
+            fatalError("Unreachable")
+        case let .error(diagnostic):
+            return .error(diagnostic)
+        }
+    }
+}
+
+enum StreamResult {
+    case dontConsume
+    case doConsume(RawStringIr)
+    case error(Diagnostic)
+}

Sources/Grammar/Context.swift

@@ -0,0 +1,8 @@
+//
+//  Context.swift
+//  atom
+//
+//  Created by George Elsham on 31/03/2026.
+//
+
+struct GrammarContext {}

Sources/Grammar/Pipeline.swift

@@ -0,0 +1,171 @@
+//
+//  Pipeline.swift
+//  atom
+//
+//  Created by George Elsham on 26/03/2026.
+//
+
+struct GrammarPipelineSource {
+    var heads: [GrammarPipelineLiteral: GrammarPipelineHead]
+    var wildcard: GrammarPipelineHead
+    var empty: GrammarPipelineHead
+
+    init(match: any GrammarMatch.Type) {
+        self.init(match: match, rest: [])
+    }
+
+    init?(parts: [any Grammar.Type].SubSequence) {
+        let id = parts.map { ObjectIdentifier($0) }
+        if let source = partsCache[id] {
+            guard let source else {
+                return nil
+            }
+            self = source
+        } else {
+            let source = GrammarPipelineSource(uncachedParts: parts)
+            partsCache.updateValue(source, forKey: id)
+            guard let source else {
+                return nil
+            }
+            self = source
+        }
+    }
+
+    private init() {
+        heads = [:]
+        wildcard = GrammarPipelineHead(bodies: [])
+        empty = GrammarPipelineHead(bodies: [])
+    }
+
+    private init(literal: any GrammarLiteral.Type, rest: [any Grammar.Type].SubSequence) {
+        let literal = GrammarPipelineLiteral(value: literal)
+        let head = GrammarPipelineHead(bodies: [GrammarPipelineBody(rest: rest)])
+        heads = [literal: head]
+        wildcard = GrammarPipelineHead(bodies: [])
+        empty = GrammarPipelineHead(bodies: [])
+    }
+
+    private init(wildcardWithRest rest: [any Grammar.Type].SubSequence) {
+        heads = [:]
+        wildcard = GrammarPipelineHead(bodies: [GrammarPipelineBody(rest: rest)])
+        empty = GrammarPipelineHead(bodies: [])
+    }
+
+    private init(match: any GrammarMatch.Type, rest: [any Grammar.Type].SubSequence) {
+        let patterns = match.patterns.map { pattern in
+            pattern.anyParts()
+        }
+        self = GrammarPipelineSource(patterns: patterns, rest: rest)
+    }
+
+    private init(patterns: [[any Grammar.Type]], rest: [any Grammar.Type].SubSequence) {
+        var source = GrammarPipelineSource()
+        for parts in patterns {
+            guard let pipelines = GrammarPipelineSource(parts: .SubSequence(parts)) else {
+                let body = GrammarPipelineBody(rest: rest)
+                source.empty.bodies.append(body)
+                continue
+            }
+            source.merge(with: pipelines, rest: rest)
+        }
+        self = source
+    }
+
+    private init?(uncachedParts parts: [any Grammar.Type].SubSequence) {
+        guard let first = parts.first else {
+            return nil
+        }
+        let rest = parts.dropFirst()
+
+        if let literal = first as? any GrammarLiteral.Type {
+            if literal == Literal.Wildcard.self {
+                self = GrammarPipelineSource(wildcardWithRest: rest)
+            } else {
+                self = GrammarPipelineSource(literal: literal, rest: rest)
+            }
+        } else if let match = first as? any GrammarMatch.Type {
+            var source = GrammarPipelineSource(match: match, rest: rest)
+            if source.canAcceptNothing(), let new = Self(parts: rest) {
+                source.merge(with: new, rest: [])
+            }
+            self = source
+        } else {
+            fatalError("Unreachable")
+        }
+    }
+
+    private mutating func combine(with other: GrammarPipelineHead, literal: GrammarPipelineLiteral) {
+        if heads[literal] == nil {
+            heads[literal] = other
+        } else {
+            heads[literal]!.bodies.append(contentsOf: other.bodies)
+        }
+    }
+
+    private mutating func merge(with other: GrammarPipelineSource, rest: [any Grammar.Type].SubSequence) {
+        for (literal, var head) in other.heads {
+            for index in head.bodies.indices {
+                head.bodies[index].rest.append(contentsOf: rest)
+            }
+            combine(with: head, literal: literal)
+        }
+
+        var other = other
+
+        for index in other.wildcard.bodies.indices {
+            other.wildcard.bodies[index].rest.append(contentsOf: rest)
+        }
+        wildcard.bodies.append(contentsOf: other.wildcard.bodies)
+
+        for index in other.empty.bodies.indices {
+            other.empty.bodies[index].rest.append(contentsOf: rest)
+        }
+        empty.bodies.append(contentsOf: other.empty.bodies)
+    }
+
+    func canAcceptNothing() -> Bool {
+        empty.bodies.contains { body in
+            body.rest.allSatisfy(isEmpty(grammar:))
+        }
+    }
+}
+
+struct GrammarPipelineHead {
+    var bodies: [GrammarPipelineBody]
+}
+
+struct GrammarPipelineBody {
+    var rest: [any Grammar.Type].SubSequence
+}
+
+struct GrammarPipelineLiteral: Hashable {
+    let value: any GrammarLiteral.Type
+
+    func hash(into hasher: inout Hasher) {
+        hasher.combine(ObjectIdentifier(value))
+    }
+
+    static func == (lhs: GrammarPipelineLiteral, rhs: GrammarPipelineLiteral) -> Bool {
+        lhs.value == rhs.value
+    }
+}
+
+nonisolated(unsafe) private var partsCache: [[ObjectIdentifier]: GrammarPipelineSource?] = [:]
+
+private func isEmpty(grammar: any Grammar.Type) -> Bool {
+    if grammar is any GrammarLiteral.Type {
+        return false
+    } else if let match = grammar as? any GrammarMatch.Type {
+        patternLoop: for pattern in match.patterns {
+            for part in pattern.anyParts() {
+                if !isEmpty(grammar: part) {
+                    continue patternLoop
+                }
+            }
+            return true
+        }
+        return false
+    } else {
+        fatalError("Unreachable")
+    }
+}