From 598a4d8958b73fe00fa64804e2df115ad3db3c4d Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Spahn Date: Sat, 18 Jul 2026 17:16:11 +0200 Subject: [PATCH] fix(cimnotebook): stop highlighting bare "<" operator as IRI in IntelliJ (GH-52) - The hand-written lexer treated every "<" as the start of an IRI reference and consumed to ">" or end of line, so comparisons like FILTER(?a < ?b) colored the rest of the line as an IRI. - "<" now only opens an IRI token when a closing ">" follows with no character that IRIREF forbids in between (whitespace/controls, <"{}|^`\), matching the TextMate rule the VS Code extension already uses; otherwise it lexes as the less-than operator. - Add JUnit 5 test wiring to the Gradle build and lexer regression tests. --- cimnotebook/intellij/build.gradle.kts | 6 + .../cimnotebook/intellij/CimnotebookLexer.kt | 26 ++-- .../intellij/CimnotebookLexerTest.kt | 142 ++++++++++++++++++ 3 files changed, 165 insertions(+), 9 deletions(-) create mode 100644 cimnotebook/intellij/src/test/kotlin/de/soptim/opencgmes/cimnotebook/intellij/CimnotebookLexerTest.kt diff --git a/cimnotebook/intellij/build.gradle.kts b/cimnotebook/intellij/build.gradle.kts index ba8ef9d6..d989a202 100644 --- a/cimnotebook/intellij/build.gradle.kts +++ b/cimnotebook/intellij/build.gradle.kts @@ -117,6 +117,12 @@ dependencies { // Update lsp4ijVersion in gradle.properties to the latest available release. plugins("com.redhat.devtools.lsp4ij:${providers.gradleProperty("lsp4ijVersion").get()}") } + testImplementation("org.junit.jupiter:junit-jupiter:5.14.3") + testRuntimeOnly("org.junit.platform:junit-platform-launcher") +} + +tasks.test { + useJUnitPlatform() } // --------------------------------------------------------------------------- diff --git a/cimnotebook/intellij/src/main/kotlin/de/soptim/opencgmes/cimnotebook/intellij/CimnotebookLexer.kt b/cimnotebook/intellij/src/main/kotlin/de/soptim/opencgmes/cimnotebook/intellij/CimnotebookLexer.kt index eb008578..c723437c 100644 --- a/cimnotebook/intellij/src/main/kotlin/de/soptim/opencgmes/cimnotebook/intellij/CimnotebookLexer.kt +++ b/cimnotebook/intellij/src/main/kotlin/de/soptim/opencgmes/cimnotebook/intellij/CimnotebookLexer.kt @@ -67,6 +67,19 @@ class CimnotebookLexer : LexerBase() { // Returns the char at position i, or NUL if out of bounds. private fun at(i: Int): Char = if (i < myBufferEnd) myBuffer[i] else '\u0000' + // Whether the '<' at myTokenEnd opens an IRIREF: a '>' must follow before any + // character the SPARQL grammar excludes there (whitespace/controls and <"{}|^`\). + private fun hasIriTail(): Boolean { + var i = myTokenEnd + 1 + while (i < myBufferEnd) { + val ch = myBuffer[i] + if (ch == '>') return true + if (ch <= ' ' || ch in "<\"{}|^`\\") return false + i++ + } + return false + } + private fun nextToken(): IElementType { val c = at(myTokenEnd) @@ -112,16 +125,11 @@ class CimnotebookLexer : LexerBase() { return SparqlTokenTypes.STRING } - // IRI reference <…> - if (c == '<') { + // IRI reference <…> — only when a closing '>' follows with none of the + // characters IRIREF forbids in between; a bare '<' is the less-than operator. + if (c == '<' && hasIriTail()) { + while (myBuffer[myTokenEnd] != '>') myTokenEnd++ myTokenEnd++ - while (myTokenEnd < myBufferEnd && - myBuffer[myTokenEnd] != '>' && - myBuffer[myTokenEnd] != '\n' - ) { - myTokenEnd++ - } - if (myTokenEnd < myBufferEnd) myTokenEnd++ return SparqlTokenTypes.IRI } diff --git a/cimnotebook/intellij/src/test/kotlin/de/soptim/opencgmes/cimnotebook/intellij/CimnotebookLexerTest.kt b/cimnotebook/intellij/src/test/kotlin/de/soptim/opencgmes/cimnotebook/intellij/CimnotebookLexerTest.kt new file mode 100644 index 00000000..84d558ae --- /dev/null +++ b/cimnotebook/intellij/src/test/kotlin/de/soptim/opencgmes/cimnotebook/intellij/CimnotebookLexerTest.kt @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2026 SOPTIM AG + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package de.soptim.opencgmes.cimnotebook.intellij + +import com.intellij.psi.tree.IElementType +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test + +class CimnotebookLexerTest { + private fun tokens(text: String): List> { + val lexer = CimnotebookLexer() + lexer.start(text, 0, text.length, 0) + val result = mutableListOf>() + while (true) { + val type = lexer.tokenType ?: break + result.add(type to text.substring(lexer.tokenStart, lexer.tokenEnd)) + lexer.advance() + } + return result + } + + private fun significantTokens(text: String) = tokens(text).filter { it.first != SparqlTokenTypes.WHITESPACE } + + @Test + fun `less-than comparison is an operator, not an IRI`() { + val actual = significantTokens("FILTER(?a < ?b)") + val expected = + listOf( + SparqlTokenTypes.KEYWORD to "FILTER", + SparqlTokenTypes.PUNCTUATION to "(", + SparqlTokenTypes.VARIABLE to "?a", + SparqlTokenTypes.OPERATOR to "<", + SparqlTokenTypes.VARIABLE to "?b", + SparqlTokenTypes.PUNCTUATION to ")", + ) + assertEquals(expected, actual) + } + + @Test + fun `comparison operators inside a function call`() { + val actual = significantTokens("BIND(IF(?a < ?b, ?a, ?b) AS ?min)") + val expected = + listOf( + SparqlTokenTypes.KEYWORD to "BIND", + SparqlTokenTypes.PUNCTUATION to "(", + SparqlTokenTypes.KEYWORD to "IF", + SparqlTokenTypes.PUNCTUATION to "(", + SparqlTokenTypes.VARIABLE to "?a", + SparqlTokenTypes.OPERATOR to "<", + SparqlTokenTypes.VARIABLE to "?b", + SparqlTokenTypes.PUNCTUATION to ",", + SparqlTokenTypes.VARIABLE to "?a", + SparqlTokenTypes.PUNCTUATION to ",", + SparqlTokenTypes.VARIABLE to "?b", + SparqlTokenTypes.PUNCTUATION to ")", + SparqlTokenTypes.KEYWORD to "AS", + SparqlTokenTypes.VARIABLE to "?min", + SparqlTokenTypes.PUNCTUATION to ")", + ) + assertEquals(expected, actual) + } + + @Test + fun `less-than does not swallow a later greater-than on the same line`() { + val actual = significantTokens("FILTER(?a < ?b && ?c > ?d)") + val expected = + listOf( + SparqlTokenTypes.KEYWORD to "FILTER", + SparqlTokenTypes.PUNCTUATION to "(", + SparqlTokenTypes.VARIABLE to "?a", + SparqlTokenTypes.OPERATOR to "<", + SparqlTokenTypes.VARIABLE to "?b", + SparqlTokenTypes.OPERATOR to "&", + SparqlTokenTypes.OPERATOR to "&", + SparqlTokenTypes.VARIABLE to "?c", + SparqlTokenTypes.OPERATOR to ">", + SparqlTokenTypes.VARIABLE to "?d", + SparqlTokenTypes.PUNCTUATION to ")", + ) + assertEquals(expected, actual) + } + + @Test + fun `iri reference is lexed as a single IRI token`() { + val actual = significantTokens("FROM ") + val expected = + listOf( + SparqlTokenTypes.KEYWORD to "FROM", + SparqlTokenTypes.IRI to "", + ) + assertEquals(expected, actual) + } + + @Test + fun `less-than-or-equal is lexed as operators`() { + val actual = significantTokens("FILTER(?a <= ?b)") + val expected = + listOf( + SparqlTokenTypes.KEYWORD to "FILTER", + SparqlTokenTypes.PUNCTUATION to "(", + SparqlTokenTypes.VARIABLE to "?a", + SparqlTokenTypes.OPERATOR to "<", + SparqlTokenTypes.OPERATOR to "=", + SparqlTokenTypes.VARIABLE to "?b", + SparqlTokenTypes.PUNCTUATION to ")", + ) + assertEquals(expected, actual) + } + + @Test + fun `unclosed angle bracket at end of line is an operator`() { + val actual = significantTokens("?a <\n?b") + val expected = + listOf( + SparqlTokenTypes.VARIABLE to "?a", + SparqlTokenTypes.OPERATOR to "<", + SparqlTokenTypes.VARIABLE to "?b", + ) + assertEquals(expected, actual) + } + + @Test + fun `iri candidate containing a quote is not an IRI`() { + val actual = significantTokens("?a <\"x\"> ?b") + assertEquals(SparqlTokenTypes.OPERATOR to "<", actual[1]) + } +}