Skip to content

feat(ide): директивы компиляции и ключевые слова объявления различимы в семантической подсветке #429

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
flamber-dev wants to merge 1 commit into
base: develop
Choose a base branch
from
+371 −17
Open

feat(ide): директивы компиляции и ключевые слова объявления различимы в семантической подсветке #429

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions crates/bsl-analyzer/src/lsp/to_proto.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,7 @@ pub fn semantic_tokens_legend() -> SemanticTokensLegend {
SemanticTokenType::ENUM_MEMBER,
SemanticTokenType::NAMESPACE,
SemanticTokenType::CLASS,
SemanticTokenType::MODIFIER,
];

let token_modifiers = vec![
Expand Down Expand Up @@ -387,6 +388,7 @@ fn token_type_index(tag: HlTag) -> u32 {
HlTag::EnumMember => 13,
HlTag::Namespace => 14,
HlTag::Class => 15,
HlTag::CompilerDirective => 16,
}
}

Expand Down
105 changes: 105 additions & 0 deletions crates/bsl-analyzer/tests/semantic_tokens_directives_and_declarations.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
//! What an LSP client decodes for compilation directives, extension
//! annotations and declaration keywords: every emitted token type and
//! modifier must exist in the advertised legend under the expected name.

use bsl_analyzer::lsp::{semantic_tokens, semantic_tokens_legend};
use ide::highlight;
use ide_db::base_db::{SourceDatabase, SourceRoot, SourceRootId};
use ide_db::RootDatabaseImpl;
use line_index::LineIndex;
use vfs::{FileId, FileSet, VfsPath};

/// `(text, token type, modifiers)` for every token, decoded through the legend
/// exactly as a client would.
fn decoded_tokens(source: &str) -> Vec<(String, String, Vec<String>)> {
let mut db = RootDatabaseImpl::new();
let file_id = FileId(0);
let mut files = FileSet::default();
files.insert(file_id, VfsPath::new("/test/Module.bsl".to_string()));
db.set_source_root(SourceRootId(0), SourceRoot::new_local(files));
db.set_file_source_root(file_id, SourceRootId(0));
db.set_file_text(file_id, source);
let highlights = highlight(&db, file_id).highlights;

let legend = semantic_tokens_legend();
let line_index = LineIndex::new(source);
let tokens = semantic_tokens(&line_index, source, &highlights);
let lines: Vec<&str> = source.split('\n').collect();

let mut line = 0usize;
let mut start = 0usize;
let mut out = Vec::new();
for token in tokens {
if token.delta_line > 0 {
line += token.delta_line as usize;
start = token.delta_start as usize;
} else {
start += token.delta_start as usize;
}
let utf16: Vec<u16> = lines[line].encode_utf16().collect();
let text = String::from_utf16(&utf16[start..start + token.length as usize]).unwrap();
let ty = legend
.token_types
.get(token.token_type as usize)
.unwrap_or_else(|| {
panic!("token type {} of {text:?} is not in the legend", token.token_type)
})
.as_str()
.to_string();
let mods = legend
.token_modifiers
.iter()
.enumerate()
.filter(|(bit, _)| token.token_modifiers_bitset & (1 << bit) != 0)
.map(|(_, modifier)| modifier.as_str().to_string())
.collect();
out.push((text, ty, mods));
}
out
}

fn decoded<'a>(
tokens: &'a [(String, String, Vec<String>)],
text: &str,
) -> &'a (String, String, Vec<String>) {
tokens
.iter()
.find(|(t, _, _)| t == text)
.unwrap_or_else(|| panic!("no token {text:?}: {tokens:?}"))
}

#[test]
fn legend_advertises_the_modifier_token_type() {
let legend = semantic_tokens_legend();
assert!(
legend.token_types.iter().any(|ty| ty.as_str() == "modifier"),
"legend: {:?}",
legend.token_types
);
assert!(legend.token_modifiers.iter().any(|m| m.as_str() == "declaration"));
}

#[test]
fn client_decodes_directives_annotations_and_declaration_keywords() {
let source = "&НаСервере\n&После(\"Исходный\")\nПроцедура Расш_Исходный(Знач Параметр) Экспорт\n Если Параметр Тогда\n Возврат;\n КонецЕсли;\nКонецПроцедуры\n";
let tokens = decoded_tokens(source);

let (_, ty, mods) = decoded(&tokens, "&НаСервере");
assert_eq!(ty, "modifier");
assert!(mods.is_empty(), "{mods:?}");

let (_, ty, _) = decoded(&tokens, "&После");
assert_eq!(ty, "decorator");

for keyword in ["Процедура", "Знач", "Экспорт", "КонецПроцедуры"]
{
let (_, ty, mods) = decoded(&tokens, keyword);
assert_eq!(ty, "keyword", "{keyword}");
assert!(mods.iter().any(|m| m == "declaration"), "{keyword}: {mods:?}");
}
for keyword in ["Если", "Тогда", "Возврат", "КонецЕсли"] {
let (_, ty, mods) = decoded(&tokens, keyword);
assert_eq!(ty, "keyword", "{keyword}");
assert!(!mods.iter().any(|m| m == "declaration"), "{keyword}: {mods:?}");
}
}
13 changes: 12 additions & 1 deletion crates/ide/src/syntax_highlighting.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ pub enum HlTag {
Comment,
Preprocessor,
Annotation,
CompilerDirective,
Property,
Operator,
UnresolvedReference,
Expand All @@ -47,6 +48,7 @@ impl HlTag {
HlTag::Comment => "comment",
HlTag::Preprocessor => "macro",
HlTag::Annotation => "decorator",
HlTag::CompilerDirective => "modifier",
HlTag::Property => "property",
HlTag::Operator => "operator",
HlTag::UnresolvedReference => "unresolvedReference",
Expand Down Expand Up @@ -291,18 +293,27 @@ fn highlight_token<DB: RootDatabase>(
}
}

let mut modifiers = HlMod::new();
let tag = if kind.is_boolean_literal() {
HlTag::BooleanLiteral
} else if kind.is_string_literal() {
HlTag::StringLiteral
} else if kind.is_number_literal() {
HlTag::NumberLiteral
} else if kind.is_keyword() {
// Declaration keywords stay `keyword` for clients that ignore
// modifiers; the standard `declaration` modifier lets a theme colour
// them apart from control-flow keywords.
if kind.is_declaration_keyword() {
modifiers = modifiers.with(HlMod::DECLARATION);
}
HlTag::Keyword
} else if kind == SyntaxKind::COMMENT {
HlTag::Comment
} else if kind.is_preprocessor() {
HlTag::Preprocessor
} else if kind.is_compilation_directive() {
HlTag::CompilerDirective
} else if kind.is_annotation() {
HlTag::Annotation
} else if kind.is_operator() {
Expand All @@ -311,7 +322,7 @@ fn highlight_token<DB: RootDatabase>(
return None;
};

Some(HlRange { range, tag, modifiers: HlMod::new() })
Some(HlRange { range, tag, modifiers })
}

/// The name in a region directive (`#Область БыстрыйЗапуск`) is the label of a
Expand Down
32 changes: 16 additions & 16 deletions crates/ide/src/syntax_highlighting/token_stream_tests.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,21 +121,21 @@ fn module_shape_token_stream() {
code,
expect![[r##"
0..15 Preprocessor "#Область"
35..45 Keyword "Перем"
35..45 Keyword [DECLARATION] "Перем"
46..84 Variable [EXPORT+DECLARATION] "МодульнаяПеременная"
85..99 Keyword "Экспорт"
85..99 Keyword [DECLARATION] "Экспорт"
101..122 Comment "// модульная"
123..133 Keyword "Перем"
123..133 Keyword [DECLARATION] "Перем"
134..170 Variable [DECLARATION] "ЛокальнаяМодульная"
173..192 Annotation "&НаСервере"
193..211 Keyword "Процедура"
173..192 CompilerDirective "&НаСервере"
193..211 Keyword [DECLARATION] "Процедура"
212..232 Procedure [EXPORT+DEFINITION] "Обработать"
233..250 Parameter [DECLARATION] "Параметр1"
252..269 Parameter [DECLARATION] "Параметр2"
270..271 Operator "="
272..274 NumberLiteral "10"
276..290 Keyword "Экспорт"
295..305 Keyword "Перем"
276..290 Keyword [DECLARATION] "Экспорт"
295..305 Keyword [DECLARATION] "Перем"
306..316 Variable [DECLARATION] "Явная"
322..332 Variable "Явная"
333..334 Operator "="
Expand All @@ -154,8 +154,8 @@ fn module_shape_token_stream() {
519..520 NumberLiteral "2"
527..543 BuiltinFunction [EXPORT] "Сообщить"
544..558 StringLiteral "\"готово\""
561..589 Keyword "КонецПроцедуры"
591..605 Keyword "Функция"
561..589 Keyword [DECLARATION] "КонецПроцедуры"
591..605 Keyword [DECLARATION] "Функция"
606..624 Function [DEFINITION] "Вычислить"
631..640 Preprocessor "#Если"
654..664 Keyword "Тогда"
Expand All @@ -165,7 +165,7 @@ fn module_shape_token_stream() {
718..732 Keyword "Возврат"
733..741 BooleanLiteral "Ложь"
747..766 Preprocessor "#КонецЕсли"
767..791 Keyword "КонецФункции"
767..791 Keyword [DECLARATION] "КонецФункции"
792..817 Preprocessor "#КонецОбласти"
819..855 Variable "ЛокальнаяМодульная"
856..857 Operator "="
Expand All @@ -189,7 +189,7 @@ fn mdo_and_typed_member_token_stream() {
check_token_stream_with_config(
code,
expect![[r#"
0..18 Keyword "Процедура"
0..18 Keyword [DECLARATION] "Процедура"
19..27 Procedure [DEFINITION] "Тест"
28..44 Parameter [DECLARATION] "Значение"
50..74 Variable [DECLARATION] "НаборЗаписей"
Expand All @@ -206,7 +206,7 @@ fn mdo_and_typed_member_token_stream() {
334..344 Keyword "Новый"
382..406 Variable "НаборЗаписей"
407..423 Function [EXPORT] "Записать"
427..455 Keyword "КонецПроцедуры"
427..455 Keyword [DECLARATION] "КонецПроцедуры"
"#]],
);
}
Expand All @@ -232,7 +232,7 @@ fn sdbl_query_token_stream() {
check_token_stream(
code,
expect![[r#"
0..14 Keyword "Функция"
0..14 Keyword [DECLARATION] "Функция"
15..29 Function [DEFINITION] "Выбрать"
30..44 Parameter [DECLARATION] "Партнер"
50..62 Variable [DECLARATION] "Запрос"
Expand Down Expand Up @@ -263,7 +263,7 @@ fn sdbl_query_token_stream() {
454..476 Property "Регистратор"
483..497 Keyword "Возврат"
498..510 Variable "Запрос"
512..536 Keyword "КонецФункции"
512..536 Keyword [DECLARATION] "КонецФункции"
"#]],
);
}
Expand All @@ -281,10 +281,10 @@ fn documentation_token_stream() {
23..26 Comment " - "
26..32 Type [DOCUMENTATION] "String"
32..39 Comment " - text"
40..49 Keyword "Procedure"
40..49 Keyword [DECLARATION] "Procedure"
50..54 Procedure [DEFINITION] "Test"
55..60 Parameter [DECLARATION] "Value"
62..74 Keyword "EndProcedure"
62..74 Keyword [DECLARATION] "EndProcedure"
"#]],
);
}
Loading
Loading