TudbuT/helix-mods
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Incremental parsing: rough draft.

Browse source
This commit is contained in:
Blaž Hrastnik 2020-09-17 14:57:49 +09:00
parent 31999d6528
commit 088f8a82af
11 changed files with 693 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,2 +1,3 @@
target
helix-term/rustfmt.toml
helix-syntax/languages/

1
Cargo.lock generated
View file

@ -369,6 +369,7 @@ name = "helix-core"
version = "0.1.0"
dependencies = [
"anyhow",
"helix-syntax",
"ropey",
"smallvec",
"tendril",

4
Cargo.toml
View file

@ -6,5 +6,5 @@ members = [
]
# Build helix-syntax in release mode to make the code path faster in development.
[profile.dev.package."helix-syntax"]
opt-level = 3
# [profile.dev.package."helix-syntax"]
# opt-level = 3

2
helix-core/Cargo.toml
View file

@ -7,6 +7,8 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
helix-syntax = { path = "../helix-syntax" }
ropey = "1.2.0"
anyhow = "1.0.31"
smallvec = "1.4.0"

3
helix-core/src/lib.rs
View file

@ -1,6 +1,7 @@
#![allow(unused)]
pub mod commands;
pub mod graphemes;
mod position;
mod selection;
pub mod state;
pub mod syntax;
@ -9,8 +10,10 @@ mod transaction;
pub use ropey::{Rope, RopeSlice};
pub use tendril::StrTendril as Tendril;
pub use position::Position;
pub use selection::Range as SelectionRange;
pub use selection::Selection;
pub use syntax::Syntax;
pub use state::State;

57
helix-core/src/position.rs Normal file
View file

@ -0,0 +1,57 @@
/// Represents a single point in a text buffer. Zero indexed.
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub struct Position {
pub row: usize,
pub col: usize,
}
impl Position {
pub const fn new(row: usize, col: usize) -> Self {
Self { row, col }
}
pub const fn is_zero(self) -> bool {
self.row == 0 && self.col == 0
}
// TODO: generalize
pub fn traverse(self, text: &crate::Tendril) -> Self {
let Self { mut row, mut col } = self;
// TODO: there should be a better way here
for ch in text.chars() {
if ch == '\n' {
row += 1;
col = 0;
} else {
col += 1;
}
}
Self { row, col }
}
}
impl From<(usize, usize)> for Position {
fn from(tuple: (usize, usize)) -> Self {
Position {
row: tuple.0,
col: tuple.1,
}
}
}
impl Into<tree_sitter::Point> for Position {
fn into(self) -> tree_sitter::Point {
tree_sitter::Point::new(self.row, self.col)
}
}
#[cfg(test)]
mod test {
use super::*;
#[test]
fn test_ordering() {
// (0, 5) is less than (1, 0)
assert!(Position::new(0, 5) < Position::new(1, 0));
}
}

73
helix-core/src/state.rs
View file

@ -1,5 +1,5 @@
use crate::graphemes::{nth_next_grapheme_boundary, nth_prev_grapheme_boundary, RopeGraphemes};
use crate::{Rope, RopeSlice, Selection, SelectionRange};
use crate::{Position, Rope, RopeSlice, Selection, SelectionRange, Syntax};
use anyhow::Error;
use std::path::PathBuf;
@ -17,6 +17,9 @@ pub struct State {
pub(crate) doc: Rope,
pub(crate) selection: Selection,
pub(crate) mode: Mode,
//
pub syntax: Option<Syntax>,
}
#[derive(Copy, Clone, PartialEq, Eq)]
@ -40,6 +43,7 @@ impl State {
doc,
selection: Selection::single(0, 0),
mode: Mode::Normal,
syntax: None,
}
}
@ -54,6 +58,29 @@ impl State {
let mut state = Self::new(doc);
state.path = Some(path);
let language = helix_syntax::get_language(&helix_syntax::LANG::Rust);
let mut highlight_config = crate::syntax::HighlightConfiguration::new(
language,
&std::fs::read_to_string(
"../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
)
.unwrap(),
&std::fs::read_to_string(
"../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
)
.unwrap(),
"", // locals.scm
)
.unwrap();
// TODO: config.configure(scopes) is now delayed, is that ok?
// TODO: get_language is called twice
let syntax = Syntax::new(helix_syntax::LANG::Rust, &state.doc, highlight_config);
state.syntax = Some(syntax);
Ok(state)
}
@ -175,29 +202,29 @@ impl State {
}
/// Coordinates are a 0-indexed line and column pair.
type Coords = (usize, usize); // line, col
pub type Coords = (usize, usize); // line, col
/// Convert a character index to (line, column) coordinates.
pub fn coords_at_pos(text: &RopeSlice, pos: usize) -> Coords {
pub fn coords_at_pos(text: &RopeSlice, pos: usize) -> Position {
let line = text.char_to_line(pos);
let line_start = text.line_to_char(line);
let col = text.slice(line_start..pos).len_chars();
(line, col)
Position::new(line, col)
}
/// Convert (line, column) coordinates to a character index.
pub fn pos_at_coords(text: &RopeSlice, coords: Coords) -> usize {
let (line, col) = coords;
let line_start = text.line_to_char(line);
pub fn pos_at_coords(text: &RopeSlice, coords: Position) -> usize {
let Position { row, col } = coords;
let line_start = text.line_to_char(row);
nth_next_grapheme_boundary(text, line_start, col)
}
fn move_vertically(text: &RopeSlice, dir: Direction, pos: usize, count: usize) -> usize {
let (line, col) = coords_at_pos(text, pos);
let Position { row, col } = coords_at_pos(text, pos);
let new_line = match dir {
Direction::Backward => line.saturating_sub(count),
Direction::Forward => std::cmp::min(line.saturating_add(count), text.len_lines() - 1),
Direction::Backward => row.saturating_sub(count),
Direction::Forward => std::cmp::min(row.saturating_add(count), text.len_lines() - 1),
};
// convert to 0-indexed, subtract another 1 because len_chars() counts \n
@ -210,7 +237,7 @@ fn move_vertically(text: &RopeSlice, dir: Direction, pos: usize, count: usize) -
col
};
pos_at_coords(text, (new_line, new_col))
pos_at_coords(text, Position::new(new_line, new_col))
}
#[cfg(test)]
@ -220,32 +247,32 @@ mod test {
#[test]
fn test_coords_at_pos() {
let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
assert_eq!(coords_at_pos(&text.slice(..), 0), (0, 0));
assert_eq!(coords_at_pos(&text.slice(..), 5), (0, 5)); // position on \n
assert_eq!(coords_at_pos(&text.slice(..), 6), (1, 0)); // position on w
assert_eq!(coords_at_pos(&text.slice(..), 7), (1, 1)); // position on o
assert_eq!(coords_at_pos(&text.slice(..), 10), (1, 4)); // position on d
assert_eq!(coords_at_pos(&text.slice(..), 0), (0, 0).into());
assert_eq!(coords_at_pos(&text.slice(..), 5), (0, 5).into()); // position on \n
assert_eq!(coords_at_pos(&text.slice(..), 6), (1, 0).into()); // position on w
assert_eq!(coords_at_pos(&text.slice(..), 7), (1, 1).into()); // position on o
assert_eq!(coords_at_pos(&text.slice(..), 10), (1, 4).into()); // position on d
}
#[test]
fn test_pos_at_coords() {
let text = Rope::from("ḧëḷḷö\nẅöṛḷḋ");
assert_eq!(pos_at_coords(&text.slice(..), (0, 0)), 0);
assert_eq!(pos_at_coords(&text.slice(..), (0, 5)), 5); // position on \n
assert_eq!(pos_at_coords(&text.slice(..), (1, 0)), 6); // position on w
assert_eq!(pos_at_coords(&text.slice(..), (1, 1)), 7); // position on o
assert_eq!(pos_at_coords(&text.slice(..), (1, 4)), 10); // position on d
assert_eq!(pos_at_coords(&text.slice(..), (0, 0).into()), 0);
assert_eq!(pos_at_coords(&text.slice(..), (0, 5).into()), 5); // position on \n
assert_eq!(pos_at_coords(&text.slice(..), (1, 0).into()), 6); // position on w
assert_eq!(pos_at_coords(&text.slice(..), (1, 1).into()), 7); // position on o
assert_eq!(pos_at_coords(&text.slice(..), (1, 4).into()), 10); // position on d
}
#[test]
fn test_vertical_move() {
let text = Rope::from("abcd\nefg\nwrs");
let pos = pos_at_coords(&text.slice(..), (0, 4));
let pos = pos_at_coords(&text.slice(..), (0, 4).into());
let slice = text.slice(..);
assert_eq!(
coords_at_pos(&slice, move_vertically(&slice, Direction::Forward, pos, 1)),
(1, 2)
(1, 2).into()
);
}
}

620
helix-core/src/syntax.rs
View file

@ -1,66 +1,397 @@
// pub struct Syntax {
// parser: Parser,
// }
use crate::{Change, Rope, RopeSlice, Transaction};
pub use helix_syntax::LANG;
pub use helix_syntax::{get_language, get_language_name};
//impl Syntax {
// // buffer, grammar, config, grammars, sync_timeout?
// pub fn new() -> Self {
// unimplemented!()
// // make a new root layer
// // track markers of injections
// //
// // track scope_descriptor: a Vec of scopes for item in tree
// //
// // fetch grammar for parser based on language string
// // update root layer
// }
pub struct Syntax {
grammar: Language,
parser: Parser,
cursors: Vec<QueryCursor>,
// // fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
config: HighlightConfiguration,
// // call this on transaction.apply() -> buffer_changed(changes)
// //
// // fn parse(language, old_tree, ranges)
// //
// // fn tree() -> Tree
// //
// // <!--update_for_injection(grammar)-->
root_layer: LanguageLayer,
}
// // Highlighting
// // fn highlight_iter() -> iterates over all the scopes
// // on_tokenize
// // on_change_highlighting
impl Syntax {
// buffer, grammar, config, grammars, sync_timeout?
pub fn new(language: LANG, source: &Rope, config: HighlightConfiguration) -> Self {
// fetch grammar for parser based on language string
let grammar = get_language(&language);
let parser = Parser::new();
// // Commenting
// // comment_strings_for_pos
// // is_commented
let root_layer = LanguageLayer::new();
// // Indentation
// // suggested_indent_for_line_at_buffer_row
// // suggested_indent_for_buffer_row
// // indent_level_for_line
// track markers of injections
// track scope_descriptor: a Vec of scopes for item in tree
// // TODO: Folding
let mut syntax = Self {
grammar,
parser,
cursors: Vec::new(),
config,
root_layer,
};
// // Syntax APIs
// // get_syntax_node_containing_range ->
// // ...
// // get_syntax_node_at_pos
// // buffer_range_for_scope_at_pos
//}
// update root layer
syntax.root_layer.parse(
&mut syntax.parser,
&syntax.config,
source,
0,
vec![Range {
start_byte: 0,
end_byte: usize::MAX,
start_point: Point::new(0, 0),
end_point: Point::new(usize::MAX, usize::MAX),
}],
);
syntax
}
pub fn configure(&mut self, scopes: &[String]) {
self.config.configure(scopes)
}
pub fn update(&mut self, source: &Rope, changeset: &ChangeSet) -> Result<(), Error> {
self.root_layer
.update(&mut self.parser, &self.config, source, changeset)
// TODO: deal with injections and update them too
}
// fn buffer_changed -> call layer.update(range, new_text) on root layer and then all marker layers
// call this on transaction.apply() -> buffer_changed(changes)
//
// fn parse(language, old_tree, ranges)
//
fn tree(&self) -> &Tree {
self.root_layer.tree()
}
//
// <!--update_for_injection(grammar)-->
// Highlighting
/// Iterate over the highlighted regions for a given slice of source code.
pub fn highlight_iter<'a>(
&'a mut self,
source: &'a [u8],
cancellation_flag: Option<&'a AtomicUsize>,
mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
// prevents them from being moved. But both of these values are really just
// pointers, so it's actually ok to move them.
let mut cursor = QueryCursor::new(); // reuse a pool
let tree_ref = unsafe { mem::transmute::<_, &'static Tree>(self.tree()) };
let cursor_ref = unsafe { mem::transmute::<_, &'static mut QueryCursor>(&mut cursor) };
let query_ref = unsafe { mem::transmute::<_, &'static mut Query>(&mut self.config.query) };
let config_ref =
unsafe { mem::transmute::<_, &'static HighlightConfiguration>(&self.config) };
let captures = cursor_ref
.captures(query_ref, tree_ref.root_node(), move |n: Node| {
&source[n.byte_range()]
})
.peekable();
// manually craft the root layer based on the existing tree
let layer = HighlightIterLayer {
highlight_end_stack: Vec::new(),
scope_stack: vec![LocalScope {
inherits: false,
range: 0..usize::MAX,
local_defs: Vec::new(),
}],
cursor,
depth: 0,
_tree: None,
captures,
config: config_ref,
ranges: vec![Range {
start_byte: 0,
end_byte: usize::MAX,
start_point: Point::new(0, 0),
end_point: Point::new(usize::MAX, usize::MAX),
}],
};
let mut result = HighlightIter {
source,
byte_offset: 0,
injection_callback,
cancellation_flag,
highlighter: self,
iter_count: 0,
layers: vec![layer],
next_event: None,
last_highlight_range: None,
};
result.sort_layers();
Ok(result)
}
// on_tokenize
// on_change_highlighting
// Commenting
// comment_strings_for_pos
// is_commented
// Indentation
// suggested_indent_for_line_at_buffer_row
// suggested_indent_for_buffer_row
// indent_level_for_line
// TODO: Folding
// Syntax APIs
// get_syntax_node_containing_range ->
// ...
// get_syntax_node_at_pos
// buffer_range_for_scope_at_pos
}
pub struct LanguageLayer {
// mode
// grammar
// depth
// tree: Tree,
// grammar
// depth
tree: Option<Tree>,
}
// impl LanguageLayer {
// // fn highlight_iter() -> same as Mode but for this layer. Mode composits these
// // fn buffer_changed
// // fn update(range)
// // fn update_injections()
// }
use crate::state::{coords_at_pos, Coords};
use crate::transaction::{ChangeSet, Operation};
use crate::Tendril;
impl LanguageLayer {
pub fn new() -> Self {
Self { tree: None }
}
fn tree(&self) -> &Tree {
// TODO: no unwrap
self.tree.as_ref().unwrap()
}
fn parse<'a>(
&mut self,
parser: &mut Parser,
config: &HighlightConfiguration,
source: &Rope,
mut depth: usize,
mut ranges: Vec<Range>,
) -> Result<(), Error> {
if parser.set_included_ranges(&ranges).is_ok() {
parser
.set_language(config.language)
.map_err(|_| Error::InvalidLanguage)?;
// unsafe { syntax.parser.set_cancellation_flag(cancellation_flag) };
let tree = parser
.parse_with(
&mut |byte, _| {
if byte <= source.len_bytes() {
let (chunk, start_byte, _, _) = source.chunk_at_byte(byte);
chunk[byte - start_byte..].as_bytes()
} else {
// out of range
&[]
}
},
self.tree.as_ref(),
)
.ok_or(Error::Cancelled)?;
// unsafe { syntax.parser.set_cancellation_flag(None) };
// let mut cursor = syntax.cursors.pop().unwrap_or_else(QueryCursor::new);
// Process combined injections. (ERB, EJS, etc https://github.com/tree-sitter/tree-sitter/pull/526)
// if let Some(combined_injections_query) = &config.combined_injections_query {
// let mut injections_by_pattern_index =
// vec![(None, Vec::new(), false); combined_injections_query.pattern_count()];
// let matches =
// cursor.matches(combined_injections_query, tree.root_node(), |n: Node| {
// &source[n.byte_range()]
// });
// for mat in matches {
// let entry = &mut injections_by_pattern_index[mat.pattern_index];
// let (language_name, content_node, include_children) =
// injection_for_match(config, combined_injections_query, &mat, source);
// if language_name.is_some() {
// entry.0 = language_name;
// }
// if let Some(content_node) = content_node {
// entry.1.push(content_node);
// }
// entry.2 = include_children;
// }
// for (lang_name, content_nodes, includes_children) in injections_by_pattern_index {
// if let (Some(lang_name), false) = (lang_name, content_nodes.is_empty()) {
// if let Some(next_config) = (injection_callback)(lang_name) {
// let ranges =
// Self::intersect_ranges(&ranges, &content_nodes, includes_children);
// if !ranges.is_empty() {
// queue.push((next_config, depth + 1, ranges));
// }
// }
// }
// }
// }
self.tree = Some(tree)
}
Ok(())
}
pub(crate) fn generate_edits(
text: &RopeSlice,
changeset: &ChangeSet,
) -> Vec<tree_sitter::InputEdit> {
use Operation::*;
let mut old_pos = 0;
let mut new_pos = 0;
let mut edits = Vec::new();
let mut iter = changeset.changes.iter().peekable();
// TODO; this is a lot easier with Change instead of Operation.
fn point_at_pos(text: &RopeSlice, pos: usize) -> (usize, Point) {
let byte = text.char_to_byte(pos);
let line = text.char_to_line(pos);
let line_start_byte = text.line_to_byte(line);
let col = byte - line_start_byte;
(byte, Point::new(line, col))
}
fn traverse(point: Point, text: &Tendril) -> Point {
let Point {
mut row,
mut column,
} = point;
// TODO: there should be a better way here
for ch in text.bytes() {
if ch == b'\n' {
row += 1;
column = 0;
} else {
column += 1;
}
}
Point { row, column }
}
while let Some(change) = iter.next() {
let len = match change {
Delete(i) | Retain(i) => *i,
Insert(_) => 0,
};
let old_end = old_pos + len;
match change {
Retain(_) => {
new_pos += len;
}
Delete(_) => {
let (start_byte, start_position) = point_at_pos(&text, old_pos);
let (old_end_byte, old_end_position) = point_at_pos(&text, old_end);
// TODO: Position also needs to be byte based...
// let byte = char_to_byte(old_pos)
// let line = char_to_line(old_pos)
// let line_start_byte = line_to_byte()
// Position::new(line, line_start_byte - byte)
// a subsequent ins means a replace, consume it
if let Some(Insert(s)) = iter.peek() {
iter.next();
let ins = s.chars().count();
// replacement
edits.push(tree_sitter::InputEdit {
start_byte, // old_pos to byte
old_end_byte, // old_end to byte
new_end_byte: start_byte + s.len(), // old_pos to byte + s.len()
start_position, // old pos to coords
old_end_position, // old_end to coords
new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
});
new_pos += ins;
} else {
// deletion
edits.push(tree_sitter::InputEdit {
start_byte, // old_pos to byte
old_end_byte, // old_end to byte
new_end_byte: start_byte, // old_pos to byte
start_position, // old pos to coords
old_end_position, // old_end to coords
new_end_position: start_position, // old pos to coords
});
};
}
Insert(s) => {
let (start_byte, start_position) = point_at_pos(&text, old_pos);
let ins = s.chars().count();
// insert
edits.push(tree_sitter::InputEdit {
start_byte, // old_pos to byte
old_end_byte: start_byte, // same
new_end_byte: start_byte + s.len(), // old_pos + s.len()
start_position, // old pos to coords
old_end_position: start_position, // same
new_end_position: traverse(start_position, s), // old pos + chars, newlines matter too (iter over)
});
new_pos += ins;
}
}
old_pos = old_end;
}
edits
}
fn update(
&mut self,
parser: &mut Parser,
config: &HighlightConfiguration,
source: &Rope,
changeset: &ChangeSet,
) -> Result<(), Error> {
if changeset.is_empty() {
return Ok(());
}
let edits = Self::generate_edits(&source.slice(..), changeset);
// Notify the tree about all the changes
for edit in edits {
self.tree.as_mut().unwrap().edit(&edit);
}
self.parse(
parser,
config,
source,
0,
// TODO: what to do about this range on update
vec![Range {
start_byte: 0,
end_byte: usize::MAX,
start_point: Point::new(0, 0),
end_point: Point::new(usize::MAX, usize::MAX),
}],
)
}
// fn highlight_iter() -> same as Mode but for this layer. Mode composits these
// fn buffer_changed
// fn update(range)
// fn update_injections()
}
// -- refactored from tree-sitter-highlight to be able to retain state
// TODO: add seek() to iter
@ -169,7 +500,7 @@ where
{
source: &'a [u8],
byte_offset: usize,
highlighter: &'a mut Highlighter,
highlighter: &'a mut Syntax,
injection_callback: F,
cancellation_flag: Option<&'a AtomicUsize>,
layers: Vec<HighlightIterLayer<'a>>,
@ -179,7 +510,7 @@ where
}
struct HighlightIterLayer<'a> {
_tree: Tree,
_tree: Option<Tree>,
cursor: QueryCursor,
captures: iter::Peekable<QueryCaptures<'a, &'a [u8]>>,
config: &'a HighlightConfiguration,
@ -207,43 +538,43 @@ impl Highlighter {
&mut self.parser
}
/// Iterate over the highlighted regions for a given slice of source code.
pub fn highlight<'a>(
&'a mut self,
config: &'a HighlightConfiguration,
source: &'a [u8],
cancellation_flag: Option<&'a AtomicUsize>,
mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
let layers = HighlightIterLayer::new(
source,
self,
cancellation_flag,
&mut injection_callback,
config,
0,
vec![Range {
start_byte: 0,
end_byte: usize::MAX,
start_point: Point::new(0, 0),
end_point: Point::new(usize::MAX, usize::MAX),
}],
)?;
assert_ne!(layers.len(), 0);
let mut result = HighlightIter {
source,
byte_offset: 0,
injection_callback,
cancellation_flag,
highlighter: self,
iter_count: 0,
layers,
next_event: None,
last_highlight_range: None,
};
result.sort_layers();
Ok(result)
}
// /// Iterate over the highlighted regions for a given slice of source code.
// pub fn highlight<'a>(
// &'a mut self,
// config: &'a HighlightConfiguration,
// source: &'a [u8],
// cancellation_flag: Option<&'a AtomicUsize>,
// mut injection_callback: impl FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a,
// ) -> Result<impl Iterator<Item = Result<HighlightEvent, Error>> + 'a, Error> {
// let layers = HighlightIterLayer::new(
// source,
// self,
// cancellation_flag,
// &mut injection_callback,
// config,
// 0,
// vec![Range {
// start_byte: 0,
// end_byte: usize::MAX,
// start_point: Point::new(0, 0),
// end_point: Point::new(usize::MAX, usize::MAX),
// }],
// )?;
// assert_ne!(layers.len(), 0);
// let mut result = HighlightIter {
// source,
// byte_offset: 0,
// injection_callback,
// cancellation_flag,
// highlighter: self,
// iter_count: 0,
// layers,
// next_event: None,
// last_highlight_range: None,
// };
// result.sort_layers();
// Ok(result)
// }
}
impl HighlightConfiguration {
@ -413,7 +744,7 @@ impl<'a> HighlightIterLayer<'a> {
/// added to the returned vector.
fn new<F: FnMut(&str) -> Option<&'a HighlightConfiguration> + 'a>(
source: &'a [u8],
highlighter: &mut Highlighter,
highlighter: &mut Syntax,
cancellation_flag: Option<&'a AtomicUsize>,
injection_callback: &mut F,
mut config: &'a HighlightConfiguration,
@ -423,6 +754,8 @@ impl<'a> HighlightIterLayer<'a> {
let mut result = Vec::with_capacity(1);
let mut queue = Vec::new();
loop {
// --> Tree parsing part
if highlighter.parser.set_included_ranges(&ranges).is_ok() {
highlighter
.parser
@ -474,6 +807,8 @@ impl<'a> HighlightIterLayer<'a> {
}
}
// --> Highlighting query part
// The `captures` iterator borrows the `Tree` and the `QueryCursor`, which
// prevents them from being moved. But both of these values are really just
// pointers, so it's actually ok to move them.
@ -495,7 +830,7 @@ impl<'a> HighlightIterLayer<'a> {
}],
cursor,
depth,
_tree: tree,
_tree: Some(tree),
captures,
config,
ranges,
@ -1016,3 +1351,106 @@ fn shrink_and_clear<T>(vec: &mut Vec<T>, capacity: usize) {
}
vec.clear();
}
#[test]
fn test_parser() {
let highlight_names: Vec<String> = [
"attribute",
"constant",
"function.builtin",
"function",
"keyword",
"operator",
"property",
"punctuation",
"punctuation.bracket",
"punctuation.delimiter",
"string",
"string.special",
"tag",
"type",
"type.builtin",
"variable",
"variable.builtin",
"variable.parameter",
]
.iter()
.cloned()
.map(String::from)
.collect();
let language = get_language(&LANG::Rust);
let mut config = HighlightConfiguration::new(
language,
&std::fs::read_to_string(
"../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
)
.unwrap(),
&std::fs::read_to_string(
"../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
)
.unwrap(),
"", // locals.scm
)
.unwrap();
config.configure(&highlight_names);
let source = Rope::from_str(
"
struct Stuff {}
fn main() {}
",
);
let syntax = Syntax::new(LANG::Rust, &source, config);
let tree = syntax.root_layer.tree.unwrap();
let root = tree.root_node();
assert_eq!(root.kind(), "source_file");
assert_eq!(
root.to_sexp(),
concat!(
"(source_file ",
"(struct_item name: (type_identifier) body: (field_declaration_list)) ",
"(function_item name: (identifier) parameters: (parameters) body: (block)))"
)
);
let struct_node = root.child(0).unwrap();
assert_eq!(struct_node.kind(), "struct_item");
}
#[test]
fn test_input_edits() {
use crate::State;
use tree_sitter::InputEdit;
let mut state = State::new("hello world!\ntest 123".into());
let transaction = Transaction::change(
&state,
vec![(6, 11, Some("test".into())), (12, 17, None)].into_iter(),
);
let edits = LanguageLayer::generate_edits(&state.doc.slice(..), &transaction.changes);
// transaction.apply(&mut state);
assert_eq!(
edits,
&[
InputEdit {
start_byte: 6,
old_end_byte: 11,
new_end_byte: 10,
start_position: Point { row: 0, column: 6 },
old_end_position: Point { row: 0, column: 11 },
new_end_position: Point { row: 0, column: 10 }
},
InputEdit {
start_byte: 12,
old_end_byte: 17,
new_end_byte: 12,
start_position: Point { row: 0, column: 12 },
old_end_position: Point { row: 1, column: 4 },
new_end_position: Point { row: 0, column: 12 }
}
]
);
}

35
helix-core/src/transaction.rs
View file

@ -4,7 +4,7 @@ use crate::{Rope, Selection, SelectionRange, State, Tendril};
pub type Change = (usize, usize, Option<Tendril>);
#[derive(Debug, Clone, PartialEq, Eq)]
enum Operation {
pub(crate) enum Operation {
/// Move cursor by n characters.
Retain(usize),
/// Delete n characters.
@ -22,7 +22,7 @@ pub enum Assoc {
// ChangeSpec = Change | ChangeSet | Vec<Change>
#[derive(Debug)]
pub struct ChangeSet {
changes: Vec<Operation>,
pub(crate) changes: Vec<Operation>,
/// The required document length. Will refuse to apply changes unless it matches.
len: usize,
}
@ -307,7 +307,7 @@ impl ChangeSet {
/// a single transaction.
pub struct Transaction {
/// Changes made to the buffer.
changes: ChangeSet,
pub(crate) changes: ChangeSet,
/// When set, explicitly updates the selection.
selection: Option<Selection>,
// effects, annotations
@ -337,6 +337,14 @@ impl Transaction {
.clone()
.unwrap_or_else(|| state.selection.clone().map(&self.changes));
// TODO: no unwrap
state
.syntax
.as_mut()
.unwrap()
.update(&state.doc, &self.changes)
.unwrap();
true
}
@ -359,9 +367,15 @@ impl Transaction {
for (from, to, tendril) in changes {
// Retain from last "to" to current "from"
acc.push(Operation::Retain(from - last));
let span = to - from;
match tendril {
Some(text) => acc.push(Operation::Insert(text)),
None => acc.push(Operation::Delete(to - from)),
Some(text) => {
if span > 0 {
acc.push(Operation::Delete(span));
}
acc.push(Operation::Insert(text));
}
None => acc.push(Operation::Delete(span)),
}
last = to;
}
@ -466,4 +480,15 @@ mod test {
assert_eq!(cs.map_pos(2, Assoc::Before), 2);
assert_eq!(cs.map_pos(2, Assoc::After), 2);
}
#[test]
fn transaction_change() {
let mut state = State::new("hello world!\ntest 123".into());
let transaction = Transaction::change(
&state,
vec![(6, 11, Some("void".into())), (12, 17, None)].into_iter(),
);
transaction.apply(&mut state);
assert_eq!(state.doc, Rope::from_str("hello void! 123"));
}
}

2
helix-term/Cargo.toml
View file

@ -11,11 +11,11 @@ name = "hx"
path = "src/main.rs"
[dependencies]
helix-core = { path = "../helix-core" }
helix-syntax = { path = "../helix-syntax" }
anyhow = "1"
argh = "0.1.3"
helix-core = { path = "../helix-core" }
crossterm = { version = "0.17", features = ["event-stream"] }
smol = "1"

55
helix-term/src/editor.rs
View file

@ -38,8 +38,6 @@ pub struct Editor {
size: (u16, u16),
surface: Surface,
theme: Theme,
highlighter: Highlighter,
highlight_config: HighlightConfiguration,
}
impl Editor {
@ -49,33 +47,8 @@ impl Editor {
let mut terminal = Terminal::new(backend)?;
let size = terminal::size().unwrap();
let area = Rect::new(0, 0, size.0, size.1);
let theme = Theme::default();
// let mut parser = tree_sitter::Parser::new();
// parser.set_language(language).unwrap();
// let tree = parser.parse(source_code, None).unwrap();
let language = helix_syntax::get_language(&helix_syntax::LANG::Rust);
let highlighter = Highlighter::new();
let mut highlight_config = HighlightConfiguration::new(
language,
&std::fs::read_to_string(
"../helix-syntax/languages/tree-sitter-rust/queries/highlights.scm",
)
.unwrap(),
&std::fs::read_to_string(
"../helix-syntax/languages/tree-sitter-rust/queries/injections.scm",
)
.unwrap(),
"", // locals.scm
)
.unwrap();
highlight_config.configure(theme.scopes());
let mut editor = Editor {
terminal,
state: None,
@ -84,8 +57,6 @@ impl Editor {
surface: Surface::empty(area),
theme,
// TODO; move to state
highlighter,
highlight_config,
};
if let Some(file) = args.files.pop() {
@ -96,12 +67,19 @@ impl Editor {
}
pub fn open(&mut self, path: PathBuf) -> Result<(), Error> {
self.state = Some(State::load(path)?);
let mut state = State::load(path)?;
state
.syntax
.as_mut()
.unwrap()
.configure(self.theme.scopes());
self.state = Some(state);
Ok(())
}
fn render(&mut self) {
match &self.state {
// TODO: ideally not mut but highlights require it because of cursor cache
match &mut self.state {
Some(state) => {
let area = Rect::new(0, 0, self.size.0, self.size.1);
let mut surface = Surface::empty(area);
@ -112,12 +90,13 @@ impl Editor {
// TODO: cache highlight results
// TODO: only recalculate when state.doc is actually modified
let highlights = self
.highlighter
.highlight(&self.highlight_config, source_code.as_bytes(), None, |_| {
None
})
.unwrap();
let highlights: Vec<_> = state
.syntax
.as_mut()
.unwrap()
.highlight_iter(source_code.as_bytes(), None, |_| None)
.unwrap()
.collect(); // TODO: we collect here to avoid double borrow, fix later
let mut spans = Vec::new();
@ -235,7 +214,7 @@ impl Editor {
let coords = coords_at_pos(&state.doc().slice(..), pos);
execute!(
stdout,
cursor::MoveTo((coords.1 + 2) as u16, coords.0 as u16)
cursor::MoveTo((coords.col + 2) as u16, coords.row as u16)
);
}
None => (),