TudbuT/helix-mods
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

migrate grammar fetching/building code into helix-loader crate

Browse source 
This is a rather large refactor that moves most of the code for
loading, fetching, and building grammars into a new helix-loader
module. This works well with the [[grammars]] syntax for
languages.toml defined earlier: we only have to depend on the types
for GrammarConfiguration in helix-loader and can leave all the
[[language]] entries for helix-core.
This commit is contained in:
Michael Davis 2022-02-16 07:57:20 -06:00 • committed by Blaž Hrastnik
parent 08ee949dcb
commit 4fc991fdec
23 changed files with 419 additions and 374 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
Cargo.lock generated
View file

@ -358,12 +358,11 @@ dependencies = [
name = "helix-core"
version = "0.6.0"
dependencies = [
"anyhow",
"arc-swap",
"chrono",
"encoding_rs",
"etcetera",
"libloading",
"helix-loader",
"log",
"once_cell",
"quickcheck",
@ -397,6 +396,21 @@ dependencies = [
"which",
]
[[package]]
name = "helix-loader"
version = "0.6.0"
dependencies = [
"anyhow",
"cc",
"etcetera",
"libloading",
"once_cell",
"serde",
"threadpool",
"toml",
"tree-sitter",
]
[[package]]
name = "helix-lsp"
version = "0.6.0"
@ -432,6 +446,7 @@ dependencies = [
"grep-searcher",
"helix-core",
"helix-dap",
"helix-loader",
"helix-lsp",
"helix-tui",
"helix-view",

1
Cargo.toml
View file

@ -6,6 +6,7 @@ members = [
"helix-tui",
"helix-lsp",
"helix-dap",
"helix-loader",
"xtask",
]

2
book/src/guides/adding_languages.md
View file

@ -51,7 +51,6 @@ Grammar configuration takes these keys:
| Key | Description |
| --- | ----------- |
| `name` | The name of the tree-sitter grammar |
| `path` | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --build-grammars` to the correct path for compilation. When ommitted, the root of the grammar directory is used |
| `source` | The method of fetching the grammar - a table with a schema defined below |
Where `source` is a table with either these keys when using a grammar from a
@ -61,6 +60,7 @@ git repository:
| --- | ----------- |
| `git` | A git remote URL from which the grammar should be cloned |
| `rev` | The revision (commit hash or tag) which should be fetched |
| `subpath` | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --build-grammars` to the correct path for compilation. When omitted, the root of repository is used |
Or a `path` key with an absolute path to a locally available grammar directory.

2
docs/architecture.md
View file

@ -3,6 +3,8 @@
| ----------- | ----------- |
| helix-core | Core editing primitives, functional. |
| helix-lsp | Language server client |
| helix-dap | Debug Adapter Protocol (DAP) client |
| helix-loader | Functions for building, fetching, and loading external resources |
| helix-view | UI abstractions for use in backends, imperative shell. |
| helix-term | Terminal UI |
| helix-tui | TUI primitives, forked from tui-rs, inspired by Cursive |

6
helix-core/Cargo.toml
View file

@ -13,6 +13,8 @@ include = ["src/**/*", "README.md"]
[features]
[dependencies]
helix-loader = { version = "0.6", path = "../helix-loader" }
ropey = "1.3"
smallvec = "1.8"
smartstring = "1.0.0"
@ -33,13 +35,11 @@ toml = "0.5"
similar = "2.1"
etcetera = "0.3"
encoding_rs = "0.8"
chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
libloading = "0.7"
anyhow = "1"
etcetera = "0.3"
[dev-dependencies]
quickcheck = { version = "1", default-features = false }

27
helix-core/src/config.rs
View file

@ -1,33 +1,10 @@
use crate::merge_toml_values;
/// Default bultin-in languages.toml.
pub fn default_lang_config() -> toml::Value {
toml::from_slice(include_bytes!("../../languages.toml"))
.expect("Could not parse bultin-in languages.toml to valid toml")
}
/// User configured languages.toml file, merged with the default config.
pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
let def_lang_conf = default_lang_config();
let data = std::fs::read(crate::config_dir().join("languages.toml"));
let user_lang_conf = match data {
Ok(raw) => {
let value = toml::from_slice(&raw)?;
merge_toml_values(def_lang_conf, value)
}
Err(_) => def_lang_conf,
};
Ok(user_lang_conf)
}
/// Syntax configuration loader based on built-in languages.toml.
pub fn default_syntax_loader() -> crate::syntax::Configuration {
default_lang_config()
helix_loader::default_lang_config()
.try_into()
.expect("Could not serialize built-in languages.toml")
}
/// Syntax configuration loader based on user configured languages.toml.
pub fn user_syntax_loader() -> Result<crate::syntax::Configuration, toml::de::Error> {
user_lang_config()?.try_into()
helix_loader::user_lang_config()?.try_into()
}

2
helix-core/src/indent.rs
View file

@ -444,8 +444,6 @@ where
debugger: None,
auto_pairs: None,
}],
grammar: vec![],
grammar_selection: None,
});
// set runtime path so we can find the queries

141
helix-core/src/lib.rs
View file

@ -33,9 +33,6 @@ pub mod unicode {
pub use unicode_width as width;
}
static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
once_cell::sync::Lazy::new(runtime_dir);
pub fn find_first_non_whitespace_char(line: RopeSlice) -> Option<usize> {
line.chars().position(|ch| !ch.is_whitespace())
}
@ -85,144 +82,6 @@ pub fn find_root(root: Option<&str>, root_markers: &[String]) -> Option<std::pat
}
}
pub fn runtime_dir() -> std::path::PathBuf {
if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
return dir.into();
}
const RT_DIR: &str = "runtime";
let conf_dir = config_dir().join(RT_DIR);
if conf_dir.exists() {
return conf_dir;
}
if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
// this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
}
// fallback to location of the executable being run
std::env::current_exe()
.ok()
.and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
.unwrap()
}
pub fn config_dir() -> std::path::PathBuf {
// TODO: allow env var override
let strategy = choose_base_strategy().expect("Unable to find the config directory!");
let mut path = strategy.config_dir();
path.push("helix");
path
}
pub fn cache_dir() -> std::path::PathBuf {
// TODO: allow env var override
let strategy = choose_base_strategy().expect("Unable to find the config directory!");
let mut path = strategy.cache_dir();
path.push("helix");
path
}
pub fn config_file() -> std::path::PathBuf {
config_dir().join("config.toml")
}
pub fn lang_config_file() -> std::path::PathBuf {
config_dir().join("languages.toml")
}
pub fn log_file() -> std::path::PathBuf {
cache_dir().join("helix.log")
}
// right overrides left
pub fn merge_toml_values(left: toml::Value, right: toml::Value) -> toml::Value {
use toml::Value;
fn get_name(v: &Value) -> Option<&str> {
v.get("name").and_then(Value::as_str)
}
match (left, right) {
(Value::Array(mut left_items), Value::Array(right_items)) => {
left_items.reserve(right_items.len());
for rvalue in right_items {
let lvalue = get_name(&rvalue)
.and_then(|rname| left_items.iter().position(|v| get_name(v) == Some(rname)))
.map(|lpos| left_items.remove(lpos));
let mvalue = match lvalue {
Some(lvalue) => merge_toml_values(lvalue, rvalue),
None => rvalue,
};
left_items.push(mvalue);
}
Value::Array(left_items)
}
(Value::Table(mut left_map), Value::Table(right_map)) => {
for (rname, rvalue) in right_map {
match left_map.remove(&rname) {
Some(lvalue) => {
let merged_value = merge_toml_values(lvalue, rvalue);
left_map.insert(rname, merged_value);
}
None => {
left_map.insert(rname, rvalue);
}
}
}
Value::Table(left_map)
}
// Catch everything else we didn't handle, and use the right value
(_, value) => value,
}
}
#[cfg(test)]
mod merge_toml_tests {
use super::merge_toml_values;
#[test]
fn language_tomls() {
use toml::Value;
const USER: &str = "
[[language]]
name = \"nix\"
test = \"bbb\"
indent = { tab-width = 4, unit = \" \", test = \"aaa\" }
";
let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
.expect("Couldn't parse built-in languages config");
let user: Value = toml::from_str(USER).unwrap();
let merged = merge_toml_values(base, user);
let languages = merged.get("language").unwrap().as_array().unwrap();
let nix = languages
.iter()
.find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
.unwrap();
let nix_indent = nix.get("indent").unwrap();
// We changed tab-width and unit in indent so check them if they are the new values
assert_eq!(
nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
4
);
assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), " ");
// We added a new keys, so check them
assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
// We didn't change comment-token so it should be same
assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
}
}
pub use etcetera::home_dir;
use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
pub use ropey::{Rope, RopeBuilder, RopeSlice};
// pub use tendril::StrTendril as Tendril;

5
helix-core/src/path.rs
View file

@ -1,9 +1,10 @@
use etcetera::home_dir;
use std::path::{Component, Path, PathBuf};
/// Replaces users home directory from `path` with tilde `~` if the directory
/// is available, otherwise returns the path unchanged.
pub fn fold_home_dir(path: &Path) -> PathBuf {
if let Ok(home) = super::home_dir() {
if let Ok(home) = home_dir() {
if path.starts_with(&home) {
// it's ok to unwrap, the path starts with home dir
return PathBuf::from("~").join(path.strip_prefix(&home).unwrap());
@ -20,7 +21,7 @@ pub fn expand_tilde(path: &Path) -> PathBuf {
let mut components = path.components().peekable();
if let Some(Component::Normal(c)) = components.peek() {
if c == &"~" {
if let Ok(home) = super::home_dir() {
if let Ok(home) = home_dir() {
// it's ok to unwrap, the path starts with `~`
return home.join(path.strip_prefix("~").unwrap());
}

87
helix-core/src/syntax.rs
View file

@ -7,10 +7,6 @@ use crate::{
Rope, RopeSlice, Tendril,
};
use anyhow::{Context, Result};
use libloading::{Library, Symbol};
use tree_sitter::Language;
use arc_swap::{ArcSwap, Guard};
use slotmap::{DefaultKey as LayerId, HopSlotMap};
@ -27,33 +23,7 @@ use std::{
use once_cell::sync::{Lazy, OnceCell};
use serde::{Deserialize, Serialize};
#[cfg(unix)]
pub const DYLIB_EXTENSION: &str = "so";
#[cfg(windows)]
pub const DYLIB_EXTENSION: &str = "dll";
fn replace_dashes_with_underscores(name: &str) -> String {
name.replace('-', "_")
}
pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
let name = name.to_ascii_lowercase();
let mut library_path = runtime_path.join("grammars").join(&name);
library_path.set_extension(DYLIB_EXTENSION);
let library = unsafe { Library::new(&library_path) }
.with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
let language = unsafe {
let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
.get(language_fn_name.as_bytes())
.with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
language_fn()
};
std::mem::forget(library);
Ok(language)
}
use helix_loader::grammar::{get_language, load_runtime_file};
fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>
where
@ -81,19 +51,8 @@ where
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "kebab-case", deny_unknown_fields)]
pub struct Configuration {
#[serde(rename = "use-grammars")]
pub grammar_selection: Option<GrammarSelection>,
pub language: Vec<LanguageConfiguration>,
pub grammar: Vec<GrammarConfiguration>,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase", untagged)]
pub enum GrammarSelection {
Only(HashSet<String>),
Except(HashSet<String>),
}
// largely based on tree-sitter/cli/src/loader.rs
@ -279,29 +238,6 @@ pub struct IndentQuery {
pub outdent: HashSet<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct GrammarConfiguration {
#[serde(rename = "name")]
pub grammar_id: String, // c-sharp, rust
pub source: GrammarSource,
pub path: Option<String>,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
#[serde(untagged)]
pub enum GrammarSource {
Local {
path: String,
},
Git {
#[serde(rename = "git")]
remote: String,
#[serde(rename = "rev")]
revision: String,
},
}
#[derive(Debug)]
pub struct TextObjectQuery {
pub query: Query,
@ -398,14 +334,6 @@ impl TextObjectQuery {
}
}
pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
let path = crate::RUNTIME_DIR
.join("queries")
.join(language)
.join(filename);
std::fs::read_to_string(&path)
}
fn read_query(language: &str, filename: &str) -> String {
static INHERITS_REGEX: Lazy<Regex> =
Lazy::new(|| Regex::new(r";+\s*inherits\s*:?\s*([a-z_,()]+)\s*").unwrap());
@ -451,10 +379,7 @@ impl LanguageConfiguration {
if highlights_query.is_empty() {
None
} else {
let language = get_language(
&crate::RUNTIME_DIR,
self.grammar.as_deref().unwrap_or(&self.language_id),
)
let language = get_language(self.grammar.as_deref().unwrap_or(&self.language_id))
.map_err(|e| log::info!("{}", e))
.ok()?;
let config = HighlightConfiguration::new(
@ -2116,13 +2041,9 @@ mod test {
.map(String::from)
.collect();
let loader = Loader::new(Configuration {
language: vec![],
grammar: vec![],
grammar_selection: None,
});
let loader = Loader::new(Configuration { language: vec![] });
let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
let language = get_language("Rust").unwrap();
let config = HighlightConfiguration::new(
language,
&std::fs::read_to_string("../runtime/grammars/sources/rust/queries/highlights.scm")

23
helix-loader/Cargo.toml Normal file
View file

@ -0,0 +1,23 @@
[package]
name = "helix-loader"
version = "0.6.0"
description = "A post-modern text editor."
authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
edition = "2021"
license = "MPL-2.0"
categories = ["editor"]
repository = "https://github.com/helix-editor/helix"
homepage = "https://helix-editor.com"
[dependencies]
anyhow = "1"
serde = { version = "1.0", features = ["derive"] }
toml = "0.5"
etcetera = "0.3"
tree-sitter = "0.20"
libloading = "0.7"
once_cell = "1.9"
# cloning/compiling tree-sitter grammars
cc = { version = "1" }
threadpool = { version = "1.0" }

6
helix-loader/build.rs Normal file
View file

@ -0,0 +1,6 @@
fn main() {
println!(
"cargo:rustc-env=BUILD_TARGET={}",
std::env::var("TARGET").unwrap()
);
}

230
helix-term/src/grammars.rs → helix-loader/src/grammar.rs
View file

@ -1,63 +1,155 @@
use anyhow::{anyhow, Context, Result};
use libloading::{Library, Symbol};
use serde::{Deserialize, Serialize};
use std::fs;
use std::time::SystemTime;
use std::{
collections::HashSet,
path::{Path, PathBuf},
process::Command,
sync::mpsc::channel,
};
use tree_sitter::Language;
use helix_core::syntax::{GrammarConfiguration, GrammarSelection, GrammarSource, DYLIB_EXTENSION};
#[cfg(unix)]
const DYLIB_EXTENSION: &str = "so";
#[cfg(windows)]
const DYLIB_EXTENSION: &str = "dll";
#[derive(Debug, Serialize, Deserialize)]
struct Configuration {
#[serde(rename = "use-grammars")]
pub grammar_selection: Option<GrammarSelection>,
pub grammar: Vec<GrammarConfiguration>,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase", untagged)]
pub enum GrammarSelection {
Only(HashSet<String>),
Except(HashSet<String>),
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct GrammarConfiguration {
#[serde(rename = "name")]
pub grammar_id: String,
pub source: GrammarSource,
}
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase", untagged)]
pub enum GrammarSource {
Local {
path: String,
},
Git {
#[serde(rename = "git")]
remote: String,
#[serde(rename = "rev")]
revision: String,
subpath: Option<String>,
},
}
const BUILD_TARGET: &str = env!("BUILD_TARGET");
const REMOTE_NAME: &str = "origin";
pub fn get_language(name: &str) -> Result<Language> {
let name = name.to_ascii_lowercase();
let mut library_path = crate::runtime_dir().join("grammars").join(&name);
library_path.set_extension(DYLIB_EXTENSION);
let library = unsafe { Library::new(&library_path) }
.with_context(|| format!("Error opening dynamic library {library_path:?}"))?;
let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_"));
let language = unsafe {
let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
.get(language_fn_name.as_bytes())
.with_context(|| format!("Failed to load symbol {language_fn_name}"))?;
language_fn()
};
std::mem::forget(library);
Ok(language)
}
pub fn fetch_grammars() -> Result<()> {
run_parallel(get_grammar_configs(), fetch_grammar, "fetch")
run_parallel(get_grammar_configs()?, fetch_grammar, "fetch")
}
pub fn build_grammars() -> Result<()> {
run_parallel(get_grammar_configs(), build_grammar, "build")
run_parallel(get_grammar_configs()?, build_grammar, "build")
}
// Returns the set of grammar configurations the user requests.
// Grammars are configured in the default and user `languages.toml` and are
// merged. The `grammar_selection` key of the config is then used to filter
// down all grammars into a subset of the user's choosing.
fn get_grammar_configs() -> Result<Vec<GrammarConfiguration>> {
let config: Configuration = crate::user_lang_config()
.context("Could not parse languages.toml")?
.try_into()?;
let grammars = match config.grammar_selection {
Some(GrammarSelection::Only(selections)) => config
.grammar
.into_iter()
.filter(|grammar| selections.contains(&grammar.grammar_id))
.collect(),
Some(GrammarSelection::Except(rejections)) => config
.grammar
.into_iter()
.filter(|grammar| !rejections.contains(&grammar.grammar_id))
.collect(),
None => config.grammar,
};
Ok(grammars)
}
fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
where
F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
{
let mut n_jobs = 0;
let pool = threadpool::Builder::new().build();
let (tx, rx) = channel();
for grammar in grammars {
let tx = tx.clone();
n_jobs += 1;
pool.execute(move || {
let grammar_id = grammar.grammar_id.clone();
job(grammar).unwrap_or_else(|err| {
eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err)
});
// report progress
tx.send(1).unwrap();
tx.send(job(grammar)).unwrap();
});
}
pool.join();
if rx.try_iter().sum::<usize>() == n_jobs {
Ok(())
// TODO: print all failures instead of the first one found.
if let Some(failure) = rx.try_iter().find_map(|result| result.err()) {
Err(anyhow!(
"Failed to {} some grammar(s).\n{}",
action,
failure
))
} else {
Err(anyhow!("Failed to {} some grammar(s).", action))
Ok(())
}
}
fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
if let GrammarSource::Git { remote, revision } = grammar.source {
let grammar_dir = helix_core::runtime_dir()
if let GrammarSource::Git {
remote, revision, ..
} = grammar.source
{
let grammar_dir = crate::runtime_dir()
.join("grammars/sources")
.join(grammar.grammar_id.clone());
.join(&grammar.grammar_id);
fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory");
fs::create_dir_all(&grammar_dir).context(format!(
"Could not create grammar directory {:?}",
grammar_dir
))?;
// create the grammar dir contains a git directory
if !grammar_dir.join(".git").is_dir() {
@ -65,12 +157,12 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
}
// ensure the remote matches the configured remote
if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) {
if get_remote_url(&grammar_dir).map_or(true, |s| s != remote) {
set_remote(&grammar_dir, &remote)?;
}
// ensure the revision matches the configured revision
if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) {
if get_revision(&grammar_dir).map_or(true, |s| s != revision) {
// Fetch the exact revision from the remote.
// Supported by server-side git since v2.5.0 (July 2015),
// enabled by default on major git hosts.
@ -94,33 +186,38 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
// Sets the remote for a repository to the given URL, creating the remote if
// it does not yet exist.
fn set_remote(repository: &Path, remote_url: &str) -> Result<String> {
git(repository, ["remote", "set-url", REMOTE_NAME, remote_url])
.or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url]))
fn set_remote(repository_dir: &Path, remote_url: &str) -> Result<String> {
git(
repository_dir,
["remote", "set-url", REMOTE_NAME, remote_url],
)
.or_else(|_| git(repository_dir, ["remote", "add", REMOTE_NAME, remote_url]))
}
fn get_remote_url(repository: &Path) -> Option<String> {
git(repository, ["remote", "get-url", REMOTE_NAME]).ok()
fn get_remote_url(repository_dir: &Path) -> Option<String> {
git(repository_dir, ["remote", "get-url", REMOTE_NAME]).ok()
}
fn get_revision(repository: &Path) -> Option<String> {
git(repository, ["rev-parse", "HEAD"]).ok()
fn get_revision(repository_dir: &Path) -> Option<String> {
git(repository_dir, ["rev-parse", "HEAD"]).ok()
}
// A wrapper around 'git' commands which returns stdout in success and a
// helpful error message showing the command, stdout, and stderr in error.
fn git<I, S>(repository: &Path, args: I) -> Result<String>
fn git<I, S>(repository_dir: &Path, args: I) -> Result<String>
where
I: IntoIterator<Item = S>,
S: AsRef<std::ffi::OsStr>,
{
let output = Command::new("git")
.args(args)
.current_dir(repository)
.current_dir(repository_dir)
.output()?;
if output.status.success() {
Ok(String::from_utf8_lossy(&output.stdout).into_owned())
Ok(String::from_utf8_lossy(&output.stdout)
.trim_end()
.to_owned())
} else {
// TODO: figure out how to display the git command using `args`
Err(anyhow!(
@ -132,52 +229,37 @@ where
}
fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source {
PathBuf::from(path)
println!("{:#?}", grammar);
let grammar_dir = if let GrammarSource::Local { path } = &grammar.source {
PathBuf::from(&path)
} else {
helix_core::runtime_dir()
crate::runtime_dir()
.join("grammars/sources")
.join(grammar.grammar_id.clone())
.join(&grammar.grammar_id)
};
grammar_dir.read_dir().with_context(|| {
format!(
"The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?",
grammar_dir
)
let grammar_dir_entries = grammar_dir.read_dir().with_context(|| {
format!("Failed to read directory {grammar_dir:?}. Did you use 'hx --fetch-grammars'?")
})?;
let path = match grammar.path {
Some(ref subpath) => grammar_dir.join(subpath),
None => grammar_dir,
if grammar_dir_entries.count() == 0 {
return Err(anyhow!(
"Directory {grammar_dir:?} is empty. Did you use 'hx --fetch-grammars'?"
));
};
let path = match &grammar.source {
GrammarSource::Git {
subpath: Some(subpath),
..
} => grammar_dir.join(subpath),
_ => grammar_dir,
}
.join("src");
build_tree_sitter_library(&path, grammar)
}
// Returns the set of grammar configurations the user requests.
// Grammars are configured in the default and user `languages.toml` and are
// merged. The `grammar_selection` key of the config is then used to filter
// down all grammars into a subset of the user's choosing.
fn get_grammar_configs() -> Vec<GrammarConfiguration> {
let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");
match config.grammar_selection {
Some(GrammarSelection::Only(selections)) => config
.grammar
.into_iter()
.filter(|grammar| selections.contains(&grammar.grammar_id))
.collect(),
Some(GrammarSelection::Except(rejections)) => config
.grammar
.into_iter()
.filter(|grammar| !rejections.contains(&grammar.grammar_id))
.collect(),
None => config.grammar,
}
}
fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
let header_path = src_path;
let parser_path = src_path.join("parser.c");
@ -193,8 +275,8 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
None
}
};
let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars");
let mut library_path = parser_lib_path.join(grammar.grammar_id.clone());
let parser_lib_path = crate::runtime_dir().join("grammars");
let mut library_path = parser_lib_path.join(&grammar.grammar_id);
library_path.set_extension(DYLIB_EXTENSION);
let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
@ -210,7 +292,7 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
let mut config = cc::Build::new();
config
.cpp(true)
.opt_level(2)
.opt_level(3)
.cargo_metadata(false)
.host(BUILD_TARGET)
.target(BUILD_TARGET);
@ -245,7 +327,7 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
.arg(header_path)
.arg("-o")
.arg(&library_path)
.arg("-O2");
.arg("-O3");
if let Some(scanner_path) = scanner_path.as_ref() {
if scanner_path.extension() == Some("c".as_ref()) {
command.arg("-xc").arg("-std=c99").arg(scanner_path);
@ -294,3 +376,13 @@ fn needs_recompile(
fn mtime(path: &Path) -> Result<SystemTime> {
Ok(fs::metadata(path)?.modified()?)
}
/// Gives the contents of a file from a language's `runtime/queries/<lang>`
/// directory
pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
let path = crate::RUNTIME_DIR
.join("queries")
.join(language)
.join(filename);
std::fs::read_to_string(&path)
}

161
helix-loader/src/lib.rs Normal file
View file

@ -0,0 +1,161 @@
pub mod grammar;
use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
pub static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
once_cell::sync::Lazy::new(runtime_dir);
pub fn runtime_dir() -> std::path::PathBuf {
if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
return dir.into();
}
const RT_DIR: &str = "runtime";
let conf_dir = config_dir().join(RT_DIR);
if conf_dir.exists() {
return conf_dir;
}
if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
// this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
}
// fallback to location of the executable being run
std::env::current_exe()
.ok()
.and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
.unwrap()
}
pub fn config_dir() -> std::path::PathBuf {
// TODO: allow env var override
let strategy = choose_base_strategy().expect("Unable to find the config directory!");
let mut path = strategy.config_dir();
path.push("helix");
path
}
pub fn cache_dir() -> std::path::PathBuf {
// TODO: allow env var override
let strategy = choose_base_strategy().expect("Unable to find the config directory!");
let mut path = strategy.cache_dir();
path.push("helix");
path
}
pub fn config_file() -> std::path::PathBuf {
config_dir().join("config.toml")
}
pub fn lang_config_file() -> std::path::PathBuf {
config_dir().join("languages.toml")
}
pub fn log_file() -> std::path::PathBuf {
cache_dir().join("helix.log")
}
/// Default bultin-in languages.toml.
pub fn default_lang_config() -> toml::Value {
toml::from_slice(include_bytes!("../../languages.toml"))
.expect("Could not parse bultin-in languages.toml to valid toml")
}
/// User configured languages.toml file, merged with the default config.
pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
let def_lang_conf = default_lang_config();
let data = std::fs::read(crate::config_dir().join("languages.toml"));
let user_lang_conf = match data {
Ok(raw) => {
let value = toml::from_slice(&raw)?;
merge_toml_values(def_lang_conf, value)
}
Err(_) => def_lang_conf,
};
Ok(user_lang_conf)
}
// right overrides left
pub fn merge_toml_values(left: toml::Value, right: toml::Value) -> toml::Value {
use toml::Value;
fn get_name(v: &Value) -> Option<&str> {
v.get("name").and_then(Value::as_str)
}
match (left, right) {
(Value::Array(mut left_items), Value::Array(right_items)) => {
left_items.reserve(right_items.len());
for rvalue in right_items {
let lvalue = get_name(&rvalue)
.and_then(|rname| left_items.iter().position(|v| get_name(v) == Some(rname)))
.map(|lpos| left_items.remove(lpos));
let mvalue = match lvalue {
Some(lvalue) => merge_toml_values(lvalue, rvalue),
None => rvalue,
};
left_items.push(mvalue);
}
Value::Array(left_items)
}
(Value::Table(mut left_map), Value::Table(right_map)) => {
for (rname, rvalue) in right_map {
match left_map.remove(&rname) {
Some(lvalue) => {
let merged_value = merge_toml_values(lvalue, rvalue);
left_map.insert(rname, merged_value);
}
None => {
left_map.insert(rname, rvalue);
}
}
}
Value::Table(left_map)
}
// Catch everything else we didn't handle, and use the right value
(_, value) => value,
}
}
#[cfg(test)]
mod merge_toml_tests {
use super::merge_toml_values;
#[test]
fn language_tomls() {
use toml::Value;
const USER: &str = "
[[language]]
name = \"nix\"
test = \"bbb\"
indent = { tab-width = 4, unit = \" \", test = \"aaa\" }
";
let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
.expect("Couldn't parse built-in languages config");
let user: Value = toml::from_str(USER).unwrap();
let merged = merge_toml_values(base, user);
let languages = merged.get("language").unwrap().as_array().unwrap();
let nix = languages
.iter()
.find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
.unwrap();
let nix_indent = nix.get("indent").unwrap();
// We changed tab-width and unit in indent so check them if they are the new values
assert_eq!(
nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
4
);
assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), " ");
// We added a new keys, so check them
assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
// We didn't change comment-token so it should be same
assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
}
}

1
helix-term/Cargo.toml
View file

@ -26,6 +26,7 @@ helix-core = { version = "0.6", path = "../helix-core" }
helix-view = { version = "0.6", path = "../helix-view" }
helix-lsp = { version = "0.6", path = "../helix-lsp" }
helix-dap = { version = "0.6", path = "../helix-dap" }
helix-loader = { version = "0.6", path = "../helix-loader" }
anyhow = "1"
once_cell = "1.10"

5
helix-term/build.rs
View file

@ -14,10 +14,5 @@ fn main() {
None => env!("CARGO_PKG_VERSION").into(),
};
println!(
"cargo:rustc-env=BUILD_TARGET={}",
std::env::var("TARGET").unwrap()
);
println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
}

6
helix-term/src/application.rs
View file

@ -61,10 +61,10 @@ impl Application {
let mut compositor = Compositor::new()?;
let size = compositor.size();
let conf_dir = helix_core::config_dir();
let conf_dir = helix_loader::config_dir();
let theme_loader =
std::sync::Arc::new(theme::Loader::new(&conf_dir, &helix_core::runtime_dir()));
std::sync::Arc::new(theme::Loader::new(&conf_dir, &helix_loader::runtime_dir()));
let true_color = config.editor.true_color || crate::true_color();
let theme = config
@ -109,7 +109,7 @@ impl Application {
compositor.push(editor_view);
if args.load_tutor {
let path = helix_core::runtime_dir().join("tutor.txt");
let path = helix_loader::runtime_dir().join("tutor.txt");
editor.open(path, Action::VerticalSplit)?;
// Unset path to prevent accidentally saving to the original tutor file.
doc_mut!(editor).set_path(None)?;

2
helix-term/src/commands/typed.rs
View file

@ -828,7 +828,7 @@ fn tutor(
_args: &[Cow<str>],
_event: PromptEvent,
) -> anyhow::Result<()> {
let path = helix_core::runtime_dir().join("tutor.txt");
let path = helix_loader::runtime_dir().join("tutor.txt");
cx.editor.open(path, Action::Replace)?;
// Unset path to prevent accidentally saving to the original tutor file.
doc_mut!(cx.editor).set_path(None)?;

14
helix-term/src/health.rs
View file

@ -1,8 +1,6 @@
use crossterm::style::{Color, Print, Stylize};
use helix_core::{
config::{default_syntax_loader, user_syntax_loader},
syntax::load_runtime_file,
};
use helix_core::config::{default_syntax_loader, user_syntax_loader};
use helix_loader::grammar::load_runtime_file;
#[derive(Copy, Clone)]
pub enum TsFeature {
@ -43,10 +41,10 @@ impl TsFeature {
/// Display general diagnostics.
pub fn general() {
let config_file = helix_core::config_file();
let lang_file = helix_core::lang_config_file();
let log_file = helix_core::log_file();
let rt_dir = helix_core::runtime_dir();
let config_file = helix_loader::config_file();
let lang_file = helix_loader::lang_config_file();
let log_file = helix_loader::log_file();
let rt_dir = helix_loader::runtime_dir();
if config_file.exists() {
println!("Config file: {}", config_file.display());

1
helix-term/src/lib.rs
View file

@ -7,7 +7,6 @@ pub mod commands;
pub mod compositor;
pub mod config;
pub mod health;
pub mod grammars;
pub mod job;
pub mod keymap;
pub mod ui;

10
helix-term/src/main.rs
View file

@ -40,7 +40,7 @@ fn main() -> Result<()> {
#[tokio::main]
async fn main_impl() -> Result<i32> {
let logpath = helix_core::log_file();
let logpath = helix_loader::log_file();
let parent = logpath.parent().unwrap();
if !parent.exists() {
std::fs::create_dir_all(parent).ok();
@ -105,21 +105,21 @@ FLAGS:
}
if args.fetch_grammars {
helix_term::grammars::fetch_grammars()?;
helix_loader::grammar::fetch_grammars()?;
return Ok(0);
}
if args.build_grammars {
helix_term::grammars::build_grammars()?;
helix_loader::grammar::build_grammars()?;
return Ok(0);
}
let conf_dir = helix_core::config_dir();
let conf_dir = helix_loader::config_dir();
if !conf_dir.exists() {
std::fs::create_dir_all(&conf_dir).ok();
}
let config = match std::fs::read_to_string(helix_core::config_file()) {
let config = match std::fs::read_to_string(helix_loader::config_file()) {
Ok(config) => toml::from_str(&config)
.map(merge_keys)
.unwrap_or_else(|err| {

4
helix-term/src/ui/mod.rs
View file

@ -218,9 +218,9 @@ pub mod completers {
}
pub fn theme(_editor: &Editor, input: &str) -> Vec<Completion> {
let mut names = theme::Loader::read_names(&helix_core::runtime_dir().join("themes"));
let mut names = theme::Loader::read_names(&helix_loader::runtime_dir().join("themes"));
names.extend(theme::Loader::read_names(
&helix_core::config_dir().join("themes"),
&helix_loader::config_dir().join("themes"),
));
names.push("default".into());
names.push("base16_default".into());

12
languages.toml
View file

@ -302,8 +302,7 @@ indent = { tab-width = 2, unit = " " }
[[grammar]]
name = "typescript"
source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a" }
path = "typescript"
source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a", subpath = "typescript" }
[[language]]
name = "tsx"
@ -317,8 +316,7 @@ indent = { tab-width = 2, unit = " " }
[[grammar]]
name = "tsx"
source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a" }
path = "tsx"
source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a", subpath = "tsx" }
[[language]]
name = "css"
@ -522,8 +520,7 @@ indent = { tab-width = 2, unit = " " }
[[grammar]]
name = "ocaml"
source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a" }
path = "ocaml"
source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a", subpath = "ocaml" }
[[language]]
name = "ocaml-interface"
@ -536,8 +533,7 @@ indent = { tab-width = 2, unit = " "}
[[grammar]]
name = "ocaml-interface"
source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a" }
path = "interface"
source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a", subpath = "interface" }
[[language]]
name = "lua"