migrate grammar fetching/building code into helix-loader crate

This is a rather large refactor that moves most of the code for
loading, fetching, and building grammars into a new helix-loader
module. This works well with the [[grammars]] syntax for
languages.toml defined earlier: we only have to depend on the types
for GrammarConfiguration in helix-loader and can leave all the
[[language]] entries for helix-core.

Cargo.lock

@@ -358,12 +358,11 @@ dependencies = [
 name = "helix-core"
 version = "0.6.0"
 dependencies = [
- "anyhow",
  "arc-swap",
  "chrono",
  "encoding_rs",
  "etcetera",
- "libloading",
+ "helix-loader",
  "log",
  "once_cell",
  "quickcheck",
@@ -397,6 +396,21 @@ dependencies = [
  "which",
 ]
 
+[[package]]
+name = "helix-loader"
+version = "0.6.0"
+dependencies = [
+ "anyhow",
+ "cc",
+ "etcetera",
+ "libloading",
+ "once_cell",
+ "serde",
+ "threadpool",
+ "toml",
+ "tree-sitter",
+]
+
 [[package]]
 name = "helix-lsp"
 version = "0.6.0"
@@ -432,6 +446,7 @@ dependencies = [
  "grep-searcher",
  "helix-core",
  "helix-dap",
+ "helix-loader",
  "helix-lsp",
  "helix-tui",
  "helix-view",

Cargo.toml

@@ -6,6 +6,7 @@ members = [
   "helix-tui",
   "helix-lsp",
   "helix-dap",
+  "helix-loader",
   "xtask",
 ]
 

book/src/guides/adding_languages.md

@@ -51,7 +51,6 @@ Grammar configuration takes these keys:
 | Key      | Description                                                              |
 | ---      | -----------                                                              |
 | `name`   | The name of the tree-sitter grammar                                      |
-| `path`   | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --build-grammars` to the correct path for compilation. When ommitted, the root of the grammar directory is used |
 | `source` | The method of fetching the grammar - a table with a schema defined below |
 
 Where `source` is a table with either these keys when using a grammar from a
@@ -61,6 +60,7 @@ git repository:
 | ---    | -----------                                               |
 | `git`  | A git remote URL from which the grammar should be cloned  |
 | `rev`  | The revision (commit hash or tag) which should be fetched |
+| `subpath` | A path within the grammar directory which should be built. Some grammar repositories host multiple grammars (for example `tree-sitter-typescript` and `tree-sitter-ocaml`) in subdirectories. This key is used to point `hx --build-grammars` to the correct path for compilation. When omitted, the root of repository is used |
 
 Or a `path` key with an absolute path to a locally available grammar directory.
 

docs/architecture.md

@@ -3,6 +3,8 @@
 | -----------  | -----------                                                      |
 | helix-core   | Core editing primitives, functional.                             |
 | helix-lsp    | Language server client                                           |
+| helix-dap    | Debug Adapter Protocol (DAP) client                              |
+| helix-loader | Functions for building, fetching, and loading external resources |
 | helix-view   | UI abstractions for use in backends, imperative shell.           |
 | helix-term   | Terminal UI                                                      |
 | helix-tui    | TUI primitives, forked from tui-rs, inspired by Cursive          |

helix-core/Cargo.toml

@@ -13,6 +13,8 @@ include = ["src/**/*", "README.md"]
 [features]
 
 [dependencies]
+helix-loader = { version = "0.6", path = "../helix-loader" }
+
 ropey = "1.3"
 smallvec = "1.8"
 smartstring = "1.0.0"
@@ -33,13 +35,11 @@ toml = "0.5"
 
 similar = "2.1"
 
-etcetera = "0.3"
 encoding_rs = "0.8"
 
 chrono = { version = "0.4", default-features = false, features = ["alloc", "std"] }
 
-libloading = "0.7"
+etcetera = "0.3"
-anyhow = "1"
 
 [dev-dependencies]
 quickcheck = { version = "1", default-features = false }

helix-core/src/config.rs

@@ -1,33 +1,10 @@
-use crate::merge_toml_values;
-
-/// Default bultin-in languages.toml.
-pub fn default_lang_config() -> toml::Value {
-    toml::from_slice(include_bytes!("../../languages.toml"))
-        .expect("Could not parse bultin-in languages.toml to valid toml")
-}
-
-/// User configured languages.toml file, merged with the default config.
-pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
-    let def_lang_conf = default_lang_config();
-    let data = std::fs::read(crate::config_dir().join("languages.toml"));
-    let user_lang_conf = match data {
-        Ok(raw) => {
-            let value = toml::from_slice(&raw)?;
-            merge_toml_values(def_lang_conf, value)
-        }
-        Err(_) => def_lang_conf,
-    };
-
-    Ok(user_lang_conf)
-}
-
 /// Syntax configuration loader based on built-in languages.toml.
 pub fn default_syntax_loader() -> crate::syntax::Configuration {
-    default_lang_config()
+    helix_loader::default_lang_config()
         .try_into()
         .expect("Could not serialize built-in languages.toml")
 }
 /// Syntax configuration loader based on user configured languages.toml.
 pub fn user_syntax_loader() -> Result<crate::syntax::Configuration, toml::de::Error> {
-    user_lang_config()?.try_into()
+    helix_loader::user_lang_config()?.try_into()
 }

helix-core/src/indent.rs

@@ -444,8 +444,6 @@ where
                 debugger: None,
                 auto_pairs: None,
             }],
-            grammar: vec![],
-            grammar_selection: None,
         });
 
         // set runtime path so we can find the queries

helix-core/src/lib.rs

@@ -33,9 +33,6 @@ pub mod unicode {
     pub use unicode_width as width;
 }
 
-static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
-    once_cell::sync::Lazy::new(runtime_dir);
-
 pub fn find_first_non_whitespace_char(line: RopeSlice) -> Option<usize> {
     line.chars().position(|ch| !ch.is_whitespace())
 }
@@ -85,144 +82,6 @@ pub fn find_root(root: Option<&str>, root_markers: &[String]) -> Option<std::pat
     }
 }
 
-pub fn runtime_dir() -> std::path::PathBuf {
-    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
-        return dir.into();
-    }
-
-    const RT_DIR: &str = "runtime";
-    let conf_dir = config_dir().join(RT_DIR);
-    if conf_dir.exists() {
-        return conf_dir;
-    }
-
-    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
-        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
-        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
-    }
-
-    // fallback to location of the executable being run
-    std::env::current_exe()
-        .ok()
-        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
-        .unwrap()
-}
-
-pub fn config_dir() -> std::path::PathBuf {
-    // TODO: allow env var override
-    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
-    let mut path = strategy.config_dir();
-    path.push("helix");
-    path
-}
-
-pub fn cache_dir() -> std::path::PathBuf {
-    // TODO: allow env var override
-    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
-    let mut path = strategy.cache_dir();
-    path.push("helix");
-    path
-}
-
-pub fn config_file() -> std::path::PathBuf {
-    config_dir().join("config.toml")
-}
-
-pub fn lang_config_file() -> std::path::PathBuf {
-    config_dir().join("languages.toml")
-}
-
-pub fn log_file() -> std::path::PathBuf {
-    cache_dir().join("helix.log")
-}
-
-// right overrides left
-pub fn merge_toml_values(left: toml::Value, right: toml::Value) -> toml::Value {
-    use toml::Value;
-
-    fn get_name(v: &Value) -> Option<&str> {
-        v.get("name").and_then(Value::as_str)
-    }
-
-    match (left, right) {
-        (Value::Array(mut left_items), Value::Array(right_items)) => {
-            left_items.reserve(right_items.len());
-            for rvalue in right_items {
-                let lvalue = get_name(&rvalue)
-                    .and_then(|rname| left_items.iter().position(|v| get_name(v) == Some(rname)))
-                    .map(|lpos| left_items.remove(lpos));
-                let mvalue = match lvalue {
-                    Some(lvalue) => merge_toml_values(lvalue, rvalue),
-                    None => rvalue,
-                };
-                left_items.push(mvalue);
-            }
-            Value::Array(left_items)
-        }
-        (Value::Table(mut left_map), Value::Table(right_map)) => {
-            for (rname, rvalue) in right_map {
-                match left_map.remove(&rname) {
-                    Some(lvalue) => {
-                        let merged_value = merge_toml_values(lvalue, rvalue);
-                        left_map.insert(rname, merged_value);
-                    }
-                    None => {
-                        left_map.insert(rname, rvalue);
-                    }
-                }
-            }
-            Value::Table(left_map)
-        }
-        // Catch everything else we didn't handle, and use the right value
-        (_, value) => value,
-    }
-}
-
-#[cfg(test)]
-mod merge_toml_tests {
-    use super::merge_toml_values;
-
-    #[test]
-    fn language_tomls() {
-        use toml::Value;
-
-        const USER: &str = "
-        [[language]]
-        name = \"nix\"
-        test = \"bbb\"
-        indent = { tab-width = 4, unit = \"    \", test = \"aaa\" }
-        ";
-
-        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
-            .expect("Couldn't parse built-in languages config");
-        let user: Value = toml::from_str(USER).unwrap();
-
-        let merged = merge_toml_values(base, user);
-        let languages = merged.get("language").unwrap().as_array().unwrap();
-        let nix = languages
-            .iter()
-            .find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
-            .unwrap();
-        let nix_indent = nix.get("indent").unwrap();
-
-        // We changed tab-width and unit in indent so check them if they are the new values
-        assert_eq!(
-            nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
-            4
-        );
-        assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), "    ");
-        // We added a new keys, so check them
-        assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
-        assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
-        // We didn't change comment-token so it should be same
-        assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
-    }
-}
-
-pub use etcetera::home_dir;
-
-use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
-
 pub use ropey::{Rope, RopeBuilder, RopeSlice};
 
 // pub use tendril::StrTendril as Tendril;

helix-core/src/path.rs

@@ -1,9 +1,10 @@
+use etcetera::home_dir;
 use std::path::{Component, Path, PathBuf};
 
 /// Replaces users home directory from `path` with tilde `~` if the directory
 /// is available, otherwise returns the path unchanged.
 pub fn fold_home_dir(path: &Path) -> PathBuf {
-    if let Ok(home) = super::home_dir() {
+    if let Ok(home) = home_dir() {
         if path.starts_with(&home) {
             // it's ok to unwrap, the path starts with home dir
             return PathBuf::from("~").join(path.strip_prefix(&home).unwrap());
@@ -20,7 +21,7 @@ pub fn expand_tilde(path: &Path) -> PathBuf {
     let mut components = path.components().peekable();
     if let Some(Component::Normal(c)) = components.peek() {
         if c == &"~" {
-            if let Ok(home) = super::home_dir() {
+            if let Ok(home) = home_dir() {
                 // it's ok to unwrap, the path starts with `~`
                 return home.join(path.strip_prefix("~").unwrap());
             }

helix-core/src/syntax.rs

@@ -7,10 +7,6 @@ use crate::{
     Rope, RopeSlice, Tendril,
 };
 
-use anyhow::{Context, Result};
-use libloading::{Library, Symbol};
-use tree_sitter::Language;
-
 use arc_swap::{ArcSwap, Guard};
 use slotmap::{DefaultKey as LayerId, HopSlotMap};
 
@@ -27,33 +23,7 @@ use std::{
 use once_cell::sync::{Lazy, OnceCell};
 use serde::{Deserialize, Serialize};
 
-#[cfg(unix)]
+use helix_loader::grammar::{get_language, load_runtime_file};
-pub const DYLIB_EXTENSION: &str = "so";
-
-#[cfg(windows)]
-pub const DYLIB_EXTENSION: &str = "dll";
-
-fn replace_dashes_with_underscores(name: &str) -> String {
-    name.replace('-', "_")
-}
-
-pub fn get_language(runtime_path: &std::path::Path, name: &str) -> Result<Language> {
-    let name = name.to_ascii_lowercase();
-    let mut library_path = runtime_path.join("grammars").join(&name);
-    library_path.set_extension(DYLIB_EXTENSION);
-
-    let library = unsafe { Library::new(&library_path) }
-        .with_context(|| format!("Error opening dynamic library {:?}", &library_path))?;
-    let language_fn_name = format!("tree_sitter_{}", replace_dashes_with_underscores(&name));
-    let language = unsafe {
-        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
-            .get(language_fn_name.as_bytes())
-            .with_context(|| format!("Failed to load symbol {}", language_fn_name))?;
-        language_fn()
-    };
-    std::mem::forget(library);
-    Ok(language)
-}
 
 fn deserialize_regex<'de, D>(deserializer: D) -> Result<Option<Regex>, D::Error>
 where
@@ -81,19 +51,8 @@ where
 }
 
 #[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "kebab-case", deny_unknown_fields)]
 pub struct Configuration {
-    #[serde(rename = "use-grammars")]
-    pub grammar_selection: Option<GrammarSelection>,
     pub language: Vec<LanguageConfiguration>,
-    pub grammar: Vec<GrammarConfiguration>,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase", untagged)]
-pub enum GrammarSelection {
-    Only(HashSet<String>),
-    Except(HashSet<String>),
 }
 
 // largely based on tree-sitter/cli/src/loader.rs
@@ -279,29 +238,6 @@ pub struct IndentQuery {
     pub outdent: HashSet<String>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
-pub struct GrammarConfiguration {
-    #[serde(rename = "name")]
-    pub grammar_id: String, // c-sharp, rust
-    pub source: GrammarSource,
-    pub path: Option<String>,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "lowercase")]
-#[serde(untagged)]
-pub enum GrammarSource {
-    Local {
-        path: String,
-    },
-    Git {
-        #[serde(rename = "git")]
-        remote: String,
-        #[serde(rename = "rev")]
-        revision: String,
-    },
-}
-
 #[derive(Debug)]
 pub struct TextObjectQuery {
     pub query: Query,
@@ -398,14 +334,6 @@ impl TextObjectQuery {
     }
 }
 
-pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
-    let path = crate::RUNTIME_DIR
-        .join("queries")
-        .join(language)
-        .join(filename);
-    std::fs::read_to_string(&path)
-}
-
 fn read_query(language: &str, filename: &str) -> String {
     static INHERITS_REGEX: Lazy<Regex> =
         Lazy::new(|| Regex::new(r";+\s*inherits\s*:?\s*([a-z_,()]+)\s*").unwrap());
@@ -451,10 +379,7 @@ impl LanguageConfiguration {
         if highlights_query.is_empty() {
             None
         } else {
-            let language = get_language(
+            let language = get_language(self.grammar.as_deref().unwrap_or(&self.language_id))
-                &crate::RUNTIME_DIR,
-                self.grammar.as_deref().unwrap_or(&self.language_id),
-            )
                 .map_err(|e| log::info!("{}", e))
                 .ok()?;
             let config = HighlightConfiguration::new(
@@ -2116,13 +2041,9 @@ mod test {
         .map(String::from)
         .collect();
 
-        let loader = Loader::new(Configuration {
+        let loader = Loader::new(Configuration { language: vec![] });
-            language: vec![],
-            grammar: vec![],
-            grammar_selection: None,
-        });
 
-        let language = get_language(&crate::RUNTIME_DIR, "Rust").unwrap();
+        let language = get_language("Rust").unwrap();
         let config = HighlightConfiguration::new(
             language,
             &std::fs::read_to_string("../runtime/grammars/sources/rust/queries/highlights.scm")

helix-loader/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "helix-loader"
+version = "0.6.0"
+description = "A post-modern text editor."
+authors = ["Blaž Hrastnik <blaz@mxxn.io>"]
+edition = "2021"
+license = "MPL-2.0"
+categories = ["editor"]
+repository = "https://github.com/helix-editor/helix"
+homepage = "https://helix-editor.com"
+
+[dependencies]
+anyhow = "1"
+serde = { version = "1.0", features = ["derive"] }
+toml = "0.5"
+etcetera = "0.3"
+tree-sitter = "0.20"
+libloading = "0.7"
+once_cell = "1.9"
+
+# cloning/compiling tree-sitter grammars
+cc = { version = "1" }
+threadpool = { version = "1.0" }

helix-loader/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+    println!(
+        "cargo:rustc-env=BUILD_TARGET={}",
+        std::env::var("TARGET").unwrap()
+    );
+}

helix-loader/src/grammar.rs

@@ -1,63 +1,155 @@
 use anyhow::{anyhow, Context, Result};
+use libloading::{Library, Symbol};
+use serde::{Deserialize, Serialize};
 use std::fs;
 use std::time::SystemTime;
 use std::{
+    collections::HashSet,
     path::{Path, PathBuf},
     process::Command,
     sync::mpsc::channel,
 };
+use tree_sitter::Language;
 
-use helix_core::syntax::{GrammarConfiguration, GrammarSelection, GrammarSource, DYLIB_EXTENSION};
+#[cfg(unix)]
+const DYLIB_EXTENSION: &str = "so";
+
+#[cfg(windows)]
+const DYLIB_EXTENSION: &str = "dll";
+
+#[derive(Debug, Serialize, Deserialize)]
+struct Configuration {
+    #[serde(rename = "use-grammars")]
+    pub grammar_selection: Option<GrammarSelection>,
+    pub grammar: Vec<GrammarConfiguration>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSelection {
+    Only(HashSet<String>),
+    Except(HashSet<String>),
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct GrammarConfiguration {
+    #[serde(rename = "name")]
+    pub grammar_id: String,
+    pub source: GrammarSource,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(rename_all = "lowercase", untagged)]
+pub enum GrammarSource {
+    Local {
+        path: String,
+    },
+    Git {
+        #[serde(rename = "git")]
+        remote: String,
+        #[serde(rename = "rev")]
+        revision: String,
+        subpath: Option<String>,
+    },
+}
 
 const BUILD_TARGET: &str = env!("BUILD_TARGET");
 const REMOTE_NAME: &str = "origin";
 
+pub fn get_language(name: &str) -> Result<Language> {
+    let name = name.to_ascii_lowercase();
+    let mut library_path = crate::runtime_dir().join("grammars").join(&name);
+    library_path.set_extension(DYLIB_EXTENSION);
+
+    let library = unsafe { Library::new(&library_path) }
+        .with_context(|| format!("Error opening dynamic library {library_path:?}"))?;
+    let language_fn_name = format!("tree_sitter_{}", name.replace('-', "_"));
+    let language = unsafe {
+        let language_fn: Symbol<unsafe extern "C" fn() -> Language> = library
+            .get(language_fn_name.as_bytes())
+            .with_context(|| format!("Failed to load symbol {language_fn_name}"))?;
+        language_fn()
+    };
+    std::mem::forget(library);
+    Ok(language)
+}
+
 pub fn fetch_grammars() -> Result<()> {
-    run_parallel(get_grammar_configs(), fetch_grammar, "fetch")
+    run_parallel(get_grammar_configs()?, fetch_grammar, "fetch")
 }
 
 pub fn build_grammars() -> Result<()> {
-    run_parallel(get_grammar_configs(), build_grammar, "build")
+    run_parallel(get_grammar_configs()?, build_grammar, "build")
+}
+
+// Returns the set of grammar configurations the user requests.
+// Grammars are configured in the default and user `languages.toml` and are
+// merged. The `grammar_selection` key of the config is then used to filter
+// down all grammars into a subset of the user's choosing.
+fn get_grammar_configs() -> Result<Vec<GrammarConfiguration>> {
+    let config: Configuration = crate::user_lang_config()
+        .context("Could not parse languages.toml")?
+        .try_into()?;
+
+    let grammars = match config.grammar_selection {
+        Some(GrammarSelection::Only(selections)) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| selections.contains(&grammar.grammar_id))
+            .collect(),
+        Some(GrammarSelection::Except(rejections)) => config
+            .grammar
+            .into_iter()
+            .filter(|grammar| !rejections.contains(&grammar.grammar_id))
+            .collect(),
+        None => config.grammar,
+    };
+
+    Ok(grammars)
 }
 
 fn run_parallel<F>(grammars: Vec<GrammarConfiguration>, job: F, action: &'static str) -> Result<()>
 where
     F: Fn(GrammarConfiguration) -> Result<()> + std::marker::Send + 'static + Copy,
 {
-    let mut n_jobs = 0;
     let pool = threadpool::Builder::new().build();
     let (tx, rx) = channel();
 
     for grammar in grammars {
         let tx = tx.clone();
-        n_jobs += 1;
 
         pool.execute(move || {
-            let grammar_id = grammar.grammar_id.clone();
+            tx.send(job(grammar)).unwrap();
-            job(grammar).unwrap_or_else(|err| {
-                eprintln!("Failed to {} grammar '{}'\n{}", action, grammar_id, err)
-            });
-
-            // report progress
-            tx.send(1).unwrap();
         });
     }
     pool.join();
 
-    if rx.try_iter().sum::<usize>() == n_jobs {
+    // TODO: print all failures instead of the first one found.
-        Ok(())
+    if let Some(failure) = rx.try_iter().find_map(|result| result.err()) {
+        Err(anyhow!(
+            "Failed to {} some grammar(s).\n{}",
+            action,
+            failure
+        ))
     } else {
-        Err(anyhow!("Failed to {} some grammar(s).", action))
+        Ok(())
     }
 }
 
 fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
-    if let GrammarSource::Git { remote, revision } = grammar.source {
+    if let GrammarSource::Git {
-        let grammar_dir = helix_core::runtime_dir()
+        remote, revision, ..
+    } = grammar.source
+    {
+        let grammar_dir = crate::runtime_dir()
             .join("grammars/sources")
-            .join(grammar.grammar_id.clone());
+            .join(&grammar.grammar_id);
 
-        fs::create_dir_all(grammar_dir.clone()).expect("Could not create grammar directory");
+        fs::create_dir_all(&grammar_dir).context(format!(
+            "Could not create grammar directory {:?}",
+            grammar_dir
+        ))?;
 
         // create the grammar dir contains a git directory
         if !grammar_dir.join(".git").is_dir() {
@@ -65,12 +157,12 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
         }
 
         // ensure the remote matches the configured remote
-        if get_remote_url(&grammar_dir).map_or(true, |s| s.trim_end() != remote) {
+        if get_remote_url(&grammar_dir).map_or(true, |s| s != remote) {
             set_remote(&grammar_dir, &remote)?;
         }
 
         // ensure the revision matches the configured revision
-        if get_revision(&grammar_dir).map_or(true, |s| s.trim_end() != revision) {
+        if get_revision(&grammar_dir).map_or(true, |s| s != revision) {
             // Fetch the exact revision from the remote.
             // Supported by server-side git since v2.5.0 (July 2015),
             // enabled by default on major git hosts.
@@ -94,33 +186,38 @@ fn fetch_grammar(grammar: GrammarConfiguration) -> Result<()> {
 
 // Sets the remote for a repository to the given URL, creating the remote if
 // it does not yet exist.
-fn set_remote(repository: &Path, remote_url: &str) -> Result<String> {
+fn set_remote(repository_dir: &Path, remote_url: &str) -> Result<String> {
-    git(repository, ["remote", "set-url", REMOTE_NAME, remote_url])
+    git(
-        .or_else(|_| git(repository, ["remote", "add", REMOTE_NAME, remote_url]))
+        repository_dir,
+        ["remote", "set-url", REMOTE_NAME, remote_url],
+    )
+    .or_else(|_| git(repository_dir, ["remote", "add", REMOTE_NAME, remote_url]))
 }
 
-fn get_remote_url(repository: &Path) -> Option<String> {
+fn get_remote_url(repository_dir: &Path) -> Option<String> {
-    git(repository, ["remote", "get-url", REMOTE_NAME]).ok()
+    git(repository_dir, ["remote", "get-url", REMOTE_NAME]).ok()
 }
 
-fn get_revision(repository: &Path) -> Option<String> {
+fn get_revision(repository_dir: &Path) -> Option<String> {
-    git(repository, ["rev-parse", "HEAD"]).ok()
+    git(repository_dir, ["rev-parse", "HEAD"]).ok()
 }
 
 // A wrapper around 'git' commands which returns stdout in success and a
 // helpful error message showing the command, stdout, and stderr in error.
-fn git<I, S>(repository: &Path, args: I) -> Result<String>
+fn git<I, S>(repository_dir: &Path, args: I) -> Result<String>
 where
     I: IntoIterator<Item = S>,
     S: AsRef<std::ffi::OsStr>,
 {
     let output = Command::new("git")
         .args(args)
-        .current_dir(repository)
+        .current_dir(repository_dir)
         .output()?;
 
     if output.status.success() {
-        Ok(String::from_utf8_lossy(&output.stdout).into_owned())
+        Ok(String::from_utf8_lossy(&output.stdout)
+            .trim_end()
+            .to_owned())
     } else {
         // TODO: figure out how to display the git command using `args`
         Err(anyhow!(
@@ -132,52 +229,37 @@ where
 }
 
 fn build_grammar(grammar: GrammarConfiguration) -> Result<()> {
-    let grammar_dir = if let GrammarSource::Local { ref path } = grammar.source {
+    println!("{:#?}", grammar);
-        PathBuf::from(path)
+    let grammar_dir = if let GrammarSource::Local { path } = &grammar.source {
+        PathBuf::from(&path)
     } else {
-        helix_core::runtime_dir()
+        crate::runtime_dir()
             .join("grammars/sources")
-            .join(grammar.grammar_id.clone())
+            .join(&grammar.grammar_id)
     };
 
-    grammar_dir.read_dir().with_context(|| {
+    let grammar_dir_entries = grammar_dir.read_dir().with_context(|| {
-        format!(
+        format!("Failed to read directory {grammar_dir:?}. Did you use 'hx --fetch-grammars'?")
-            "The directory {:?} is empty, you probably need to use 'hx --fetch-grammars'?",
-            grammar_dir
-        )
     })?;
 
-    let path = match grammar.path {
+    if grammar_dir_entries.count() == 0 {
-        Some(ref subpath) => grammar_dir.join(subpath),
+        return Err(anyhow!(
-        None => grammar_dir,
+            "Directory {grammar_dir:?} is empty. Did you use 'hx --fetch-grammars'?"
+        ));
+    };
+
+    let path = match &grammar.source {
+        GrammarSource::Git {
+            subpath: Some(subpath),
+            ..
+        } => grammar_dir.join(subpath),
+        _ => grammar_dir,
     }
     .join("src");
 
     build_tree_sitter_library(&path, grammar)
 }
 
-// Returns the set of grammar configurations the user requests.
-// Grammars are configured in the default and user `languages.toml` and are
-// merged. The `grammar_selection` key of the config is then used to filter
-// down all grammars into a subset of the user's choosing.
-fn get_grammar_configs() -> Vec<GrammarConfiguration> {
-    let config = helix_core::config::user_syntax_loader().expect("Could not parse languages.toml");
-
-    match config.grammar_selection {
-        Some(GrammarSelection::Only(selections)) => config
-            .grammar
-            .into_iter()
-            .filter(|grammar| selections.contains(&grammar.grammar_id))
-            .collect(),
-        Some(GrammarSelection::Except(rejections)) => config
-            .grammar
-            .into_iter()
-            .filter(|grammar| !rejections.contains(&grammar.grammar_id))
-            .collect(),
-        None => config.grammar,
-    }
-}
-
 fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) -> Result<()> {
     let header_path = src_path;
     let parser_path = src_path.join("parser.c");
@@ -193,8 +275,8 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
             None
         }
     };
-    let parser_lib_path = helix_core::runtime_dir().join("../runtime/grammars");
+    let parser_lib_path = crate::runtime_dir().join("grammars");
-    let mut library_path = parser_lib_path.join(grammar.grammar_id.clone());
+    let mut library_path = parser_lib_path.join(&grammar.grammar_id);
     library_path.set_extension(DYLIB_EXTENSION);
 
     let recompile = needs_recompile(&library_path, &parser_path, &scanner_path)
@@ -210,7 +292,7 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
     let mut config = cc::Build::new();
     config
         .cpp(true)
-        .opt_level(2)
+        .opt_level(3)
         .cargo_metadata(false)
         .host(BUILD_TARGET)
         .target(BUILD_TARGET);
@@ -245,7 +327,7 @@ fn build_tree_sitter_library(src_path: &Path, grammar: GrammarConfiguration) ->
             .arg(header_path)
             .arg("-o")
             .arg(&library_path)
-            .arg("-O2");
+            .arg("-O3");
         if let Some(scanner_path) = scanner_path.as_ref() {
             if scanner_path.extension() == Some("c".as_ref()) {
                 command.arg("-xc").arg("-std=c99").arg(scanner_path);
@@ -294,3 +376,13 @@ fn needs_recompile(
 fn mtime(path: &Path) -> Result<SystemTime> {
     Ok(fs::metadata(path)?.modified()?)
 }
+
+/// Gives the contents of a file from a language's `runtime/queries/<lang>`
+/// directory
+pub fn load_runtime_file(language: &str, filename: &str) -> Result<String, std::io::Error> {
+    let path = crate::RUNTIME_DIR
+        .join("queries")
+        .join(language)
+        .join(filename);
+    std::fs::read_to_string(&path)
+}

helix-loader/src/lib.rs

@@ -0,0 +1,161 @@
+pub mod grammar;
+
+use etcetera::base_strategy::{choose_base_strategy, BaseStrategy};
+
+pub static RUNTIME_DIR: once_cell::sync::Lazy<std::path::PathBuf> =
+    once_cell::sync::Lazy::new(runtime_dir);
+
+pub fn runtime_dir() -> std::path::PathBuf {
+    if let Ok(dir) = std::env::var("HELIX_RUNTIME") {
+        return dir.into();
+    }
+
+    const RT_DIR: &str = "runtime";
+    let conf_dir = config_dir().join(RT_DIR);
+    if conf_dir.exists() {
+        return conf_dir;
+    }
+
+    if let Ok(dir) = std::env::var("CARGO_MANIFEST_DIR") {
+        // this is the directory of the crate being run by cargo, we need the workspace path so we take the parent
+        return std::path::PathBuf::from(dir).parent().unwrap().join(RT_DIR);
+    }
+
+    // fallback to location of the executable being run
+    std::env::current_exe()
+        .ok()
+        .and_then(|path| path.parent().map(|path| path.to_path_buf().join(RT_DIR)))
+        .unwrap()
+}
+
+pub fn config_dir() -> std::path::PathBuf {
+    // TODO: allow env var override
+    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
+    let mut path = strategy.config_dir();
+    path.push("helix");
+    path
+}
+
+pub fn cache_dir() -> std::path::PathBuf {
+    // TODO: allow env var override
+    let strategy = choose_base_strategy().expect("Unable to find the config directory!");
+    let mut path = strategy.cache_dir();
+    path.push("helix");
+    path
+}
+
+pub fn config_file() -> std::path::PathBuf {
+    config_dir().join("config.toml")
+}
+
+pub fn lang_config_file() -> std::path::PathBuf {
+    config_dir().join("languages.toml")
+}
+
+pub fn log_file() -> std::path::PathBuf {
+    cache_dir().join("helix.log")
+}
+
+/// Default bultin-in languages.toml.
+pub fn default_lang_config() -> toml::Value {
+    toml::from_slice(include_bytes!("../../languages.toml"))
+        .expect("Could not parse bultin-in languages.toml to valid toml")
+}
+
+/// User configured languages.toml file, merged with the default config.
+pub fn user_lang_config() -> Result<toml::Value, toml::de::Error> {
+    let def_lang_conf = default_lang_config();
+    let data = std::fs::read(crate::config_dir().join("languages.toml"));
+    let user_lang_conf = match data {
+        Ok(raw) => {
+            let value = toml::from_slice(&raw)?;
+            merge_toml_values(def_lang_conf, value)
+        }
+        Err(_) => def_lang_conf,
+    };
+
+    Ok(user_lang_conf)
+}
+
+// right overrides left
+pub fn merge_toml_values(left: toml::Value, right: toml::Value) -> toml::Value {
+    use toml::Value;
+
+    fn get_name(v: &Value) -> Option<&str> {
+        v.get("name").and_then(Value::as_str)
+    }
+
+    match (left, right) {
+        (Value::Array(mut left_items), Value::Array(right_items)) => {
+            left_items.reserve(right_items.len());
+            for rvalue in right_items {
+                let lvalue = get_name(&rvalue)
+                    .and_then(|rname| left_items.iter().position(|v| get_name(v) == Some(rname)))
+                    .map(|lpos| left_items.remove(lpos));
+                let mvalue = match lvalue {
+                    Some(lvalue) => merge_toml_values(lvalue, rvalue),
+                    None => rvalue,
+                };
+                left_items.push(mvalue);
+            }
+            Value::Array(left_items)
+        }
+        (Value::Table(mut left_map), Value::Table(right_map)) => {
+            for (rname, rvalue) in right_map {
+                match left_map.remove(&rname) {
+                    Some(lvalue) => {
+                        let merged_value = merge_toml_values(lvalue, rvalue);
+                        left_map.insert(rname, merged_value);
+                    }
+                    None => {
+                        left_map.insert(rname, rvalue);
+                    }
+                }
+            }
+            Value::Table(left_map)
+        }
+        // Catch everything else we didn't handle, and use the right value
+        (_, value) => value,
+    }
+}
+
+#[cfg(test)]
+mod merge_toml_tests {
+    use super::merge_toml_values;
+
+    #[test]
+    fn language_tomls() {
+        use toml::Value;
+
+        const USER: &str = "
+        [[language]]
+        name = \"nix\"
+        test = \"bbb\"
+        indent = { tab-width = 4, unit = \"    \", test = \"aaa\" }
+        ";
+
+        let base: Value = toml::from_slice(include_bytes!("../../languages.toml"))
+            .expect("Couldn't parse built-in languages config");
+        let user: Value = toml::from_str(USER).unwrap();
+
+        let merged = merge_toml_values(base, user);
+        let languages = merged.get("language").unwrap().as_array().unwrap();
+        let nix = languages
+            .iter()
+            .find(|v| v.get("name").unwrap().as_str().unwrap() == "nix")
+            .unwrap();
+        let nix_indent = nix.get("indent").unwrap();
+
+        // We changed tab-width and unit in indent so check them if they are the new values
+        assert_eq!(
+            nix_indent.get("tab-width").unwrap().as_integer().unwrap(),
+            4
+        );
+        assert_eq!(nix_indent.get("unit").unwrap().as_str().unwrap(), "    ");
+        // We added a new keys, so check them
+        assert_eq!(nix.get("test").unwrap().as_str().unwrap(), "bbb");
+        assert_eq!(nix_indent.get("test").unwrap().as_str().unwrap(), "aaa");
+        // We didn't change comment-token so it should be same
+        assert_eq!(nix.get("comment-token").unwrap().as_str().unwrap(), "#");
+    }
+}

helix-term/Cargo.toml

@@ -26,6 +26,7 @@ helix-core = { version = "0.6", path = "../helix-core" }
 helix-view = { version = "0.6", path = "../helix-view" }
 helix-lsp = { version = "0.6", path = "../helix-lsp" }
 helix-dap = { version = "0.6", path = "../helix-dap" }
+helix-loader = { version = "0.6", path = "../helix-loader" }
 
 anyhow = "1"
 once_cell = "1.10"

helix-term/build.rs

@@ -14,10 +14,5 @@ fn main() {
         None => env!("CARGO_PKG_VERSION").into(),
     };
 
-    println!(
-        "cargo:rustc-env=BUILD_TARGET={}",
-        std::env::var("TARGET").unwrap()
-    );
-
     println!("cargo:rustc-env=VERSION_AND_GIT_HASH={}", version);
 }

helix-term/src/application.rs

@@ -61,10 +61,10 @@ impl Application {
         let mut compositor = Compositor::new()?;
         let size = compositor.size();
 
-        let conf_dir = helix_core::config_dir();
+        let conf_dir = helix_loader::config_dir();
 
         let theme_loader =
-            std::sync::Arc::new(theme::Loader::new(&conf_dir, &helix_core::runtime_dir()));
+            std::sync::Arc::new(theme::Loader::new(&conf_dir, &helix_loader::runtime_dir()));
 
         let true_color = config.editor.true_color || crate::true_color();
         let theme = config
@@ -109,7 +109,7 @@ impl Application {
         compositor.push(editor_view);
 
         if args.load_tutor {
-            let path = helix_core::runtime_dir().join("tutor.txt");
+            let path = helix_loader::runtime_dir().join("tutor.txt");
             editor.open(path, Action::VerticalSplit)?;
             // Unset path to prevent accidentally saving to the original tutor file.
             doc_mut!(editor).set_path(None)?;

helix-term/src/commands/typed.rs

@@ -828,7 +828,7 @@ fn tutor(
     _args: &[Cow<str>],
     _event: PromptEvent,
 ) -> anyhow::Result<()> {
-    let path = helix_core::runtime_dir().join("tutor.txt");
+    let path = helix_loader::runtime_dir().join("tutor.txt");
     cx.editor.open(path, Action::Replace)?;
     // Unset path to prevent accidentally saving to the original tutor file.
     doc_mut!(cx.editor).set_path(None)?;

helix-term/src/health.rs

@@ -1,8 +1,6 @@
 use crossterm::style::{Color, Print, Stylize};
-use helix_core::{
+use helix_core::config::{default_syntax_loader, user_syntax_loader};
-    config::{default_syntax_loader, user_syntax_loader},
+use helix_loader::grammar::load_runtime_file;
-    syntax::load_runtime_file,
-};
 
 #[derive(Copy, Clone)]
 pub enum TsFeature {
@@ -43,10 +41,10 @@ impl TsFeature {
 
 /// Display general diagnostics.
 pub fn general() {
-    let config_file = helix_core::config_file();
+    let config_file = helix_loader::config_file();
-    let lang_file = helix_core::lang_config_file();
+    let lang_file = helix_loader::lang_config_file();
-    let log_file = helix_core::log_file();
+    let log_file = helix_loader::log_file();
-    let rt_dir = helix_core::runtime_dir();
+    let rt_dir = helix_loader::runtime_dir();
 
     if config_file.exists() {
         println!("Config file: {}", config_file.display());

helix-term/src/lib.rs

@@ -7,7 +7,6 @@ pub mod commands;
 pub mod compositor;
 pub mod config;
 pub mod health;
-pub mod grammars;
 pub mod job;
 pub mod keymap;
 pub mod ui;

helix-term/src/main.rs

@@ -40,7 +40,7 @@ fn main() -> Result<()> {
 
 #[tokio::main]
 async fn main_impl() -> Result<i32> {
-    let logpath = helix_core::log_file();
+    let logpath = helix_loader::log_file();
     let parent = logpath.parent().unwrap();
     if !parent.exists() {
         std::fs::create_dir_all(parent).ok();
@@ -105,21 +105,21 @@ FLAGS:
     }
 
     if args.fetch_grammars {
-        helix_term::grammars::fetch_grammars()?;
+        helix_loader::grammar::fetch_grammars()?;
         return Ok(0);
     }
 
     if args.build_grammars {
-        helix_term::grammars::build_grammars()?;
+        helix_loader::grammar::build_grammars()?;
         return Ok(0);
     }
 
-    let conf_dir = helix_core::config_dir();
+    let conf_dir = helix_loader::config_dir();
     if !conf_dir.exists() {
         std::fs::create_dir_all(&conf_dir).ok();
     }
 
-    let config = match std::fs::read_to_string(helix_core::config_file()) {
+    let config = match std::fs::read_to_string(helix_loader::config_file()) {
         Ok(config) => toml::from_str(&config)
             .map(merge_keys)
             .unwrap_or_else(|err| {

helix-term/src/ui/mod.rs

@@ -218,9 +218,9 @@ pub mod completers {
     }
 
     pub fn theme(_editor: &Editor, input: &str) -> Vec<Completion> {
-        let mut names = theme::Loader::read_names(&helix_core::runtime_dir().join("themes"));
+        let mut names = theme::Loader::read_names(&helix_loader::runtime_dir().join("themes"));
         names.extend(theme::Loader::read_names(
-            &helix_core::config_dir().join("themes"),
+            &helix_loader::config_dir().join("themes"),
         ));
         names.push("default".into());
         names.push("base16_default".into());

languages.toml

@@ -302,8 +302,7 @@ indent = { tab-width = 2, unit = "  " }
 
 [[grammar]]
 name = "typescript"
-source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a" }
+source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a", subpath = "typescript" }
-path = "typescript"
 
 [[language]]
 name = "tsx"
@@ -317,8 +316,7 @@ indent = { tab-width = 2, unit = "  " }
 
 [[grammar]]
 name = "tsx"
-source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a" }
+source = { git = "https://github.com/tree-sitter/tree-sitter-typescript", rev = "3e897ea5925f037cfae2e551f8e6b12eec2a201a", subpath = "tsx" }
-path = "tsx"
 
 [[language]]
 name = "css"
@@ -522,8 +520,7 @@ indent = { tab-width = 2, unit = "  " }
 
 [[grammar]]
 name = "ocaml"
-source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a" }
+source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a", subpath = "ocaml" }
-path = "ocaml"
 
 [[language]]
 name = "ocaml-interface"
@@ -536,8 +533,7 @@ indent = { tab-width = 2, unit = "  "}
 
 [[grammar]]
 name = "ocaml-interface"
-source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a" }
+source = { git = "https://github.com/tree-sitter/tree-sitter-ocaml", rev = "23d419ba45789c5a47d31448061557716b02750a", subpath = "interface" }
-path = "interface"
 
 [[language]]
 name = "lua"