WIP: Add embedded rust

Cargo.lock

@@ -2,6 +2,12 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "multicall"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "428f6ba17d0c927e57c15a86cf5d7d07a2f35b3fbf15b1eb36b7075459e150a3"
+
 [[package]]
 name = "once_cell"
 version = "1.17.1"
@@ -16,8 +22,9 @@ checksum = "b03f7fbd470aa8b3ad163c85cce8bccfc11cc9c44ef12da0a4eddd98bd307352"
 
 [[package]]
 name = "spl"
-version = "0.0.4"
+version = "0.1.0"
 dependencies = [
+ "multicall",
  "once_cell",
  "readformat",
 ]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spl"
-version = "0.0.4"
+version = "0.1.0"
 edition = "2021"
 description = "Stack Pogramming Language: A simple, concise scripting language."
 license = "MIT"
@@ -10,3 +10,4 @@ authors = ["TudbuT"]
 [dependencies]
 readformat = "0.1"
 once_cell = "1.17"
+multicall = "0.1.4"

net.spl

@@ -1,8 +1,9 @@
 
+"the net namespace allows any other constructs and namespaces in it. They can be added";
+"using \"Name\" net:register after which net:Name is available to become a construct";
 construct net namespace {
     ;
     register { | with name this ;
-        name "net" dyn-def-field;
-        this "net" settype =net
+        name "net" this register-field
     }
 }

rust-test.spl

@@ -0,0 +1,10 @@
+func main { |
+	1 rusty-test _str println
+	0
+}
+func rusty-test @rust !{
+	println!("hii");
+	let v = #pop:Mega#;
+	#push(v + 1)#;
+}
+

src/lexer.rs

@@ -1,4 +1,4 @@
-use std::sync::Arc;
+use std::{mem, sync::Arc};
 
 use crate::runtime::*;
 use readformat::*;
@@ -14,10 +14,7 @@ pub enum LexerError {
 }
 
 pub fn lex(input: String) -> Result<Words, LexerError> {
-    let mut str_words = Vec::new();
-    for line in input.split('\n') {
-        str_words.append(&mut parse_line(line));
-    }
+    let str_words = parse(input);
     Ok(read_block(&str_words[..], false)?.1)
 }
 
@@ -52,6 +49,17 @@ fn read_block(str_words: &[String], isfn: bool) -> Result<(Option<u32>, Words, u
                         block.0.ok_or(LexerError::FunctionBlockExpected)?,
                         block.1,
                     )));
+                } else if let Some(dat) =
+                    readf1("func\0{}\0@rust", str_words[i..=i + 2].join("\0").as_str())
+                {
+                    i += 3;
+                    words.push(Word::Key(Keyword::FuncOf(
+                        dat.to_owned(),
+                        str_words[i][2..].to_owned(),
+                        FuncImplType::Rust,
+                    )));
+                } else {
+                    return Err(LexerError::FunctionBlockExpected);
                 }
             }
             "{" => {
@@ -66,6 +74,9 @@ fn read_block(str_words: &[String], isfn: bool) -> Result<(Option<u32>, Words, u
                     run_as_base: false,
                 }))))
             }
+            x if x.len() >= 2 && &x[0..2] == "!{" => {
+                words.push(Word::Const(Value::Str(x[2..].to_owned())));
+            }
             "<{" => {
                 let block = read_block(&str_words[i + 1..], false)?;
                 i += block.2 + 1;
@@ -241,67 +252,104 @@ fn read_block(str_words: &[String], isfn: bool) -> Result<(Option<u32>, Words, u
     Ok((rem, Words { words }, i))
 }
 
-fn parse_line(line: &str) -> Vec<String> {
+fn parse(input: String) -> Vec<String> {
     let mut words = Vec::new();
-    let mut in_string = false;
-    let mut escaping = false;
-    let mut was_in_string = false;
     let mut s = String::new();
-    for c in line.chars() {
-        if in_string {
-            if escaping {
+
+    let mut exclam = false;
+    let mut raw = 0;
+
+    for line in input.split('\n') {
+        let mut in_string = false;
+        let mut escaping = false;
+        let mut was_in_string = false;
+        for c in line.chars() {
+            if in_string {
+                if escaping {
+                    if raw == 0 {
+                        if c == '\\' {
+                            s += "\\";
+                        }
+                        if c == 'n' {
+                            s += "\n";
+                        }
+                        if c == 'r' {
+                            s += "\r";
+                        }
+                        if c == '"' {
+                            s += "\"";
+                        }
+                        escaping = false;
+                        continue;
+                    } else {
+                        escaping = false;
+                    }
+                } else if c == '"' {
+                    in_string = false;
+                    escaping = false;
+                    was_in_string = true;
+                    if raw == 0 {
+                        continue;
+                    }
+                }
                 if c == '\\' {
-                    s += "\\";
-                }
-                if c == 'n' {
-                    s += "\n";
-                }
-                if c == 'r' {
-                    s += "\r";
+                    escaping = true;
+                    if raw == 0 {
+                        continue;
+                    }
                 }
+            } else {
                 if c == '"' {
                     s += "\"";
-                }
-                escaping = false;
-                continue;
-            } else if c == '"' {
-                in_string = false;
-                escaping = false;
-                was_in_string = true;
-                continue;
-            }
-            if c == '\\' {
-                escaping = true;
-                continue;
-            }
-        } else {
-            if c == '"' {
-                s += "\"";
-                in_string = true;
-                continue;
-            }
-            if c == ';' && was_in_string {
-                s = String::new();
-                continue;
-            }
-            if c == '(' || c == ')' {
-                continue;
-            }
-            if c == ' ' || c == '\t' {
-                if s.is_empty() {
+                    in_string = true;
                     continue;
                 }
-                words.push(s);
-                s = String::new();
-                was_in_string = false;
-                continue;
+                if raw == 0 {
+                    if c == ';' && was_in_string {
+                        s = String::new();
+                        continue;
+                    }
+                    if c == '(' || c == ')' {
+                        continue;
+                    }
+                    if c == ' ' || c == '\t' {
+                        if s.is_empty() {
+                            continue;
+                        }
+                        words.push(s);
+                        s = String::new();
+                        was_in_string = false;
+                        continue;
+                    }
+                    if c == '{' && exclam {
+                        raw = 1;
+                    }
+                    exclam = false;
+                    if c == '!' {
+                        exclam = true;
+                    }
+                } else {
+                    if c == '{' {
+                        raw += 1;
+                    }
+                    if c == '}' {
+                        raw -= 1;
+                    }
+                    if raw == 0 {
+                        words.push(mem::take(&mut s));
+                        continue;
+                    }
+                }
             }
+            was_in_string = false;
+            s += String::from(c).as_str();
+        }
+        if !s.is_empty() && raw == 0 {
+            words.push(mem::take(&mut s));
         }
-        was_in_string = false;
-        s += String::from(c).as_str();
     }
     if !s.is_empty() {
-        words.push(s);
+        words.push(mem::take(&mut s));
     }
     words
 }

src/lib.rs

@@ -29,6 +29,7 @@
 pub mod dyn_fns;
 pub mod lexer;
 pub mod mutex;
+pub mod oxidizer;
 pub mod runtime;
 pub mod sasm;
 pub mod std_fns;

src/main.rs

@@ -1,13 +1,26 @@
-use spl::{find_in_splpath, start_file};
+use spl::{find_in_splpath, lex, oxidizer::RustAppBuilder, start_file};
 
-use std::env::args;
+use std::{env::args, fs};
 
 fn main() {
-    if let Err(x) = start_file(
-        &args()
-            .nth(1)
-            .unwrap_or_else(|| find_in_splpath("repl.spl").expect("no file to be run")),
-    ) {
+    let mut args = args().skip(1);
+    let arg = &args
+        .next()
+        .unwrap_or_else(|| find_in_splpath("repl.spl").expect("no file to be run"));
+    if arg == "--build" {
+        let file = args.next().unwrap();
+        let data = fs::read_to_string(file.clone()).expect("unable to read specified file");
+        println!("Building SPL with specified natives file...");
+        let mut builder = RustAppBuilder::new();
+        println!("Embedding source...");
+        builder.add_source(file, data.to_owned());
+        println!("Preparing rust code...");
+        builder.prepare(lex(data.to_owned()).expect("invalid SPL in natives file."));
+        println!("Building...");
+        println!("Built! Binary is {}", builder.build().unwrap().get_binary());
+        return;
+    }
+    if let Err(x) = start_file(arg) {
         println!("{x:?}");
     }
 }

src/oxidizer/mod.rs

@@ -0,0 +1,183 @@
+//! This module creates a rust application that runs the desired SPL.
+//! At its current stage, this is just parsing and rewriting `@rust` functions from SPL into actual rust.
+//! The future plan is for this to effectively become a compiler.
+
+use std::{
+    collections::{hash_map::DefaultHasher, HashMap},
+    env, fs,
+    hash::{Hash, Hasher},
+    io,
+    process::{Child, Command},
+};
+
+use crate::{FuncImplType, Keyword, Word, Words};
+
+mod splrs;
+
+/// A specially compiled SPL version with custom parameters included.
+pub struct RustApp {
+    /// The path to the binary
+    binary: String,
+}
+
+impl RustApp {
+    /// Gets the path to the binary
+    pub fn get_binary(&self) -> &str {
+        &self.binary
+    }
+
+    /// Executes the binary with some args
+    pub fn execute(&self, args: Vec<&str>) -> Result<Child, io::Error> {
+        Command::new(self.binary.clone()).args(args).spawn()
+    }
+}
+
+/// A rust function which was embedded in SPL
+pub struct RustFunction {
+    fn_name: String,
+    content: String,
+}
+
+/// A builder for [`RustApp`]s. This is work-in-progress.
+pub struct RustAppBuilder {
+    rust_functions: Vec<RustFunction>,
+    to_embed: HashMap<String, String>,
+    default_file: String,
+    name: Option<String>,
+}
+
+impl Hash for RustAppBuilder {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        state.write_usize(self.rust_functions.len());
+        for f in &self.rust_functions {
+            f.fn_name.hash(state);
+        }
+        for (k, _) in &self.to_embed {
+            k.hash(state);
+        }
+    }
+}
+
+impl RustAppBuilder {
+    pub fn new() -> RustAppBuilder {
+        Self {
+            default_file: "repl.spl".to_owned(),
+            ..Default::default()
+        }
+    }
+
+    /// Embeds a file into the desired app
+    pub fn add_source(&mut self, name: String, source: String) {
+        self.to_embed.insert(name, source);
+    }
+
+    /// Sets the name of the folder it will sit in.
+    pub fn set_name(&mut self, name: String) {
+        self.name = Some(name);
+    }
+
+    /// Adds all `@rust` functions from the given SPL code's top level. Does NOT scan for lower levels at this time.
+    pub fn prepare(&mut self, spl: Words) -> bool {
+        let mut needs_new = false;
+        for word in spl.words {
+            match word {
+                Word::Key(Keyword::FuncOf(name, content, FuncImplType::Rust)) => {
+                    self.rust_functions.push(splrs::to_rust(name, content));
+                    needs_new = true;
+                }
+                _ => (),
+            }
+        }
+        needs_new
+    }
+
+    /// Sets the default file to start when none is provided. This will not work if the file is not also embedded.
+    pub fn set_default_file(&mut self, name: String) {
+        self.default_file = name;
+    }
+
+    /// Builds the desired app, including literally building it using cargo.
+    pub fn build(self) -> Result<RustApp, io::Error> {
+        // we need a temp folder!
+        let tmp = "."; // TODO replace?
+        let name = match self.name {
+            Some(x) => x,
+            None => {
+                let mut hash = DefaultHasher::new();
+                self.hash(&mut hash);
+                let hash = hash.finish();
+                hash.to_string()
+            }
+        };
+        let _ = Command::new("cargo")
+            .arg("new")
+            .arg(format!("spl-{name}"))
+            .current_dir(tmp)
+            .spawn()
+            .unwrap()
+            .wait_with_output();
+        Command::new("cargo")
+            .arg("add")
+            .arg(format!("spl@{}", env!("CARGO_PKG_VERSION")))
+            .current_dir(format!("{tmp}/spl-{name}"))
+            .spawn()
+            .unwrap()
+            .wait_with_output()?;
+        let mut runtime_init = String::new();
+        let mut code = String::new();
+        for func in self.rust_functions.into_iter().enumerate() {
+            code += &format!(
+                "fn spl_oxidizer_{}(stack: &mut Stack) -> OError {{ {} Ok(()) }}",
+                func.0, func.1.content
+            );
+            runtime_init += &format!(
+                "rt.native_functions.insert({:?}, (0, FuncImpl::Native(spl_oxidizer_{})));",
+                func.1.fn_name, func.0
+            )
+        }
+        for (name, data) in self.to_embed.into_iter() {
+            runtime_init += &format!("rt.embedded_files.insert({:?}, {:?});", name, data);
+        }
+        fs::write(
+            format!("{tmp}/spl-{name}/src/main.rs"),
+            stringify! {
+                use spl::{runtime::*, *};
+
+                use std::env::args;
+
+                pub fn start_file(path: &str) -> Result<Stack, Error> {
+                    let mut rt = Runtime::new();
+                    runtime_init
+                    rt.set();
+                    (start_file_in_runtime(path), Runtime::reset()).0
+                }
+
+                fn main() {
+                    if let Err(x) = start_file(
+                        &args()
+                            .nth(1)
+                            .unwrap_or_else(|| find_in_splpath(default_file).expect("no file to be run")),
+                    ) {
+                        println!("{x:?}");
+                    }
+                }
+            }.to_owned().replace("default_file", &self.default_file).replace("runtime_init", &runtime_init) + &code,
+        )?;
+        Command::new("cargo")
+            .arg("build")
+            .arg("--release")
+            .current_dir(format!("{tmp}/spl-{name}"))
+            .spawn()
+            .unwrap()
+            .wait_with_output()?;
+        Ok(RustApp {
+            binary: format!("{tmp}/spl-{name}/target/release/spl-{name}"),
+        })
+    }
+}
+
+impl Default for RustAppBuilder {
+    fn default() -> Self {
+        Self::new()
+    }
+}

src/oxidizer/splrs.rs

@@ -0,0 +1,83 @@
+use readformat::readf1;
+
+use super::RustFunction;
+
+/// Parses a #-expression and returns the string to be inserted in its place.
+fn parse_hash_expr(s: String, name: &str) -> String {
+    if &s == "pop" {
+        return "stack.pop().lock_ro()".to_owned();
+    }
+    if &s == "pop_mut" {
+        return "stack.pop().lock()".to_owned();
+    }
+    if &s == "pop:Array" {
+        return format!("{{ require_array_on_stack!(tmp, stack, {name:?}); tmp }}");
+    }
+    if &s == "pop_mut:Array" {
+        return format!("{{ require_mut_array_on_stack!(tmp, stack, {name:?}); tmp }}");
+    }
+    if let Some(s) = readf1("pop:{}", &s) {
+        return format!("{{ require_on_stack!(tmp, {s}, stack, {name:?}); tmp }}");
+    }
+    if let Some(s) = readf1("push({})", &s) {
+        return format!("stack.push(({s}).spl())");
+    }
+    panic!("invalid #-expr - this error will be handled in the future")
+}
+
+pub fn to_rust(name: String, mut splrs: String) -> RustFunction {
+    RustFunction {
+        content: {
+            loop {
+                let mut did_anything = false;
+
+                let mut rs = String::new();
+                let mut in_str = false;
+                let mut escaping = false;
+                let mut hash_expr = None;
+                let mut brace = 0;
+                for c in splrs.chars() {
+                    dbg!(c, &rs, in_str, escaping, &hash_expr, brace);
+                    if in_str {
+                        if escaping {
+                            escaping = false;
+                        } else if c == '"' {
+                            in_str = false;
+                        }
+                        if c == '\\' {
+                            escaping = true;
+                        }
+                    } else if c == '"' {
+                        in_str = true;
+                    }
+                    if !in_str && c == '#' && hash_expr.is_none() {
+                        hash_expr = Some(String::new());
+                        did_anything = true;
+                        continue;
+                    }
+                    if let Some(ref mut expr) = hash_expr {
+                        if c == '#' && brace == 0 {
+                            rs += &parse_hash_expr(expr.to_owned(), &name);
+                            hash_expr = None;
+                            continue;
+                        }
+                        expr.push(c);
+                        if c == '(' {
+                            brace += 1;
+                        }
+                        if c == ')' {
+                            brace -= 1;
+                        }
+                        continue;
+                    }
+                    rs += String::from(c).as_str();
+                }
+                if !did_anything {
+                    break rs;
+                }
+                splrs = rs;
+            }
+        },
+        fn_name: name,
+    }
+}

src/runtime.rs

@@ -63,6 +63,8 @@ pub struct Runtime {
     types_by_id: HashMap<u32, AMType>,
     next_stream_id: u128,
     streams: HashMap<u128, Arc<Mut<Stream>>>,
+    pub embedded_files: HashMap<&'static str, &'static str>,
+    pub native_functions: HashMap<&'static str, (u32, FuncImpl)>,
 }
 
 impl Debug for Runtime {
@@ -90,6 +92,8 @@ impl Runtime {
             types_by_id: HashMap::new(),
             next_stream_id: 0,
             streams: HashMap::new(),
+            embedded_files: HashMap::new(),
+            native_functions: HashMap::new(),
         };
         let _ = rt.make_type("null".to_owned(), Ok); // infallible
         let _ = rt.make_type("int".to_owned(), Ok); // infallible
@@ -100,6 +104,7 @@ impl Runtime {
         let _ = rt.make_type("func".to_owned(), Ok); // infallible
         let _ = rt.make_type("array".to_owned(), Ok); // infallible
         let _ = rt.make_type("str".to_owned(), Ok); // infallible
+        stdlib::register(&mut rt);
         rt
     }
 
@@ -146,6 +151,14 @@ impl Runtime {
         self.streams.remove(&id);
     }
 
+    pub fn load_native_function(&self, name: &str) -> &(u32, FuncImpl) {
+        self.native_functions.get(name).unwrap_or_else(|| {
+            panic!(
+                "It seems the native function {name} was not compiled into this program. Stopping."
+            )
+        })
+    }
+
     pub fn reset() {
         RUNTIME.with(|x| *x.borrow_mut() = None);
     }
@@ -605,6 +618,11 @@ impl Stack {
     }
 }
 
+#[derive(Clone, Debug)]
+pub enum FuncImplType {
+    Rust,
+}
+
 /// An SPL keyword. Used to deviate from normal linear code structure.
 ///
 /// This is different from a [Word], which are any SPL code.
@@ -680,6 +698,11 @@ pub enum Keyword {
     ///
     /// see [Keyword::ObjPush]
     ObjPop,
+    /// func <name> @<type> !{ <content> }
+    ///
+    /// Defines function <name> with <type> impl type
+    /// equivalent to "<content>" "<name>" "<type>" dyn-func-of
+    FuncOf(String, String, FuncImplType),
 }
 
 /// Any SPL value that is not a construct.
@@ -1067,6 +1090,22 @@ where
     }
 }
 
+macro_rules! impl_to_object {
+    ($kind:ident, $type:ty) => {
+        impl From<$type> for Object {
+            fn from(value: $type) -> Object {
+                Value::$kind(value).into()
+            }
+        }
+    };
+}
+
+impl_to_object!(Int, i32);
+impl_to_object!(Long, i64);
+impl_to_object!(Mega, i128);
+impl_to_object!(Float, f32);
+impl_to_object!(Double, f64);
+
 /// Finds a file in the SPL_PATH, or returns the internal [stdlib] version of it.
 pub fn find_in_splpath(path: &str) -> Result<String, String> {
     if Path::new(path).exists() {
@@ -1076,15 +1115,14 @@ pub fn find_in_splpath(path: &str) -> Result<String, String> {
     if Path::new(&s).exists() {
         Ok(s)
     } else {
-        match path {
-            "std.spl" => Err(stdlib::STD.to_owned()),
-            "net.spl" => Err(stdlib::NET.to_owned()),
-            "iter.spl" => Err(stdlib::ITER.to_owned()),
-            "http.spl" => Err(stdlib::HTTP.to_owned()),
-            "stream.spl" => Err(stdlib::STREAM.to_owned()),
-            "messaging.spl" => Err(stdlib::MESSAGING.to_owned()),
-            _ => Ok(path.to_owned()),
-        }
+        runtime(|x| {
+            for (&p, &data) in &x.embedded_files {
+                if path == p {
+                    return Err(data.to_owned());
+                }
+            }
+            Ok(path.to_owned()) // fails later
+        })
     }
 }
 
@@ -1231,6 +1269,20 @@ impl Words {
                             .expect("invalid word generation. objpop without objpush!");
                         stack.push(o);
                     }
+                    Keyword::FuncOf(name, _, _) => runtime(|x| {
+                        let f = x.load_native_function(&name);
+                        stack.define_func(
+                            name.to_owned(),
+                            Arc::new(Func {
+                                ret_count: f.0,
+                                to_call: f.1.clone(),
+                                origin: stack.get_frame(),
+                                run_as_base: false,
+                                fname: None,
+                                name,
+                            }),
+                        )
+                    }),
                 },
                 Word::Const(x) => {
                     if option_env!("SPLDEBUG").is_some() {

src/std_fns.rs

@@ -602,20 +602,19 @@ pub fn import(stack: &mut Stack) -> OError {
     let Value::Str(mut s) = stack.pop().lock_ro().native.clone() else {
         return stack.err(ErrorKind::InvalidCall("import".to_owned()))
     };
-    let fallback = match s
+    let fallback = s
         .as_str()
         .rsplit_once(|x| x == '/' || x == '#')
         .map(|(.., x)| x)
-        .unwrap_or(s.as_str())
-    {
-        "std.spl" => Some(stdlib::STD),
-        "net.spl" => Some(stdlib::NET),
-        "iter.spl" => Some(stdlib::ITER),
-        "http.spl" => Some(stdlib::HTTP),
-        "stream.spl" => Some(stdlib::STREAM),
-        "messaging.spl" => Some(stdlib::MESSAGING),
-        _ => None,
-    };
+        .unwrap_or(s.as_str());
+    let fallback = runtime(|x| {
+        for (&p, &data) in &x.embedded_files {
+            if fallback == p {
+                return Some(data.to_owned());
+            }
+        }
+        None
+    });
     if let Some(x) = s.strip_prefix('#') {
         s = find_in_splpath(x).unwrap_or(x.to_owned());
     } else if let Some(x) = s.strip_prefix('@') {

src/stdlib.rs

@@ -1,6 +1,21 @@
+use crate::Runtime;
+use multicall::multicall;
+
 pub const STD: &str = include_str!("../std.spl");
 pub const NET: &str = include_str!("../net.spl");
 pub const ITER: &str = include_str!("../iter.spl");
 pub const HTTP: &str = include_str!("../http.spl");
 pub const STREAM: &str = include_str!("../stream.spl");
 pub const MESSAGING: &str = include_str!("../messaging.spl");
+
+pub fn register(runtime: &mut Runtime) {
+    multicall! {
+        &mut runtime.embedded_files:
+        insert("std.spl", STD);
+        insert("net.spl", NET);
+        insert("iter.spl", ITER);
+        insert("http.spl", HTTP);
+        insert("stream.spl", STREAM);
+        insert("messaging.spl", MESSAGING);
+    }
+}

std.spl

@@ -426,6 +426,14 @@ func -- { mega |
     1 -
 }
 
+func times { | with amount callable ;
+    def i 0 =i
+    while { i amount lt } {
+        i callable call
+        i ++ =i
+    }
+}
+
 def _'has-been-called 0 =_'has-been-called
 func _ { | 
     _'has-been-called not if {
@@ -462,3 +470,9 @@ func update-types { |
 }
 update-types
 
+"Adds a field to a namespace and initially sets it to the field's name.";
+func register-field { | with field-name namespace-name namespace ;
+    field-name namespace-name dyn-def-field;
+    namespace namespace-name settype ("=" namespace-name concat) dyn-call
+}
+