diff --git a/dagrs/.gitignore b/dagrs/.gitignore index ae9ea50078ef99e806d95bb3ac4275a15df0c52f..a20fd1a2edab16a5a8bbd7d906bb3ed554f87538 100644 --- a/dagrs/.gitignore +++ b/dagrs/.gitignore @@ -16,6 +16,7 @@ Cargo.lock /target #**/*.rs.bk +example.log # IDE .idea diff --git a/dagrs/Cargo.toml b/dagrs/Cargo.toml index c6c3c9f1d9f920f8841907ea683042c566e14918..1045810497c730d21dacd27bae0a69c882323ec2 100644 --- a/dagrs/Cargo.toml +++ b/dagrs/Cargo.toml @@ -9,4 +9,7 @@ edition = "2021" yaml-rust = "0.4.5" lazy_static = "1.4.0" bimap = "0.6.1" +deno_core = "0.121.0" +log = "0.4.14" +simplelog = "^0.10.0" clap = { version = "3.0.14", features = ["derive"] } \ No newline at end of file diff --git a/dagrs/src/dag_engine.rs b/dagrs/src/dag_engine.rs index efac4e9563b2ee27d14e4633a8cadd242d3408a4..d7e5fec34a33353e3b5f563f327421469d20ec80 100644 --- a/dagrs/src/dag_engine.rs +++ b/dagrs/src/dag_engine.rs @@ -140,13 +140,15 @@ impl DagEngine { self.print_seq(&seq); seq.iter() .map(|id| { - println!("Executing Task[name: {}]", self.tasks[id].get_name()); - self.tasks[id].run(); + info!("Executing Task[name: {}]", self.tasks[id].get_name()); + if let Some(v) = self.tasks[id].run() { + info!("Task[name: {}] exec done, success: {}, return value: {}", self.tasks[id].get_name(), v.success, v.value); + } }) .count(); true } else { - println!("Loop Detect"); + error!("Loop Detect"); false } } @@ -161,10 +163,10 @@ impl DagEngine { /// } /// ``` fn print_seq(&self, seq: &Vec) { - print!("[Start]"); + let mut res = String::from("[Start]"); seq.iter() - .map(|id| print!(" -> {}", self.tasks[id].get_name())) + .map(|id| res.push_str(&format!(" -> {}", self.tasks[id].get_name()))) .count(); - println!(" -> [End]"); + info!("{} -> [End]", res); } } diff --git a/dagrs/src/error_handler.rs b/dagrs/src/error_handler.rs index 051f02d679d479b0407e20722d0c85051a0a903b..ef8456d37089b0c55c75e2a2b78c01ba573f1b67 100644 --- a/dagrs/src/error_handler.rs +++ b/dagrs/src/error_handler.rs @@ -21,8 +21,8 @@ pub enum FormatError { StartWordError, /// A task have no name filed, `String` points out task's id. NoName(String), - /// No run scripts - NoRunScript(String), + /// Run field format error + RunScriptError(String) } #[derive(Debug, PartialEq, Eq)] @@ -46,7 +46,7 @@ impl Display for FormatError { match self { Self::StartWordError => write!(f, "YAML file not start with 'dagrs:'"), Self::NoName(id) => write!(f, "Task[ID:{}] name not found", id), - Self::NoRunScript(id) => write!(f, "Task[ID:{}] run script not found", id), + Self::RunScriptError(id) => write!(f, "Task[ID:{}] run script format error", id), } } } diff --git a/dagrs/src/main.rs b/dagrs/src/main.rs index 02d07268fb011c818e0271b7b65fd85e80042c97..4f1d30e1ea5d1180d9fea07843ffa0ecc4514d1c 100644 --- a/dagrs/src/main.rs +++ b/dagrs/src/main.rs @@ -1,15 +1,24 @@ +extern crate bimap; extern crate clap; +extern crate deno_core; extern crate lazy_static; extern crate yaml_rust; -extern crate bimap; +#[macro_use] +extern crate log; +extern crate simplelog; mod dag_engine; mod error_handler; mod graph; mod task; +use std::fs::File; + use clap::Parser; use dag_engine::DagEngine; +use simplelog::*; + +use crate::task::Retval; #[derive(Parser)] #[clap(version)] @@ -18,34 +27,73 @@ struct Args { /// YAML file path #[clap(short, long)] filepath: String, + /// Log file path + #[clap(short, long)] + logpath: Option, } - fn main() { let args = Args::parse(); let mut dagrs: DagEngine = DagEngine::new(); + let log_path = if let Some(s) = args.logpath { + s + } else { + "./example.log".to_owned() + }; + + init_logger(&log_path); if let Err(e) = dagrs.run_from_yaml(&args.filepath) { - println!("[Error] {}", e); + error!("[Error] {}", e); } } +/// Get a new logger, which will store logs into given path. +fn init_logger(path: &str) { + CombinedLogger::init(vec![ + TermLogger::new( + LevelFilter::Info, + Config::default(), + TerminalMode::Mixed, + ColorChoice::Auto, + ), + WriteLogger::new( + LevelFilter::Info, + Config::default(), + File::create(path).unwrap(), + ), + ]) + .unwrap(); +} + +#[test] +fn test_runscript() { + let res = DagEngine::new().run_from_yaml("test/test_dag1.yaml").unwrap(); + assert_eq!(res, true) +} + #[test] fn test_dag() { - let res = DagEngine::new().run_from_yaml("test/test_dag.yaml").unwrap(); + let res = DagEngine::new() + .run_from_yaml("test/test_dag2.yaml") + .unwrap(); assert_eq!(res, true) } #[test] fn test_loop() { - let res = DagEngine::new().run_from_yaml("test/test_loop1.yaml").unwrap(); + let res = DagEngine::new() + .run_from_yaml("test/test_loop1.yaml") + .unwrap(); assert_eq!(res, false) } #[test] fn test_complex_loop() { - let res = DagEngine::new().run_from_yaml("test/test_loop2.yaml").unwrap(); + let res = DagEngine::new() + .run_from_yaml("test/test_loop2.yaml") + .unwrap(); assert_eq!(res, false) } @@ -69,14 +117,15 @@ fn test_format_error2() { ); } - #[test] fn test_rely_error() { use error_handler::{DagError, InnerError}; let res = DagEngine::new().run_from_yaml("test/test_error3.yaml"); assert_eq!( res, - Err(DagError::inner_error(InnerError::RelyTaskIllegal("任务1".into()))) + Err(DagError::inner_error(InnerError::RelyTaskIllegal( + "任务1".into() + ))) ); } @@ -86,40 +135,85 @@ fn test_no_runscript() { let res = DagEngine::new().run_from_yaml("test/test_error4.yaml"); assert_eq!( res, - Err(DagError::format_error(FormatError::NoRunScript("a".into()))) + Err(DagError::format_error(FormatError::RunScriptError( + "a".into() + ))) ); } #[test] -fn test_prom() { +fn test_prom1() { use crate::task::{TaskTrait, TaskWrapper}; - struct T1 {} impl TaskTrait for T1 { - fn run(&self) { + fn run(&self) -> Option { println!("T1!"); + None } } struct T2 {} impl TaskTrait for T2 { - fn run(&self) { + fn run(&self) -> Option { println!("T2!"); + None } } - let mut t1 = TaskWrapper::new(T1{}, "Task 1"); - let mut t2 = TaskWrapper::new(T2{}, "Task 2"); - let mut t3 = TaskWrapper::new(T1{}, "Task 3"); + let mut t1 = TaskWrapper::new(T1 {}, "Task 1"); + let mut t2 = TaskWrapper::new(T2 {}, "Task 2"); + let t3 = TaskWrapper::new(T1 {}, "Task 3"); t2.add_relys(&[&t1, &t3]); t1.add_relys(&[&t3]); + let mut dag = DagEngine::new(); + dag.add_task(t1); + dag.add_task(t2); + dag.add_task(t3); + + dag.run().unwrap(); +} + +#[test] +fn test_prom2() { + use crate::task::{RunScript, RunType, TaskTrait, TaskWrapper}; + struct T { + run_script: RunScript, + } + + impl TaskTrait for T { + fn run(&self) -> Option { + Some(self.run_script.exec()) + } + } + + let mut t1 = TaskWrapper::new( + T { + run_script: RunScript::new("echo T1", RunType::SH), + }, + "Task 1", + ); + let mut t2 = TaskWrapper::new( + T { + run_script: RunScript::new("echo T2", RunType::SH), + }, + "Task 2", + ); + let t3 = TaskWrapper::new( + T { + run_script: RunScript::new(r#"Deno.core.print("T3\n")"#, RunType::DENO), + }, + "Task 3", + ); + + t2.add_relys(&[&t1, &t3]); + t1.add_relys(&[&t3]); let mut dag = DagEngine::new(); dag.add_task(t1); dag.add_task(t2); dag.add_task(t3); - dag.run(); -} \ No newline at end of file + dag.run().unwrap(); +} diff --git a/dagrs/src/task.rs b/dagrs/src/task.rs index 0b6955c3d5d4aa8cf61db7db0554690129cb80b7..0ae361d651fc8e70a7ee563353be906cd60d121a 100644 --- a/dagrs/src/task.rs +++ b/dagrs/src/task.rs @@ -1,17 +1,84 @@ //! Task Implementations, used to store task infos use crate::error_handler::{DagError, FormatError, InnerError}; +use deno_core::{JsRuntime, RuntimeOptions}; use lazy_static::lazy_static; use std::process::Command; use std::sync::Mutex; use std::{collections::HashMap, fs::File, io::Read}; use yaml_rust::{Yaml, YamlLoader}; +#[derive(Debug)] +pub struct RunScript { + script: String, + executor: RunType, +} + +#[derive(Debug)] +pub enum RunType { + SH, + DENO, +} + +pub struct Retval { + pub success: bool, + pub value: String, +} + +impl RunScript { + pub fn new(script: &str, executor: RunType) -> Self { + Self { + script: script.to_owned(), + executor, + } + } + + pub fn exec(&self) -> Retval { + match self.executor { + RunType::SH => self.run_sh(), + RunType::DENO => self.run_deno(), + } + } + + fn run_sh(&self) -> Retval { + let output = Command::new("sh") + .arg("-c") + .arg(&self.script) + .output() + .unwrap(); + // Reprint result + print!("{}", String::from_utf8(output.stdout).unwrap()); + + Retval { + success: output.status.success(), + value: "".into(), + } + } + + fn run_deno(&self) -> Retval { + let output = JsRuntime::new(RuntimeOptions { + ..Default::default() + }) + .execute_script("", &self.script); + + match output { + Ok(val) => Retval { + success: true, + value: format!("{:?}", val), + }, + Err(e) => Retval { + success: false, + value: format!("{}", e), + }, + } + } +} + /// Task Trait. /// /// Any struct implements this trait can be added into dagrs. pub trait TaskTrait { - fn run(&self); + fn run(&self) -> Option; } /// Wrapper for task that impl [`TaskTrait`]. @@ -83,7 +150,7 @@ impl TaskWrapper { self.name.to_owned() } - pub fn run(&self) { + pub fn run(&self) -> Option { self.inner.run() } } @@ -110,7 +177,7 @@ lazy_static! { /// Task Struct for YAML file. struct YamlTaskInner { /// Running Script - run: String, + run: RunScript, } /// Task struct for YAML file. @@ -129,14 +196,8 @@ pub struct YamlTask { } impl TaskTrait for YamlTaskInner { - fn run(&self) { - let output = Command::new("bash") - .arg("-c") - .arg(self.run.clone()) - .output() - .unwrap(); - println!("Exec stdout: {}", String::from_utf8(output.stdout).unwrap()); - println!("Exec stderr: {}", String::from_utf8(output.stderr).unwrap()); + fn run(&self) -> Option { + Some(self.run.exec()) } } @@ -166,12 +227,26 @@ impl YamlTask { .to_owned(); // Get run script - let run = info["run"] - .as_str() - .ok_or(DagError::format_error(FormatError::NoRunScript( - id.to_owned(), - )))? - .to_owned(); + let run = &info["run"]; + + let executor = match run["type"].as_str().ok_or(DagError::format_error( + FormatError::RunScriptError(id.into()), + ))? { + "sh" => RunType::SH, + "deno" => RunType::DENO, + _ => { + return Err(DagError::format_error(FormatError::RunScriptError( + id.into(), + ))) + } + }; + + let run_script = + run["script"] + .as_str() + .ok_or(DagError::format_error(FormatError::RunScriptError( + id.into(), + )))?; // relys can be empty let mut relys = Vec::new(); @@ -182,7 +257,9 @@ impl YamlTask { .count(); } - let inner = YamlTaskInner { run }; + let inner = YamlTaskInner { + run: RunScript::new(run_script, executor), + }; Ok(YamlTask { yaml_id: id.to_string(), diff --git a/dagrs/test/test.sh b/dagrs/test/test.sh new file mode 100755 index 0000000000000000000000000000000000000000..d470797a74afa9cfce32b8cd12d669552c31c648 --- /dev/null +++ b/dagrs/test/test.sh @@ -0,0 +1 @@ +echo "exec sh file success" \ No newline at end of file diff --git a/dagrs/test/test_dag1.yaml b/dagrs/test/test_dag1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93b481efadfd87556ef7c75da27497149a4cd25f --- /dev/null +++ b/dagrs/test/test_dag1.yaml @@ -0,0 +1,12 @@ +dagrs: + a: + name: 任务1 + rely: [b] + run: + type: sh + script: ./test/test.sh + b: + name: "任务2" + run: + type: deno + script: print("Hello!") \ No newline at end of file diff --git a/dagrs/test/test_dag.yaml b/dagrs/test/test_dag2.yaml similarity index 44% rename from dagrs/test/test_dag.yaml rename to dagrs/test/test_dag2.yaml index c5f8f7064893ffda879c909e52d1297193695b06..e46a70faaad7d0d1ded1570c221a73b2c890ecec 100644 --- a/dagrs/test/test_dag.yaml +++ b/dagrs/test/test_dag2.yaml @@ -2,31 +2,47 @@ dagrs: a: name: "任务1" rely: [b, c] - run: "1" + run: + type: sh + script: echo a b: name: "任务2" rely: [c, f, g] - run: "2" + run: + type: sh + script: echo b c: name: "任务3" rely: [e, g] - run: "3" + run: + type: sh + script: echo c d: name: "任务4" rely: [c, e] - run: "4" + run: + type: sh + script: echo d e: name: "任务5" rely: [h] - run: "5" + run: + type: sh + script: echo e f: name: "任务6" rely: [g] - run: "6" + run: + type: deno + script: Deno.core.print("f\n") g: name: "任务7" rely: [h] - run: "7" + run: + type: deno + script: Deno.core.print("g\n") h: name: "任务8" - run: "8" \ No newline at end of file + run: + type: sh + script: ./test/test.sh \ No newline at end of file diff --git a/dagrs/test/test_error1.yaml b/dagrs/test/test_error1.yaml index 34e3ea5ee353b2e8c2fe4ac30d3913da7d8cf801..f442cc7ca07eb9d3188448e5a31ec322993309d9 100644 --- a/dagrs/test/test_error1.yaml +++ b/dagrs/test/test_error1.yaml @@ -2,7 +2,11 @@ dagrs: a: # no name rely: [b] - run: "echo 123" + run: + type: sh + script: echo x b: name: "任务2" - run: "echo 456" \ No newline at end of file + run: + type: sh + script: echo x \ No newline at end of file diff --git a/dagrs/test/test_error2.yaml b/dagrs/test/test_error2.yaml index 6488564ba3159298b7375d0c60c0d09acd20c8d6..5d22582edd980f264a7e6e61406bd8a5725bfc5d 100644 --- a/dagrs/test/test_error2.yaml +++ b/dagrs/test/test_error2.yaml @@ -1,4 +1,6 @@ a: name: "任务1" rely: [a] - run: "echo a" \ No newline at end of file + run: + type: sh + script: echo x \ No newline at end of file diff --git a/dagrs/test/test_error3.yaml b/dagrs/test/test_error3.yaml index 2ce0ba83d8853a7fcaa2f0d186dd08c40a883d49..0e0328a5d0fe0f129b66ed6354f322e4522ae9bc 100644 --- a/dagrs/test/test_error3.yaml +++ b/dagrs/test/test_error3.yaml @@ -2,4 +2,6 @@ dagrs: a: name: "任务1" rely: [b] - run: "echo a" \ No newline at end of file + run: + type: sh + script: echo x \ No newline at end of file diff --git a/dagrs/test/test_loop1.yaml b/dagrs/test/test_loop1.yaml index 957be6b8db782413562805b40ccc850b977fe0cd..cdbdaf3b618e26c5b5c11634fdbb3ef974d76747 100644 --- a/dagrs/test/test_loop1.yaml +++ b/dagrs/test/test_loop1.yaml @@ -2,20 +2,30 @@ dagrs: a: name: "任务1" rely: [b, c] - run: "1" + run: + type: sh + script: echo x b: name: "任务2" rely: [c] - run: "2" + run: + type: sh + script: echo x c: name: "任务3" rely: [d] - run: "3" + run: + type: sh + script: echo x d: name: "任务4" rely: [e] - run: "4" + run: + type: sh + script: echo x e: name: "任务5" rely: [c] - run: "4" \ No newline at end of file + run: + type: sh + script: echo x \ No newline at end of file diff --git a/dagrs/test/test_loop2.yaml b/dagrs/test/test_loop2.yaml index 697430a068f0168882781d3e0d74f7353a1aa426..b87c79d4ce22eb6f010f4e4c40854d33f28539c0 100644 --- a/dagrs/test/test_loop2.yaml +++ b/dagrs/test/test_loop2.yaml @@ -2,32 +2,48 @@ dagrs: a: name: "任务1" rely: [b, c] - run: "1" + run: + type: sh + script: echo x b: name: "任务2" rely: [c, f, g] - run: "2" + run: + type: sh + script: echo x c: name: "任务3" rely: [e, g] - run: "3" + run: + type: sh + script: echo x d: name: "任务4" rely: [c, e] - run: "4" + run: + type: sh + script: echo x e: name: "任务5" rely: [h] - run: "5" + run: + type: sh + script: echo x f: name: "任务6" rely: [g] - run: "6" + run: + type: sh + script: echo x g: name: "任务7" rely: [h] - run: "7" + run: + type: sh + script: echo x h: name: "任务8" rely: [f] - run: "8" \ No newline at end of file + run: + type: sh + script: echo x \ No newline at end of file