A local-first private AI assistant for everyday use. Runs on-device models with encrypted P2P sync, and supports sharing chats publicly on ATProto.
refactor: Using separate runtime depending OS
Currently this is implemented using Enums which makes
the runtime system as closed and not easily pluggable.
We will revisit later once this is much more polished to
make the implementation flexible to add runtimes by anybody
by traits
+181
-78
+6
HACKING.md
+6
HACKING.md
fixtures/a.modelfile
tilekit/fixtures/a.modelfile
fixtures/a.modelfile
tilekit/fixtures/a.modelfile
fixtures/llama_bad.Modelfile
tilekit/fixtures/llama_bad.Modelfile
fixtures/llama_bad.Modelfile
tilekit/fixtures/llama_bad.Modelfile
fixtures/mistral.modelfile
tilekit/fixtures/mistral.modelfile
fixtures/mistral.modelfile
tilekit/fixtures/mistral.modelfile
+1
tiles/Cargo.toml
+1
tiles/Cargo.toml
+10
-10
tiles/src/commands/mod.rs
+10
-10
tiles/src/commands/mod.rs
···
2
2
3
3
use anyhow::Result;
4
4
use tilekit::{modelfile, modelfile::Modelfile};
5
-
use tiles::{core::health, runner::mlx};
6
-
5
+
use tiles::runtime::Runtime;
6
+
use tiles::{core::health, runtime::RunArgs};
7
7
const DEFAULT_MODELFILE: &str = "
8
8
FROM driaforall/mem-agent-mlx-4bit
9
9
";
10
10
11
-
pub async fn run(modelfile: Option<String>) {
11
+
pub async fn run(runtime: &Runtime, modelfile: Option<String>) {
12
12
let modelfile_parse_result: Result<Modelfile, String> = if let Some(modelfile_str) = modelfile {
13
13
modelfile::parse_from_file(modelfile_str.as_str())
14
14
} else {
15
15
modelfile::parse(DEFAULT_MODELFILE)
16
16
};
17
-
18
17
match modelfile_parse_result {
19
18
Ok(modelfile) => {
20
-
mlx::run(modelfile).await;
19
+
let run_args = RunArgs { modelfile };
20
+
runtime.run(run_args).await;
21
21
}
22
-
Err(err) => println!("{}", err),
22
+
Err(_err) => println!("Invalid Modelfile"),
23
23
}
24
24
}
25
25
···
27
27
health::check_health();
28
28
}
29
29
30
-
pub async fn start_server() {
31
-
let _ = mlx::start_server_daemon().await;
30
+
pub async fn start_server(runtime: &Runtime) {
31
+
let _ = runtime.start_server_daemon().await;
32
32
}
33
33
34
-
pub async fn stop_server() {
35
-
let _ = mlx::stop_server_daemon().await;
34
+
pub async fn stop_server(runtime: &Runtime) {
35
+
let _ = runtime.stop_server_daemon().await;
36
36
}
+1
-1
tiles/src/lib.rs
+1
-1
tiles/src/lib.rs
+6
-4
tiles/src/main.rs
+6
-4
tiles/src/main.rs
···
1
1
use std::error::Error;
2
2
3
3
use clap::{Args, Parser, Subcommand};
4
+
use tiles::runtime::build_runtime;
4
5
mod commands;
5
6
#[derive(Debug, Parser)]
6
7
#[command(name = "tiles")]
···
38
39
/// Stops the daemon py server
39
40
Stop,
40
41
}
41
-
#[tokio::main]
42
+
#[tokio::main(flavor = "current_thread")]
42
43
pub async fn main() -> Result<(), Box<dyn Error>> {
43
44
let cli = Cli::parse();
45
+
let runtime = build_runtime();
44
46
match cli.command {
45
47
Commands::Run { modelfile_path } => {
46
-
commands::run(modelfile_path).await;
48
+
commands::run(&runtime, modelfile_path).await;
47
49
}
48
50
Commands::Health => {
49
51
commands::check_health();
50
52
}
51
53
Commands::Server(server) => match server.command {
52
-
Some(ServerCommands::Start) => commands::start_server().await,
53
-
Some(ServerCommands::Stop) => commands::stop_server().await,
54
+
Some(ServerCommands::Start) => commands::start_server(&runtime).await,
55
+
Some(ServerCommands::Stop) => commands::stop_server(&runtime).await,
54
56
_ => println!("Expected start or stop"),
55
57
},
56
58
}
+82
-62
tiles/src/runner/mlx.rs
tiles/src/runtime/mlx.rs
+82
-62
tiles/src/runner/mlx.rs
tiles/src/runtime/mlx.rs
···
12
12
use std::{io, process::Command};
13
13
use tilekit::modelfile::Modelfile;
14
14
use tokio::time::sleep;
15
+
16
+
pub struct MLXRuntime {}
17
+
18
+
impl MLXRuntime {}
15
19
pub struct ChatResponse {
16
20
// think: String,
17
21
reply: String,
18
22
code: String,
19
23
}
20
24
21
-
pub async fn run(modelfile: Modelfile) {
22
-
let model = modelfile.from.as_ref().unwrap();
23
-
if model.starts_with("driaforall/mem-agent") {
24
-
let _res = run_model_with_server(modelfile).await;
25
-
} else {
26
-
run_model_by_sub_process(modelfile);
25
+
impl Default for MLXRuntime {
26
+
fn default() -> Self {
27
+
Self::new()
28
+
}
29
+
}
30
+
31
+
impl MLXRuntime {
32
+
pub fn new() -> Self {
33
+
MLXRuntime {}
34
+
}
35
+
36
+
pub async fn run(&self, run_args: super::RunArgs) {
37
+
let model = run_args.modelfile.from.as_ref().unwrap();
38
+
if model.starts_with("driaforall/mem-agent") {
39
+
let _res = run_model_with_server(self, run_args.modelfile).await;
40
+
} else {
41
+
run_model_by_sub_process(run_args.modelfile);
42
+
}
43
+
}
44
+
45
+
#[allow(clippy::zombie_processes)]
46
+
pub async fn start_server_daemon(&self) -> Result<()> {
47
+
// check if the server is running
48
+
// start server as a child process
49
+
// save the pid in a file under ~/.config/tiles/server_pid
50
+
51
+
if (ping().await).is_ok() {
52
+
println!("server is already up");
53
+
return Ok(());
54
+
}
55
+
56
+
let config_dir = get_config_dir()?;
57
+
let mut server_dir = get_server_dir()?;
58
+
let pid_file = config_dir.join("server.pid");
59
+
fs::create_dir_all(&config_dir).context("Failed to create config directory")?;
60
+
61
+
let stdout_log = File::create(config_dir.join("server.out.log"))?;
62
+
let stderr_log = File::create(config_dir.join("server.err.log"))?;
63
+
let server_path = server_dir.join(".venv/bin/python3");
64
+
server_dir.pop();
65
+
let child = Command::new(server_path)
66
+
.args(["-m", "server.main"])
67
+
.current_dir(server_dir)
68
+
.stdin(Stdio::null())
69
+
.stdout(Stdio::from(stdout_log))
70
+
.stderr(Stdio::from(stderr_log))
71
+
.spawn()
72
+
.expect("failed to start server");
73
+
74
+
fs::create_dir_all(&config_dir).context("Failed to create config directory")?;
75
+
std::fs::write(pid_file, child.id().to_string()).unwrap();
76
+
println!("Server started with PID {}", child.id());
77
+
Ok(())
78
+
}
79
+
80
+
pub async fn stop_server_daemon(&self) -> Result<()> {
81
+
if (ping().await).is_err() {
82
+
println!("Server is not running");
83
+
return Ok(());
84
+
}
85
+
let pid_file = get_config_dir()?.join("server.pid");
86
+
87
+
if !pid_file.exists() {
88
+
eprintln!("server pid doesnt exist");
89
+
return Ok(());
90
+
}
91
+
92
+
let pid = std::fs::read_to_string(&pid_file).unwrap();
93
+
Command::new("kill").arg(pid.trim()).status().unwrap();
94
+
std::fs::remove_file(pid_file).unwrap();
95
+
println!("Server stopped.");
96
+
Ok(())
27
97
}
28
98
}
29
99
···
82
152
}
83
153
}
84
154
85
-
#[allow(clippy::zombie_processes)]
86
-
pub async fn start_server_daemon() -> Result<()> {
87
-
// check if the server is running
88
-
// start server as a child process
89
-
// save the pid in a file under ~/.config/tiles/server_pid
90
-
91
-
if (ping().await).is_ok() {
92
-
println!("server is already up");
93
-
return Ok(());
94
-
}
95
-
96
-
let config_dir = get_config_dir()?;
97
-
let mut server_dir = get_server_dir()?;
98
-
let pid_file = config_dir.join("server.pid");
99
-
fs::create_dir_all(&config_dir).context("Failed to create config directory")?;
100
-
101
-
let stdout_log = File::create(config_dir.join("server.out.log"))?;
102
-
let stderr_log = File::create(config_dir.join("server.err.log"))?;
103
-
let server_path = server_dir.join(".venv/bin/python3");
104
-
server_dir.pop();
105
-
let child = Command::new(server_path)
106
-
.args(["-m", "server.main"])
107
-
.current_dir(server_dir)
108
-
.stdin(Stdio::null())
109
-
.stdout(Stdio::from(stdout_log))
110
-
.stderr(Stdio::from(stderr_log))
111
-
.spawn()
112
-
.expect("failed to start server");
113
-
114
-
fs::create_dir_all(&config_dir).context("Failed to create config directory")?;
115
-
std::fs::write(pid_file, child.id().to_string()).unwrap();
116
-
println!("Server started with PID {}", child.id());
117
-
Ok(())
118
-
}
119
-
120
-
pub async fn stop_server_daemon() -> Result<()> {
121
-
if (ping().await).is_err() {
122
-
println!("Server is not running");
123
-
return Ok(());
124
-
}
125
-
let pid_file = get_config_dir()?.join("server.pid");
126
-
127
-
if !pid_file.exists() {
128
-
eprintln!("server pid doesnt exist");
129
-
return Ok(());
130
-
}
131
-
132
-
let pid = std::fs::read_to_string(&pid_file).unwrap();
133
-
Command::new("kill").arg(pid.trim()).status().unwrap();
134
-
std::fs::remove_file(pid_file).unwrap();
135
-
println!("Server stopped.");
136
-
Ok(())
137
-
}
138
-
async fn run_model_with_server(modelfile: Modelfile) -> reqwest::Result<()> {
155
+
async fn run_model_with_server(
156
+
mlx_runtime: &MLXRuntime,
157
+
modelfile: Modelfile,
158
+
) -> reqwest::Result<()> {
139
159
if !cfg!(debug_assertions) {
140
-
let _res = start_server_daemon().await;
160
+
let _res = mlx_runtime.start_server_daemon().await;
141
161
let _ = wait_until_server_is_up().await;
142
162
}
143
163
let stdin = io::stdin();
···
160
180
"exit" => {
161
181
println!("Exiting interactive mode");
162
182
if !cfg!(debug_assertions) {
163
-
let _res = stop_server_daemon().await;
183
+
let _res = mlx_runtime.stop_server_daemon().await;
164
184
}
165
185
break;
166
186
}
-1
tiles/src/runner/mod.rs
-1
tiles/src/runner/mod.rs
···
1
-
pub mod mlx;
+26
tiles/src/runtime/cpu.rs
+26
tiles/src/runtime/cpu.rs
···
1
+
use anyhow::Result;
2
+
3
+
pub struct CPURuntime {}
4
+
5
+
impl Default for CPURuntime {
6
+
fn default() -> Self {
7
+
Self::new()
8
+
}
9
+
}
10
+
11
+
impl CPURuntime {
12
+
pub fn new() -> Self {
13
+
CPURuntime {}
14
+
}
15
+
pub async fn run(&self, _run_args: super::RunArgs) {
16
+
unimplemented!()
17
+
}
18
+
19
+
pub async fn start_server_daemon(&self) -> Result<()> {
20
+
unimplemented!()
21
+
}
22
+
23
+
pub async fn stop_server_daemon(&self) -> Result<()> {
24
+
unimplemented!()
25
+
}
26
+
}
+49
tiles/src/runtime/mod.rs
+49
tiles/src/runtime/mod.rs
···
1
+
#[allow(unused_imports)]
2
+
use crate::runtime::cpu::CPURuntime;
3
+
use crate::runtime::mlx::MLXRuntime;
4
+
use anyhow::Result;
5
+
use tilekit::modelfile::Modelfile;
6
+
pub mod cpu;
7
+
pub mod mlx;
8
+
9
+
pub struct RunArgs {
10
+
pub modelfile: Modelfile,
11
+
}
12
+
13
+
pub enum Runtime {
14
+
Mlx(MLXRuntime),
15
+
Cpu(CPURuntime),
16
+
}
17
+
18
+
impl Runtime {
19
+
pub async fn run(&self, run_args: RunArgs) {
20
+
match self {
21
+
Runtime::Mlx(runtime) => runtime.run(run_args).await,
22
+
Runtime::Cpu(runtime) => runtime.run(run_args).await,
23
+
}
24
+
}
25
+
26
+
pub async fn start_server_daemon(&self) -> Result<()> {
27
+
match self {
28
+
Runtime::Mlx(runtime) => runtime.start_server_daemon().await,
29
+
Runtime::Cpu(runtime) => runtime.start_server_daemon().await,
30
+
}
31
+
}
32
+
33
+
pub async fn stop_server_daemon(&self) -> Result<()> {
34
+
match self {
35
+
Runtime::Mlx(runtime) => runtime.stop_server_daemon().await,
36
+
Runtime::Cpu(runtime) => runtime.stop_server_daemon().await,
37
+
}
38
+
}
39
+
}
40
+
41
+
#[cfg(target_os = "macos")]
42
+
pub fn build_runtime() -> Runtime {
43
+
Runtime::Mlx(MLXRuntime::new())
44
+
}
45
+
46
+
#[cfg(not(target_os = "macos"))]
47
+
pub fn build_runtime() -> Runtime {
48
+
Runtime::Cpu(CPURuntime::new())
49
+
}