⌜ ᐸ ᐊ ᐯ Ⲷ Ⲡ ⌟
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

rust and shell script setup #1

open opened by yzzxyz.roomy.chat targeting main from dev
Labels

None yet.

assignee

None yet.

Participants 1
AT URI
at://did:plc:5lewiq2q3rliu3ypkx57zasr/sh.tangled.repo.pull/3mkw7rrdx3t22
+720
Diff #0
+72
README.md
··· 1 + # ⌜ ᐸ ᐊ ᐯ Ⲷ Ⲡ ⌟ 2 + 3 + ## Inference 4 + 5 + Install mesh-llm: 6 + 7 + It runs on CPU or any GPU/Graphics Card: cuda/metal/rocm/vulkan/cpu 8 + 9 + ```sh 10 + curl -fsSL https://raw.githubusercontent.com/Mesh-LLM/mesh-llm/main/install.sh | bash 11 + ``` 12 + 13 + For Linux: 14 + 15 + Force GPU accelerator type for your local device: 16 + 17 + ```sh 18 + just build --backend cpu 19 + ``` 20 + 21 + Symlink binaries: 22 + 23 + ```sh 24 + ln -s ./targets/release ~/.local/bin/ 25 + ``` 26 + 27 + If encountering NVCC issues try switching CUDA driver versions, web searching, or ping us. 28 + 29 + Ping us and request access token. 30 + With access token, join as a new node: 31 + 32 + ``` 33 + mesh-llm —join <access_token> 34 + ``` 35 + 36 + Look for: `OK Joined mesh` `OK bootstrap_proxy: API ready (bootstrap) : http://localhost:9337/` 37 + 38 + http://localhost:9337/ Chat 39 + http://localhost:3131/ Dashboard 40 + http://localhost:9337/v1 API for programmatic requests 41 + Leave your session open and undisturbed for best results (and try to stay online) 42 + 43 + | Goal | Command Syntax | 44 + | ------------------------- | ------------------------------- | 45 + | Join as client only node | `mesh-llm client —join eyJ…V19` | 46 + | Join and share GPU node | `mesh-llm —join eyJ…V19` | 47 + | Unload model | `mesh-llm unload Qwen3 0.6B` | 48 + | Load model | `mesh-llm load Qwen3.5 0.8B` | 49 + | Connection status | `mesh-llm status` | 50 + | Set VRAM total limit | `mesh-llm —max-vram 8` | 51 + | Force split model tensors | `mesh-llm —model … --split` | 52 + | Bind port | `mesh-llm --bind-port 7842` | 53 + 54 + Build mesh-llm for Linux/MacOS/Windows (needs CMAKE/programmer tools): 55 + 56 + ``` 57 + cargo run setup.rs 58 + ``` 59 + 60 + Build mesh-llm for non-M-series MAC (x86_64): 61 + 62 + ``` 63 + sh build-mac-cpu.sh --backend cpu 64 + ``` 65 + 66 + ## Training 67 + 68 + flower + iroh. not yet implemented 69 + 70 + ### Data Store 71 + 72 + not yet implemented
+167
build-mac-cpu-mesh-llm.sh
··· 1 + #!/bin/zsh 2 + # build-mac.sh — build llama.cpp + mesh-llm on macOS Apple Silicon 3 + # 4 + # Usage: 5 + # scripts/build-mac.sh 6 + 7 + setopt errexit nounset pipefail 8 + 9 + SCRIPT_DIR="${0:A:h}" 10 + REPO_ROOT="${SCRIPT_DIR:h}" 11 + 12 + LLAMA_DIR="${MESH_LLM_LLAMA_DIR:-$REPO_ROOT/.deps/llama.cpp}" 13 + BUILD_DIR="$LLAMA_DIR/build" 14 + MESH_DIR="mesh-llm" 15 + UI_DIR="$MESH_DIR/ui" 16 + 17 + CLEAN=0 18 + BACKEND="" 19 + LLAMA_TARGETS="${MESH_LLM_LLAMA_TARGETS:-}" 20 + 21 + while [[ $# -gt 0 ]]; do 22 + case "$1" in 23 + --clean) 24 + CLEAN=1 25 + shift 26 + ;; 27 + --backend) 28 + BACKEND="${2:-}" 29 + shift 2 30 + ;; 31 + esac 32 + done 33 + 34 + compiler_launcher_flags=() 35 + rustc_wrapper="" 36 + 37 + detect_jobs() { 38 + sysctl -n hw.ncpu 2>/dev/null || echo 4 39 + } 40 + 41 + configure_compiler_cache() { 42 + local cache_bin="" 43 + if (( ${+commands[sccache]} )); then 44 + cache_bin="sccache" 45 + rustc_wrapper="$cache_bin" 46 + elif (( ${+commands[ccache]} )); then 47 + cache_bin="ccache" 48 + else 49 + return 0 50 + fi 51 + 52 + echo "Using compiler cache: $cache_bin" 53 + compiler_launcher_flags=( 54 + -DCMAKE_C_COMPILER_LAUNCHER="$cache_bin" 55 + -DCMAKE_CXX_COMPILER_LAUNCHER="$cache_bin" 56 + ) 57 + 58 + if [[ -n "$rustc_wrapper" ]]; then 59 + echo "Using Rust compiler wrapper: $rustc_wrapper" 60 + fi 61 + } 62 + 63 + stage_dev_runtime_binaries() { 64 + local backend="$1" 65 + local target_dir="$2" 66 + local source_bin_dir="$BUILD_DIR/bin" 67 + 68 + mkdir -p "$target_dir" 69 + rm -f "$target_dir/rpc-server" "$target_dir/llama-server" 70 + 71 + for name in rpc-server llama-server; do 72 + local source="$source_bin_dir/$name" 73 + if [[ ! -f "$source" ]]; then 74 + echo "Error: expected llama.cpp binary not found: $source" >&2 75 + exit 1 76 + fi 77 + cp "$source" "$target_dir/$name-$backend" 78 + done 79 + 80 + for name in llama-moe-analyze llama-moe-split; do 81 + local source="$source_bin_dir/$name" 82 + if [[ -f "$source" ]]; then 83 + cp "$source" "$target_dir/$name" 84 + fi 85 + done 86 + 87 + echo "Staged llama.cpp runtime binaries in $target_dir with '$backend' flavor names." 88 + } 89 + 90 + LLAMA_WORKDIR="$LLAMA_DIR" "scripts/prepare-llama.sh" "${MESH_LLM_LLAMA_PIN_SHA:-pinned}" 91 + 92 + configure_compiler_cache 93 + 94 + cmake_flags=( 95 + -B "$BUILD_DIR" 96 + -S "$LLAMA_DIR" 97 + -DGGML_RPC=ON 98 + -DBUILD_SHARED_LIBS=OFF 99 + -DLLAMA_OPENSSL=OFF 100 + ) 101 + 102 + if [[ "$BACKEND" == "cpu" ]]; then 103 + echo cpu 104 + cmake_flags+=( 105 + -DGGML_VULKAN=OFF 106 + -DGGML_METAL=OFF 107 + ) 108 + elif [[ "$BACKEND" == "vulkan" ]]; then 109 + echo vulkan 110 + cmake_flags+=( 111 + -DGGML_VULKAN=ON 112 + -DGGML_METAL=OFF 113 + ) 114 + else 115 + echo metal 116 + cmake_flags+=( 117 + -DGGML_VULKAN=OFF 118 + -DGGML_METAL=ON 119 + ) 120 + fi 121 + if (( ${+commands[ninja]} )); then 122 + cmake_flags=(-G Ninja "${cmake_flags[@]}") 123 + fi 124 + 125 + cmake_flags+=("${compiler_launcher_flags[@]}") 126 + 127 + echo "Configuring llama.cpp for macOS..." 128 + cmake "${cmake_flags[@]}" 129 + 130 + echo "Building llama.cpp..." 131 + cmake --build "$BUILD_DIR" --config Release --parallel "$(detect_jobs)" 132 + echo "Build complete: $BUILD_DIR/bin/" 133 + 134 + if [[ -d "$MESH_DIR" ]]; then 135 + echo "Building mesh-llm..." 136 + if [[ -d "$UI_DIR" ]]; then 137 + "$SCRIPT_DIR/build-ui.sh" "$UI_DIR" 138 + fi 139 + 140 + if [[ -n "$rustc_wrapper" ]]; then 141 + ( 142 + cd "$REPO_ROOT" 143 + RUSTC_WRAPPER="$rustc_wrapper" cargo build --release 144 + ) 145 + else 146 + ( 147 + cd "$REPO_ROOT" 148 + cargo build --release 149 + ) 150 + fi 151 + 152 + if [[ "$BACKEND" == "cpu" ]]; then 153 + ( 154 + stage_dev_runtime_binaries "cpu" "target/release" 155 + ) 156 + elif [[ "$BACKEND" == "vulkan" ]]; then 157 + ( 158 + stage_dev_runtime_binaries "vulkan" "$REPO_ROOT/target/release" 159 + ) 160 + else 161 + ( 162 + stage_dev_runtime_binaries "metal" "$REPO_ROOT/target/release" 163 + 164 + ) 165 + fi 166 + echo "Mesh binary: target/release/mesh-llm" 167 + fi
+218
install_prereq.sh
··· 1 + #!/usr/bin/env bash 2 + set -euo pipefail 3 + 4 + NAME="Check required tools and install missing ones" 5 + 6 + platform() { 7 + case "$(uname_s)" in 8 + Darwin) echo "Darwin" ;; 9 + Linux) echo "Linux" ;; 10 + CYGWIN*|MSYS*) echo "Windows" ;; 11 + *) echo "Unknown" ;; 12 + esac 13 + } 14 + 15 + check_cmake() { 16 + if command -v cmake >/dev/null; then 17 + return 0 18 + fi 19 + return 1 20 + } 21 + 22 + check_rust() { 23 + if command -v cargo >/dev/null; then 24 + return 0 25 + fi 26 + return 1 27 + } 28 + 29 + check_xet() { 30 + if command -v xet >/dev/null; then 31 + return 0 32 + fi 33 + return 1 34 + } 35 + 36 + check_npm() { 37 + if command -v npm >/dev/null; then 38 + return 0 39 + fi 40 + if [[ -n "${NVM_HOME:-}" ]]; then 41 + return 0 42 + fi 43 + if [[ -f "$HOME/.nvm/nvm.sh" ]]; then 44 + return 0 45 + fi 46 + return 1 47 + } 48 + 49 + run_version_cmake() { 50 + local out 51 + out=$(cmake --version 2>&1 | head -1) || true 52 + if [[ -n "$out" ]]; then 53 + echo "[OK] CMake: $out" 54 + else 55 + echo "[OK] CMake: installed (no version available)" 56 + fi 57 + } 58 + 59 + run_version_rust() { 60 + local out 61 + out=$(cargo --version 2>&1) || true 62 + if [[ -n "$out" ]]; then 63 + echo "[OK] Rust/Cargo: $out" 64 + else 65 + echo "[OK] Rust/Cargo: installed (no version available)" 66 + fi 67 + } 68 + 69 + run_version_xet() { 70 + local out 71 + out=$(xet --version 2>&1) || true 72 + if [[ -n "$out" ]]; then 73 + echo "[OK] Git Xet: $out" 74 + else 75 + echo "[OK] Git Xet: installed (no version available)" 76 + fi 77 + } 78 + 79 + run_version_npm() { 80 + local out 81 + out=$(npm --version 2>&1) || true 82 + if [[ -n "$out" ]]; then 83 + echo "[OK] npm: $out" 84 + else 85 + echo "[OK] npm: installed (no version available)" 86 + fi 87 + } 88 + 89 + install_cmake() { 90 + local sys 91 + sys=$(platform) 92 + case "$sys" in 93 + Darwin) 94 + if command -v brew >/dev/null; then 95 + echo "Run: brew install cmake" 96 + return 1 97 + fi 98 + ;; 99 + Linux) 100 + if command -v apt >/dev/null; then 101 + echo "Run: sudo apt install cmake" 102 + return 1 103 + elif command -v dnf >/dev/null; then 104 + echo "Run: sudo dnf install cmake" 105 + return 1 106 + elif command -v pacman >/dev/null; then 107 + echo "Run: sudo pacman install cmake" 108 + return 1 109 + fi 110 + ;; 111 + esac 112 + echo "No automatic install method for CMake on this platform." 113 + return 1 114 + } 115 + 116 + install_rust() { 117 + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh 118 + } 119 + 120 + install_xet() { 121 + curl --proto '=https' --tlsv1.2 -sSf \ 122 + "https://raw.githubusercontent.com/huggingface/xet-core/refs/heads/main/git_xet/install.sh" | sh 123 + } 124 + 125 + install_npm() { 126 + curl -o- "https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh" | bash 127 + } 128 + 129 + prompt_install() { 130 + local name="$1" 131 + read -p "$name not found. Install now? [y/N]: " resp 132 + case "${resp,,}" in 133 + y|yes) return 0 ;; 134 + *) return 1 ;; 135 + esac 136 + } 137 + 138 + do_install() { 139 + local tool="$1" 140 + local result 141 + 142 + case "$tool" in 143 + cmake) 144 + if ! install_cmake; then 145 + return 1 146 + fi 147 + ;; 148 + rust) 149 + echo "Installing Rust via rustup..." 150 + if [[ "$(platform)" == "Darwin" ]]; then 151 + sudo sh -c "$(install_rust)" 152 + else 153 + eval "$(install_rust)" 154 + fi 155 + ;; 156 + xet) 157 + echo "Installing Git Xet..." 158 + if [[ "$(platform)" == "Darwin" ]]; then 159 + sudo sh -c "$(install_xet)" 160 + else 161 + eval "$(install_xet)" 162 + fi 163 + ;; 164 + npm) 165 + echo "Installing nvm..." 166 + eval "$(install_npm)" 167 + ;; 168 + esac 169 + } 170 + 171 + main() { 172 + local sys 173 + local installed=0 174 + local total=4 175 + local missing="" 176 + 177 + sys=$(platform) 178 + 179 + echo "Platform: $sys" 180 + echo "Bash: ${BASH_VERSION:-unknown}\n" 181 + 182 + for tool in cmake rust xet npm; do 183 + case "$tool" in 184 + cmake) check_fn="check_$tool"; run_fn="run_version_$tool" ;; 185 + rust) check_fn="check_$tool"; run_fn="run_version_$tool" ;; 186 + xet) check_fn="check_$tool"; run_fn="run_version_$tool" ;; 187 + npm) check_fn="check_$tool"; run_fn="run_version_$tool" ;; 188 + esac 189 + 190 + if "$check_fn"; then 191 + echo "[OK] ${tool^} found" 192 + "$run_fn" 193 + ((installed++)) 194 + else 195 + echo "${tool^} not found" 196 + if prompt_install "${tool^}"; then 197 + if do_install "$tool"; then 198 + "$run_fn" 199 + ((installed++)) 200 + fi 201 + else 202 + missing="$missing $tool," 203 + fi 204 + else 205 + missing="$missing $tool," 206 + fi 207 + fi 208 + done 209 + 210 + echo "" 211 + echo "Tools available: $installed/$total" 212 + 213 + if [[ -n "$missing" ]]; then 214 + echo "Missing:${missing%,}" 215 + fi 216 + } 217 + 218 + main "$@"
+263
setup.rs
··· 1 + //! Clone mesh-llm, detect GPU card version, then invoke `just build backend`. 2 + 3 + use std::io::Write; 4 + use std::path::Path; 5 + use std::process::{Command, Stdio}; 6 + 7 + const REPO_URL: &str = "https://github.com/Mesh-LLM/mesh-llm"; 8 + const CLOPE_DIR: &str = ".deps/mesh-llm"; 9 + 10 + fn die(msg: &str) { 11 + eprintln!("ERROR: {}", msg); 12 + std::process::exit(1); 13 + } 14 + 15 + fn main() { 16 + if let Err(e) = run() { 17 + eprintln!("ERROR: {}", e); 18 + std::process::exit(1); 19 + } 20 + } 21 + 22 + fn run() -> Result<(), Box<dyn std::error::Error>> { 23 + clone_repo()?; 24 + cd_to_clone(); 25 + println!("Detecting GPU back-end ..."); 26 + let backend = detect_backend()?; 27 + execute_just_build(&backend)?; 28 + Ok(()) 29 + } 30 + 31 + fn clone_repo() -> Result<(), Box<dyn std::error::Error>> { 32 + let path = Path::new(CLOPE_DIR); 33 + 34 + if !path.exists() { 35 + print!("Cloning {} ... ", REPO_URL); 36 + std::io::stdout().flush(); 37 + 38 + std::fs::create_dir_all(".deps")?; 39 + 40 + let status = Command::new("git") 41 + .args(["clone", "--quiet", REPO_URL, CLOPE_DIR]) 42 + .current_dir(Path::new(".deps")) 43 + .stdout(Stdio::null()) 44 + .stderr(Stdio::null()) 45 + .status(); 46 + 47 + match status { 48 + Ok(s) if s.success() => { 49 + println!(" done"); 50 + } 51 + _ => { 52 + let code = match status { 53 + Ok(s) => format!("{:?}", s.code()), 54 + Err(_) => "signal".to_string(), 55 + }; 56 + return Err(format!("git clone failed with exit code {}", code).into()); 57 + } 58 + } 59 + } else { 60 + println!("Clone already exists at {}, using it.", CLOPE_DIR); 61 + } 62 + 63 + Ok(()) 64 + } 65 + 66 + fn cd_to_clone() -> Result<(), Box<dyn std::error::Error>> { 67 + std::env::set_current_dir(std::path::Path::new(CLOPE_DIR))?; 68 + Ok(()) 69 + } 70 + 71 + fn run_bash(script: &str) -> Result<String, Box<dyn std::error::Error>> { 72 + let out = Command::new("/bin/bash") 73 + .arg("--noconf") 74 + .arg("-c") 75 + .arg(script) 76 + .stdout(Stdio::piped()) 77 + .output(); 78 + 79 + match out { 80 + Ok(o) => { 81 + if o.status.success() { 82 + Ok(String::from_utf8_lossy(&o.stdout).trim().to_string()) 83 + } else { 84 + Ok("".to_string()) 85 + } 86 + } 87 + Err(e) => Err(Box::new(e)), 88 + } 89 + } 90 + 91 + fn has_command(name: &str) -> bool { 92 + matches!( 93 + Command::new("command").args(["-v", name]).status(), 94 + Ok(s) if s.success(), 95 + ) 96 + } 97 + 98 + fn detect_backend() -> Result<String, Box<dyn std::error::Error>> { 99 + // ── 1. Try nvidia-smi + detect-cuda-arch.sh 100 + print!("Checking CUDA ... "); 101 + std::io::stdout().flush(); 102 + 103 + if has_command("nvidia-smi") { 104 + println!("found"); 105 + print!("Running detect-cuda-arch.sh ... "); 106 + std::io::stdout().flush(); 107 + 108 + let sm_list = run_bash("scripts/detect-cuda-arch.sh"); 109 + 110 + match sm_list { 111 + Ok(list) if !list.is_empty() && list != "" => { 112 + println!("SM values detected: {}", list); 113 + return Ok("cuda".to_string()); 114 + } 115 + _ => { 116 + println!("script failed or returned empty — falling back"); 117 + } 118 + } 119 + } else { 120 + println!("nvidia-smi not in PATH"); 121 + } 122 + 123 + // ── 2. Try reem-smi / rocminfo + detect-rocm-arch.sh 124 + print!("Checking ROCm ... "); 125 + std::io::stdout().flush(); 126 + 127 + if has_command("reem-smi") || has_command("rocminfo") { 128 + println!("found"); 129 + print!("Running detect-rocm-arch.sh ... "); 130 + std::io::stdout().flush(); 131 + 132 + let gfx_list = run_bash("scripts/detect-rocm-arch.sh"); 133 + 134 + match gfx_list { 135 + Ok(list) if !list.is_empty() && list != "" => { 136 + println!("gfx values detected: {}", list); 137 + return Ok("rocm".to_string()); 138 + } 139 + _ => { 140 + println!("script failed or returned empty — falling back"); 141 + } 142 + } 143 + } else { 144 + println!("ROCm tools not found"); 145 + } 146 + 147 + // ── 3. Apple Silicon on macOS (Metal backend) 148 + #[cfg(target_os = "macos")] 149 + { 150 + let output = Command::new("uname").arg("-m").output().ok(); 151 + if let Some(o) = output { 152 + let arch = String::from_utf8_lossy(&o.stdout).trim().to_string(); 153 + if arch == "arm" { 154 + println!("Apple Silicon Mac detected (backing with Metal)"); 155 + return Ok("metal".to_string()); 156 + } 157 + } 158 + } 159 + 160 + // ── 4. Vulkan via vulkaninfo or SDK headers 161 + print!("Checking Vulkan ... "); 162 + std::io::stdout().flush(); 163 + 164 + if check_vulkan() { 165 + println!("vulkan present"); 166 + 167 + if has_command("vulkaninfo") { 168 + print!("Running vulkaninfo --summary ... "); 169 + std::io::stdout().flush(); 170 + 171 + match run_bash("vulkaninfo --summary") { 172 + Ok(info) if !info.is_empty() && info != "" => { 173 + println!("GPU: {}", parse_gpu_name_from_vulkaninfo_output(&info)); 174 + return Ok("vulkan".to_string()); 175 + } 176 + _ => { 177 + println!("could not detect GPU model — falling through"); 178 + } 179 + } 180 + } else { 181 + println!("no vulkaninfo binary, but SDK present — using vulkan backend"); 182 + return Ok("vulkan".to_string()); 183 + } 184 + } else { 185 + println!("Vulkan SDK / vulkaninfo not found") 186 + } 187 + 188 + // ── 5. Fall back to CPU only 189 + println!("No GPU acceleration available — using cpu backend"); 190 + Ok("cpu".to_string()) 191 + } 192 + 193 + fn execute_just_build(backend: &str) -> Result<(), Box<dyn std::error::Error>> { 194 + println!("Running `just build backend=\"{}\"` ...", backend); 195 + std::io::stdout().flush(); 196 + 197 + let out = Command::new("just") 198 + .arg(format!("build backend={}", backend)) 199 + .current_dir(std::path::Path::new(".")) 200 + .stderr(Stdio::piped()) 201 + .output(); 202 + 203 + match out { 204 + Ok(o) => { 205 + if o.status.success() { 206 + println!("Build succeeded with exit code {:?}", o.status.code()); 207 + Ok(()) 208 + } else { 209 + let exit_code = o.status.code(); 210 + let code_str = match exit_code.map(|c| c.to_string()) { 211 + Some(s) => s, 212 + None => "signal".to_string(), 213 + }; 214 + let msg = format!("`just build` exited with code {}", code_str); 215 + return Err(msg.into()); 216 + } 217 + } 218 + Err(e) => { 219 + let msg = format!( 220 + "failed to run `just`: {} (hint: make sure `just` is installed and in PATH; see README.md).", 221 + e, 222 + ); 223 + return Err(msg.into()); 224 + } 225 + } 226 + } 227 + 228 + fn check_vulkan() -> bool { 229 + if has_command("vulkaninfo") { 230 + return true; 231 + } 232 + 233 + const VULKAN_HEADER_PATHS: [&str; 3] = [ 234 + "/usr/include/vulkan/vulkan.h", 235 + "/usr/local/include/vulkan/vulkan.h", 236 + "/opt/homebrew/Cellar/vulkan-loader/2024.12.18/include/vulkan/vulkan.h", 237 + ]; 238 + 239 + for p in VULKAN_HEADER_PATHS.iter() { 240 + if std::path::PathBuf::from(*p).exists() { 241 + return true; 242 + } 243 + } 244 + 245 + false 246 + } 247 + 248 + fn parse_gpu_name_from_vulkaninfo_output(output: &str) -> String { 249 + let lines: Vec<&str> = output.lines().collect(); 250 + 251 + for line in lines.iter().take(20) { 252 + if line.starts_with("GPU #") || line.contains(": Device") || line.contains(": GPU") { 253 + if let Some(pos) = line.find(':') { 254 + let after_colon = line[pos + 1..].trim(); 255 + if !after_colon.is_empty() && !after_colon.starts_with("{") { 256 + return after_colon.to_string(); 257 + } 258 + } 259 + } 260 + } 261 + 262 + "unknown device".to_string() 263 + }

History

1 round 0 comments
sign up or login to add to the discussion
yzzxyz.roomy.chat submitted #0
2 commits
expand
rust and shell script setup
update readme
merge conflicts detected
expand
expand 0 comments