⌜ ᐸ ᐊ ᐯ Ⲷ Ⲡ ⌟
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

rust and shell script setup #2

open opened by yzzxyz.roomy.chat targeting main from dev
Labels

None yet.

assignee

None yet.

Participants 1
AT URI
at://did:plc:5lewiq2q3rliu3ypkx57zasr/sh.tangled.repo.pull/3mkxde3togw22
+660 -35
Diff #0
+76
README.md
··· 1 + # ⌜ ᐸ ᐊ ᐯ Ⲷ Ⲡ ⌟ 2 + 3 + ## Inference 4 + 5 + Install mesh-llm: 6 + 7 + It runs on CPU or any GPU/Graphics Card: cuda/metal/rocm/vulkan/cpu 8 + 9 + ```sh 10 + curl -fsSL https://raw.githubusercontent.com/Mesh-LLM/mesh-llm/main/install.sh | bash 11 + ``` 12 + 13 + For Linux: 14 + 15 + Force GPU accelerator type for your local device: 16 + 17 + ```sh 18 + just build --backend cpu 19 + ``` 20 + 21 + Symlink binaries: 22 + 23 + ```sh 24 + ln -s ./targets/release ~/.local/bin/ 25 + ``` 26 + 27 + If encountering NVCC issues try switching CUDA driver versions, web searching, or ping us. 28 + 29 + Ping us and request access token. 30 + With access token, join as a new node: 31 + 32 + ``` 33 + mesh-llm —join <access_token> 34 + ``` 35 + 36 + Look for: `OK Joined mesh` `OK bootstrap_proxy: API ready (bootstrap) : http://localhost:9337/` 37 + 38 + http://localhost:9337/ Chat 39 + http://localhost:3131/ Dashboard 40 + http://localhost:9337/v1 API for programmatic requests 41 + Leave your session open and undisturbed for best results (and try to stay online) 42 + 43 + | Goal | Command Syntax | 44 + | ------------------------- | ------------------------------- | 45 + | Join as client only node | `mesh-llm client —join eyJ…V19` | 46 + | Join and share GPU node | `mesh-llm —join eyJ…V19` | 47 + | Unload model | `mesh-llm unload Qwen3 0.6B` | 48 + | Load model | `mesh-llm load Qwen3.5 0.8B` | 49 + | Connection status | `mesh-llm status` | 50 + | Set VRAM total limit | `mesh-llm —max-vram 8` | 51 + | Force split model tensors | `mesh-llm —model … --split` | 52 + | Bind port | `mesh-llm --bind-port 7842` | 53 + 54 + Build mesh-llm for Linux/MacOS/Windows (needs CMAKE/programmer tools): 55 + 56 + ``` 57 + cd mesh-build-scripts 58 + sh install-prereq.sh 59 + sh setup.sh 60 + ``` 61 + 62 + Build mesh-llm for non-M-series MAC (x86_64): 63 + 64 + ``` 65 + cd mesh-build-scripts 66 + sh install-prereq.sh 67 + sh build-mac-cpu.sh --backend cpu 68 + ``` 69 + 70 + ## Training 71 + 72 + flower + iroh. not yet implemented 73 + 74 + ### Data Store 75 + 76 + not yet implemented
+167
build-mac-cpu-mesh-llm.sh
··· 1 + #!/bin/zsh 2 + # build-mac.sh — build llama.cpp + mesh-llm on macOS Apple Silicon 3 + # 4 + # Usage: 5 + # scripts/build-mac.sh 6 + 7 + setopt errexit nounset pipefail 8 + 9 + SCRIPT_DIR="${0:A:h}" 10 + REPO_ROOT="${SCRIPT_DIR:h}" 11 + 12 + LLAMA_DIR="${MESH_LLM_LLAMA_DIR:-$REPO_ROOT/.deps/llama.cpp}" 13 + BUILD_DIR="$LLAMA_DIR/build" 14 + MESH_DIR="mesh-llm" 15 + UI_DIR="$MESH_DIR/ui" 16 + 17 + CLEAN=0 18 + BACKEND="" 19 + LLAMA_TARGETS="${MESH_LLM_LLAMA_TARGETS:-}" 20 + 21 + while [[ $# -gt 0 ]]; do 22 + case "$1" in 23 + --clean) 24 + CLEAN=1 25 + shift 26 + ;; 27 + --backend) 28 + BACKEND="${2:-}" 29 + shift 2 30 + ;; 31 + esac 32 + done 33 + 34 + compiler_launcher_flags=() 35 + rustc_wrapper="" 36 + 37 + detect_jobs() { 38 + sysctl -n hw.ncpu 2>/dev/null || echo 4 39 + } 40 + 41 + configure_compiler_cache() { 42 + local cache_bin="" 43 + if (( ${+commands[sccache]} )); then 44 + cache_bin="sccache" 45 + rustc_wrapper="$cache_bin" 46 + elif (( ${+commands[ccache]} )); then 47 + cache_bin="ccache" 48 + else 49 + return 0 50 + fi 51 + 52 + echo "Using compiler cache: $cache_bin" 53 + compiler_launcher_flags=( 54 + -DCMAKE_C_COMPILER_LAUNCHER="$cache_bin" 55 + -DCMAKE_CXX_COMPILER_LAUNCHER="$cache_bin" 56 + ) 57 + 58 + if [[ -n "$rustc_wrapper" ]]; then 59 + echo "Using Rust compiler wrapper: $rustc_wrapper" 60 + fi 61 + } 62 + 63 + stage_dev_runtime_binaries() { 64 + local backend="$1" 65 + local target_dir="$2" 66 + local source_bin_dir="$BUILD_DIR/bin" 67 + 68 + mkdir -p "$target_dir" 69 + rm -f "$target_dir/rpc-server" "$target_dir/llama-server" 70 + 71 + for name in rpc-server llama-server; do 72 + local source="$source_bin_dir/$name" 73 + if [[ ! -f "$source" ]]; then 74 + echo "Error: expected llama.cpp binary not found: $source" >&2 75 + exit 1 76 + fi 77 + cp "$source" "$target_dir/$name-$backend" 78 + done 79 + 80 + for name in llama-moe-analyze llama-moe-split; do 81 + local source="$source_bin_dir/$name" 82 + if [[ -f "$source" ]]; then 83 + cp "$source" "$target_dir/$name" 84 + fi 85 + done 86 + 87 + echo "Staged llama.cpp runtime binaries in $target_dir with '$backend' flavor names." 88 + } 89 + 90 + LLAMA_WORKDIR="$LLAMA_DIR" "scripts/prepare-llama.sh" "${MESH_LLM_LLAMA_PIN_SHA:-pinned}" 91 + 92 + configure_compiler_cache 93 + 94 + cmake_flags=( 95 + -B "$BUILD_DIR" 96 + -S "$LLAMA_DIR" 97 + -DGGML_RPC=ON 98 + -DBUILD_SHARED_LIBS=OFF 99 + -DLLAMA_OPENSSL=OFF 100 + ) 101 + 102 + if [[ "$BACKEND" == "cpu" ]]; then 103 + echo cpu 104 + cmake_flags+=( 105 + -DGGML_VULKAN=OFF 106 + -DGGML_METAL=OFF 107 + ) 108 + elif [[ "$BACKEND" == "vulkan" ]]; then 109 + echo vulkan 110 + cmake_flags+=( 111 + -DGGML_VULKAN=ON 112 + -DGGML_METAL=OFF 113 + ) 114 + else 115 + echo metal 116 + cmake_flags+=( 117 + -DGGML_VULKAN=OFF 118 + -DGGML_METAL=ON 119 + ) 120 + fi 121 + if (( ${+commands[ninja]} )); then 122 + cmake_flags=(-G Ninja "${cmake_flags[@]}") 123 + fi 124 + 125 + cmake_flags+=("${compiler_launcher_flags[@]}") 126 + 127 + echo "Configuring llama.cpp for macOS..." 128 + cmake "${cmake_flags[@]}" 129 + 130 + echo "Building llama.cpp..." 131 + cmake --build "$BUILD_DIR" --config Release --parallel "$(detect_jobs)" 132 + echo "Build complete: $BUILD_DIR/bin/" 133 + 134 + if [[ -d "$MESH_DIR" ]]; then 135 + echo "Building mesh-llm..." 136 + if [[ -d "$UI_DIR" ]]; then 137 + "$SCRIPT_DIR/build-ui.sh" "$UI_DIR" 138 + fi 139 + 140 + if [[ -n "$rustc_wrapper" ]]; then 141 + ( 142 + cd "$REPO_ROOT" 143 + RUSTC_WRAPPER="$rustc_wrapper" cargo build --release 144 + ) 145 + else 146 + ( 147 + cd "$REPO_ROOT" 148 + cargo build --release 149 + ) 150 + fi 151 + 152 + if [[ "$BACKEND" == "cpu" ]]; then 153 + ( 154 + stage_dev_runtime_binaries "cpu" "target/release" 155 + ) 156 + elif [[ "$BACKEND" == "vulkan" ]]; then 157 + ( 158 + stage_dev_runtime_binaries "vulkan" "$REPO_ROOT/target/release" 159 + ) 160 + else 161 + ( 162 + stage_dev_runtime_binaries "metal" "$REPO_ROOT/target/release" 163 + 164 + ) 165 + fi 166 + echo "Mesh binary: target/release/mesh-llm" 167 + fi
+218
install_prereq.sh
··· 1 + #!/usr/bin/env bash 2 + set -euo pipefail 3 + 4 + NAME="Check required tools and install missing ones" 5 + 6 + platform() { 7 + case "$(uname_s)" in 8 + Darwin) echo "Darwin" ;; 9 + Linux) echo "Linux" ;; 10 + CYGWIN*|MSYS*) echo "Windows" ;; 11 + *) echo "Unknown" ;; 12 + esac 13 + } 14 + 15 + check_cmake() { 16 + if command -v cmake >/dev/null; then 17 + return 0 18 + fi 19 + return 1 20 + } 21 + 22 + check_rust() { 23 + if command -v cargo >/dev/null; then 24 + return 0 25 + fi 26 + return 1 27 + } 28 + 29 + check_xet() { 30 + if command -v xet >/dev/null; then 31 + return 0 32 + fi 33 + return 1 34 + } 35 + 36 + check_npm() { 37 + if command -v npm >/dev/null; then 38 + return 0 39 + fi 40 + if [[ -n "${NVM_HOME:-}" ]]; then 41 + return 0 42 + fi 43 + if [[ -f "$HOME/.nvm/nvm.sh" ]]; then 44 + return 0 45 + fi 46 + return 1 47 + } 48 + 49 + run_version_cmake() { 50 + local out 51 + out=$(cmake --version 2>&1 | head -1) || true 52 + if [[ -n "$out" ]]; then 53 + echo "[OK] CMake: $out" 54 + else 55 + echo "[OK] CMake: installed (no version available)" 56 + fi 57 + } 58 + 59 + run_version_rust() { 60 + local out 61 + out=$(cargo --version 2>&1) || true 62 + if [[ -n "$out" ]]; then 63 + echo "[OK] Rust/Cargo: $out" 64 + else 65 + echo "[OK] Rust/Cargo: installed (no version available)" 66 + fi 67 + } 68 + 69 + run_version_xet() { 70 + local out 71 + out=$(xet --version 2>&1) || true 72 + if [[ -n "$out" ]]; then 73 + echo "[OK] Git Xet: $out" 74 + else 75 + echo "[OK] Git Xet: installed (no version available)" 76 + fi 77 + } 78 + 79 + run_version_npm() { 80 + local out 81 + out=$(npm --version 2>&1) || true 82 + if [[ -n "$out" ]]; then 83 + echo "[OK] npm: $out" 84 + else 85 + echo "[OK] npm: installed (no version available)" 86 + fi 87 + } 88 + 89 + install_cmake() { 90 + local sys 91 + sys=$(platform) 92 + case "$sys" in 93 + Darwin) 94 + if command -v brew >/dev/null; then 95 + echo "Run: brew install cmake" 96 + return 1 97 + fi 98 + ;; 99 + Linux) 100 + if command -v apt >/dev/null; then 101 + echo "Run: sudo apt install cmake" 102 + return 1 103 + elif command -v dnf >/dev/null; then 104 + echo "Run: sudo dnf install cmake" 105 + return 1 106 + elif command -v pacman >/dev/null; then 107 + echo "Run: sudo pacman install cmake" 108 + return 1 109 + fi 110 + ;; 111 + esac 112 + echo "No automatic install method for CMake on this platform." 113 + return 1 114 + } 115 + 116 + install_rust() { 117 + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh 118 + } 119 + 120 + install_xet() { 121 + curl --proto '=https' --tlsv1.2 -sSf \ 122 + "https://raw.githubusercontent.com/huggingface/xet-core/refs/heads/main/git_xet/install.sh" | sh 123 + } 124 + 125 + install_npm() { 126 + curl -o- "https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.4/install.sh" | bash 127 + } 128 + 129 + prompt_install() { 130 + local name="$1" 131 + read -p "$name not found. Install now? [y/N]: " resp 132 + case "${resp,,}" in 133 + y|yes) return 0 ;; 134 + *) return 1 ;; 135 + esac 136 + } 137 + 138 + do_install() { 139 + local tool="$1" 140 + local result 141 + 142 + case "$tool" in 143 + cmake) 144 + if ! install_cmake; then 145 + return 1 146 + fi 147 + ;; 148 + rust) 149 + echo "Installing Rust via rustup..." 150 + if [[ "$(platform)" == "Darwin" ]]; then 151 + sudo sh -c "$(install_rust)" 152 + else 153 + eval "$(install_rust)" 154 + fi 155 + ;; 156 + xet) 157 + echo "Installing Git Xet..." 158 + if [[ "$(platform)" == "Darwin" ]]; then 159 + sudo sh -c "$(install_xet)" 160 + else 161 + eval "$(install_xet)" 162 + fi 163 + ;; 164 + npm) 165 + echo "Installing nvm..." 166 + eval "$(install_npm)" 167 + ;; 168 + esac 169 + } 170 + 171 + main() { 172 + local sys 173 + local installed=0 174 + local total=4 175 + local missing="" 176 + 177 + sys=$(platform) 178 + 179 + echo "Platform: $sys" 180 + echo "Bash: ${BASH_VERSION:-unknown}\n" 181 + 182 + for tool in cmake rust xet npm; do 183 + case "$tool" in 184 + cmake) check_fn="check_$tool"; run_fn="run_version_$tool" ;; 185 + rust) check_fn="check_$tool"; run_fn="run_version_$tool" ;; 186 + xet) check_fn="check_$tool"; run_fn="run_version_$tool" ;; 187 + npm) check_fn="check_$tool"; run_fn="run_version_$tool" ;; 188 + esac 189 + 190 + if "$check_fn"; then 191 + echo "[OK] ${tool^} found" 192 + "$run_fn" 193 + ((installed++)) 194 + else 195 + echo "${tool^} not found" 196 + if prompt_install "${tool^}"; then 197 + if do_install "$tool"; then 198 + "$run_fn" 199 + ((installed++)) 200 + fi 201 + else 202 + missing="$missing $tool," 203 + fi 204 + else 205 + missing="$missing $tool," 206 + fi 207 + fi 208 + done 209 + 210 + echo "" 211 + echo "Tools available: $installed/$total" 212 + 213 + if [[ -n "$missing" ]]; then 214 + echo "Missing:${missing%,}" 215 + fi 216 + } 217 + 218 + main "$@"
+8 -35
build-mac-cpu-mesh-llm.sh mesh-build-scripts/build-mac-cpu-mesh-llm.sh
··· 91 91 92 92 configure_compiler_cache 93 93 94 + echo cpu 95 + 94 96 cmake_flags=( 95 97 -B "$BUILD_DIR" 96 98 -S "$LLAMA_DIR" 97 99 -DGGML_RPC=ON 98 100 -DBUILD_SHARED_LIBS=OFF 99 101 -DLLAMA_OPENSSL=OFF 100 - ) 101 - 102 - if [[ "$BACKEND" == "cpu" ]]; then 103 - echo cpu 104 - cmake_flags+=( 105 - -DGGML_VULKAN=OFF 106 - -DGGML_METAL=OFF 107 - ) 108 - elif [[ "$BACKEND" == "vulkan" ]]; then 109 - echo vulkan 110 - cmake_flags+=( 111 - -DGGML_VULKAN=ON 112 - -DGGML_METAL=OFF 113 - ) 114 - else 115 - echo metal 116 - cmake_flags+=( 117 - -DGGML_VULKAN=OFF 118 - -DGGML_METAL=ON 102 + -DGGML_VULKAN=OFF 103 + -DGGML_METAL=OFF 119 104 ) 120 - fi 105 + 106 + 121 107 if (( ${+commands[ninja]} )); then 122 108 cmake_flags=(-G Ninja "${cmake_flags[@]}") 123 109 fi ··· 134 120 if [[ -d "$MESH_DIR" ]]; then 135 121 echo "Building mesh-llm..." 136 122 if [[ -d "$UI_DIR" ]]; then 137 - "$SCRIPT_DIR/build-ui.sh" "$UI_DIR" 123 + "$SCRIPT_DIR/scripts/build-ui.sh" "$UI_DIR" 138 124 fi 139 125 140 126 if [[ -n "$rustc_wrapper" ]]; then ··· 149 135 ) 150 136 fi 151 137 152 - if [[ "$BACKEND" == "cpu" ]]; then 153 - ( 154 - stage_dev_runtime_binaries "cpu" "target/release" 155 - ) 156 - elif [[ "$BACKEND" == "vulkan" ]]; then 157 - ( 158 - stage_dev_runtime_binaries "vulkan" "$REPO_ROOT/target/release" 159 - ) 160 - else 161 - ( 162 - stage_dev_runtime_binaries "metal" "$REPO_ROOT/target/release" 163 - 164 - ) 165 - fi 138 + stage_dev_runtime_binaries "cpu" "target/release" 166 139 echo "Mesh binary: target/release/mesh-llm" 167 140 fi
install_prereq.sh mesh-build-scripts/install_prereq.sh
+191
mesh-build-scripts/setup.sh
··· 1 + #!/usr/bin/env bash 2 + set -euo pipefail 3 + 4 + NAME="Clone mesh-llm, detect GPU card version, then invoke build backend" 5 + 6 + REPO_URL="https://github.com/Mesh-LLM/mesh-llm" 7 + CLOPE_DIR=".deps/mesh-llm" 8 + 9 + die() { 10 + echo "ERROR: $1" >&2 11 + exit 1 12 + } 13 + 14 + has_cmd() { 15 + command -v "$1" > /dev/null 16 + } 17 + 18 + platform() { 19 + case "$(uname_s)" in 20 + Darwin) echo "Darwin" ;; 21 + Linux) echo "Linux" ;; 22 + CYGWIN*|MSYS*) echo "Windows" ;; 23 + *) echo "Unknown" ;; 24 + esac 25 + } 26 + 27 + detect_backend() { 28 + local out 29 + 30 + if has_cmd nvidia-smi; then 31 + echo -n "Checking CUDA ... " 32 + echo "found" 33 + echo -n "Running detect-cuda-arch.sh ... " 34 + 35 + if [[ ! -f "scripts/detect-cuda-arch.sh" ]]; then 36 + die "detect-cuda-arch.sh not found — clone may be corrupted" 37 + fi 38 + 39 + if out=$(bash scripts/detect-cuda-arch.sh); then 40 + if [[ -n "$out" && "$out" != "" ]]; then 41 + echo "SM values detected: $out" 42 + echo "cuda" 43 + return 44 + fi 45 + fi 46 + echo "script failed or returned empty — falling back" 47 + else 48 + echo "nvidia-smi not in PATH" 49 + fi 50 + 51 + if has_cmd reem-smi || has_cmd rocminfo; then 52 + echo -n "Checking ROCm ... " 53 + echo "found" 54 + echo -n "Running detect-rocm-arch.sh ... " 55 + 56 + if [[ ! -f "scripts/detect-rocm-arch.sh" ]]; then 57 + die "detect-rocm-arch.sh not found — clone may be corrupted" 58 + fi 59 + 60 + if out=$(bash scripts/detect-rocm-arch.sh); then 61 + if [[ -n "$out" && "$out" != "" ]]; then 62 + echo "gfx values detected: $out" 63 + echo "rocm" 64 + return 65 + fi 66 + fi 67 + echo "script failed or returned empty — falling back" 68 + else 69 + echo "ROCm tools not found" 70 + fi 71 + 72 + local arch="" 73 + if [[ "$(platform)" == "Darwin" ]]; then 74 + arch=$(uname -m) || true 75 + if [[ "$arch" == "arm" ]]; then 76 + echo "Apple Silicon Mac detected (backing with Metal)" 77 + echo "metal" 78 + return 79 + fi 80 + fi 81 + 82 + echo -n "Checking Vulkan ... " 83 + if check_vulkan; then 84 + echo "vulkan present" 85 + 86 + if has_cmd vulkaninfo; then 87 + echo -n "Running vulkaninfo --summary ... " 88 + 89 + if out=$(vulkaninfo --summary 2>&1); then 90 + if [[ -n "$out" && "$out" != "" ]]; then 91 + echo "GPU: $(parse_gpu_name_from_vulkaninfo_output "$out")" 92 + echo "vulkan" 93 + return 94 + fi 95 + fi 96 + echo "could not detect GPU model — falling through" 97 + else 98 + echo "no vulkaninfo binary, but SDK present — using vulkan backend" 99 + echo "vulkan" 100 + return 101 + fi 102 + else 103 + echo "Vulkan SDK / vulkaninfo not found" 104 + fi 105 + 106 + echo "No GPU acceleration available — using cpu backend" 107 + echo "cpu" 108 + } 109 + 110 + check_vulkan() { 111 + if has_cmd vulkaninfo; then 112 + return 0 113 + fi 114 + 115 + local header_paths=( 116 + "/usr/include/vulkan/vulkan.h", 117 + "/usr/local/include/vulkan/vulkan.h", 118 + "/opt/homebrew/Cellar/vulkan-loader/2024.12.18/include/vulkan/vulkan.h" 119 + ) 120 + 121 + for p in "${header_paths[@]}"; do 122 + if [[ -f "$p" ]]; then 123 + return 0 124 + fi 125 + done 126 + 127 + return 1 128 + } 129 + 130 + parse_gpu_name_from_vulkaninfo_output() { 131 + local out="$1" 132 + 133 + while IFS= read -r line; do 134 + if [[ "$line" == *"GPU #"* ]] || [[ "$line" == *": Device"* ]] || [[ "$line" == *": GPU"* ]]; then 135 + if [[ "$line" == *":"* ]]; then 136 + local after_colon="${line#*:}" 137 + after_colon="${after_colon#"${after_colon%%[^ ]*}"}" 138 + after_colon="${after_colon#"${after_colon##[![:space:]]}"}" 139 + after_colon="$(echo "$after_colon" | xargs)" 140 + 141 + if [[ -n "$after_colon" && "$after_colon" != "{"* ]]; then 142 + echo "$after_colon" 143 + return 144 + fi 145 + fi 146 + fi 147 + done <<< "$out" 148 + 149 + echo "unknown device" 150 + } 151 + 152 + clone_repo() { 153 + if [[ ! -d "$CLOPE_DIR" ]]; then 154 + echo -n "Cloning $REPO_URL ... " 155 + 156 + mkdir -p .deps 157 + 158 + if git clone --quiet "$REPO_URL" "$CLOPE_DIR"; then 159 + echo "done" 160 + else 161 + die "git clone failed with exit code $?" 162 + fi 163 + else 164 + echo "Clone already exists at $CLOPE_DIR, using it." 165 + fi 166 + } 167 + 168 + execute_just_build() { 169 + local backend="$1" 170 + 171 + echo "Running \`just build backend=\"$backend\"\` ..." 172 + 173 + if ! just "build backend=$backend"; then 174 + die "\`just build\` exited with non-zero status (hint: make sure \`just\` is installed and in PATH)" 175 + fi 176 + 177 + echo "Build succeeded" 178 + } 179 + 180 + main() { 181 + clone_repo 182 + 183 + cd "$CLOPE_DIR" || die "failed to cd into $CLOPE_DIR" 184 + 185 + local backend 186 + backend=$(detect_backend) || die "detection failed" 187 + 188 + execute_just_build "$backend" 189 + } 190 + 191 + main "$@"

History

1 round 0 comments
sign up or login to add to the discussion
yzzxyz.roomy.chat submitted #0
4 commits
expand
rust and shell script setup
update readme
migrate to sh entirely for building
update docs
merge conflicts detected
expand
expand 0 comments