Personal NixOS config
Enable Devstral 2 image support
+5
-1
+5
-1
configuration.nix
+5
-1
configuration.nix
···
99
99
# Download: huggingface-cli download unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF \
100
100
# --include "*Q4_K_XL*" \
101
101
# --local-dir /srv/devstral-2
102
+
# huggingface-cli download unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF \
103
+
# --include "*mmproj-F16*" \
104
+
# --local-dir /srv/devstral-2
102
105
{
103
106
Name = "devstral-2";
104
107
ListenPort = "8081";
···
107
110
Command = "${llamaCppCuda}/bin/llama-server";
108
111
Args = builtins.concatStringsSep " " [
109
112
"-m /srv/devstral-2/Devstral-Small-2-24B-Instruct-2512-UD-Q4_K_XL.gguf"
113
+
"--mmproj /srv/devstral-2/mmproj-F16.gguf"
110
114
"--jinja" # Enable Jinja2 templates for chat formatting
111
115
"-ngl 99" # Offload all layers to GPU
112
116
"--threads -1" # Auto-detect threads
···
122
126
HealthcheckIntervalMilliseconds = 500;
123
127
RestartOnConnectionFailure = true;
124
128
ResourceRequirements = {
125
-
VRAM = 23000; # ~23GB for Q4_K_XL quantization at a 96k context
129
+
VRAM = 23800; # ~23GB for Q4_K_XL quantization at a 96k context with image support
126
130
};
127
131
}
128
132
# Add more models here - LMP will swap them based on VRAM availability