Personal NixOS config
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

Use Q6 Devstral Small 2 with 80k ctxlen

+10 -12
+10 -12
configuration.nix
··· 97 97 # Devstral 2 - 24B coding agent model from Mistral AI 98 98 # (unsloth version has fixed system prompts!) 99 99 # Download: huggingface-cli download unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF \ 100 - # --include "*Q4_K_XL*" \ 101 - # --local-dir /srv/devstral-2 102 - # huggingface-cli download unsloth/Devstral-Small-2-24B-Instruct-2512-GGUF \ 103 - # --include "*mmproj-F16*" \ 100 + # --include "*Q6_K_XL*" \ 104 101 # --local-dir /srv/devstral-2 105 102 { 106 103 Name = "devstral-2"; ··· 109 106 ProxyTargetPort = "18081"; 110 107 Command = "${llamaCppCuda}/bin/llama-server"; 111 108 Args = builtins.concatStringsSep " " [ 112 - "-m /srv/devstral-2/Devstral-Small-2-24B-Instruct-2512-UD-Q4_K_XL.gguf" 113 - "--mmproj /srv/devstral-2/mmproj-F16.gguf" 109 + "-m /srv/devstral-2/Devstral-Small-2-24B-Instruct-2512-UD-Q6_K_XL.gguf" 114 110 "--jinja" # Enable Jinja2 templates for chat formatting 115 111 "-ngl 99" # Offload all layers to GPU 116 112 "--threads -1" # Auto-detect threads 117 - "--ctx-size 98304" # 96k context 113 + "--ctx-size 81920" # 80k context (Q6_K_XL needs more VRAM for weights) 118 114 "-fa on" # Flash attention 119 - "--cache-type-k q8_0" # Quantize K cache to save VRAM 120 - "--cache-type-v q8_0" # Quantize V cache to save VRAM 115 + "--cache-type-k q4_0" # Quantize K cache to save VRAM 116 + "--cache-type-v q4_0" # Quantize V cache to save VRAM 121 117 "--host 127.0.0.1" 122 118 "--port 18081" 123 119 ]; ··· 126 122 HealthcheckIntervalMilliseconds = 500; 127 123 RestartOnConnectionFailure = true; 128 124 ResourceRequirements = { 129 - VRAM = 23800; # ~23GB for Q4_K_XL quantization at a 96k context with image support 125 + VRAM = 23800; # ~23GB for Q6_K_XL quantization at 96k context 130 126 }; 131 127 } 132 128 # GLM 4.7 Flash - 30B MoE model (only ~3.6B active params!) ··· 439 435 440 436 # Development tools 441 437 devcontainer 442 - python3Packages.huggingface-hub 443 - python3Packages.hf-xet # Blazing fast HuggingFace downloads via XET protocol 438 + (python3.withPackages (ps: with ps; [ 439 + huggingface-hub 440 + hf-xet # Blazing fast HuggingFace downloads via XET protocol 441 + ])) 444 442 llamaCppCuda 445 443 unstable.claude-code 446 444 ];