A local-first private AI assistant for everyday use. Runs on-device models with encrypted P2P sync, and supports sharing chats publicly on ATProto.
Moved model downloading ownership to CLI (#38)
* feat: Moved model downloading ownership to cli from py server
* refactor: passing CI
* fix: using String's ends_with instead of contains
* fix: changed the default relay_count to 10
* fix: changed retry_count to relay_count
authored by
Anandu Pavanan
and committed by
GitHub
fe14a70e
5a322205
+517
-42
+383
Cargo.lock
+383
Cargo.lock
···
3
3
version = 4
4
4
5
5
[[package]]
6
+
name = "adler2"
7
+
version = "2.0.1"
8
+
source = "registry+https://github.com/rust-lang/crates.io-index"
9
+
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
10
+
11
+
[[package]]
6
12
name = "anstream"
7
13
version = "0.6.21"
8
14
source = "registry+https://github.com/rust-lang/crates.io-index"
···
81
87
version = "3.19.1"
82
88
source = "registry+https://github.com/rust-lang/crates.io-index"
83
89
checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510"
90
+
91
+
[[package]]
92
+
name = "byteorder"
93
+
version = "1.5.0"
94
+
source = "registry+https://github.com/rust-lang/crates.io-index"
95
+
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
84
96
85
97
[[package]]
86
98
name = "bytes"
···
151
163
checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
152
164
153
165
[[package]]
166
+
name = "console"
167
+
version = "0.15.11"
168
+
source = "registry+https://github.com/rust-lang/crates.io-index"
169
+
checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
170
+
dependencies = [
171
+
"encode_unicode",
172
+
"libc",
173
+
"once_cell",
174
+
"unicode-width",
175
+
"windows-sys 0.59.0",
176
+
]
177
+
178
+
[[package]]
154
179
name = "core-foundation"
155
180
version = "0.9.4"
156
181
source = "registry+https://github.com/rust-lang/crates.io-index"
···
167
192
checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
168
193
169
194
[[package]]
195
+
name = "crc32fast"
196
+
version = "1.5.0"
197
+
source = "registry+https://github.com/rust-lang/crates.io-index"
198
+
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
199
+
dependencies = [
200
+
"cfg-if",
201
+
]
202
+
203
+
[[package]]
204
+
name = "dirs"
205
+
version = "6.0.0"
206
+
source = "registry+https://github.com/rust-lang/crates.io-index"
207
+
checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e"
208
+
dependencies = [
209
+
"dirs-sys",
210
+
]
211
+
212
+
[[package]]
213
+
name = "dirs-sys"
214
+
version = "0.5.0"
215
+
source = "registry+https://github.com/rust-lang/crates.io-index"
216
+
checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
217
+
dependencies = [
218
+
"libc",
219
+
"option-ext",
220
+
"redox_users",
221
+
"windows-sys 0.61.2",
222
+
]
223
+
224
+
[[package]]
170
225
name = "displaydoc"
171
226
version = "0.2.5"
172
227
source = "registry+https://github.com/rust-lang/crates.io-index"
···
178
233
]
179
234
180
235
[[package]]
236
+
name = "encode_unicode"
237
+
version = "1.0.0"
238
+
source = "registry+https://github.com/rust-lang/crates.io-index"
239
+
checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
240
+
241
+
[[package]]
181
242
name = "encoding_rs"
182
243
version = "0.8.35"
183
244
source = "registry+https://github.com/rust-lang/crates.io-index"
···
215
276
checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff"
216
277
217
278
[[package]]
279
+
name = "flate2"
280
+
version = "1.1.5"
281
+
source = "registry+https://github.com/rust-lang/crates.io-index"
282
+
checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb"
283
+
dependencies = [
284
+
"crc32fast",
285
+
"miniz_oxide",
286
+
]
287
+
288
+
[[package]]
218
289
name = "fnv"
219
290
version = "1.0.7"
220
291
source = "registry+https://github.com/rust-lang/crates.io-index"
···
245
316
]
246
317
247
318
[[package]]
319
+
name = "futures"
320
+
version = "0.3.31"
321
+
source = "registry+https://github.com/rust-lang/crates.io-index"
322
+
checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
323
+
dependencies = [
324
+
"futures-channel",
325
+
"futures-core",
326
+
"futures-executor",
327
+
"futures-io",
328
+
"futures-sink",
329
+
"futures-task",
330
+
"futures-util",
331
+
]
332
+
333
+
[[package]]
248
334
name = "futures-channel"
249
335
version = "0.3.31"
250
336
source = "registry+https://github.com/rust-lang/crates.io-index"
···
261
347
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
262
348
263
349
[[package]]
350
+
name = "futures-executor"
351
+
version = "0.3.31"
352
+
source = "registry+https://github.com/rust-lang/crates.io-index"
353
+
checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
354
+
dependencies = [
355
+
"futures-core",
356
+
"futures-task",
357
+
"futures-util",
358
+
]
359
+
360
+
[[package]]
264
361
name = "futures-io"
265
362
version = "0.3.31"
266
363
source = "registry+https://github.com/rust-lang/crates.io-index"
···
295
392
source = "registry+https://github.com/rust-lang/crates.io-index"
296
393
checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
297
394
dependencies = [
395
+
"futures-channel",
298
396
"futures-core",
299
397
"futures-io",
300
398
"futures-macro",
···
359
457
version = "0.5.0"
360
458
source = "registry+https://github.com/rust-lang/crates.io-index"
361
459
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
460
+
461
+
[[package]]
462
+
name = "hermit-abi"
463
+
version = "0.5.2"
464
+
source = "registry+https://github.com/rust-lang/crates.io-index"
465
+
checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
466
+
467
+
[[package]]
468
+
name = "hf-hub"
469
+
version = "0.4.3"
470
+
source = "registry+https://github.com/rust-lang/crates.io-index"
471
+
checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97"
472
+
dependencies = [
473
+
"dirs",
474
+
"futures",
475
+
"http",
476
+
"indicatif",
477
+
"libc",
478
+
"log",
479
+
"native-tls",
480
+
"num_cpus",
481
+
"rand",
482
+
"reqwest",
483
+
"serde",
484
+
"serde_json",
485
+
"thiserror",
486
+
"tokio",
487
+
"ureq",
488
+
"windows-sys 0.60.2",
489
+
]
362
490
363
491
[[package]]
364
492
name = "http"
···
592
720
]
593
721
594
722
[[package]]
723
+
name = "indicatif"
724
+
version = "0.17.11"
725
+
source = "registry+https://github.com/rust-lang/crates.io-index"
726
+
checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
727
+
dependencies = [
728
+
"console",
729
+
"number_prefix",
730
+
"portable-atomic",
731
+
"unicode-width",
732
+
"web-time",
733
+
]
734
+
735
+
[[package]]
595
736
name = "ipnet"
596
737
version = "2.11.0"
597
738
source = "registry+https://github.com/rust-lang/crates.io-index"
···
636
777
checksum = "c5a2d376baa530d1238d133232d15e239abad80d05838b4b59354e5268af431f"
637
778
638
779
[[package]]
780
+
name = "libredox"
781
+
version = "0.1.12"
782
+
source = "registry+https://github.com/rust-lang/crates.io-index"
783
+
checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616"
784
+
dependencies = [
785
+
"bitflags",
786
+
"libc",
787
+
]
788
+
789
+
[[package]]
639
790
name = "linux-raw-sys"
640
791
version = "0.11.0"
641
792
source = "registry+https://github.com/rust-lang/crates.io-index"
···
666
817
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
667
818
668
819
[[package]]
820
+
name = "miniz_oxide"
821
+
version = "0.8.9"
822
+
source = "registry+https://github.com/rust-lang/crates.io-index"
823
+
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
824
+
dependencies = [
825
+
"adler2",
826
+
"simd-adler32",
827
+
]
828
+
829
+
[[package]]
669
830
name = "mio"
670
831
version = "1.1.1"
671
832
source = "registry+https://github.com/rust-lang/crates.io-index"
···
703
864
]
704
865
705
866
[[package]]
867
+
name = "num_cpus"
868
+
version = "1.17.0"
869
+
source = "registry+https://github.com/rust-lang/crates.io-index"
870
+
checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
871
+
dependencies = [
872
+
"hermit-abi",
873
+
"libc",
874
+
]
875
+
876
+
[[package]]
877
+
name = "number_prefix"
878
+
version = "0.4.0"
879
+
source = "registry+https://github.com/rust-lang/crates.io-index"
880
+
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
881
+
882
+
[[package]]
706
883
name = "once_cell"
707
884
version = "1.21.3"
708
885
source = "registry+https://github.com/rust-lang/crates.io-index"
···
759
936
]
760
937
761
938
[[package]]
939
+
name = "option-ext"
940
+
version = "0.2.0"
941
+
source = "registry+https://github.com/rust-lang/crates.io-index"
942
+
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
943
+
944
+
[[package]]
762
945
name = "owo-colors"
763
946
version = "4.2.3"
764
947
source = "registry+https://github.com/rust-lang/crates.io-index"
···
789
972
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
790
973
791
974
[[package]]
975
+
name = "portable-atomic"
976
+
version = "1.13.0"
977
+
source = "registry+https://github.com/rust-lang/crates.io-index"
978
+
checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950"
979
+
980
+
[[package]]
792
981
name = "potential_utf"
793
982
version = "0.1.4"
794
983
source = "registry+https://github.com/rust-lang/crates.io-index"
···
798
987
]
799
988
800
989
[[package]]
990
+
name = "ppv-lite86"
991
+
version = "0.2.21"
992
+
source = "registry+https://github.com/rust-lang/crates.io-index"
993
+
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
994
+
dependencies = [
995
+
"zerocopy",
996
+
]
997
+
998
+
[[package]]
801
999
name = "proc-macro2"
802
1000
version = "1.0.104"
803
1001
source = "registry+https://github.com/rust-lang/crates.io-index"
···
820
1018
version = "5.3.0"
821
1019
source = "registry+https://github.com/rust-lang/crates.io-index"
822
1020
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
1021
+
1022
+
[[package]]
1023
+
name = "rand"
1024
+
version = "0.9.2"
1025
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1026
+
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
1027
+
dependencies = [
1028
+
"rand_chacha",
1029
+
"rand_core",
1030
+
]
1031
+
1032
+
[[package]]
1033
+
name = "rand_chacha"
1034
+
version = "0.9.0"
1035
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1036
+
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
1037
+
dependencies = [
1038
+
"ppv-lite86",
1039
+
"rand_core",
1040
+
]
1041
+
1042
+
[[package]]
1043
+
name = "rand_core"
1044
+
version = "0.9.3"
1045
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1046
+
checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
1047
+
dependencies = [
1048
+
"getrandom 0.3.4",
1049
+
]
1050
+
1051
+
[[package]]
1052
+
name = "redox_users"
1053
+
version = "0.5.2"
1054
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1055
+
checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
1056
+
dependencies = [
1057
+
"getrandom 0.2.16",
1058
+
"libredox",
1059
+
"thiserror",
1060
+
]
823
1061
824
1062
[[package]]
825
1063
name = "reqwest"
···
898
1136
source = "registry+https://github.com/rust-lang/crates.io-index"
899
1137
checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
900
1138
dependencies = [
1139
+
"log",
901
1140
"once_cell",
1141
+
"ring",
902
1142
"rustls-pki-types",
903
1143
"rustls-webpki",
904
1144
"subtle",
···
1031
1271
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
1032
1272
1033
1273
[[package]]
1274
+
name = "simd-adler32"
1275
+
version = "0.3.8"
1276
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1277
+
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
1278
+
1279
+
[[package]]
1034
1280
name = "slab"
1035
1281
version = "0.4.11"
1036
1282
source = "registry+https://github.com/rust-lang/crates.io-index"
···
1050
1296
dependencies = [
1051
1297
"libc",
1052
1298
"windows-sys 0.60.2",
1299
+
]
1300
+
1301
+
[[package]]
1302
+
name = "socks"
1303
+
version = "0.3.4"
1304
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1305
+
checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
1306
+
dependencies = [
1307
+
"byteorder",
1308
+
"libc",
1309
+
"winapi",
1053
1310
]
1054
1311
1055
1312
[[package]]
···
1136
1393
]
1137
1394
1138
1395
[[package]]
1396
+
name = "thiserror"
1397
+
version = "2.0.17"
1398
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1399
+
checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
1400
+
dependencies = [
1401
+
"thiserror-impl",
1402
+
]
1403
+
1404
+
[[package]]
1405
+
name = "thiserror-impl"
1406
+
version = "2.0.17"
1407
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1408
+
checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
1409
+
dependencies = [
1410
+
"proc-macro2",
1411
+
"quote",
1412
+
"syn",
1413
+
]
1414
+
1415
+
[[package]]
1139
1416
name = "tilekit"
1140
1417
version = "0.1.0"
1141
1418
dependencies = [
···
1149
1426
"anyhow",
1150
1427
"clap",
1151
1428
"futures-util",
1429
+
"hf-hub",
1152
1430
"owo-colors",
1153
1431
"reqwest",
1154
1432
"serde",
···
1303
1581
checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
1304
1582
1305
1583
[[package]]
1584
+
name = "unicode-width"
1585
+
version = "0.2.2"
1586
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1587
+
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
1588
+
1589
+
[[package]]
1306
1590
name = "untrusted"
1307
1591
version = "0.9.0"
1308
1592
source = "registry+https://github.com/rust-lang/crates.io-index"
1309
1593
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
1310
1594
1311
1595
[[package]]
1596
+
name = "ureq"
1597
+
version = "2.12.1"
1598
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1599
+
checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d"
1600
+
dependencies = [
1601
+
"base64",
1602
+
"flate2",
1603
+
"log",
1604
+
"native-tls",
1605
+
"once_cell",
1606
+
"rustls",
1607
+
"rustls-pki-types",
1608
+
"serde",
1609
+
"serde_json",
1610
+
"socks",
1611
+
"url",
1612
+
"webpki-roots 0.26.11",
1613
+
]
1614
+
1615
+
[[package]]
1312
1616
name = "url"
1313
1617
version = "2.5.7"
1314
1618
source = "registry+https://github.com/rust-lang/crates.io-index"
···
1444
1748
]
1445
1749
1446
1750
[[package]]
1751
+
name = "web-time"
1752
+
version = "1.1.0"
1753
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1754
+
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
1755
+
dependencies = [
1756
+
"js-sys",
1757
+
"wasm-bindgen",
1758
+
]
1759
+
1760
+
[[package]]
1761
+
name = "webpki-roots"
1762
+
version = "0.26.11"
1763
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1764
+
checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9"
1765
+
dependencies = [
1766
+
"webpki-roots 1.0.5",
1767
+
]
1768
+
1769
+
[[package]]
1770
+
name = "webpki-roots"
1771
+
version = "1.0.5"
1772
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1773
+
checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c"
1774
+
dependencies = [
1775
+
"rustls-pki-types",
1776
+
]
1777
+
1778
+
[[package]]
1779
+
name = "winapi"
1780
+
version = "0.3.9"
1781
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1782
+
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
1783
+
dependencies = [
1784
+
"winapi-i686-pc-windows-gnu",
1785
+
"winapi-x86_64-pc-windows-gnu",
1786
+
]
1787
+
1788
+
[[package]]
1789
+
name = "winapi-i686-pc-windows-gnu"
1790
+
version = "0.4.0"
1791
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1792
+
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
1793
+
1794
+
[[package]]
1795
+
name = "winapi-x86_64-pc-windows-gnu"
1796
+
version = "0.4.0"
1797
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1798
+
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
1799
+
1800
+
[[package]]
1447
1801
name = "windows-link"
1448
1802
version = "0.2.1"
1449
1803
source = "registry+https://github.com/rust-lang/crates.io-index"
···
1483
1837
version = "0.52.0"
1484
1838
source = "registry+https://github.com/rust-lang/crates.io-index"
1485
1839
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
1840
+
dependencies = [
1841
+
"windows-targets 0.52.6",
1842
+
]
1843
+
1844
+
[[package]]
1845
+
name = "windows-sys"
1846
+
version = "0.59.0"
1847
+
source = "registry+https://github.com/rust-lang/crates.io-index"
1848
+
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
1486
1849
dependencies = [
1487
1850
"windows-targets 0.52.6",
1488
1851
]
···
1667
2030
"quote",
1668
2031
"syn",
1669
2032
"synstructure",
2033
+
]
2034
+
2035
+
[[package]]
2036
+
name = "zerocopy"
2037
+
version = "0.8.31"
2038
+
source = "registry+https://github.com/rust-lang/crates.io-index"
2039
+
checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3"
2040
+
dependencies = [
2041
+
"zerocopy-derive",
2042
+
]
2043
+
2044
+
[[package]]
2045
+
name = "zerocopy-derive"
2046
+
version = "0.8.31"
2047
+
source = "registry+https://github.com/rust-lang/crates.io-index"
2048
+
checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a"
2049
+
dependencies = [
2050
+
"proc-macro2",
2051
+
"quote",
2052
+
"syn",
1670
2053
]
1671
2054
1672
2055
[[package]]
+2
-1
server/backend/mlx.py
+2
-1
server/backend/mlx.py
···
12
12
13
13
logger = logging.getLogger("app")
14
14
15
-
from typing import Any, Dict, List, Optional, Union
15
+
from typing import Any, Dict, Iterator, List, Optional, Union
16
16
17
17
_model_cache: Dict[str, MLXRunner] = {}
18
18
_default_max_tokens: Optional[int] = None # Use dynamic model-aware limits by default
···
181
181
def count_tokens(text: str) -> int:
182
182
"""Rough token count estimation."""
183
183
return int(len(text.split()) * 1.3) # Approximation, convert to int
184
+
+1
-1
tiles/Cargo.toml
+1
-1
tiles/Cargo.toml
+2
tiles/src/core/mod.rs
+2
tiles/src/core/mod.rs
+1
-1
tiles/src/lib.rs
+1
-1
tiles/src/lib.rs
+9
-7
tiles/src/main.rs
+9
-7
tiles/src/main.rs
···
1
1
use std::error::Error;
2
2
3
3
use clap::{Args, Parser, Subcommand};
4
-
use tiles::runtime::{build_runtime, RunArgs};
4
+
use tiles::runtime::{RunArgs, build_runtime};
5
5
mod commands;
6
6
#[derive(Debug, Parser)]
7
7
#[command(name = "tiles")]
···
31
31
32
32
#[derive(Debug, Args)]
33
33
struct RunFlags {
34
-
/// Number of chat retries before giving up (default: 6)
35
-
#[arg(short = 'r', long, default_value_t = 6)]
36
-
retry_count: u32,
37
-
34
+
/// Max times cli communicates with the model until it gets a proper reply for a user prompt
35
+
#[arg(short = 'r', long, default_value_t = 10)]
36
+
relay_count: u32,
38
37
// Future flags go here:
39
38
// #[arg(long, default_value_t = 6969)]
40
39
// port: u16,
···
61
60
let cli = Cli::parse();
62
61
let runtime = build_runtime();
63
62
match cli.command {
64
-
Commands::Run { modelfile_path, flags } => {
63
+
Commands::Run {
64
+
modelfile_path,
65
+
flags,
66
+
} => {
65
67
let run_args = RunArgs {
66
68
modelfile_path,
67
-
retry_count: flags.retry_count,
69
+
relay_count: flags.relay_count,
68
70
};
69
71
commands::run(&runtime, run_args).await;
70
72
}
+33
-31
tiles/src/runtime/mlx.rs
+33
-31
tiles/src/runtime/mlx.rs
···
1
+
use crate::runtime::RunArgs;
2
+
use crate::utils::hf_model_downloader::*;
1
3
use anyhow::{Context, Result};
2
4
use futures_util::StreamExt;
3
5
use owo_colors::OwoColorize;
···
12
14
use std::{io, process::Command};
13
15
use tilekit::modelfile::Modelfile;
14
16
use tokio::time::sleep;
15
-
16
17
pub struct MLXRuntime {}
17
18
18
19
impl MLXRuntime {}
···
37
38
const DEFAULT_MODELFILE: &str = "FROM driaforall/mem-agent-mlx-4bit";
38
39
39
40
// Parse modelfile
40
-
let modelfile_parse_result = if let Some(modelfile_str) = run_args.modelfile_path {
41
+
let modelfile_parse_result = if let Some(modelfile_str) = &run_args.modelfile_path {
41
42
tilekit::modelfile::parse_from_file(modelfile_str.as_str())
42
43
} else {
43
44
tilekit::modelfile::parse(DEFAULT_MODELFILE)
···
53
54
54
55
let model = modelfile.from.as_ref().unwrap();
55
56
if model.starts_with("driaforall/mem-agent") {
56
-
let _res = run_model_with_server(self, modelfile, run_args.retry_count).await;
57
+
let _res = run_model_with_server(self, modelfile, &run_args).await;
57
58
} else {
58
59
run_model_by_sub_process(modelfile);
59
60
}
···
172
173
async fn run_model_with_server(
173
174
mlx_runtime: &MLXRuntime,
174
175
modelfile: Modelfile,
175
-
retry_count: u32,
176
+
run_args: &RunArgs,
176
177
) -> reqwest::Result<()> {
177
178
if !cfg!(debug_assertions) {
178
179
let _res = mlx_runtime.start_server_daemon().await;
179
180
let _ = wait_until_server_is_up().await;
180
181
}
181
-
let stdin = io::stdin();
182
-
let mut stdout = io::stdout();
183
182
// loading the model from mem-agent via daemon server
184
183
let memory_path = get_memory_path()
185
184
.context("Retrieving memory_path failed")
186
185
.unwrap();
187
186
let modelname = modelfile.from.as_ref().unwrap();
188
-
load_model(modelname, &memory_path).await.unwrap();
187
+
match load_model(modelname, &memory_path).await {
188
+
Ok(_) => start_repl(mlx_runtime, modelname, run_args).await,
189
+
Err(err) => println!("{}", err),
190
+
}
191
+
Ok(())
192
+
}
193
+
194
+
async fn start_repl(mlx_runtime: &MLXRuntime, modelname: &str, run_args: &RunArgs) {
195
+
let stdin = io::stdin();
196
+
let mut stdout = io::stdout();
189
197
println!("Running in interactive mode");
190
198
// TODO: Handle "enter" key press or any key press when repl is processing an input
191
199
loop {
···
203
211
break;
204
212
}
205
213
_ => {
206
-
let mut remaining_count = retry_count;
214
+
let mut remaining_count = run_args.relay_count;
207
215
let mut g_reply: String = "".to_owned();
208
216
let mut python_code: String = "".to_owned();
209
217
loop {
210
218
if remaining_count > 0 {
211
-
let chat_start = remaining_count == retry_count;
219
+
let chat_start = remaining_count == run_args.relay_count;
212
220
if let Ok(response) = chat(input, modelname, chat_start, &python_code).await
213
221
{
214
222
if response.reply.is_empty() {
···
233
241
}
234
242
}
235
243
}
236
-
Ok(())
237
244
}
238
245
239
246
async fn ping() -> Result<(), String> {
···
252
259
"model": model_name,
253
260
"memory_path": memory_path
254
261
});
262
+
263
+
//TODO: Fix the unwrap here
255
264
let res = client
256
265
.post("http://127.0.0.1:6969/start")
257
266
.json(&body)
···
260
269
.unwrap();
261
270
match res.status() {
262
271
StatusCode::OK => Ok(()),
263
-
StatusCode::NOT_FOUND => download_model(model_name).await,
272
+
StatusCode::NOT_FOUND => {
273
+
println!("Downloading {}\n", model_name);
274
+
match pull_model(model_name).await {
275
+
Ok(_) => {
276
+
println!("\nDownloading completed \n");
277
+
Ok(())
278
+
}
279
+
Err(err) => Err(err),
280
+
}
281
+
}
264
282
_ => {
265
283
println!("err {:?}", res);
266
-
Ok(())
284
+
Err(format!(
285
+
"Failed to load model {} due to {:?}",
286
+
model_name, res
287
+
))
267
288
}
268
-
}
269
-
}
270
-
271
-
async fn download_model(model_name: &str) -> Result<(), String> {
272
-
println!("Downloading the model {} ....", model_name);
273
-
let client = Client::new();
274
-
let body = json!({
275
-
"model": model_name
276
-
});
277
-
let res = client
278
-
.post("http://127.0.0.1:6969/download")
279
-
.json(&body)
280
-
.send()
281
-
.await
282
-
.unwrap();
283
-
if res.status() == 200 {
284
-
Ok(())
285
-
} else {
286
-
Err(String::from("Downloading model failed"))
287
289
}
288
290
}
289
291
+1
-1
tiles/src/runtime/mod.rs
+1
-1
tiles/src/runtime/mod.rs
+84
tiles/src/utils/hf_model_downloader.rs
+84
tiles/src/utils/hf_model_downloader.rs
···
1
+
/// Manages model snapshot downloading from HuggingFace
2
+
use std::{env, path::PathBuf};
3
+
4
+
use hf_hub::api::{
5
+
Siblings,
6
+
tokio::{ApiBuilder, ApiError},
7
+
};
8
+
9
+
/// Download the entire model (including snapshot) for the given model name
10
+
pub async fn pull_model(model_name: &str) -> Result<(), String> {
11
+
snapshot_download(model_name).await
12
+
}
13
+
14
+
pub async fn snapshot_download(modelname: &str) -> Result<(), String> {
15
+
let allow_patterns = [
16
+
".json",
17
+
".txt",
18
+
".safetensors",
19
+
".md",
20
+
".gitattributes",
21
+
"LICENSE",
22
+
];
23
+
let api_build_result = ApiBuilder::new()
24
+
.with_progress(true)
25
+
.with_cache_dir(PathBuf::from(get_model_cache()))
26
+
.build();
27
+
28
+
match api_build_result {
29
+
Ok(api) => {
30
+
let repo = api.model(modelname.to_owned());
31
+
match repo.info().await {
32
+
Ok(repo_info) => {
33
+
let filtered_siblings = repo_info
34
+
.siblings
35
+
.iter()
36
+
.filter(|sibling| {
37
+
allow_patterns
38
+
.iter()
39
+
.any(|pat| sibling.rfilename.ends_with(pat))
40
+
})
41
+
.collect::<Vec<&Siblings>>();
42
+
43
+
for sibling in filtered_siblings {
44
+
if repo.get(&sibling.rfilename).await.is_err() {
45
+
return Err(format!(
46
+
"{:?} failed to download, retry again",
47
+
&sibling.rfilename,
48
+
));
49
+
}
50
+
}
51
+
}
52
+
Err(err) => return Err(format_hf_api_error(err)),
53
+
};
54
+
}
55
+
Err(err) => return Err(format_hf_api_error(err)),
56
+
}
57
+
58
+
Ok(())
59
+
}
60
+
61
+
fn format_hf_api_error(api_error: ApiError) -> String {
62
+
match api_error {
63
+
ApiError::RequestError(err) => err.to_string(),
64
+
ApiError::TooManyRetries(err) => err.to_string(),
65
+
_err => "Something unexpected happened, check your internet connection".to_owned(),
66
+
}
67
+
}
68
+
69
+
fn get_model_cache() -> String {
70
+
let default_cache = format!(
71
+
"{}/.cache/huggingface",
72
+
env::home_dir().unwrap().to_str().unwrap()
73
+
);
74
+
let cache_root = if let Ok(home) = env::var("HF_HOME") {
75
+
home.to_owned()
76
+
} else {
77
+
default_cache
78
+
};
79
+
80
+
format!("{}/hub", cache_root)
81
+
}
82
+
83
+
#[cfg(test)]
84
+
mod tests {}
+1
tiles/src/utils/mod.rs
+1
tiles/src/utils/mod.rs
···
1
+
pub mod hf_model_downloader;