A local-first private AI assistant for everyday use. Runs on-device models with encrypted P2P sync, and supports sharing chats publicly on ATProto.
10
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat: Added encoding decoding of delta chats + e2e tests for sync

madclaws 18a068d1 d74caebb

+1138 -78
+593 -2
Cargo.lock
··· 409 409 checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" 410 410 411 411 [[package]] 412 + name = "asn1-rs" 413 + version = "0.7.1" 414 + source = "registry+https://github.com/rust-lang/crates.io-index" 415 + checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" 416 + dependencies = [ 417 + "asn1-rs-derive", 418 + "asn1-rs-impl", 419 + "displaydoc", 420 + "nom 7.1.3", 421 + "num-traits", 422 + "rusticata-macros", 423 + "thiserror 2.0.18", 424 + "time", 425 + ] 426 + 427 + [[package]] 428 + name = "asn1-rs-derive" 429 + version = "0.6.0" 430 + source = "registry+https://github.com/rust-lang/crates.io-index" 431 + checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" 432 + dependencies = [ 433 + "proc-macro2", 434 + "quote", 435 + "syn 2.0.117", 436 + "synstructure", 437 + ] 438 + 439 + [[package]] 440 + name = "asn1-rs-impl" 441 + version = "0.2.0" 442 + source = "registry+https://github.com/rust-lang/crates.io-index" 443 + checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" 444 + dependencies = [ 445 + "proc-macro2", 446 + "quote", 447 + "syn 2.0.117", 448 + ] 449 + 450 + [[package]] 412 451 name = "assert-json-diff" 413 452 version = "2.0.2" 414 453 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 712 751 ] 713 752 714 753 [[package]] 754 + name = "bao-tree" 755 + version = "0.16.0" 756 + source = "registry+https://github.com/rust-lang/crates.io-index" 757 + checksum = "06384416b1825e6e04fde63262fda2dc408f5b64c02d04e0d8b70ae72c17a52b" 758 + dependencies = [ 759 + "blake3", 760 + "bytes", 761 + "futures-lite", 762 + "genawaiter", 763 + "iroh-io", 764 + "positioned-io", 765 + "range-collections", 766 + "self_cell", 767 + "serde", 768 + "smallvec", 769 + "tokio", 770 + ] 771 + 772 + [[package]] 715 773 name = "base-x" 716 774 version = "0.2.11" 717 775 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 756 814 version = "1.8.3" 757 815 source = "registry+https://github.com/rust-lang/crates.io-index" 758 816 checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" 817 + 818 + [[package]] 819 + name = "binary-merge" 820 + version = "0.1.2" 821 + source = "registry+https://github.com/rust-lang/crates.io-index" 822 + checksum = "597bb81c80a54b6a4381b23faba8d7774b144c94cbd1d6fe3f1329bd776554ab" 759 823 760 824 [[package]] 761 825 name = "bincode" ··· 915 979 ] 916 980 917 981 [[package]] 982 + name = "cesu8" 983 + version = "1.1.0" 984 + source = "registry+https://github.com/rust-lang/crates.io-index" 985 + checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" 986 + 987 + [[package]] 918 988 name = "cfg-if" 919 989 version = "1.0.4" 920 990 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 933 1003 checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" 934 1004 dependencies = [ 935 1005 "iana-time-zone", 1006 + "js-sys", 936 1007 "num-traits", 937 1008 "serde", 1009 + "wasm-bindgen", 938 1010 "windows-link", 939 1011 ] 940 1012 ··· 1036 1108 checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" 1037 1109 1038 1110 [[package]] 1111 + name = "combine" 1112 + version = "4.6.7" 1113 + source = "registry+https://github.com/rust-lang/crates.io-index" 1114 + checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" 1115 + dependencies = [ 1116 + "bytes", 1117 + "memchr", 1118 + ] 1119 + 1120 + [[package]] 1039 1121 name = "concurrent-queue" 1040 1122 version = "2.5.0" 1041 1123 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1507 1589 ] 1508 1590 1509 1591 [[package]] 1592 + name = "der-parser" 1593 + version = "10.0.0" 1594 + source = "registry+https://github.com/rust-lang/crates.io-index" 1595 + checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" 1596 + dependencies = [ 1597 + "asn1-rs", 1598 + "displaydoc", 1599 + "nom 7.1.3", 1600 + "num-bigint", 1601 + "num-traits", 1602 + "rusticata-macros", 1603 + ] 1604 + 1605 + [[package]] 1510 1606 name = "deranged" 1511 1607 version = "0.5.8" 1512 1608 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1895 1991 1896 1992 [[package]] 1897 1993 name = "env_logger" 1898 - version = "0.11.9" 1994 + version = "0.11.10" 1899 1995 source = "registry+https://github.com/rust-lang/crates.io-index" 1900 - checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" 1996 + checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" 1901 1997 dependencies = [ 1998 + "anstream", 1999 + "anstyle", 1902 2000 "env_filter", 2001 + "jiff", 1903 2002 "log", 1904 2003 ] 1905 2004 ··· 2374 2473 ] 2375 2474 2376 2475 [[package]] 2476 + name = "genawaiter" 2477 + version = "0.99.1" 2478 + source = "registry+https://github.com/rust-lang/crates.io-index" 2479 + checksum = "c86bd0361bcbde39b13475e6e36cb24c329964aa2611be285289d1e4b751c1a0" 2480 + dependencies = [ 2481 + "futures-core", 2482 + "genawaiter-macro", 2483 + "genawaiter-proc-macro", 2484 + "proc-macro-hack", 2485 + ] 2486 + 2487 + [[package]] 2488 + name = "genawaiter-macro" 2489 + version = "0.99.1" 2490 + source = "registry+https://github.com/rust-lang/crates.io-index" 2491 + checksum = "0b32dfe1fdfc0bbde1f22a5da25355514b5e450c33a6af6770884c8750aedfbc" 2492 + 2493 + [[package]] 2494 + name = "genawaiter-proc-macro" 2495 + version = "0.99.1" 2496 + source = "registry+https://github.com/rust-lang/crates.io-index" 2497 + checksum = "784f84eebc366e15251c4a8c3acee82a6a6f427949776ecb88377362a9621738" 2498 + dependencies = [ 2499 + "proc-macro-error", 2500 + "proc-macro-hack", 2501 + "proc-macro2", 2502 + "quote", 2503 + "syn 1.0.109", 2504 + ] 2505 + 2506 + [[package]] 2377 2507 name = "generator" 2378 2508 version = "0.8.8" 2379 2509 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3014 3144 ] 3015 3145 3016 3146 [[package]] 3147 + name = "inplace-vec-builder" 3148 + version = "0.1.1" 3149 + source = "registry+https://github.com/rust-lang/crates.io-index" 3150 + checksum = "cf64c2edc8226891a71f127587a2861b132d2b942310843814d5001d99a1d307" 3151 + dependencies = [ 3152 + "smallvec", 3153 + ] 3154 + 3155 + [[package]] 3017 3156 name = "integer-encoding" 3018 3157 version = "3.0.4" 3019 3158 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3145 3284 ] 3146 3285 3147 3286 [[package]] 3287 + name = "iroh-blobs" 3288 + version = "0.99.0" 3289 + source = "registry+https://github.com/rust-lang/crates.io-index" 3290 + checksum = "51b06914e77bd07bc1b3600096be66e2a63d391e8f4a901f61771630e20f2116" 3291 + dependencies = [ 3292 + "arrayvec", 3293 + "bao-tree", 3294 + "bytes", 3295 + "cfg_aliases", 3296 + "chrono", 3297 + "data-encoding", 3298 + "derive_more", 3299 + "futures-lite", 3300 + "genawaiter", 3301 + "hex", 3302 + "iroh", 3303 + "iroh-base", 3304 + "iroh-io", 3305 + "iroh-metrics", 3306 + "iroh-tickets", 3307 + "irpc", 3308 + "n0-error", 3309 + "n0-future", 3310 + "nested_enum_utils", 3311 + "noq", 3312 + "postcard", 3313 + "rand 0.9.2", 3314 + "range-collections", 3315 + "redb", 3316 + "ref-cast", 3317 + "reflink-copy", 3318 + "self_cell", 3319 + "serde", 3320 + "smallvec", 3321 + "tokio", 3322 + "tracing", 3323 + ] 3324 + 3325 + [[package]] 3148 3326 name = "iroh-gossip" 3149 3327 version = "0.97.0" 3150 3328 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3175 3353 ] 3176 3354 3177 3355 [[package]] 3356 + name = "iroh-io" 3357 + version = "0.6.2" 3358 + source = "registry+https://github.com/rust-lang/crates.io-index" 3359 + checksum = "e0a5feb781017b983ff1b155cd1faf8174da2acafd807aa482876da2d7e6577a" 3360 + dependencies = [ 3361 + "bytes", 3362 + "futures-lite", 3363 + "pin-project", 3364 + "smallvec", 3365 + "tokio", 3366 + ] 3367 + 3368 + [[package]] 3178 3369 name = "iroh-metrics" 3179 3370 version = "0.38.3" 3180 3371 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3269 3460 source = "registry+https://github.com/rust-lang/crates.io-index" 3270 3461 checksum = "4f47b7c52662d673df377b5ac40c121c7ff56eb764e520fae6543686132f7957" 3271 3462 dependencies = [ 3463 + "futures-buffered", 3272 3464 "futures-util", 3273 3465 "irpc-derive", 3274 3466 "n0-error", 3275 3467 "n0-future", 3468 + "noq", 3469 + "postcard", 3470 + "rcgen", 3471 + "rustls", 3276 3472 "serde", 3473 + "smallvec", 3277 3474 "tokio", 3278 3475 "tokio-util", 3279 3476 "tracing", ··· 3312 3509 checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" 3313 3510 3314 3511 [[package]] 3512 + name = "jiff" 3513 + version = "0.2.23" 3514 + source = "registry+https://github.com/rust-lang/crates.io-index" 3515 + checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" 3516 + dependencies = [ 3517 + "jiff-static", 3518 + "log", 3519 + "portable-atomic", 3520 + "portable-atomic-util", 3521 + "serde_core", 3522 + ] 3523 + 3524 + [[package]] 3525 + name = "jiff-static" 3526 + version = "0.2.23" 3527 + source = "registry+https://github.com/rust-lang/crates.io-index" 3528 + checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" 3529 + dependencies = [ 3530 + "proc-macro2", 3531 + "quote", 3532 + "syn 2.0.117", 3533 + ] 3534 + 3535 + [[package]] 3536 + name = "jni" 3537 + version = "0.21.1" 3538 + source = "registry+https://github.com/rust-lang/crates.io-index" 3539 + checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" 3540 + dependencies = [ 3541 + "cesu8", 3542 + "cfg-if", 3543 + "combine", 3544 + "jni-sys 0.3.1", 3545 + "log", 3546 + "thiserror 1.0.69", 3547 + "walkdir", 3548 + "windows-sys 0.45.0", 3549 + ] 3550 + 3551 + [[package]] 3552 + name = "jni-sys" 3553 + version = "0.3.1" 3554 + source = "registry+https://github.com/rust-lang/crates.io-index" 3555 + checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" 3556 + dependencies = [ 3557 + "jni-sys 0.4.1", 3558 + ] 3559 + 3560 + [[package]] 3561 + name = "jni-sys" 3562 + version = "0.4.1" 3563 + source = "registry+https://github.com/rust-lang/crates.io-index" 3564 + checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" 3565 + dependencies = [ 3566 + "jni-sys-macros", 3567 + ] 3568 + 3569 + [[package]] 3570 + name = "jni-sys-macros" 3571 + version = "0.4.1" 3572 + source = "registry+https://github.com/rust-lang/crates.io-index" 3573 + checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" 3574 + dependencies = [ 3575 + "quote", 3576 + "syn 2.0.117", 3577 + ] 3578 + 3579 + [[package]] 3315 3580 name = "jobserver" 3316 3581 version = "0.1.34" 3317 3582 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3860 4125 ] 3861 4126 3862 4127 [[package]] 4128 + name = "nested_enum_utils" 4129 + version = "0.2.3" 4130 + source = "registry+https://github.com/rust-lang/crates.io-index" 4131 + checksum = "b1d5475271bdd36a4a2769eac1ef88df0f99428ea43e52dfd8b0ee5cb674695f" 4132 + dependencies = [ 4133 + "proc-macro-crate", 4134 + "proc-macro2", 4135 + "quote", 4136 + "syn 2.0.117", 4137 + ] 4138 + 4139 + [[package]] 3863 4140 name = "netdev" 3864 4141 version = "0.40.1" 3865 4142 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4063 4340 "rustc-hash", 4064 4341 "rustls", 4065 4342 "rustls-pki-types", 4343 + "rustls-platform-verifier", 4066 4344 "slab", 4067 4345 "sorted-index-buffer", 4068 4346 "thiserror 2.0.18", ··· 4289 4567 ] 4290 4568 4291 4569 [[package]] 4570 + name = "oid-registry" 4571 + version = "0.8.1" 4572 + source = "registry+https://github.com/rust-lang/crates.io-index" 4573 + checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" 4574 + dependencies = [ 4575 + "asn1-rs", 4576 + ] 4577 + 4578 + [[package]] 4292 4579 name = "once_cell" 4293 4580 version = "1.21.4" 4294 4581 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4481 4768 checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" 4482 4769 4483 4770 [[package]] 4771 + name = "pem" 4772 + version = "3.0.6" 4773 + source = "registry+https://github.com/rust-lang/crates.io-index" 4774 + checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" 4775 + dependencies = [ 4776 + "base64", 4777 + "serde_core", 4778 + ] 4779 + 4780 + [[package]] 4484 4781 name = "pem-rfc7468" 4485 4782 version = "0.7.0" 4486 4783 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4651 4948 ] 4652 4949 4653 4950 [[package]] 4951 + name = "portable-atomic-util" 4952 + version = "0.2.6" 4953 + source = "registry+https://github.com/rust-lang/crates.io-index" 4954 + checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" 4955 + dependencies = [ 4956 + "portable-atomic", 4957 + ] 4958 + 4959 + [[package]] 4654 4960 name = "portmapper" 4655 4961 version = "0.15.0" 4656 4962 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4678 4984 "tower-layer", 4679 4985 "tracing", 4680 4986 "url", 4987 + ] 4988 + 4989 + [[package]] 4990 + name = "positioned-io" 4991 + version = "0.3.5" 4992 + source = "registry+https://github.com/rust-lang/crates.io-index" 4993 + checksum = "d4ec4b80060f033312b99b6874025d9503d2af87aef2dd4c516e253fbfcdada7" 4994 + dependencies = [ 4995 + "libc", 4996 + "winapi", 4681 4997 ] 4682 4998 4683 4999 [[package]] ··· 4758 5074 ] 4759 5075 4760 5076 [[package]] 5077 + name = "proc-macro-error" 5078 + version = "0.4.12" 5079 + source = "registry+https://github.com/rust-lang/crates.io-index" 5080 + checksum = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" 5081 + dependencies = [ 5082 + "proc-macro-error-attr", 5083 + "proc-macro2", 5084 + "quote", 5085 + "syn 1.0.109", 5086 + "version_check", 5087 + ] 5088 + 5089 + [[package]] 5090 + name = "proc-macro-error-attr" 5091 + version = "0.4.12" 5092 + source = "registry+https://github.com/rust-lang/crates.io-index" 5093 + checksum = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" 5094 + dependencies = [ 5095 + "proc-macro2", 5096 + "quote", 5097 + "syn 1.0.109", 5098 + "syn-mid", 5099 + "version_check", 5100 + ] 5101 + 5102 + [[package]] 5103 + name = "proc-macro-hack" 5104 + version = "0.5.20+deprecated" 5105 + source = "registry+https://github.com/rust-lang/crates.io-index" 5106 + checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" 5107 + 5108 + [[package]] 4761 5109 name = "proc-macro2" 4762 5110 version = "1.0.106" 4763 5111 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4957 5305 ] 4958 5306 4959 5307 [[package]] 5308 + name = "range-collections" 5309 + version = "0.4.6" 5310 + source = "registry+https://github.com/rust-lang/crates.io-index" 5311 + checksum = "861706ea9c4aded7584c5cd1d241cec2ea7f5f50999f236c22b65409a1f1a0d0" 5312 + dependencies = [ 5313 + "binary-merge", 5314 + "inplace-vec-builder", 5315 + "ref-cast", 5316 + "serde", 5317 + "smallvec", 5318 + ] 5319 + 5320 + [[package]] 4960 5321 name = "rayon" 4961 5322 version = "1.11.0" 4962 5323 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4977 5338 ] 4978 5339 4979 5340 [[package]] 5341 + name = "rcgen" 5342 + version = "0.14.7" 5343 + source = "registry+https://github.com/rust-lang/crates.io-index" 5344 + checksum = "10b99e0098aa4082912d4c649628623db6aba77335e4f4569ff5083a6448b32e" 5345 + dependencies = [ 5346 + "pem", 5347 + "ring", 5348 + "rustls-pki-types", 5349 + "time", 5350 + "x509-parser", 5351 + "yasna", 5352 + ] 5353 + 5354 + [[package]] 5355 + name = "redb" 5356 + version = "2.6.3" 5357 + source = "registry+https://github.com/rust-lang/crates.io-index" 5358 + checksum = "8eca1e9d98d5a7e9002d0013e18d5a9b000aee942eb134883a82f06ebffb6c01" 5359 + dependencies = [ 5360 + "libc", 5361 + ] 5362 + 5363 + [[package]] 4980 5364 name = "redox_syscall" 4981 5365 version = "0.5.18" 4982 5366 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5014 5398 "proc-macro2", 5015 5399 "quote", 5016 5400 "syn 2.0.117", 5401 + ] 5402 + 5403 + [[package]] 5404 + name = "reflink-copy" 5405 + version = "0.1.29" 5406 + source = "registry+https://github.com/rust-lang/crates.io-index" 5407 + checksum = "13362233b147e57674c37b802d216b7c5e3dcccbed8967c84f0d8d223868ae27" 5408 + dependencies = [ 5409 + "cfg-if", 5410 + "libc", 5411 + "rustix", 5412 + "windows", 5017 5413 ] 5018 5414 5019 5415 [[package]] ··· 5254 5650 ] 5255 5651 5256 5652 [[package]] 5653 + name = "rusticata-macros" 5654 + version = "4.1.0" 5655 + source = "registry+https://github.com/rust-lang/crates.io-index" 5656 + checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" 5657 + dependencies = [ 5658 + "nom 7.1.3", 5659 + ] 5660 + 5661 + [[package]] 5257 5662 name = "rustix" 5258 5663 version = "1.1.4" 5259 5664 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5282 5687 ] 5283 5688 5284 5689 [[package]] 5690 + name = "rustls-native-certs" 5691 + version = "0.8.3" 5692 + source = "registry+https://github.com/rust-lang/crates.io-index" 5693 + checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" 5694 + dependencies = [ 5695 + "openssl-probe", 5696 + "rustls-pki-types", 5697 + "schannel", 5698 + "security-framework 3.7.0", 5699 + ] 5700 + 5701 + [[package]] 5285 5702 name = "rustls-pki-types" 5286 5703 version = "1.14.0" 5287 5704 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5292 5709 ] 5293 5710 5294 5711 [[package]] 5712 + name = "rustls-platform-verifier" 5713 + version = "0.6.2" 5714 + source = "registry+https://github.com/rust-lang/crates.io-index" 5715 + checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" 5716 + dependencies = [ 5717 + "core-foundation 0.10.1", 5718 + "core-foundation-sys", 5719 + "jni", 5720 + "log", 5721 + "once_cell", 5722 + "rustls", 5723 + "rustls-native-certs", 5724 + "rustls-platform-verifier-android", 5725 + "rustls-webpki", 5726 + "security-framework 3.7.0", 5727 + "security-framework-sys", 5728 + "webpki-root-certs", 5729 + "windows-sys 0.61.2", 5730 + ] 5731 + 5732 + [[package]] 5733 + name = "rustls-platform-verifier-android" 5734 + version = "0.1.1" 5735 + source = "registry+https://github.com/rust-lang/crates.io-index" 5736 + checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" 5737 + 5738 + [[package]] 5295 5739 name = "rustls-webpki" 5296 5740 version = "0.103.10" 5297 5741 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5335 5779 version = "1.0.23" 5336 5780 source = "registry+https://github.com/rust-lang/crates.io-index" 5337 5781 checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" 5782 + 5783 + [[package]] 5784 + name = "same-file" 5785 + version = "1.0.6" 5786 + source = "registry+https://github.com/rust-lang/crates.io-index" 5787 + checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" 5788 + dependencies = [ 5789 + "winapi-util", 5790 + ] 5338 5791 5339 5792 [[package]] 5340 5793 name = "scc" ··· 5733 6186 version = "1.15.1" 5734 6187 source = "registry+https://github.com/rust-lang/crates.io-index" 5735 6188 checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" 6189 + dependencies = [ 6190 + "serde", 6191 + ] 5736 6192 5737 6193 [[package]] 5738 6194 name = "smol_str" ··· 5924 6380 ] 5925 6381 5926 6382 [[package]] 6383 + name = "syn-mid" 6384 + version = "0.5.4" 6385 + source = "registry+https://github.com/rust-lang/crates.io-index" 6386 + checksum = "fea305d57546cc8cd04feb14b62ec84bf17f50e3f7b12560d7bfa9265f39d9ed" 6387 + dependencies = [ 6388 + "proc-macro2", 6389 + "quote", 6390 + "syn 1.0.109", 6391 + ] 6392 + 6393 + [[package]] 5927 6394 name = "sync_wrapper" 5928 6395 version = "1.0.2" 5929 6396 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 6080 6547 "axum-macros", 6081 6548 "clap", 6082 6549 "data-encoding", 6550 + "env_logger", 6083 6551 "futures-util", 6084 6552 "hf-hub", 6085 6553 "iroh", 6554 + "iroh-blobs", 6086 6555 "iroh-gossip", 6087 6556 "iroh-tickets", 6088 6557 "keyring", 6558 + "log", 6089 6559 "owo-colors", 6090 6560 "postcard", 6091 6561 "reqwest", ··· 6708 7178 version = "0.9.5" 6709 7179 source = "registry+https://github.com/rust-lang/crates.io-index" 6710 7180 checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" 7181 + 7182 + [[package]] 7183 + name = "walkdir" 7184 + version = "2.5.0" 7185 + source = "registry+https://github.com/rust-lang/crates.io-index" 7186 + checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" 7187 + dependencies = [ 7188 + "same-file", 7189 + "winapi-util", 7190 + ] 6711 7191 6712 7192 [[package]] 6713 7193 name = "want" ··· 6869 7349 ] 6870 7350 6871 7351 [[package]] 7352 + name = "webpki-root-certs" 7353 + version = "1.0.6" 7354 + source = "registry+https://github.com/rust-lang/crates.io-index" 7355 + checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" 7356 + dependencies = [ 7357 + "rustls-pki-types", 7358 + ] 7359 + 7360 + [[package]] 6872 7361 name = "webpki-roots" 6873 7362 version = "0.26.11" 6874 7363 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 6907 7396 version = "0.4.0" 6908 7397 source = "registry+https://github.com/rust-lang/crates.io-index" 6909 7398 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 7399 + 7400 + [[package]] 7401 + name = "winapi-util" 7402 + version = "0.1.11" 7403 + source = "registry+https://github.com/rust-lang/crates.io-index" 7404 + checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" 7405 + dependencies = [ 7406 + "windows-sys 0.61.2", 7407 + ] 6910 7408 6911 7409 [[package]] 6912 7410 name = "winapi-x86_64-pc-windows-gnu" ··· 7028 7526 7029 7527 [[package]] 7030 7528 name = "windows-sys" 7529 + version = "0.45.0" 7530 + source = "registry+https://github.com/rust-lang/crates.io-index" 7531 + checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" 7532 + dependencies = [ 7533 + "windows-targets 0.42.2", 7534 + ] 7535 + 7536 + [[package]] 7537 + name = "windows-sys" 7031 7538 version = "0.48.0" 7032 7539 source = "registry+https://github.com/rust-lang/crates.io-index" 7033 7540 checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" ··· 7073 7580 7074 7581 [[package]] 7075 7582 name = "windows-targets" 7583 + version = "0.42.2" 7584 + source = "registry+https://github.com/rust-lang/crates.io-index" 7585 + checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" 7586 + dependencies = [ 7587 + "windows_aarch64_gnullvm 0.42.2", 7588 + "windows_aarch64_msvc 0.42.2", 7589 + "windows_i686_gnu 0.42.2", 7590 + "windows_i686_msvc 0.42.2", 7591 + "windows_x86_64_gnu 0.42.2", 7592 + "windows_x86_64_gnullvm 0.42.2", 7593 + "windows_x86_64_msvc 0.42.2", 7594 + ] 7595 + 7596 + [[package]] 7597 + name = "windows-targets" 7076 7598 version = "0.48.5" 7077 7599 source = "registry+https://github.com/rust-lang/crates.io-index" 7078 7600 checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" ··· 7130 7652 7131 7653 [[package]] 7132 7654 name = "windows_aarch64_gnullvm" 7655 + version = "0.42.2" 7656 + source = "registry+https://github.com/rust-lang/crates.io-index" 7657 + checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" 7658 + 7659 + [[package]] 7660 + name = "windows_aarch64_gnullvm" 7133 7661 version = "0.48.5" 7134 7662 source = "registry+https://github.com/rust-lang/crates.io-index" 7135 7663 checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" ··· 7148 7676 7149 7677 [[package]] 7150 7678 name = "windows_aarch64_msvc" 7679 + version = "0.42.2" 7680 + source = "registry+https://github.com/rust-lang/crates.io-index" 7681 + checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" 7682 + 7683 + [[package]] 7684 + name = "windows_aarch64_msvc" 7151 7685 version = "0.48.5" 7152 7686 source = "registry+https://github.com/rust-lang/crates.io-index" 7153 7687 checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" ··· 7166 7700 7167 7701 [[package]] 7168 7702 name = "windows_i686_gnu" 7703 + version = "0.42.2" 7704 + source = "registry+https://github.com/rust-lang/crates.io-index" 7705 + checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" 7706 + 7707 + [[package]] 7708 + name = "windows_i686_gnu" 7169 7709 version = "0.48.5" 7170 7710 source = "registry+https://github.com/rust-lang/crates.io-index" 7171 7711 checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" ··· 7196 7736 7197 7737 [[package]] 7198 7738 name = "windows_i686_msvc" 7739 + version = "0.42.2" 7740 + source = "registry+https://github.com/rust-lang/crates.io-index" 7741 + checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" 7742 + 7743 + [[package]] 7744 + name = "windows_i686_msvc" 7199 7745 version = "0.48.5" 7200 7746 source = "registry+https://github.com/rust-lang/crates.io-index" 7201 7747 checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" ··· 7214 7760 7215 7761 [[package]] 7216 7762 name = "windows_x86_64_gnu" 7763 + version = "0.42.2" 7764 + source = "registry+https://github.com/rust-lang/crates.io-index" 7765 + checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" 7766 + 7767 + [[package]] 7768 + name = "windows_x86_64_gnu" 7217 7769 version = "0.48.5" 7218 7770 source = "registry+https://github.com/rust-lang/crates.io-index" 7219 7771 checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" ··· 7232 7784 7233 7785 [[package]] 7234 7786 name = "windows_x86_64_gnullvm" 7787 + version = "0.42.2" 7788 + source = "registry+https://github.com/rust-lang/crates.io-index" 7789 + checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" 7790 + 7791 + [[package]] 7792 + name = "windows_x86_64_gnullvm" 7235 7793 version = "0.48.5" 7236 7794 source = "registry+https://github.com/rust-lang/crates.io-index" 7237 7795 checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" ··· 7247 7805 version = "0.53.1" 7248 7806 source = "registry+https://github.com/rust-lang/crates.io-index" 7249 7807 checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" 7808 + 7809 + [[package]] 7810 + name = "windows_x86_64_msvc" 7811 + version = "0.42.2" 7812 + source = "registry+https://github.com/rust-lang/crates.io-index" 7813 + checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" 7250 7814 7251 7815 [[package]] 7252 7816 name = "windows_x86_64_msvc" ··· 7443 8007 ] 7444 8008 7445 8009 [[package]] 8010 + name = "x509-parser" 8011 + version = "0.18.1" 8012 + source = "registry+https://github.com/rust-lang/crates.io-index" 8013 + checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" 8014 + dependencies = [ 8015 + "asn1-rs", 8016 + "data-encoding", 8017 + "der-parser", 8018 + "lazy_static", 8019 + "nom 7.1.3", 8020 + "oid-registry", 8021 + "ring", 8022 + "rusticata-macros", 8023 + "thiserror 2.0.18", 8024 + "time", 8025 + ] 8026 + 8027 + [[package]] 7446 8028 name = "xml-rs" 7447 8029 version = "0.8.28" 7448 8030 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 7455 8037 checksum = "d7d8a75eaf6557bb84a65ace8609883db44a29951042ada9b393151532e41fcb" 7456 8038 dependencies = [ 7457 8039 "xml-rs", 8040 + ] 8041 + 8042 + [[package]] 8043 + name = "yasna" 8044 + version = "0.5.2" 8045 + source = "registry+https://github.com/rust-lang/crates.io-index" 8046 + checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" 8047 + dependencies = [ 8048 + "time", 7458 8049 ] 7459 8050 7460 8051 [[package]]
+3
Cargo.toml
··· 5 5 "tiles", 6 6 ] 7 7 8 + 9 + [workspace.lints.clippy] 10 + pedantic = "warn"
+1 -1
tilekit/src/modelfile.rs
··· 38 38 } 39 39 } 40 40 41 - #[derive(Debug, Clone, Copy, serde::Serialize)] 41 + #[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)] 42 42 #[serde(rename_all = "lowercase")] 43 43 pub enum Role { 44 44 System,
+15 -16
tilekit/src/optimize.rs
··· 113 113 } 114 114 115 115 pub async fn optimize( 116 - modelfile_path: String, 116 + modelfile_path: &str, 117 117 data_path: Option<String>, 118 - model: String, 118 + model: &str, 119 119 ) -> Result<Modelfile, String> { 120 120 println!("Optimizing Modelfile: {}", modelfile_path); 121 121 ··· 136 136 println!("Current SYSTEM prompt: \"{}\"", system_prompt); 137 137 138 138 // 2. Configure DSRs 139 - let lm = LM::builder().model(model).build().await.map_err(|e| { 140 - format!( 141 - "Error configuring LM: {}. Make sure appropriate API keys are set.", 142 - e 143 - ) 144 - })?; 139 + let lm = LM::builder() 140 + .model(model.to_owned()) 141 + .build() 142 + .await 143 + .map_err(|e| { 144 + format!( 145 + "Error configuring LM: {}. Make sure appropriate API keys are set.", 146 + e 147 + ) 148 + })?; 145 149 146 150 configure(lm, ChatAdapter); 147 151 ··· 336 340 337 341 #[tokio::test] 338 342 async fn test_optimize_missing_file() { 339 - let result = optimize( 340 - "nonexistent_file.modelfile".to_string(), 341 - None, 342 - "openai:gpt-4o-mini".to_string(), 343 - ) 344 - .await; 343 + let result = optimize("nonexistent_file.modelfile", None, "openai:gpt-4o-mini").await; 345 344 346 345 assert!(result.is_err()); 347 346 assert!(result.unwrap_err().contains("Error reading Modelfile")); ··· 355 354 std::fs::write(&temp_file, "FROM llama3.2\n").unwrap(); 356 355 357 356 let result = optimize( 358 - temp_file.to_string_lossy().to_string(), 357 + temp_file.to_str().expect("File path is invalid"), 359 358 None, 360 - "openai:gpt-4o-mini".to_string(), 359 + "openai:gpt-4o-mini", 361 360 ) 362 361 .await; 363 362
+3
tiles/Cargo.toml
··· 28 28 postcard = "1.1.3" 29 29 data-encoding = "2.10.0" 30 30 sha2 = "0.10.9" 31 + iroh-blobs = "0.99.0" 32 + log = "0.4.29" 33 + env_logger = "0.11.10" 31 34 32 35 [dev-dependencies] 33 36 tempfile = "3"
+228 -11
tiles/src/core/chats.rs
··· 33 33 pub content: String, 34 34 } 35 35 36 - #[derive(Debug)] 36 + #[derive(Debug, serde::Serialize, serde::Deserialize)] 37 37 pub struct Chats { 38 38 pub id: String, 39 39 content: String, ··· 122 122 Err(err) => Err(<rusqlite::Error as Into<anyhow::Error>>::into(err)), 123 123 } 124 124 } 125 - /// Return list of rows.. 126 - /// encoding is the job of network modules 125 + /// Return list of rows for the given `user_id` since `last_row_counter` 127 126 pub fn get_delta(conn: &Connection, user_id: &str, last_row_couter: i64) -> Result<Vec<Chats>> { 128 127 let mut stmt = conn.prepare("select id, user_id, content, resp_id, role, context_id, created_at, updated_at , row_counter from chats where user_id = ?1 and row_counter > ?2 order by id")?; 129 128 ··· 156 155 Ok(chats) 157 156 } 158 157 159 - pub fn apply_delta(chat_conn: &mut Connection, delta_chats: Vec<Chats>) -> Result<()> { 160 - // bulk insert 161 - // TODO: Handle primary key conflict, for now upsert it, later 162 - // do LWW based on iss of UCAN 158 + pub fn apply_delta(chat_conn: &mut Connection, delta_chats: &Vec<Chats>) -> Result<()> { 159 + // TODO: Handle primary key conflict, for now reject it (in a way its impossible to have this scenario, and if its occuring then that means 160 + // some issue in syncing, so ignore it, by rejecting it), later 161 + // do LWW based on issuer of UCAN 163 162 let txn = chat_conn.transaction()?; 164 163 { 165 164 let mut stmt = txn.prepare("insert into chats(id, user_id, content, resp_id, role, context_id, created_at, updated_at, row_counter) values (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)")?; 166 165 167 166 for chat in delta_chats { 168 - stmt.execute(params![ 167 + match stmt.execute(params![ 169 168 &chat.id.to_string(), 170 169 &chat.user_id, 171 170 &chat.content, ··· 175 174 &chat.created_at.to_string(), 176 175 &chat.updated_at.to_string(), 177 176 &chat.row_counter, 178 - ])?; 177 + ]) { 178 + Err(rusqlite::Error::SqliteFailure(_, Some(reason))) 179 + if reason == "UNIQUE constraint failed: chats.id" => 180 + { 181 + log::error!( 182 + "err in writing row {:?}, already exists, skipping", 183 + &chat.id 184 + ); 185 + } 186 + _ => (), 187 + } 179 188 } 180 189 } 181 190 txn.commit()?; ··· 183 192 Ok(()) 184 193 } 185 194 195 + pub fn get_encoded_delta( 196 + conn: &Connection, 197 + user_id: &str, 198 + last_row_couter: i64, 199 + ) -> Result<Vec<u8>> { 200 + let delta = get_delta(conn, user_id, last_row_couter)?; 201 + Ok(encode_delta_to_bytes(&delta)) 202 + } 203 + 204 + fn encode_delta_to_bytes(delta_chats: &Vec<Chats>) -> Vec<u8> { 205 + postcard::to_stdvec(delta_chats).expect("Failed to convert to bytes with postcard") 206 + } 207 + 208 + fn decode_delta_from_bytes(bytes: &[u8]) -> Result<Vec<Chats>> { 209 + postcard::from_bytes(bytes).map_err(Into::into) 210 + } 211 + 186 212 #[cfg(test)] 187 213 mod tests { 214 + 188 215 use std::time::{SystemTime, UNIX_EPOCH}; 189 216 190 217 use rusqlite::Connection; ··· 194 221 use crate::{ 195 222 core::{ 196 223 accounts::{ACCOUNT, User}, 197 - chats::{apply_delta, get_delta, get_last_row_counter, save_chat}, 224 + chats::{ 225 + apply_delta, decode_delta_from_bytes, encode_delta_to_bytes, get_delta, 226 + get_last_row_counter, save_chat, 227 + }, 198 228 }, 199 229 runtime::mlx::ChatResponse, 230 + utils::test_logger, 200 231 }; 201 232 202 233 #[test] ··· 366 397 367 398 let rows = get_delta(&conn, &user.user_id, 0).unwrap(); 368 399 assert_eq!(rows.len(), 4); 369 - assert!(apply_delta(&mut conn_2, rows).is_ok()); 400 + assert!(apply_delta(&mut conn_2, &rows).is_ok()); 401 + let rows = get_delta(&conn_2, &user.user_id, 0).unwrap(); 402 + assert_eq!(rows.len(), 4); 403 + } 404 + 405 + #[test] 406 + fn test_e2e_delta_roundtrip() { 407 + let conn = setup_db_schema(); 408 + let mut conn_2 = setup_db_schema(); 409 + let user = create_user(); 410 + let input = "2+2"; 411 + let _chat_1 = save_chat(&conn, &user, input, None).expect("chat should be saved"); 412 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 413 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 414 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 415 + 416 + let rows = get_delta(&conn, &user.user_id, 0).unwrap(); 417 + assert_eq!(rows.len(), 4); 418 + let chat_bytes = encode_delta_to_bytes(&rows); 419 + let decoded_chat = decode_delta_from_bytes(&chat_bytes).unwrap(); 420 + assert!(apply_delta(&mut conn_2, &decoded_chat).is_ok()); 421 + let rows = get_delta(&conn_2, &user.user_id, 0).unwrap(); 422 + assert_eq!(rows.len(), 4); 423 + } 424 + 425 + #[test] 426 + fn test_e2e_delta_roundtrip_w_empty_bytes() { 427 + let conn = setup_db_schema(); 428 + let mut conn_2 = setup_db_schema(); 429 + let user = create_user(); 430 + let input = "2+2"; 431 + let _chat_1 = save_chat(&conn, &user, input, None).expect("chat should be saved"); 432 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 433 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 434 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 435 + 436 + let rows = get_delta(&conn, &user.user_id, 4).unwrap(); 437 + assert_eq!(rows.len(), 0); 438 + let chat_bytes = encode_delta_to_bytes(&rows); 439 + let decoded_chat = decode_delta_from_bytes(&chat_bytes).unwrap(); 440 + assert!(apply_delta(&mut conn_2, &decoded_chat).is_ok()); 441 + let rows = get_delta(&conn_2, &user.user_id, 0).unwrap(); 442 + assert_eq!(rows.len(), 0); 443 + } 444 + 445 + #[test] 446 + fn test_non_zero_last_counter_delta() { 447 + let conn = setup_db_schema(); 448 + let mut _conn_2 = setup_db_schema(); 449 + let user = create_user(); 450 + let input = "2+2"; 451 + let chat_1 = save_chat(&conn, &user, input, None).expect("chat should be saved"); 452 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 453 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 454 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 455 + let rows = get_delta(&conn, &user.user_id, chat_1.row_counter).unwrap(); 456 + assert_eq!(rows.len(), 3); 457 + } 458 + 459 + #[test] 460 + fn test_duplicate_row_apply() { 461 + test_logger(); 462 + let conn = setup_db_schema(); 463 + let mut conn_2 = setup_db_schema(); 464 + let user = create_user(); 465 + let input = "2+2"; 466 + let _chat_1 = save_chat(&conn, &user, input, None).expect("chat should be saved"); 467 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 468 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 469 + let _ = save_chat(&conn, &user, input, None).expect("chat should be saved"); 470 + 471 + let rows = get_delta(&conn, &user.user_id, 0).unwrap(); 472 + assert_eq!(rows.len(), 4); 473 + let chat_bytes = encode_delta_to_bytes(&rows); 474 + let decoded_chat = decode_delta_from_bytes(&chat_bytes).unwrap(); 475 + assert!(apply_delta(&mut conn_2, &decoded_chat).is_ok()); 476 + let rows = get_delta(&conn_2, &user.user_id, 0).unwrap(); 477 + assert_eq!(rows.len(), 4); 478 + assert!(apply_delta(&mut conn_2, &decoded_chat).is_ok()); 370 479 let rows = get_delta(&conn_2, &user.user_id, 0).unwrap(); 371 480 assert_eq!(rows.len(), 4); 372 481 } 373 482 483 + #[test] 484 + fn test_e2e_syncing_both_ways_w_eventual_consistency() { 485 + test_logger(); 486 + let mut conn = setup_db_schema(); 487 + let mut conn_2 = setup_db_schema(); 488 + let user_a = create_user_by_id("user_a"); 489 + let user_b = create_user_by_id("user_b"); 490 + 491 + // Node user A adds stuff 492 + let input = "2+2"; 493 + let _chat_1 = save_chat(&conn, &user_a, input, None).expect("chat should be saved"); 494 + let _ = save_chat(&conn, &user_a, input, None).expect("chat should be saved"); 495 + let _ = save_chat(&conn, &user_a, input, None).expect("chat should be saved"); 496 + let _ = save_chat(&conn, &user_a, input, None).expect("chat should be saved"); 497 + 498 + // Node user B adds stuff 499 + let input = "4+4"; 500 + let _chat_1 = save_chat(&conn_2, &user_b, input, None).expect("chat should be saved"); 501 + let _ = save_chat(&conn_2, &user_b, input, None).expect("chat should be saved"); 502 + let _ = save_chat(&conn_2, &user_b, input, None).expect("chat should be saved"); 503 + let _ = save_chat(&conn_2, &user_b, input, None).expect("chat should be saved"); 504 + 505 + // Node A wants to sync with Node B 506 + 507 + // 1. So its sends last_row_counter of Node B to Node B and hopefully 508 + // it sends the diff since then and last_row_counter of Node A back.. 509 + 510 + let user_b_last_entry_of_user_a = get_last_row_counter(&conn, &user_b.user_id).unwrap(); 511 + 512 + // user_b is extracting the row's of it since the given last_row_counter 513 + let user_bs_diff_rows = 514 + get_delta(&conn_2, &user_b.user_id, user_b_last_entry_of_user_a).unwrap(); 515 + 516 + assert_eq!(user_bs_diff_rows.len(), 4); 517 + 518 + // user_bs diff is encoded 519 + let user_b_chat_bytes = encode_delta_to_bytes(&user_bs_diff_rows); 520 + 521 + // send to user_a and its decoded 522 + let user_b_decoded_chat = decode_delta_from_bytes(&user_b_chat_bytes).unwrap(); 523 + 524 + // Now user_a is gonna apply the user_b diff 525 + assert!(apply_delta(&mut conn, &user_b_decoded_chat).is_ok()); 526 + 527 + // Just checking if we user_a has all 8 rows 528 + 529 + let user_a_rows = conn 530 + .query_row("select count(*) from chats", [], |row| { 531 + row.get::<usize, i64>(0) 532 + }) 533 + .unwrap(); 534 + 535 + assert_eq!(user_a_rows, 8); 536 + 537 + // cool, now lets do the reverse sync, user B syncs user A stuff 538 + 539 + let user_a_last_entry_of_user_b = get_last_row_counter(&conn_2, &user_a.user_id).unwrap(); 540 + 541 + // user_a is extracting the row's of it since the given last_row_counter 542 + let user_as_diff_rows = 543 + get_delta(&conn, &user_a.user_id, user_a_last_entry_of_user_b).unwrap(); 544 + 545 + assert_eq!(user_as_diff_rows.len(), 4); 546 + 547 + // user_as diff is encoded 548 + let user_a_chat_bytes = encode_delta_to_bytes(&user_as_diff_rows); 549 + 550 + // send to user_b and its decoded 551 + let user_a_decoded_chat = decode_delta_from_bytes(&user_a_chat_bytes).unwrap(); 552 + 553 + // Now user_b is gonna apply the user_b diff 554 + assert!(apply_delta(&mut conn_2, &user_a_decoded_chat).is_ok()); 555 + 556 + // Just checking eventual consistency 557 + 558 + let user_a_rows = conn 559 + .query_row("select count(*) from chats", [], |row| { 560 + row.get::<usize, i64>(0) 561 + }) 562 + .unwrap(); 563 + 564 + let user_b_rows = conn_2 565 + .query_row("select count(*) from chats", [], |row| { 566 + row.get::<usize, i64>(0) 567 + }) 568 + .unwrap(); 569 + 570 + assert_eq!(user_a_rows, user_b_rows); 571 + } 572 + 374 573 struct SavedChatRow { 375 574 content: String, 376 575 resp_id: Option<String>, ··· 400 599 User { 401 600 id: Uuid::now_v7(), 402 601 user_id: String::from("did"), 602 + username: String::from("nickname"), 603 + account_type: ACCOUNT::LOCAL, 604 + active_profile: true, 605 + root: true, 606 + created_at: SystemTime::now() 607 + .duration_since(UNIX_EPOCH) 608 + .expect("time went backwards") 609 + .as_secs(), 610 + updated_at: SystemTime::now() 611 + .duration_since(UNIX_EPOCH) 612 + .expect("time went backwards") 613 + .as_secs(), 614 + } 615 + } 616 + fn create_user_by_id(user_id: &str) -> User { 617 + User { 618 + id: Uuid::now_v7(), 619 + user_id: String::from(user_id), 403 620 username: String::from("nickname"), 404 621 account_type: ACCOUNT::LOCAL, 405 622 active_profile: true,
+275 -44
tiles/src/core/network/mod.rs
··· 9 9 }; 10 10 11 11 use anyhow::Result; 12 + use axum::body::Bytes; 12 13 use futures_util::{StreamExt, TryStreamExt}; 13 14 use iroh::{ 14 15 Endpoint, EndpointId, NET_REPORT_TIMEOUT, PublicKey, ··· 17 18 endpoint_info::UserData, 18 19 protocol::Router, 19 20 }; 21 + use iroh_blobs::{BlobsProtocol, store::mem::MemStore, ticket::BlobTicket}; 20 22 use iroh_gossip::{ 21 23 Gossip, TopicId, 22 24 api::{Event, GossipReceiver, GossipSender}, 23 25 }; 26 + 24 27 use rusqlite::Connection; 25 - use tilekit::accounts::{get_did_from_public_key, get_random_bytes, get_random_bytes_32}; 28 + use tilekit::accounts::{ 29 + get_did_from_public_key, get_public_key_from_did, get_random_bytes, get_random_bytes_32, 30 + }; 26 31 use tokio::task::spawn_blocking; 27 32 use uuid::Uuid; 28 33 ··· 30 35 accounts::{ 31 36 self, get_app_secret_key, get_current_user, get_user_by_user_id, save_peer_account_db, 32 37 }, 38 + chats::{get_delta, get_encoded_delta, get_last_row_counter}, 33 39 network::ticket::{EndpointUserData, LinkTicket}, 34 40 storage::db::{DBTYPE, get_db_conn}, 35 41 }; ··· 64 70 } 65 71 } 66 72 67 - #[derive(serde::Serialize, serde::Deserialize)] 73 + #[derive(serde::Serialize, serde::Deserialize, Debug)] 68 74 #[allow(clippy::enum_variant_names)] 69 75 enum MessageBody { 70 - LinkRequest { ticket: String }, 76 + LinkRequest { 77 + ticket: String, 78 + }, 71 79 LinkAccepted, 72 - LinkRejected { reason: String }, 80 + LinkRejected { 81 + reason: String, 82 + }, 83 + SyncStart { 84 + last_row_counter: Option<i64>, 85 + }, 86 + SyncSendDeltaInfo { 87 + blob_ticket: String, 88 + last_row_counter: Option<i64>, 89 + }, 90 + SyncEnd, 73 91 } 74 92 75 - // Entrypoint of network connection 76 - // pub async fn init(ticket: Option<&str>) -> Result<()> { 77 - // if let Some(ticket_addr) = ticket { 78 - // let sender_endpoint = Endpoint::bind(presets::N0).await?; 79 - // println!("{:?}", sender_endpoint.addr()); 80 - // let se_clone = sender_endpoint.clone(); 81 - // let send_pinger = Ping::new(); 82 - // let rtt = send_pinger 83 - // .ping( 84 - // &sender_endpoint, 85 - // EndpointTicket::from_str(ticket_addr)? 86 - // .endpoint_addr() 87 - // .clone(), 88 - // ) 89 - // .await?; 90 - 91 - // println!("ping took: {:?} to complete", rtt); 92 - // se_clone.close().await; 93 - // } else { 94 - // let endpoint = Endpoint::bind(presets::N0).await?; 95 - // let ep = endpoint.clone(); 96 - // let ep2 = endpoint.clone(); 97 - // endpoint.online().await; 98 - 99 - // let ping = Ping::new(); 100 - 101 - // let ticket = EndpointTicket::new(endpoint.addr()); 102 - 103 - // println!("ticket\n{:?}", ticket.to_string()); 104 - 105 - // let recv_router = Router::builder(ep).accept(iroh_ping::ALPN, ping).spawn(); 106 - // ep2.close().await; 107 - // recv_router.shutdown().await?; 108 - // } 109 - // Ok(()) 110 - // } 111 - 112 93 pub async fn link(ticket: Option<String>) -> Result<()> { 113 94 let user_db_conn = get_db_conn(DBTYPE::COMMON)?; 114 95 let user = get_current_user(&user_db_conn)?; 115 96 let endpoint = create_endpoint(&user).await?; 116 97 let is_online = is_online(&endpoint).await; 117 98 let mut bootstrap_ids: Vec<EndpointId> = vec![]; 118 - // if ticket's there, then this is link enable sender's command, e;se receiver end 99 + // if ticket's there, then this is link enable sender's command, else receiver end 119 100 if let Some(ticket) = ticket { 120 101 let (endpoint_id, mut did, mut nickname, topic_value) = parse_link_ticket(&ticket)?; 121 102 ··· 242 223 if cfg!(debug_assertions) { 243 224 println!("In {}:, some event {:?}", user.username, event); 244 225 } 226 + // TODO: Damn refactor the loop, its getting bigger 245 227 if let Event::Received(msg) = event { 246 228 let pub_key = msg.delivered_from; 247 229 let msg = NetworkMessage::from_bytes(&msg.content)?; ··· 333 315 msg.from_nickname, msg.from_did, reason 334 316 ); 335 317 } 318 + msg_body => { 319 + eprintln!("Invalid link message {:?}", msg_body) 320 + } 336 321 } 337 322 } 338 323 } 339 324 Ok(()) 340 325 } 341 326 327 + async fn sync_subscribe_loop( 328 + mut receiver: GossipReceiver, 329 + sender: GossipSender, 330 + user: accounts::User, 331 + user_db_conn: Connection, 332 + store: MemStore, 333 + endpoint: Endpoint, 334 + ) -> Result<()> { 335 + while let Some(event) = receiver.try_next().await? { 336 + if cfg!(debug_assertions) { 337 + println!("SYNC_LOOP: In {}:, some event {:?}", user.username, event); 338 + } 339 + if let Event::Received(msg) = event { 340 + let pub_key = msg.delivered_from; 341 + let msg = NetworkMessage::from_bytes(&msg.content)?; 342 + if !is_did_valid(&msg.from_did, pub_key)? { 343 + eprintln!( 344 + "Incoming peer DID {} invalid, blocking request", 345 + msg.from_did 346 + ); 347 + continue; 348 + } 349 + match msg.body { 350 + MessageBody::SyncStart { 351 + last_row_counter: _, 352 + } => { 353 + // TODO: REJECT SYNC_REQUESTS FROM NON_PEERS 354 + println!("Received sync start"); 355 + on_sync_start_event(&sender, &store, &msg, pub_key, &user).await?; 356 + } 357 + MessageBody::SyncSendDeltaInfo { 358 + blob_ticket: _, 359 + last_row_counter: _, 360 + } => { 361 + on_sync_send_delta_info(&sender, &store, &msg, pub_key, &user, &endpoint) 362 + .await?; 363 + } 364 + MessageBody::SyncEnd => { 365 + println!("sync end, can close"); 366 + } 367 + msg_body => { 368 + println!("Invalid sync message {:?}", msg_body) 369 + } 370 + } 371 + } 372 + } 373 + Ok(()) 374 + } 342 375 async fn create_endpoint(user: &accounts::User) -> Result<Endpoint> { 343 376 // In release mode, we will build the endpoint using 344 377 // tiles keypair in keychain ··· 360 393 } 361 394 } 362 395 396 + pub async fn sync(did: Option<String>) -> Result<()> { 397 + let user_db_conn = get_db_conn(DBTYPE::COMMON)?; 398 + let user = get_current_user(&user_db_conn)?; 399 + let chat_db_conn = get_db_conn(DBTYPE::CHAT)?; 400 + let endpoint = create_endpoint(&user).await?; 401 + let is_online = is_online(&endpoint).await; 402 + if let Some(receiver_did) = did { 403 + // INITIATOR BLOCK 404 + // The sync gossip topic is basically derived from the receiver's 405 + // DID, so that initiator's can directly connect w/o any 406 + // initial handshake 407 + let receiver_pub_key = get_public_key_from_did(&receiver_did)?; 408 + let receiver_endpoint_id = PublicKey::from_bytes(&receiver_pub_key)?; 409 + println!("receiver endpoint id {:?}", receiver_endpoint_id); 410 + let sync_topic = format!("sync:{}", receiver_did); 411 + let sync_topic_id = create_topic_id(&sync_topic); 412 + 413 + let (sender, mut receiver, recv_router, store) = 414 + create_sync_network(&endpoint, sync_topic_id, vec![receiver_endpoint_id]).await?; 415 + println!("\nConnecting to {}.....", receiver_did); 416 + receiver.joined().await?; 417 + tokio::spawn(sync_subscribe_loop( 418 + receiver, 419 + sender.clone(), 420 + user.clone(), 421 + user_db_conn, 422 + store, 423 + endpoint.clone(), 424 + )); 425 + 426 + // get the last_row_counter 427 + // 428 + let receiver_last_row_counter = get_last_row_counter(&chat_db_conn, &receiver_did)?; 429 + let sync_start_msg = NetworkMessage::new( 430 + &user, 431 + is_online, 432 + MessageBody::SyncStart { 433 + last_row_counter: Some(receiver_last_row_counter), 434 + }, 435 + ); 436 + sender.broadcast(sync_start_msg.to_bytes().into()).await?; 437 + 438 + println!( 439 + "\nSent sync start request to {}({})", 440 + user.username, user.user_id 441 + ); 442 + tokio::signal::ctrl_c().await?; 443 + recv_router.shutdown().await?; 444 + } else { 445 + // RECEIVER BLOCK 446 + // The sync gossip topic is basically derived from the receiver's 447 + // public-key, so that initiator's can directly connect w/o any 448 + // initial handshake 449 + 450 + println!("endpointId {:?}", endpoint.id()); 451 + let did = if cfg!(debug_assertions) { 452 + let pub_key = endpoint.id(); 453 + &get_did_from_public_key(pub_key.as_bytes())? 454 + } else { 455 + &user.user_id 456 + }; 457 + 458 + let sync_topic = format!("sync:{}", did); 459 + let sync_topic_id = create_topic_id(&sync_topic); 460 + let (sender, receiver, recv_router, store) = 461 + create_sync_network(&endpoint, sync_topic_id, vec![]).await?; 462 + 463 + tokio::spawn(sync_subscribe_loop( 464 + receiver, 465 + sender.clone(), 466 + user.clone(), 467 + user_db_conn, 468 + store, 469 + endpoint.clone(), 470 + )); 471 + println!("Ready to accept sync requests from peers..."); 472 + 473 + // Since in dev, we use create endpoints, at the initiator side 474 + // we can use the DID derived from this, instead of actual ones 475 + // for the network to form correctly 476 + if cfg!(debug_assertions) { 477 + println!("Use this DID {} in dev for testing", did); 478 + } 479 + tokio::signal::ctrl_c().await?; 480 + recv_router.shutdown().await?; 481 + } 482 + endpoint.close().await; 483 + Ok(()) 484 + } 485 + 486 + // Router with gossip and blob protocol 487 + async fn create_sync_network( 488 + endpoint: &Endpoint, 489 + topic_id: TopicId, 490 + bootstrap_ids: Vec<iroh::PublicKey>, 491 + ) -> Result<(GossipSender, GossipReceiver, Router, MemStore)> { 492 + let gossip = Gossip::builder().spawn(endpoint.clone()); 493 + let store = MemStore::new(); 494 + let blobs = BlobsProtocol::new(&store, None); 495 + let recv_router = Router::builder(endpoint.clone()) 496 + .accept(iroh_gossip::ALPN, gossip.clone()) 497 + .accept(iroh_blobs::ALPN, blobs.clone()) 498 + .spawn(); 499 + 500 + let (goss_sender, goss_receiver) = gossip.subscribe(topic_id, bootstrap_ids).await?.split(); 501 + 502 + Ok((goss_sender, goss_receiver, recv_router, store)) 503 + } 504 + 363 505 fn create_topic_id(topic_name: &str) -> TopicId { 364 506 let mut hasher = Sha256::new(); 365 507 hasher.update(topic_name.as_bytes()); ··· 461 603 Ok(get_did_from_public_key(&pub_key)? == did) 462 604 } 463 605 } 464 - // fn subsribe_mdns_events(mdns_events) {} 465 - //TODO: Add tests, can we get some from iroh reference? 606 + 607 + async fn on_sync_start_event( 608 + sender: &GossipSender, 609 + store: &MemStore, 610 + msg: &NetworkMessage, 611 + delivered_from: PublicKey, 612 + user: &accounts::User, 613 + // chat_db_conn: &Connection, 614 + ) -> Result<()> { 615 + println!("Received sync start"); 616 + if let MessageBody::SyncStart { 617 + last_row_counter: lrc, 618 + } = &msg.body 619 + { 620 + // let chat_delta = get_encoded_delta( 621 + // &chat_db_conn, 622 + // &user.user_id, 623 + // lrc.expect("Expected a valid last row counter"), 624 + // ); 625 + // let chat_delta_bytes = 626 + let rand_bytes = get_random_bytes_32().to_vec(); 627 + println!("rand bytes\n{:?}", rand_bytes); 628 + let tag = store 629 + .blobs() 630 + .add_bytes(Into::<Bytes>::into(rand_bytes)) 631 + .await?; 632 + 633 + let ticket = BlobTicket::new(delivered_from.into(), tag.hash, tag.format); 634 + let delta_info = NetworkMessage::new( 635 + &user, 636 + msg.is_online, 637 + MessageBody::SyncSendDeltaInfo { 638 + blob_ticket: ticket.to_string(), 639 + last_row_counter: Some(0), 640 + }, 641 + ); 642 + sender.broadcast(delta_info.to_bytes().into()).await?; 643 + println!("Sent blob ticket {}", ticket.to_string()); 644 + } 645 + Ok(()) 646 + } 647 + 648 + async fn on_sync_send_delta_info( 649 + sender: &GossipSender, 650 + store: &MemStore, 651 + msg: &NetworkMessage, 652 + delivered_from: PublicKey, 653 + user: &accounts::User, 654 + endpoint: &Endpoint, 655 + ) -> Result<()> { 656 + if let MessageBody::SyncSendDeltaInfo { 657 + blob_ticket, 658 + last_row_counter, 659 + } = &msg.body 660 + { 661 + let ticket: BlobTicket = blob_ticket.parse()?; 662 + let downloader = store.downloader(&endpoint); 663 + downloader 664 + .download(ticket.hash(), Some(delivered_from)) 665 + .await?; 666 + 667 + let data = store.blobs().get_bytes(ticket.hash()).await?; 668 + println!("rand bytes received {:?}", data.to_vec()); 669 + 670 + println!("finished download"); 671 + if let Some(_row_counter) = last_row_counter { 672 + let rand_bytes = get_random_bytes_32().to_vec(); 673 + println!("rand bytes\n{:?}", rand_bytes); 674 + let tag = store 675 + .blobs() 676 + .add_bytes(Into::<Bytes>::into(rand_bytes)) 677 + .await?; 678 + 679 + let ticket = BlobTicket::new(delivered_from.into(), tag.hash, tag.format); 680 + let delta_info = NetworkMessage::new( 681 + &user, 682 + msg.is_online, 683 + MessageBody::SyncSendDeltaInfo { 684 + blob_ticket: ticket.to_string(), 685 + last_row_counter: None, 686 + }, 687 + ); 688 + sender.broadcast(delta_info.to_bytes().into()).await?; 689 + } else { 690 + let stop_req = NetworkMessage::new(&user, msg.is_online, MessageBody::SyncEnd); 691 + sender.broadcast(stop_req.to_bytes().into()).await?; 692 + println!("sync end, can close"); 693 + } 694 + } 695 + Ok(()) 696 + }
+15 -2
tiles/src/main.rs
··· 1 + #![warn(clippy::pedantic)] 2 + 1 3 use std::error::Error; 2 4 3 5 use clap::{Args, Parser, Subcommand}; 4 6 use tiles::{ 5 - core::{self, network::link}, 7 + core::{ 8 + self, 9 + network::{link, sync}, 10 + }, 6 11 daemon::{start_cmd, start_server, stop_cmd}, 7 12 runtime::{RunArgs, build_runtime}, 8 13 utils::installer, ··· 66 71 67 72 /// Link with other devices p2p 68 73 Link(LinkArgs), 74 + 75 + /// Syncs the chats to peers 76 + Sync { 77 + /// The DID of the peer you want to sync 78 + did: Option<String>, 79 + }, 69 80 } 70 81 71 82 #[derive(Debug, Args)] ··· 174 185 } 175 186 #[tokio::main] 176 187 pub async fn main() -> Result<(), Box<dyn Error>> { 188 + env_logger::init(); 177 189 let cli = Cli::parse(); 178 190 let runtime = build_runtime(); 179 191 match cli.command { ··· 232 244 data, 233 245 model, 234 246 }) => { 235 - let modelfile = commands::optimize(modelfile_path.clone(), data, model).await?; 247 + let modelfile = commands::optimize(&modelfile_path, data, &model).await?; 236 248 std::fs::write(&modelfile_path, modelfile.to_string())?; 237 249 println!("Successfully updated {}", modelfile_path); 238 250 } ··· 263 275 show_peers()?; 264 276 } 265 277 }, 278 + Some(Commands::Sync { did }) => sync(did).await?, 266 279 } 267 280 Ok(()) 268 281 }
+1 -2
tiles/src/utils/config.rs
··· 96 96 .get("data") 97 97 .expect("Failed to get data") 98 98 .as_table() 99 - .expect("Failed to parse to table (data)") 100 - .clone(); 99 + .expect("Failed to parse to table (data)"); 101 100 102 101 if let Some(path) = data_config 103 102 .get("path")
+4
tiles/src/utils/mod.rs
··· 10 10 .expect("time went backwards") 11 11 .as_secs() 12 12 } 13 + 14 + pub fn test_logger() { 15 + let _ = env_logger::builder().is_test(true).try_init(); 16 + }