+40 -650

Cargo.lock

reviewed

··· 29 29 30 30 [[package]] 31 31 name = "ahash" 32 32 - version = "0.7.8" 33 33 - source = "registry+https://github.com/rust-lang/crates.io-index" 34 34 - checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" 35 35 - dependencies = [ 36 36 - "getrandom 0.2.16", 37 37 - "once_cell", 38 38 - "version_check", 39 39 - ] 40 40 - 41 41 - [[package]] 42 42 - name = "ahash" 43 32 version = "0.8.12" 44 33 source = "registry+https://github.com/rust-lang/crates.io-index" 45 34 checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" 46 35 dependencies = [ 47 36 "cfg-if", 48 48 - "const-random", 49 49 - "getrandom 0.3.4", 50 37 "once_cell", 51 38 "version_check", 52 39 "zerocopy", ··· 127 114 ] 128 115 129 116 [[package]] 130 130 - name = "arrayvec" 131 131 - version = "0.7.6" 132 132 - source = "registry+https://github.com/rust-lang/crates.io-index" 133 133 - checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" 134 134 - 135 135 - [[package]] 136 136 - name = "arrow" 137 137 - version = "56.2.0" 138 138 - source = "registry+https://github.com/rust-lang/crates.io-index" 139 139 - checksum = "6e833808ff2d94ed40d9379848a950d995043c7fb3e81a30b383f4c6033821cc" 140 140 - dependencies = [ 141 141 - "arrow-arith", 142 142 - "arrow-array", 143 143 - "arrow-buffer", 144 144 - "arrow-cast", 145 145 - "arrow-data", 146 146 - "arrow-ord", 147 147 - "arrow-row", 148 148 - "arrow-schema", 149 149 - "arrow-select", 150 150 - "arrow-string", 151 151 - ] 152 152 - 153 153 - [[package]] 154 154 - name = "arrow-arith" 155 155 - version = "56.2.0" 156 156 - source = "registry+https://github.com/rust-lang/crates.io-index" 157 157 - checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" 158 158 - dependencies = [ 159 159 - "arrow-array", 160 160 - "arrow-buffer", 161 161 - "arrow-data", 162 162 - "arrow-schema", 163 163 - "chrono", 164 164 - "num", 165 165 - ] 166 166 - 167 167 - [[package]] 168 168 - name = "arrow-array" 169 169 - version = "56.2.0" 170 170 - source = "registry+https://github.com/rust-lang/crates.io-index" 171 171 - checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" 172 172 - dependencies = [ 173 173 - "ahash 0.8.12", 174 174 - "arrow-buffer", 175 175 - "arrow-data", 176 176 - "arrow-schema", 177 177 - "chrono", 178 178 - "half", 179 179 - "hashbrown 0.16.0", 180 180 - "num", 181 181 - ] 182 182 - 183 183 - [[package]] 184 184 - name = "arrow-buffer" 185 185 - version = "56.2.0" 186 186 - source = "registry+https://github.com/rust-lang/crates.io-index" 187 187 - checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" 188 188 - dependencies = [ 189 189 - "bytes", 190 190 - "half", 191 191 - "num", 192 192 - ] 193 193 - 194 194 - [[package]] 195 195 - name = "arrow-cast" 196 196 - version = "56.2.0" 197 197 - source = "registry+https://github.com/rust-lang/crates.io-index" 198 198 - checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" 199 199 - dependencies = [ 200 200 - "arrow-array", 201 201 - "arrow-buffer", 202 202 - "arrow-data", 203 203 - "arrow-schema", 204 204 - "arrow-select", 205 205 - "atoi", 206 206 - "base64", 207 207 - "chrono", 208 208 - "comfy-table", 209 209 - "half", 210 210 - "lexical-core", 211 211 - "num", 212 212 - "ryu", 213 213 - ] 214 214 - 215 215 - [[package]] 216 216 - name = "arrow-data" 217 217 - version = "56.2.0" 218 218 - source = "registry+https://github.com/rust-lang/crates.io-index" 219 219 - checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" 220 220 - dependencies = [ 221 221 - "arrow-buffer", 222 222 - "arrow-schema", 223 223 - "half", 224 224 - "num", 225 225 - ] 226 226 - 227 227 - [[package]] 228 228 - name = "arrow-ord" 229 229 - version = "56.2.0" 230 230 - source = "registry+https://github.com/rust-lang/crates.io-index" 231 231 - checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" 232 232 - dependencies = [ 233 233 - "arrow-array", 234 234 - "arrow-buffer", 235 235 - "arrow-data", 236 236 - "arrow-schema", 237 237 - "arrow-select", 238 238 - ] 239 239 - 240 240 - [[package]] 241 241 - name = "arrow-row" 242 242 - version = "56.2.0" 243 243 - source = "registry+https://github.com/rust-lang/crates.io-index" 244 244 - checksum = "9d07ba24522229d9085031df6b94605e0f4b26e099fb7cdeec37abd941a73753" 245 245 - dependencies = [ 246 246 - "arrow-array", 247 247 - "arrow-buffer", 248 248 - "arrow-data", 249 249 - "arrow-schema", 250 250 - "half", 251 251 - ] 252 252 - 253 253 - [[package]] 254 254 - name = "arrow-schema" 255 255 - version = "56.2.0" 256 256 - source = "registry+https://github.com/rust-lang/crates.io-index" 257 257 - checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" 258 258 - dependencies = [ 259 259 - "bitflags", 260 260 - ] 261 261 - 262 262 - [[package]] 263 263 - name = "arrow-select" 264 264 - version = "56.2.0" 265 265 - source = "registry+https://github.com/rust-lang/crates.io-index" 266 266 - checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" 267 267 - dependencies = [ 268 268 - "ahash 0.8.12", 269 269 - "arrow-array", 270 270 - "arrow-buffer", 271 271 - "arrow-data", 272 272 - "arrow-schema", 273 273 - "num", 274 274 - ] 275 275 - 276 276 - [[package]] 277 277 - name = "arrow-string" 278 278 - version = "56.2.0" 279 279 - source = "registry+https://github.com/rust-lang/crates.io-index" 280 280 - checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" 281 281 - dependencies = [ 282 282 - "arrow-array", 283 283 - "arrow-buffer", 284 284 - "arrow-data", 285 285 - "arrow-schema", 286 286 - "arrow-select", 287 287 - "memchr", 288 288 - "num", 289 289 - "regex", 290 290 - "regex-syntax", 291 291 - ] 292 292 - 293 293 - [[package]] 294 117 name = "astral-tokio-tar" 295 118 version = "0.5.6" 296 119 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 328 151 "proc-macro2", 329 152 "quote", 330 153 "syn 2.0.108", 331 331 - ] 332 332 - 333 333 - [[package]] 334 334 - name = "atoi" 335 335 - version = "2.0.0" 336 336 - source = "registry+https://github.com/rust-lang/crates.io-index" 337 337 - checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" 338 338 - dependencies = [ 339 339 - "num-traits", 340 154 ] 341 155 342 156 [[package]] ··· 444 258 checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" 445 259 446 260 [[package]] 447 447 - name = "bitvec" 448 448 - version = "1.0.1" 449 449 - source = "registry+https://github.com/rust-lang/crates.io-index" 450 450 - checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" 451 451 - dependencies = [ 452 452 - "funty", 453 453 - "radium", 454 454 - "tap", 455 455 - "wyz", 456 456 - ] 457 457 - 458 458 - [[package]] 459 261 name = "block-buffer" 460 262 version = "0.10.4" 461 263 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 495 297 source = "registry+https://github.com/rust-lang/crates.io-index" 496 298 checksum = "ad8646f98db542e39fc66e68a20b2144f6a732636df7c2354e74645faaa433ce" 497 299 dependencies = [ 498 498 - "borsh-derive", 499 300 "cfg_aliases", 500 301 ] 501 302 502 303 [[package]] 503 503 - name = "borsh-derive" 504 504 - version = "1.5.7" 505 505 - source = "registry+https://github.com/rust-lang/crates.io-index" 506 506 - checksum = "fdd1d3c0c2f5833f22386f252fe8ed005c7f59fdcddeef025c01b4c3b9fd9ac3" 507 507 - dependencies = [ 508 508 - "once_cell", 509 509 - "proc-macro-crate", 510 510 - "proc-macro2", 511 511 - "quote", 512 512 - "syn 2.0.108", 513 513 - ] 514 514 - 515 515 - [[package]] 516 304 name = "btree-range-map" 517 305 version = "0.7.2" 518 306 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 543 331 checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" 544 332 545 333 [[package]] 546 546 - name = "bytecheck" 547 547 - version = "0.6.12" 548 548 - source = "registry+https://github.com/rust-lang/crates.io-index" 549 549 - checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" 550 550 - dependencies = [ 551 551 - "bytecheck_derive", 552 552 - "ptr_meta", 553 553 - "simdutf8", 554 554 - ] 555 555 - 556 556 - [[package]] 557 557 - name = "bytecheck_derive" 558 558 - version = "0.6.12" 559 559 - source = "registry+https://github.com/rust-lang/crates.io-index" 560 560 - checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" 561 561 - dependencies = [ 562 562 - "proc-macro2", 563 563 - "quote", 564 564 - "syn 1.0.109", 565 565 - ] 566 566 - 567 567 - [[package]] 568 334 name = "bytecount" 569 335 version = "0.6.9" 570 336 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 626 392 ] 627 393 628 394 [[package]] 629 629 - name = "cast" 630 630 - version = "0.3.0" 631 631 - source = "registry+https://github.com/rust-lang/crates.io-index" 632 632 - checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" 633 633 - 634 634 - [[package]] 635 395 name = "cbor4ii" 636 396 version = "0.2.14" 637 397 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 647 407 checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3" 648 408 dependencies = [ 649 409 "find-msvc-tools", 650 650 - "jobserver", 651 651 - "libc", 652 410 "shlex", 653 411 ] 654 412 ··· 775 533 checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" 776 534 777 535 [[package]] 778 778 - name = "comfy-table" 779 779 - version = "7.1.2" 780 780 - source = "registry+https://github.com/rust-lang/crates.io-index" 781 781 - checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" 782 782 - dependencies = [ 783 783 - "strum 0.26.3", 784 784 - "strum_macros 0.26.4", 785 785 - "unicode-width 0.2.2", 786 786 - ] 787 787 - 788 788 - [[package]] 789 536 name = "compression-codecs" 790 537 version = "0.4.31" 791 538 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 810 557 checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" 811 558 812 559 [[package]] 813 813 - name = "const-random" 814 814 - version = "0.1.18" 815 815 - source = "registry+https://github.com/rust-lang/crates.io-index" 816 816 - checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" 817 817 - dependencies = [ 818 818 - "const-random-macro", 819 819 - ] 820 820 - 821 821 - [[package]] 822 822 - name = "const-random-macro" 823 823 - version = "0.1.16" 824 824 - source = "registry+https://github.com/rust-lang/crates.io-index" 825 825 - checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" 826 826 - dependencies = [ 827 827 - "getrandom 0.2.16", 828 828 - "once_cell", 829 829 - "tiny-keccak", 830 830 - ] 831 831 - 832 832 - [[package]] 833 560 name = "const-str" 834 561 version = "0.4.3" 835 562 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1110 837 ] 1111 838 1112 839 [[package]] 1113 1113 - name = "duckdb" 1114 1114 - version = "1.4.2" 1115 1115 - source = "registry+https://github.com/rust-lang/crates.io-index" 1116 1116 - checksum = "e46d5568337ee1f7ea8779e1d9aa2eafcdf156458713ce65afb246c5d2cf5850" 1117 1117 - dependencies = [ 1118 1118 - "arrow", 1119 1119 - "cast", 1120 1120 - "chrono", 1121 1121 - "fallible-iterator", 1122 1122 - "fallible-streaming-iterator", 1123 1123 - "hashlink", 1124 1124 - "libduckdb-sys", 1125 1125 - "num-integer", 1126 1126 - "r2d2", 1127 1127 - "rust_decimal", 1128 1128 - "strum 0.27.2", 1129 1129 - "uuid", 1130 1130 - ] 1131 1131 - 1132 1132 - [[package]] 1133 840 name = "dyn-clone" 1134 841 version = "1.0.20" 1135 842 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1203 910 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" 1204 911 dependencies = [ 1205 912 "libc", 1206 1206 - "windows-sys 0.52.0", 913 913 + "windows-sys 0.61.2", 1207 914 ] 1208 915 1209 916 [[package]] ··· 1280 987 "axum", 1281 988 "chrono", 1282 989 "clap", 1283 1283 - "duckdb", 1284 990 "eyre", 1285 991 "futures", 1286 992 "inventory", ··· 1294 1000 "metrics-exporter-prometheus", 1295 1001 "mini-moka", 1296 1002 "r2d2", 1003 1003 + "r2d2_sqlite", 1297 1004 "reqwest", 1005 1005 + "rusqlite", 1298 1006 "serde", 1299 1007 "tempfile", 1300 1008 "tokio", ··· 1358 1066 ] 1359 1067 1360 1068 [[package]] 1361 1361 - name = "funty" 1362 1362 - version = "2.0.0" 1363 1363 - source = "registry+https://github.com/rust-lang/crates.io-index" 1364 1364 - checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" 1365 1365 - 1366 1366 - [[package]] 1367 1069 name = "futf" 1368 1070 version = "0.1.5" 1369 1071 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1612 1314 dependencies = [ 1613 1315 "cfg-if", 1614 1316 "crunchy", 1615 1615 - "num-traits", 1616 1317 "zerocopy", 1617 1318 ] 1618 1319 ··· 1621 1322 version = "0.12.3" 1622 1323 source = "registry+https://github.com/rust-lang/crates.io-index" 1623 1324 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 1624 1624 - dependencies = [ 1625 1625 - "ahash 0.7.8", 1626 1626 - ] 1627 1325 1628 1326 [[package]] 1629 1327 name = "hashbrown" ··· 2322 2020 ] 2323 2021 2324 2022 [[package]] 2325 2325 - name = "jobserver" 2326 2326 - version = "0.1.34" 2327 2327 - source = "registry+https://github.com/rust-lang/crates.io-index" 2328 2328 - checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" 2329 2329 - dependencies = [ 2330 2330 - "getrandom 0.3.4", 2331 2331 - "libc", 2332 2332 - ] 2333 2333 - 2334 2334 - [[package]] 2335 2023 name = "jose-b64" 2336 2024 version = "0.1.2" 2337 2025 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2410 2098 ] 2411 2099 2412 2100 [[package]] 2413 2413 - name = "lexical-core" 2414 2414 - version = "1.0.6" 2415 2415 - source = "registry+https://github.com/rust-lang/crates.io-index" 2416 2416 - checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" 2417 2417 - dependencies = [ 2418 2418 - "lexical-parse-float", 2419 2419 - "lexical-parse-integer", 2420 2420 - "lexical-util", 2421 2421 - "lexical-write-float", 2422 2422 - "lexical-write-integer", 2423 2423 - ] 2424 2424 - 2425 2425 - [[package]] 2426 2426 - name = "lexical-parse-float" 2427 2427 - version = "1.0.6" 2428 2428 - source = "registry+https://github.com/rust-lang/crates.io-index" 2429 2429 - checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" 2430 2430 - dependencies = [ 2431 2431 - "lexical-parse-integer", 2432 2432 - "lexical-util", 2433 2433 - ] 2434 2434 - 2435 2435 - [[package]] 2436 2436 - name = "lexical-parse-integer" 2437 2437 - version = "1.0.6" 2438 2438 - source = "registry+https://github.com/rust-lang/crates.io-index" 2439 2439 - checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" 2440 2440 - dependencies = [ 2441 2441 - "lexical-util", 2442 2442 - ] 2443 2443 - 2444 2444 - [[package]] 2445 2445 - name = "lexical-util" 2446 2446 - version = "1.0.7" 2447 2447 - source = "registry+https://github.com/rust-lang/crates.io-index" 2448 2448 - checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" 2449 2449 - 2450 2450 - [[package]] 2451 2451 - name = "lexical-write-float" 2452 2452 - version = "1.0.6" 2453 2453 - source = "registry+https://github.com/rust-lang/crates.io-index" 2454 2454 - checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" 2455 2455 - dependencies = [ 2456 2456 - "lexical-util", 2457 2457 - "lexical-write-integer", 2458 2458 - ] 2459 2459 - 2460 2460 - [[package]] 2461 2461 - name = "lexical-write-integer" 2462 2462 - version = "1.0.6" 2463 2463 - source = "registry+https://github.com/rust-lang/crates.io-index" 2464 2464 - checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" 2465 2465 - dependencies = [ 2466 2466 - "lexical-util", 2467 2467 - ] 2468 2468 - 2469 2469 - [[package]] 2470 2101 name = "libbz2-rs-sys" 2471 2102 version = "0.2.2" 2472 2103 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2479 2110 checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" 2480 2111 2481 2112 [[package]] 2482 2482 - name = "libduckdb-sys" 2483 2483 - version = "1.4.2" 2484 2484 - source = "registry+https://github.com/rust-lang/crates.io-index" 2485 2485 - checksum = "6650a7ea86fce24fe1fbf5b037671a8b77c59d135703fc6085b8a1827e66e977" 2486 2486 - dependencies = [ 2487 2487 - "cc", 2488 2488 - "flate2", 2489 2489 - "pkg-config", 2490 2490 - "serde", 2491 2491 - "serde_json", 2492 2492 - "tar", 2493 2493 - "vcpkg", 2494 2494 - ] 2495 2495 - 2496 2496 - [[package]] 2497 2113 name = "libm" 2498 2114 version = "0.2.15" 2499 2115 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2508 2124 "bitflags", 2509 2125 "libc", 2510 2126 "redox_syscall", 2127 2127 + ] 2128 2128 + 2129 2129 + [[package]] 2130 2130 + name = "libsqlite3-sys" 2131 2131 + version = "0.35.0" 2132 2132 + source = "registry+https://github.com/rust-lang/crates.io-index" 2133 2133 + checksum = "133c182a6a2c87864fe97778797e46c7e999672690dc9fa3ee8e241aa4a9c13f" 2134 2134 + dependencies = [ 2135 2135 + "pkg-config", 2136 2136 + "vcpkg", 2511 2137 ] 2512 2138 2513 2139 [[package]] ··· 2641 2267 source = "registry+https://github.com/rust-lang/crates.io-index" 2642 2268 checksum = "5d5312e9ba3771cfa961b585728215e3d972c950a3eed9252aa093d6301277e8" 2643 2269 dependencies = [ 2644 2644 - "ahash 0.8.12", 2270 2270 + "ahash", 2645 2271 "portable-atomic", 2646 2272 ] 2647 2273 ··· 2689 2315 dependencies = [ 2690 2316 "cfg-if", 2691 2317 "miette-derive", 2692 2692 - "unicode-width 0.1.14", 2318 2318 + "unicode-width", 2693 2319 ] 2694 2320 2695 2321 [[package]] ··· 2838 2464 ] 2839 2465 2840 2466 [[package]] 2841 2841 - name = "num" 2842 2842 - version = "0.4.3" 2843 2843 - source = "registry+https://github.com/rust-lang/crates.io-index" 2844 2844 - checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" 2845 2845 - dependencies = [ 2846 2846 - "num-bigint", 2847 2847 - "num-complex", 2848 2848 - "num-integer", 2849 2849 - "num-iter", 2850 2850 - "num-rational", 2851 2851 - "num-traits", 2852 2852 - ] 2853 2853 - 2854 2854 - [[package]] 2855 2855 - name = "num-bigint" 2856 2856 - version = "0.4.6" 2857 2857 - source = "registry+https://github.com/rust-lang/crates.io-index" 2858 2858 - checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" 2859 2859 - dependencies = [ 2860 2860 - "num-integer", 2861 2861 - "num-traits", 2862 2862 - ] 2863 2863 - 2864 2864 - [[package]] 2865 2467 name = "num-bigint-dig" 2866 2468 version = "0.8.5" 2867 2469 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2878 2480 ] 2879 2481 2880 2482 [[package]] 2881 2881 - name = "num-complex" 2882 2882 - version = "0.4.6" 2883 2883 - source = "registry+https://github.com/rust-lang/crates.io-index" 2884 2884 - checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" 2885 2885 - dependencies = [ 2886 2886 - "num-traits", 2887 2887 - ] 2888 2888 - 2889 2889 - [[package]] 2890 2483 name = "num-conv" 2891 2484 version = "0.1.0" 2892 2485 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2908 2501 checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" 2909 2502 dependencies = [ 2910 2503 "autocfg", 2911 2911 - "num-integer", 2912 2912 - "num-traits", 2913 2913 - ] 2914 2914 - 2915 2915 - [[package]] 2916 2916 - name = "num-rational" 2917 2917 - version = "0.4.2" 2918 2918 - source = "registry+https://github.com/rust-lang/crates.io-index" 2919 2919 - checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" 2920 2920 - dependencies = [ 2921 2921 - "num-bigint", 2922 2504 "num-integer", 2923 2505 "num-traits", 2924 2506 ] ··· 3232 2814 ] 3233 2815 3234 2816 [[package]] 3235 3235 - name = "proc-macro-crate" 3236 3236 - version = "3.4.0" 3237 3237 - source = "registry+https://github.com/rust-lang/crates.io-index" 3238 3238 - checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" 3239 3239 - dependencies = [ 3240 3240 - "toml_edit", 3241 3241 - ] 3242 3242 - 3243 3243 - [[package]] 3244 2817 name = "proc-macro-error" 3245 2818 version = "1.0.4" 3246 2819 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3287 2860 ] 3288 2861 3289 2862 [[package]] 3290 3290 - name = "ptr_meta" 3291 3291 - version = "0.1.4" 3292 3292 - source = "registry+https://github.com/rust-lang/crates.io-index" 3293 3293 - checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" 3294 3294 - dependencies = [ 3295 3295 - "ptr_meta_derive", 3296 3296 - ] 3297 3297 - 3298 3298 - [[package]] 3299 3299 - name = "ptr_meta_derive" 3300 3300 - version = "0.1.4" 3301 3301 - source = "registry+https://github.com/rust-lang/crates.io-index" 3302 3302 - checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" 3303 3303 - dependencies = [ 3304 3304 - "proc-macro2", 3305 3305 - "quote", 3306 3306 - "syn 1.0.109", 3307 3307 - ] 3308 3308 - 3309 3309 - [[package]] 3310 2863 name = "pulldown-cmark" 3311 2864 version = "0.9.6" 3312 2865 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3384 2937 "once_cell", 3385 2938 "socket2 0.6.1", 3386 2939 "tracing", 3387 3387 - "windows-sys 0.52.0", 2940 2940 + "windows-sys 0.60.2", 3388 2941 ] 3389 2942 3390 2943 [[package]] ··· 3414 2967 ] 3415 2968 3416 2969 [[package]] 3417 3417 - name = "radium" 3418 3418 - version = "0.7.0" 2970 2970 + name = "r2d2_sqlite" 2971 2971 + version = "0.31.0" 3419 2972 source = "registry+https://github.com/rust-lang/crates.io-index" 3420 3420 - checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" 2973 2973 + checksum = "63417e83dc891797eea3ad379f52a5986da4bca0d6ef28baf4d14034dd111b0c" 2974 2974 + dependencies = [ 2975 2975 + "r2d2", 2976 2976 + "rusqlite", 2977 2977 + "uuid", 2978 2978 + ] 3421 2979 3422 2980 [[package]] 3423 2981 name = "rand" ··· 3567 3125 checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" 3568 3126 3569 3127 [[package]] 3570 3570 - name = "rend" 3571 3571 - version = "0.4.2" 3572 3572 - source = "registry+https://github.com/rust-lang/crates.io-index" 3573 3573 - checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" 3574 3574 - dependencies = [ 3575 3575 - "bytecheck", 3576 3576 - ] 3577 3577 - 3578 3578 - [[package]] 3579 3128 name = "reqwest" 3580 3129 version = "0.12.24" 3581 3130 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3654 3203 ] 3655 3204 3656 3205 [[package]] 3657 3657 - name = "rkyv" 3658 3658 - version = "0.7.45" 3659 3659 - source = "registry+https://github.com/rust-lang/crates.io-index" 3660 3660 - checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" 3661 3661 - dependencies = [ 3662 3662 - "bitvec", 3663 3663 - "bytecheck", 3664 3664 - "bytes", 3665 3665 - "hashbrown 0.12.3", 3666 3666 - "ptr_meta", 3667 3667 - "rend", 3668 3668 - "rkyv_derive", 3669 3669 - "seahash", 3670 3670 - "tinyvec", 3671 3671 - "uuid", 3672 3672 - ] 3673 3673 - 3674 3674 - [[package]] 3675 3675 - name = "rkyv_derive" 3676 3676 - version = "0.7.45" 3677 3677 - source = "registry+https://github.com/rust-lang/crates.io-index" 3678 3678 - checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" 3679 3679 - dependencies = [ 3680 3680 - "proc-macro2", 3681 3681 - "quote", 3682 3682 - "syn 1.0.109", 3683 3683 - ] 3684 3684 - 3685 3685 - [[package]] 3686 3206 name = "rsa" 3687 3207 version = "0.9.8" 3688 3208 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 3703 3223 ] 3704 3224 3705 3225 [[package]] 3706 3706 - name = "rust_decimal" 3707 3707 - version = "1.39.0" 3226 3226 + name = "rusqlite" 3227 3227 + version = "0.37.0" 3708 3228 source = "registry+https://github.com/rust-lang/crates.io-index" 3709 3709 - checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" 3229 3229 + checksum = "165ca6e57b20e1351573e3729b958bc62f0e48025386970b6e4d29e7a7e71f3f" 3710 3230 dependencies = [ 3711 3711 - "arrayvec", 3712 3712 - "borsh", 3713 3713 - "bytes", 3714 3714 - "num-traits", 3715 3715 - "rand 0.8.5", 3716 3716 - "rkyv", 3717 3717 - "serde", 3718 3718 - "serde_json", 3231 3231 + "bitflags", 3232 3232 + "chrono", 3233 3233 + "fallible-iterator", 3234 3234 + "fallible-streaming-iterator", 3235 3235 + "hashlink", 3236 3236 + "libsqlite3-sys", 3237 3237 + "smallvec", 3238 3238 + "uuid", 3719 3239 ] 3720 3240 3721 3241 [[package]] ··· 3734 3254 "errno", 3735 3255 "libc", 3736 3256 "linux-raw-sys", 3737 3737 - "windows-sys 0.52.0", 3257 3257 + "windows-sys 0.61.2", 3738 3258 ] 3739 3259 3740 3260 [[package]] ··· 3858 3378 version = "1.2.0" 3859 3379 source = "registry+https://github.com/rust-lang/crates.io-index" 3860 3380 checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" 3861 3861 - 3862 3862 - [[package]] 3863 3863 - name = "seahash" 3864 3864 - version = "4.1.0" 3865 3865 - source = "registry+https://github.com/rust-lang/crates.io-index" 3866 3866 - checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" 3867 3381 3868 3382 [[package]] 3869 3383 name = "sec1" ··· 4137 3651 checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" 4138 3652 4139 3653 [[package]] 4140 4140 - name = "simdutf8" 4141 4141 - version = "0.1.5" 4142 4142 - source = "registry+https://github.com/rust-lang/crates.io-index" 4143 4143 - checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" 4144 4144 - 4145 4145 - [[package]] 4146 3654 name = "siphasher" 4147 3655 version = "1.0.1" 4148 3656 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4295 3803 version = "0.11.1" 4296 3804 source = "registry+https://github.com/rust-lang/crates.io-index" 4297 3805 checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" 4298 4298 - 4299 4299 - [[package]] 4300 4300 - name = "strum" 4301 4301 - version = "0.26.3" 4302 4302 - source = "registry+https://github.com/rust-lang/crates.io-index" 4303 4303 - checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" 4304 4304 - 4305 4305 - [[package]] 4306 4306 - name = "strum" 4307 4307 - version = "0.27.2" 4308 4308 - source = "registry+https://github.com/rust-lang/crates.io-index" 4309 4309 - checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" 4310 4310 - dependencies = [ 4311 4311 - "strum_macros 0.27.2", 4312 4312 - ] 4313 4313 - 4314 4314 - [[package]] 4315 4315 - name = "strum_macros" 4316 4316 - version = "0.26.4" 4317 4317 - source = "registry+https://github.com/rust-lang/crates.io-index" 4318 4318 - checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" 4319 4319 - dependencies = [ 4320 4320 - "heck 0.5.0", 4321 4321 - "proc-macro2", 4322 4322 - "quote", 4323 4323 - "rustversion", 4324 4324 - "syn 2.0.108", 4325 4325 - ] 4326 4326 - 4327 4327 - [[package]] 4328 4328 - name = "strum_macros" 4329 4329 - version = "0.27.2" 4330 4330 - source = "registry+https://github.com/rust-lang/crates.io-index" 4331 4331 - checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" 4332 4332 - dependencies = [ 4333 4333 - "heck 0.5.0", 4334 4334 - "proc-macro2", 4335 4335 - "quote", 4336 4336 - "syn 2.0.108", 4337 4337 - ] 4338 3806 4339 3807 [[package]] 4340 3808 name = "subtle" ··· 4412 3880 checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" 4413 3881 4414 3882 [[package]] 4415 4415 - name = "tap" 4416 4416 - version = "1.0.1" 4417 4417 - source = "registry+https://github.com/rust-lang/crates.io-index" 4418 4418 - checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" 4419 4419 - 4420 4420 - [[package]] 4421 4421 - name = "tar" 4422 4422 - version = "0.4.44" 4423 4423 - source = "registry+https://github.com/rust-lang/crates.io-index" 4424 4424 - checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" 4425 4425 - dependencies = [ 4426 4426 - "filetime", 4427 4427 - "libc", 4428 4428 - "xattr", 4429 4429 - ] 4430 4430 - 4431 4431 - [[package]] 4432 3883 name = "tempfile" 4433 3884 version = "3.23.0" 4434 3885 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4438 3889 "getrandom 0.3.4", 4439 3890 "once_cell", 4440 3891 "rustix", 4441 4441 - "windows-sys 0.52.0", 3892 3892 + "windows-sys 0.61.2", 4442 3893 ] 4443 3894 4444 3895 [[package]] ··· 4533 3984 ] 4534 3985 4535 3986 [[package]] 4536 4536 - name = "tiny-keccak" 4537 4537 - version = "2.0.2" 4538 4538 - source = "registry+https://github.com/rust-lang/crates.io-index" 4539 4539 - checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" 4540 4540 - dependencies = [ 4541 4541 - "crunchy", 4542 4542 - ] 4543 4543 - 4544 4544 - [[package]] 4545 3987 name = "tinystr" 4546 3988 version = "0.8.2" 4547 3989 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4676 4118 ] 4677 4119 4678 4120 [[package]] 4679 4679 - name = "toml_datetime" 4680 4680 - version = "0.7.3" 4681 4681 - source = "registry+https://github.com/rust-lang/crates.io-index" 4682 4682 - checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" 4683 4683 - dependencies = [ 4684 4684 - "serde_core", 4685 4685 - ] 4686 4686 - 4687 4687 - [[package]] 4688 4688 - name = "toml_edit" 4689 4689 - version = "0.23.7" 4690 4690 - source = "registry+https://github.com/rust-lang/crates.io-index" 4691 4691 - checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" 4692 4692 - dependencies = [ 4693 4693 - "indexmap 2.12.0", 4694 4694 - "toml_datetime", 4695 4695 - "toml_parser", 4696 4696 - "winnow", 4697 4697 - ] 4698 4698 - 4699 4699 - [[package]] 4700 4700 - name = "toml_parser" 4701 4701 - version = "1.0.4" 4702 4702 - source = "registry+https://github.com/rust-lang/crates.io-index" 4703 4703 - checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" 4704 4704 - dependencies = [ 4705 4705 - "winnow", 4706 4706 - ] 4707 4707 - 4708 4708 - [[package]] 4709 4121 name = "tower" 4710 4122 version = "0.5.2" 4711 4123 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4888 4300 checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" 4889 4301 4890 4302 [[package]] 4891 4891 - name = "unicode-width" 4892 4892 - version = "0.2.2" 4893 4893 - source = "registry+https://github.com/rust-lang/crates.io-index" 4894 4894 - checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" 4895 4895 - 4896 4896 - [[package]] 4897 4303 name = "unicode-xid" 4898 4304 version = "0.2.6" 4899 4305 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 4953 4359 source = "registry+https://github.com/rust-lang/crates.io-index" 4954 4360 checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" 4955 4361 dependencies = [ 4362 4362 + "getrandom 0.3.4", 4956 4363 "js-sys", 4364 4364 + "rand 0.9.2", 4957 4365 "wasm-bindgen", 4958 4366 ] 4959 4367 ··· 5149 4557 source = "registry+https://github.com/rust-lang/crates.io-index" 5150 4558 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" 5151 4559 dependencies = [ 5152 5152 - "windows-sys 0.48.0", 4560 4560 + "windows-sys 0.61.2", 5153 4561 ] 5154 4562 5155 4563 [[package]] ··· 5509 4917 checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" 5510 4918 5511 4919 [[package]] 5512 5512 - name = "winnow" 5513 5513 - version = "0.7.13" 5514 5514 - source = "registry+https://github.com/rust-lang/crates.io-index" 5515 5515 - checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" 5516 5516 - dependencies = [ 5517 5517 - "memchr", 5518 5518 - ] 5519 5519 - 5520 5520 - [[package]] 5521 4920 name = "winreg" 5522 4921 version = "0.50.0" 5523 4922 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 5538 4937 version = "0.6.2" 5539 4938 source = "registry+https://github.com/rust-lang/crates.io-index" 5540 4939 checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" 5541 5541 - 5542 5542 - [[package]] 5543 5543 - name = "wyz" 5544 5544 - version = "0.5.1" 5545 5545 - source = "registry+https://github.com/rust-lang/crates.io-index" 5546 5546 - checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" 5547 5547 - dependencies = [ 5548 5548 - "tap", 5549 5549 - ] 5550 4940 5551 4941 [[package]] 5552 4942 name = "xattr"

+2 -1

Cargo.toml

reviewed

··· 9 9 axum = { version = "0.8", features = ["json"] } 10 10 chrono = { version = "0.4", features = ["serde"] } 11 11 clap = { version = "4.5", features = ["derive", "env"] } 12 12 - duckdb = { version = "1.4", features = ["bundled", "chrono", "r2d2", "uuid"] } 13 12 eyre = "0.6" 14 13 futures = "0.3.31" 15 14 inventory = "0.3.21" ··· 23 22 metrics-exporter-prometheus = { version = "0.18.0", default-features = false, features = ["http-listener"] } 24 23 mini-moka = { version = "0.10.3", features = ["sync"] } 25 24 r2d2 = "0.8" 25 25 + r2d2_sqlite = "0.31.0" 26 26 reqwest = "0.12.24" 27 27 + rusqlite = { version = "0.37.0", features = ["chrono", "uuid"] } 27 28 serde = { version = "1.0.228", features = ["derive"] } 28 29 tempfile = "3.23.0" 29 30 tokio = { version = "1.42", features = ["full"] }

+18 -17

src/analysis.rs

reviewed

··· 1 1 + use crate::SqliteConnection; 1 2 use chrono::prelude::*; 2 2 - use duckdb::{Connection, params}; 3 3 + use rusqlite::params; 3 4 4 5 #[derive(Debug)] 5 6 pub struct TopAlbumsResp { ··· 11 12 } 12 13 13 14 pub fn get_top_albums( 14 14 - conn: &Connection, 15 15 + conn: &SqliteConnection, 15 16 did: &str, 16 17 start: DateTime<Utc>, 17 18 end: DateTime<Utc>, 18 19 limit: i64, 19 19 - ) -> duckdb::Result<Vec<TopAlbumsResp>> { 20 20 + ) -> rusqlite::Result<Vec<TopAlbumsResp>> { 20 21 let mut stmt = conn.prepare_cached( 21 22 r"WITH albums AS ( 22 22 - SELECT release_mbid, count: count(*) FROM scrobbles 23 23 + SELECT release_mbid, count(*) as count FROM scrobbles 23 24 WHERE did=$1 AND created_at BETWEEN $2 AND $3 24 25 GROUP BY release_mbid 25 26 ORDER BY count DESC LIMIT $4) 26 26 - SELECT release.gid, release.name, artist_credit.name, release_group, count FROM mbz.release 27 27 + SELECT release.gid, release.name, artist_credit.name, release_group, count FROM release 27 28 INNER JOIN albums ON release_mbid = release.gid 28 28 - INNER JOIN mbz.artist_credit ON artist_credit.id = release.artist_credit 29 29 + INNER JOIN artist_credit ON artist_credit.id = release.artist_credit 29 30 ORDER BY count DESC", 30 31 )?; 31 32 ··· 51 52 pub count: i64, 52 53 } 53 54 pub fn get_top_artists( 54 54 - conn: &Connection, 55 55 + conn: &SqliteConnection, 55 56 did: &str, 56 57 start: DateTime<Utc>, 57 58 end: DateTime<Utc>, 58 59 limit: i64, 59 59 - ) -> duckdb::Result<Vec<TopArtistResp>> { 60 60 + ) -> rusqlite::Result<Vec<TopArtistResp>> { 60 61 let mut stmt = conn.prepare_cached( 61 62 r"WITH artists AS ( 62 62 - SELECT id: artist_credit_name.artist, count: count(*) FROM scrobbles 63 63 - INNER JOIN mbz.track ON track.gid = track_mbid 64 64 - INNER JOIN mbz.artist_credit_name ON artist_credit_name.artist_credit = track.artist_credit 63 63 + SELECT artist_credit_name.artist as id, count(*) as count FROM scrobbles 64 64 + INNER JOIN track ON track.gid = track_mbid 65 65 + INNER JOIN artist_credit_name ON artist_credit_name.artist_credit = track.artist_credit 65 66 WHERE did=$1 AND created_at BETWEEN $2 AND $3 66 67 GROUP BY artist_credit_name.artist 67 68 ORDER BY count DESC limit $4) 68 68 - SELECT gid, name, count FROM mbz.artist 69 69 + SELECT gid, name, count FROM artist 69 70 INNER JOIN artists ON artists.id = artist.id 70 71 ORDER BY count DESC" 71 72 )?; ··· 92 93 } 93 94 94 95 pub fn get_top_tracks( 95 95 - conn: &Connection, 96 96 + conn: &SqliteConnection, 96 97 did: &str, 97 98 start: DateTime<Utc>, 98 99 end: DateTime<Utc>, 99 100 limit: i64, 100 100 - ) -> duckdb::Result<Vec<TopTracksResp>> { 101 101 + ) -> rusqlite::Result<Vec<TopTracksResp>> { 101 102 let mut stmt = conn.prepare_cached( 102 103 r"WITH tracks AS ( 103 103 - SELECT track_mbid, count: count(*) FROM scrobbles 104 104 + SELECT track_mbid, count(*) as count FROM scrobbles 104 105 WHERE did=$1 AND created_at BETWEEN $2 AND $3 105 106 GROUP BY track_mbid 106 107 ORDER BY count DESC LIMIT $4) 107 107 - SELECT track.gid, track.name, artist_credit.name, count FROM mbz.track 108 108 + SELECT track.gid, track.name, artist_credit.name, count FROM track 108 109 INNER JOIN tracks on track_mbid = track.gid 109 109 - INNER JOIN mbz.artist_credit ON artist_credit.id = track.artist_credit 110 110 + INNER JOIN artist_credit ON artist_credit.id = track.artist_credit 110 111 ORDER BY count DESC", 111 112 )?; 112 113

+4 -9

src/ingest/mod.rs

reviewed

··· 1 1 - use duckdb::DuckdbConnectionManager; 1 1 + use crate::SqlitePool; 2 2 use futures::StreamExt; 3 3 use jacquard::StreamErrorKind; 4 4 use jacquard::jetstream::CommitOperation; ··· 6 6 use jacquard::xrpc::{SubscriptionClient, TungsteniteSubscriptionClient}; 7 7 use jacquard_api::app_rocksky::scrobble::Scrobble as RockskyScrobble; 8 8 use jacquard_api::fm_teal::alpha::feed::play::Play as TealAlphaPlay; 9 9 - use std::sync::Arc; 10 9 use tracing::instrument; 11 10 12 11 mod scrobbles; 13 12 mod tap; 14 13 15 15 - pub async fn tapstream(db: Arc<DuckdbConnectionManager>, tap: String) -> eyre::Result<()> { 14 14 + pub async fn tapstream(db: SqlitePool, tap: String) -> eyre::Result<()> { 16 15 let client = TungsteniteSubscriptionClient::from_base_uri(tap.parse()?); 17 16 18 17 let stream = client.subscribe(&tap::TapParams {}).await?; ··· 50 49 Ok(()) 51 50 } 52 51 53 53 - fn handle_message(db: &DuckdbConnectionManager, message: tap::TapMessage<'_>) -> eyre::Result<()> { 52 52 + fn handle_message(db: &SqlitePool, message: tap::TapMessage<'_>) -> eyre::Result<()> { 54 53 match message { 55 54 tap::TapMessage::Record { id, record, .. } => handle_record(db, id, record), 56 55 tap::TapMessage::User { id, user, .. } => handle_user(id, user), ··· 58 57 } 59 58 60 59 #[instrument(skip(db, commit), fields(nsid=commit.collection.as_str(), did=commit.did.as_str(), rkey=commit.rkey.as_str()))] 61 61 - fn handle_record( 62 62 - db: &DuckdbConnectionManager, 63 63 - id: i64, 64 64 - commit: tap::TapRecord<'_>, 65 65 - ) -> eyre::Result<()> { 60 60 + fn handle_record(db: &SqlitePool, id: i64, commit: tap::TapRecord<'_>) -> eyre::Result<()> { 66 61 if commit.action == CommitOperation::Delete { 67 62 return Ok(()); 68 63 }

+19 -15

src/ingest/scrobbles.rs

reviewed

··· 1 1 use crate::mbz::{FindMbzData, try_find_mbz_data}; 2 2 + use crate::{SqliteConnection, SqlitePool}; 2 3 use chrono::prelude::*; 3 4 use chrono::{DurationRound, TimeDelta}; 4 4 - use duckdb::{Connection, DuckdbConnectionManager, OptionalExt, params}; 5 5 use jacquard::types::datetime::Datetime; 6 6 use jacquard_api::app_rocksky::scrobble::Scrobble as RockskyScrobble; 7 7 use jacquard_api::fm_teal::alpha::feed::play::Play as TealAlphaPlay; 8 8 - use r2d2::ManageConnection; 8 8 + use rusqlite::{OptionalExtension, params}; 9 9 10 10 pub fn scrobble_teal<'a>( 11 11 - db: &DuckdbConnectionManager, 11 11 + db: &SqlitePool, 12 12 did: &str, 13 13 rkey: &str, 14 14 scrobble: TealAlphaPlay<'a>, 15 15 ) -> eyre::Result<()> { 16 16 - let conn = db.connect()?; 16 16 + let conn = db.get()?; 17 17 let created = scrobble.played_time.clone().unwrap_or(Datetime::now()); 18 18 let created = created.as_ref().to_utc(); 19 19 ··· 51 51 } 52 52 53 53 pub fn scrobble_rocksky<'a>( 54 54 - db: &DuckdbConnectionManager, 54 54 + db: &SqlitePool, 55 55 did: &str, 56 56 rkey: &str, 57 57 scrobble: RockskyScrobble<'a>, 58 58 ) -> eyre::Result<()> { 59 59 - let conn = db.connect()?; 59 59 + let conn = db.get()?; 60 60 let created = scrobble.created_at.as_ref().to_utc(); 61 61 62 62 if let Some(rkey) = check_duplicate_scrobble(&conn, did, &scrobble.title, created)? { ··· 80 80 } 81 81 82 82 fn resolve_and_insert_scrobble( 83 83 - conn: &Connection, 83 83 + conn: &SqliteConnection, 84 84 did: &str, 85 85 rkey: &str, 86 86 track: &str, 87 87 find: FindMbzData<'_>, 88 88 created: DateTime<Utc>, 89 89 - ) -> duckdb::Result<()> { 89 89 + ) -> rusqlite::Result<()> { 90 90 + let now = Utc::now(); 91 91 + 90 92 if let Some(data) = try_find_mbz_data(conn, &find)? { 91 93 conn.execute( 92 94 r"INSERT INTO scrobbles (did, rkey, track_name, track_mbid, release_name, 93 93 - release_mbid, release_group_name, release_group_mbid, artists, created_at, debug) 94 94 - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT DO NOTHING", 95 95 + release_mbid, release_group_name, release_group_mbid, artists, created_at, indexed_at, debug) 96 96 + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12) ON CONFLICT DO NOTHING", 95 97 params![ 96 98 did, 97 99 rkey, ··· 103 105 data.release_group_gid, 104 106 data.artists, 105 107 created, 108 108 + now, 106 109 data.debug, 107 110 ], 108 111 )?; 109 112 } else { 110 113 // oh dear, oh dear, oh dear - run with the data from the search... 111 114 conn.execute( 112 112 - r"INSERT INTO scrobbles (did, rkey, track_name, release_name, release_mbid, artists, created_at) 113 113 - VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT DO NOTHING", 115 115 + r"INSERT INTO scrobbles (did, rkey, track_name, release_name, release_mbid, artists, created_at, indexed_at) 116 116 + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) ON CONFLICT DO NOTHING", 114 117 params![ 115 118 did, 116 119 rkey, ··· 119 122 find.release_mbid, 120 123 find.artist_name, 121 124 created, 125 125 + now, 122 126 ], 123 127 )?; 124 128 } ··· 127 131 } 128 132 129 133 fn check_duplicate_scrobble( 130 130 - conn: &Connection, 134 134 + conn: &SqliteConnection, 131 135 did: &str, 132 136 track: &str, 133 137 created: DateTime<Utc>, 134 134 - ) -> duckdb::Result<Option<String>> { 138 138 + ) -> eyre::Result<Option<String>> { 135 139 // try to round the timestamp down to just seconds - if it fails, just use the og timestamp. 136 140 let created = created 137 141 .duration_trunc(TimeDelta::seconds(1)) 138 142 .unwrap_or(created); 139 143 let mut stmt = conn.prepare_cached( 140 140 - "SELECT rkey FROM scrobbles WHERE did = $1 AND track_name = $2 AND created_at = $3", 144 144 + "SELECT rkey FROM scrobbles WHERE did = ?1 AND track_name = ?2 AND created_at = ?3", 141 145 )?; 142 146 143 147 let rkey = stmt

+2 -2

src/init.sql

reviewed

··· 13 13 14 14 debug text, 15 15 16 16 - created_at timestamptz not null, 17 17 - indexed_at timestamptz not null default now(), 16 16 + created_at datetime not null, 17 17 + indexed_at datetime not null, 18 18 19 19 primary key (did, rkey) 20 20 );

+14 -15

src/main.rs

reviewed

··· 1 1 use clap::Parser; 2 2 - use std::sync::Arc; 3 2 use metrics_exporter_prometheus::PrometheusBuilder; 4 4 - use r2d2::ManageConnection; 3 3 + use r2d2_sqlite::SqliteConnectionManager; 5 4 6 5 mod analysis; 7 6 mod config; ··· 11 10 mod server; 12 11 mod utils; 13 12 13 13 + pub type SqlitePool = r2d2::Pool<SqliteConnectionManager>; 14 14 + pub type SqliteConnection = r2d2::PooledConnection<SqliteConnectionManager>; 15 15 + 14 16 #[tokio::main] 15 17 async fn main() -> eyre::Result<()> { 16 18 tracing_subscriber::fmt::init(); ··· 18 20 19 21 let config = config::Config::parse(); 20 22 21 21 - let ddb = duckdb::DuckdbConnectionManager::file(config.db)?; 22 22 - { 23 23 - let conn = ddb.connect()?; 24 24 - conn.execute_batch(include_str!("init.sql"))?; 25 25 - if config.duckdb_ui { 26 26 - conn.execute("CALL start_ui_server();", [])?; 27 27 - } 28 28 - } 23 23 + let manager = SqliteConnectionManager::file(&config.db).with_init(|conn| { 24 24 + conn.pragma_update(None, "journal_mode", "WAL")?; 25 25 + conn.pragma_update(None, "synchronous", "NORMAL")?; 26 26 + conn.pragma_update(None, "cache_size", -16 * 1024)?; // 64MB 27 27 + conn.execute_batch(include_str!("init.sql")) 28 28 + }); 29 29 + let pool = r2d2::Pool::new(manager)?; 29 30 30 30 - let ddb = Arc::new(ddb); 31 31 + let replication_agent = mbz::ReplicationAgent::new(pool.clone(), config.db, config.mb_agent); 31 32 32 32 - let replication_agent = mbz::ReplicationAgent::new(ddb.clone(), config.mb_agent); 33 33 - 34 33 let mut tasks = tokio::task::JoinSet::new(); 35 34 36 36 - tasks.spawn(server::start_server(ddb.clone(), config.port)); 37 37 - tasks.spawn(ingest::tapstream(ddb, config.tap)); 35 35 + tasks.spawn(server::start_server(pool.clone(), config.port)); 36 36 + tasks.spawn(ingest::tapstream(pool, config.tap)); 38 37 tasks.spawn(replication_agent.start()); 39 38 40 39 tasks.join_all().await;

+47 -59

src/mbz/find_by_fts.sql

reviewed

··· 1 1 - with 2 2 - releases as ( 3 3 - select *, rel_score: mbz.fts_release ($1, $2) 4 4 - from mbz.release 5 5 - where 6 6 - ( 7 7 - rel_score is not null 8 8 - and rel_score > 5 9 9 - and $2 IS NULL 10 10 - ) 11 11 - or ( 12 12 - $2 IS NOT NULL 13 13 - and release.gid = $2 14 14 - ) 15 15 - ), 16 16 - recordings as ( 17 17 - select *, rec_score: mbz.fts_recording ($3, $4) 18 18 - from mbz.recording 19 19 - where 20 20 - ( 21 21 - rec_score is not null 22 22 - and rec_score > 5 23 23 - and $4 IS NULL 24 24 - ) 25 25 - or ( 26 26 - $4 IS NOT NULL 27 27 - and recording.gid = $4 28 28 - ) 29 29 - ), 30 30 - artists as ( 31 31 - select *, artist_score: mbz.fts_artists ($5, coalesce($2, $4)), 32 32 - from mbz.artist_credit 33 33 - where 34 34 - artist_score is not null 35 35 - and artist_score > 2.5 36 36 - ) 37 37 - select 38 38 - track_name: track.name, 39 39 - track_gid: track.gid, 40 40 - release_gid: releases.gid, 41 41 - release_name: releases.name, 42 42 - recording_gid: recordings.gid, 43 43 - release_grp_gid: release_group.gid, 44 44 - release_grp_name: release_group.name, 45 45 - artists: artists.name, 46 46 - rel_score, 47 47 - rec_score, 48 48 - artist_score: coalesce(artist_score, 0) 49 49 - from mbz.track 50 50 - inner join recordings on recordings.id = track.recording 51 51 - inner join mbz.medium on medium.id = track.medium 52 52 - inner join releases on releases.id = medium.release 53 53 - inner join mbz.release_group on release_group.id = releases.release_group 54 54 - left join artists on artists.id = recordings.artist_credit 55 55 - where is_data_track = false 56 56 - order by 57 57 - rel_score desc, 58 58 - rec_score desc, 59 59 - artist_score desc 1 1 + with releases as (select "release".*, rank 2 2 + from fts_release 3 3 + left join "release" on id = fts_release.ROWID 4 4 + where fts_release.name match ?1 5 5 + and ?2 IS NULL 6 6 + union 7 7 + select *, 0 as rank 8 8 + from "release" 9 9 + where gid = ?2 10 10 + and ?2 IS NOT NULL 11 11 + order by rank), 12 12 + recordings as (select recording.*, rank 13 13 + from fts_recording 14 14 + left join recording on id = fts_recording.ROWID 15 15 + where fts_recording.name match ?3 16 16 + and ?4 IS NULL 17 17 + union 18 18 + select *, 0 as rank 19 19 + from recording 20 20 + where gid = ?4 21 21 + and ?4 IS NOT NULL 22 22 + order by rank), 23 23 + artists as (select artist_credit.*, rank 24 24 + from fts_artist_credit 25 25 + inner join artist_credit on artist_credit.id = fts_artist_credit.ROWID 26 26 + where fts_artist_credit.name match ?5 27 27 + and ?2 IS NULL 28 28 + and ?4 IS NULL 29 29 + order by rank) 30 30 + select track.name, 31 31 + track.gid, 32 32 + releases.gid, 33 33 + releases.name, 34 34 + recordings.gid, 35 35 + release_group.gid, 36 36 + release_group.name, 37 37 + artists.name, 38 38 + coalesce(releases.rank,0), 39 39 + coalesce(recordings.rank,0), 40 40 + coalesce(artists.rank,0) 41 41 + from track 42 42 + inner join recordings on recordings.id = track.recording 43 43 + inner join medium on medium.id = track.medium 44 44 + inner join releases on releases.id = medium.release 45 45 + inner join release_group on release_group.id = releases.release_group 46 46 + left join artists on artists.id = recordings.artist_credit 47 47 + order by releases.rank, recordings.rank, artists.rank 60 48 limit 10;

+31 -41

src/mbz/find_by_isrc.sql

reviewed

··· 1 1 - with 2 2 - releases as ( 3 3 - select *, rel_score: mbz.fts_release ($1, $2) 4 4 - from mbz.release 5 5 - where 6 6 - ( 7 7 - rel_score is not null 8 8 - and rel_score > 2 9 9 - and $2 IS NULL 10 10 - ) 11 11 - or ( 12 12 - $2 IS NOT NULL 13 13 - and release.gid = $2 14 14 - ) 15 15 - ) 16 16 - select 17 17 - track_name: track.name, 18 18 - track_gid: track.gid, 19 19 - release_gid: releases.gid, 20 20 - release_name: releases.name, 21 21 - recording_gid: recording.gid, 22 22 - release_grp_gid: release_group.gid, 23 23 - release_grp_name: release_group.name, 24 24 - artists: artist_credit.name, 25 25 - rel_score, 26 26 - rec_score: 10, 27 27 - artist_score: 10 28 28 - from mbz.track 29 29 - inner join mbz.isrc on isrc.recording = track.recording 30 30 - inner join mbz.recording on recording.id = isrc.recording 31 31 - inner join mbz.medium on medium.id = track.medium 32 32 - inner join releases on releases.id = medium.release 33 33 - inner join mbz.release_group on release_group.id = releases.release_group 34 34 - left join mbz.artist_credit on artist_credit.id = recording.artist_credit 35 35 - where 36 36 - is_data_track = false 37 37 - and isrc.isrc = $3 38 38 - order by 39 39 - rel_score desc, 40 40 - rec_score desc, 41 41 - artist_score desc 1 1 + with releases as (select "release".*, rank 2 2 + from fts_release 3 3 + left join "release" on id = fts_release.ROWID 4 4 + where fts_release.name match ?1 5 5 + and ?2 IS NULL 6 6 + union 7 7 + select *, 0 as rank 8 8 + from "release" 9 9 + where gid = ?2 10 10 + and ?2 IS NOT NULL 11 11 + order by rank) 12 12 + select track.name, 13 13 + track.gid, 14 14 + releases.gid, 15 15 + releases.name, 16 16 + recording.gid, 17 17 + release_group.gid, 18 18 + release_group.name, 19 19 + artist_credit.name, 20 20 + coalesce(releases.rank, 0), 21 21 + 0, 22 22 + 0 23 23 + from track 24 24 + inner join isrc on isrc.recording = track.recording 25 25 + inner join recording on recording.id = isrc.recording 26 26 + inner join medium on medium.id = track.medium 27 27 + inner join releases on releases.id = medium.release 28 28 + inner join release_group on release_group.id = releases.release_group 29 29 + left join artist_credit on artist_credit.id = recording.artist_credit 30 30 + where isrc.isrc = ?3 31 31 + order by releases.rank desc 42 32 limit 10;

-27

src/mbz/find_by_mbid_all.sql

reviewed

··· 1 1 - select 2 2 - track_name: track.name, 3 3 - track_gid: track.gid, 4 4 - release_gid: release.gid, 5 5 - release_name: release.name, 6 6 - recording_gid: recording.gid, 7 7 - release_grp_gid: release_group.gid, 8 8 - release_grp_name: release_group.name, 9 9 - artists: artist_credit.name, 10 10 - rel_score: 10, 11 11 - rec_score: 10, 12 12 - artist_score: 10 13 13 - from mbz.track 14 14 - inner join mbz.recording on recording.id = track.recording 15 15 - inner join mbz.medium on medium.id = track.medium 16 16 - inner join mbz.release on release.id = medium.release 17 17 - inner join mbz.release_group on release_group.id = release.release_group 18 18 - left join mbz.artist_credit on artist_credit.id = recording.artist_credit 19 19 - where 20 20 - is_data_track = false 21 21 - and recording.gid = $1 22 22 - and release.gid = $2 23 23 - order by 24 24 - rel_score desc, 25 25 - rec_score desc, 26 26 - artist_score desc 27 27 - limit 10;

+119 -123

src/mbz/init.sql

reviewed

··· 1 1 - install fts; 2 2 - load fts; 3 3 - 4 1 begin; 5 5 - create schema mbz; 6 2 7 3 -- TODO MAYBE: l_*_* link tables? 8 4 -- TODO MAYBE: label, label_alias, label_gid_redirect, release_label ?? 9 5 10 10 - create table mbz.artist 6 6 + create table if not exists artist 11 7 ( 12 8 id bigint primary key, 13 13 - gid uuid not null, 14 14 - name text not null, 15 15 - sort_name text not null, 9 9 + gid text not null, 10 10 + name text not null, 11 11 + sort_name text not null, 16 12 begin_date_year int, 17 13 begin_date_month int, 18 14 begin_date_day int, ··· 24 20 gender int, 25 21 comment text, 26 22 edits_pending int, 27 27 - last_update timestamptz, 28 28 - ended bool not null, 23 23 + last_update datetime, 24 24 + ended text not null, 29 25 begin_area int, 30 30 - end_area int, 26 26 + end_area int 31 27 ); 32 28 33 33 - create table mbz.artist_alias 29 29 + create table if not exists artist_alias 34 30 ( 35 31 id bigint primary key, 36 36 - artist bigint not null, 37 37 - name text not null, 32 32 + artist bigint not null references artist (id), 33 33 + name text not null, 38 34 locale text, 39 35 edits_pending int, 40 40 - last_update timestamptz, 36 36 + last_update datetime, 41 37 type int, 42 42 - sort_name text not null, 38 38 + sort_name text not null, 43 39 begin_date_year int, 44 40 begin_date_month int, 45 41 begin_date_day int, 46 42 end_date_year int, 47 43 end_date_month int, 48 44 end_date_day int, 49 49 - primary_for_locale bool not null, 50 50 - ended bool not null, 45 45 + primary_for_locale text not null, 46 46 + ended text not null 51 47 ); 52 48 53 53 - create table mbz.artist_credit 49 49 + create table if not exists artist_credit 54 50 ( 55 51 id bigint primary key, 56 56 - name text not null, 57 57 - artist_count int not null, 58 58 - ref_count int not null, 59 59 - created timestamptz not null, 52 52 + name text not null, 53 53 + artist_count int not null, 54 54 + ref_count int not null, 55 55 + created datetime not null, 60 56 edits_pending int, 61 61 - gid uuid not null, 57 57 + gid text not null 62 58 ); 63 59 64 64 - create table mbz.artist_credit_name 60 60 + create table if not exists artist_credit_name 65 61 ( 66 66 - artist_credit bigint not null, 62 62 + artist_credit bigint not null references artist_credit (id), 67 63 position int not null, 68 68 - artist bigint not null, 64 64 + artist bigint not null references artist (id), 69 65 name text not null, 70 66 join_phrase text, 71 67 72 68 primary key (artist_credit, position) 73 69 ); 74 70 75 75 - create table mbz.artist_gid_redirect 71 71 + create table if not exists artist_gid_redirect 76 72 ( 77 73 gid text primary key, 78 78 - new_id bigint not null, 79 79 - created timestamptz not null 74 74 + new_id bigint not null references artist (id), 75 75 + created datetime not null 80 76 ); 81 77 82 82 - create table mbz.artist_tag 78 78 + create table if not exists artist_tag 83 79 ( 84 84 - artist bigint not null, 85 85 - tag bigint not null, 80 80 + artist bigint not null references artist (id), 81 81 + tag bigint not null references tag (id), 86 82 count int, 87 87 - last_update timestamptz, 83 83 + last_update datetime, 88 84 89 85 primary key (artist, tag) 90 86 ); 91 87 92 92 - create table mbz.genre 88 88 + 89 89 + create table if not exists genre 93 90 ( 94 91 id bigint primary key, 95 95 - gid uuid not null, 92 92 + gid text not null, 96 93 name text not null, 97 94 comment text, 98 95 edits_pending int, 99 99 - last_update timestamptz 96 96 + last_update datetime 100 97 ); 101 98 102 102 - create table mbz.genre_alias 99 99 + create table if not exists genre_alias 103 100 ( 104 101 id bigint primary key, 105 105 - genre bigint not null, 106 106 - name text not null, 102 102 + genre bigint not null references genre (id), 103 103 + name text not null, 107 104 locale text, 108 105 edits_pending int, 109 109 - last_update timestamptz, 106 106 + last_update datetime, 110 107 type int, 111 111 - sort_name text not null, 108 108 + sort_name text not null, 112 109 begin_date_year int, 113 110 begin_date_month int, 114 111 begin_date_day int, 115 112 end_date_year int, 116 113 end_date_month int, 117 114 end_date_day int, 118 118 - primary_for_locale bool not null, 119 119 - ended bool not null, 115 115 + primary_for_locale text not null, 116 116 + ended text not null 120 117 ); 121 118 122 122 - create table mbz.isrc 119 119 + create table if not exists isrc 123 120 ( 124 121 id bigint primary key, 125 125 - recording bigint not null, 122 122 + recording bigint not null references recording (id), 126 123 isrc text not null, 127 124 source int, 128 125 edits_pending int, 129 129 - created timestamptz 126 126 + created datetime 130 127 ); 131 128 132 132 - create table mbz.medium 129 129 + create table if not exists medium 133 130 ( 134 131 id bigint primary key, 135 135 - release bigint not null, 132 132 + "release" bigint not null references "release" (id), 136 133 position int not null, 137 134 format int, 138 135 name text, 139 136 edits_pending int, 140 140 - last_update timestamptz, 137 137 + last_update datetime, 141 138 track_count int, 142 142 - gid uuid not null, 139 139 + gid text not null 143 140 ); 144 141 145 145 - create table mbz.recording 142 142 + create table if not exists recording 146 143 ( 147 144 id bigint primary key, 148 148 - gid uuid not null, 149 149 - name text not null, 150 150 - artist_credit bigint not null, 145 145 + gid text not null, 146 146 + name text not null, 147 147 + artist_credit bigint not null references artist_credit (id), 151 148 length int, 152 149 comment text, 153 150 edits_pending int, 154 154 - last_update timestamptz, 155 155 - video bool not null, 151 151 + last_update datetime, 152 152 + video text not null 156 153 ); 157 154 158 158 - create table mbz.recording_alias 155 155 + create table if not exists recording_alias 159 156 ( 160 157 id bigint primary key, 161 161 - recording bigint not null, 162 162 - name text not null, 158 158 + recording bigint not null references recording (id), 159 159 + name text not null, 163 160 locale text, 164 161 edits_pending int, 165 165 - last_update timestamptz, 162 162 + last_update datetime, 166 163 type int, 167 167 - sort_name text not null, 164 164 + sort_name text not null, 168 165 begin_date_year int, 169 166 begin_date_month int, 170 167 begin_date_day int, 171 168 end_date_year int, 172 169 end_date_month int, 173 170 end_date_day int, 174 174 - primary_for_locale bool not null, 175 175 - ended bool not null, 171 171 + primary_for_locale text not null, 172 172 + ended text not null 176 173 ); 177 174 178 178 - create table mbz.recording_gid_redirect 175 175 + create table if not exists recording_gid_redirect 179 176 ( 180 177 gid text primary key, 181 181 - new_id bigint not null, 182 182 - created timestamptz not null 178 178 + new_id bigint not null references recording (id), 179 179 + created datetime not null 183 180 ); 184 181 185 185 - create table mbz.recording_tag 182 182 + create table if not exists recording_tag 186 183 ( 187 187 - recording bigint not null, 188 188 - tag bigint not null, 184 184 + recording bigint not null references recording (id), 185 185 + tag bigint not null references tag (id), 189 186 count int, 190 190 - last_updated timestamptz, 187 187 + last_updated datetime, 191 188 192 189 primary key (recording, tag) 193 190 ); 194 191 195 195 - create table mbz.release 192 192 + create table if not exists "release" 196 193 ( 197 194 id bigint primary key, 198 198 - gid uuid not null, 195 195 + gid text not null, 199 196 name text not null, 200 200 - artist_credit bigint not null, 201 201 - release_group bigint not null, 197 197 + artist_credit bigint not null references artist_credit (id), 198 198 + release_group bigint not null references release_group (id), 202 199 status int, 203 200 packaging int, 204 201 language int, ··· 207 204 comment text, 208 205 edits_pending int, 209 206 quality int, 210 210 - last_update timestamptz 207 207 + last_update datetime 211 208 ); 212 209 213 213 - create table mbz.release_alias 210 210 + create table if not exists release_alias 214 211 ( 215 212 id bigint primary key, 216 216 - release bigint not null, 217 217 - name text not null, 213 213 + "release" bigint not null references "release" (id), 214 214 + name text not null, 218 215 locale text, 219 216 edits_pending int, 220 220 - last_update timestamptz, 217 217 + last_update datetime, 221 218 type int, 222 222 - sort_name text not null, 219 219 + sort_name text not null, 223 220 begin_date_year int, 224 221 begin_date_month int, 225 222 begin_date_day int, 226 223 end_date_year int, 227 224 end_date_month int, 228 225 end_date_day int, 229 229 - primary_for_locale bool not null, 230 230 - ended bool not null, 226 226 + primary_for_locale text not null, 227 227 + ended text not null 231 228 ); 232 229 233 233 - create table mbz.release_gid_redirect 230 230 + create table if not exists release_gid_redirect 234 231 ( 235 232 gid text primary key, 236 236 - new_id bigint not null, 237 237 - created timestamptz not null 233 233 + new_id bigint not null, 234 234 + created datetime not null 238 235 ); 239 236 240 240 - create table mbz.release_group 237 237 + create table if not exists release_group 241 238 ( 242 239 id bigint primary key, 243 243 - gid uuid not null, 240 240 + gid text not null, 244 241 name text not null, 245 245 - artist_credit bigint not null, 242 242 + artist_credit bigint not null references artist_credit (id), 246 243 type int, 247 244 comment text, 248 245 edits_pending int, 249 249 - last_update timestamptz 246 246 + last_update datetime 250 247 ); 251 248 252 252 - create table mbz.release_group_alias 249 249 + create table if not exists release_group_alias 253 250 ( 254 251 id bigint primary key, 255 255 - release_group bigint not null, 256 256 - name text not null, 252 252 + release_group bigint not null references release_group (id), 253 253 + name text not null, 257 254 locale text, 258 255 edits_pending int, 259 259 - last_update timestamptz, 256 256 + last_update datetime, 260 257 type int, 261 261 - sort_name text not null, 258 258 + sort_name text not null, 262 259 begin_date_year int, 263 260 begin_date_month int, 264 261 begin_date_day int, 265 262 end_date_year int, 266 263 end_date_month int, 267 264 end_date_day int, 268 268 - primary_for_locale bool not null, 269 269 - ended bool not null, 265 265 + primary_for_locale text not null, 266 266 + ended text not null 270 267 ); 271 268 272 272 - create table mbz.release_group_gid_redirect 269 269 + create table if not exists release_group_gid_redirect 273 270 ( 274 271 gid text primary key, 275 275 - new_id bigint not null, 276 276 - created timestamptz not null 272 272 + new_id bigint not null, 273 273 + created datetime not null 277 274 ); 278 275 279 279 - create table mbz.release_group_tag 276 276 + create table if not exists release_group_tag 280 277 ( 281 281 - release_group bigint not null, 282 282 - tag bigint not null, 278 278 + release_group bigint not null references release_group (id), 279 279 + tag bigint not null references tag (id), 283 280 count int, 284 284 - last_update timestamptz, 281 281 + last_update datetime, 285 282 286 283 primary key (release_group, tag) 287 284 ); 288 285 289 289 - create table mbz.release_tag 286 286 + create table if not exists release_tag 290 287 ( 291 291 - release bigint not null, 292 292 - tag bigint not null, 288 288 + "release" bigint not null references "release" (id), 289 289 + tag bigint not null references tag (id), 293 290 count int, 294 294 - last_update timestamptz, 291 291 + last_update datetime, 295 292 296 296 - primary key (release, tag) 293 293 + primary key ("release", tag) 297 294 ); 298 295 299 299 - create table mbz.tag 296 296 + create table if not exists tag 300 297 ( 301 298 id bigint primary key, 302 299 name text not null, 303 303 - ref_count bigint, 300 300 + ref_count bigint 304 301 ); 305 302 306 306 - create table mbz.track 303 303 + create table if not exists track 307 304 ( 308 305 id bigint primary key, 309 309 - gid uuid not null, 310 310 - recording bigint not null, 311 311 - medium bigint not null, 306 306 + gid text not null, 307 307 + recording bigint not null references recording (id), 308 308 + medium bigint not null references medium (id), 312 309 position int, 313 310 number text, 314 314 - name text not null, 315 315 - artist_credit bigint not null, 311 311 + name text not null, 312 312 + artist_credit bigint not null references artist_credit (id), 316 313 length int, 317 314 edits_pending int, 318 318 - last_updated timestamptz, 319 319 - is_data_track bool not null 320 320 - 315 315 + last_updated datetime, 316 316 + is_data_track text not null 321 317 ); 322 318 323 323 - create table mbz.track_gid_redirect 319 319 + create table if not exists track_gid_redirect 324 320 ( 325 321 gid text primary key, 326 326 - new_id bigint not null, 327 327 - created timestamptz not null 322 322 + new_id bigint not null, 323 323 + created datetime not null 328 324 ); 329 325 commit;

+6 -18

src/mbz/init_fts.sql

reviewed

··· 1 1 - PRAGMA create_fts_index('mbz.artist_credit', 'id', 'name'); 2 2 - PRAGMA create_fts_index('mbz.recording', 'id', 'name'); 3 3 - PRAGMA create_fts_index('mbz.release', 'id', 'name', 'comment'); 4 4 - 5 5 - create macro if not exists mbz.fts_release (term, gid) AS CASE 6 6 - WHEN gid IS NOT NULL THEN 10 7 7 - ELSE fts_mbz_release.match_bm25 (release.id, term) * if (lower(release.name) = lower(term), 2.5, 1) 8 8 - END; 9 9 - 10 10 - create macro if not exists mbz.fts_recording (term, gid) AS CASE 11 11 - WHEN gid IS NOT NULL THEN 10 12 12 - ELSE fts_mbz_recording.match_bm25 (recording.id, term) * if (lower(recording.name) = lower(term), 2.5, 1) 13 13 - END; 1 1 + create virtual table fts_artist_credit using fts5(name, content='artist_credit', content_rowid='id'); 2 2 + create virtual table fts_recording using fts5(name, content='recording', content_rowid='id'); 3 3 + create virtual table fts_release using fts5(name, comment, content='release', content_rowid='id'); 14 4 15 15 - -- like the above but `trig` takes the release/recording mbid/isrc to disable the query 16 16 - create macro if not exists mbz.fts_artists(term, trig) AS CASE 17 17 - WHEN trig IS NOT NULL THEN 10 18 18 - ELSE fts_mbz_artist_credit.match_bm25(artist_credit.id, term) * if (lower(artist_credit.name) = lower(term), 2.5, 1) 19 19 - END; 5 5 + insert into fts_artist_credit(fts_artist_credit) values ('rebuild'); 6 6 + insert into fts_recording(fts_recording) values ('rebuild'); 7 7 + insert into fts_release(fts_release) values ('rebuild');

+48

src/mbz/init_index.sql

reviewed

··· 1 1 + create index if not exists artist__gid_idx on artist (gid); 2 2 + 3 3 + create index if not exists artist_alias__artist_idx on artist_alias (artist); 4 4 + 5 5 + create index if not exists artist_credit_name__artist_credit_idx on artist_credit_name (artist_credit); 6 6 + create index if not exists artist_credit_name__artist_idx on artist_credit_name (artist_credit); 7 7 + 8 8 + create index if not exists artist_tag__tag_idx on artist_tag (tag); 9 9 + create index if not exists artist_tag__artist_idx on artist_tag (artist); 10 10 + 11 11 + create index if not exists genre__gid_idx on genre (gid); 12 12 + 13 13 + create index if not exists genre_alias__genre_idx on genre_alias (genre); 14 14 + 15 15 + create index if not exists isrc__recording_idx on isrc (recording); 16 16 + 17 17 + create index if not exists medium__gid_idx on medium (gid); 18 18 + create index if not exists medium__release_idx on medium ("release"); 19 19 + 20 20 + create index if not exists recording__artist_credit_idx on recording (artist_credit); 21 21 + create index if not exists recording__gid_idx on recording (gid); 22 22 + 23 23 + create index if not exists recording_alias__recording_idx on recording_alias (recording); 24 24 + 25 25 + create index if not exists recording_tag__tag_idx on recording_tag (tag); 26 26 + create index if not exists recording_tag__recording_idx on recording_tag (recording); 27 27 + 28 28 + create index if not exists release__artist_credit_idx on "release" (artist_credit); 29 29 + create index if not exists release__gid_idx on "release" (gid); 30 30 + create index if not exists release__release_group_idx on "release" (release_group); 31 31 + 32 32 + create index if not exists release_alias__recording_idx on release_alias ("release"); 33 33 + 34 34 + create index if not exists release_group__artist_credit_idx on release_group (artist_credit); 35 35 + create index if not exists release_group__gid_idx on release_group (gid); 36 36 + 37 37 + create index if not exists release_group_alias__release_group_idx on release_group_alias (release_group); 38 38 + 39 39 + create index if not exists release_group_tag__tag_idx on release_group_tag (tag); 40 40 + create index if not exists release_group_tag__release_group_idx on release_group_tag (release_group); 41 41 + 42 42 + create index if not exists release_tag__tag_idx on release_tag (tag); 43 43 + create index if not exists release_tag__release_idx on release_tag ("release"); 44 44 + 45 45 + create index if not exists track__artist_credit_idx on track ("artist_credit"); 46 46 + create index if not exists track__gid_idx on track (gid); 47 47 + create index if not exists track__medium_idx on track (medium); 48 48 + create index if not exists track__recording_idx on track (recording);

+4 -4

src/mbz/mod.rs

reviewed

··· 1 1 - use duckdb::{Connection, params}; 1 1 + use crate::SqliteConnection; 2 2 use std::sync::LazyLock; 3 3 4 4 mod lookup_cache; ··· 9 9 pub use query::{FindMbzData, MbzQueryRes}; 10 10 11 11 static CACHE: LazyLock<lookup_cache::LookupCache> = LazyLock::new(lookup_cache::LookupCache::new); 12 12 - 12 12 + 13 13 pub fn try_find_mbz_data( 14 14 - conn: &Connection, 14 14 + conn: &SqliteConnection, 15 15 opts: &FindMbzData, 16 16 - ) -> duckdb::Result<Option<MbzQueryRes>> { 16 16 + ) -> rusqlite::Result<Option<MbzQueryRes>> { 17 17 if let Some(entry) = CACHE.get(build_cache_key(opts)) { 18 18 return Ok(Some(entry)); 19 19 }

+28 -39

src/mbz/query.rs

reviewed

··· 1 1 - use duckdb::{Connection, params}; 1 1 + use crate::SqliteConnection; 2 2 + use rusqlite::params; 2 3 use std::time::Instant; 3 4 4 5 #[derive(Debug, Default)] ··· 27 28 } 28 29 29 30 impl MbzQueryRes { 30 30 - fn from_row(row: &duckdb::Row, start: Option<Instant>) -> duckdb::Result<Self> { 31 31 + fn from_row(row: &rusqlite::Row, start: Option<Instant>) -> rusqlite::Result<Self> { 31 32 let debug = MbzDebug::from_row(row, start)?; 32 33 33 34 Ok(MbzQueryRes { ··· 53 54 } 54 55 55 56 impl MbzDebug { 56 56 - pub fn from_row(row: &duckdb::Row, start: Option<Instant>) -> duckdb::Result<Self> { 57 57 + pub fn from_row(row: &rusqlite::Row, start: Option<Instant>) -> rusqlite::Result<Self> { 57 58 Ok(MbzDebug { 58 59 release: row.get(8)?, 59 60 recording: row.get(9)?, ··· 73 74 } 74 75 } 75 76 76 76 - pub fn find_mbz_data(conn: &Connection, query: &FindMbzData) -> duckdb::Result<Vec<MbzQueryRes>> { 77 77 + pub fn find_mbz_data( 78 78 + conn: &SqliteConnection, 79 79 + query: &FindMbzData, 80 80 + ) -> rusqlite::Result<Vec<MbzQueryRes>> { 77 81 let start = Instant::now(); 78 82 79 83 if let Some(isrc) = query.isrc { 80 84 find_by_isrc(conn, start, isrc, query.release_name, query.release_mbid) 81 81 - } else if let Some((rel, rec)) = query.release_mbid.zip(query.recording_mbid) { 82 82 - find_by_mbid_all(conn, start, rec, rel) 83 83 - } /*else if query.release_mbid.is_some() || query.recording_mbid.is_some() { 84 84 - find_by_mbid(conn, start) 85 85 - }*/ else { 85 85 + } else { 86 86 find_by_fts(conn, start, query) 87 87 } 88 88 } 89 89 90 90 - // when recording xor release mbids are set 91 91 - fn find_by_mbid(conn: &Connection, ts: Instant) -> duckdb::Result<Vec<MbzQueryRes>> { 92 92 - Ok(vec![]) 93 93 - } 94 94 - 95 95 - // when recording AND release mbids are set 96 96 - fn find_by_mbid_all( 97 97 - conn: &Connection, 98 98 - ts: Instant, 99 99 - recording: &str, 100 100 - release: &str, 101 101 - ) -> duckdb::Result<Vec<MbzQueryRes>> { 102 102 - let mut stmt = conn.prepare_cached(include_str!("find_by_mbid_all.sql"))?; 103 103 - 104 104 - stmt.query_map(params![recording, release], |row| { 105 105 - MbzQueryRes::from_row(row, Some(ts)) 106 106 - })? 107 107 - .collect() 108 108 - } 109 109 - 110 90 // when isrc is set. release may or may not be set 111 91 fn find_by_isrc( 112 112 - conn: &Connection, 92 92 + conn: &SqliteConnection, 113 93 ts: Instant, 114 94 isrc: &str, 115 95 release_name: Option<&str>, 116 96 release_mbid: Option<&str>, 117 117 - ) -> duckdb::Result<Vec<MbzQueryRes>> { 97 97 + ) -> rusqlite::Result<Vec<MbzQueryRes>> { 118 98 let mut stmt = conn.prepare_cached(include_str!("find_by_isrc.sql"))?; 119 99 120 120 - stmt.query_map(params![release_name, release_mbid, isrc], |row| { 121 121 - MbzQueryRes::from_row(row, Some(ts)) 122 122 - })? 100 100 + stmt.query_map( 101 101 + params![fts5_safe_opt(release_name), release_mbid, isrc], 102 102 + |row| MbzQueryRes::from_row(row, Some(ts)), 103 103 + )? 123 104 .collect() 124 105 } 125 106 126 107 // when there are only names (this is absolutely worst case!) 127 108 fn find_by_fts( 128 128 - conn: &Connection, 109 109 + conn: &SqliteConnection, 129 110 ts: Instant, 130 111 query: &FindMbzData, 131 131 - ) -> duckdb::Result<Vec<MbzQueryRes>> { 112 112 + ) -> rusqlite::Result<Vec<MbzQueryRes>> { 132 113 let mut stmt = conn.prepare_cached(include_str!("find_by_fts.sql"))?; 133 114 134 115 stmt.query_map( 135 116 params![ 136 136 - query.release_name, 117 117 + fts5_safe_opt(query.release_name), 137 118 query.release_mbid, 138 138 - query.track_name, 119 119 + fts5_safe(query.track_name), 139 120 query.recording_mbid, 140 140 - query.artist_name, 121 121 + fts5_safe_opt(query.artist_name), 141 122 ], 142 123 |row| MbzQueryRes::from_row(row, Some(ts)), 143 124 )? 144 125 .collect() 145 126 } 127 127 + 128 128 + fn fts5_safe(input: &str) -> String { 129 129 + format!(r#""{input}""#) 130 130 + } 131 131 + 132 132 + fn fts5_safe_opt(input: Option<&str>) -> Option<String> { 133 133 + input.map(fts5_safe) 134 134 + }

+79 -59

src/mbz/replica.rs

reviewed

··· 1 1 + use crate::SqlitePool; 1 2 use async_compression::tokio::bufread::BzDecoder; 2 2 - use duckdb::{DuckdbConnectionManager, OptionalExt, params}; 3 3 use futures::{StreamExt, TryStreamExt}; 4 4 - use r2d2::ManageConnection; 5 4 use reqwest::Client; 5 5 + use rusqlite::OptionalExtension; 6 6 use std::path::Path; 7 7 use std::str::FromStr; 8 8 - use std::sync::Arc; 9 8 use tokio::io::{AsyncBufRead, AsyncRead, AsyncReadExt}; 10 9 use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; 11 10 use tokio_util::compat::FuturesAsyncReadCompatExt; ··· 78 77 } 79 78 80 79 pub struct ReplicationAgent { 81 81 - db: Arc<DuckdbConnectionManager>, 80 80 + db: SqlitePool, 82 81 client: Client, 82 82 + db_path: String, 83 83 } 84 84 85 85 impl ReplicationAgent { 86 86 - pub fn new(db: Arc<DuckdbConnectionManager>, agent: String) -> Self { 86 86 + pub fn new(db: SqlitePool, db_path: String, agent: String) -> Self { 87 87 let client = Client::builder().user_agent(agent).build().unwrap(); 88 88 89 89 - ReplicationAgent { db, client } 89 89 + ReplicationAgent { 90 90 + db, 91 91 + client, 92 92 + db_path, 93 93 + } 90 94 } 91 95 92 96 pub async fn start(self) -> eyre::Result<()> { 93 93 - if !check_mbz_schema(&self.db)? { 97 97 + if !self.check_mbz_tables()? { 94 98 self.init().await?; 95 99 } else { 96 100 debug!("skipping initial load - schema mbz exists"); ··· 112 116 113 117 self.init_dump(MusicbrainzDump::Primary, dir).await?; 114 118 self.init_dump(MusicbrainzDump::Derived, dir).await?; 119 119 + debug!("musicbrainz dump imported successfully"); 115 120 116 116 - debug!("musicbrainz dump imported successfully, building FTS"); 121 121 + debug!("building indices"); 122 122 + self.connect_and_run_sql_batch(include_str!("init_index.sql"))?; 123 123 + debug!("building FTS"); 117 124 self.connect_and_run_sql_batch(include_str!("init_fts.sql"))?; 118 118 - debug!("finished building FTS"); 119 125 120 126 tmp.close()?; 121 127 ··· 132 138 }; 133 139 134 140 let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); 135 135 - let db = self.db.clone(); 136 141 let d = dir.to_path_buf(); 137 142 138 138 - let compressed = replica_get_dump(&self.client, dump).await?; 139 139 - let bh = tokio::task::spawn_blocking(move || replica_import_dump(&db, rx, &d)); 143 143 + let compressed = self.download_dump(dump).await?; 144 144 + let db = self.db_path.clone(); 145 145 + let bh = tokio::task::spawn_blocking(move || replica_import_dump(db, rx, &d)); 140 146 141 147 replica_extract(compressed, wanted, dir, tx).await?; 142 148 bh.await??; ··· 144 150 Ok(()) 145 151 } 146 152 147 147 - fn connect_and_run_sql_batch(&self, batch: &str) -> duckdb::Result<()> { 148 148 - self.db.connect()?.execute_batch(batch) 153 153 + fn connect_and_run_sql_batch(&self, batch: &str) -> eyre::Result<()> { 154 154 + self.db.get()?.execute_batch(batch)?; 155 155 + Ok(()) 149 156 } 150 150 - } 151 157 152 152 - fn check_mbz_schema(db: &DuckdbConnectionManager) -> eyre::Result<bool> { 153 153 - let conn = db.connect()?; 154 154 - let maybe_schema = conn 155 155 - .query_row( 156 156 - "SELECT schema_name FROM duckdb_schemas() WHERE schema_name='mbz'", 157 157 - params![], 158 158 - |row| row.get::<_, String>(0), 159 159 - ) 160 160 - .optional()?; 158 158 + fn check_mbz_tables(&self) -> eyre::Result<bool> { 159 159 + let conn = self.db.get()?; 160 160 + let exists: Option<String> = conn 161 161 + .query_one( 162 162 + "SELECT name FROM sqlite_master WHERE type='table' AND name='track'", 163 163 + [], 164 164 + |row| row.get(0), 165 165 + ) 166 166 + .optional()?; 161 167 162 162 - Ok(maybe_schema.is_some()) 163 163 - } 168 168 + Ok(exists.is_some()) 169 169 + } 164 170 165 165 - /// Downloads the full Musicbrainz dump file 166 166 - async fn replica_get_dump( 167 167 - client: &Client, 168 168 - dump: MusicbrainzDump, 169 169 - ) -> eyre::Result<impl AsyncBufRead> { 170 170 - let latest = client 171 171 - .get(format!("{MBZ_FTP_DUMP}/LATEST")) 172 172 - .send() 173 173 - .await? 174 174 - .error_for_status()? 175 175 - .text() 176 176 - .await?; 177 177 - let latest = latest.trim(); 171 171 + /// Downloads the full Musicbrainz dump file 172 172 + async fn download_dump(&self, dump: MusicbrainzDump) -> eyre::Result<impl AsyncBufRead> { 173 173 + let latest = self 174 174 + .client 175 175 + .get(format!("{MBZ_FTP_DUMP}/LATEST")) 176 176 + .send() 177 177 + .await? 178 178 + .error_for_status()? 179 179 + .text() 180 180 + .await?; 181 181 + let latest = latest.trim(); 178 182 179 179 - info!("downloading musicbrainz dump {dump}: {latest}"); 180 180 - let dump_res = client 181 181 - .get(format!("{MBZ_FTP_DUMP}/{latest}/{dump}.tar.bz2")) 182 182 - .send() 183 183 - .await? 184 184 - .error_for_status()?; 185 185 - let dump_stream = dump_res.bytes_stream(); 186 186 - let dump = dump_stream 187 187 - .map_err(futures::io::Error::other) 188 188 - .into_async_read() 189 189 - .compat(); 183 183 + info!("downloading musicbrainz dump {dump}: {latest}"); 184 184 + let dump_res = self 185 185 + .client 186 186 + .get(format!("{MBZ_FTP_DUMP}/{latest}/{dump}.tar.bz2")) 187 187 + .send() 188 188 + .await? 189 189 + .error_for_status()?; 190 190 + let dump_stream = dump_res.bytes_stream(); 191 191 + let dump = dump_stream 192 192 + .map_err(futures::io::Error::other) 193 193 + .into_async_read() 194 194 + .compat(); 190 195 191 191 - Ok(dump) 196 196 + Ok(dump) 197 197 + } 192 198 } 193 199 194 194 - async fn replica_extract<T>(compressed: T, wanted: &[&str], tmpdir: &Path, tx: UnboundedSender<String>) -> eyre::Result<u32> 200 200 + async fn replica_extract<T>( 201 201 + compressed: T, 202 202 + wanted: &[&str], 203 203 + tmpdir: &Path, 204 204 + tx: UnboundedSender<String>, 205 205 + ) -> eyre::Result<u32> 195 206 where 196 207 T: AsyncRead + AsyncBufRead + Unpin, 197 208 { ··· 235 246 Ok(seq_replication) 236 247 } 237 248 238 238 - /// Imports the downloaded dump into duckdb 249 249 + /// Imports the downloaded dump into DB. This shells out to duckdb because their csv parser is good. 239 250 fn replica_import_dump( 240 240 - db: &DuckdbConnectionManager, 251 251 + db: String, 241 252 mut recv: UnboundedReceiver<String>, 242 253 dump: &Path, 243 254 ) -> eyre::Result<()> { 244 244 - let db = db.connect()?; 245 255 let dump = dump.join("mbdump"); 246 256 247 257 while let Some(table) = recv.blocking_recv() { 248 258 debug!("importing dump - mbz.{table}"); 249 249 - db.execute( 250 250 - &format!("COPY mbz.{table} FROM ? (NULLSTR '\\N', QUOTE '')"), 251 251 - params![dump.join(&table).to_string_lossy()], 252 252 - )?; 259 259 + 260 260 + let out = std::process::Command::new("duckdb") 261 261 + .arg(&db) 262 262 + .arg(format!( 263 263 + "COPY {table} FROM '{}' (NULLSTR '\\N', QUOTE '', HEADER false)", 264 264 + dump.join(&table).to_string_lossy() 265 265 + )) 266 266 + .output()?; 267 267 + 268 268 + if !out.status.success() { 269 269 + let stderr = String::from_utf8(out.stderr)?; 270 270 + eyre::bail!("failed to import table {table}: {stderr}"); 271 271 + } 272 272 + 253 273 debug!("finished importing dump mbz.{table}") 254 274 } 255 275

+3 -4

src/server/mod.rs

reviewed

··· 1 1 + use crate::SqlitePool; 1 2 use crate::lex::queries; 2 3 use axum::Router; 3 3 - use duckdb::DuckdbConnectionManager; 4 4 use jacquard_axum::IntoRouter; 5 5 use std::net::SocketAddr; 6 6 - use std::sync::Arc; 7 6 use tokio::net::TcpListener; 8 7 use tower_http::cors::{AllowHeaders, AllowOrigin, CorsLayer}; 9 8 use tower_http::trace::TraceLayer; ··· 13 12 14 13 #[derive(Clone)] 15 14 pub struct GlobalState { 16 16 - pub db: Arc<DuckdbConnectionManager>, 15 15 + pub db: SqlitePool, 17 16 } 18 17 19 19 - pub async fn start_server(db: Arc<DuckdbConnectionManager>, port: u16) -> eyre::Result<()> { 18 18 + pub async fn start_server(db: SqlitePool, port: u16) -> eyre::Result<()> { 20 19 let cors = CorsLayer::new() 21 20 .allow_headers(AllowHeaders::any()) 22 21 .allow_origin(AllowOrigin::any());

+3 -3

src/server/summary.rs

reviewed

··· 1 1 use super::GlobalState; 2 2 - use crate::analysis; 2 2 + use crate::{analysis, SqliteConnection}; 3 3 use crate::lex::queries::{ 4 4 GetAnnualSummaryOutput, GetAnnualSummaryRequest, GetPeriodSummaryOutput, 5 5 GetPeriodSummaryRequest, ··· 99 99 } 100 100 101 101 fn get_core_summary( 102 102 - conn: &duckdb::Connection, 102 102 + conn: &SqliteConnection, 103 103 did: &str, 104 104 start: DateTime<Utc>, 105 105 end: DateTime<Utc>, 106 106 - ) -> duckdb::Result<( 106 106 + ) -> rusqlite::Result<( 107 107 Vec<TopAlbumEntry<'static>>, 108 108 Vec<TopArtistEntry<'static>>, 109 109 Vec<TopTrackEntry<'static>>,

+5 -6

src/utils.rs

reviewed

··· 1 1 - use duckdb::{Connection, DuckdbConnectionManager}; 2 1 use jacquard_api::app_bsky::actor::ProfileViewBasic; 3 2 use jacquard_api::app_bsky::actor::get_profile::GetProfile; 4 3 use jacquard_common::types::did::Did; ··· 6 5 use jacquard_common::xrpc::XrpcExt; 7 6 use jacquard_identity::JacquardResolver; 8 7 use jacquard_identity::resolver::{HandleStep, IdentityResolver, PlcSource, ResolverOptions}; 9 9 - use r2d2::ManageConnection; 10 8 use std::sync::LazyLock; 11 9 use tokio::task::{JoinHandle, spawn_blocking}; 10 10 + use crate::{SqliteConnection, SqlitePool}; 12 11 13 12 const BSKY_PUBLIC_API: &str = "https://public.api.bsky.app"; 14 13 static CLIENT: LazyLock<reqwest::Client> = LazyLock::new(reqwest::Client::new); ··· 23 22 ) 24 23 }); 25 24 26 26 - pub fn db_call<F, R>(db: &DuckdbConnectionManager, func: F) -> JoinHandle<duckdb::Result<R>> 25 25 + pub fn db_call<F, R>(db: &SqlitePool, func: F) -> JoinHandle<eyre::Result<R>> 27 26 where 28 28 - F: Fn(Connection) -> duckdb::Result<R> + Send + Sync + 'static, 27 27 + F: Fn(SqliteConnection) -> rusqlite::Result<R> + Send + Sync + 'static, 29 28 R: Send + Sync + 'static, 30 29 { 31 31 - let conn = db.connect(); 32 32 - spawn_blocking(move || func(conn?)) 30 30 + let conn = db.get(); 31 31 + spawn_blocking(move || Ok(func(conn?)?)) 33 32 } 34 33 35 34 pub async fn get_public_profile(did: &Did<'_>) -> eyre::Result<ProfileViewBasic<'static>> {