From 49e7340c1905423b9db9216f7026ac8d86c9391c Mon Sep 17 00:00:00 2001 From: Julia Lange Date: Tue, 6 May 2025 13:50:57 -0700 Subject: [PATCH] Src, move out of ./rust, add missing wip files Moves file contents out of ./rust since I'm planning to go with rust for the backend and the lexicons will be used for codegen soon anyways. I also added extra files that have been in the works that I have been accidentally sprinkling into main.rs already. God forbid I need to cherry-pick anything from so far back in the history. It will make git blame ugly though. --- rust/Cargo.lock => Cargo.lock | 637 ++++++++++++++++++++++++++++++- rust/Cargo.toml => Cargo.toml | 5 + rust/flake.lock => flake.lock | 6 +- rust/flake.nix => flake.nix | 0 rust/src/atproto.rs | 1 - src/atproto.rs | 70 ++++ src/db.rs | 113 ++++++ src/injester.rs | 30 ++ {rust/src => src}/main.rs | 5 + {rust/src => src}/router.rs | 0 {rust/src => src}/router/xrpc.rs | 0 11 files changed, 846 insertions(+), 21 deletions(-) rename rust/Cargo.lock => Cargo.lock (82%) rename rust/Cargo.toml => Cargo.toml (77%) rename rust/flake.lock => flake.lock (89%) rename rust/flake.nix => flake.nix (100%) delete mode 100644 rust/src/atproto.rs create mode 100644 src/atproto.rs create mode 100644 src/db.rs create mode 100644 src/injester.rs rename {rust/src => src}/main.rs (89%) rename {rust/src => src}/router.rs (100%) rename {rust/src => src}/router/xrpc.rs (100%) diff --git a/rust/Cargo.lock b/Cargo.lock similarity index 82% rename from rust/Cargo.lock rename to Cargo.lock index 0d927f1..9032f39 100644 --- a/rust/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,18 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy 0.7.35", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -47,6 +59,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anyhow" +version = "1.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" + [[package]] name = "async-lock" version = "3.4.0" @@ -58,6 +76,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atoi" version = "2.0.0" @@ -76,7 +105,7 @@ dependencies = [ "atrium-common", "atrium-xrpc", "chrono", - "http", + "http 1.3.1", "ipld-core", "langtag", "regex", @@ -102,13 +131,23 @@ dependencies = [ "web-time", ] +[[package]] +name = "atrium-lex" +version = "0.1.0" +source = "git+https://github.com/sugyan/atrium.git?rev=f162f815a04b5ecb0421b390d521c883c41d5f75#f162f815a04b5ecb0421b390d521c883c41d5f75" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + [[package]] name = "atrium-xrpc" version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "18a9e526cb2ed3e0a2ca78c3ce2a943d9041a68e067dadf42923b523771e07df" dependencies = [ - "http", + "http 1.3.1", "serde", "serde_html_form", "serde_json", @@ -132,7 +171,7 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http", + "http 1.3.1", "http-body", "http-body-util", "hyper", @@ -164,7 +203,7 @@ checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ "bytes", "futures-core", - "http", + "http 1.3.1", "http-body", "http-body-util", "mime", @@ -208,6 +247,18 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cbbc9d0964165b47557570cce6c952866c2678457aca742aafc9fb771d30270" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -238,6 +289,31 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bon" +version = "3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced38439e7a86a4761f7f7d5ded5ff009135939ecb464a24452eaa4c1696af7d" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce61d2d3844c6b8d31b2353d9f66cf5e632b3e9549583fe3cac2f4f6136725e" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "bumpalo" version = "3.17.0" @@ -262,6 +338,8 @@ version = "1.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04da6a0d40b948dfc4fa8f5bbf402b0fc1a64a28dbf7d12ffd683550f2c1b63a" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -315,6 +393,16 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -397,6 +485,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "dashmap" version = "6.1.0" @@ -448,6 +571,47 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +dependencies = [ + "powerfmt", + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "digest" version = "0.10.7" @@ -502,6 +666,22 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "esquema-codegen" +version = "0.1.0" +source = "git+https://github.com/fatfingers23/esquema.git?branch=main#9ef00b9d631b746bd6396fb46ba255eb9360e43f" +dependencies = [ + "atrium-lex", + "heck 0.4.1", + "itertools", + "prettyplease", + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -548,6 +728,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", + "nanorand", "spin", ] @@ -686,8 +867,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -708,6 +891,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.14.5" @@ -734,6 +923,12 @@ dependencies = [ "hashbrown 0.15.2", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -773,6 +968,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.3.1" @@ -791,7 +997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.3.1", ] [[package]] @@ -802,7 +1008,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", + "http 1.3.1", "http-body", "pin-project-lite", ] @@ -828,7 +1034,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", + "http 1.3.1", "http-body", "httparse", "httpdate", @@ -846,7 +1052,7 @@ checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-util", - "http", + "http 1.3.1", "http-body", "hyper", "pin-project-lite", @@ -996,6 +1202,12 @@ dependencies = [ "syn", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.0.3" @@ -1017,6 +1229,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.9.0" @@ -1038,12 +1261,31 @@ dependencies = [ "serde_bytes", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jobserver" +version = "0.1.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +dependencies = [ + "getrandom 0.3.2", + "libc", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -1175,6 +1417,16 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "metrics" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25dea7ac8057892855ec285c440160265225438c3c45072613c25a4b26e98ef5" +dependencies = [ + "ahash", + "portable-atomic", +] + [[package]] name = "mime" version = "0.3.17" @@ -1245,6 +1497,15 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.15", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1272,6 +1533,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -1317,6 +1584,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + [[package]] name = "overload" version = "0.1.1" @@ -1412,13 +1685,29 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy", + "zerocopy 0.8.24", +] + +[[package]] +name = "prettyplease" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +dependencies = [ + "proc-macro2", + "syn", ] [[package]] @@ -1528,6 +1817,44 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.15", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rocketman" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a3aae946adbfdcf80cad8793e02d8eb94be06c925331aa56aeb446795893361" +dependencies = [ + "anyhow", + "async-trait", + "bon", + "derive_builder", + "flume", + "futures-util", + "metrics", + "rand", + "serde", + "serde_json", + "tokio", + "tokio-tungstenite", + "tracing", + "tracing-subscriber", + "url", + "zstd", +] + [[package]] name = "rsa" version = "0.9.8" @@ -1555,7 +1882,10 @@ dependencies = [ "atrium-api", "axum", "axum-macros", - "http", + "esquema-codegen", + "http 1.3.1", + "regex", + "rocketman", "serde", "serde_json", "sqlx", @@ -1590,6 +1920,49 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.20" @@ -1602,6 +1975,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -1614,6 +1996,39 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.26" @@ -1656,7 +2071,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d2de91cf02bbc07cde38891769ccd5d4f073d22a40683aa4bc7a95781aaa2c4" dependencies = [ "form_urlencoded", - "indexmap", + "indexmap 2.9.0", "itoa", "ryu", "serde", @@ -1684,6 +2099,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1696,6 +2122,34 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" +dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap 1.9.3", + "serde", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "881b6f881b17d13214e5d494c939ebab463d01264ce1811e9d4ac3a882e7695f" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha1" version = "0.10.6" @@ -1809,7 +2263,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f743f2a3cea30a58cd479013f75550e879009e3a02f616f18ca699335aa248c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "crc", "crossbeam-queue", @@ -1821,7 +2275,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.2", "hashlink", - "indexmap", + "indexmap 2.9.0", "log", "memchr", "once_cell", @@ -1858,7 +2312,7 @@ checksum = "882ceaa29cade31beca7129b6beeb05737f44f82dbe2a9806ecea5a7093d00b7" dependencies = [ "dotenvy", "either", - "heck", + "heck 0.5.0", "hex", "once_cell", "proc-macro2", @@ -1883,7 +2337,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0afdd3aa7a629683c2d750c2df343025545087081ab5942593a5288855b1b7a7" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "bytes", @@ -1925,7 +2379,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0bedbe1bbb5e2615ef347a5e9d8cd7680fb63e77d9dafc0f29be15e53f1ebe6" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "crc", @@ -1996,6 +2450,12 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "subtle" version = "2.6.1" @@ -2099,6 +2559,37 @@ dependencies = [ "once_cell", ] +[[package]] +name = "time" +version = "0.3.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" + +[[package]] +name = "time-macros" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -2151,6 +2642,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" @@ -2162,6 +2663,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" +dependencies = [ + "futures-util", + "log", + "rustls", + "rustls-native-certs", + "tokio", + "tokio-rustls", + "tungstenite", + "webpki-roots", +] + [[package]] name = "tower" version = "0.5.2" @@ -2263,6 +2780,26 @@ dependencies = [ "syn", ] +[[package]] +name = "tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" +dependencies = [ + "byteorder", + "bytes", + "data-encoding", + "http 0.2.12", + "httparse", + "log", + "rand", + "rustls", + "sha1", + "thiserror 1.0.69", + "url", + "utf-8", +] + [[package]] name = "typenum" version = "1.18.0" @@ -2302,6 +2839,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.4" @@ -2313,6 +2856,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf16_iter" version = "1.0.5" @@ -2441,6 +2990,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.25.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" + [[package]] name = "whoami" version = "1.6.0" @@ -2789,13 +3344,33 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive 0.7.35", +] + [[package]] name = "zerocopy" version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.8.24", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2857,3 +3432,31 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/rust/Cargo.toml b/Cargo.toml similarity index 77% rename from rust/Cargo.toml rename to Cargo.toml index c6b4dba..4f4b2c9 100644 --- a/rust/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,12 @@ atrium-api = { version = "0.25.2", default-features = false } axum = { version = "0.8.3", features = ["json"] } axum-macros = "0.5.0" http = "1.3.1" +regex = "1.11.1" +rocketman = "0.2.0" serde = "1.0.219" serde_json = "1.0.140" sqlx = { version = "0.8.5", features = ["postgres", "runtime-tokio"] } tokio = { version = "1.44.2", features = ["macros", "rt-multi-thread"] } + +[build-dependencies] +esquema-codegen = { git = "https://github.com/fatfingers23/esquema.git", branch = "main" } diff --git a/rust/flake.lock b/flake.lock similarity index 89% rename from rust/flake.lock rename to flake.lock index 4e92e1f..8f3aa7b 100644 --- a/rust/flake.lock +++ b/flake.lock @@ -41,11 +41,11 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1744857263, - "narHash": "sha256-M4X/CnquHozzgwDk+CbFb8Sb4rSGJttfNOKcpRwziis=", + "lastModified": 1746585402, + "narHash": "sha256-Pf+ufu6bYNA1+KQKHnGMNEfTwpD9ZIcAeLoE2yPWIP0=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "9f3d63d569536cd661a4adcf697e32eb08d61e31", + "rev": "72dd969389583664f87aa348b3458f2813693617", "type": "github" }, "original": { diff --git a/rust/flake.nix b/flake.nix similarity index 100% rename from rust/flake.nix rename to flake.nix diff --git a/rust/src/atproto.rs b/rust/src/atproto.rs deleted file mode 100644 index 4405e7f..0000000 --- a/rust/src/atproto.rs +++ /dev/null @@ -1 +0,0 @@ -pub use atrium_api::types::string::Nsid; diff --git a/src/atproto.rs b/src/atproto.rs new file mode 100644 index 0000000..0fdc930 --- /dev/null +++ b/src/atproto.rs @@ -0,0 +1,70 @@ +use atrium_api::types::string::{ + AtIdentifier, + RecordKey, +}; +use regex::Regex; + +pub use atrium_api::types::string::{ + Nsid, + Did, + Handle, +}; + +enum Authority { + Did(Did), + Handle(Handle), +} + +impl Authority { + pub fn new(authority: String) -> Result { + } +} + +// This implementation does not support Query or Fragments, and thus follows +// the following schema: "at://" AUTHORITY [ "/" COLLECTION [ "/" RKEY ] ] +pub struct Uri { + authority: Authority, + collection: Option, + rkey: Option, +} + +// TODO: Replace super basic URI regex with real uri parsing +const URI_REGEX: Regex = Regex::new( + r"/^at:\/\/([\w\.\-_~:]+)(?:\/([\w\.\-_~:]+)(?:)\/([\w\.\-_~:]+))?$/i" + ).expect("valid regex"); + +impl Uri { + pub fn new(uri: String) -> Result { + let Some(captures) = URI_REGEX.captures(&uri) else { + return Err("Invalid Uri"); + }; + // TODO: Convert authority if its a did or a handle + let Some(Ok(authority)) = captures.get(1).map(|mtch| { + Authority::new(mtch.as_str().to_string()) + }) else { + return Err("Invalid Authority") + }; + let collection = captures.get(2).map(|mtch| { + Nsid::new(mtch.as_str().to_string()) + }); + let rkey = captures.get(3).map(|mtch| { + RecordKey::new(mtch.as_str().to_string()) + }); + Ok(Uri { authority, collection, rkey }) + } + + pub fn as_string(&self) -> String { + let mut uri = String::from("at://"); + uri.push_str(match &self.authority { + Authority::Handle(h) => &*h, + Authority::Did(d) => &*d, + }); + if let Some(nsid) = &self.collection { + uri.push_str(&*nsid); + } + if let Some(rkey) = &self.rkey { + uri.push_str(&*rkey); + } + uri + } +} diff --git a/src/db.rs b/src/db.rs new file mode 100644 index 0000000..78fb640 --- /dev/null +++ b/src/db.rs @@ -0,0 +1,113 @@ +use crate::atproto::{ + Did, + Uri, +} +use sqlx::{ + query, + Database, + Pool, + Postgres, + pool::PoolOptions, + postgres::{ + PgConnectOptions, + PgSslMode, + }, + Result, +}; +use std::string::ToString; + +pub struct Db { + pool: Pool +} + +pub struct User { + userdid: Did, + handle: Handle, +} + +#[non_exhaustive] +enum Role { + Owner, + Participant +} + +impl ToString for Role { + fn to_string(&self) -> String { + match *self { + Role::Owner => "owner".to_string(), + Role::Participant => "participant".to_string(), + } + } +} + +struct Participant { + participantdid: Did, + role: Role, +} + +pub struct Session { + sessionuri: Uri, + label: Option, + participants: Vec, +} + +impl Db { + async fn connect() -> Result { + let conn = PgConnectOptions::new() + .host("localhost") + .port(5432) + .username("postgres") + .password("062217") + .database("anisky") + .ssl_mode(PgSslMode::Disable); + + let pool = match PoolOptions::new().connect_with(conn).await { + Ok(p) => p, + Err(e) => return Err(e), + }; + + Ok(Db { pool }) + } + // + // pub async fn add_user(&self, user: &User) -> Result<()> { + // query!(r#" + // INSERT INTO users(userdid, handle) VALUES ($1, $2) + // "#, + // user.userdid, user.handle + // ).execute(self.pool).await?; + // Ok(()) + // } + // + // pub async fn add_session(&self, session: &Session) -> Result<()> { + // let mut transaction = self.pool.begin().await?; + // + // query!(r#" + // INSERT INTO sessions(sessionuri, label) VALUES ($1, $2) + // "#, + // session.sessionuri, session.label + // ).execute(&mut *transaction).await?; + // + // for participant in session.participants { + // query!(r#" + // INSERT INTO participants(sessionuri, userdid, role) VALUES ($1, $2, $3) + // "#, + // session.sessionuri, participant.userdid, participant.role.to_string() + // ).execute(&mut *transaction).await?; + // } + // + // transaction.commit().await + // } + // + // pub async fn add_participant(&self, session: Session, + // participant: Participant) -> Result { + // query!(r#" + // INSERT INTO participants(sessionuri, userdid, role) VALUES ($1, $2, $3) + // "#, + // session.sessionuri, participant.userdid, participant.role.to_string() + // ).execute(self.pool).await?; + // + // session.participants.push(participant); + // + // Ok(session) + // } +} diff --git a/src/injester.rs b/src/injester.rs new file mode 100644 index 0000000..722390e --- /dev/null +++ b/src/injester.rs @@ -0,0 +1,30 @@ +use crate::db::Db; +use rocketman::{ + options::JetstreamOptions + ingestion::LexiconIngestor +}; + +enum Injester { + Jetstream(Jetstream) +} + +struct SpoorJetstream; + +impl LexiconIngestor for SpoorJetstream { + async fn ingest(&self, message: Event) -> Result<()> { + if let Some(commit) = &message.commit { + match commit.operation { + Operation::Create | Operation::Update => { + + } + } + } else { + return Err("Message has no commit"); + } + Ok(()) + } +} + +pub async fn start_ingester(db: Db) { + let opts = JetstreamOptions::builder() +} diff --git a/rust/src/main.rs b/src/main.rs similarity index 89% rename from rust/src/main.rs rename to src/main.rs index 1a6eb1e..993e55f 100644 --- a/rust/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ use crate::{ atproto::Nsid, + injester::start_injester, router::{ Router, Endpoint, @@ -14,6 +15,7 @@ use crate::{ use http::status::StatusCode; mod atproto; +mod injester; mod router; mod db; @@ -24,6 +26,9 @@ async fn main() { let post_nsid = Nsid::new(String::from("me.woach.post")).expect("me.woach.post is a valid nsid"); router = router.add_endpoint(Endpoint::new_xrpc_query(get_nsid, test)); router = router.add_endpoint(Endpoint::new_xrpc_procedure(post_nsid, test2)); + tokio::spawn(async move { + start_injester(); + }); router.serve().await; } diff --git a/rust/src/router.rs b/src/router.rs similarity index 100% rename from rust/src/router.rs rename to src/router.rs diff --git a/rust/src/router/xrpc.rs b/src/router/xrpc.rs similarity index 100% rename from rust/src/router/xrpc.rs rename to src/router/xrpc.rs