From a335f14b14b53b4519b3394468c3c8130fb86110 Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Sat, 8 Nov 2025 13:12:07 -0500 Subject: [PATCH] revert to septa api for rt tracking --- api/Cargo.lock | 489 ++++++++++++++++++++++++++++-- api/Cargo.toml | 5 + api/src/controllers/route.rs | 25 +- api/src/database.rs | 179 ----------- api/src/main.rs | 26 +- api/src/services/mod.rs | 2 + api/src/services/trip_tracking.rs | 5 +- api/src/templates.rs | 10 +- api/templates/route.html | 4 +- api/templates/route_symbol.html | 8 +- api/templates/routes.html | 8 +- data_loader/config.yaml | 7 +- data_loader/src/main.rs | 93 ------ libseptastic/src/direction.rs | 4 +- libseptastic/src/lib.rs | 1 + libseptastic/src/stop.rs | 6 +- libseptastic/src/stop_schedule.rs | 17 +- shell.nix | 1 + 18 files changed, 535 insertions(+), 355 deletions(-) delete mode 100644 api/src/database.rs diff --git a/api/Cargo.lock b/api/Cargo.lock index 7428f95..0ae836b 100644 --- a/api/Cargo.lock +++ b/api/Cargo.lock @@ -103,7 +103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -237,7 +237,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -391,6 +391,15 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "askama" version = "0.14.0" @@ -418,7 +427,7 @@ dependencies = [ "rustc-hash", "serde", "serde_derive", - "syn", + "syn 2.0.104", ] [[package]] @@ -717,6 +726,12 @@ version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + [[package]] name = "byteorder" version = "1.5.0" @@ -738,6 +753,34 @@ dependencies = [ "bytes", ] +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "camino" version = "1.1.12" @@ -862,7 +905,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -905,6 +948,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.4.0" @@ -1004,6 +1053,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -1013,6 +1083,12 @@ dependencies = [ "cipher", ] +[[package]] +name = "deflate64" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" + [[package]] name = "der" version = "0.7.10" @@ -1033,6 +1109,28 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "derive_more" version = "0.99.20" @@ -1043,7 +1141,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.104", ] [[package]] @@ -1063,7 +1161,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "unicode-xid", ] @@ -1098,7 +1196,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1241,6 +1339,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.2" @@ -1248,6 +1352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -1393,7 +1498,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1454,9 +1559,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1493,6 +1600,39 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gtfs-realtime" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421e7aa1a3a540a6f2e046cb349b184be285e16be332aff043e188b2b563c2ff" +dependencies = [ + "prost", + "prost-build", + "prost-derive", + "serde", +] + +[[package]] +name = "gtfs-structures" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3211c82a20a529763951e9072e4d85c06b6c8d4efa75d08859ccb58e702334e3" +dependencies = [ + "bytes", + "chrono", + "csv", + "derivative", + "futures", + "itertools", + "reqwest", + "rgb", + "serde", + "serde_derive", + "sha2", + "thiserror 1.0.69", + "zip 2.4.2", +] + [[package]] name = "h2" version = "0.3.26" @@ -1952,6 +2092,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1979,7 +2128,7 @@ checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -2026,6 +2175,12 @@ dependencies = [ "spin", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + [[package]] name = "libc" version = "0.2.174" @@ -2069,6 +2224,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -2129,6 +2293,37 @@ dependencies = [ "value-bag", ] +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "lzma-rust2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a" +dependencies = [ + "crc", + "sha2", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "md-5" version = "0.10.6" @@ -2182,6 +2377,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "native-tls" version = "0.2.14" @@ -2302,7 +2503,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -2352,6 +2553,16 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -2367,6 +2578,16 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "phf" version = "0.12.1" @@ -2508,6 +2729,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppmd-rust" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2517,6 +2744,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff24dfcda44452b9816fff4cd4227e1bb73ff5a2f1bc1105aa92fb8565ce44d2" +dependencies = [ + "proc-macro2", + "syn 2.0.104", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -2526,6 +2763,58 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.104", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "prost-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.40" @@ -2686,6 +2975,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rgb" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.17.14" @@ -2888,13 +3186,18 @@ dependencies = [ "dotenv", "env_logger", "futures-util", + "gtfs-realtime", + "gtfs-structures", "libseptastic", "log", + "prost", "reqwest", "serde", "serde_json", + "serde_yaml", "sqlx", "sqlx-cli", + "zip 5.1.1", ] [[package]] @@ -2914,7 +3217,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -2941,6 +3244,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2994,6 +3310,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "siphasher" version = "1.0.1" @@ -3137,7 +3459,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.104", ] [[package]] @@ -3161,7 +3483,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn", + "syn 2.0.104", "tokio", "url", ] @@ -3298,6 +3620,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.104" @@ -3326,7 +3659,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3399,7 +3732,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3410,7 +3743,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3497,7 +3830,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3609,7 +3942,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3688,6 +4021,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -3799,7 +4138,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.104", "wasm-bindgen-shared", ] @@ -3834,7 +4173,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3911,7 +4250,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3922,7 +4261,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -4206,6 +4545,15 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yoke" version = "0.8.0" @@ -4226,7 +4574,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "synstructure", ] @@ -4247,7 +4595,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -4267,7 +4615,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "synstructure", ] @@ -4276,6 +4624,20 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] [[package]] name = "zerotrie" @@ -4307,7 +4669,82 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.5.2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom 0.3.3", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror 2.0.12", + "time", + "xz2", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zip" +version = "5.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f852905151ac8d4d06fdca66520a661c09730a74c6d4e2b0f27b436b382e532" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.6.0", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "getrandom 0.3.3", + "hmac", + "indexmap", + "lzma-rust2", + "memchr", + "pbkdf2", + "ppmd-rust", + "sha1", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zlib-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" + +[[package]] +name = "zopfli" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", ] [[package]] diff --git a/api/Cargo.toml b/api/Cargo.toml index 9094583..8c3d8a5 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -22,3 +22,8 @@ reqwest = { version = "0.12.22", features = [ "json", "blocking" ] } sqlx-cli = "0.8.6" futures-util = "0.3.31" actix-session = { version = "0.11.0", features = ["cookie-session"] } +serde_yaml = "0.9.34" +gtfs-structures = "0.45.1" +zip = "5.1.1" +gtfs-realtime = "0.2.0" +prost = "0.14.1" diff --git a/api/src/controllers/route.rs b/api/src/controllers/route.rs index 2bd6ae8..09e85cd 100644 --- a/api/src/controllers/route.rs +++ b/api/src/controllers/route.rs @@ -1,12 +1,10 @@ use actix_web::{get, web::{self, Data}, HttpRequest, HttpResponse, Responder}; use anyhow::anyhow; -use std::{time::Instant, sync::Arc}; -use libseptastic::{route::RouteType, stop_schedule::Trip}; +use std::{collections::HashSet, sync::Arc, time::Instant}; +use libseptastic::{direction, route::RouteType, stop_schedule::Trip}; use serde::{Serialize, Deserialize}; -use askama::Template; use crate::AppState; -use crate::database; #[get("/routes")] async fn get_routes_html(req: HttpRequest, state: Data>) -> impl Responder { @@ -15,8 +13,7 @@ async fn get_routes_html(req: HttpRequest, state: Data>) -> impl R async move { let start_time = Instant::now(); - let all_routes = database::get_all_routes(&mut statex.database.begin().await?).await?; - + let all_routes: Vec = statex.gtfs_service.get_routes(); let rr_routes = all_routes.clone().into_iter().filter(|x| x.route_type == RouteType::RegionalRail).collect(); let subway_routes = all_routes.clone().into_iter().filter(|x| x.route_type == RouteType::SubwayElevated).collect(); let trolley_routes = all_routes.clone().into_iter().filter(|x| x.route_type == RouteType::Trolley).collect(); @@ -40,7 +37,7 @@ async fn get_routes_html(req: HttpRequest, state: Data>) -> impl R #[get("/routes.json")] async fn get_routes_json(state: Data>) -> impl Responder { - let all_routes = database::get_all_routes(&mut state.database.begin().await.unwrap()).await.unwrap(); + let all_routes: Vec = state.gtfs_service.get_routes(); HttpResponse::Ok().json(all_routes) } @@ -58,11 +55,15 @@ pub struct RouteResponse { } async fn get_route_info(route_id: String, state: Data>) -> ::anyhow::Result { - let mut tx = state.database.begin().await?; + let route = state.gtfs_service.get_route(route_id.clone())?; + let mut trips = state.gtfs_service.get_schedule(route_id)?; - let route = database::get_route_by_id(route_id.clone(), &mut tx).await?; - let directions = database::get_direction_by_route_id(route_id.clone(), &mut tx).await?; - let mut trips = database::get_schedule_by_route_id(route_id.clone(), &mut tx).await?; + let mut seen = HashSet::new(); + let directions: Vec<_> = trips + .iter() + .map(|x| x.direction.clone()) + .filter(|dir| seen.insert(dir.direction.to_string())) + .collect(); state.trip_tracking_service.annotate_trips(&mut trips); @@ -127,7 +128,7 @@ async fn api_get_route(state: Data>, path: web::Path) -> i #[get("/api/route/{route_id}/schedule")] async fn api_get_schedule(state: Data>, path: web::Path) -> impl Responder { let route_id = path.into_inner(); - let route_r = database::get_schedule_by_route_id(route_id, &mut state.database.begin().await.unwrap()).await; + let route_r: anyhow::Result = Ok(5); if let Ok(route) = route_r { HttpResponse::Ok().json(route) diff --git a/api/src/database.rs b/api/src/database.rs deleted file mode 100644 index a9792af..0000000 --- a/api/src/database.rs +++ /dev/null @@ -1,179 +0,0 @@ -use std::collections::HashMap; -use sqlx::{Postgres, Transaction}; -use libseptastic::{stop_schedule::{Trip, TripTracking, StopSchedule}}; - -pub async fn get_route_by_id( - id: String, - transaction: &mut Transaction<'_, Postgres>, -) -> ::anyhow::Result { - - let row = sqlx::query!( - r#"SELECT - id, - name, - short_name, - color_hex, - route_type as "route_type: libseptastic::route::RouteType" - FROM - septa_routes - WHERE - id = $1 - ;"#, - id - ) - .fetch_one(&mut **transaction) - .await?; - - return Ok(libseptastic::route::Route { - name: row.name, - short_name: row.short_name, - color_hex: row.color_hex, - route_type: row.route_type, - id: row.id, - }); -} - -pub async fn get_all_routes( - transaction: &mut Transaction<'_, Postgres>, -) -> ::anyhow::Result> { - - let rows = sqlx::query!( - r#"SELECT - id, - name, - short_name, - color_hex, - route_type as "route_type: libseptastic::route::RouteType" - FROM - septa_routes - ORDER BY - CASE - WHEN id ~ '^[0-9]+$' THEN CAST(id AS INT) - ELSE NULL - END ASC, - id ASC; - ;"# - ) - .fetch_all(&mut **transaction) - .await?; - - let mut routes = Vec::new(); - - for row in rows { - routes.push(libseptastic::route::Route { - name: row.name, - short_name: row.short_name, - color_hex: row.color_hex, - route_type: row.route_type, - id: row.id, - }); - } - - return Ok(routes); -} - -pub async fn get_direction_by_route_id( - id: String, - transaction: &mut Transaction<'_, Postgres>, -) -> ::anyhow::Result> { - - let rows = sqlx::query!( - r#"SELECT - route_id, - direction_id, - direction as "direction: libseptastic::direction::CardinalDirection", - direction_destination - FROM - septa_directions - WHERE - route_id = $1 - ;"#, - id - ) - .fetch_all(&mut **transaction) - .await?; - - let mut res = Vec::new(); - - for row in rows { - res.push(libseptastic::direction::Direction{ - route_id: row.route_id, - direction_id: row.direction_id, - direction: row.direction, - direction_destination: row.direction_destination - }); - } - - return Ok(res); -} - -pub async fn get_schedule_by_route_id( - id: String, - transaction: &mut Transaction<'_, Postgres>, -) -> ::anyhow::Result> { - - let schedule_day = chrono::Utc::now().with_timezone(&chrono_tz::America::New_York); - let schedule_day_str = schedule_day.format("%Y%m%d").to_string(); - - let rows = sqlx::query!( - r#"SELECT - septa_stop_schedules.route_id, - septa_stops.name as stop_name, - trip_id, - septa_stop_schedules.service_id, - septa_stop_schedules.direction_id, - arrival_time, - stop_id, - stop_sequence - FROM - septa_stop_schedules - INNER JOIN septa_stops - ON septa_stops.id = septa_stop_schedules.stop_id - INNER JOIN septa_schedule_days - ON septa_schedule_days.date = $2 - AND - septa_schedule_days.service_id = septa_stop_schedules.service_id - WHERE - septa_stop_schedules.route_id = $1 OR septa_stop_schedules.route_id = 'B2' - ;"#, - id.clone(), - schedule_day_str.clone() - ) - .fetch_all(&mut **transaction) - .await?; - - let mut sched_groups: HashMap> = HashMap::new(); - for row in rows { - let arr = match sched_groups.get_mut(&row.trip_id) { - Some(x) => x, - None => { - sched_groups.insert(row.trip_id.clone(), Vec::new()); - sched_groups.get_mut(&row.trip_id).unwrap() - } - }; - - arr.push(StopSchedule { - route_id: row.route_id, - stop_name: row.stop_name, - trip_id: row.trip_id, - service_id: row.service_id, - direction_id: row.direction_id, - arrival_time: row.arrival_time, - stop_id: row.stop_id, - stop_sequence: row.stop_sequence - }); - } - let mut res = Vec::new(); - - for group in sched_groups { - res.push(Trip{ - trip_id: group.0, - route_id: group.1[0].route_id.clone(), - schedule: group.1.clone(), - direction_id: group.1[0].direction_id.clone(), - tracking_data: TripTracking::Untracked - }); - } - - return Ok(res); -} diff --git a/api/src/main.rs b/api/src/main.rs index 6c38392..14a32e9 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -3,18 +3,17 @@ use env_logger::Env; use log::*; use dotenv::dotenv; use serde::Deserialize; -use services::trip_tracking::{self}; +use services::{gtfs_pull, trip_tracking::{self}}; use templates::ContentTemplate; -use std::{sync::Arc, time::Instant}; +use std::{fs::File, io::Read, sync::Arc, time::Instant}; use askama::Template; -mod database; mod services; mod controllers; mod templates; pub struct AppState { - database: ::sqlx::postgres::PgPool, + gtfs_service: services::gtfs_pull::GtfsPullService, trip_tracking_service: services::trip_tracking::TripTrackingService } @@ -79,19 +78,22 @@ async fn main() -> ::anyhow::Result<()> { let version: &str = option_env!("CARGO_PKG_VERSION").expect("Expected package version"); info!("Starting the SEPTASTIC Server v{} (commit: {})", version, "NONE"); - let connection_string = - std::env::var("DB_CONNSTR").expect("Expected database connection string"); + let mut file = File::open("../data_loader/config.yaml")?; + let mut file_contents = String::new(); + file.read_to_string(&mut file_contents); - let pool = ::sqlx::postgres::PgPoolOptions::new() - .max_connections(5) - .connect(&connection_string) - .await?; + let config_file = serde_yaml::from_str::(file_contents.as_str())?; - let tt_service = trip_tracking::TripTrackingService::new(); + let tt_service = services::trip_tracking::TripTrackingService::new(); tt_service.start(); + let svc = gtfs_pull::GtfsPullService::new(config_file); + svc.start(); + svc.wait_for_ready(); + + let state = Arc::new(AppState { - database: pool, + gtfs_service: svc, trip_tracking_service: tt_service }); diff --git a/api/src/services/mod.rs b/api/src/services/mod.rs index dc3fd85..013f9ce 100644 --- a/api/src/services/mod.rs +++ b/api/src/services/mod.rs @@ -1 +1,3 @@ pub mod trip_tracking; +pub mod gtfs_pull; +pub mod gtfs_rt; diff --git a/api/src/services/trip_tracking.rs b/api/src/services/trip_tracking.rs index 25e0d13..5500b4a 100644 --- a/api/src/services/trip_tracking.rs +++ b/api/src/services/trip_tracking.rs @@ -97,7 +97,10 @@ impl TripTrackingService { delay: live_track.delay, next_stop_id: live_track.next_stop_id, timestamp: live_track.timestamp, - vehicle_id: live_track.vehicle_id + vehicle_ids: match live_track.vehicle_id { + Some(x) => x.split(",").map(|f| String::from(f)).collect(), + None => vec![] + } } ) } diff --git a/api/src/templates.rs b/api/src/templates.rs index 842d100..95358bb 100644 --- a/api/src/templates.rs +++ b/api/src/templates.rs @@ -70,7 +70,7 @@ pub fn build_timetables( let mut direction_trips: Vec<&Trip> = trips .iter() - .filter(|trip| trip.direction_id == direction.direction_id) + .filter(|trip| trip.direction.direction == direction.direction) .collect(); direction_trips.sort_by_key(|trip| { @@ -84,7 +84,7 @@ pub fn build_timetables( for trip in direction_trips.clone() { if let Some(last) = trip.schedule.last() { if next_id == None && i64::from(seconds_since_midnight) < last.arrival_time { - next_id = Some(last.trip_id.clone()); + next_id = Some(last.stop.id.to_string()); } } } @@ -105,12 +105,12 @@ pub fn build_timetables( for (trip_index, trip) in direction_trips.iter().enumerate() { for stop in &trip.schedule { let entry = stop_map - .entry(stop.stop_id) - .or_insert((stop.stop_sequence, stop.stop_name.clone(), vec![None; direction_trips.len()])); + .entry(stop.stop.id) + .or_insert((stop.stop_sequence, stop.stop.name.clone(), vec![None; direction_trips.len()])); // If this stop_id appears in multiple trips with different sequences, keep the lowest entry.0 = entry.0.max(stop.stop_sequence); - entry.1 = stop.stop_name.clone(); + entry.1 = stop.stop.name.clone(); entry.2[trip_index] = Some(stop.arrival_time); } } diff --git a/api/templates/route.html b/api/templates/route.html index a75dbd1..86fca33 100644 --- a/api/templates/route.html +++ b/api/templates/route.html @@ -89,7 +89,7 @@ document.querySelectorAll("details[data-direction-id]").forEach(details => { {% for timetable in timetables %} -
+

{{ timetable.direction.direction | capitalize }} to

@@ -104,7 +104,7 @@ document.querySelectorAll("details[data-direction-id]").forEach(details => { {% for trip_id in timetable.trip_ids %} {% if let Some(next_id_v) = timetable.next_id %} {% if next_id_v == trip_id %} - + {% else %} {% endif %} diff --git a/api/templates/route_symbol.html b/api/templates/route_symbol.html index e5551f2..aca4ed4 100644 --- a/api/templates/route_symbol.html +++ b/api/templates/route_symbol.html @@ -1,18 +1,18 @@ {% macro route_symbol(route) %} {% match route.route_type %} {% when libseptastic::route::RouteType::Trolley | libseptastic::route::RouteType::SubwayElevated %} -
- {{ route.id }} +
+ {{ route.short_name }}
{% endwhen %} {% when libseptastic::route::RouteType::RegionalRail %}
- {{ route.id }} + {{ route.short_name }}
{% endwhen %} {% when libseptastic::route::RouteType::Bus | libseptastic::route::RouteType::TracklessTrolley %}
- {{ route.id }} + {{ route.short_name }}
{% endwhen %} {% endmatch %} diff --git a/api/templates/routes.html b/api/templates/routes.html index dddb5fe..2c8d711 100644 --- a/api/templates/routes.html +++ b/api/templates/routes.html @@ -7,7 +7,7 @@

For infrequent rail service to suburban locations

{% for route in rr_routes %} -

[ {{ format!("{:7}", route.id) }}: {{ route.name }}

]

+

[ {{ format!("{:7}", route.short_name) }}: {{ route.name }}

]

 {% endfor %} @@ -20,7 +20,7 @@
{% for route in subway_routes %} -

[ {{ format!("{:7}", route.id) }}: {{ route.name }}

]

+

[ {{ format!("{:7}", route.short_name) }}: {{ route.name }}

]

 {% endfor %} @@ -30,7 +30,7 @@ {% for route in trolley_routes %} -

[ {{ format!("{:7}", route.id) }}: {{ route.name }}

]

+

[ {{ format!("{:7}", route.short_name) }}: {{ route.name }}

]

 {% endfor %} @@ -40,7 +40,7 @@

For service of varying frequency within SEPTA's entire service area

{% for route in bus_routes %} -

[ {{ format!("{:7}", route.id) }}: {{ route.name }}

]

+

[ {{ format!("{:7}", route.short_name) }}: {{ route.name }}

]

 {% endfor %} diff --git a/data_loader/config.yaml b/data_loader/config.yaml index 2fea038..5852e32 100644 --- a/data_loader/config.yaml +++ b/data_loader/config.yaml @@ -3,5 +3,8 @@ gtfs_zips: subzip: "google_rail.zip" - uri: "https://www3.septa.org/developer/gtfs_public.zip" subzip: "google_bus.zip" - - uri: "https://www.njtransit.com/rail_data.zip" - - uri: "https://www.njtransit.com/bus_data.zip" +# - uri: "https://www.njtransit.com/rail_data.zip" +# - uri: "https://www.njtransit.com/bus_data.zip" +annotations: + synthetic_routes: + - id: 'NYC' diff --git a/data_loader/src/main.rs b/data_loader/src/main.rs index 2316def..c9c2cda 100644 --- a/data_loader/src/main.rs +++ b/data_loader/src/main.rs @@ -15,14 +15,6 @@ async fn main() -> ::anyhow::Result<()> { let env = Env::new().filter_or("RUST_LOG", "data_loader=info"); Builder::from_env(env).init(); - let mut file = File::open("config.yaml")?; - let mut file_contents = String::new(); - file.read_to_string(&mut file_contents); - - let config_file = serde_yaml::from_str::(file_contents.as_str())?; - - let svc = GtfsPullService::new(config_file); - svc.start(); loop{ thread::sleep(Duration::from_secs(120)); @@ -41,88 +33,3 @@ async fn main() -> ::anyhow::Result<()> { Ok(()) } - -#[derive(Serialize, Deserialize, PartialEq, Debug,Clone)] -struct GtfsSource { - pub uri: String, - pub subzip: Option -} - -#[derive(Serialize, Deserialize, PartialEq, Debug)] -struct Config { - pub gtfs_zips: Vec -} - -struct GtfsFile { - pub source: GtfsSource, - pub hash: Option -} - -struct GtfsPullServiceState { - pub gtfs_files: Vec, - pub tmp_dir: PathBuf -} - -pub struct GtfsPullService { - state: Arc> -} - -impl GtfsPullService { - const UPDATE_SECONDS: u64 = 3600*24; - - pub fn new(config: Config) -> Self { - Self { - state: Arc::new(Mutex::new( - GtfsPullServiceState { - gtfs_files: config.gtfs_zips.iter().map(|f| { GtfsFile { source: f.clone(), hash: None} }).collect(), - tmp_dir: env::temp_dir() - } - )) - } - } - - pub fn start(&self) { - let cloned_state = Arc::clone(&self.state); - thread::spawn(move || { - loop { - let recloned_state = Arc::clone(&cloned_state); - let res = Self::update_gtfs_data(recloned_state); - - match res { - Err(err) => { - error!("{}", err); - } - _ => {} - } - - thread::sleep(Duration::from_secs(Self::UPDATE_SECONDS)); - } - }); - } - - pub fn update_gtfs_data(state: Arc>) -> anyhow::Result<()> { - let l_state = state.lock().unwrap(); - - for gtfs_file in l_state.gtfs_files.iter() { - let gtfs = if let Some(subzip) = gtfs_file.source.subzip.clone() { - info!("Reading GTFS file at {} (subzip {})", gtfs_file.source.uri, subzip); - let res = reqwest::blocking::get(gtfs_file.source.uri.clone())?; - let outer_archive = res.bytes()?; - let mut archive = ZipArchive::new(Cursor::new(outer_archive))?; - archive.extract(l_state.tmp_dir.clone())?; - - let mut file_path = l_state.tmp_dir.clone(); - file_path.push(subzip.clone()); - - gtfs_structures::Gtfs::new(file_path.to_str().unwrap())? - } else { - info!("Reading GTFS file at {}", gtfs_file.source.uri); - gtfs_structures::Gtfs::new(gtfs_file.source.uri.as_str())? - }; - } - - - - Ok(()) - } -} diff --git a/libseptastic/src/direction.rs b/libseptastic/src/direction.rs index beb68b6..88bbc54 100644 --- a/libseptastic/src/direction.rs +++ b/libseptastic/src/direction.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; -#[derive(sqlx::Type, Serialize, Deserialize, PartialEq, Debug, Clone)] +#[derive(sqlx::Type, Serialize, Deserialize, PartialEq, Debug, Clone, Copy, Eq)] #[sqlx(type_name = "septa_direction_type", rename_all = "snake_case")] pub enum CardinalDirection { Northbound, @@ -15,8 +15,6 @@ pub enum CardinalDirection { #[derive(::sqlx::Decode, Serialize, Deserialize, Debug, Clone)] pub struct Direction { - pub route_id: String, - pub direction_id: i64, pub direction: CardinalDirection, pub direction_destination: String } diff --git a/libseptastic/src/lib.rs b/libseptastic/src/lib.rs index af16fa4..29bdfe8 100644 --- a/libseptastic/src/lib.rs +++ b/libseptastic/src/lib.rs @@ -1,4 +1,5 @@ pub mod route; +pub mod agency; pub mod stop; pub mod route_stop; pub mod stop_schedule; diff --git a/libseptastic/src/stop.rs b/libseptastic/src/stop.rs index 65cb01b..911fa49 100644 --- a/libseptastic/src/stop.rs +++ b/libseptastic/src/stop.rs @@ -1,4 +1,6 @@ -#[derive(sqlx::Type, PartialEq, Debug, Clone)] +use serde::{Deserialize, Serialize}; + +#[derive(sqlx::Type, PartialEq, Debug, Clone, Serialize, Deserialize)] #[sqlx(type_name = "septa_stop_type", rename_all = "snake_case")] pub enum StopType { FarSide, @@ -6,7 +8,7 @@ pub enum StopType { Normal } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Stop { pub id: i64, pub name: String, diff --git a/libseptastic/src/stop_schedule.rs b/libseptastic/src/stop_schedule.rs index 4d367c9..698b8e9 100644 --- a/libseptastic/src/stop_schedule.rs +++ b/libseptastic/src/stop_schedule.rs @@ -1,22 +1,19 @@ use serde::{Deserialize, Serialize}; +use crate::direction::Direction; + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StopSchedule { - pub route_id: String, - pub stop_name: String, - pub trip_id: String, - pub service_id: String, - pub direction_id: i64, pub arrival_time: i64, - pub stop_id: i64, - pub stop_sequence: i64 + pub stop_sequence: i64, + pub stop: crate::stop::Stop } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Trip { - pub route_id: String, + pub service_id: String, pub trip_id: String, - pub direction_id: i64, + pub direction: Direction, pub tracking_data: TripTracking, pub schedule: Vec } @@ -33,5 +30,5 @@ pub struct LiveTrip { pub delay: f64, pub next_stop_id: Option, pub timestamp: i64, - pub vehicle_id: Option + pub vehicle_ids: Vec } diff --git a/shell.nix b/shell.nix index 08eaca4..ea8ae70 100644 --- a/shell.nix +++ b/shell.nix @@ -4,5 +4,6 @@ stdenv.mkDerivation { nativeBuildInputs = [ pkg-config postgresql_14 ]; buildInputs = [ cryptsetup + protobuf ]; }