From 167cffc868a81b639305b905aa7f87202b37f89f Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Mon, 6 Oct 2025 23:01:09 -0400 Subject: [PATCH 1/4] init gtfs setup --- data_loader/Cargo.lock | 20 ++ data_loader/Cargo.toml | 1 + data_loader/config.yaml | 4 + data_loader/src/fetchers/mod.rs | 12 - data_loader/src/fetchers/septa.rs | 258 -------------------- data_loader/src/main.rs | 124 ++++++---- data_loader/src/septa/direction.rs | 85 ------- data_loader/src/septa/mod.rs | 7 - data_loader/src/septa/ridership.rs | 216 ---------------- data_loader/src/septa/route.rs | 187 -------------- data_loader/src/septa/route_stop.rs | 78 ------ data_loader/src/septa/schedule_day.rs | 70 ------ data_loader/src/septa/stop.rs | 108 -------- data_loader/src/septa/stop_schedule.rs | 127 ---------- data_loader/src/septa_json/direction.rs | 19 -- data_loader/src/septa_json/mod.rs | 6 - data_loader/src/septa_json/ridership.rs | 37 --- data_loader/src/septa_json/route.rs | 42 ---- data_loader/src/septa_json/route_stop.rs | 31 --- data_loader/src/septa_json/schedule_day.rs | 18 -- data_loader/src/septa_json/stop_schedule.rs | 27 -- data_loader/src/traits.rs | 23 -- 22 files changed, 99 insertions(+), 1401 deletions(-) create mode 100644 data_loader/config.yaml delete mode 100644 data_loader/src/fetchers/mod.rs delete mode 100644 data_loader/src/fetchers/septa.rs delete mode 100644 data_loader/src/septa/direction.rs delete mode 100644 data_loader/src/septa/mod.rs delete mode 100644 data_loader/src/septa/ridership.rs delete mode 100644 data_loader/src/septa/route.rs delete mode 100644 data_loader/src/septa/route_stop.rs delete mode 100644 data_loader/src/septa/schedule_day.rs delete mode 100644 data_loader/src/septa/stop.rs delete mode 100644 data_loader/src/septa/stop_schedule.rs delete mode 100644 data_loader/src/septa_json/direction.rs delete mode 100644 data_loader/src/septa_json/mod.rs delete mode 100644 data_loader/src/septa_json/ridership.rs delete mode 100644 data_loader/src/septa_json/route.rs delete mode 100644 data_loader/src/septa_json/route_stop.rs delete mode 100644 data_loader/src/septa_json/schedule_day.rs delete mode 100644 data_loader/src/septa_json/stop_schedule.rs delete mode 100644 data_loader/src/traits.rs diff --git a/data_loader/Cargo.lock b/data_loader/Cargo.lock index 51ba98b..62f9412 100644 --- a/data_loader/Cargo.lock +++ b/data_loader/Cargo.lock @@ -287,6 +287,7 @@ dependencies = [ "serde-map-to-array", "serde_json", "serde_repr", + "serde_yaml", "sqlx", "thiserror", "tokio", @@ -1668,6 +1669,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2272,6 +2286,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" diff --git a/data_loader/Cargo.toml b/data_loader/Cargo.toml index 00ecaa0..4b39f8c 100644 --- a/data_loader/Cargo.toml +++ b/data_loader/Cargo.toml @@ -17,3 +17,4 @@ libseptastic = { path = "../libseptastic/" } env_logger = "0.11.8" log = "0.4.27" reqwest = { version = "0.12.22", features = [ "json", "blocking" ] } +serde_yaml = "0.9.34" diff --git a/data_loader/config.yaml b/data_loader/config.yaml new file mode 100644 index 0000000..9b680f7 --- /dev/null +++ b/data_loader/config.yaml @@ -0,0 +1,4 @@ +gtfs_zips: + - "https://www3.septa.org/developer/gtfs_public.zip" + - "https://www.njtransit.com/rail_data.zip" + - "https://www.njtransit.com/bus_data.zip" diff --git a/data_loader/src/fetchers/mod.rs b/data_loader/src/fetchers/mod.rs deleted file mode 100644 index 7edc9ae..0000000 --- a/data_loader/src/fetchers/mod.rs +++ /dev/null @@ -1,12 +0,0 @@ -use libseptastic::{direction::Direction, route::Route, route_stop::RouteStop, schedule_day::ScheduleDay, stop::Stop, stop_schedule::StopSchedule}; - -pub mod septa; - -pub struct AuthorityInfo { - pub routes: Vec, - pub stops: Vec, - pub route_stops: Vec, - pub stop_schedules: Vec, - pub schedule_days: Vec, - pub directions: Vec -} diff --git a/data_loader/src/fetchers/septa.rs b/data_loader/src/fetchers/septa.rs deleted file mode 100644 index 8a40774..0000000 --- a/data_loader/src/fetchers/septa.rs +++ /dev/null @@ -1,258 +0,0 @@ -use dotenv::dotenv; -use libseptastic::stop::Stop; -use crate::septa::route_stop; -use crate::septa_json::direction::Direction; -use crate::septa_json::route::Route; -use crate::septa_json::route_stop::RouteStop; -use crate::septa_json::schedule_day::{Calendar, ScheduleDay}; -use crate::septa_json::stop_schedule::StopSchedule; -use sqlx::postgres::PgPoolOptions; -use std::collections::{HashMap, HashSet, VecDeque}; -use std::sync::{Arc, Mutex}; -use std::{fs, thread}; -use std::path::Path; -use std::time::SystemTime; -use env_logger::{Builder, Env}; -use log::{error, info, warn}; -use crate::traits::*; - -use super::AuthorityInfo; - -async fn convert_routes(json_data: Vec) -> ::anyhow::Result> { - let mut parsed_data: Vec = Vec::new(); - - for json_single in json_data { - parsed_data.push(*libseptastic::route::Route::from_septa_json(json_single)?); - } - - Ok(parsed_data) -} - -async fn convert_stops(json_data: Vec) -> ::anyhow::Result> { - let mut parsed_data: Vec = Vec::new(); - - for json_single in json_data { - parsed_data.push(*libseptastic::stop::Stop::from_septa_json(json_single)?); - } - - Ok(parsed_data) -} - -async fn convert_route_stops(json_data: Vec) -> ::anyhow::Result> { - let mut parsed_data: Vec = Vec::new(); - - for json_single in json_data { - parsed_data.push(*libseptastic::route_stop::RouteStop::from_septa_json(json_single)?); - } - - Ok(parsed_data) -} - -async fn convert_stop_schedules(json_data:Vec) -> ::anyhow::Result> { - let mut parsed_data: Vec = Vec::new(); - - for json_single in json_data { - parsed_data.push(*libseptastic::stop_schedule::StopSchedule::from_septa_json(json_single)?); - } - - - Ok(parsed_data) -} - - -async fn convert_schedule_days(json_data: crate::septa_json::schedule_day::Calendar) -> ::anyhow::Result> { - let mut parsed_data: Vec = Vec::new(); - - parsed_data.append(&mut *Vec::::from_septa_json(json_data)?); - - Ok(parsed_data) -} - -async fn convert_directions(json_data: Vec) -> ::anyhow::Result> { - let mut parsed_data: Vec = Vec::new(); - - for json_single in json_data { - parsed_data.push(*libseptastic::direction::Direction::from_septa_json(json_single)?); - } - - Ok(parsed_data) -} - - -pub struct SeptaFetcher{ -} - -impl SeptaFetcher { - const HOST: &str = "https://flat-api.septa.org"; - - async fn fetch_version() -> ::anyhow::Result { - let unix_seconds = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?.as_secs(); - let mut version = reqwest::get(format!("{}/version.txt?t={}", Self::HOST, unix_seconds)).await?.text().await?; - version = version.trim().to_string(); - Ok(version) - } - - async fn fetch_routes(version: &String) -> ::anyhow::Result> { - Ok(reqwest::get(format!("{}/routes.json?v={}", Self::HOST, version)).await?.json::>().await?) - } - - async fn fetch_route_stops(routes: &mut Vec, version: &String) -> ::anyhow::Result> { - let mut all_route_stops: Vec = Vec::new(); - - let mut i = 0; - while i < routes.len() { - let route = &routes[i]; - if let Ok(route_stops_json) = reqwest::get(format!("{}/stops/{}/stops.json?v={}", Self::HOST, route.route_id, version)).await { - if let Ok(mut route_stops) = route_stops_json.json::>().await { - all_route_stops.append(&mut route_stops); - i += 1; - continue; - } - } - - warn!("Encountered failure downloading route stops for Route {}, removing", route.route_id); - routes.swap_remove(i); - } - - if routes.len() == 0 { - error!("Unable to download SEPTA routes"); - return Err(anyhow::anyhow!("Unable to successfully download any SEPTA routes")); - } - - Ok(all_route_stops) - } - - async fn fetch_directions(version: &String) -> ::anyhow::Result> { - Ok(reqwest::get(format!("{}/directions.json?v={}", Self::HOST, version)).await?.json::>().await?) - } - - async fn fetch_calendar(version: &String) -> ::anyhow::Result { - Ok(reqwest::get(format!("{}/calendar.json?v={}", Self::HOST, version)).await?.json::().await?) - } - - - async fn fetch_schedules(stops: &Vec, version: &String) -> ::anyhow::Result> { - const NUM_THREADS: usize = 10; - - let mut threads = Vec::with_capacity(NUM_THREADS); - let mut stop_ids: HashSet = HashSet::new(); - let results = Arc::new(Mutex::new(Vec::new())); - let mut worker_queues: Vec> = Vec::with_capacity(NUM_THREADS); - - for _ in 0..NUM_THREADS { - worker_queues.push(VecDeque::new()); - } - - for stop in stops { - stop_ids.insert(stop.stop_id); - } - - let mut cur_thread_no = 0; - for stop_id in stop_ids { - worker_queues[cur_thread_no].push_back((stop_id, 0)); - cur_thread_no = (cur_thread_no + 1) % NUM_THREADS; - } - - - for thread_no in 0..NUM_THREADS { - let mut worker_queue = worker_queues[thread_no].clone(); - let local_result = results.clone(); - let local_version = version.clone(); - - threads.push(thread::spawn(move || { - let mut stop_schedules: Vec = Vec::new(); - - while let Some(stop_id) = worker_queue.pop_front() { - let uri = format!("{}/schedules/stop_schedule/{}.json?v={}", Self::HOST, stop_id.0, local_version); - - match (||{reqwest::blocking::get(uri)?.json::>()})() { - Ok(mut element) => (stop_schedules.append(&mut element)), - Err(_) => { - if stop_id.1 < 5 { - error!("Error downloading stop schedule for {} (try {}). Retrying.", stop_id.0, stop_id.1); - worker_queue.push_back((stop_id.0, stop_id.1 + 1)); - } else { - error!("Error downloading stop schedule for {} (try {}). Giving up.", stop_id.0, stop_id.1); - } - } - } - } - - if let Ok(mut res) = local_result.lock() { - res.append(&mut stop_schedules); - } - })); - } - - for thread in threads { - thread.join(); - } - - Ok(results.lock().unwrap().clone()) - } - - pub async fn fetch_septa_data() -> ::anyhow::Result { - let version = Self::fetch_version().await?; - info!("Got SEPTA schedule version {}", version); - - let mut routes = Self::fetch_routes(&version).await?; - info!("Discovered {} SEPTA routes", routes.len()); - - routes = routes.into_iter().filter(|x| x.release_name == "20250928").collect(); - - let route_stops = Self::fetch_route_stops(&mut routes, &version).await?; - info!("Stop data for {} stops on {} routes successfully downloaded", route_stops.len(), routes.len()); - - let directions = Self::fetch_directions(&version).await?; - info!("Directions data successfully downloaded for {} route directions", directions.len()); - - let mut calendar = Self::fetch_calendar(&version).await?; - info!("Calendar data successfully downloaded for {} days", calendar.len()); - for entry in calendar.clone() { - if entry.1.release_name != "20250928" { - calendar.remove(&entry.0.clone()); - } - } - - let mut schedule_stops = Self::fetch_schedules(&route_stops, &version).await?; - info!("Schedule data downloaded for {} scheduled stops", schedule_stops.len()); - schedule_stops = schedule_stops.into_iter().filter(|x| x.release_name == "20250928").collect(); - - let mut stop_map: HashMap = HashMap::new(); - let mut stop_set: HashSet = HashSet::new(); - - for stop in route_stops.iter() { - stop_map.insert(stop.stop_name.clone(), stop.stop_id); - stop_set.insert(stop.stop_id); - } - - - let mut route_map: HashSet = HashSet::new(); - - for route in routes.iter() { - if route_map.contains(&route.route_id) { - error!("Duplicate route found for {}", route.route_id.clone()); - } - route_map.insert(route.route_id.clone()); - } - - let filtered_directions: Vec<_> = directions.iter() - .filter(|direction| { - let keep = route_map.contains(&direction.route); - if !keep { - warn!( - "Removing route '{}' from direction data (This data has old and new wayfinding values)", - direction.route - ); - } - keep - }) - .cloned() - .collect(); - - info!("Data is valid"); - - - Ok(AuthorityInfo { routes: convert_routes(routes).await?, stops: convert_stops(route_stops.clone()).await?, route_stops: convert_route_stops(route_stops).await?, stop_schedules: convert_stop_schedules(schedule_stops).await?, schedule_days: convert_schedule_days(calendar).await?, directions: convert_directions(filtered_directions).await? }) - } -} diff --git a/data_loader/src/main.rs b/data_loader/src/main.rs index e51f72b..8c684ab 100644 --- a/data_loader/src/main.rs +++ b/data_loader/src/main.rs @@ -1,27 +1,11 @@ +use std::{clone, env, fs::File, io::{Read, Write}, path::{Path, PathBuf}, sync::{Arc, Mutex}, thread, time::Duration}; + use dotenv::dotenv; -use libseptastic::stop::Stop; -use septa::route_stop; -use septa_json::direction::Direction; -use septa_json::route::Route; -use septa_json::route_stop::RouteStop; -use septa_json::schedule_day::{Calendar, ScheduleDay}; -use septa_json::stop_schedule::StopSchedule; +use serde::{Deserialize, Serialize}; use sqlx::postgres::PgPoolOptions; -use std::collections::{HashMap, HashSet}; -use std::sync::{Arc, Mutex}; -use std::{fs, thread}; -use std::path::Path; -use std::time::SystemTime; use env_logger::{Builder, Env}; use log::{error, info, warn}; -pub mod traits; -use traits::*; - -pub mod septa_json; -pub mod septa; -pub mod fetchers; - #[tokio::main] async fn main() -> ::anyhow::Result<()> { @@ -29,46 +13,86 @@ async fn main() -> ::anyhow::Result<()> { let env = Env::new().filter_or("RUST_LOG", "data_loader=info"); Builder::from_env(env).init(); + + let mut file = File::open("config.yaml")?; + let mut file_contents = String::new(); + file.read_to_string(&mut file_contents); + + let config_file = serde_yaml::from_str::(file_contents.as_str()); + let database_url = std::env::var("DATABASE_URL").expect("Database URL"); let pool = PgPoolOptions::new() .max_connections(5) .connect(&database_url) .await?; - - let septa_data = fetchers::septa::SeptaFetcher::fetch_septa_data().await?; - + let mut tx = pool.begin().await?; - libseptastic::route::Route::create_table(&mut tx).await?; - libseptastic::direction::Direction::create_table(&mut tx).await?; - libseptastic::stop::Stop::create_table(&mut tx).await?; - libseptastic::route_stop::RouteStop::create_table(&mut tx).await?; - libseptastic::stop_schedule::StopSchedule::create_table(&mut tx).await?; - libseptastic::schedule_day::ScheduleDay::create_table(&mut tx).await?; - - - - info!("Inserting Route Data"); - libseptastic::route::Route::insert_many(septa_data.routes, &mut tx).await?; - - info!("Inserting Direction Data"); - libseptastic::direction::Direction::insert_many(septa_data.directions, &mut tx).await?; - - info!("Inserting Stop Data"); - libseptastic::stop::Stop::insert_many(septa_data.stops, &mut tx).await?; - - info!("Inserting Route-Stop Data"); - libseptastic::route_stop::RouteStop::insert_many(septa_data.route_stops, &mut tx).await?; - - info!("Inserting Stop Schedule Data"); - libseptastic::stop_schedule::StopSchedule::insert_many(septa_data.stop_schedules, &mut tx).await?; - - info!("Inserting Schedule Day Data"); - libseptastic::schedule_day::ScheduleDay::insert_many(septa_data.schedule_days, &mut tx).await?; - - tx.commit().await?; pool.close().await; Ok(()) } + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +struct Config { + gtfs_zips: Vec +} + +struct GtfsFile { + pub url: String, + pub hash: Option +} + +struct GtfsPullServiceState { + pub gtfs_files: Vec, + pub tmp_dir: PathBuf +} + +pub struct GtfsPullService { + state: Arc> +} + +impl GtfsPullService { + const UPDATE_SECONDS: u64 = 3600*24; + + pub fn new(config: Config) -> Self { + Self { + state: Arc::new(Mutex::new( + GtfsPullServiceState { + gtfs_files: config.gtfs_zips.iter().map(|f| { GtfsFile { url: f.clone(), hash: None} }).collect(), + tmp_dir: env::temp_dir() + } + )) + } + } + + pub fn start(&self) { + let cloned_state = Arc::clone(&self.state); + thread::spawn(move || { + loop { + let recloned_state = Arc::clone(&cloned_state); + let res = Self::update_gtfs_data(recloned_state); + + match res { + Err(err) => { + error!("{}", err); + } + _ => {} + } + + thread::sleep(Duration::from_secs(Self::UPDATE_SECONDS)); + } + }); + } + + pub fn update_gtfs_data(state: Arc>) -> anyhow::Result<()> { + let l_state = state.lock().unwrap(); + + for gtfs_file in l_state.gtfs_files.iter() { + let resp = reqwest::blocking::get(gtfs_file.url.clone())?; + } + + Ok(()) + } +} diff --git a/data_loader/src/septa/direction.rs b/data_loader/src/septa/direction.rs deleted file mode 100644 index 515e775..0000000 --- a/data_loader/src/septa/direction.rs +++ /dev/null @@ -1,85 +0,0 @@ -use libseptastic::direction::{CardinalDirection, Direction}; -use sqlx::{Postgres, Transaction}; -use crate::traits::{DbObj, FromSeptaJson}; -use crate::septa_json; - - -impl DbObj for Direction { - async fn create_table(tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - sqlx::query(" - CREATE TABLE IF NOT EXISTS septa_directions ( - route_id TEXT NOT NULL, - direction_id BIGINT NOT NULL, - direction septa_direction_type NOT NULL, - direction_destination TEXT NOT NULL, - - FOREIGN KEY (route_id) REFERENCES septa_routes(id) ON DELETE CASCADE, - PRIMARY KEY (route_id, direction_id) - - ); - ") - .execute(&mut **tx) - .await?; - - Ok(()) - } - async fn insert_many(dirs: Vec, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - let mut route_ids: Vec = Vec::new(); - let mut direction_ids: Vec = Vec::new(); - let mut directions: Vec = Vec::new(); - let mut direction_destinations: Vec = Vec::new(); - - for dir in dirs { - route_ids.push(dir.route_id); - direction_ids.push(dir.direction_id); - directions.push(dir.direction); - direction_destinations.push(dir.direction_destination); - } - - sqlx::query(" - INSERT INTO septa_directions ( - route_id, - direction_id, - direction, - direction_destination - ) - SELECT * FROM UNNEST( - $1::text[], - $2::bigint[], - $3::septa_direction_type[], - $4::text[] - ); - ") - .bind(&route_ids[..]) - .bind(&direction_ids[..]) - .bind(&directions[..]) - .bind(&direction_destinations[..]) - .execute(&mut **tx) - .await?; - - Ok(()) - } - - async fn insert(&self, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - Self::insert_many(vec![self.clone()], tx).await?; - Ok(()) - } -} - -impl FromSeptaJson for Direction { - fn from_septa_json(json_dir: septa_json::direction::Direction) -> ::anyhow::Result> { - Ok(Box::new(Direction { route_id: json_dir.route, direction_id: json_dir.direction.parse::()?, - direction: match json_dir.true_direction.as_str().clone() { - "Eastbound" => Ok(libseptastic::direction::CardinalDirection::Eastbound), - "Westbound" => Ok(libseptastic::direction::CardinalDirection::Westbound), - "Outbound" => Ok(libseptastic::direction::CardinalDirection::Outbound), - "Inbound" => Ok(libseptastic::direction::CardinalDirection::Inbound), - "Southbound" => Ok(libseptastic::direction::CardinalDirection::Southbound), - "Northbound" => Ok(libseptastic::direction::CardinalDirection::Northbound), - "LOOP" => Ok(libseptastic::direction::CardinalDirection::Loop), - "Loop" => Ok(libseptastic::direction::CardinalDirection::Loop), - _ => Err(anyhow::anyhow!(format!("Unable to find right direction {}", json_dir.true_direction))) - }? - , direction_destination: json_dir.direction_destination })) - } -} diff --git a/data_loader/src/septa/mod.rs b/data_loader/src/septa/mod.rs deleted file mode 100644 index 07ad21e..0000000 --- a/data_loader/src/septa/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -pub mod route; -pub mod direction; -pub mod stop; -pub mod route_stop; -pub mod stop_schedule; -pub mod schedule_day; -pub mod ridership; diff --git a/data_loader/src/septa/ridership.rs b/data_loader/src/septa/ridership.rs deleted file mode 100644 index 4049608..0000000 --- a/data_loader/src/septa/ridership.rs +++ /dev/null @@ -1,216 +0,0 @@ -use std::collections::HashMap; - -use libseptastic::direction::CardinalDirection; -use libseptastic::ridership::Ridership; -use sqlx::{Postgres, Transaction}; -use crate::traits::{DbObj, FromSeptaJson, FromSeptaJsonAndStations}; -use crate::septa_json; - - -impl DbObj for Ridership { - async fn create_table(tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - sqlx::query(" - CREATE TABLE IF NOT EXISTS septa_ridership ( - route_id TEXT NOT NULL, - stop_id bigint NOT NULL, - direction septa_direction_type, - exp_ons bigint, - exp_offs bigint, - ons bigint, - offs bigint, - year bigint, - - PRIMARY KEY (route_id, stop_id, direction), - - FOREIGN KEY (route_id) REFERENCES septa_routes(id) ON DELETE CASCADE, - FOREIGN KEY (stop_id) REFERENCES septa_stops(id) ON DELETE CASCADE - ); - ") - .execute(&mut **tx) - .await?; - - Ok(()) - } - - async fn insert_many(dirs: Vec, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - let mut route_ids: Vec = Vec::new(); - let mut stop_ids: Vec = Vec::new(); - let mut directions: Vec = Vec::new(); - let mut exp_onss: Vec = Vec::new(); - let mut exp_offss: Vec = Vec::new(); - let mut onss: Vec = Vec::new(); - let mut offss: Vec = Vec::new(); - let mut years: Vec = Vec::new(); - - for dir in dirs { - route_ids.push(dir.route_id); - stop_ids.push(dir.stop_id); - directions.push(dir.direction); - exp_onss.push(dir.exp_ons); - exp_offss.push(dir.exp_offs); - onss.push(dir.ons); - offss.push(dir.offs); - years.push(dir.year); - } - - sqlx::query(" - INSERT INTO - septa_ridership - ( - route_id, - stop_id, - direction, - exp_ons, - exp_offs, - ons, - offs, - year - ) - SELECT * FROM UNNEST( - $1::text[], - $2::bigint[], - $3::septa_direction_type[], - $4::bigint[], - $5::bigint[], - $6::bigint[], - $7::bigint[], - $8::bigint[] - ) - ON CONFLICT DO NOTHING - ; - ") - .bind(&route_ids[..]) - .bind(&stop_ids[..]) - .bind(&directions[..]) - .bind(&exp_onss[..]) - .bind(&exp_offss[..]) - .bind(&onss[..]) - .bind(&offss[..]) - .bind(&years[..]) - .execute(&mut **tx) - .await?; - - Ok(()) - } - - async fn insert(&self, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - Self::insert_many(vec![self.clone()], tx).await?; - Ok(()) - } -} - -impl FromSeptaJsonAndStations for Ridership { - fn from_septa_json(json_rider: septa_json::ridership::Ridership, stop_map: &HashMap) -> ::anyhow::Result> { - Ok(Box::new(Ridership { - route_id: match json_rider.route_id.as_str() { - // SEPTA Metro - "MFL" => "L1", - "BSL" => "B1", - "MFO" => "L1 OWL", - "BSO" => "B1 OWL", - "J" => "41", - "L" => "51", - "G" => "63", - "H" => "71", - "XH" => "81", - "R" => "82", - "101" => "D1", - "102" => "D2", - "15" => "G1", - "NHSL" => "M1", - "10" => "T1", - "34" => "T2", - "13" => "T3", - "11" => "T4", - "36" => "T5", - // Regional Rail Remap - "Airport" => "AIR", - "Chestnut Hill West" => "CHW", - "Chestnut Hill East" => "CHE", - "Cynwyd" => "CYN", - "Fox Chase" => "FOX", - "Lansdale/Doylestown" => "LAN", - "Media/Wawa" => "MED", - "Manyunk/Norristown" => "NOR", - "Paoli/Thorndale" => "PAO", - "Trenton" => "TRE", - "Warminster" => "WAR", - "Wilmington/Newark" => "WIL", - "West Trenton" => "WTR", - any => any - }.to_string(), - stop_id: { - if json_rider.stop_code != "" { - Ok(json_rider.stop_code.parse::()?) - } else { - if let Some(sid) = stop_map.get(&(match json_rider.stop_name.as_str() { - "Churchmans Crossing" => "Churchman's Crossing", - "Prospect Park" => "Prospect Park - Moore", - "Highland Ave." => "Highland Avenue", - "Chester TC" => "Chester Transit Center", - "University City" => "Penn Medicine Station", - "Wynnefield" => "Wynnefield Avenue", - "Temple" => "Temple University", - "Fern Rock" => "Fern Rock T C", - "Jenkintown" => "Jenkintown Wyncote", - "Cornwells Height" => "Cornwells Heights", - "Levittown" => "Levittown Station", - "9th Street" => "9th Street Lansdale", - "Del Val College" => "Delaware Valley College", - "Sedwick" => "Sedgwick", - "Allen Lane" => "Richard Allen Lane", - "Elm Street" => "Norristown - Elm Street", - "Norristown" => "Norristown Transit Center", - "Neshaminy Falls" => "Neshaminy", - "Moylan Rose Valley" => "Moylan-Rose Valley", - "Morton" => "Morton-Rutledge", - "Easwick" => "Eastwick", - "30th Street Station" => "Gray 30th Street", - "Holmesburg Jct" => "Holmesburg Junction", - any => any - }.to_string())) { - Ok(sid.clone()) - } else { - Err(::anyhow::anyhow!("Station {} not found", json_rider.stop_name)) - } - } - }?, - exp_ons: json_rider.exp_ons.parse()?, - exp_offs: json_rider.exp_offs.parse()?, - ons: json_rider.ons.parse()?, - offs: json_rider.offs.parse()?, - year: 2024, //FIXME FIXME! Actually parse - direction: match json_rider.direction.as_str() { - "Eastbound" => Ok(CardinalDirection::Eastbound), - "Westbound" => Ok(CardinalDirection::Westbound), - "Northbound" => Ok(CardinalDirection::Northbound), - "Southbound" => Ok(CardinalDirection::Southbound), - "Inbound" => Ok(CardinalDirection::Inbound), - "Outbound" => Ok(CardinalDirection::Outbound), - "Loop" => Ok(CardinalDirection::Loop), - "" => match json_rider.route_id.as_str() { - "AIR" => Ok(CardinalDirection::Inbound), - "CHE" => Ok(CardinalDirection::Outbound), - "CHW" => Ok(CardinalDirection::Inbound), - "CYN" => Ok(CardinalDirection::Inbound), - "FOX" => Ok(CardinalDirection::Outbound), - "LAN" => Ok(CardinalDirection::Outbound), - "NOR" => Ok(CardinalDirection::Outbound), - "MED" => Ok(CardinalDirection::Inbound), - "PAO" => Ok(CardinalDirection::Inbound), - "TRE" => Ok(CardinalDirection::Inbound), - "WAR" => Ok(CardinalDirection::Outbound), - "WTR" => Ok(CardinalDirection::Outbound), - "WIL" => Ok(CardinalDirection::Inbound), - _ => Err(anyhow::anyhow!("Bad dir value RR route id {}", json_rider.route_id)) - }, - _ => Err( - ::anyhow::anyhow!( - "Unknown true direction {} for Route {}", - json_rider.direction, - json_rider.route_id - ) - ) - }?})) - } -} diff --git a/data_loader/src/septa/route.rs b/data_loader/src/septa/route.rs deleted file mode 100644 index d5ee633..0000000 --- a/data_loader/src/septa/route.rs +++ /dev/null @@ -1,187 +0,0 @@ -use sqlx::{Postgres, Transaction}; -use crate::traits::{DbObj, FromSeptaJson}; -use crate::septa_json; -use libseptastic::route::{Route, RouteType, InterlinedRoute}; - -impl DbObj for Route { - - async fn create_table(tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - sqlx::query(" - CREATE TYPE septa_route_type AS ENUM ( - 'trolley', - 'subway_elevated', - 'regional_rail', - 'bus', - 'trackless_trolley' - ); - ") - .execute(&mut **tx) - .await?; - - sqlx::query(" - CREATE TYPE septa_direction_type AS ENUM ( - 'northbound', - 'southbound', - 'eastbound', - 'westbound', - 'inbound', - 'outbound', - 'loop' - ); - ") - .execute(&mut **tx) - .await?; - - sqlx::query(" - CREATE TABLE IF NOT EXISTS septa_routes ( - id VARCHAR(8) PRIMARY KEY, - name VARCHAR(64) NOT NULL, - short_name VARCHAR(32) NOT NULL, - color_hex VARCHAR(6) NOT NULL, - route_type septa_route_type NOT NULL - ); - ") - .execute(&mut **tx) - .await?; - - sqlx::query(" - CREATE TABLE IF NOT EXISTS interlined_routes ( - id VARCHAR(8) PRIMARY KEY, - name VARCHAR(64) NOT NULL - ); - ") - .execute(&mut **tx) - .await?; - - sqlx::query(" - CREATE TABLE IF NOT EXISTS interlined_routes_subroutes ( - interline_id VARCHAR(8), - route_id VARCHAR(8), - FOREIGN KEY (interline_id) REFERENCES interlined_routes(id), - FOREIGN KEY (route_id) REFERENCES septa_routes(id) - ); - ") - .execute(&mut **tx) - .await?; - - Ok(()) - } - -async fn insert_many(routes: Vec, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - let mut names = Vec::new(); - let mut short_names = Vec::new(); - let mut color_hexes = Vec::new(); - let mut route_types: Vec = Vec::new(); - let mut ids = Vec::new(); - - for route in routes { - ids.push(route.id); - names.push(route.name); - short_names.push(route.short_name); - color_hexes.push(route.color_hex); - route_types.push(route.route_type); - } - - sqlx::query(" - INSERT INTO - septa_routes - ( - id, - name, - short_name, - color_hex, - route_type - ) - SELECT * FROM UNNEST( - $1::text[], - $2::text[], - $3::text[], - $4::text[], - $5::septa_route_type[] - ); - ") - .bind(&ids) - .bind(&names) - .bind(&short_names) - .bind(&color_hexes) - .bind(&route_types) - .execute(&mut **tx) - .await?; - - let ir: InterlinedRoute = InterlinedRoute { - interline_id: String::from("B"), - interline_name: String::from("Broad Street Line"), - interlined_routes: vec![String::from("B1"), String::from("B2"), String::from("B3")] - }; - - sqlx::query(" - INSERT INTO - interlined_routes - ( - id, - name - ) - VALUES - ( - $1, - $2 - ); - ") - .bind(&ir.interline_id) - .bind(&ir.interline_name) - .execute(&mut **tx) - .await?; - - for route in ir.interlined_routes { - sqlx::query(" - INSERT INTO - interlined_routes_subroutes - ( - interline_id, - route_id - ) - VALUES - ( - $1, - $2 - ); - ") - .bind(&ir.interline_id) - .bind(&route) - .execute(&mut **tx) - .await?; - } - - Ok(()) -} - async fn insert(&self, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - Self::insert_many(vec![self.clone()], tx).await?; - - Ok(()) - } -} - -impl FromSeptaJson for RouteType { - fn from_septa_json(json_rt: septa_json::route::RouteType) -> ::anyhow::Result> { - Ok(Box::new(match json_rt { - septa_json::route::RouteType::Trolley => RouteType::Trolley, - septa_json::route::RouteType::SubwayElevated => RouteType::SubwayElevated, - septa_json::route::RouteType::RegionalRail => RouteType::RegionalRail, - septa_json::route::RouteType::Bus => RouteType::Bus, - septa_json::route::RouteType::TracklessTrolley => RouteType::TracklessTrolley, - })) - } -} - -impl FromSeptaJson for Route { - fn from_septa_json(json_route: septa_json::route::Route) -> ::anyhow::Result> { - Ok(Box::new(Route { - name: json_route.route_long_name, - short_name: json_route.route_short_name, - color_hex: json_route.route_color, - route_type: *RouteType::from_septa_json(json_route.route_type)?, - id: json_route.route_id, - // FIXME: Actually get direction - })) - } -} diff --git a/data_loader/src/septa/route_stop.rs b/data_loader/src/septa/route_stop.rs deleted file mode 100644 index a626346..0000000 --- a/data_loader/src/septa/route_stop.rs +++ /dev/null @@ -1,78 +0,0 @@ -use sqlx::{Postgres, Transaction}; -use crate::traits::{DbObj, FromSeptaJson}; -use crate::septa_json; - -use libseptastic::route_stop::RouteStop; - - -impl DbObj for RouteStop { - async fn create_table(tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - sqlx::query(" - CREATE TABLE IF NOT EXISTS septa_route_stops ( - route_id TEXT NOT NULL, - stop_id BIGINT NOT NULL, - direction_id BIGINT NOT NULL, - stop_sequence BIGINT NOT NULL, - - PRIMARY KEY (route_id, stop_id, direction_id), - - FOREIGN KEY (route_id) REFERENCES septa_routes(id) ON DELETE CASCADE, - FOREIGN KEY (stop_id) REFERENCES septa_stops(id) ON DELETE CASCADE - ); - ") - .execute(&mut **tx) - .await?; - - Ok(()) - } - async fn insert_many(rses: Vec, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - let mut route_ids: Vec = Vec::new(); - let mut stop_ids: Vec = Vec::new(); - let mut direction_ids: Vec = Vec::new(); - let mut stop_sequences: Vec = Vec::new(); - - for rs in rses { - route_ids.push(rs.route_id.clone()); - stop_ids.push(rs.stop_id.clone()); - direction_ids.push(rs.direction_id.clone()); - stop_sequences.push(rs.stop_sequence.clone()); - } - - sqlx::query(" - INSERT INTO - septa_route_stops - ( - route_id, - stop_id, - direction_id, - stop_sequence - ) - SELECT * FROM UNNEST($1::text[], $2::bigint[], $3::bigint[], $4::bigint[]) - ON CONFLICT DO NOTHING - ; - ") - .bind(&route_ids[..]) - .bind(&stop_ids[..]) - .bind(&direction_ids[..]) - .bind(&stop_sequences[..]) - .execute(&mut **tx) - .await?; - Ok(()) - } - - async fn insert(&self, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - Self::insert_many(vec![self.clone()], tx).await?; - Ok(()) - } -} - -impl FromSeptaJson for RouteStop { - fn from_septa_json(json_stops: septa_json::route_stop::RouteStop) -> ::anyhow::Result> { - Ok(Box::new(RouteStop{ - route_id: json_stops.route_id, - stop_id: json_stops.stop_id, - direction_id: json_stops.direction_id, - stop_sequence: json_stops.stop_sequence - })) - } -} diff --git a/data_loader/src/septa/schedule_day.rs b/data_loader/src/septa/schedule_day.rs deleted file mode 100644 index 6c1b816..0000000 --- a/data_loader/src/septa/schedule_day.rs +++ /dev/null @@ -1,70 +0,0 @@ -use sqlx::{Postgres, Transaction}; -use crate::traits::{DbObj, FromSeptaJson}; -use crate::septa_json; - -use libseptastic::schedule_day::ScheduleDay; - -impl DbObj for ScheduleDay { - async fn create_table(tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - sqlx::query(" - CREATE TABLE IF NOT EXISTS septa_schedule_days ( - date TEXT NOT NULL, - service_id TEXT NOT NULL, - PRIMARY KEY (date, service_id) - ); - ") - .execute(&mut **tx) - .await?; - - Ok(()) - } - async fn insert_many(scheds: Vec, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - let mut dates: Vec = Vec::new(); - let mut service_ids: Vec = Vec::new(); - - for sched in scheds { - dates.push(sched.date); - service_ids.push(sched.service_id); - } - - sqlx::query(" - - INSERT INTO septa_schedule_days ( - date, - service_id - ) - SELECT * FROM UNNEST( - $1::text[], - $2::text[] - ) - ON CONFLICT DO NOTHING - ; - ") - .bind(&dates[..]) - .bind(&service_ids[..]) - .execute(&mut **tx) - .await?; - - Ok(()) - } - - async fn insert(&self, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - Self::insert_many(vec![self.clone()], tx).await?; - Ok(()) - } -} - - -impl FromSeptaJson for Vec { - fn from_septa_json(json_sched: septa_json::schedule_day::Calendar) -> ::anyhow::Result>> { - let mut res: Vec = Vec::new(); - - for (day, sched) in json_sched { - for svc in sched.service_id { - res.push(ScheduleDay { date: day.clone(), service_id: svc }); - } - } - - Ok(Box::new(res)) - } -} diff --git a/data_loader/src/septa/stop.rs b/data_loader/src/septa/stop.rs deleted file mode 100644 index d01f53c..0000000 --- a/data_loader/src/septa/stop.rs +++ /dev/null @@ -1,108 +0,0 @@ -use sqlx::{Postgres, Transaction}; -use crate::traits::{DbObj, FromSeptaJson}; -use crate::septa_json; - -use libseptastic::stop::{Stop, StopType}; - -impl DbObj for Stop { - async fn create_table(tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - sqlx::query(" - CREATE TYPE septa_stop_type AS ENUM ( - 'far_side', - 'middle_block_near_side', - 'normal' - ); - ") - .execute(&mut **tx) - .await?; - - sqlx::query(" - CREATE TABLE IF NOT EXISTS septa_stops ( - id BIGINT PRIMARY KEY, - name VARCHAR(128) NOT NULL, - lat DOUBLE PRECISION NOT NULL, - lng DOUBLE PRECISION NOT NULL, - stop_type septa_stop_type NOT NULL - ); - ") - .execute(&mut **tx) - .await?; - - Ok(()) - } - async fn insert_many(stations: Vec, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - let mut ids: Vec = Vec::new(); - let mut names: Vec = Vec::new(); - let mut lats: Vec = Vec::new(); - let mut lngs: Vec = Vec::new(); - let mut stop_types: Vec = Vec::new(); - - for station in stations { - ids.push(station.id); - names.push(station.name); - lats.push(station.lat); - lngs.push(station.lng); - stop_types.push(station.stop_type); - } - - sqlx::query(" - INSERT INTO - septa_stops - ( - id, - name, - lat, - lng, - stop_type - ) - SELECT * FROM UNNEST( - $1::bigint[], - $2::text[], - $3::double precision[], - $4::double precision[], - $5::septa_stop_type[] - ) - ON CONFLICT DO NOTHING - ; - ") - .bind(&ids[..]) - .bind(&names[..]) - .bind(&lats[..]) - .bind(&lngs[..]) - .bind(&stop_types[..]) - .execute(&mut **tx) - .await?; - - Ok(()) - } - - async fn insert(&self, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - Self::insert_many(vec![self.clone()], tx).await?; - Ok(()) - } -} - -impl FromSeptaJson for Stop { - fn from_septa_json(json_station: septa_json::route_stop::RouteStop) -> ::anyhow::Result> { - let mut name = json_station.stop_name; - let mut stop_type = StopType::Normal; - - if let Some(new_name) = name.strip_suffix("- MNBS") { - stop_type = StopType::MiddleBlockNearSide; - name = new_name.to_string(); - } - - if let Some(new_name) = name.strip_suffix("- FS") { - stop_type = StopType::FarSide; - name = new_name.to_string(); - } - - Ok(Box::new(Stop { - name, - id: json_station.stop_id, - lat: json_station.stop_lat, - lng: json_station.stop_lon, - stop_type - })) - } -} diff --git a/data_loader/src/septa/stop_schedule.rs b/data_loader/src/septa/stop_schedule.rs deleted file mode 100644 index ef7a1e6..0000000 --- a/data_loader/src/septa/stop_schedule.rs +++ /dev/null @@ -1,127 +0,0 @@ -use sqlx::{Postgres, Transaction}; -use crate::traits::{DbObj, FromSeptaJson}; -use crate::septa_json; - -use libseptastic::stop_schedule::StopSchedule; - -impl DbObj for StopSchedule { - async fn create_table(tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - sqlx::query(" - CREATE TABLE IF NOT EXISTS septa_stop_schedules ( - route_id TEXT NOT NULL, - trip_id TEXT NOT NULL, - service_id TEXT NOT NULL, - direction_id BIGINT NOT NULL, - arrival_time BIGINT NOT NULL, - stop_id BIGINT NOT NULL, - stop_sequence BIGINT NOT NULL, - - PRIMARY KEY (trip_id, stop_id), - - FOREIGN KEY (route_id) REFERENCES septa_routes(id) ON DELETE CASCADE, - FOREIGN KEY (stop_id) REFERENCES septa_stops(id) ON DELETE CASCADE - ); - ") - //FOREIGN KEY (route_id, direction_id) REFERENCES septa_directions(route_id, direction_id) ON DELETE CASCADE, - .execute(&mut **tx) - .await?; - - sqlx::query(" - CREATE INDEX septa_stop_schedule_trip_id_idx ON septa_stop_schedules (trip_id); - ").execute(&mut **tx).await?; - - sqlx::query(" - CREATE INDEX septa_stop_schedule_service_id_idx ON septa_stop_schedules (service_id); - ").execute(&mut **tx).await?; - - Ok(()) - } - async fn insert_many(scheds: Vec, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - let mut route_ids: Vec = Vec::new(); - let mut trip_ids: Vec = Vec::new(); - let mut service_ids: Vec = Vec::new(); - let mut direction_ids: Vec = Vec::new(); - let mut arrival_times: Vec = Vec::new(); - let mut stop_ids: Vec = Vec::new(); - let mut stop_sequences: Vec = Vec::new(); - - for sched in scheds { - route_ids.push(sched.route_id); - trip_ids.push(sched.trip_id); - service_ids.push(sched.service_id); - direction_ids.push(sched.direction_id); - arrival_times.push(sched.arrival_time); - stop_ids.push(sched.stop_id); - stop_sequences.push(sched.stop_sequence); - } - - sqlx::query(" - - INSERT INTO septa_stop_schedules ( - route_id, - trip_id, - service_id, - direction_id, - arrival_time, - stop_id, - stop_sequence - ) - SELECT * FROM UNNEST( - $1::text[], - $2::text[], - $3::text[], - $4::bigint[], - $5::bigint[], - $6::bigint[], - $7::bigint[] - ) - ON CONFLICT DO NOTHING - ; - ") - .bind(&route_ids[..]) - .bind(&trip_ids[..]) - .bind(&service_ids[..]) - .bind(&direction_ids[..]) - .bind(&arrival_times[..]) - .bind(&stop_ids[..]) - .bind(&stop_sequences[..]) - .execute(&mut **tx) - .await?; - - Ok(()) - } - - async fn insert(&self, tx: &mut Transaction<'_, Postgres>) -> anyhow::Result<()> { - Self::insert_many(vec![self.clone()], tx).await?; - Ok(()) - } -} - -impl FromSeptaJson for StopSchedule { - fn from_septa_json(json_sched: septa_json::stop_schedule::StopSchedule) -> ::anyhow::Result> { - let time_parts: Vec<&str> = json_sched.arrival_time.split(":").collect(); - - let arrival_time: i64 = { - let hour: i64 = time_parts[0].parse::()?; - let minute: i64 = time_parts[1].parse::()?; - let second: i64 = time_parts[2].parse::()?; - - (hour*3600) + (minute * 60) + second - }; - - Ok(Box::new(StopSchedule { - route_id: json_sched.route_id, - trip_id: match json_sched.trip_id{ - septa_json::stop_schedule::TripId::RegionalRail(x) => x, - septa_json::stop_schedule::TripId::Other(y) => y.to_string() - }, - stop_name: String::from(""), - service_id: json_sched.service_id, - // FIXME: Actually get direction - direction_id: json_sched.direction_id, - arrival_time, - stop_id: json_sched.stop_id, - stop_sequence: json_sched.stop_sequence - })) - } -} diff --git a/data_loader/src/septa_json/direction.rs b/data_loader/src/septa_json/direction.rs deleted file mode 100644 index ce15bc6..0000000 --- a/data_loader/src/septa_json/direction.rs +++ /dev/null @@ -1,19 +0,0 @@ -use crate::traits::*; -use serde::Deserialize; - - -#[derive(Debug, Clone, Deserialize)] -pub struct Direction { - #[serde(rename="Route")] - pub route: String, - #[serde(rename="DirectionDescription")] - pub direction_destination: String, - #[serde(rename="Direction")] - pub direction: String, - #[serde(rename="TrueDirection")] - pub true_direction: String, - #[serde(rename="Mode")] - pub mode: String -} - -impl SeptaJson for Direction {} diff --git a/data_loader/src/septa_json/mod.rs b/data_loader/src/septa_json/mod.rs deleted file mode 100644 index d28e3e1..0000000 --- a/data_loader/src/septa_json/mod.rs +++ /dev/null @@ -1,6 +0,0 @@ -pub mod route; -pub mod route_stop; -pub mod stop_schedule; -pub mod schedule_day; -pub mod direction; -pub mod ridership; diff --git a/data_loader/src/septa_json/ridership.rs b/data_loader/src/septa_json/ridership.rs deleted file mode 100644 index bc9dd33..0000000 --- a/data_loader/src/septa_json/ridership.rs +++ /dev/null @@ -1,37 +0,0 @@ -use crate::traits::*; -use serde::Deserialize; - -#[derive(Debug, Clone, Deserialize)] -pub struct Ridership { - #[serde(rename = "Stop ID")] - pub stop_id: Option, - - #[serde(rename = "Stop Code")] - pub stop_code: String, - - #[serde(rename = "Stop")] - pub stop_name: String, - - #[serde(rename = "Route")] - pub route_id: String, - - #[serde(rename = "Direction")] - pub direction: String, - - #[serde(rename = "Ons")] - pub ons: String, - - #[serde(rename = "Offs")] - pub offs: String, - - #[serde(rename = "Exp_Ons")] - pub exp_ons: String, - - #[serde(rename = "Exp_Offs")] - pub exp_offs: String, - - #[serde(rename = "Year")] - pub year: String -} - -impl SeptaJson for Ridership {} diff --git a/data_loader/src/septa_json/route.rs b/data_loader/src/septa_json/route.rs deleted file mode 100644 index 93e6967..0000000 --- a/data_loader/src/septa_json/route.rs +++ /dev/null @@ -1,42 +0,0 @@ -use crate::traits::*; -use serde_repr::*; -use serde::Deserialize; - -#[derive(Serialize_repr, Deserialize_repr, PartialEq, Debug)] -#[repr(u8)] -pub enum RouteType { - Trolley= 0, - SubwayElevated = 1, - RegionalRail = 2, - Bus = 3, - TracklessTrolley = 11 -} - - -#[derive(Debug, Deserialize)] -pub struct Document { - #[serde(rename = "type")] - pub doc_type: String, - pub title: String, - pub url: String, - pub link_label: Option, -} - -#[derive(Debug, Deserialize)] -pub struct Route { - pub release_name: String, - pub route_id: String, - pub route_short_name: String, - pub route_long_name: String, - pub route_type: RouteType, - pub route_color: String, - pub route_text_color: String, - pub route_frequency_text: String, - pub is_frequent_bus: bool, - pub route_color_dark: String, - pub route_color_text_dark: String, - pub documents: Vec -} - -impl SeptaJson for Route {} -impl SeptaJson for RouteType {} diff --git a/data_loader/src/septa_json/route_stop.rs b/data_loader/src/septa_json/route_stop.rs deleted file mode 100644 index 2eca64d..0000000 --- a/data_loader/src/septa_json/route_stop.rs +++ /dev/null @@ -1,31 +0,0 @@ -use crate::traits::*; -use serde::{self, Deserialize, Deserializer}; - -pub fn de_string_or_int<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - use serde::de::Error; - use serde_json::Value; - - match Value::deserialize(deserializer)? { - Value::Number(n) => n.as_i64().ok_or_else(|| Error::custom("Invalid number")), - Value::String(s) => s.parse::().map_err(|_| Error::custom("Invalid string number")), - _ => Err(Error::custom("Expected a string or number")), - } -} - -#[derive(Debug, Clone, Deserialize)] -pub struct RouteStop { - pub route_id: String, - pub direction_id: i64, - #[serde(deserialize_with = "de_string_or_int")] - pub stop_id: i64, - pub stop_name: String, - pub stop_sequence: i64, - pub stop_lat: f64, - pub stop_lon: f64, -} - -impl SeptaJson for RouteStop {} -impl SeptaJson for Vec {} diff --git a/data_loader/src/septa_json/schedule_day.rs b/data_loader/src/septa_json/schedule_day.rs deleted file mode 100644 index ad9886a..0000000 --- a/data_loader/src/septa_json/schedule_day.rs +++ /dev/null @@ -1,18 +0,0 @@ -use std::collections::{BTreeMap, HashMap, HashSet}; - -use crate::traits::*; -use serde::{Deserialize, Serialize}; - - -#[derive(Default, Debug, Serialize, Deserialize, Clone)] -pub struct ScheduleDay { - pub release_name: String, - pub special: bool, - pub service_id: Vec, - pub service_added: Vec, - pub service_removed: Vec -} - -pub type Calendar = BTreeMap; - -impl SeptaJson for Calendar {} diff --git a/data_loader/src/septa_json/stop_schedule.rs b/data_loader/src/septa_json/stop_schedule.rs deleted file mode 100644 index 65a98bf..0000000 --- a/data_loader/src/septa_json/stop_schedule.rs +++ /dev/null @@ -1,27 +0,0 @@ -use crate::traits::*; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(untagged)] -pub enum TripId { - RegionalRail(String), - Other(i64) -} - -#[derive(Debug, Clone, Deserialize)] -pub struct StopSchedule { - pub route_id: String, - pub block_id: i64, - pub release_name: String, - pub trip_id: TripId, - pub service_id: String, - pub trip_headsign: Option, - pub direction_id: i64, - pub arrival_time: String, - pub stop_id: i64, - pub stop_name: String, - pub stop_sequence: i64, - pub last_stop_id: i64 -} - -impl SeptaJson for StopSchedule {} diff --git a/data_loader/src/traits.rs b/data_loader/src/traits.rs deleted file mode 100644 index 20eb2cc..0000000 --- a/data_loader/src/traits.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::collections::HashMap; - -use sqlx::{Postgres, Transaction}; - - - -pub trait SeptaJson {} - -pub trait FromSeptaJson { - fn from_septa_json(septa_json: T) -> ::anyhow::Result>; -} - -pub trait FromSeptaJsonAndStations { - fn from_septa_json(septa_json: T, stop_map: &HashMap) -> ::anyhow::Result>; -} - -pub trait DbObj { - fn create_table(tx: &mut Transaction<'_, Postgres>) -> impl std::future::Future< Output = ::anyhow::Result<()>> + Send; - fn insert(&self, tx: &mut Transaction<'_, Postgres>) -> impl std::future::Future< Output = ::anyhow::Result<()>> + Send; - fn insert_many(s: Vec, tx: &mut Transaction<'_, Postgres>) -> impl std::future::Future< Output = ::anyhow::Result<()>> + Send; -} - -pub trait Fetchable {} From f4079920359cb7e64ea19fe9c135bd50f538a2fa Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Wed, 8 Oct 2025 07:27:15 -0400 Subject: [PATCH 2/4] mostly works for fetching gtfs zips --- data_loader/Cargo.lock | 668 ++++++++++++++++++++++++++++++++++++++-- data_loader/Cargo.toml | 2 + data_loader/config.yaml | 9 +- data_loader/src/main.rs | 42 ++- 4 files changed, 679 insertions(+), 42 deletions(-) diff --git a/data_loader/Cargo.lock b/data_loader/Cargo.lock index 62f9412..922ab58 100644 --- a/data_loader/Cargo.lock +++ b/data_loader/Cargo.lock @@ -17,6 +17,17 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -32,6 +43,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.19" @@ -88,6 +108,15 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "atoi" version = "2.0.0" @@ -160,6 +189,12 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + [[package]] name = "byteorder" version = "1.5.0" @@ -172,12 +207,42 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cc" version = "1.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -187,6 +252,29 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link 0.2.1", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -208,6 +296,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "core-foundation" version = "0.9.4" @@ -235,9 +329,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.2.1" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -248,6 +342,15 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -273,6 +376,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +dependencies = [ + "memchr", +] + [[package]] name = "data_loader" version = "0.1.0" @@ -280,6 +404,7 @@ dependencies = [ "anyhow", "dotenv", "env_logger", + "gtfs-structures", "libseptastic", "log", "reqwest", @@ -289,10 +414,17 @@ dependencies = [ "serde_repr", "serde_yaml", "sqlx", - "thiserror", + "thiserror 2.0.12", "tokio", + "zip 5.1.1", ] +[[package]] +name = "deflate64" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" + [[package]] name = "der" version = "0.7.10" @@ -304,6 +436,37 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "digest" version = "0.10.7" @@ -324,7 +487,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", ] [[package]] @@ -424,6 +587,17 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "flate2" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5a4e564e38c699f2880d3fda590bedc2e69f3f84cd48b457bd892ce61d0aa9" +dependencies = [ + "crc32fast", + "libz-rs-sys", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.1" @@ -471,6 +645,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -515,6 +704,17 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "futures-sink" version = "0.3.31" @@ -533,8 +733,10 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -571,9 +773,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -582,6 +786,27 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "gtfs-structures" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3211c82a20a529763951e9072e4d85c06b6c8d4efa75d08859ccb58e702334e3" +dependencies = [ + "bytes", + "chrono", + "csv", + "derivative", + "futures", + "itertools", + "reqwest", + "rgb", + "serde", + "serde_derive", + "sha2", + "thiserror 1.0.69", + "zip 2.4.2", +] + [[package]] name = "h2" version = "0.4.11" @@ -778,6 +1003,30 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "iana-time-zone" +version = "0.1.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -893,7 +1142,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", ] [[package]] @@ -927,6 +1176,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -949,6 +1207,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -976,7 +1243,17 @@ checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.3", + "libc", ] [[package]] @@ -998,6 +1275,12 @@ dependencies = [ "spin", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + [[package]] name = "libc" version = "0.2.172" @@ -1029,6 +1312,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -1057,6 +1349,37 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "lzma-rust2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a" +dependencies = [ + "crc", + "sha2", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "md-5" version = "0.10.6" @@ -1086,6 +1409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -1133,6 +1457,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -1207,7 +1537,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", ] [[package]] @@ -1257,6 +1587,16 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -1326,6 +1666,18 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppmd-rust" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1469,6 +1821,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rgb" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.17.14" @@ -1631,7 +1992,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", ] [[package]] @@ -1654,7 +2015,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", ] [[package]] @@ -1695,9 +2056,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.8" +version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures", @@ -1729,6 +2090,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "slab" version = "0.4.9" @@ -1816,7 +2183,7 @@ dependencies = [ "serde_json", "sha2", "smallvec", - "thiserror", + "thiserror 2.0.12", "tokio", "tokio-stream", "tracing", @@ -1833,7 +2200,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.101", ] [[package]] @@ -1856,7 +2223,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn", + "syn 2.0.101", "tokio", "url", ] @@ -1898,7 +2265,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 2.0.12", "tracing", "whoami", ] @@ -1935,7 +2302,7 @@ dependencies = [ "smallvec", "sqlx-core", "stringprep", - "thiserror", + "thiserror 2.0.12", "tracing", "whoami", ] @@ -1959,7 +2326,7 @@ dependencies = [ "serde", "serde_urlencoded", "sqlx-core", - "thiserror", + "thiserror 2.0.12", "tracing", "url", ] @@ -1987,6 +2354,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.101" @@ -2015,7 +2393,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", ] [[package]] @@ -2052,13 +2430,33 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.12", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", ] [[package]] @@ -2069,9 +2467,28 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", ] +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + [[package]] name = "tinystr" version = "0.7.6" @@ -2123,7 +2540,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", ] [[package]] @@ -2235,7 +2652,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", ] [[package]] @@ -2391,7 +2808,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.101", "wasm-bindgen-shared", ] @@ -2426,7 +2843,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2460,21 +2877,62 @@ dependencies = [ "wasite", ] +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link 0.2.1", + "windows-result 0.4.1", + "windows-strings 0.5.1", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] + [[package]] name = "windows-link" version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + [[package]] name = "windows-registry" version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b8a9ed28765efc97bbc954883f4e6796c33a06546ebafacbabee9696967499e" dependencies = [ - "windows-link", - "windows-result", - "windows-strings", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", ] [[package]] @@ -2483,7 +2941,16 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ - "windows-link", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link 0.2.1", ] [[package]] @@ -2492,7 +2959,16 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" dependencies = [ - "windows-link", + "windows-link 0.1.3", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link 0.2.1", ] [[package]] @@ -2664,6 +3140,15 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yoke" version = "0.7.5" @@ -2684,7 +3169,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", "synstructure", ] @@ -2705,7 +3190,7 @@ checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", ] [[package]] @@ -2725,7 +3210,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", "synstructure", ] @@ -2734,6 +3219,20 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", +] [[package]] name = "zerovec" @@ -2754,5 +3253,108 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.101", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.5.2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom 0.3.3", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror 2.0.12", + "time", + "xz2", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zip" +version = "5.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f852905151ac8d4d06fdca66520a661c09730a74c6d4e2b0f27b436b382e532" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.6.0", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "getrandom 0.3.3", + "hmac", + "indexmap", + "lzma-rust2", + "memchr", + "pbkdf2", + "ppmd-rust", + "sha1", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zlib-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" + +[[package]] +name = "zopfli" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", ] diff --git a/data_loader/Cargo.toml b/data_loader/Cargo.toml index 4b39f8c..5c49f46 100644 --- a/data_loader/Cargo.toml +++ b/data_loader/Cargo.toml @@ -18,3 +18,5 @@ env_logger = "0.11.8" log = "0.4.27" reqwest = { version = "0.12.22", features = [ "json", "blocking" ] } serde_yaml = "0.9.34" +gtfs-structures = "0.45.1" +zip = "5.1.1" diff --git a/data_loader/config.yaml b/data_loader/config.yaml index 9b680f7..2fea038 100644 --- a/data_loader/config.yaml +++ b/data_loader/config.yaml @@ -1,4 +1,7 @@ gtfs_zips: - - "https://www3.septa.org/developer/gtfs_public.zip" - - "https://www.njtransit.com/rail_data.zip" - - "https://www.njtransit.com/bus_data.zip" + - uri: "https://www3.septa.org/developer/gtfs_public.zip" + subzip: "google_rail.zip" + - uri: "https://www3.septa.org/developer/gtfs_public.zip" + subzip: "google_bus.zip" + - uri: "https://www.njtransit.com/rail_data.zip" + - uri: "https://www.njtransit.com/bus_data.zip" diff --git a/data_loader/src/main.rs b/data_loader/src/main.rs index 8c684ab..2316def 100644 --- a/data_loader/src/main.rs +++ b/data_loader/src/main.rs @@ -1,10 +1,11 @@ -use std::{clone, env, fs::File, io::{Read, Write}, path::{Path, PathBuf}, sync::{Arc, Mutex}, thread, time::Duration}; +use std::{clone, env, fs::{self, File}, io::{Cursor, Read, Write}, path::{Path, PathBuf}, sync::{Arc, Mutex}, thread, time::Duration}; use dotenv::dotenv; use serde::{Deserialize, Serialize}; use sqlx::postgres::PgPoolOptions; use env_logger::{Builder, Env}; use log::{error, info, warn}; +use zip::ZipArchive; #[tokio::main] @@ -18,7 +19,14 @@ async fn main() -> ::anyhow::Result<()> { let mut file_contents = String::new(); file.read_to_string(&mut file_contents); - let config_file = serde_yaml::from_str::(file_contents.as_str()); + let config_file = serde_yaml::from_str::(file_contents.as_str())?; + + let svc = GtfsPullService::new(config_file); + svc.start(); + + loop{ + thread::sleep(Duration::from_secs(120)); + } let database_url = std::env::var("DATABASE_URL").expect("Database URL"); let pool = PgPoolOptions::new() @@ -34,13 +42,19 @@ async fn main() -> ::anyhow::Result<()> { Ok(()) } +#[derive(Serialize, Deserialize, PartialEq, Debug,Clone)] +struct GtfsSource { + pub uri: String, + pub subzip: Option +} + #[derive(Serialize, Deserialize, PartialEq, Debug)] struct Config { - gtfs_zips: Vec + pub gtfs_zips: Vec } struct GtfsFile { - pub url: String, + pub source: GtfsSource, pub hash: Option } @@ -60,7 +74,7 @@ impl GtfsPullService { Self { state: Arc::new(Mutex::new( GtfsPullServiceState { - gtfs_files: config.gtfs_zips.iter().map(|f| { GtfsFile { url: f.clone(), hash: None} }).collect(), + gtfs_files: config.gtfs_zips.iter().map(|f| { GtfsFile { source: f.clone(), hash: None} }).collect(), tmp_dir: env::temp_dir() } )) @@ -90,9 +104,25 @@ impl GtfsPullService { let l_state = state.lock().unwrap(); for gtfs_file in l_state.gtfs_files.iter() { - let resp = reqwest::blocking::get(gtfs_file.url.clone())?; + let gtfs = if let Some(subzip) = gtfs_file.source.subzip.clone() { + info!("Reading GTFS file at {} (subzip {})", gtfs_file.source.uri, subzip); + let res = reqwest::blocking::get(gtfs_file.source.uri.clone())?; + let outer_archive = res.bytes()?; + let mut archive = ZipArchive::new(Cursor::new(outer_archive))?; + archive.extract(l_state.tmp_dir.clone())?; + + let mut file_path = l_state.tmp_dir.clone(); + file_path.push(subzip.clone()); + + gtfs_structures::Gtfs::new(file_path.to_str().unwrap())? + } else { + info!("Reading GTFS file at {}", gtfs_file.source.uri); + gtfs_structures::Gtfs::new(gtfs_file.source.uri.as_str())? + }; } + + Ok(()) } } From a335f14b14b53b4519b3394468c3c8130fb86110 Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Sat, 8 Nov 2025 13:12:07 -0500 Subject: [PATCH 3/4] revert to septa api for rt tracking --- api/Cargo.lock | 489 ++++++++++++++++++++++++++++-- api/Cargo.toml | 5 + api/src/controllers/route.rs | 25 +- api/src/database.rs | 179 ----------- api/src/main.rs | 26 +- api/src/services/mod.rs | 2 + api/src/services/trip_tracking.rs | 5 +- api/src/templates.rs | 10 +- api/templates/route.html | 4 +- api/templates/route_symbol.html | 8 +- api/templates/routes.html | 8 +- data_loader/config.yaml | 7 +- data_loader/src/main.rs | 93 ------ libseptastic/src/direction.rs | 4 +- libseptastic/src/lib.rs | 1 + libseptastic/src/stop.rs | 6 +- libseptastic/src/stop_schedule.rs | 17 +- shell.nix | 1 + 18 files changed, 535 insertions(+), 355 deletions(-) delete mode 100644 api/src/database.rs diff --git a/api/Cargo.lock b/api/Cargo.lock index 7428f95..0ae836b 100644 --- a/api/Cargo.lock +++ b/api/Cargo.lock @@ -103,7 +103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" dependencies = [ "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -237,7 +237,7 @@ dependencies = [ "actix-router", "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -391,6 +391,15 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "askama" version = "0.14.0" @@ -418,7 +427,7 @@ dependencies = [ "rustc-hash", "serde", "serde_derive", - "syn", + "syn 2.0.104", ] [[package]] @@ -717,6 +726,12 @@ version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +[[package]] +name = "bytemuck" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" + [[package]] name = "byteorder" version = "1.5.0" @@ -738,6 +753,34 @@ dependencies = [ "bytes", ] +[[package]] +name = "bzip2" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" +dependencies = [ + "bzip2-sys", +] + +[[package]] +name = "bzip2" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea8dcd42434048e4f7a304411d9273a411f647446c1234a65ce0554923f4cff" +dependencies = [ + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "camino" version = "1.1.12" @@ -862,7 +905,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -905,6 +948,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + [[package]] name = "convert_case" version = "0.4.0" @@ -1004,6 +1053,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -1013,6 +1083,12 @@ dependencies = [ "cipher", ] +[[package]] +name = "deflate64" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" + [[package]] name = "der" version = "0.7.10" @@ -1033,6 +1109,28 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derivative" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "derive_more" version = "0.99.20" @@ -1043,7 +1141,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.104", ] [[package]] @@ -1063,7 +1161,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "unicode-xid", ] @@ -1098,7 +1196,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1241,6 +1339,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.2" @@ -1248,6 +1352,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", + "libz-rs-sys", "miniz_oxide", ] @@ -1393,7 +1498,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -1454,9 +1559,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -1493,6 +1600,39 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gtfs-realtime" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421e7aa1a3a540a6f2e046cb349b184be285e16be332aff043e188b2b563c2ff" +dependencies = [ + "prost", + "prost-build", + "prost-derive", + "serde", +] + +[[package]] +name = "gtfs-structures" +version = "0.45.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3211c82a20a529763951e9072e4d85c06b6c8d4efa75d08859ccb58e702334e3" +dependencies = [ + "bytes", + "chrono", + "csv", + "derivative", + "futures", + "itertools", + "reqwest", + "rgb", + "serde", + "serde_derive", + "sha2", + "thiserror 1.0.69", + "zip 2.4.2", +] + [[package]] name = "h2" version = "0.3.26" @@ -1952,6 +2092,15 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1979,7 +2128,7 @@ checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -2026,6 +2175,12 @@ dependencies = [ "spin", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + [[package]] name = "libc" version = "0.2.174" @@ -2069,6 +2224,15 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libz-rs-sys" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" +dependencies = [ + "zlib-rs", +] + [[package]] name = "linux-raw-sys" version = "0.3.8" @@ -2129,6 +2293,37 @@ dependencies = [ "value-bag", ] +[[package]] +name = "lzma-rs" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "297e814c836ae64db86b36cf2a557ba54368d03f6afcd7d947c266692f71115e" +dependencies = [ + "byteorder", + "crc", +] + +[[package]] +name = "lzma-rust2" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a" +dependencies = [ + "crc", + "sha2", +] + +[[package]] +name = "lzma-sys" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "md-5" version = "0.10.6" @@ -2182,6 +2377,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "native-tls" version = "0.2.14" @@ -2302,7 +2503,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -2352,6 +2553,16 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -2367,6 +2578,16 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "phf" version = "0.12.1" @@ -2508,6 +2729,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppmd-rust" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c834641d8ad1b348c9ee86dec3b9840d805acd5f24daa5f90c788951a52ff59b" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -2517,6 +2744,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff24dfcda44452b9816fff4cd4227e1bb73ff5a2f1bc1105aa92fb8565ce44d2" +dependencies = [ + "proc-macro2", + "syn 2.0.104", +] + [[package]] name = "proc-macro2" version = "1.0.95" @@ -2526,6 +2763,58 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6c3320f9abac597dcbc668774ef006702672474aad53c6d596b62e487b40b1" +dependencies = [ + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn 2.0.104", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.104", +] + +[[package]] +name = "prost-types" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b4db3d6da204ed77bb26ba83b6122a73aeb2e87e25fbf7ad2e84c4ccbf8f72" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.40" @@ -2686,6 +2975,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rgb" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6a884d2998352bb4daf0183589aec883f16a6da1f4dde84d8e2e9a5409a1ce" +dependencies = [ + "bytemuck", +] + [[package]] name = "ring" version = "0.17.14" @@ -2888,13 +3186,18 @@ dependencies = [ "dotenv", "env_logger", "futures-util", + "gtfs-realtime", + "gtfs-structures", "libseptastic", "log", + "prost", "reqwest", "serde", "serde_json", + "serde_yaml", "sqlx", "sqlx-cli", + "zip 5.1.1", ] [[package]] @@ -2914,7 +3217,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -2941,6 +3244,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1" version = "0.10.6" @@ -2994,6 +3310,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "siphasher" version = "1.0.1" @@ -3137,7 +3459,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn", + "syn 2.0.104", ] [[package]] @@ -3161,7 +3483,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn", + "syn 2.0.104", "tokio", "url", ] @@ -3298,6 +3620,17 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.104" @@ -3326,7 +3659,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3399,7 +3732,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3410,7 +3743,7 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3497,7 +3830,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3609,7 +3942,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3688,6 +4021,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -3799,7 +4138,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn", + "syn 2.0.104", "wasm-bindgen-shared", ] @@ -3834,7 +4173,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3911,7 +4250,7 @@ checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -3922,7 +4261,7 @@ checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -4206,6 +4545,15 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "xz2" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yoke" version = "0.8.0" @@ -4226,7 +4574,7 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "synstructure", ] @@ -4247,7 +4595,7 @@ checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", ] [[package]] @@ -4267,7 +4615,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", "synstructure", ] @@ -4276,6 +4624,20 @@ name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] [[package]] name = "zerotrie" @@ -4307,7 +4669,82 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.104", +] + +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.5.2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "deflate64", + "displaydoc", + "flate2", + "getrandom 0.3.3", + "hmac", + "indexmap", + "lzma-rs", + "memchr", + "pbkdf2", + "sha1", + "thiserror 2.0.12", + "time", + "xz2", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zip" +version = "5.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f852905151ac8d4d06fdca66520a661c09730a74c6d4e2b0f27b436b382e532" +dependencies = [ + "aes", + "arbitrary", + "bzip2 0.6.0", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "getrandom 0.3.3", + "hmac", + "indexmap", + "lzma-rust2", + "memchr", + "pbkdf2", + "ppmd-rust", + "sha1", + "time", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zlib-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" + +[[package]] +name = "zopfli" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfc5ee405f504cd4984ecc6f14d02d55cfda60fa4b689434ef4102aae150cd7" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", ] [[package]] diff --git a/api/Cargo.toml b/api/Cargo.toml index 9094583..8c3d8a5 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -22,3 +22,8 @@ reqwest = { version = "0.12.22", features = [ "json", "blocking" ] } sqlx-cli = "0.8.6" futures-util = "0.3.31" actix-session = { version = "0.11.0", features = ["cookie-session"] } +serde_yaml = "0.9.34" +gtfs-structures = "0.45.1" +zip = "5.1.1" +gtfs-realtime = "0.2.0" +prost = "0.14.1" diff --git a/api/src/controllers/route.rs b/api/src/controllers/route.rs index 2bd6ae8..09e85cd 100644 --- a/api/src/controllers/route.rs +++ b/api/src/controllers/route.rs @@ -1,12 +1,10 @@ use actix_web::{get, web::{self, Data}, HttpRequest, HttpResponse, Responder}; use anyhow::anyhow; -use std::{time::Instant, sync::Arc}; -use libseptastic::{route::RouteType, stop_schedule::Trip}; +use std::{collections::HashSet, sync::Arc, time::Instant}; +use libseptastic::{direction, route::RouteType, stop_schedule::Trip}; use serde::{Serialize, Deserialize}; -use askama::Template; use crate::AppState; -use crate::database; #[get("/routes")] async fn get_routes_html(req: HttpRequest, state: Data>) -> impl Responder { @@ -15,8 +13,7 @@ async fn get_routes_html(req: HttpRequest, state: Data>) -> impl R async move { let start_time = Instant::now(); - let all_routes = database::get_all_routes(&mut statex.database.begin().await?).await?; - + let all_routes: Vec = statex.gtfs_service.get_routes(); let rr_routes = all_routes.clone().into_iter().filter(|x| x.route_type == RouteType::RegionalRail).collect(); let subway_routes = all_routes.clone().into_iter().filter(|x| x.route_type == RouteType::SubwayElevated).collect(); let trolley_routes = all_routes.clone().into_iter().filter(|x| x.route_type == RouteType::Trolley).collect(); @@ -40,7 +37,7 @@ async fn get_routes_html(req: HttpRequest, state: Data>) -> impl R #[get("/routes.json")] async fn get_routes_json(state: Data>) -> impl Responder { - let all_routes = database::get_all_routes(&mut state.database.begin().await.unwrap()).await.unwrap(); + let all_routes: Vec = state.gtfs_service.get_routes(); HttpResponse::Ok().json(all_routes) } @@ -58,11 +55,15 @@ pub struct RouteResponse { } async fn get_route_info(route_id: String, state: Data>) -> ::anyhow::Result { - let mut tx = state.database.begin().await?; + let route = state.gtfs_service.get_route(route_id.clone())?; + let mut trips = state.gtfs_service.get_schedule(route_id)?; - let route = database::get_route_by_id(route_id.clone(), &mut tx).await?; - let directions = database::get_direction_by_route_id(route_id.clone(), &mut tx).await?; - let mut trips = database::get_schedule_by_route_id(route_id.clone(), &mut tx).await?; + let mut seen = HashSet::new(); + let directions: Vec<_> = trips + .iter() + .map(|x| x.direction.clone()) + .filter(|dir| seen.insert(dir.direction.to_string())) + .collect(); state.trip_tracking_service.annotate_trips(&mut trips); @@ -127,7 +128,7 @@ async fn api_get_route(state: Data>, path: web::Path) -> i #[get("/api/route/{route_id}/schedule")] async fn api_get_schedule(state: Data>, path: web::Path) -> impl Responder { let route_id = path.into_inner(); - let route_r = database::get_schedule_by_route_id(route_id, &mut state.database.begin().await.unwrap()).await; + let route_r: anyhow::Result = Ok(5); if let Ok(route) = route_r { HttpResponse::Ok().json(route) diff --git a/api/src/database.rs b/api/src/database.rs deleted file mode 100644 index a9792af..0000000 --- a/api/src/database.rs +++ /dev/null @@ -1,179 +0,0 @@ -use std::collections::HashMap; -use sqlx::{Postgres, Transaction}; -use libseptastic::{stop_schedule::{Trip, TripTracking, StopSchedule}}; - -pub async fn get_route_by_id( - id: String, - transaction: &mut Transaction<'_, Postgres>, -) -> ::anyhow::Result { - - let row = sqlx::query!( - r#"SELECT - id, - name, - short_name, - color_hex, - route_type as "route_type: libseptastic::route::RouteType" - FROM - septa_routes - WHERE - id = $1 - ;"#, - id - ) - .fetch_one(&mut **transaction) - .await?; - - return Ok(libseptastic::route::Route { - name: row.name, - short_name: row.short_name, - color_hex: row.color_hex, - route_type: row.route_type, - id: row.id, - }); -} - -pub async fn get_all_routes( - transaction: &mut Transaction<'_, Postgres>, -) -> ::anyhow::Result> { - - let rows = sqlx::query!( - r#"SELECT - id, - name, - short_name, - color_hex, - route_type as "route_type: libseptastic::route::RouteType" - FROM - septa_routes - ORDER BY - CASE - WHEN id ~ '^[0-9]+$' THEN CAST(id AS INT) - ELSE NULL - END ASC, - id ASC; - ;"# - ) - .fetch_all(&mut **transaction) - .await?; - - let mut routes = Vec::new(); - - for row in rows { - routes.push(libseptastic::route::Route { - name: row.name, - short_name: row.short_name, - color_hex: row.color_hex, - route_type: row.route_type, - id: row.id, - }); - } - - return Ok(routes); -} - -pub async fn get_direction_by_route_id( - id: String, - transaction: &mut Transaction<'_, Postgres>, -) -> ::anyhow::Result> { - - let rows = sqlx::query!( - r#"SELECT - route_id, - direction_id, - direction as "direction: libseptastic::direction::CardinalDirection", - direction_destination - FROM - septa_directions - WHERE - route_id = $1 - ;"#, - id - ) - .fetch_all(&mut **transaction) - .await?; - - let mut res = Vec::new(); - - for row in rows { - res.push(libseptastic::direction::Direction{ - route_id: row.route_id, - direction_id: row.direction_id, - direction: row.direction, - direction_destination: row.direction_destination - }); - } - - return Ok(res); -} - -pub async fn get_schedule_by_route_id( - id: String, - transaction: &mut Transaction<'_, Postgres>, -) -> ::anyhow::Result> { - - let schedule_day = chrono::Utc::now().with_timezone(&chrono_tz::America::New_York); - let schedule_day_str = schedule_day.format("%Y%m%d").to_string(); - - let rows = sqlx::query!( - r#"SELECT - septa_stop_schedules.route_id, - septa_stops.name as stop_name, - trip_id, - septa_stop_schedules.service_id, - septa_stop_schedules.direction_id, - arrival_time, - stop_id, - stop_sequence - FROM - septa_stop_schedules - INNER JOIN septa_stops - ON septa_stops.id = septa_stop_schedules.stop_id - INNER JOIN septa_schedule_days - ON septa_schedule_days.date = $2 - AND - septa_schedule_days.service_id = septa_stop_schedules.service_id - WHERE - septa_stop_schedules.route_id = $1 OR septa_stop_schedules.route_id = 'B2' - ;"#, - id.clone(), - schedule_day_str.clone() - ) - .fetch_all(&mut **transaction) - .await?; - - let mut sched_groups: HashMap> = HashMap::new(); - for row in rows { - let arr = match sched_groups.get_mut(&row.trip_id) { - Some(x) => x, - None => { - sched_groups.insert(row.trip_id.clone(), Vec::new()); - sched_groups.get_mut(&row.trip_id).unwrap() - } - }; - - arr.push(StopSchedule { - route_id: row.route_id, - stop_name: row.stop_name, - trip_id: row.trip_id, - service_id: row.service_id, - direction_id: row.direction_id, - arrival_time: row.arrival_time, - stop_id: row.stop_id, - stop_sequence: row.stop_sequence - }); - } - let mut res = Vec::new(); - - for group in sched_groups { - res.push(Trip{ - trip_id: group.0, - route_id: group.1[0].route_id.clone(), - schedule: group.1.clone(), - direction_id: group.1[0].direction_id.clone(), - tracking_data: TripTracking::Untracked - }); - } - - return Ok(res); -} diff --git a/api/src/main.rs b/api/src/main.rs index 6c38392..14a32e9 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -3,18 +3,17 @@ use env_logger::Env; use log::*; use dotenv::dotenv; use serde::Deserialize; -use services::trip_tracking::{self}; +use services::{gtfs_pull, trip_tracking::{self}}; use templates::ContentTemplate; -use std::{sync::Arc, time::Instant}; +use std::{fs::File, io::Read, sync::Arc, time::Instant}; use askama::Template; -mod database; mod services; mod controllers; mod templates; pub struct AppState { - database: ::sqlx::postgres::PgPool, + gtfs_service: services::gtfs_pull::GtfsPullService, trip_tracking_service: services::trip_tracking::TripTrackingService } @@ -79,19 +78,22 @@ async fn main() -> ::anyhow::Result<()> { let version: &str = option_env!("CARGO_PKG_VERSION").expect("Expected package version"); info!("Starting the SEPTASTIC Server v{} (commit: {})", version, "NONE"); - let connection_string = - std::env::var("DB_CONNSTR").expect("Expected database connection string"); + let mut file = File::open("../data_loader/config.yaml")?; + let mut file_contents = String::new(); + file.read_to_string(&mut file_contents); - let pool = ::sqlx::postgres::PgPoolOptions::new() - .max_connections(5) - .connect(&connection_string) - .await?; + let config_file = serde_yaml::from_str::(file_contents.as_str())?; - let tt_service = trip_tracking::TripTrackingService::new(); + let tt_service = services::trip_tracking::TripTrackingService::new(); tt_service.start(); + let svc = gtfs_pull::GtfsPullService::new(config_file); + svc.start(); + svc.wait_for_ready(); + + let state = Arc::new(AppState { - database: pool, + gtfs_service: svc, trip_tracking_service: tt_service }); diff --git a/api/src/services/mod.rs b/api/src/services/mod.rs index dc3fd85..013f9ce 100644 --- a/api/src/services/mod.rs +++ b/api/src/services/mod.rs @@ -1 +1,3 @@ pub mod trip_tracking; +pub mod gtfs_pull; +pub mod gtfs_rt; diff --git a/api/src/services/trip_tracking.rs b/api/src/services/trip_tracking.rs index 25e0d13..5500b4a 100644 --- a/api/src/services/trip_tracking.rs +++ b/api/src/services/trip_tracking.rs @@ -97,7 +97,10 @@ impl TripTrackingService { delay: live_track.delay, next_stop_id: live_track.next_stop_id, timestamp: live_track.timestamp, - vehicle_id: live_track.vehicle_id + vehicle_ids: match live_track.vehicle_id { + Some(x) => x.split(",").map(|f| String::from(f)).collect(), + None => vec![] + } } ) } diff --git a/api/src/templates.rs b/api/src/templates.rs index 842d100..95358bb 100644 --- a/api/src/templates.rs +++ b/api/src/templates.rs @@ -70,7 +70,7 @@ pub fn build_timetables( let mut direction_trips: Vec<&Trip> = trips .iter() - .filter(|trip| trip.direction_id == direction.direction_id) + .filter(|trip| trip.direction.direction == direction.direction) .collect(); direction_trips.sort_by_key(|trip| { @@ -84,7 +84,7 @@ pub fn build_timetables( for trip in direction_trips.clone() { if let Some(last) = trip.schedule.last() { if next_id == None && i64::from(seconds_since_midnight) < last.arrival_time { - next_id = Some(last.trip_id.clone()); + next_id = Some(last.stop.id.to_string()); } } } @@ -105,12 +105,12 @@ pub fn build_timetables( for (trip_index, trip) in direction_trips.iter().enumerate() { for stop in &trip.schedule { let entry = stop_map - .entry(stop.stop_id) - .or_insert((stop.stop_sequence, stop.stop_name.clone(), vec![None; direction_trips.len()])); + .entry(stop.stop.id) + .or_insert((stop.stop_sequence, stop.stop.name.clone(), vec![None; direction_trips.len()])); // If this stop_id appears in multiple trips with different sequences, keep the lowest entry.0 = entry.0.max(stop.stop_sequence); - entry.1 = stop.stop_name.clone(); + entry.1 = stop.stop.name.clone(); entry.2[trip_index] = Some(stop.arrival_time); } } diff --git a/api/templates/route.html b/api/templates/route.html index a75dbd1..86fca33 100644 --- a/api/templates/route.html +++ b/api/templates/route.html @@ -89,7 +89,7 @@ document.querySelectorAll("details[data-direction-id]").forEach(details => { {% for timetable in timetables %} -
+

{{ timetable.direction.direction | capitalize }} to

@@ -104,7 +104,7 @@ document.querySelectorAll("details[data-direction-id]").forEach(details => { {% for trip_id in timetable.trip_ids %} {% if let Some(next_id_v) = timetable.next_id %} {% if next_id_v == trip_id %} - + {% else %} {% endif %} diff --git a/api/templates/route_symbol.html b/api/templates/route_symbol.html index e5551f2..aca4ed4 100644 --- a/api/templates/route_symbol.html +++ b/api/templates/route_symbol.html @@ -1,18 +1,18 @@ {% macro route_symbol(route) %} {% match route.route_type %} {% when libseptastic::route::RouteType::Trolley | libseptastic::route::RouteType::SubwayElevated %} -
- {{ route.id }} +
+ {{ route.short_name }}
{% endwhen %} {% when libseptastic::route::RouteType::RegionalRail %}
- {{ route.id }} + {{ route.short_name }}
{% endwhen %} {% when libseptastic::route::RouteType::Bus | libseptastic::route::RouteType::TracklessTrolley %}
- {{ route.id }} + {{ route.short_name }}
{% endwhen %} {% endmatch %} diff --git a/api/templates/routes.html b/api/templates/routes.html index dddb5fe..2c8d711 100644 --- a/api/templates/routes.html +++ b/api/templates/routes.html @@ -7,7 +7,7 @@

For infrequent rail service to suburban locations

{% for route in rr_routes %} -

[ {{ format!("{:7}", route.id) }}: {{ route.name }}

]

+

[ {{ format!("{:7}", route.short_name) }}: {{ route.name }}

]

 {% endfor %} @@ -20,7 +20,7 @@
{% for route in subway_routes %} -

[ {{ format!("{:7}", route.id) }}: {{ route.name }}

]

+

[ {{ format!("{:7}", route.short_name) }}: {{ route.name }}

]

 {% endfor %} @@ -30,7 +30,7 @@ {% for route in trolley_routes %} -

[ {{ format!("{:7}", route.id) }}: {{ route.name }}

]

+

[ {{ format!("{:7}", route.short_name) }}: {{ route.name }}

]

 {% endfor %} @@ -40,7 +40,7 @@

For service of varying frequency within SEPTA's entire service area

{% for route in bus_routes %} -

[ {{ format!("{:7}", route.id) }}: {{ route.name }}

]

+

[ {{ format!("{:7}", route.short_name) }}: {{ route.name }}

]

 {% endfor %} diff --git a/data_loader/config.yaml b/data_loader/config.yaml index 2fea038..5852e32 100644 --- a/data_loader/config.yaml +++ b/data_loader/config.yaml @@ -3,5 +3,8 @@ gtfs_zips: subzip: "google_rail.zip" - uri: "https://www3.septa.org/developer/gtfs_public.zip" subzip: "google_bus.zip" - - uri: "https://www.njtransit.com/rail_data.zip" - - uri: "https://www.njtransit.com/bus_data.zip" +# - uri: "https://www.njtransit.com/rail_data.zip" +# - uri: "https://www.njtransit.com/bus_data.zip" +annotations: + synthetic_routes: + - id: 'NYC' diff --git a/data_loader/src/main.rs b/data_loader/src/main.rs index 2316def..c9c2cda 100644 --- a/data_loader/src/main.rs +++ b/data_loader/src/main.rs @@ -15,14 +15,6 @@ async fn main() -> ::anyhow::Result<()> { let env = Env::new().filter_or("RUST_LOG", "data_loader=info"); Builder::from_env(env).init(); - let mut file = File::open("config.yaml")?; - let mut file_contents = String::new(); - file.read_to_string(&mut file_contents); - - let config_file = serde_yaml::from_str::(file_contents.as_str())?; - - let svc = GtfsPullService::new(config_file); - svc.start(); loop{ thread::sleep(Duration::from_secs(120)); @@ -41,88 +33,3 @@ async fn main() -> ::anyhow::Result<()> { Ok(()) } - -#[derive(Serialize, Deserialize, PartialEq, Debug,Clone)] -struct GtfsSource { - pub uri: String, - pub subzip: Option -} - -#[derive(Serialize, Deserialize, PartialEq, Debug)] -struct Config { - pub gtfs_zips: Vec -} - -struct GtfsFile { - pub source: GtfsSource, - pub hash: Option -} - -struct GtfsPullServiceState { - pub gtfs_files: Vec, - pub tmp_dir: PathBuf -} - -pub struct GtfsPullService { - state: Arc> -} - -impl GtfsPullService { - const UPDATE_SECONDS: u64 = 3600*24; - - pub fn new(config: Config) -> Self { - Self { - state: Arc::new(Mutex::new( - GtfsPullServiceState { - gtfs_files: config.gtfs_zips.iter().map(|f| { GtfsFile { source: f.clone(), hash: None} }).collect(), - tmp_dir: env::temp_dir() - } - )) - } - } - - pub fn start(&self) { - let cloned_state = Arc::clone(&self.state); - thread::spawn(move || { - loop { - let recloned_state = Arc::clone(&cloned_state); - let res = Self::update_gtfs_data(recloned_state); - - match res { - Err(err) => { - error!("{}", err); - } - _ => {} - } - - thread::sleep(Duration::from_secs(Self::UPDATE_SECONDS)); - } - }); - } - - pub fn update_gtfs_data(state: Arc>) -> anyhow::Result<()> { - let l_state = state.lock().unwrap(); - - for gtfs_file in l_state.gtfs_files.iter() { - let gtfs = if let Some(subzip) = gtfs_file.source.subzip.clone() { - info!("Reading GTFS file at {} (subzip {})", gtfs_file.source.uri, subzip); - let res = reqwest::blocking::get(gtfs_file.source.uri.clone())?; - let outer_archive = res.bytes()?; - let mut archive = ZipArchive::new(Cursor::new(outer_archive))?; - archive.extract(l_state.tmp_dir.clone())?; - - let mut file_path = l_state.tmp_dir.clone(); - file_path.push(subzip.clone()); - - gtfs_structures::Gtfs::new(file_path.to_str().unwrap())? - } else { - info!("Reading GTFS file at {}", gtfs_file.source.uri); - gtfs_structures::Gtfs::new(gtfs_file.source.uri.as_str())? - }; - } - - - - Ok(()) - } -} diff --git a/libseptastic/src/direction.rs b/libseptastic/src/direction.rs index beb68b6..88bbc54 100644 --- a/libseptastic/src/direction.rs +++ b/libseptastic/src/direction.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; -#[derive(sqlx::Type, Serialize, Deserialize, PartialEq, Debug, Clone)] +#[derive(sqlx::Type, Serialize, Deserialize, PartialEq, Debug, Clone, Copy, Eq)] #[sqlx(type_name = "septa_direction_type", rename_all = "snake_case")] pub enum CardinalDirection { Northbound, @@ -15,8 +15,6 @@ pub enum CardinalDirection { #[derive(::sqlx::Decode, Serialize, Deserialize, Debug, Clone)] pub struct Direction { - pub route_id: String, - pub direction_id: i64, pub direction: CardinalDirection, pub direction_destination: String } diff --git a/libseptastic/src/lib.rs b/libseptastic/src/lib.rs index af16fa4..29bdfe8 100644 --- a/libseptastic/src/lib.rs +++ b/libseptastic/src/lib.rs @@ -1,4 +1,5 @@ pub mod route; +pub mod agency; pub mod stop; pub mod route_stop; pub mod stop_schedule; diff --git a/libseptastic/src/stop.rs b/libseptastic/src/stop.rs index 65cb01b..911fa49 100644 --- a/libseptastic/src/stop.rs +++ b/libseptastic/src/stop.rs @@ -1,4 +1,6 @@ -#[derive(sqlx::Type, PartialEq, Debug, Clone)] +use serde::{Deserialize, Serialize}; + +#[derive(sqlx::Type, PartialEq, Debug, Clone, Serialize, Deserialize)] #[sqlx(type_name = "septa_stop_type", rename_all = "snake_case")] pub enum StopType { FarSide, @@ -6,7 +8,7 @@ pub enum StopType { Normal } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Stop { pub id: i64, pub name: String, diff --git a/libseptastic/src/stop_schedule.rs b/libseptastic/src/stop_schedule.rs index 4d367c9..698b8e9 100644 --- a/libseptastic/src/stop_schedule.rs +++ b/libseptastic/src/stop_schedule.rs @@ -1,22 +1,19 @@ use serde::{Deserialize, Serialize}; +use crate::direction::Direction; + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StopSchedule { - pub route_id: String, - pub stop_name: String, - pub trip_id: String, - pub service_id: String, - pub direction_id: i64, pub arrival_time: i64, - pub stop_id: i64, - pub stop_sequence: i64 + pub stop_sequence: i64, + pub stop: crate::stop::Stop } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct Trip { - pub route_id: String, + pub service_id: String, pub trip_id: String, - pub direction_id: i64, + pub direction: Direction, pub tracking_data: TripTracking, pub schedule: Vec } @@ -33,5 +30,5 @@ pub struct LiveTrip { pub delay: f64, pub next_stop_id: Option, pub timestamp: i64, - pub vehicle_id: Option + pub vehicle_ids: Vec } diff --git a/shell.nix b/shell.nix index 08eaca4..ea8ae70 100644 --- a/shell.nix +++ b/shell.nix @@ -4,5 +4,6 @@ stdenv.mkDerivation { nativeBuildInputs = [ pkg-config postgresql_14 ]; buildInputs = [ cryptsetup + protobuf ]; } From 58e2097016f8e4614f249eccfd20eb9f22ae5fc7 Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Sat, 8 Nov 2025 13:14:10 -0500 Subject: [PATCH 4/4] copy config --- Dockerfile | 1 + api/config.yaml | 10 ++++++++++ api/src/main.rs | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 api/config.yaml diff --git a/Dockerfile b/Dockerfile index d39a18b..d27c95e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,6 +19,7 @@ WORKDIR /app EXPOSE 8080 COPY --from=build /api/target/release/septastic_api /app/septastic_api +COPY --from=build /api/config.yaml /app/config.yaml COPY api/assets /app/assets COPY api/templates /app/templates diff --git a/api/config.yaml b/api/config.yaml new file mode 100644 index 0000000..5852e32 --- /dev/null +++ b/api/config.yaml @@ -0,0 +1,10 @@ +gtfs_zips: + - uri: "https://www3.septa.org/developer/gtfs_public.zip" + subzip: "google_rail.zip" + - uri: "https://www3.septa.org/developer/gtfs_public.zip" + subzip: "google_bus.zip" +# - uri: "https://www.njtransit.com/rail_data.zip" +# - uri: "https://www.njtransit.com/bus_data.zip" +annotations: + synthetic_routes: + - id: 'NYC' diff --git a/api/src/main.rs b/api/src/main.rs index 14a32e9..eb1ba8a 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -78,7 +78,7 @@ async fn main() -> ::anyhow::Result<()> { let version: &str = option_env!("CARGO_PKG_VERSION").expect("Expected package version"); info!("Starting the SEPTASTIC Server v{} (commit: {})", version, "NONE"); - let mut file = File::open("../data_loader/config.yaml")?; + let mut file = File::open("./config.yaml")?; let mut file_contents = String::new(); file.read_to_string(&mut file_contents);