init gtfs setup

data_loader/src/main.rs

@@ -1,27 +1,11 @@
+use std::{clone, env, fs::File, io::{Read, Write}, path::{Path, PathBuf}, sync::{Arc, Mutex}, thread, time::Duration};
+
 use dotenv::dotenv;
-use libseptastic::stop::Stop;
-use septa::route_stop;
-use septa_json::direction::Direction;
-use septa_json::route::Route;
-use septa_json::route_stop::RouteStop;
-use septa_json::schedule_day::{Calendar, ScheduleDay};
-use septa_json::stop_schedule::StopSchedule;
+use serde::{Deserialize, Serialize};
 use sqlx::postgres::PgPoolOptions;
-use std::collections::{HashMap, HashSet};
-use std::sync::{Arc, Mutex};
-use std::{fs, thread};
-use std::path::Path;
-use std::time::SystemTime;
 use env_logger::{Builder, Env};
 use log::{error, info, warn};
 
-pub mod traits;
-use traits::*;
-
-pub mod septa_json;
-pub mod septa;
-pub mod fetchers;
-
 
 #[tokio::main]
 async fn main() -> ::anyhow::Result<()> {
@@ -29,46 +13,86 @@ async fn main() -> ::anyhow::Result<()> {
 
     let env = Env::new().filter_or("RUST_LOG", "data_loader=info");
     Builder::from_env(env).init();
+
+    let mut file = File::open("config.yaml")?;
+    let mut file_contents = String::new();
+    file.read_to_string(&mut file_contents);
+
+    let config_file = serde_yaml::from_str::<Config>(file_contents.as_str());
+
     let database_url = std::env::var("DATABASE_URL").expect("Database URL");
     let pool = PgPoolOptions::new()
            .max_connections(5)
            .connect(&database_url)
            .await?;
-
-    let septa_data = fetchers::septa::SeptaFetcher::fetch_septa_data().await?;
-
+    
     let mut tx = pool.begin().await?;
 
-    libseptastic::route::Route::create_table(&mut tx).await?;
-    libseptastic::direction::Direction::create_table(&mut tx).await?;
-    libseptastic::stop::Stop::create_table(&mut tx).await?;
-    libseptastic::route_stop::RouteStop::create_table(&mut tx).await?;
-    libseptastic::stop_schedule::StopSchedule::create_table(&mut tx).await?;
-    libseptastic::schedule_day::ScheduleDay::create_table(&mut tx).await?;
-
-
-
-    info!("Inserting Route Data");
-    libseptastic::route::Route::insert_many(septa_data.routes, &mut tx).await?;
-
-    info!("Inserting Direction Data");
-    libseptastic::direction::Direction::insert_many(septa_data.directions, &mut tx).await?;
-
-    info!("Inserting Stop Data");
-    libseptastic::stop::Stop::insert_many(septa_data.stops, &mut tx).await?;
-
-    info!("Inserting Route-Stop Data");
-    libseptastic::route_stop::RouteStop::insert_many(septa_data.route_stops, &mut tx).await?;
-
-    info!("Inserting Stop Schedule Data");
-    libseptastic::stop_schedule::StopSchedule::insert_many(septa_data.stop_schedules, &mut tx).await?;
-
-    info!("Inserting Schedule Day Data");
-    libseptastic::schedule_day::ScheduleDay::insert_many(septa_data.schedule_days, &mut tx).await?;
-
-
     tx.commit().await?;
     pool.close().await;
 
     Ok(())
 }
+
+#[derive(Serialize, Deserialize, PartialEq, Debug)]
+struct Config {
+    gtfs_zips: Vec<String>
+}
+
+struct GtfsFile {
+    pub url: String,
+    pub hash: Option<String>
+}
+
+struct GtfsPullServiceState {
+    pub gtfs_files: Vec<GtfsFile>,
+    pub tmp_dir: PathBuf
+}
+
+pub struct GtfsPullService {
+    state: Arc<Mutex<GtfsPullServiceState>>
+}
+
+impl GtfsPullService {
+    const UPDATE_SECONDS: u64 = 3600*24;
+
+    pub fn new(config: Config) -> Self {
+        Self {
+            state: Arc::new(Mutex::new(
+                           GtfsPullServiceState { 
+                               gtfs_files: config.gtfs_zips.iter().map(|f| { GtfsFile { url: f.clone(), hash: None} }).collect(),
+                               tmp_dir: env::temp_dir()
+                           }
+                    ))
+        }
+    }
+
+    pub fn start(&self) {
+        let cloned_state = Arc::clone(&self.state);
+        thread::spawn(move || {
+            loop {
+                let recloned_state = Arc::clone(&cloned_state);
+                let res = Self::update_gtfs_data(recloned_state);
+                
+                match res {
+                    Err(err) => {
+                        error!("{}", err);
+                    }
+                    _ => {}
+                }
+
+                thread::sleep(Duration::from_secs(Self::UPDATE_SECONDS));
+            }
+        });
+    }
+
+    pub fn update_gtfs_data(state: Arc<Mutex<GtfsPullServiceState>>) -> anyhow::Result<()> {
+        let l_state = state.lock().unwrap();
+
+        for gtfs_file in l_state.gtfs_files.iter() {
+            let resp = reqwest::blocking::get(gtfs_file.url.clone())?;
+        }
+
+        Ok(())
+    }
+}