restructure modules,use tokio::sync::Mutex, add stats watcher thread

.gitignore

@@ -1,3 +1,5 @@
 *.pem
 *.lock
-target/
+target/
+*.stats*
+.vscode

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pulse"
-version = "0.0.1"
+version = "0.0.2"
 authors = ["Jerboa"]
 
 edition="2021"

src/lib.rs

@@ -1,13 +1,20 @@
 pub mod web;
 pub mod server;
-
+pub mod stats;
 pub mod util;
 
 #[cfg(feature = "http")]
 pub mod server_http;
 
 const DEBUG: bool = true;
 
+/// Completely drop Github POST requests concerning private repos
+pub const IGNORE_PRIVATE_REPOS: bool = true;
+
+/// Process Github POST requests concerning private repos
+/// but never send outbound trafic (e.g. Discord)
+pub const DONT_MESSAGE_ON_PRIVATE_REPOS: bool = true;
+
 pub fn debug(msg: String, context: Option<String>)
 {
     if DEBUG == false { return }

src/main.rs

@@ -1,4 +1,5 @@
-use pulse::{server::Server, web::request::discord::model::Webhook};
+use pulse::{server::Server, web::discord::request::model::Webhook, stats};
+use tokio::task::spawn;
 
 #[tokio::main]
 async fn main() {
@@ -97,6 +98,7 @@ async fn main() {
         "./key.pem".to_string()
     };
 
+    let _stats_watcher = spawn(stats::io::watch(Webhook::new(disc_url.clone())));
 
     let server = Server::new(0,0,0,0, port,token, Webhook::new(disc_url));
 

src/main_http.rs

@@ -1,6 +1,7 @@
 #[cfg(feature = "http")]
 
-use pulse::{server_http::ServerHttp, web::request::discord::model::Webhook};
+use pulse::{server_http::ServerHttp, web::discord::request::model::Webhook, stats};
+use tokio::task::spawn;
 
 #[cfg(feature = "http")]
 #[tokio::main]
@@ -66,6 +67,7 @@ async fn main() {
         3030
     };
 
+    let _stats_watcher = spawn(stats::io::watch(Webhook::new(disc_url.clone())));
 
     let server = ServerHttp::new(0,0,0,0, port,token, Webhook::new(disc_url));
 

src/server.rs

@@ -1,13 +1,16 @@
 use crate::web::
 {
     throttle::{IpThrottler, handle_throttle},
-    response::github::{github_filter::filter_github, model::GithubConfig},
-    request::discord::model::Webhook
+    github::{response::github_filter::filter_github, model::{GithubConfig, GithubStats}},
+    discord::request::model::Webhook
 };
 
+use crate::stats;
+
 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
 use std::path::PathBuf;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
+use tokio::sync::Mutex;
 
 use axum::
 {
@@ -33,7 +36,7 @@ impl Server
         d: u8,
         port: u16,
         token: String,
-        disc: Webhook
+        disc: Webhook,
     ) 
     -> Server
     {
@@ -46,7 +49,7 @@ impl Server
 
         let throttle_state = Arc::new(Mutex::new(requests));
 
-        let github = GithubConfig::new(token, disc);
+        let github = Arc::new(Mutex::new(GithubConfig::new(token, disc.clone(), GithubStats::new())));
 
         Server
         {
@@ -55,7 +58,6 @@ impl Server
             .route("/", post(|| async move {  }))
             .layer(middleware::from_fn_with_state(github, filter_github))
             .layer(middleware::from_fn_with_state(throttle_state.clone(), handle_throttle))
-
         }
     }
 

src/server_http.rs

@@ -1,17 +1,16 @@
-
 use crate::web::
 {
     throttle::{IpThrottler, handle_throttle},
-    response::github::{github_filter::filter_github, model::GithubConfig},
-    request::discord::model::Webhook
+    github::{response::github_filter::filter_github, model::{GithubConfig, GithubStats}},
+    discord::request::model::Webhook
 };
 
+use crate::stats;
 
-use std::clone;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
+use tokio::sync::Mutex;
 
-use axum::extract::State;
 use axum::
 {
     routing::post, 
@@ -48,16 +47,15 @@ impl ServerHttp
 
         let throttle_state = Arc::new(Mutex::new(requests));
 
-        let github = GithubConfig::new(token, disc);
-
+        let github = Arc::new(Mutex::new(GithubConfig::new(token, disc.clone(), GithubStats::new())));
+        
         ServerHttp
         {
             addr: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(a,b,c,d)), port),
             router: Router::new()
             .route("/", post(|| async move {  }))
             .layer(middleware::from_fn_with_state(github, filter_github))
             .layer(middleware::from_fn_with_state(throttle_state.clone(), handle_throttle))
-
         }
     }
 

src/stats/io.rs

@@ -0,0 +1,156 @@
+use std::cmp::min;
+use std::path::Path;
+use std::{sync::Arc, collections::HashMap};
+
+use chrono::{Local, Datelike, Timelike, DateTime};
+use tokio::sync::Mutex;
+
+use crate::DONT_MESSAGE_ON_PRIVATE_REPOS;
+use crate::util::{write_file, read_file_utf8};
+use crate::web::discord::request::model::Webhook;
+use crate::web::discord::request::post::post;
+use crate::web::github::model::{GithubStats, GithubRepoStats, GithubConfig};
+
+use std::time::{Duration, SystemTime};
+use std::thread::sleep;
+
+const STATS_PATH: &str = "repo.stats";
+
+pub async fn collect(stats: Arc<Mutex<GithubConfig>>, data: HashMap<String, serde_json::Value>)
+{
+
+    let mut held_stats = stats.lock().await;
+
+    let mut name = match data["repository"]["name"].as_str()
+    {
+        Some(s) => s.to_string(),
+        None => return
+    };
+
+    let push = if data.contains_key("pusher")
+    {
+        1
+    }
+    else 
+    {
+        0
+    };
+
+    if data["repository"]["private"].as_bool().is_some_and(|x|x)
+    {
+        name = format!("{}_private", name);
+    }
+
+    let stars = data["repository"]["stargazers_count"].as_u64().unwrap();
+
+    let new_stats = GithubRepoStats {stars: stars, pushes: push};
+
+    if !held_stats.get_stats().repos.contains_key(&name)
+    {
+        held_stats.get_stats().repos.insert(name.to_string(), GithubRepoStats::new());
+    }
+
+    held_stats.get_stats().repos.get_mut(&name).unwrap().update(new_stats);
+
+
+    if Path::new(STATS_PATH).exists()
+    {
+        match std::fs::copy(STATS_PATH, format!("{}.bk",STATS_PATH))
+        {
+            Ok(_) => {},
+            Err(why) => 
+            {
+                crate::debug(format!("error {} copying stats to {}.bk", why, STATS_PATH), None);
+                return
+            }
+        }
+    }
+
+    match serde_json::to_string_pretty(held_stats.get_stats())
+    {
+        Ok(se) => 
+        {
+            write_file(STATS_PATH, se.as_bytes())
+        },
+        Err(why) => 
+        {
+            crate::debug(format!("error {} writing stats to {}", why, STATS_PATH), None);
+            return
+        }
+    }
+
+    crate::debug(format!("wrote data"), None);
+}
+
+pub async fn watch(disc: Webhook)
+{
+    let mut last_message = SystemTime::UNIX_EPOCH;
+    loop 
+    {
+        let date = Local::now();
+
+        if date.weekday() == chrono::Weekday::Fri && last_message.elapsed().unwrap().as_secs() > 24*60*60
+        {
+            last_message = SystemTime::now();
+
+            let data = match read_file_utf8(STATS_PATH)
+            {
+                Some(d) => d,
+                None =>
+                {
+                    crate::debug(format!("error reading stats at {}", STATS_PATH), None);
+                    break
+                }
+            };
+
+            let stats: GithubStats = match serde_json::from_str(&data)
+            {
+                Ok(data) => {data},
+                Err(why) => 
+                {
+                    crate::debug(format!("error {} reading stats at {}", why, STATS_PATH), None);
+                    break
+                }
+            };
+
+            let mut pushes: Vec<(u64, u64, String)> = Vec::new();
+
+            for repo in stats.repos.into_iter()
+            {
+                if repo.0.contains("private") && DONT_MESSAGE_ON_PRIVATE_REPOS
+                {
+                    continue;
+                }
+
+                pushes.push((repo.1.pushes, repo.1.stars, repo.0));
+            }
+
+            if pushes.len() == 0
+            {
+                break;
+            }
+
+            pushes.sort_by(| a:&(u64, u64, String), b:&(u64, u64, String) | b.0.partial_cmp(&a.0).unwrap());
+
+            if pushes[0].0 == 0 
+            {
+                continue;
+            }
+
+            let mut msg = "Top activity this week :bar_chart:\n".to_string();
+
+            for i in 0..min(pushes.len(), 3)
+            {
+                msg.push_str(format!("    {} | {} pushes | {} stars\n", pushes[i].2, pushes[i].0, pushes[i].1).as_str());
+            }
+
+            match post(disc.clone(), msg).await
+            {
+                Ok(_) => {},
+                Err(e) => {crate::debug(format!("error posting message to discord {}", e), Some("stats watch".to_string()))}
+            }
+        }
+
+        sleep(Duration::from_secs(60*60));
+    }
+}

src/stats/mod.rs

@@ -0,0 +1 @@
+pub mod io;

src/util.rs

@@ -1,4 +1,4 @@
-use std::fmt::Write;
+use std::{fmt::Write, fs::File, io::{Write as ioWrite, Read}};
 use regex::Regex;
 
 pub fn dump_bytes(v: &[u8]) -> String 
@@ -25,4 +25,32 @@ pub fn strip_control_characters(s: String) -> String
 {
     let re = Regex::new(r"[\u0000-\u001F]").unwrap().replace_all(&s, "");
     return re.to_string()
+}
+
+pub fn write_file(path: &str, data: &[u8])
+{
+    let mut file = File::create(path).unwrap();
+    file.write_all(data).unwrap();
+}
+
+pub fn read_file_utf8(path: &str) -> Option<String>
+{
+    let mut file = match File::open(path) {
+        Err(why) => 
+        {
+            crate::debug(format!("error reading file to utf8, {}", why), None);
+            return None
+        },
+        Ok(file) => file,
+    };
+
+    let mut s = String::new();
+    match file.read_to_string(&mut s) {
+        Err(why) => 
+        {
+            crate::debug(format!("error reading file to utf8, {}", why), None);
+            return None
+        },
+        Ok(_) => Some(s)
+    }
 }

src/web/discord/mod.rs

@@ -0,0 +1 @@
+pub mod request;

src/web/request/discord/post.rs → src/web/discord/request/post.rs

@@ -2,7 +2,7 @@
 
 use std::collections::HashMap;
 
-use crate::web::request::discord::model::Webhook;
+use crate::web::discord::request::model::Webhook;
 
 /// Send a simple plaintext string message, msg, to the webhook w
 /// 
@@ -15,7 +15,7 @@ use crate::web::request::discord::model::Webhook;
 /// # Example
 /// ```rust
 /// 
-/// use pulse::web::request::discord::{model::Webhook, post::post};
+/// use pulse::web::discord::request::{model::Webhook, post::post};
 /// 
 /// pub async fn post_to_discord(){
 ///     let w = Webhook::new("https://discord.com/api/webhooks/xxx/yyy".to_string());
@@ -37,6 +37,7 @@ use crate::web::request::discord::model::Webhook;
 pub async fn post(w: Webhook, msg: String) -> Result<String, reqwest::Error>
 {
 
+    crate::debug(format!("Posting to Discord {:?}", msg), None);
     let client = reqwest::Client::new();
 
     let mut map = HashMap::new();

src/web/github/mod.rs

@@ -0,0 +1,2 @@
+pub mod response;
+pub mod model;