diff --git a/README.md b/README.md index 6e5f6e0..ec1ae8a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Code of https://status.d420.de +[screenshot](screenshot.png) + - install rust - Copy .env.example to .env and adjust - cargo run --release \ No newline at end of file diff --git a/entities/src/host.rs b/entities/src/host.rs index c60a91c..30d4daf 100644 --- a/entities/src/host.rs +++ b/entities/src/host.rs @@ -1,6 +1,9 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 -use sea_orm::{entity::prelude::*, sea_query::{Expr, SimpleExpr}}; +use sea_orm::{ + entity::prelude::*, + sea_query::{Expr, SimpleExpr}, +}; use serde::Serialize; #[derive(Copy, Clone, Default, Debug, DeriveEntity)] diff --git a/entities/src/mod.rs b/entities/src/mod.rs index 2630023..43d3bd2 100644 --- a/entities/src/mod.rs +++ b/entities/src/mod.rs @@ -2,8 +2,8 @@ pub mod prelude; -pub mod host; pub mod health_check; +pub mod host; // has to be re-added pub mod state; diff --git a/entities/src/prelude.rs b/entities/src/prelude.rs index 5e2a4c7..bf3284e 100644 --- a/entities/src/prelude.rs +++ b/entities/src/prelude.rs @@ -1,4 +1,4 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 -pub use super::host::Entity as Host; pub use super::health_check::Entity as UpdateCheck; +pub use super::host::Entity as Host; diff --git a/migration/src/m20230803_154714_version_url.rs b/migration/src/m20230803_154714_version_url.rs index d950723..ac43e6e 100644 --- a/migration/src/m20230803_154714_version_url.rs +++ b/migration/src/m20230803_154714_version_url.rs @@ -6,7 +6,6 @@ use sea_orm_migration::{ #[derive(DeriveMigrationName)] pub struct Migration; - #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { diff --git a/scanner/src/about_parser.rs b/scanner/src/about_parser.rs index 9000b2f..d4f0c7b 100644 --- a/scanner/src/about_parser.rs +++ b/scanner/src/about_parser.rs @@ -43,7 +43,10 @@ impl AboutParser { let mut a_elems = p_elem.select(&self.selector_a); let link = a_elems.next().ok_or(AboutParseError::NoCommitLinkFound)?; - let url = link.value().attr("href").map(|v|v.trim().to_owned()) + let url = link + .value() + .attr("href") + .map(|v| v.trim().to_owned()) .ok_or(AboutParseError::NoValidHref)?; let link_text = link.text().fold(String::new(), |mut acc, text| { acc.push_str(text); @@ -54,7 +57,7 @@ impl AboutParser { } Ok(AboutParsed { url, - version_name: link_text + version_name: link_text, }) } @@ -78,6 +81,9 @@ mod test { let parser = AboutParser::new(); let res = parser.parse_about_version(html).unwrap(); assert_eq!(&res.version_name, "2023.07.22-72d8f35"); - assert_eq!(res.url, String::from("https://github.com/zedeus/nitter/commit/72d8f35")); + assert_eq!( + res.url, + String::from("https://github.com/zedeus/nitter/commit/72d8f35") + ); } } diff --git a/scanner/src/cache_update.rs b/scanner/src/cache_update.rs index 4d2ea3b..e940464 100644 --- a/scanner/src/cache_update.rs +++ b/scanner/src/cache_update.rs @@ -13,8 +13,8 @@ use sea_orm::QueryFilter; use sea_orm::QueryOrder; use sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement}; -use crate::LatestCheck; use crate::version_check::fetch_git_state; +use crate::LatestCheck; use crate::{Result, Scanner}; #[derive(Debug, FromQueryResult)] @@ -51,9 +51,9 @@ impl Scanner { pub(crate) async fn generate_cache_data(&self) -> Result { let hosts = self.query_hosts_enabled().await?; let config_c = self.inner.config.clone(); - let current_version = tokio::task::spawn_blocking(move || { - fetch_git_state(config_c) - }).await.unwrap()?; + let current_version = tokio::task::spawn_blocking(move || fetch_git_state(config_c)) + .await + .unwrap()?; if hosts.is_empty() { return Ok(CacheData { hosts: Vec::new(), @@ -78,14 +78,25 @@ impl Scanner { let latest_check: HashMap = latest_check.into_iter().map(|v| (v.host, v)).collect(); - let mut ping_data = self.query_pings(time_now - self.inner.config.ping_range).await?; - + let mut ping_data = self + .query_pings(time_now - self.inner.config.ping_range) + .await?; + let mut host_statistics = Vec::with_capacity(hosts.len()); let default_health_check = LatestCheck::default(); for host in hosts { - let points_3h: f64 = 0.3 * stats_3h.get(&host.id).map_or(0.0, |stats|stats.good as f64 / stats.total as f64); - let points_30d: f64 = 0.2 * stats_30d.get(&host.id).map_or(0.0, |stats|stats.good as f64 / stats.total as f64); - let points_120d: f64 = 0.2 * stats_120d.get(&host.id).map_or(0.0, |stats|stats.good as f64 / stats.total as f64); + let points_3h: f64 = 0.3 + * stats_3h + .get(&host.id) + .map_or(0.0, |stats| stats.good as f64 / stats.total as f64); + let points_30d: f64 = 0.2 + * stats_30d + .get(&host.id) + .map_or(0.0, |stats| stats.good as f64 / stats.total as f64); + let points_120d: f64 = 0.2 + * stats_120d + .get(&host.id) + .map_or(0.0, |stats| stats.good as f64 / stats.total as f64); let points_version = 0.1 * host .version @@ -101,10 +112,17 @@ impl Scanner { // }; let points = (points * 100.0) as i32; - let latest_version = host.version_url.as_ref().map_or(false, |url|current_version.is_same_version(&url)); - let is_upstream = host.version_url.as_ref().map_or(false, |url|current_version.is_same_repo(&url)); - - let is_bad_host = (!last_check.healthy) && self.inner.config.bad_hosts.contains(&host.domain); + let latest_version = host + .version_url + .as_ref() + .map_or(false, |url| current_version.is_same_version(&url)); + let is_upstream = host + .version_url + .as_ref() + .map_or(false, |url| current_version.is_same_repo(&url)); + + let is_bad_host = + (!last_check.healthy) && self.inner.config.bad_hosts.contains(&host.domain); let host_ping_data = ping_data.remove(&host.id); host_statistics.push(CacheHost { @@ -115,10 +133,10 @@ impl Scanner { rss: host.rss, version: host.version, healthy: last_check.healthy, - ping_max: host_ping_data.as_ref().and_then(|v|v.max), - ping_min: host_ping_data.as_ref().and_then(|v|v.min), - ping_avg: host_ping_data.as_ref().and_then(|v|v.avg), - recent_pings: host_ping_data.map(|v|v.pings).unwrap_or_default(), + ping_max: host_ping_data.as_ref().and_then(|v| v.max), + ping_min: host_ping_data.as_ref().and_then(|v| v.min), + ping_avg: host_ping_data.as_ref().and_then(|v| v.avg), + recent_pings: host_ping_data.map(|v| v.pings).unwrap_or_default(), is_latest_version: latest_version, is_upstream, version_url: host.version_url, @@ -142,7 +160,7 @@ impl Scanner { .await?) } - async fn query_pings(&self, age: DateTimeUtc) -> Result> { + async fn query_pings(&self, age: DateTimeUtc) -> Result> { #[derive(Debug, FromQueryResult, Default)] struct PingEntry { host: i32, @@ -168,12 +186,14 @@ impl Scanner { let mut map = HashMap::with_capacity(100); let mut iter = last_pings.iter(); let first = match iter.next() { - None => {return Ok(HashMap::new());}, + None => { + return Ok(HashMap::new()); + } Some(v) => v, }; let mut current_entry = LastPings::default(); let mut last_host = first.host; - let mut non_null_entries = first.ping.as_ref().map_or(0, |_|1); + let mut non_null_entries = first.ping.as_ref().map_or(0, |_| 1); if let Some(ping) = first.ping.as_ref() { current_entry.avg = Some(*ping); non_null_entries += 1; @@ -192,14 +212,14 @@ impl Scanner { non_null_entries = 0; std::mem::swap(&mut new_entry, &mut current_entry); // insert back the old (swapped) entry - assert_eq!(map.insert(last_host, new_entry).is_some(),false); + assert_eq!(map.insert(last_host, new_entry).is_some(), false); last_host = ping.host; } if let Some(ping) = ping.ping.as_ref() { current_entry.avg = Some(current_entry.avg.unwrap_or(0) + ping); non_null_entries += 1; - current_entry.min = Some(current_entry.min.map_or(*ping,|v|v.min(*ping))); - current_entry.max = Some(current_entry.max.map_or(*ping,|v|v.max(*ping))); + current_entry.min = Some(current_entry.min.map_or(*ping, |v| v.min(*ping))); + current_entry.max = Some(current_entry.max.map_or(*ping, |v| v.max(*ping))); } current_entry.pings.push(ping.ping); } @@ -207,7 +227,7 @@ impl Scanner { if let Some(sum) = current_entry.avg { current_entry.avg = Some(sum / non_null_entries); } - assert_eq!(map.insert(last_host, current_entry).is_some(),false); + assert_eq!(map.insert(last_host, current_entry).is_some(), false); Ok(map) } @@ -235,25 +255,29 @@ impl Scanner { } /// Timestamp of last healthy host check - async fn query_last_healthy(&self) -> Result> { + async fn query_last_healthy(&self) -> Result> { #[derive(Debug, FromQueryResult)] struct LastHealthyEntry { host: i32, time: i64, } - let last_healthy_times = LastHealthyEntry::find_by_statement(Statement::from_sql_and_values( - DbBackend::Sqlite, - r#" + let last_healthy_times = + LastHealthyEntry::find_by_statement(Statement::from_sql_and_values( + DbBackend::Sqlite, + r#" SELECT u.host,MAX(u.time) as time FROM health_check u JOIN host h ON h.id = u.host WHERE h.enabled = true AND u.healthy = true GROUP BY u.host "#, - [], - )) - .all(&self.inner.db) - .await?; - let last_healthy_times: HashMap<_,_> = last_healthy_times.into_iter().map(|v| (v.host, Utc.timestamp_opt(v.time, 0).unwrap())).collect(); + [], + )) + .all(&self.inner.db) + .await?; + let last_healthy_times: HashMap<_, _> = last_healthy_times + .into_iter() + .map(|v| (v.host, Utc.timestamp_opt(v.time, 0).unwrap())) + .collect(); Ok(last_healthy_times) } @@ -262,7 +286,7 @@ impl Scanner { &self, from: DateTimeUtc, to: DateTimeUtc, - ) -> Result> { + ) -> Result> { let stats: Vec = HostStats::find_by_statement(Statement::from_sql_and_values( DbBackend::Sqlite, r#"SELECT u.host, COUNT(CASE WHEN healthy = true THEN 1 END) as good,COUNT(*) as total FROM health_check u @@ -273,7 +297,7 @@ impl Scanner { )) .all(&self.inner.db) .await?; - let stats: HashMap<_,_> = stats.into_iter().map(|v| (v.host, v)).collect(); + let stats: HashMap<_, _> = stats.into_iter().map(|v| (v.host, v)).collect(); Ok(stats) } } diff --git a/scanner/src/instance_check.rs b/scanner/src/instance_check.rs index 2702af8..9cf47e9 100644 --- a/scanner/src/instance_check.rs +++ b/scanner/src/instance_check.rs @@ -14,10 +14,10 @@ use sea_orm::{ActiveModelTrait, ActiveValue}; use tokio::task::JoinSet; use tracing::instrument; +use crate::about_parser::AboutParsed; use crate::FetchError; use crate::Result; use crate::Scanner; -use crate::about_parser::AboutParsed; impl Scanner { /// Check uptime for host and create a new uptime entry in the database @@ -34,7 +34,10 @@ impl Scanner { for model in hosts.into_iter() { let scanner = self.clone(); - let muted_host = last_check.iter().find(|v|v.host == model.id).map_or(false,|check|!check.healthy); + let muted_host = last_check + .iter() + .find(|v| v.host == model.id) + .map_or(false, |check| !check.healthy); join_set.spawn(async move { scanner.health_check_host(model, muted_host).await; }); @@ -133,7 +136,12 @@ impl Scanner { false => { if !mute { // 404 = disabled - tracing::debug!(url=url.as_str(), code = code, content = content, "rss content not found"); + tracing::debug!( + url = url.as_str(), + code = code, + content = content, + "rss content not found" + ); } return false; } diff --git a/scanner/src/lib.rs b/scanner/src/lib.rs index 4666ad6..88ecfd1 100644 --- a/scanner/src/lib.rs +++ b/scanner/src/lib.rs @@ -18,8 +18,9 @@ use reqwest::{ Client, Url, }; use sea_orm::{ - sea_query::OnConflict, ActiveModelTrait, ActiveValue, ColumnTrait, DatabaseConnection, - EntityTrait, QueryFilter, TransactionTrait, FromQueryResult, Statement, DbBackend, ConnectionTrait, + sea_query::OnConflict, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, + DatabaseConnection, DbBackend, EntityTrait, FromQueryResult, QueryFilter, Statement, + TransactionTrait, }; use thiserror::Error; use tokio::{task::JoinSet, time::sleep}; @@ -83,9 +84,7 @@ impl FetchError { /// Returns the http status code, if applicable fn http_status_code(&self) -> Option { match self { - FetchError::Reqwest(e) => { - e.status().map(|v|v.as_u16()) - }, + FetchError::Reqwest(e) => e.status().map(|v| v.as_u16()), FetchError::HttpResponseStatus(code, _, _) => Some(*code), FetchError::KnownHttpResponseStatus(code, _) => Some(*code), FetchError::Captcha | FetchError::RetrievingBody(_, _) => None, @@ -93,7 +92,12 @@ impl FetchError { } } -pub fn run_scanner(db: DatabaseConnection, config: ScannerConfig, cache: Cache, disable_startup_scan: bool) -> Result<()> { +pub fn run_scanner( + db: DatabaseConnection, + config: ScannerConfig, + cache: Cache, + disable_startup_scan: bool, +) -> Result<()> { let scanner = Scanner::new(db, config, cache); tokio::spawn(async move { @@ -194,9 +198,9 @@ impl Scanner { async fn sleep_till_deadline(&self) { let delay_instance_check = self.last_uptime_check() + self.inner.config.instance_check_interval; - + let delay_list_update = self.last_list_fetch() + self.inner.config.list_fetch_interval; - + let delay = delay_instance_check.min(delay_list_update); tracing::trace!(duration=?delay-Instant::now(),"scanner sleeping"); sleep(delay.duration_since(Instant::now())).await; @@ -264,7 +268,10 @@ impl Scanner { // detect already offline host and prevent log spam let muted_host = match self.inner.config.auto_mute { false => false, - true => last_status.iter().find(|v|v.domain == instance.domain).map_or(false,|check|!check.healthy), + true => last_status + .iter() + .find(|v| v.domain == instance.domain) + .map_or(false, |check| !check.healthy), }; // tracing::trace!(muted_host,instance=?instance,last_status=?last_status); join_set.spawn(async move { @@ -278,12 +285,12 @@ impl Scanner { Ok(mut url) => { let rss = scanner_c.has_rss(&mut url, muted_host).await; match scanner_c.nitter_version(&mut url, muted_host).await { - Some(version) => (rss,Some(version.version_name),Some(version.url)), - None => (rss,None,None), + Some(version) => (rss, Some(version.version_name), Some(version.url)), + None => (rss, None, None), } } }; - + host::ActiveModel { id: ActiveValue::NotSet, domain: ActiveValue::Set(instance.domain), @@ -376,7 +383,10 @@ impl Scanner { Ok((code, body)) } - pub async fn query_latest_check(&self, connection: &T) -> Result> { + pub async fn query_latest_check( + &self, + connection: &T, + ) -> Result> { let health_checks = LatestCheck::find_by_statement(Statement::from_sql_and_values( DbBackend::Sqlite, r#" diff --git a/scanner/src/version_check.rs b/scanner/src/version_check.rs index 87a0b35..12daa0a 100644 --- a/scanner/src/version_check.rs +++ b/scanner/src/version_check.rs @@ -1,5 +1,5 @@ use entities::state::scanner::ScannerConfig; -use git2::{Remote, Direction}; +use git2::{Direction, Remote}; use crate::{Result, ScannerError}; @@ -19,20 +19,21 @@ impl CurrentVersion { pub fn is_same_version(&self, url: &str) -> bool { // look for last path segment (don't parse) and look if that matches // from the start - self.is_same_repo(url) && match url.split('/').last() { - Some(other_version) => { - if other_version.len() == self.version.len() { - // we get the long version - other_version.starts_with(&self.version) - } else if other_version.len() == SHORT_COMMIT_LEN { - other_version.starts_with(&self.version[..SHORT_COMMIT_LEN]) - } else { - // everything else is no short commit id.. - false + self.is_same_repo(url) + && match url.split('/').last() { + Some(other_version) => { + if other_version.len() == self.version.len() { + // we get the long version + other_version.starts_with(&self.version) + } else if other_version.len() == SHORT_COMMIT_LEN { + other_version.starts_with(&self.version[..SHORT_COMMIT_LEN]) + } else { + // everything else is no short commit id.. + false + } } + None => false, } - None => false - } } } @@ -41,13 +42,21 @@ pub(crate) fn fetch_git_state(config: ScannerConfig) -> Result { remote.connect(Direction::Fetch)?; - let reference = format!("refs/heads/{}",config.source_git_branch); - let commit = remote.list()?.into_iter().find(|v|v.name() == &reference) - .map(|v|v.oid().to_string()); + let reference = format!("refs/heads/{}", config.source_git_branch); + let commit = remote + .list()? + .into_iter() + .find(|v| v.name() == &reference) + .map(|v| v.oid().to_string()); remote.disconnect()?; - Ok(commit.map(|commit|CurrentVersion { version: commit, config }).ok_or(ScannerError::GitBranch)?) + Ok(commit + .map(|commit| CurrentVersion { + version: commit, + config, + }) + .ok_or(ScannerError::GitBranch)?) } #[cfg(test)] @@ -60,4 +69,4 @@ mod test { fn test_git() { fetch_git_state(Config::test_defaults()).unwrap(); } -} \ No newline at end of file +} diff --git a/server/src/website.rs b/server/src/website.rs index b01ccc4..36efd33 100644 --- a/server/src/website.rs +++ b/server/src/website.rs @@ -26,7 +26,7 @@ pub async fn instances( let end = Instant::now(); drop(guard); let templating_time = end - start; - tracing::trace!(templating_time=templating_time.as_millis()); + tracing::trace!(templating_time = templating_time.as_millis()); res }; res.headers_mut().insert( diff --git a/src/main.rs b/src/main.rs index 07fb0e5..32823b3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,9 +14,12 @@ fn main() -> miette::Result<()> { let build_mode = "debug mode"; #[cfg(not(debug_assertions))] let build_mode = "release mode"; - println!("Starting {} {} licensed under {}, {build_mode}", - env!("CARGO_PKG_NAME"),env!("CARGO_PKG_VERSION"), - env!("CARGO_PKG_LICENSE")); + println!( + "Starting {} {} licensed under {}, {build_mode}", + env!("CARGO_PKG_NAME"), + env!("CARGO_PKG_VERSION"), + env!("CARGO_PKG_LICENSE") + ); dotenvy::dotenv() .into_diagnostic() .wrap_err_with(|| "Failed to load .env file!")?; @@ -74,9 +77,14 @@ async fn _main() -> miette::Result<()> { let disable_startup_scan = require_env_str("DISABLE_STARTUP_SCAN")? == "true"; - scanner::run_scanner(pool.clone(), scanner_config.clone(), cache.clone(),disable_startup_scan) - .into_diagnostic() - .wrap_err("Failed starting background scanner")?; + scanner::run_scanner( + pool.clone(), + scanner_config.clone(), + cache.clone(), + disable_startup_scan, + ) + .into_diagnostic() + .wrap_err("Failed starting background scanner")?; let addr = std::net::SocketAddr::from(([127, 0, 0, 1], port)); server::start(&addr, pool, server_config, scanner_config, cache)