diff --git a/entities/src/host.rs b/entities/src/host.rs index cb9e308..fd5355b 100644 --- a/entities/src/host.rs +++ b/entities/src/host.rs @@ -1,10 +1,7 @@ //! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 -use sea_orm::{ - entity::prelude::*, - sea_query::{Expr, SimpleExpr}, -}; -use serde::{Serialize, Deserialize}; +use sea_orm::entity::prelude::*; +use serde::Serialize; #[derive(Copy, Clone, Default, Debug, DeriveEntity)] pub struct Entity; @@ -32,7 +29,7 @@ pub struct Model { pub updated: i64, } -#[derive(Copy, Clone, Debug, PartialEq ,Eq, EnumIter, DeriveActiveEnum, Serialize)] +#[derive(Copy, Clone, Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, Serialize)] #[sea_orm(rs_type = "i32", db_type = "Integer")] pub enum Connectivity { #[sea_orm(num_value = 0)] diff --git a/entities/src/state/error_cache.rs b/entities/src/state/error_cache.rs index fe54d81..e192ec3 100644 --- a/entities/src/state/error_cache.rs +++ b/entities/src/state/error_cache.rs @@ -1,5 +1,3 @@ -use std::collections::VecDeque; - use chrono::Utc; use sea_orm::prelude::DateTimeUtc; use serde::Serialize; @@ -41,4 +39,4 @@ impl HostError { http_status: Some(http_status as _), } } -} \ No newline at end of file +} diff --git a/entities/src/state/mod.rs b/entities/src/state/mod.rs index f1b0f42..03c0e3b 100644 --- a/entities/src/state/mod.rs +++ b/entities/src/state/mod.rs @@ -10,9 +10,9 @@ use serde::Serialize; use crate::host::Connectivity; -pub mod scanner; /// Log for recent host errors pub mod error_cache; +pub mod scanner; pub type Cache = RwLock; @@ -62,7 +62,7 @@ pub struct CacheHost { /// Country from the wiki pub country: String, /// Last health checks time formatted, healthy - pub recent_checks: Vec<(String,bool)>, + pub recent_checks: Vec<(String, bool)>, /// Percentage of healthy checks since first seen pub healthy_percentage_overall: u8, pub connectivity: Option, diff --git a/migration/src/lib.rs b/migration/src/lib.rs index b8601d5..6824fd4 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -8,8 +8,6 @@ mod m20230829_201916_country; mod m20230914_231514_connectivity; mod m20231011_231223_errors; - - pub struct Migrator; #[async_trait::async_trait] diff --git a/migration/src/m20230729_010231_datetime_rowid.rs b/migration/src/m20230729_010231_datetime_rowid.rs index 196fcd8..1864b72 100644 --- a/migration/src/m20230729_010231_datetime_rowid.rs +++ b/migration/src/m20230729_010231_datetime_rowid.rs @@ -2,9 +2,7 @@ use std::borrow::Cow; use sea_orm_migration::{ prelude::*, - sea_orm::{ - prelude::DateTimeUtc, DbBackend, DeriveEntityModel, FromQueryResult, RuntimeErr, Statement, - }, + sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, RuntimeErr, Statement}, }; #[derive(DeriveMigrationName)] diff --git a/migration/src/m20230729_230909_datetime_int_host.rs b/migration/src/m20230729_230909_datetime_int_host.rs index 3842a65..0230410 100644 --- a/migration/src/m20230729_230909_datetime_int_host.rs +++ b/migration/src/m20230729_230909_datetime_int_host.rs @@ -90,4 +90,4 @@ impl MigrationTrait for Migration { async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { panic!("Can't migrate down"); } -} \ No newline at end of file +} diff --git a/migration/src/m20230803_154714_version_url.rs b/migration/src/m20230803_154714_version_url.rs index ac43e6e..a475906 100644 --- a/migration/src/m20230803_154714_version_url.rs +++ b/migration/src/m20230803_154714_version_url.rs @@ -1,7 +1,4 @@ -use sea_orm_migration::{ - prelude::*, - sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement}, -}; +use sea_orm_migration::prelude::*; #[derive(DeriveMigrationName)] pub struct Migration; diff --git a/migration/src/m20230829_201916_country.rs b/migration/src/m20230829_201916_country.rs index be6d128..8a2dd3d 100644 --- a/migration/src/m20230829_201916_country.rs +++ b/migration/src/m20230829_201916_country.rs @@ -19,4 +19,4 @@ impl MigrationTrait for Migration { async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { panic!("Can't migrate down"); } -} \ No newline at end of file +} diff --git a/migration/src/m20230914_231514_connectivity.rs b/migration/src/m20230914_231514_connectivity.rs index 3bde050..2190823 100644 --- a/migration/src/m20230914_231514_connectivity.rs +++ b/migration/src/m20230914_231514_connectivity.rs @@ -19,4 +19,4 @@ impl MigrationTrait for Migration { async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { panic!("Can't migrate down"); } -} \ No newline at end of file +} diff --git a/migration/src/m20231011_231223_errors.rs b/migration/src/m20231011_231223_errors.rs index 32b8df1..ceb36b1 100644 --- a/migration/src/m20231011_231223_errors.rs +++ b/migration/src/m20231011_231223_errors.rs @@ -27,4 +27,4 @@ impl MigrationTrait for Migration { async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { panic!("Can't migrate down"); } -} \ No newline at end of file +} diff --git a/scanner/src/about_parser.rs b/scanner/src/about_parser.rs index d4f0c7b..bed9c03 100644 --- a/scanner/src/about_parser.rs +++ b/scanner/src/about_parser.rs @@ -1,6 +1,5 @@ // SPDX-License-Identifier: AGPL-3.0-only use regex::{Regex, RegexBuilder}; -use reqwest::Url; use scraper::{Html, Selector}; use thiserror::Error; diff --git a/scanner/src/cache_update.rs b/scanner/src/cache_update.rs index 403c228..7d9eb9b 100644 --- a/scanner/src/cache_update.rs +++ b/scanner/src/cache_update.rs @@ -156,7 +156,7 @@ impl Scanner { is_bad_host, country: host.country, healthy_percentage_overall: healthy_percentage_total.remove(&host.id).unwrap_or(0), - recent_checks: self.query_latest_health_checks(22,host.id).await?, + recent_checks: self.query_latest_health_checks(22, host.id).await?, }) } host_statistics.sort_unstable_by_key(|k| k.points); @@ -318,19 +318,18 @@ impl Scanner { } /// Query total up percentage for all hosts - async fn query_healthy_percentage( - &self, - ) -> Result> { - let stats: Vec = HostHealthyPercentage::find_by_statement(Statement::from_sql_and_values( - DbBackend::Sqlite, - r#"SELECT u.host, CAST(AVG(healthy) * 100 as INT) as healthy FROM health_check u + async fn query_healthy_percentage(&self) -> Result> { + let stats: Vec = + HostHealthyPercentage::find_by_statement(Statement::from_sql_and_values( + DbBackend::Sqlite, + r#"SELECT u.host, CAST(AVG(healthy) * 100 as INT) as healthy FROM health_check u JOIN host h ON h.id = u.host WHERE h.enabled = true GROUP BY u.host"#, - [], - )) - .all(&self.inner.db) - .await?; + [], + )) + .all(&self.inner.db) + .await?; let stats: HashMap<_, _> = stats.into_iter().map(|v| (v.host, v.healthy)).collect(); Ok(stats) } @@ -341,28 +340,33 @@ impl Scanner { // How many to retrieve amount: i32, host: i32, - ) -> Result> { + ) -> Result> { #[derive(Debug, FromQueryResult)] pub struct HostHealthCheck { healthy: bool, time: i64, } - let health_checks: Vec = HostHealthCheck::find_by_statement(Statement::from_sql_and_values( - DbBackend::Sqlite, - r#"SELECT healthy, time FROM health_check u + let health_checks: Vec = + HostHealthCheck::find_by_statement(Statement::from_sql_and_values( + DbBackend::Sqlite, + r#"SELECT healthy, time FROM health_check u JOIN host h ON h.id = u.host WHERE h.enabled = true AND host = $1 ORDER BY time DESC LIMIT $2"#, - [host.into(), amount.into()], - )) - .all(&self.inner.db) - .await?; + [host.into(), amount.into()], + )) + .all(&self.inner.db) + .await?; // transform to correct time format - let health_checks: Vec<_> = health_checks.into_iter().rev().map(|entry|{ - let time = Utc.timestamp_opt(entry.time, 0).unwrap(); - (time.format("%Y.%m.%d %H:%M").to_string(),entry.healthy) - }).collect(); + let health_checks: Vec<_> = health_checks + .into_iter() + .rev() + .map(|entry| { + let time = Utc.timestamp_opt(entry.time, 0).unwrap(); + (time.format("%Y.%m.%d %H:%M").to_string(), entry.healthy) + }) + .collect(); Ok(health_checks) } } diff --git a/scanner/src/instance_check.rs b/scanner/src/instance_check.rs index 466bac5..5ae52db 100644 --- a/scanner/src/instance_check.rs +++ b/scanner/src/instance_check.rs @@ -1,14 +1,12 @@ // SPDX-License-Identifier: AGPL-3.0-only //! Instance health/uptime checking code -use std::net::IpAddr; use std::time::Instant; use chrono::Utc; -use entities::{health_check, check_errors}; -use entities::host::Connectivity; use entities::state::error_cache::HostError; +use entities::{check_errors, health_check}; use entities::{host, prelude::*}; -use reqwest::{Url, ClientBuilder}; +use reqwest::Url; use sea_orm::prelude::DateTimeUtc; use sea_orm::ColumnTrait; use sea_orm::EntityTrait; @@ -18,7 +16,6 @@ use tokio::task::JoinSet; use tracing::instrument; use crate::about_parser::AboutParsed; -use crate::FetchError; use crate::Result; use crate::Scanner; @@ -63,8 +60,13 @@ impl Scanner { if !muted { tracing::error!(error=?e, url=host.url,"failed to parse instance URL"); } - self.insert_failed_health_check(host.id, now,HostError::new_message(format!("Not a valid URL")), None) - .await; + self.insert_failed_health_check( + host.id, + now, + HostError::new_message(format!("Not a valid URL")), + None, + ) + .await; return; } Ok(v) => v, @@ -83,8 +85,13 @@ impl Scanner { "couldn't ping host: {e}, marking as dead" ); } - self.insert_failed_health_check(host.id, now, e.to_host_error(), Some(took_ms as _)) - .await; + self.insert_failed_health_check( + host.id, + now, + e.to_host_error(), + Some(took_ms as _), + ) + .await; } Ok((http_code, content)) => { if !muted { @@ -121,7 +128,11 @@ impl Scanner { self.insert_failed_health_check( host.id, now, - HostError::new(format!("profile content mismatch"), content, http_code), + HostError::new( + format!("profile content mismatch"), + content, + http_code, + ), Some(took_ms as _), ) .await; diff --git a/scanner/src/instance_parser.rs b/scanner/src/instance_parser.rs index 265fdc6..8457b41 100644 --- a/scanner/src/instance_parser.rs +++ b/scanner/src/instance_parser.rs @@ -196,10 +196,10 @@ impl InstanceParser { #[cfg(test)] mod test { - use std::{collections::HashMap, io::BufWriter}; - use std::io::Write; - use tracing_test::traced_test; use csv; + use std::io::Write; + use std::{collections::HashMap, io::BufWriter}; + use tracing_test::traced_test; use super::*; #[test] @@ -208,11 +208,10 @@ mod test { let html = include_str!("../test_data/instancelist.html"); let parser = InstanceParser::new(); let res = parser.parse_instancelist(html, &[], "", true).unwrap(); - + // writeback for new tests // write_data(res.values()); - // adjust when updating the test html let expected: HashMap = expected_data() .into_iter() diff --git a/scanner/src/lib.rs b/scanner/src/lib.rs index ee4dac5..01ff573 100644 --- a/scanner/src/lib.rs +++ b/scanner/src/lib.rs @@ -1,7 +1,8 @@ // SPDX-License-Identifier: AGPL-3.0-only use std::{ + net::IpAddr, sync::{Arc, Mutex}, - time::Instant, net::IpAddr, + time::Instant, }; use about_parser::AboutParser; @@ -9,14 +10,14 @@ use chrono::Utc; use entities::{ host, prelude::*, - state::{scanner::ScannerConfig, Cache, AppState, error_cache::HostError}, + state::{error_cache::HostError, scanner::ScannerConfig, AppState}, }; use instance_parser::InstanceParser; use profile_parser::ProfileParser; use regex::{Regex, RegexBuilder}; use reqwest::{ header::{HeaderMap, HeaderValue}, - Client, Url, ClientBuilder, + Client, ClientBuilder, Url, }; use sea_orm::{ sea_query::OnConflict, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, @@ -94,9 +95,15 @@ impl FetchError { fn to_host_error(self) -> HostError { match self { FetchError::Reqwest(e) => HostError::new_message(e.to_string()), - FetchError::HttpResponseStatus(http_status, _code_msg, http_body) => HostError::new(format!("failed to fetch"), http_body, http_status), - FetchError::KnownHttpResponseStatus(http_status, _) => HostError::new_without_body(self.to_string(), http_status), - FetchError::RetrievingBody(_url, reqwest_error) => HostError::new_message(reqwest_error.to_string()), + FetchError::HttpResponseStatus(http_status, _code_msg, http_body) => { + HostError::new(format!("failed to fetch"), http_body, http_status) + } + FetchError::KnownHttpResponseStatus(http_status, _) => { + HostError::new_without_body(self.to_string(), http_status) + } + FetchError::RetrievingBody(_url, reqwest_error) => { + HostError::new_message(reqwest_error.to_string()) + } FetchError::Captcha => HostError::new_message(format!("Captcha detected")), } } @@ -150,7 +157,7 @@ impl Scanner { for header in HEADERS { headers.insert(header[0], HeaderValue::from_static(header[1])); } - let user_agent = format!("nitter-status (+{}/about)",config.website_url); + let user_agent = format!("nitter-status (+{}/about)", config.website_url); let http_client = Client::builder() .cookie_store(true) .brotli(true) @@ -164,12 +171,22 @@ impl Scanner { http_client } - async fn new(db: DatabaseConnection, config: ScannerConfig, app_state: AppState) -> Result { + async fn new( + db: DatabaseConnection, + config: ScannerConfig, + app_state: AppState, + ) -> Result { let mut builder_regex_rss = RegexBuilder::new(&config.rss_content); builder_regex_rss.case_insensitive(true); let http_client = Self::client_builder(&config); - let client_ipv4 = Scanner::client_builder(&config).local_address("0.0.0.0".parse::().unwrap()).build().unwrap(); - let client_ipv6 = Scanner::client_builder(&config).local_address("::".parse::().unwrap()).build().unwrap(); + let client_ipv4 = Scanner::client_builder(&config) + .local_address("0.0.0.0".parse::().unwrap()) + .build() + .unwrap(); + let client_ipv6 = Scanner::client_builder(&config) + .local_address("::".parse::().unwrap()) + .build() + .unwrap(); let scanner = Self { inner: Arc::new(InnerScanner { db, @@ -305,7 +322,12 @@ impl Scanner { let connectivity = scanner_c.check_connectivity(url.as_str()).await; let rss = scanner_c.has_rss(&mut url, muted_host).await; match scanner_c.nitter_version(&mut url, muted_host).await { - Some(version) => (connectivity, rss, Some(version.version_name), Some(version.url)), + Some(version) => ( + connectivity, + rss, + Some(version.version_name), + Some(version.url), + ), None => (connectivity, rss, None, None), } } @@ -366,18 +388,26 @@ impl Scanner { /// Check ipv4/6 connectivity of host async fn check_connectivity(&self, url: &str) -> Option { - let ipv4 = self.inner.client_ipv4.get(url).send().await.map_or(false, |res|{ - res.status().is_success() - }); - let ipv6 = self.inner.client_ipv6.get(url).send().await.map_or(false, |res|{ - res.status().is_success() - }); + let ipv4 = self + .inner + .client_ipv4 + .get(url) + .send() + .await + .map_or(false, |res| res.status().is_success()); + let ipv6 = self + .inner + .client_ipv6 + .get(url) + .send() + .await + .map_or(false, |res| res.status().is_success()); - match (ipv4,ipv6) { - (true,true) => Some(host::Connectivity::All), - (false,true) => Some(host::Connectivity::IPv6), - (true,false) => Some(host::Connectivity::IPv4), - (false,false) => None, + match (ipv4, ipv6) { + (true, true) => Some(host::Connectivity::All), + (false, true) => Some(host::Connectivity::IPv6), + (true, false) => Some(host::Connectivity::IPv4), + (false, false) => None, } } @@ -457,11 +487,11 @@ mod test { use super::*; use chrono::Duration; + use entities::health_check; use entities::state::scanner::Config; use migration::MigratorTrait; use sea_orm::{ConnectOptions, Database}; use tokio::{fs::File, io::AsyncWriteExt}; - use entities::health_check; async fn db_init() -> DatabaseConnection { let db = Database::connect(ConnectOptions::new( @@ -469,8 +499,7 @@ mod test { )) .await .unwrap(); - migration::Migrator::up(&db, None) - .await.unwrap(); + migration::Migrator::up(&db, None).await.unwrap(); db } @@ -482,7 +511,7 @@ mod test { let db = db_init().await; let time: chrono::DateTime = Utc::now(); for v in 0..365 { - let entry_time = time.checked_sub_signed(Duration::minutes(15*v)).unwrap(); + let entry_time = time.checked_sub_signed(Duration::minutes(15 * v)).unwrap(); health_check::ActiveModel { time: ActiveValue::Set(entry_time.timestamp()), host: ActiveValue::Set(208692), @@ -491,7 +520,8 @@ mod test { response_code: ActiveValue::Set(Some(200)), } .insert(&db) - .await.unwrap(); + .await + .unwrap(); } } @@ -499,7 +529,9 @@ mod test { #[ignore] async fn update_instacelist() { let db = db_init().await; - let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap(); + let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()) + .await + .unwrap(); let res = scanner.fetch_instancelist().await.unwrap(); let mut file = File::create("test_data/instancelist.html").await.unwrap(); file.write_all(&res.as_bytes()).await.unwrap(); @@ -509,7 +541,9 @@ mod test { #[ignore] async fn fetch_test() { let db = db_init().await; - let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap(); + let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()) + .await + .unwrap(); let (_, res) = scanner.fetch_url("example.com/jack").await.unwrap(); let mut file = File::create("test_data/blocked.html").await.unwrap(); file.write_all(&res.as_bytes()).await.unwrap(); @@ -519,7 +553,9 @@ mod test { #[ignore] async fn stats_test() { let db = db_init().await; - let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap(); + let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()) + .await + .unwrap(); dbg!(scanner.generate_cache_data().await.unwrap()); } @@ -527,10 +563,21 @@ mod test { #[ignore] async fn connectivity_test() { let db = db_init().await; - let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap(); - assert_eq!(scanner.check_connectivity("https://v4.ipv6test.app").await,Some(host::Connectivity::IPv4)); - assert_eq!(scanner.check_connectivity("https://ipv6test.app").await,Some(host::Connectivity::All)); - assert_eq!(scanner.check_connectivity("https://v6.ipv6test.app").await,Some(host::Connectivity::IPv6)); + let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()) + .await + .unwrap(); + assert_eq!( + scanner.check_connectivity("https://v4.ipv6test.app").await, + Some(host::Connectivity::IPv4) + ); + assert_eq!( + scanner.check_connectivity("https://ipv6test.app").await, + Some(host::Connectivity::All) + ); + assert_eq!( + scanner.check_connectivity("https://v6.ipv6test.app").await, + Some(host::Connectivity::IPv6) + ); } } diff --git a/scanner/src/profile_parser.rs b/scanner/src/profile_parser.rs index ad868d6..a7b44a2 100644 --- a/scanner/src/profile_parser.rs +++ b/scanner/src/profile_parser.rs @@ -1,7 +1,5 @@ // SPDX-License-Identifier: AGPL-3.0-only //! Parse profile pages for verification -use regex::{Regex, RegexBuilder}; -use reqwest::Url; use scraper::{Html, Selector}; use thiserror::Error; @@ -15,15 +13,12 @@ pub enum ProfileParseError { NoProfileCard, #[error("No timeline div found!")] NoTimeline, - #[error("No timeline-item div found!")] - NoTimelineItem, } pub(crate) struct ProfileParser { selector_profile_card_name: Selector, selector_timeline: Selector, selector_timeline_item: Selector, - regex: Regex, } #[derive(Debug)] @@ -60,14 +55,11 @@ impl ProfileParser { } pub fn new() -> Self { - let mut builder = RegexBuilder::new(r#"^((\d+\.\d+\.\d+)|[a-zA-Z0-9]{7,})"#); - builder.case_insensitive(true); Self { selector_profile_card_name: Selector::parse(".profile-card-username") .expect(EXPECT_CSS_SELCTOR), selector_timeline: Selector::parse(".timeline").expect(EXPECT_CSS_SELCTOR), selector_timeline_item: Selector::parse(".timeline-item").expect(EXPECT_CSS_SELCTOR), - regex: builder.build().expect("failed to generate regex"), } } } diff --git a/server/src/admin.rs b/server/src/admin.rs index 1e5f51e..01b4363 100644 --- a/server/src/admin.rs +++ b/server/src/admin.rs @@ -1,48 +1,48 @@ use std::collections::HashSet; use std::sync::Arc; -use axum::Form; -use axum::Json; use axum::extract::Path; use axum::extract::State; use axum::response::Html; use axum::response::IntoResponse; use axum::response::Redirect; +use axum::Form; +use axum::Json; +use constant_time_eq::constant_time_eq; use entities::check_errors; use entities::health_check; +use entities::host; use entities::state::AppState; +use hyper::header::REFERER; use hyper::HeaderMap; use hyper::StatusCode; -use hyper::header::REFERER; use reqwest::Client; use reqwest::Url; +use sea_orm::ColumnTrait; use sea_orm::DatabaseConnection; use sea_orm::EntityTrait; +use sea_orm::QueryFilter; use sea_orm::QueryOrder; use sea_orm::QuerySelect; use serde::Deserialize; use serde::Serialize; +use sha2::{Digest, Sha256}; use thiserror::Error; use tower_sessions::Session; -use constant_time_eq::constant_time_eq; -use sha2::{Sha256, Digest}; -use entities::host; -use sea_orm::ColumnTrait; -use sea_orm::QueryFilter; -use trust_dns_resolver::AsyncResolver; use trust_dns_resolver::config::ResolverConfig; use trust_dns_resolver::config::ResolverOpts; +use trust_dns_resolver::AsyncResolver; -use crate::ADMIN_OVERVIEW_URL; use crate::Config; -use crate::LOGIN_URL; use crate::Result; use crate::ServerError; +use crate::ADMIN_OVERVIEW_URL; +use crate::LOGIN_URL; #[derive(Serialize, Deserialize, Default)] pub struct ActiveLogin { /// Hosts this session has access to. - hosts: HashSet + hosts: HashSet, } const LOGIN_KEY: &'static str = "LOGIN"; @@ -53,7 +53,7 @@ pub enum LoginError { #[error("Invalid instance URL '{0}'")] InstanceUrl(String), #[error("Invalid response at '{0}'")] - InvalidResponse(Url,String), + InvalidResponse(Url, String), #[error("Key mismatch")] KeyMismatch, #[error("No public instance host found with domain '{0}'")] @@ -80,34 +80,41 @@ pub async fn login( State(ref login_client): State, State(ref db): State, session: Session, - Form(input): Form + Form(input): Form, ) -> Result { tracing::debug!(login=?input); - let host = host::Entity::find().filter(host::Column::Domain.eq(input.domain.trim())) - .one(db) - .await?; + let host = host::Entity::find() + .filter(host::Column::Domain.eq(input.domain.trim())) + .one(db) + .await?; - if host.as_ref().map(|host|get_session_login(&session).map(|login|login.hosts.contains(&host.id)).is_ok()).unwrap_or_default() { + if host + .as_ref() + .map(|host| { + get_session_login(&session) + .map(|login| login.hosts.contains(&host.id)) + .is_ok() + }) + .unwrap_or_default() + { // already logged in return Ok(Redirect::temporary(&input.referrer).into_response()); } - match login_inner(config, login_client, &input, host).await { + match login_inner(config, login_client, &input, host).await { Ok(host) => { let session_value = match get_session_login(&session) { Ok(mut session) => { session.hosts.insert(host.id); session - }, + } Err(_) => { let mut ids = HashSet::with_capacity(1); ids.insert(host.id); - ActiveLogin { - hosts: ids, - } + ActiveLogin { hosts: ids } } }; - session.insert(LOGIN_KEY,session_value)?; + session.insert(LOGIN_KEY, session_value)?; let referrer = input.referrer.trim(); let location = match referrer.trim().is_empty() && referrer != LOGIN_URL { true => ADMIN_OVERVIEW_URL, @@ -116,7 +123,7 @@ pub async fn login( let mut res = Redirect::temporary(location).into_response(); *res.status_mut() = StatusCode::FOUND; Ok(res) - }, + } Err(e) => { tracing::debug!(login_error=?e); let mut context = tera::Context::new(); @@ -127,15 +134,15 @@ pub async fn login( context.insert("VERIFY_TOKEN_NAME", &config.login_token_name); match e { LoginError::InvalidResponse(_, val) - | LoginError::ServerResponse(_,val) - | LoginError::DNSNoValidEntry(val) - | LoginError::InvalidHash(val) => context.insert("QUOTE", &val), - _ => (), + | LoginError::ServerResponse(_, val) + | LoginError::DNSNoValidEntry(val) + | LoginError::InvalidHash(val) => context.insert("QUOTE", &val), + _ => (), } let mut res = Html(template.render("login.html.j2", &context)?).into_response(); *res.status_mut() = StatusCode::FORBIDDEN; Ok(res.into_response()) - }, + } } } @@ -145,8 +152,8 @@ async fn login_inner( input: &LoginInput, host: Option, ) -> LoginResult { - let host = host.ok_or_else(||LoginError::HostNotFound(input.domain.clone()))?; - + let host = host.ok_or_else(|| LoginError::HostNotFound(input.domain.clone()))?; + match input.verification_method { VerificationMethod::DNS => { let entries = fetch_host_dns(&host.domain, &config).await?; @@ -156,10 +163,10 @@ async fn login_inner( } } Err(LoginError::DNSNoValidEntry(entries.join(","))) - }, + } VerificationMethod::HTTP => { let fetched_key = fetch_host_txt(&host.url, login_client, &config).await?; - verify_key(&fetched_key, &input.key).map(|_|host) + verify_key(&fetched_key, &input.key).map(|_| host) } } } @@ -169,30 +176,45 @@ pub struct LoginInput { domain: String, key: String, referrer: String, - verification_method: VerificationMethod + verification_method: VerificationMethod, } #[derive(Serialize, Deserialize, Debug)] enum VerificationMethod { DNS, - HTTP + HTTP, } -async fn fetch_host_txt(instance_url: &str, client: &Client, config: &Config) -> LoginResult { - let mut request_url = Url::parse(&instance_url).map_err(|_|LoginError::InstanceUrl(instance_url.to_string()))?; - request_url.set_path(&[".",&config.login_token_name].concat()); +async fn fetch_host_txt( + instance_url: &str, + client: &Client, + config: &Config, +) -> LoginResult { + let mut request_url = + Url::parse(&instance_url).map_err(|_| LoginError::InstanceUrl(instance_url.to_string()))?; + request_url.set_path(&[".", &config.login_token_name].concat()); request_url.set_query(None); - let result = client.get(request_url.clone()).send().await.map_err(|e|LoginError::HttpFailure(request_url.clone(), e))?; - match true { // result.status().is_success() - true => Ok(result.text().await.map_err(|e: reqwest::Error|LoginError::InvalidResponse(request_url,e.to_string()))?), - false => Err(LoginError::ServerResponse(result.status().as_u16(),result.text().await.unwrap_or_default())), + let result = client + .get(request_url.clone()) + .send() + .await + .map_err(|e| LoginError::HttpFailure(request_url.clone(), e))?; + match true { + // result.status().is_success() + true => Ok(result.text().await.map_err(|e: reqwest::Error| { + LoginError::InvalidResponse(request_url, e.to_string()) + })?), + false => Err(LoginError::ServerResponse( + result.status().as_u16(), + result.text().await.unwrap_or_default(), + )), } } async fn fetch_host_dns(instance_domain: &str, config: &Config) -> LoginResult> { // TODO: cache resolver ? let resolver = AsyncResolver::tokio(ResolverConfig::cloudflare_tls(), ResolverOpts::default()); let hashed_key = resolver - .txt_lookup(format!("{}.{}.",&config.login_token_name, instance_domain)) + .txt_lookup(format!("{}.{}.", &config.login_token_name, instance_domain)) .await?; let mut entries = Vec::with_capacity(2); @@ -212,14 +234,14 @@ fn verify_key(hash: &str, key: &str) -> LoginResult<()> { let mut hasher: Sha256 = Sha256::new(); hasher.update(key); let res = hasher.finalize(); - let mut decoded_hash = [0;32]; - base16ct::mixed::decode(hash, &mut decoded_hash).map_err(|e|{ + let mut decoded_hash = [0; 32]; + base16ct::mixed::decode(hash, &mut decoded_hash).map_err(|e| { tracing::trace!(hash_error=?e); LoginError::InvalidHash(hash.to_owned()) })?; - match constant_time_eq(&res,&decoded_hash) { + match constant_time_eq(&res, &decoded_hash) { false => Err(LoginError::KeyMismatch), - true => Ok(()) + true => Ok(()), } } @@ -229,20 +251,19 @@ pub async fn login_view( headers: HeaderMap, ) -> Result { tracing::debug!(headers=?headers); - let referrer = headers.get(REFERER).and_then(|v|v.to_str().ok()); + let referrer = headers.get(REFERER).and_then(|v| v.to_str().ok()); let mut context = tera::Context::new(); - context.insert("REFERRER", &referrer);// FIXME: won't work, handle this in the error part to extract the current situation + context.insert("REFERRER", &referrer); // FIXME: won't work, handle this in the error part to extract the current situation context.insert("VERIFY_TOKEN_NAME", &config.login_token_name); let res = Html(template.render("login.html.j2", &context)?).into_response(); Ok(res) } - pub async fn overview( State(ref app_state): State, State(ref template): State>, State(ref db): State, - session: Session + session: Session, ) -> Result { tracing::info!(?session); @@ -250,11 +271,14 @@ pub async fn overview( let mut context = tera::Context::new(); let res = { - let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?; + let guard = app_state + .cache + .read() + .map_err(|_| ServerError::MutexFailure)?; let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string(); context.insert("last_updated", &time); context.insert("instances", &hosts); - + let res = Html(template.render("admin.html.j2", &context)?).into_response(); drop(guard); res @@ -265,12 +289,16 @@ pub async fn overview( pub async fn history_json( State(ref db): State, Path(host): Path, - session: Session + session: Session, ) -> Result { - let host = get_specific_login_host(host,&session,db).await?; + let host = get_specific_login_host(host, &session, db).await?; - let history: Vec = health_check::Entity::find().filter(health_check::Column::Host.eq(host.id)) - .order_by_asc(health_check::Column::Time).limit(20).all(db).await?; + let history: Vec = health_check::Entity::find() + .filter(health_check::Column::Host.eq(host.id)) + .order_by_asc(health_check::Column::Time) + .limit(20) + .all(db) + .await?; Ok(Json(history).into_response()) } @@ -280,19 +308,22 @@ pub async fn history_view( State(ref template): State>, State(ref db): State, Path(host): Path, - session: Session + session: Session, ) -> Result { tracing::info!(?session); - let host = get_specific_login_host(host,&session,db).await?; + let host = get_specific_login_host(host, &session, db).await?; let mut context = tera::Context::new(); let res = { - let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?; + let guard = app_state + .cache + .read() + .map_err(|_| ServerError::MutexFailure)?; let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string(); context.insert("last_updated", &time); context.insert("HOST_DOMAIN", &host.domain); - + let res = Html(template.render("errors_admin.html.j2", &context)?).into_response(); drop(guard); res @@ -305,23 +336,30 @@ pub async fn errors_view( State(ref template): State>, State(ref db): State, Path(host): Path, - session: Session + session: Session, ) -> Result { tracing::info!(?session); - let host = get_specific_login_host(host,&session,db).await?; + let host = get_specific_login_host(host, &session, db).await?; - let errors = check_errors::Entity::find().filter(check_errors::Column::Host.eq(host.id)) - .order_by_desc(check_errors::Column::Time).limit(20).all(db).await?; + let errors = check_errors::Entity::find() + .filter(check_errors::Column::Host.eq(host.id)) + .order_by_desc(check_errors::Column::Time) + .limit(20) + .all(db) + .await?; let mut context = tera::Context::new(); let res = { - let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?; + let guard = app_state + .cache + .read() + .map_err(|_| ServerError::MutexFailure)?; let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string(); context.insert("last_updated", &time); context.insert("ERRORS", &errors); context.insert("HOST_DOMAIN", &host.domain); - + let res = Html(template.render("errors_admin.html.j2", &context)?).into_response(); drop(guard); res @@ -330,28 +368,40 @@ pub async fn errors_view( } /// Get all [host::Model] for current [Session] -async fn get_all_login_hosts(session: &Session, db: &DatabaseConnection) -> Result> { +async fn get_all_login_hosts( + session: &Session, + db: &DatabaseConnection, +) -> Result> { let login = get_session_login(&session)?; - let host_res = host::Entity::find().filter(host::Column::Id.is_in(login.hosts.iter().map(|v|*v))) - .all(db).await?; + let host_res = host::Entity::find() + .filter(host::Column::Id.is_in(login.hosts.iter().map(|v| *v))) + .all(db) + .await?; Ok(host_res) } /// Get wanted [host::Model] for current [Session] if valid for this user -async fn get_specific_login_host(wanted_host_id: i32, session: &Session, db: &DatabaseConnection) -> Result { +async fn get_specific_login_host( + wanted_host_id: i32, + session: &Session, + db: &DatabaseConnection, +) -> Result { let login = get_session_login(&session)?; if !login.hosts.contains(&wanted_host_id) { return Err(ServerError::MissingPermission); } - let host_res = host::Entity::find().filter(host::Column::Id.eq(wanted_host_id)).one(db).await?; + let host_res = host::Entity::find() + .filter(host::Column::Id.eq(wanted_host_id)) + .one(db) + .await?; match host_res { None => { session.delete(); return Err(ServerError::HostNotFound(wanted_host_id)); - }, + } Some(host) => Ok(host), } } @@ -359,9 +409,9 @@ async fn get_specific_login_host(wanted_host_id: i32, session: &Session, db: &Da /// Check for valid session and return the stored [Login] data fn get_session_login(session: &Session) -> Result { if session.active() { - if let Some(u) = session.get(LOGIN_KEY).map_err(|_|ServerError::NoLogin)? { + if let Some(u) = session.get(LOGIN_KEY).map_err(|_| ServerError::NoLogin)? { return Ok(u); } } Err(ServerError::NoLogin) -} \ No newline at end of file +} diff --git a/server/src/api.rs b/server/src/api.rs index 8410924..fb500fe 100644 --- a/server/src/api.rs +++ b/server/src/api.rs @@ -2,7 +2,7 @@ use crate::{Result, ServerError}; use axum::response::IntoResponse; use axum::{extract::State, Json}; -use entities::state::{Cache, AppState}; +use entities::state::AppState; use hyper::http::HeaderValue; use std::sync::Arc; @@ -11,7 +11,10 @@ pub async fn instances( State(ref config): State>, ) -> Result { let mut res = { - let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?; + let guard = app_state + .cache + .read() + .map_err(|_| ServerError::MutexFailure)?; let res = Json(&*guard).into_response(); drop(guard); res @@ -20,6 +23,9 @@ pub async fn instances( "cache-control", HeaderValue::from_str(&format!("public, max-age={}", config.max_age)).unwrap(), ); - res.headers_mut().insert("X-Robots-Tag", HeaderValue::from_static("noindex, nofollow")); + res.headers_mut().insert( + "X-Robots-Tag", + HeaderValue::from_static("noindex, nofollow"), + ); Ok(res) } diff --git a/server/src/lib.rs b/server/src/lib.rs index 7a49984..012d10e 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1,7 +1,14 @@ // SPDX-License-Identifier: AGPL-3.0-only use std::{borrow::Cow, net::SocketAddr, sync::Arc}; -use axum::{extract::DefaultBodyLimit, http::HeaderValue, response::{Html, Redirect}, routing::{get, get_service}, Router, error_handling::HandleErrorLayer}; +use axum::{ + error_handling::HandleErrorLayer, + extract::DefaultBodyLimit, + http::HeaderValue, + response::{Html, Redirect}, + routing::{get, get_service}, + Router, +}; use entities::state::{scanner::ScannerConfig, AppState}; use hyper::{header, StatusCode}; use reqwest::Client; @@ -10,14 +17,17 @@ use tera::Tera; use thiserror::Error; use tower::ServiceBuilder; use tower_http::{ - cors::CorsLayer, limit::RequestBodyLimitLayer, services::{ServeDir, ServeFile}, - set_header::SetResponseHeaderLayer, trace::TraceLayer, + cors::CorsLayer, + limit::RequestBodyLimitLayer, + services::{ServeDir, ServeFile}, + set_header::SetResponseHeaderLayer, + trace::TraceLayer, }; -use tower_sessions::{SessionManagerLayer, cookie::SameSite}; +use tower_sessions::{cookie::SameSite, SessionManagerLayer}; +mod admin; mod api; mod website; -mod admin; const LOGIN_URL: &'static str = "/admin/login"; const ADMIN_OVERVIEW_URL: &'static str = "/admin"; @@ -70,8 +80,8 @@ pub async fn start( .with_same_site(SameSite::Strict) .with_max_age(time::Duration::seconds(config.session_ttl_seconds as _)), ); - - let user_agent = format!("nitter-status (+{}/about)",scanner_config.website_url); + + let user_agent = format!("nitter-status (+{}/about)", scanner_config.website_url); let login_client = Client::builder() .cookie_store(false) .brotli(true) @@ -81,7 +91,8 @@ pub async fn start( .user_agent(user_agent) .connect_timeout(std::time::Duration::from_secs(3)) .timeout(std::time::Duration::from_secs(10)) - .build().unwrap(); + .build() + .unwrap(); let config = Arc::new(config); let mut tera = Tera::new("server/templates/*")?; @@ -172,14 +183,15 @@ impl axum::response::IntoResponse for ServerError { fn into_response(self) -> axum::response::Response { use ServerError::*; let msg = match &self { - NoLogin => { + NoLogin => { let mut resp = Redirect::temporary(LOGIN_URL).into_response(); // *resp.status_mut() = StatusCode::FOUND; // have to use a 301, [Redirect] 307 won't work for referrer return resp; - }, - MissingPermission => - (StatusCode::FORBIDDEN,Cow::Borrowed("Missing permission to access this resource")) - , + } + MissingPermission => ( + StatusCode::FORBIDDEN, + Cow::Borrowed("Missing permission to access this resource"), + ), MutexFailure | Templating(_) | DBError(_) | SessionError(_) | HostNotFound(_) => ( StatusCode::INTERNAL_SERVER_ERROR, Cow::Borrowed("Internal Server Error"), diff --git a/server/src/website.rs b/server/src/website.rs index dcfaeb8..0503222 100644 --- a/server/src/website.rs +++ b/server/src/website.rs @@ -6,9 +6,8 @@ use crate::Result; use crate::ServerError; use axum::response::IntoResponse; use axum::{extract::State, response::Html}; -use entities::state::AppState; use entities::state::scanner::ScannerConfig; -use entities::state::Cache; +use entities::state::AppState; use hyper::http::HeaderValue; pub async fn instances( @@ -18,7 +17,10 @@ pub async fn instances( ) -> Result { let mut context = tera::Context::new(); let mut res = { - let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?; + let guard = app_state + .cache + .read() + .map_err(|_| ServerError::MutexFailure)?; context.insert("instances", &guard.hosts); let time = guard.last_update.format("%Y.%m.%d %H:%M").to_string(); context.insert("last_updated", &time); @@ -61,7 +63,10 @@ pub async fn about( &scanner_config.ping_range.num_hours(), ); { - let guard = app_state.cache.read().map_err(|_| ServerError::MutexFailure)?; + let guard = app_state + .cache + .read() + .map_err(|_| ServerError::MutexFailure)?; context.insert("latest_commit", &guard.latest_commit); } diff --git a/src/main.rs b/src/main.rs index 104e751..4dac232 100644 --- a/src/main.rs +++ b/src/main.rs @@ -82,7 +82,8 @@ async fn _main() -> miette::Result<()> { scanner_config.clone(), cache.clone(), disable_startup_scan, - ).await + ) + .await .into_diagnostic() .wrap_err("Failed starting background scanner")?; @@ -115,12 +116,10 @@ fn read_scanner_cfg() -> miette::Result { let profile_posts_min = require_env_str("PROFILE_POSTS_MIN")? .parse() .expect("PROFILE_POSTS_MIN must be a positive number"); - let rss_content = require_env_str("RSS_CONTENT")?; let additional_hosts: Vec = require_env_vec_str("ADDITIONAL_HOSTS")?; let additional_host_country = require_env_str("ADDITIONAL_HOSTS_COUNTRY")?; let rss_content = require_env_str("RSS_CONTENT")?; let bad_hosts: Vec = require_env_vec_str("BAD_HOSTS")?; - let referrer = require_env_str("REFERER")?; let auto_mute = require_env_str("AUTO_MUTE")? == "true"; let source_git_branch = require_env_str("ORIGIN_SOURCE_GIT_BRANCH")?; let source_git_url = require_env_str("ORIGIN_SOURCE_GIT_URL")?; @@ -164,8 +163,8 @@ async fn test_init(db: &DatabaseConnection) -> miette::Result<()> { fn read_server_config(instance_ping_interval: usize) -> miette::Result { let site_url = require_env_str("SITE_URL")?; let session_ttl_seconds = require_env_str("SESSION_TTL_SECONDS")? - .parse() - .expect("SESSION_TTL_SECONDS must be a positive number"); + .parse() + .expect("SESSION_TTL_SECONDS must be a positive number"); let login_token_name = require_env_str("LOGIN_TOKEN_NAME")?; Ok(server::Config {