From 280b95523606d0dfaef1d0ac46521e225fa16e4d Mon Sep 17 00:00:00 2001 From: 4cuck <40539855+4cuck@users.noreply.github.com> Date: Fri, 15 Sep 2023 01:23:00 +0200 Subject: [PATCH] ipv detection --- Cargo.lock | 1 + entities/src/host.rs | 16 ++++- entities/src/state/mod.rs | 3 + migration/src/lib.rs | 2 + .../src/m20230914_231514_connectivity.rs | 22 +++++++ scanner/Cargo.toml | 3 + scanner/src/cache_update.rs | 1 + scanner/src/instance_check.rs | 4 +- scanner/src/lib.rs | 65 ++++++++++++++++--- server/templates/instances.html.j2 | 3 + 10 files changed, 108 insertions(+), 12 deletions(-) create mode 100644 migration/src/m20230914_231514_connectivity.rs diff --git a/Cargo.lock b/Cargo.lock index 0ac8bdb..0fb2403 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2777,6 +2777,7 @@ dependencies = [ "csv", "entities", "git2", + "migration", "regex", "reqwest", "scraper", diff --git a/entities/src/host.rs b/entities/src/host.rs index 7cf4c64..cb9e308 100644 --- a/entities/src/host.rs +++ b/entities/src/host.rs @@ -4,7 +4,7 @@ use sea_orm::{ entity::prelude::*, sea_query::{Expr, SimpleExpr}, }; -use serde::Serialize; +use serde::{Serialize, Deserialize}; #[derive(Copy, Clone, Default, Debug, DeriveEntity)] pub struct Entity; @@ -27,10 +27,22 @@ pub struct Model { pub version: Option, pub country: String, pub version_url: Option, + pub connectivity: Option, /// Last time the url and enabled were updated, *not* the rss pub updated: i64, } +#[derive(Copy, Clone, Debug, PartialEq ,Eq, EnumIter, DeriveActiveEnum, Serialize)] +#[sea_orm(rs_type = "i32", db_type = "Integer")] +pub enum Connectivity { + #[sea_orm(num_value = 0)] + All = 0, + #[sea_orm(num_value = 1)] + IPv4 = 1, + #[sea_orm(num_value = 2)] + IPv6 = 2, +} + #[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)] pub enum Column { Id, @@ -40,6 +52,7 @@ pub enum Column { Country, VersionUrl, Enabled, + Connectivity, Rss, Updated, } @@ -75,6 +88,7 @@ impl ColumnTrait for Column { Self::Enabled => ColumnType::Integer.def(), Self::Rss => ColumnType::Integer.def(), Self::Updated => ColumnType::Integer.def(), + Self::Connectivity => ColumnType::Integer.def().null(), } } diff --git a/entities/src/state/mod.rs b/entities/src/state/mod.rs index a1adf2c..596beb4 100644 --- a/entities/src/state/mod.rs +++ b/entities/src/state/mod.rs @@ -8,6 +8,8 @@ use chrono::Utc; use sea_orm::prelude::DateTimeUtc; use serde::Serialize; +use crate::host::Connectivity; + pub mod scanner; pub type Cache = Arc>; @@ -53,4 +55,5 @@ pub struct CacheHost { pub recent_checks: Vec<(String,bool)>, /// Percentage of healthy checks since first seen pub healthy_percentage_overall: u8, + pub connectivity: Option, } diff --git a/migration/src/lib.rs b/migration/src/lib.rs index f42fa54..3825445 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -5,6 +5,7 @@ mod m20230729_010231_datetime_rowid; mod m20230729_230909_datetime_int_host; mod m20230803_154714_version_url; mod m20230829_201916_country; +mod m20230914_231514_connectivity; pub struct Migrator; @@ -18,6 +19,7 @@ impl MigratorTrait for Migrator { Box::new(m20230729_230909_datetime_int_host::Migration), Box::new(m20230803_154714_version_url::Migration), Box::new(m20230829_201916_country::Migration), + Box::new(m20230914_231514_connectivity::Migration), ] } } diff --git a/migration/src/m20230914_231514_connectivity.rs b/migration/src/m20230914_231514_connectivity.rs new file mode 100644 index 0000000..3bde050 --- /dev/null +++ b/migration/src/m20230914_231514_connectivity.rs @@ -0,0 +1,22 @@ +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let cmd = r#"ALTER TABLE "host" ADD COLUMN "connectivity" INT;"#; + let db = manager.get_connection(); + db.execute_unprepared("BEGIN EXCLUSIVE").await?; + tracing::info!("adding connectivity column.."); + db.execute_unprepared(cmd).await?; + db.execute_unprepared("COMMIT TRANSACTION").await?; + db.execute_unprepared("VACUUM").await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + panic!("Can't migrate down"); + } +} \ No newline at end of file diff --git a/scanner/Cargo.toml b/scanner/Cargo.toml index 26f7ac3..1590748 100644 --- a/scanner/Cargo.toml +++ b/scanner/Cargo.toml @@ -22,5 +22,8 @@ serde = { workspace = true, features = ["derive"] } tracing-test = { workspace = true } csv = "1.2.2" +[dev-dependencies.migration] +path = "../migration" + [dependencies.entities] path = "../entities" \ No newline at end of file diff --git a/scanner/src/cache_update.rs b/scanner/src/cache_update.rs index 7718705..ebdd158 100644 --- a/scanner/src/cache_update.rs +++ b/scanner/src/cache_update.rs @@ -142,6 +142,7 @@ impl Scanner { url: host.url, domain: host.domain, points, + connectivity: host.connectivity, rss: host.rss, version: host.version, healthy: last_check.healthy, diff --git a/scanner/src/instance_check.rs b/scanner/src/instance_check.rs index 124f5a9..9c285cc 100644 --- a/scanner/src/instance_check.rs +++ b/scanner/src/instance_check.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: AGPL-3.0-only //! Instance health/uptime checking code +use std::net::IpAddr; use std::time::Instant; use chrono::Utc; use entities::health_check; +use entities::host::Connectivity; use entities::{host, prelude::*}; -use reqwest::Url; +use reqwest::{Url, ClientBuilder}; use sea_orm::prelude::DateTimeUtc; use sea_orm::ColumnTrait; use sea_orm::EntityTrait; diff --git a/scanner/src/lib.rs b/scanner/src/lib.rs index 5ac2fbc..347d1f5 100644 --- a/scanner/src/lib.rs +++ b/scanner/src/lib.rs @@ -1,7 +1,7 @@ // SPDX-License-Identifier: AGPL-3.0-only use std::{ sync::{Arc, Mutex}, - time::Instant, + time::Instant, net::IpAddr, }; use about_parser::AboutParser; @@ -16,7 +16,7 @@ use profile_parser::ProfileParser; use regex::{Regex, RegexBuilder}; use reqwest::{ header::{HeaderMap, HeaderValue}, - Client, Url, + Client, Url, ClientBuilder, }; use sea_orm::{ sea_query::OnConflict, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, @@ -123,6 +123,8 @@ struct InnerScanner { last_list_fetch: Mutex, last_uptime_check: Mutex, rss_check_regex: Regex, + client_ipv4: Client, + client_ipv6: Client, } #[derive(Debug, FromQueryResult, Default)] @@ -133,7 +135,7 @@ pub struct LatestCheck { } impl Scanner { - async fn new(db: DatabaseConnection, config: ScannerConfig, cache: Cache) -> Result { + pub(crate) fn client_builder(config: &ScannerConfig) -> ClientBuilder { let mut headers = HeaderMap::with_capacity(HEADERS.len()); for header in HEADERS { headers.insert(header[0], HeaderValue::from_static(header[1])); @@ -148,14 +150,23 @@ impl Scanner { .connect_timeout(std::time::Duration::from_secs(3)) .timeout(std::time::Duration::from_secs(10)) .default_headers(headers); + http_client + } + + async fn new(db: DatabaseConnection, config: ScannerConfig, cache: Cache) -> Result { let mut builder_regex_rss = RegexBuilder::new(&config.rss_content); builder_regex_rss.case_insensitive(true); + let http_client = Self::client_builder(&config); + let client_ipv4 = Scanner::client_builder(&config).local_address("0.0.0.0".parse::().unwrap()).build().unwrap(); + let client_ipv6 = Scanner::client_builder(&config).local_address("::".parse::().unwrap()).build().unwrap(); let scanner = Self { inner: Arc::new(InnerScanner { db, cache, - config, client: http_client.build().unwrap(), + config, + client_ipv4, + client_ipv6, instance_parser: InstanceParser::new(), about_parser: AboutParser::new(), profile_parser: ProfileParser::new(), @@ -272,18 +283,19 @@ impl Scanner { }; // tracing::trace!(muted_host,instance=?instance,last_status=?last_status); join_set.spawn(async move { - let (rss, version, version_url) = match Url::parse(&instance.url) { + let (connectivity, rss, version, version_url) = match Url::parse(&instance.url) { Err(_) => { if !muted_host { tracing::info!(url = instance.url, "Instance URL invalid"); } - (false, None, None) + (None, false, None, None) } Ok(mut url) => { + let connectivity = scanner_c.check_connectivity(url.as_str()).await; let rss = scanner_c.has_rss(&mut url, muted_host).await; match scanner_c.nitter_version(&mut url, muted_host).await { - Some(version) => (rss, Some(version.version_name), Some(version.url)), - None => (rss, None, None), + Some(version) => (connectivity, rss, Some(version.version_name), Some(version.url)), + None => (connectivity, rss, None, None), } } }; @@ -298,6 +310,7 @@ impl Scanner { version_url: ActiveValue::Set(version_url), rss: ActiveValue::Set(rss), updated: ActiveValue::Set(time.timestamp()), + connectivity: ActiveValue::Set(connectivity), } }); } @@ -313,6 +326,7 @@ impl Scanner { host::Column::Version, host::Column::VersionUrl, host::Column::Country, + host::Column::Connectivity, ]) .to_owned(), ) @@ -339,6 +353,23 @@ impl Scanner { Ok(body) } + /// Check ipv4/6 connectivity of host + async fn check_connectivity(&self, url: &str) -> Option { + let ipv4 = self.inner.client_ipv4.get(url).send().await.map_or(false, |res|{ + res.status().is_success() + }); + let ipv6 = self.inner.client_ipv6.get(url).send().await.map_or(false, |res|{ + res.status().is_success() + }); + + match (ipv4,ipv6) { + (true,true) => Some(host::Connectivity::All), + (false,true) => Some(host::Connectivity::IPv6), + (true,false) => Some(host::Connectivity::IPv4), + (false,false) => None, + } + } + async fn fetch_url(&self, url: &str) -> std::result::Result<(u16, String), FetchError> { let fetch_res = self.inner.client.get(url).send().await?; let code = fetch_res.status().as_u16(); @@ -416,16 +447,20 @@ mod test { use super::*; use chrono::Duration; use entities::state::scanner::Config; + use migration::MigratorTrait; use sea_orm::{ConnectOptions, Database}; use tokio::{fs::File, io::AsyncWriteExt}; use entities::health_check; async fn db_init() -> DatabaseConnection { - Database::connect(ConnectOptions::new( + let db = Database::connect(ConnectOptions::new( "sqlite:./test_db.db?mode=rwc".to_owned(), )) .await - .unwrap() + .unwrap(); + migration::Migrator::up(&db, None) + .await.unwrap(); + db } // only for generating fake data @@ -476,6 +511,16 @@ mod test { let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap(); dbg!(scanner.generate_cache_data().await.unwrap()); } + + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + #[ignore] + async fn connectivity_test() { + let db = db_init().await; + let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap(); + assert_eq!(scanner.check_connectivity("https://v4.ipv6test.app").await,Some(host::Connectivity::IPv4)); + assert_eq!(scanner.check_connectivity("https://ipv6test.app").await,Some(host::Connectivity::All)); + assert_eq!(scanner.check_connectivity("https://v6.ipv6test.app").await,Some(host::Connectivity::IPv6)); + } } impl std::fmt::Debug for Scanner { diff --git a/server/templates/instances.html.j2 b/server/templates/instances.html.j2 index 6024487..5d6313e 100644 --- a/server/templates/instances.html.j2 +++ b/server/templates/instances.html.j2 @@ -33,6 +33,7 @@ All Healthy % RSS Version + Connectivity Points @@ -111,6 +112,8 @@ outdated {%- endif -%} + {# {%- if host.connectivity == -%} #} + {{host.connectivity}} {{host.points}} {%- endfor -%}