ipv detection

Cargo.lock

@@ -2777,6 +2777,7 @@ dependencies = [
  "csv",
  "entities",
  "git2",
+ "migration",
  "regex",
  "reqwest",
  "scraper",

entities/src/host.rs

@@ -4,7 +4,7 @@ use sea_orm::{
     entity::prelude::*,
     sea_query::{Expr, SimpleExpr},
 };
-use serde::Serialize;
+use serde::{Serialize, Deserialize};
 
 #[derive(Copy, Clone, Default, Debug, DeriveEntity)]
 pub struct Entity;
@@ -27,10 +27,22 @@ pub struct Model {
     pub version: Option<String>,
     pub country: String,
     pub version_url: Option<String>,
+    pub connectivity: Option<Connectivity>,
     /// Last time the url and enabled were updated, *not* the rss
     pub updated: i64,
 }
 
+#[derive(Copy, Clone, Debug, PartialEq ,Eq, EnumIter, DeriveActiveEnum, Serialize)]
+#[sea_orm(rs_type = "i32", db_type = "Integer")]
+pub enum Connectivity {
+    #[sea_orm(num_value = 0)]
+    All = 0,
+    #[sea_orm(num_value = 1)]
+    IPv4 = 1,
+    #[sea_orm(num_value = 2)]
+    IPv6 = 2,
+}
+
 #[derive(Copy, Clone, Debug, EnumIter, DeriveColumn)]
 pub enum Column {
     Id,
@@ -40,6 +52,7 @@ pub enum Column {
     Country,
     VersionUrl,
     Enabled,
+    Connectivity,
     Rss,
     Updated,
 }
@@ -75,6 +88,7 @@ impl ColumnTrait for Column {
             Self::Enabled => ColumnType::Integer.def(),
             Self::Rss => ColumnType::Integer.def(),
             Self::Updated => ColumnType::Integer.def(),
+            Self::Connectivity => ColumnType::Integer.def().null(),
         }
     }
 

entities/src/state/mod.rs

@@ -8,6 +8,8 @@ use chrono::Utc;
 use sea_orm::prelude::DateTimeUtc;
 use serde::Serialize;
 
+use crate::host::Connectivity;
+
 pub mod scanner;
 
 pub type Cache = Arc<RwLock<CacheData>>;
@@ -53,4 +55,5 @@ pub struct CacheHost {
     pub recent_checks: Vec<(String,bool)>,
     /// Percentage of healthy checks since first seen
     pub healthy_percentage_overall: u8,
+    pub connectivity: Option<Connectivity>,
 }

migration/src/lib.rs

@@ -5,6 +5,7 @@ mod m20230729_010231_datetime_rowid;
 mod m20230729_230909_datetime_int_host;
 mod m20230803_154714_version_url;
 mod m20230829_201916_country;
+mod m20230914_231514_connectivity;
 
 
 pub struct Migrator;
@@ -18,6 +19,7 @@ impl MigratorTrait for Migrator {
             Box::new(m20230729_230909_datetime_int_host::Migration),
             Box::new(m20230803_154714_version_url::Migration),
             Box::new(m20230829_201916_country::Migration),
+            Box::new(m20230914_231514_connectivity::Migration),
         ]
     }
 }

migration/src/m20230914_231514_connectivity.rs

@@ -0,0 +1,22 @@
+use sea_orm_migration::prelude::*;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        let cmd = r#"ALTER TABLE "host" ADD COLUMN "connectivity" INT;"#;
+        let db = manager.get_connection();
+        db.execute_unprepared("BEGIN EXCLUSIVE").await?;
+        tracing::info!("adding connectivity column..");
+        db.execute_unprepared(cmd).await?;
+        db.execute_unprepared("COMMIT TRANSACTION").await?;
+        db.execute_unprepared("VACUUM").await?;
+        Ok(())
+    }
+
+    async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
+        panic!("Can't migrate down");
+    }
+}

scanner/Cargo.toml

@@ -22,5 +22,8 @@ serde = { workspace = true, features = ["derive"] }
 tracing-test = { workspace = true }
 csv = "1.2.2"
 
+[dev-dependencies.migration]
+path = "../migration"
+
 [dependencies.entities]
 path = "../entities"

scanner/src/cache_update.rs

@@ -142,6 +142,7 @@ impl Scanner {
                 url: host.url,
                 domain: host.domain,
                 points,
+                connectivity: host.connectivity,
                 rss: host.rss,
                 version: host.version,
                 healthy: last_check.healthy,

scanner/src/instance_check.rs

@@ -1,11 +1,13 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 //! Instance health/uptime checking code
+use std::net::IpAddr;
 use std::time::Instant;
 
 use chrono::Utc;
 use entities::health_check;
+use entities::host::Connectivity;
 use entities::{host, prelude::*};
-use reqwest::Url;
+use reqwest::{Url, ClientBuilder};
 use sea_orm::prelude::DateTimeUtc;
 use sea_orm::ColumnTrait;
 use sea_orm::EntityTrait;

scanner/src/lib.rs

@@ -1,7 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 use std::{
     sync::{Arc, Mutex},
-    time::Instant,
+    time::Instant, net::IpAddr,
 };
 
 use about_parser::AboutParser;
@@ -16,7 +16,7 @@ use profile_parser::ProfileParser;
 use regex::{Regex, RegexBuilder};
 use reqwest::{
     header::{HeaderMap, HeaderValue},
-    Client, Url,
+    Client, Url, ClientBuilder,
 };
 use sea_orm::{
     sea_query::OnConflict, ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait,
@@ -123,6 +123,8 @@ struct InnerScanner {
     last_list_fetch: Mutex<Instant>,
     last_uptime_check: Mutex<Instant>,
     rss_check_regex: Regex,
+    client_ipv4: Client,
+    client_ipv6: Client,
 }
 
 #[derive(Debug, FromQueryResult, Default)]
@@ -133,7 +135,7 @@ pub struct LatestCheck {
 }
 
 impl Scanner {
-    async fn new(db: DatabaseConnection, config: ScannerConfig, cache: Cache) -> Result<Self> {
+    pub(crate) fn client_builder(config: &ScannerConfig) -> ClientBuilder {
         let mut headers = HeaderMap::with_capacity(HEADERS.len());
         for header in HEADERS {
             headers.insert(header[0], HeaderValue::from_static(header[1]));
@@ -148,14 +150,23 @@ impl Scanner {
             .connect_timeout(std::time::Duration::from_secs(3))
             .timeout(std::time::Duration::from_secs(10))
             .default_headers(headers);
+        http_client
+    }
+
+    async fn new(db: DatabaseConnection, config: ScannerConfig, cache: Cache) -> Result<Self> {
         let mut builder_regex_rss = RegexBuilder::new(&config.rss_content);
         builder_regex_rss.case_insensitive(true);
+        let http_client = Self::client_builder(&config);
+        let client_ipv4 = Scanner::client_builder(&config).local_address("0.0.0.0".parse::<IpAddr>().unwrap()).build().unwrap();
+        let client_ipv6 = Scanner::client_builder(&config).local_address("::".parse::<IpAddr>().unwrap()).build().unwrap();
         let scanner = Self {
             inner: Arc::new(InnerScanner {
                 db,
                 cache,
-                config,
                 client: http_client.build().unwrap(),
+                config,
+                client_ipv4,
+                client_ipv6,
                 instance_parser: InstanceParser::new(),
                 about_parser: AboutParser::new(),
                 profile_parser: ProfileParser::new(),
@@ -272,18 +283,19 @@ impl Scanner {
             };
             // tracing::trace!(muted_host,instance=?instance,last_status=?last_status);
             join_set.spawn(async move {
-                let (rss, version, version_url) = match Url::parse(&instance.url) {
+                let (connectivity, rss, version, version_url) = match Url::parse(&instance.url) {
                     Err(_) => {
                         if !muted_host {
                             tracing::info!(url = instance.url, "Instance URL invalid");
                         }
-                        (false, None, None)
+                        (None, false, None, None)
                     }
                     Ok(mut url) => {
+                        let connectivity = scanner_c.check_connectivity(url.as_str()).await;
                         let rss = scanner_c.has_rss(&mut url, muted_host).await;
                         match scanner_c.nitter_version(&mut url, muted_host).await {
-                            Some(version) => (rss, Some(version.version_name), Some(version.url)),
-                            None => (rss, None, None),
+                            Some(version) => (connectivity, rss, Some(version.version_name), Some(version.url)),
+                            None => (connectivity, rss, None, None),
                         }
                     }
                 };
@@ -298,6 +310,7 @@ impl Scanner {
                     version_url: ActiveValue::Set(version_url),
                     rss: ActiveValue::Set(rss),
                     updated: ActiveValue::Set(time.timestamp()),
+                    connectivity: ActiveValue::Set(connectivity),
                 }
             });
         }
@@ -313,6 +326,7 @@ impl Scanner {
                             host::Column::Version,
                             host::Column::VersionUrl,
                             host::Column::Country,
+                            host::Column::Connectivity,
                         ])
                         .to_owned(),
                 )
@@ -339,6 +353,23 @@ impl Scanner {
         Ok(body)
     }
 
+    /// Check ipv4/6 connectivity of host
+    async fn check_connectivity(&self, url: &str) -> Option<host::Connectivity> {
+        let ipv4 = self.inner.client_ipv4.get(url).send().await.map_or(false, |res|{
+            res.status().is_success()
+        });
+        let ipv6 = self.inner.client_ipv6.get(url).send().await.map_or(false, |res|{
+            res.status().is_success()
+        });
+
+        match (ipv4,ipv6) {
+            (true,true) => Some(host::Connectivity::All),
+            (false,true) => Some(host::Connectivity::IPv6),
+            (true,false) => Some(host::Connectivity::IPv4),
+            (false,false) => None,
+        }
+    }
+
     async fn fetch_url(&self, url: &str) -> std::result::Result<(u16, String), FetchError> {
         let fetch_res = self.inner.client.get(url).send().await?;
         let code = fetch_res.status().as_u16();
@@ -416,16 +447,20 @@ mod test {
     use super::*;
     use chrono::Duration;
     use entities::state::scanner::Config;
+    use migration::MigratorTrait;
     use sea_orm::{ConnectOptions, Database};
     use tokio::{fs::File, io::AsyncWriteExt};
     use entities::health_check;
 
     async fn db_init() -> DatabaseConnection {
-        Database::connect(ConnectOptions::new(
+        let db = Database::connect(ConnectOptions::new(
             "sqlite:./test_db.db?mode=rwc".to_owned(),
         ))
         .await
-        .unwrap()
+        .unwrap();
+        migration::Migrator::up(&db, None)
+            .await.unwrap();
+        db
     }
 
     // only for generating fake data
@@ -476,6 +511,16 @@ mod test {
         let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap();
         dbg!(scanner.generate_cache_data().await.unwrap());
     }
+
+    #[tokio::test(flavor = "multi_thread", worker_threads = 1)]
+    #[ignore]
+    async fn connectivity_test() {
+        let db = db_init().await;
+        let scanner = Scanner::new(db, Config::test_defaults(), entities::state::new()).await.unwrap();
+        assert_eq!(scanner.check_connectivity("https://v4.ipv6test.app").await,Some(host::Connectivity::IPv4));
+        assert_eq!(scanner.check_connectivity("https://ipv6test.app").await,Some(host::Connectivity::All));
+        assert_eq!(scanner.check_connectivity("https://v6.ipv6test.app").await,Some(host::Connectivity::IPv6));
+    }
 }
 
 impl std::fmt::Debug for Scanner {

server/templates/instances.html.j2

@@ -33,6 +33,7 @@
           <th scope="col">All Healthy %</th>
           <th scope="col">RSS</th>
           <th scope="col">Version</th>
+          <th scope="col">Connectivity</th>
           <th scope="col">Points</th>
           </tr>
       </thead>
@@ -111,6 +112,8 @@
                 <span class="ms-1 badge bg-info">outdated</span>
               {%- endif -%}
           </td>
+          {# <td>{%- if host.connectivity ==  -%}</td> #}
+          <td>{{host.connectivity}}</td>
           <td>{{host.points}}</td>
           </tr>
           {%- endfor -%}