mark bad hosts

.env.example

@@ -22,7 +22,9 @@ PROFILE_CONTENT='jack.?\(@jack\)'
 RSS_CONTENT='<rss xmlns\:atom'
 # additional instances to include always
 ADDITIONAL_HOSTS="https://nitter.net"
-# Referer to use for requests
+# hosts known to be bad, as comma separated domains
+BAD_HOSTS="tweet.whateveritworks.org"
+# Referrer to use for requests
 REFERER="https://localhost"
 # Ping AVG period in hours
 PING_RANGE_H="3"

entities/src/state/mod.rs

@@ -45,4 +45,6 @@ pub struct CacheHost {
     pub is_upstream: bool,
     /// Whether the source is from the latest upstream commit
     pub is_latest_version: bool,
+    /// Whether this host is known to be bad (ip blocking)
+    pub is_bad_host: bool,
 }

entities/src/state/scanner.rs

@@ -21,7 +21,7 @@ pub struct Config {
     /// List of additional hosts to include during health checks
     pub additional_hosts: Vec<String>,
     /// Referer to use
-    pub referer: String,
+    pub referrer: String,
     /// Duration to average the ping/response times over
     pub ping_range: chrono::Duration,
     /// don't emit errors for hosts which are already listed as down
@@ -30,6 +30,8 @@ pub struct Config {
     pub source_git_url: String,
     /// Git branch to fetch the current commit from
     pub source_git_branch: String,
+    /// Hosts known to be bad (ip block), domain per entry
+    pub bad_hosts: Vec<String>,
 }
 
 impl Config {
@@ -44,11 +46,12 @@ impl Config {
             profile_content: String::from(r#"jack.?\(@jack\)"#),
             rss_content: String::from(r#"<rss xmlns\:atom"#),
             additional_hosts: vec![String::from("https://nitter.net")],
-            referer: String::from(""),
+            referrer: String::from(""),
             ping_range: chrono::Duration::hours(3),
             auto_mute: true,
             source_git_branch: String::from("master"),
             source_git_url: String::from("https://github.com/zedeus/nitter.git"),
+            bad_hosts: vec![String::from("tweet.whateveritworks.org")],
         })
     }
 }

scanner/src/cache_update.rs

@@ -103,6 +103,8 @@ impl Scanner {
 
             let latest_version = host.version_url.as_ref().map_or(false, |url|current_version.is_same_version(&url));
             let is_upstream = host.version_url.as_ref().map_or(false, |url|current_version.is_same_repo(&url));
+            
+            let is_bad_host = (!last_check.healthy) && self.inner.config.bad_hosts.contains(&host.domain);
 
             let host_ping_data = ping_data.remove(&host.id);
             host_statistics.push(CacheHost {
@@ -120,6 +122,7 @@ impl Scanner {
                 is_latest_version: latest_version,
                 is_upstream,
                 version_url: host.version_url,
+                is_bad_host,
             })
         }
         host_statistics.sort_unstable_by_key(|k| k.points);

scanner/src/lib.rs

@@ -134,7 +134,7 @@ impl Scanner {
         for header in HEADERS {
             headers.insert(header[0], HeaderValue::from_static(header[1]));
         }
-        headers.insert("Referer", HeaderValue::from_str(&config.referer).unwrap());
+        headers.insert("Referer", HeaderValue::from_str(&config.referrer).unwrap());
         let http_client = Client::builder()
             .cookie_store(true)
             .brotli(true)

server/templates/instances.html.j2

@@ -30,7 +30,14 @@
           {% for host in instances -%}
           <tr>
           <td><a href="{{host.url}}">{{host.domain}}</a></td>
-          <td>{% if host.healthy -%} ✅ {%- else -%} ❌ {%- endif -%}</td>
+          <td>
+          {%- if host.is_bad_host -%}
+            <div title="Host known for blocking healthchecks">❓</div>
+          {%- elif host.healthy -%}
+            ✅
+          {%- else -%}
+            ❌
+          {%- endif -%}</td>
           <td>
             {%- if host.recent_pings|length > 0 -%}
               {%- set height = 57 -%}

src/main.rs

@@ -105,12 +105,9 @@ fn read_scanner_cfg() -> miette::Result<ScannerConfig> {
     let about_path = require_env_str("ABOUT_PATH")?;
     let profile_content = require_env_str("PROFILE_CONTENT")?;
     let rss_content = require_env_str("RSS_CONTENT")?;
-    let additional_hosts: Vec<String> = require_env_str("ADDITIONAL_HOSTS")?
-        .trim()
-        .split(",")
-        .map(|v| v.trim().to_owned())
-        .collect();
-    let referer = require_env_str("REFERER")?;
+    let additional_hosts: Vec<String> = require_env_vec_str("ADDITIONAL_HOSTS")?;
+    let bad_hosts: Vec<String> = require_env_vec_str("BAD_HOSTS")?;
+    let referrer = require_env_str("REFERER")?;
     let auto_mute = require_env_str("AUTO_MUTE")? == "true";
     let source_git_branch = require_env_str("ORIGIN_SOURCE_GIT_BRANCH")?;
     let source_git_url = require_env_str("ORIGIN_SOURCE_GIT_URL")?;
@@ -125,11 +122,12 @@ fn read_scanner_cfg() -> miette::Result<ScannerConfig> {
         profile_content,
         rss_content,
         additional_hosts,
-        referer,
+        referrer,
         ping_range: chrono::Duration::hours(ping_range as _),
         auto_mute,
         source_git_branch,
         source_git_url,
+        bad_hosts,
     }))
 }
 
@@ -157,6 +155,14 @@ fn read_server_config(instance_ping_interval: usize) -> miette::Result<server::C
     })
 }
 
+fn require_env_vec_str(name: &str) -> miette::Result<Vec<String>> {
+    Ok(require_env_str(name)?
+        .trim()
+        .split(",")
+        .map(|v| v.trim().to_owned())
+        .collect())
+}
+
 fn require_env_str(name: &str) -> miette::Result<String> {
     var(name).map_err(|v| miette::miette!("missing `{}` in environment: {:?}", name, v))
 }