update score and add last_healthy

also updates the about page

entities/src/state/mod.rs

@@ -37,4 +37,5 @@ pub struct CacheHost {
     pub ping_avg: Option<i32>,
     pub version: Option<String>,
     pub healthy: bool,
+    pub last_healthy: Option<DateTimeUtc>,
 }

scanner/src/cache_update.rs

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 use std::collections::HashMap;
 
+use chrono::TimeZone;
 use chrono::{Days, Utc};
 use entities::host;
 use entities::prelude::*;
@@ -56,15 +57,15 @@ impl Scanner {
         }
 
         let time_now = Utc::now();
+        let time_3h = time_now.checked_sub_days(Days::new(30)).unwrap();
         let time_30d = time_now.checked_sub_days(Days::new(30)).unwrap();
         let time_120d = time_now.checked_sub_days(Days::new(120)).unwrap();
 
+        let stats_3h = self.query_stats_range(time_3h, time_now).await?;
         let stats_30d = self.query_stats_range(time_30d, time_now).await?;
-        let stats_30d: HashMap<i32, HostStats> =
-            stats_30d.into_iter().map(|v| (v.host, v)).collect();
         let stats_120d = self.query_stats_range(time_120d, time_30d).await?;
-        let stats_120d: HashMap<i32, HostStats> =
-            stats_120d.into_iter().map(|v| (v.host, v)).collect();
+
+        let mut last_healthy_check = self.query_last_healthy().await?;
 
         let version_points = self.query_versions(time_30d).await?;
         let latest_check = self.query_latest_check(&self.inner.db).await?;
@@ -76,32 +77,27 @@ impl Scanner {
         let mut host_statistics = Vec::with_capacity(hosts.len());
         let default_health_check = LatestCheck::default();
         for host in hosts {
-            let points_30d: f64 = 0.4
-                * match stats_30d.get(&host.id) {
-                    None => 0.0,
-                    Some(stats) => stats.good as f64 / stats.total as f64,
-                };
-            let points_120d: f64 = 0.3
-                * match stats_120d.get(&host.id) {
-                    None => 0.0,
-                    Some(stats) => stats.good as f64 / stats.total as f64,
-                };
+            let points_3h: f64 = 0.3 * stats_3h.get(&host.id).map_or(0.0, |stats|stats.good as f64 / stats.total as f64);
+            let points_30d: f64 = 0.2 * stats_30d.get(&host.id).map_or(0.0, |stats|stats.good as f64 / stats.total as f64);
+            let points_120d: f64 = 0.2 * stats_120d.get(&host.id).map_or(0.0, |stats|stats.good as f64 / stats.total as f64);
             let points_version = 0.1
                 * host
                     .version
                     .as_ref()
                     .map_or(0.0, |version| *version_points.get(version).unwrap_or(&0.0));
-            let points = points_30d + points_120d + points_version;
+            let points = points_30d + points_120d + points_version + points_3h;
 
             let last_check = latest_check.get(&host.id).unwrap_or(&default_health_check);
-            // don't rank currently down instances highly
-            let points = match last_check.healthy {
-                true => (points * 100.0) as i32,
-                false => 0,
-            };
+            // // don't rank currently down instances highly
+            // let points = match last_check.healthy {
+            //     true => (points * 100.0) as i32,
+            //     false => 0,
+            // };
+            let points = (points * 100.0) as i32;
 
             let host_ping_data = ping_data.remove(&host.id);
             host_statistics.push(CacheHost {
+                last_healthy: last_healthy_check.remove(&host.id),
                 url: host.url,
                 domain: host.domain,
                 points,
@@ -148,6 +144,11 @@ impl Scanner {
         ))
         .all(&self.inner.db)
         .await?;
+        // this is pretty verbose, but allows to
+        // - calculate the avg
+        // - map by host
+        // - get a list of all past entries
+        // in one go and with one DB request
         let mut map = HashMap::with_capacity(100);
         let mut iter = last_pings.iter();
         let first = match iter.next() {
@@ -217,12 +218,35 @@ impl Scanner {
         Ok(stats_rated)
     }
 
+    /// Timestamp of last healthy host check
+    async fn query_last_healthy(&self) -> Result<HashMap<i32,DateTimeUtc>> {
+        #[derive(Debug, FromQueryResult)]
+        struct LastHealthyEntry {
+            host: i32,
+            time: i64,
+        }
+        let last_healthy_times = LastHealthyEntry::find_by_statement(Statement::from_sql_and_values(
+            DbBackend::Sqlite,
+            r#"
+            SELECT u.host,MAX(u.time) as time FROM health_check u
+            JOIN host h ON h.id = u.host
+            WHERE h.enabled = true AND u.healthy = true
+            GROUP BY u.host
+            "#,
+            [],
+        ))
+        .all(&self.inner.db)
+        .await?;
+        let last_healthy_times: HashMap<_,_> = last_healthy_times.into_iter().map(|v| (v.host, Utc.timestamp_opt(v.time, 0).unwrap())).collect();
+        Ok(last_healthy_times)
+    }
+
     /// Query uptime statistics per host
     async fn query_stats_range(
         &self,
         from: DateTimeUtc,
         to: DateTimeUtc,
-    ) -> Result<Vec<HostStats>> {
+    ) -> Result<HashMap<i32,HostStats>> {
         let stats: Vec<HostStats> = HostStats::find_by_statement(Statement::from_sql_and_values(
             DbBackend::Sqlite,
             r#"SELECT u.host, COUNT(CASE WHEN healthy = true THEN 1 END) as good,COUNT(*) as total FROM health_check u
@@ -233,6 +257,7 @@ impl Scanner {
         ))
         .all(&self.inner.db)
         .await?;
+        let stats: HashMap<_,_> = stats.into_iter().map(|v| (v.host, v)).collect();
         Ok(stats)
     }
 }

server/templates/about.html.j2

@@ -27,12 +27,11 @@
     <p>Explanations</p>
     <ul>
       <li><code>Healthy</code> stands for hosts which are reachable and pass a content check.</li>
-      <li><code>Ping AVG</code> is the ping average over the last {{ping_avg_interval_h}} hours.
-        Note that this is the time in ms it took for an instance to respond with content. This is not a network ping.</li>
+      <li><code>Response Time</code> is the response time over the last {{ping_avg_interval_h}} hours. This is not a network ping.</li>
+      <li><code>Average</code> is the time average over the last {{ping_avg_interval_h}} hours.</li>
       <li><code>RSS</code> whether the host has RSS feeds enabled.</li>
       <li><code>Version</code> which nitter version the host reports.</li>
-      <li><code>Points</code> an instance score based on the availability over the last 30 and 120 days, together with the version.</li>
-      <li>(api only) <code>Ping</code> last health check response time (ms).</li>
+      <li><code>Points</code> is a weighted instance score based on the availability over the last 3h, 30 and 120 days, together with the version.</li>
     </ul>
 
     <p><a href="https://github.com/4cuck/nitter-status">Sourcecode</a></p>

server/templates/instances.html.j2

@@ -19,7 +19,7 @@
           <tr>
           <th scope="col">Instance</th>
           <th scope="col">Healthy</th>
-          <th scope="col">Ping</th>
+          <th scope="col">Response Time</th>
           <th scope="col">Average</th>
           <th scope="col">RSS</th>
           <th scope="col">Version</th>
@@ -53,26 +53,37 @@
               {# <!-- step size {{step_size}} -->
               <!-- ping_max {{host.ping_max}} -->
               <!-- ping_min {{host.ping_min}} --> #}
-              <svg viewBox="0 0 182 56" width="{{width - 1}}" height="{{height - 1}}">
-                {% if host.ping_avg and host.ping_min -%}
-                  {% set avg_pos = (height-(host.ping_avg - host.ping_min )/step_size) - offset -%}
-                  <line x1="1" x2="181" y1="{{avg_pos}}" y2="{{avg_pos}}" stroke="black" stroke-width="0.72727275" stroke-linecap="butt" stroke-opacity="0.3"><title>Average</title></line>
+              {% if host.ping_min -%}
+                <svg viewBox="0 0 182 56" width="{{width - 1}}" height="{{height - 1}}">
+                  {% if host.ping_avg and host.ping_min -%}
+                    {% set avg_pos = (height-(host.ping_avg - host.ping_min )/step_size) - offset -%}
+                    <line x1="1" x2="181" y1="{{avg_pos}}" y2="{{avg_pos}}" stroke="black" stroke-width="0.72727275" stroke-linecap="butt" stroke-opacity="0.3"><title>Average</title></line>
+                  {% endif -%}
+                  {% for ping in host.recent_pings -%}
+                    {% if ping -%}
+                      {% set height_bar = (ping - host.ping_min )/step_size + offset -%}
+                      {% set color_bar = "#a09df2" -%}
+                    {% else %}
+                      {% set height_bar = error_size -%}
+                      {% set color_bar = "#cc0000" -%}
+                    {% endif %}
+                    <rect x="{{(loop.index-1) * width_bar}}" y="{{height - height_bar - offset}}" width="{{width_bar}}" height="{{height_bar}}" fill="{{color_bar}}" fill-opacity="0.95"><title>{{ping | default(value="unhealthy") }}</title></rect>
+                  {% endfor -%}
+                </svg>
+              {% else -%}
+                {# <!-- no healthy entries, show last time host was seen healthy --> #}
+                {% if host.last_healthy -%}
+                Last seen healthy: {{host.last_healthy | truncate(length=10, end="") | replace(from="-", to=".")}}
+                {% else -%}
+                Never seen healthy.
                 {% endif -%}
-                {% for ping in host.recent_pings -%}
-                  {% if ping -%}
-                    {% set height_bar = (ping - host.ping_min )/step_size + offset -%}
-                    {% set color_bar = "#a09df2" -%}
-                  {% else %}
-                    {% set height_bar = error_size -%}
-                    {% set color_bar = "#cc0000" -%}
-                  {% endif %}
-                  <rect x="{{(loop.index-1) * width_bar}}" y="{{height - height_bar - offset}}" width="{{width_bar}}" height="{{height_bar}}" fill="{{color_bar}}" fill-opacity="0.95"><title>{{ping | default(value="unhealthy") }}</title></rect>
-                {% endfor -%}
-              </svg>
-            {% endif %}
+              {% endif -%}
+            {% else -%}
+              No recent data!
+            {% endif -%}
           </td>
           <td>{{host.ping_avg}}</td>
-          <td>{% if host.rss -%} ✅ {% else %} ❌ {% endif %}</td>
+          <td>{% if host.rss -%} ✅ {% else -%} ❌ {% endif -%}</td>
           <td>{{host.version}}</td>
           <td>{{host.points}}</td>
           </tr>