fetch latest git branch

Closes #1
Closes #2

scanner/src/about_parser.rs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 use regex::{Regex, RegexBuilder};
+use reqwest::Url;
 use scraper::{Html, Selector};
 use thiserror::Error;
 
@@ -15,6 +16,8 @@ pub enum AboutParseError {
     NoCommitLinkFound,
     #[error("Missing test! Found '{0}'")]
     InvalidCommitFormat(String),
+    #[error("No valid href!")]
+    NoValidHref,
 }
 
 pub(crate) struct AboutParser {
@@ -23,9 +26,14 @@ pub(crate) struct AboutParser {
     regex: Regex,
 }
 
+pub struct AboutParsed {
+    pub version_name: String,
+    pub url: String,
+}
+
 impl AboutParser {
     /// Returns the text of the version <a> element of nitters about site
-    pub fn parse_about_version(&self, html: &str) -> Result<String> {
+    pub fn parse_about_version(&self, html: &str) -> Result<AboutParsed> {
         let fragment = Html::parse_fragment(html);
         // get all <p> elements
         let p_elem = fragment
@@ -35,6 +43,8 @@ impl AboutParser {
 
         let mut a_elems = p_elem.select(&self.selector_a);
         let link = a_elems.next().ok_or(AboutParseError::NoCommitLinkFound)?;
+        let url = link.value().attr("href").map(|v|v.trim().to_owned())
+            .ok_or(AboutParseError::NoValidHref)?;
         let link_text = link.text().fold(String::new(), |mut acc, text| {
             acc.push_str(text);
             acc
@@ -42,7 +52,10 @@ impl AboutParser {
         if !self.regex.is_match(&link_text) {
             return Err(AboutParseError::InvalidCommitFormat(link_text));
         }
-        Ok(link_text)
+        Ok(AboutParsed {
+            url,
+            version_name: link_text
+        })
     }
 
     pub fn new() -> Self {
@@ -64,6 +77,7 @@ mod test {
         let html = include_str!("../test_data/about.html");
         let parser = AboutParser::new();
         let res = parser.parse_about_version(html).unwrap();
-        assert_eq!(&res, "2023.07.22-72d8f35");
+        assert_eq!(&res.version_name, "2023.07.22-72d8f35");
+        assert_eq!(res.url, String::from("https://github.com/zedeus/nitter/commit/72d8f35"));
     }
 }

scanner/src/cache_update.rs

@@ -14,6 +14,7 @@ use sea_orm::QueryOrder;
 use sea_orm::{prelude::DateTimeUtc, DbBackend, FromQueryResult, Statement};
 
 use crate::LatestCheck;
+use crate::version_check::fetch_git_state;
 use crate::{Result, Scanner};
 
 #[derive(Debug, FromQueryResult)]
@@ -49,10 +50,15 @@ impl Scanner {
     /// Generate host stats and returns a new CacheData
     pub(crate) async fn generate_cache_data(&self) -> Result<CacheData> {
         let hosts = self.query_hosts_enabled().await?;
+        let config_c = self.inner.config.clone();
+        let current_version = tokio::task::spawn_blocking(move || {
+            fetch_git_state(config_c)
+        }).await.unwrap()?;
         if hosts.is_empty() {
             return Ok(CacheData {
-                hosts: vec![],
+                hosts: Vec::new(),
                 last_update: Utc::now(),
+                latest_commit: current_version.version,
             });
         }
 
@@ -95,6 +101,9 @@ impl Scanner {
             // };
             let points = (points * 100.0) as i32;
 
+            let latest_version = host.version_url.as_ref().map_or(false, |url|current_version.is_same_version(&url));
+            let is_upstream = host.version_url.as_ref().map_or(false, |url|current_version.is_same_repo(&url));
+
             let host_ping_data = ping_data.remove(&host.id);
             host_statistics.push(CacheHost {
                 last_healthy: last_healthy_check.remove(&host.id),
@@ -108,6 +117,9 @@ impl Scanner {
                 ping_min: host_ping_data.as_ref().and_then(|v|v.min),
                 ping_avg: host_ping_data.as_ref().and_then(|v|v.avg),
                 recent_pings: host_ping_data.map(|v|v.pings).unwrap_or_default(),
+                is_latest_version: latest_version,
+                is_upstream,
+                version_url: host.version_url,
             })
         }
         host_statistics.sort_unstable_by_key(|k| k.points);
@@ -115,6 +127,7 @@ impl Scanner {
         Ok(CacheData {
             hosts: host_statistics,
             last_update: time_now,
+            latest_commit: current_version.version,
         })
     }
 

scanner/src/instance_check.rs

@@ -17,6 +17,7 @@ use tracing::instrument;
 use crate::FetchError;
 use crate::Result;
 use crate::Scanner;
+use crate::about_parser::AboutParsed;
 
 impl Scanner {
     /// Check uptime for host and create a new uptime entry in the database
@@ -147,7 +148,7 @@ impl Scanner {
     }
 
     /// Check nitter version
-    pub(crate) async fn nitter_version(&self, url: &mut Url, mute: bool) -> Option<String> {
+    pub(crate) async fn nitter_version(&self, url: &mut Url, mute: bool) -> Option<AboutParsed> {
         url.set_path(&self.inner.config.about_path);
         match self.fetch_url(url.as_str()).await {
             Ok((code, content)) => match self.inner.about_parser.parse_about_version(&content) {

scanner/src/lib.rs

@@ -31,6 +31,7 @@ mod about_parser;
 mod cache_update;
 mod instance_check;
 mod instance_parser;
+mod version_check;
 
 const CAPTCHA_TEXT: &'static str = "Enable JavaScript and cookies to continue";
 const CAPTCHA_CODE: u16 = 403;
@@ -58,6 +59,10 @@ pub enum ScannerError {
     InstanceParse(#[from] instance_parser::InstaceListError),
     #[error("Failed to fetch URL: {0}")]
     FetchError(#[from] FetchError),
+    #[error("Failed fetcing git: {0}")]
+    GitFetch(#[from] git2::Error),
+    #[error("Couldn't find git branch")]
+    GitBranch,
 }
 
 #[derive(Error, Debug)]
@@ -263,25 +268,29 @@ impl Scanner {
             };
             // tracing::trace!(muted_host,instance=?instance,last_status=?last_status);
             join_set.spawn(async move {
-                let (rss, version) = match Url::parse(&instance.url) {
+                let (rss, version, version_url) = match Url::parse(&instance.url) {
                     Err(_) => {
                         if !muted_host {
                             tracing::info!(url = instance.url, "Instance URL invalid");
                         }
-                        (false, None)
+                        (false, None, None)
                     }
                     Ok(mut url) => {
                         let rss = scanner_c.has_rss(&mut url, muted_host).await;
-                        let version = scanner_c.nitter_version(&mut url, muted_host).await;
-                        (rss, version)
+                        match scanner_c.nitter_version(&mut url, muted_host).await {
+                            Some(version) => (rss,Some(version.version_name),Some(version.url)),
+                            None => (rss,None,None),
+                        }
                     }
                 };
+                
                 host::ActiveModel {
                     id: ActiveValue::NotSet,
                     domain: ActiveValue::Set(instance.domain),
                     url: ActiveValue::Set(instance.url),
                     enabled: ActiveValue::Set(true),
                     version: ActiveValue::Set(version),
+                    version_url: ActiveValue::Set(version_url),
                     rss: ActiveValue::Set(rss),
                     updated: ActiveValue::Set(time.timestamp()),
                 }

scanner/src/version_check.rs

@@ -0,0 +1,63 @@
+use entities::state::scanner::ScannerConfig;
+use git2::{Remote, Direction};
+
+use crate::{Result, ScannerError};
+
+pub(crate) struct CurrentVersion {
+    pub version: String,
+    config: ScannerConfig,
+}
+
+const SHORT_COMMIT_LEN: usize = 7;
+
+impl CurrentVersion {
+    /// Whether a nitter version URL is from the same repo
+    pub fn is_same_repo(&self, url: &str) -> bool {
+        url.starts_with(self.config.source_git_url.trim_end_matches(".git"))
+    }
+    /// Whether a nitter version URL is pointing to the same version as this one
+    pub fn is_same_version(&self, url: &str) -> bool {
+        // look for last path segment (don't parse) and look if that matches
+        // from the start
+        self.is_same_repo(url) && match url.split('/').last() {
+            Some(other_version) => {
+                if other_version.len() == self.version.len() {
+                    // we get the long version
+                    other_version.starts_with(&self.version)
+                } else if other_version.len() == SHORT_COMMIT_LEN {
+                    other_version.starts_with(&self.version[..SHORT_COMMIT_LEN])
+                } else {
+                    // everything else is no short commit id..
+                    false
+                }
+            }
+            None => false
+        }
+    }
+}
+
+pub(crate) fn fetch_git_state(config: ScannerConfig) -> Result<CurrentVersion> {
+    let mut remote = Remote::create_detached(config.source_git_url.as_str())?;
+
+    remote.connect(Direction::Fetch)?;
+
+    let reference = format!("refs/heads/{}",config.source_git_branch);
+    let commit = remote.list()?.into_iter().find(|v|v.name() == &reference)
+        .map(|v|v.oid().to_string());
+
+    remote.disconnect()?;
+
+    Ok(commit.map(|commit|CurrentVersion { version: commit, config }).ok_or(ScannerError::GitBranch)?)
+}
+
+#[cfg(test)]
+mod test {
+    use entities::state::scanner::Config;
+
+    use super::fetch_git_state;
+
+    #[test]
+    fn test_git() {
+        fetch_git_state(Config::test_defaults()).unwrap();
+    }
+}