// SPDX-License-Identifier: AGPL-3.0-only use std::collections::HashMap; use reqwest::Url; use scraper::{ElementRef, Html, Selector}; use thiserror::Error; pub type Result = std::result::Result; pub static EXPECT_CSS_SELCTOR: &'static str = "failed to parse css selector"; static CHECKBOX: &'static str = "✅"; type InstanceMap = HashMap; #[derive(Error, Debug)] pub enum InstanceListError { #[error("No div #wiki-body found!")] NoWikiDiv, #[error("No table found containing instances!")] NoInstanceTable, #[error("Abort-on-err on, malformed table row found!")] MalformedRow, } #[derive(Debug, Eq, PartialEq)] #[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))] pub struct InstanceParsed { /// URL without any login stuff pub domain: String, /// connection URL pub url: String, /// whether this instance is marked as online pub online: bool, /// the SSL provider this instance supposedly has pub ssl_provider: String, /// the country for this instance pub country: String, } /// Instance parser. pub(crate) struct InstanceParser { selector_wiki: Selector, selector_table: Selector, selector_tr: Selector, selector_td: Selector, selector_a: Selector, } impl InstanceParser { pub fn new() -> Self { Self { selector_wiki: Selector::parse(r#"div[id="wiki-body"]"#).expect(EXPECT_CSS_SELCTOR), selector_table: Selector::parse("table").expect(EXPECT_CSS_SELCTOR), selector_tr: Selector::parse("tbody > tr").expect(EXPECT_CSS_SELCTOR), selector_td: Selector::parse("td").expect(EXPECT_CSS_SELCTOR), selector_a: Selector::parse("a").expect(EXPECT_CSS_SELCTOR), } } /// Parse a html rendered version of the instance list /// /// *abort_on_err* is just for testing and return an error for any malformed table entry pub fn parse_instancelist( &self, html: &str, additional_instances: &[String], additional_instances_country: &str, abort_on_err: bool, ) -> Result { let fragment = Html::parse_fragment(html); // wiki body by div ID let mut wiki_divs = fragment.select(&self.selector_wiki); // first result let first_wiki = wiki_divs.next().ok_or(InstanceListError::NoWikiDiv)?; // all element let mut tables = first_wiki.select(&self.selector_table); // find the one with "Online" text inside let instance_table = tables .find(|t| t.text().any(|text| text.contains("Online"))) .ok_or(InstanceListError::NoInstanceTable)?; let mut instances = HashMap::with_capacity(50); // iterate over all > inside for row in instance_table.select(&self.selector_tr) { match self.parse_row(row) { Ok(instance) => { if let Some(old) = instances.insert(instance.domain.clone(), instance) { tracing::warn!(domain = old.domain, "Parsed duplicate instance domain!"); } } Err(e) => { if abort_on_err { return Err(e); } continue; } } } for entry in additional_instances { match Url::parse(entry.as_ref()) { Ok(v) => { if let Some(domain) = v.domain() { instances.insert( domain.to_owned(), InstanceParsed { domain: domain.to_owned(), url: entry.clone(), online: true, ssl_provider: String::new(), country: additional_instances_country.to_owned(), }, ); } } Err(e) => tracing::warn!(instance=entry,error=?e,"Ignoring additional instance"), } } Ok(instances) } /// Parse a single instance table row fn parse_row(&self, row: ElementRef) -> Result { let mut cols = row.select(&self.selector_td); // get first URL column let url: String = match cols.next() { None => { tracing::error!(row=?row.html(),"Parsed instance missing URL row, skipping!"); return Err(InstanceListError::MalformedRow); } Some(col) => { // find first inside and get its "href" attribute let mut a_elems = col.select(&self.selector_a); match a_elems.next().and_then(|v| v.value().attr("href")) { None => { tracing::error!(row=?row.html(),"Parsed instance missing valid URL element, skipping!"); return Err(InstanceListError::MalformedRow); } Some(url_value) => { // trim whitespace and strip `/` at the end let trimmed = url_value.trim(); trimmed.strip_suffix("/").unwrap_or(trimmed).to_owned() } } } }; // parse URL to strip everything apart from the domain let domain = match Url::parse(&url) { Ok(parsed_url) => parsed_url.domain().map(|v| v.to_owned()).ok_or_else(|| { tracing::error!(url = url, "Parsed instance URL has no domain"); InstanceListError::MalformedRow })?, Err(e) => { tracing::error!(url=url,error=?e,"Parsed instance URL is not valid"); return Err(InstanceListError::MalformedRow); } }; // map all remaining cols into Strings let columns: Vec<_> = cols .map(|col| { col.text().fold(String::new(), |mut acc, text| { acc.push_str(text); acc }) }) .collect(); if columns.len() < 4 { tracing::error!(instance_data=?columns,"Parsed instance missing fields, skipping!"); return Err(InstanceListError::MalformedRow); } let instance = InstanceParsed { domain, url, online: columns[0] == CHECKBOX, ssl_provider: columns[3].clone(), country: columns[2].clone(), }; Ok(instance) } } #[cfg(test)] mod test { use csv; use std::collections::HashMap; use tracing_test::traced_test; use super::*; #[test] #[traced_test] fn parse() { let html = include_str!("../test_data/instancelist.html"); let parser = InstanceParser::new(); let res = parser.parse_instancelist(html, &[], "", true).unwrap(); // writeback for new tests // write_data(res.values()); // adjust when updating the test html let expected: HashMap = expected_data() .into_iter() .map(|instance| (instance.domain.clone(), instance)) .collect(); assert_eq!(res.len(), expected.len()); for (_, instance) in res.iter() { assert_eq!(Some(instance), expected.get(&instance.domain)); } } fn expected_data() -> Vec { let file = std::fs::File::open("test_data/instancelist_expected.csv").unwrap(); let mut rdr = csv::Reader::from_reader(file); let mut vec = Vec::new(); for result in rdr.deserialize() { // Notice that we need to provide a type hint for automatic // deserialization. let record: InstanceParsed = result.unwrap(); vec.push(record); } vec } }