mirror of
https://github.com/4cuck/nitter-status.git
synced 2026-10-27 00:57:39 +00:00
parse and display instance countries
This commit is contained in:
4cuck committed
1 parent
b7498ceb3d
commit
e0826ab468
16 files changed
+2678
-1770
No files matched your search
+3
-1
@@ -22,8 +22,10 @@ PROFILE_NAME='@jack'
|
||||
PROFILE_POSTS_MIN=5
|
||||
# regex content to search for to verify RSS availability
|
||||
RSS_CONTENT='<rss xmlns\:atom'
|
||||
# additional instances to include always
|
||||
# additional instances to always include
|
||||
ADDITIONAL_HOSTS="https://nitter.net"
|
||||
# country for additional instances
|
||||
ADDITIONAL_HOSTS_COUNTRY="🇳🇱"
|
||||
# hosts known to be bad, as comma separated domains
|
||||
BAD_HOSTS="tweet.whateveritworks.org"
|
||||
# Referrer to use for requests
|
||||
|
||||
Generated
+23
@@ -896,6 +896,27 @@ dependencies = [
|
||||
"syn 2.0.29",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "csv"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086"
|
||||
dependencies = [
|
||||
"csv-core",
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "csv-core"
|
||||
version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deranged"
|
||||
version = "0.3.8"
|
||||
@@ -2753,12 +2774,14 @@ name = "scanner"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"csv",
|
||||
"entities",
|
||||
"git2",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"scraper",
|
||||
"sea-orm",
|
||||
"serde",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
|
||||
@@ -25,6 +25,7 @@ pub struct Model {
|
||||
pub enabled: bool,
|
||||
pub rss: bool,
|
||||
pub version: Option<String>,
|
||||
pub country: String,
|
||||
pub version_url: Option<String>,
|
||||
/// Last time the url and enabled were updated, *not* the rss
|
||||
pub updated: i64,
|
||||
@@ -36,6 +37,7 @@ pub enum Column {
|
||||
Domain,
|
||||
Url,
|
||||
Version,
|
||||
Country,
|
||||
VersionUrl,
|
||||
Enabled,
|
||||
Rss,
|
||||
@@ -63,10 +65,12 @@ impl ColumnTrait for Column {
|
||||
type EntityName = Entity;
|
||||
fn def(&self) -> ColumnDef {
|
||||
match self {
|
||||
// required for updated -> integer
|
||||
Self::Id => ColumnType::Integer.def(),
|
||||
Self::Domain => ColumnType::String(None).def(),
|
||||
Self::Url => ColumnType::String(None).def(),
|
||||
Self::Version => ColumnType::String(None).def().null(),
|
||||
Self::Country => ColumnType::String(None).def(),
|
||||
Self::VersionUrl => ColumnType::String(None).def().null(),
|
||||
Self::Enabled => ColumnType::Integer.def(),
|
||||
Self::Rss => ColumnType::Integer.def(),
|
||||
|
||||
@@ -47,4 +47,5 @@ pub struct CacheHost {
|
||||
pub is_latest_version: bool,
|
||||
/// Whether this host is known to be bad (ip blocking)
|
||||
pub is_bad_host: bool,
|
||||
pub country: String,
|
||||
}
|
||||
@@ -22,6 +22,8 @@ pub struct Config {
|
||||
pub rss_content: String,
|
||||
/// List of additional hosts to include during health checks
|
||||
pub additional_hosts: Vec<String>,
|
||||
/// Country to use for additional hosts
|
||||
pub additional_host_country: String,
|
||||
/// Referer to use
|
||||
pub referrer: String,
|
||||
/// Duration to average the ping/response times over
|
||||
@@ -49,6 +51,7 @@ impl Config {
|
||||
profile_posts_min: 5,
|
||||
rss_content: String::from(r#"<rss xmlns\:atom"#),
|
||||
additional_hosts: vec![String::from("https://nitter.net")],
|
||||
additional_host_country: String::from("🇳🇱"),
|
||||
referrer: String::from(""),
|
||||
ping_range: chrono::Duration::hours(3),
|
||||
auto_mute: true,
|
||||
|
||||
@@ -4,6 +4,8 @@ mod m20220101_000001_create_table;
|
||||
mod m20230729_010231_datetime_rowid;
|
||||
mod m20230729_230909_datetime_int_host;
|
||||
mod m20230803_154714_version_url;
|
||||
mod m20230829_201916_country;
|
||||
|
||||
|
||||
pub struct Migrator;
|
||||
|
||||
@@ -15,6 +17,7 @@ impl MigratorTrait for Migrator {
|
||||
Box::new(m20230729_010231_datetime_rowid::Migration),
|
||||
Box::new(m20230729_230909_datetime_int_host::Migration),
|
||||
Box::new(m20230803_154714_version_url::Migration),
|
||||
Box::new(m20230829_201916_country::Migration),
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
use sea_orm_migration::prelude::*;
|
||||
|
||||
#[derive(DeriveMigrationName)]
|
||||
pub struct Migration;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigrationTrait for Migration {
|
||||
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
let cmd = r#"ALTER TABLE "host" ADD COLUMN "country" TEXT NOT NULL DEFAULT '';"#;
|
||||
let db = manager.get_connection();
|
||||
db.execute_unprepared("BEGIN EXCLUSIVE").await?;
|
||||
tracing::info!("adding version_url column..");
|
||||
db.execute_unprepared(cmd).await?;
|
||||
db.execute_unprepared("COMMIT TRANSACTION").await?;
|
||||
db.execute_unprepared("VACUUM").await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
panic!("Can't migrate down");
|
||||
}
|
||||
}
|
||||
@@ -15,9 +15,12 @@ scraper = "0.17.1"
|
||||
chrono = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
git2 = "0.17.2"
|
||||
# testing
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
|
||||
[dev-dependencies]
|
||||
tracing-test = { workspace = true }
|
||||
csv = "1.2.2"
|
||||
|
||||
[dependencies.entities]
|
||||
path = "../entities"
|
||||
@@ -144,6 +144,7 @@ impl Scanner {
|
||||
is_upstream,
|
||||
version_url: host.version_url,
|
||||
is_bad_host,
|
||||
country: host.country,
|
||||
})
|
||||
}
|
||||
host_statistics.sort_unstable_by_key(|k| k.points);
|
||||
|
||||
+36
-658
@@ -22,6 +22,7 @@ pub enum InstanceListError {
|
||||
MalformedRow,
|
||||
}
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
#[cfg_attr(test, derive(serde::Serialize, serde::Deserialize))]
|
||||
pub struct InstanceParsed {
|
||||
/// URL without any login stuff
|
||||
pub domain: String,
|
||||
@@ -29,22 +30,22 @@ pub struct InstanceParsed {
|
||||
pub url: String,
|
||||
/// whether this instance is marked as online
|
||||
pub online: bool,
|
||||
/// whether this is marked as up to date
|
||||
pub up_to_date: bool,
|
||||
/// the SSL provider this instance supposedly has
|
||||
pub ssl_provider: String,
|
||||
/// the country for this instance
|
||||
pub country: String,
|
||||
}
|
||||
|
||||
impl InstanceParsed {
|
||||
/// just for testing purposes
|
||||
#[cfg(test)]
|
||||
fn from(domain: &str, url: &str, online: bool, up_to_date: bool, ssl_provider: &str) -> Self {
|
||||
fn from(domain: &str, url: &str, online: bool, ssl_provider: &str, country: &str) -> Self {
|
||||
Self {
|
||||
domain: domain.to_owned(),
|
||||
url: url.to_owned(),
|
||||
online,
|
||||
up_to_date,
|
||||
ssl_provider: ssl_provider.to_owned(),
|
||||
country: country.to_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -76,6 +77,7 @@ impl InstanceParser {
|
||||
&self,
|
||||
html: &str,
|
||||
additional_instances: &[String],
|
||||
additional_instances_country: &str,
|
||||
abort_on_err: bool,
|
||||
) -> Result<InstanceMap> {
|
||||
let fragment = Html::parse_fragment(html);
|
||||
@@ -118,8 +120,8 @@ impl InstanceParser {
|
||||
domain: domain.to_owned(),
|
||||
url: entry.clone(),
|
||||
online: true,
|
||||
up_to_date: true,
|
||||
ssl_provider: String::new(),
|
||||
country: additional_instances_country.to_owned(),
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -185,8 +187,8 @@ impl InstanceParser {
|
||||
domain,
|
||||
url,
|
||||
online: columns[0] == CHECKBOX,
|
||||
up_to_date: columns[1] == CHECKBOX,
|
||||
ssl_provider: columns[3].clone(),
|
||||
country: columns[2].clone(),
|
||||
};
|
||||
Ok(instance)
|
||||
}
|
||||
@@ -194,9 +196,10 @@ impl InstanceParser {
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use std::collections::HashMap;
|
||||
|
||||
use std::{collections::HashMap, io::BufWriter};
|
||||
use std::io::Write;
|
||||
use tracing_test::traced_test;
|
||||
use csv;
|
||||
|
||||
use super::*;
|
||||
#[test]
|
||||
@@ -204,8 +207,11 @@ mod test {
|
||||
fn parse() {
|
||||
let html = include_str!("../test_data/instancelist.html");
|
||||
let parser = InstanceParser::new();
|
||||
let res = parser.parse_instancelist(html, &[], true).unwrap();
|
||||
dbg!(&res);
|
||||
let res = parser.parse_instancelist(html, &[], "", true).unwrap();
|
||||
|
||||
// writeback for new tests
|
||||
// write_data(res.values());
|
||||
|
||||
|
||||
// adjust when updating the test html
|
||||
let expected: HashMap<String, InstanceParsed> = expected_data()
|
||||
@@ -218,654 +224,26 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
fn write_data<'a, T: Iterator<Item = &'a InstanceParsed>>(data: T) {
|
||||
// use to write a new csv
|
||||
let file = std::fs::File::create("test_data/instancelist_expected.csv").unwrap();
|
||||
let mut wtr = csv::Writer::from_writer(file);
|
||||
for entry in data {
|
||||
wtr.serialize(entry).unwrap();
|
||||
}
|
||||
wtr.flush().unwrap();
|
||||
}
|
||||
|
||||
fn expected_data() -> Vec<InstanceParsed> {
|
||||
vec![
|
||||
InstanceParsed::from(
|
||||
"nitter.lacontrevoie.fr",
|
||||
"https://nitter.lacontrevoie.fr",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.nixnet.services",
|
||||
"https://nitter:[email protected]",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.fdn.fr",
|
||||
"https://nitter.fdn.fr",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.1d4.us",
|
||||
"https://nitter.1d4.us",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.kavin.rocks",
|
||||
"https://nitter.kavin.rocks",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.unixfox.eu",
|
||||
"https://nitter.unixfox.eu",
|
||||
true,
|
||||
true,
|
||||
"Buypass AS-983163327",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.domain.glass",
|
||||
"https://nitter.domain.glass",
|
||||
true,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"birdsite.xanny.family",
|
||||
"https://birdsite.xanny.family",
|
||||
true,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.moomoo.me",
|
||||
"https://nitter.moomoo.me",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"bird.trom.tf",
|
||||
"https://bird.trom.tf",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.it",
|
||||
"https://nitter.it",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"twitter.owacon.moe",
|
||||
"https://twitter.owacon.moe",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"notabird.site",
|
||||
"https://notabird.site",
|
||||
false,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.weiler.rocks",
|
||||
"https://nitter.weiler.rocks",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.sethforprivacy.com",
|
||||
"https://nitter.sethforprivacy.com",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.nl",
|
||||
"https://nitter.nl",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.mint.lgbt",
|
||||
"https://nitter.mint.lgbt",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.esmailelbob.xyz",
|
||||
"https://nitter.esmailelbob.xyz",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"tw.artemislena.eu",
|
||||
"https://tw.artemislena.eu",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.tiekoetter.com",
|
||||
"https://nitter.tiekoetter.com",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.privacy.com.de",
|
||||
"https://nitter.privacy.com.de",
|
||||
true,
|
||||
true,
|
||||
"ZeroSSL",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.poast.org",
|
||||
"https://nitter.poast.org",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.bird.froth.zone",
|
||||
"https://nitter.bird.froth.zone",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.cz",
|
||||
"https://nitter.cz",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.privacydev.net",
|
||||
"https://nitter.privacydev.net",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"tweet.lambda.dance",
|
||||
"https://tweet.lambda.dance",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.kylrth.com",
|
||||
"https://nitter.kylrth.com",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.tokhmi.xyz",
|
||||
"https://nitter.tokhmi.xyz",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.catalyst.sx",
|
||||
"https://nitter.catalyst.sx",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"unofficialbird.com",
|
||||
"https://unofficialbird.com",
|
||||
true,
|
||||
true,
|
||||
"ZeroSSL",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.projectsegfau.lt",
|
||||
"https://nitter.projectsegfau.lt",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.eu.projectsegfau.lt",
|
||||
"https://nitter.eu.projectsegfau.lt",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.in.projectsegfau.lt",
|
||||
"https://nitter.in.projectsegfau.lt",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"singapore.unofficialbird.com",
|
||||
"https://singapore.unofficialbird.com",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"canada.unofficialbird.com",
|
||||
"https://canada.unofficialbird.com",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"india.unofficialbird.com",
|
||||
"https://india.unofficialbird.com",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nederland.unofficialbird.com",
|
||||
"https://nederland.unofficialbird.com",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"uk.unofficialbird.com",
|
||||
"https://uk.unofficialbird.com",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from("n.l5.ca", "https://n.l5.ca", true, false, "Let's Encrypt"),
|
||||
InstanceParsed::from(
|
||||
"nitter.slipfox.xyz",
|
||||
"https://nitter.slipfox.xyz",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.soopy.moe",
|
||||
"https://nitter.soopy.moe",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.qwik.space",
|
||||
"https://nitter.qwik.space",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"read.whatever.social",
|
||||
"https://read.whatever.social",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.rawbit.ninja",
|
||||
"https://nitter.rawbit.ninja",
|
||||
true,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nt.vern.cc",
|
||||
"https://nt.vern.cc",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.ir",
|
||||
"https://nitter.ir",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.privacytools.io",
|
||||
"https://nitter.privacytools.io",
|
||||
true,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"n.sneed.network",
|
||||
"https://n.sneed.network",
|
||||
true,
|
||||
true,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.smnz.de",
|
||||
"https://nitter.smnz.de",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.twei.space",
|
||||
"https://nitter.twei.space",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.inpt.fr",
|
||||
"https://nitter.inpt.fr",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.d420.de",
|
||||
"https://nitter.d420.de",
|
||||
true,
|
||||
true,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.caioalonso.com",
|
||||
"https://nitter.caioalonso.com",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.at",
|
||||
"https://nitter.at",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.pw",
|
||||
"https://nitter.pw",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.nicfab.eu",
|
||||
"https://nitter.nicfab.eu",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"bird.habedieeh.re",
|
||||
"https://bird.habedieeh.re",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.hostux.net",
|
||||
"https://nitter.hostux.net",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.adminforge.de",
|
||||
"https://nitter.adminforge.de",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.platypush.tech",
|
||||
"https://nitter.platypush.tech",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.pufe.org",
|
||||
"https://nitter.pufe.org",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.us.projectsegfau.lt",
|
||||
"https://nitter.us.projectsegfau.lt",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from("t.com.sb", "https://t.com.sb", true, false, "ZeroSSL"),
|
||||
InstanceParsed::from(
|
||||
"nitter.kling.gg",
|
||||
"https://nitter.kling.gg",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.riverside.rocks",
|
||||
"https://nitter.riverside.rocks",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.lunar.icu",
|
||||
"https://nitter.lunar.icu",
|
||||
true,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"twitter.moe.ngo",
|
||||
"https://twitter.moe.ngo",
|
||||
true,
|
||||
false,
|
||||
"Google Trust Services LLC",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.freedit.eu",
|
||||
"https://nitter.freedit.eu",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"ntr.frail.duckdns.org",
|
||||
"https://ntr.frail.duckdns.org",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"n.opnxng.com",
|
||||
"https://n.opnxng.com",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.tux.pizza",
|
||||
"https://nitter.tux.pizza",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"t.floss.media",
|
||||
"https://t.floss.media",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"twit.hell.rodeo",
|
||||
"https://twit.hell.rodeo",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.nachtalb.io",
|
||||
"https://nitter.nachtalb.io",
|
||||
false,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"n.quadtr.ee",
|
||||
"https://n.quadtr.ee",
|
||||
false,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.altgr.xyz",
|
||||
"https://nitter.altgr.xyz",
|
||||
true,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"jote.lile.cl",
|
||||
"https://jote.lile.cl",
|
||||
true,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.one",
|
||||
"https://nitter.one",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.instances.cc",
|
||||
"https://nitter.instances.cc",
|
||||
false,
|
||||
false,
|
||||
"Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.io.lol",
|
||||
"https://nitter.io.lol",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.hu",
|
||||
"https://nitter.hu",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.no-logs.com",
|
||||
"https://nitter.no-logs.com",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.ftw.lol",
|
||||
"https://nitter:[email protected]",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"tweet.whateveritworks.org",
|
||||
"https://tweet.whateveritworks.org",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt + Cloudflare",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.fediflix.org",
|
||||
"https://nitter.fediflix.org",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.nohost.network",
|
||||
"https://nitter.nohost.network",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"twt.funami.tech",
|
||||
"https://twt.funami.tech",
|
||||
false,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.simpleprivacy.fr",
|
||||
"https://nitter.simpleprivacy.fr",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.onthescent.xyz",
|
||||
"https://nitter.onthescent.xyz",
|
||||
true,
|
||||
false,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.x86-64-unknown-linux-gnu.zip",
|
||||
"https://nitter.x86-64-unknown-linux-gnu.zip",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.private.coffee",
|
||||
"https://nitter.private.coffee",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.oksocial.net",
|
||||
"https://nitter.oksocial.net",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.services.woodland.cafe",
|
||||
"https://nitter.services.woodland.cafe",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
InstanceParsed::from(
|
||||
"nitter.datura.network",
|
||||
"https://nitter.datura.network",
|
||||
true,
|
||||
true,
|
||||
"Let's Encrypt",
|
||||
),
|
||||
]
|
||||
let file = std::fs::File::open("test_data/instancelist_expected.csv").unwrap();
|
||||
let mut rdr = csv::Reader::from_reader(file);
|
||||
let mut vec = Vec::new();
|
||||
for result in rdr.deserialize() {
|
||||
// Notice that we need to provide a type hint for automatic
|
||||
// deserialization.
|
||||
let record: InstanceParsed = result.unwrap();
|
||||
vec.push(record);
|
||||
}
|
||||
vec
|
||||
}
|
||||
}
|
||||
+4
-1
@@ -225,10 +225,11 @@ impl Scanner {
|
||||
#[instrument]
|
||||
async fn update_instacelist(&mut self) -> Result<()> {
|
||||
let start = Instant::now();
|
||||
let html = self.fetch_instancelist().await?;
|
||||
let html: String = self.fetch_instancelist().await?;
|
||||
let parsed_instances = self.inner.instance_parser.parse_instancelist(
|
||||
&html,
|
||||
&self.inner.config.additional_hosts,
|
||||
&self.inner.config.additional_host_country,
|
||||
false,
|
||||
)?;
|
||||
|
||||
@@ -292,6 +293,7 @@ impl Scanner {
|
||||
host::ActiveModel {
|
||||
id: ActiveValue::NotSet,
|
||||
domain: ActiveValue::Set(instance.domain),
|
||||
country: ActiveValue::Set(instance.country),
|
||||
url: ActiveValue::Set(instance.url),
|
||||
enabled: ActiveValue::Set(true),
|
||||
version: ActiveValue::Set(version),
|
||||
@@ -312,6 +314,7 @@ impl Scanner {
|
||||
host::Column::Rss,
|
||||
host::Column::Version,
|
||||
host::Column::VersionUrl,
|
||||
host::Column::Country,
|
||||
])
|
||||
.to_owned(),
|
||||
)
|
||||
|
||||
+2487
-1102
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,76 @@
|
||||
domain,url,online,ssl_provider,country
|
||||
nitter.privacydev.net,https://nitter.privacydev.net,true,Let's Encrypt,🇫🇷
|
||||
nitter.adminforge.de,https://nitter.adminforge.de,true,Let's Encrypt,🇩🇪
|
||||
bird.habedieeh.re,https://bird.habedieeh.re,true,Let's Encrypt,🇨🇦
|
||||
nitter.platypush.tech,https://nitter.platypush.tech,false,Let's Encrypt,🇩🇪
|
||||
nitter.us.projectsegfau.lt,https://nitter.us.projectsegfau.lt,true,Let's Encrypt,🇺🇸
|
||||
singapore.unofficialbird.com,https://singapore.unofficialbird.com,true,Let's Encrypt,🇸🇬
|
||||
nitter.smnz.de,https://nitter.smnz.de,false,Let's Encrypt,🇩🇪
|
||||
nitter.one,https://nitter.one,true,Let's Encrypt,🇸🇪
|
||||
nitter.fediflix.org,https://nitter.fediflix.org,true,Let's Encrypt,🇩🇪
|
||||
tweet.lambda.dance,https://tweet.lambda.dance,true,Let's Encrypt,🇺🇸
|
||||
nitter.mint.lgbt,https://nitter.mint.lgbt,true,Let's Encrypt,🇬🇧
|
||||
nitter.poast.org,https://nitter.poast.org,true,Let's Encrypt,🇺🇸
|
||||
n.sneed.network,https://n.sneed.network,true,Cloudflare,🇦🇺
|
||||
nitter.inpt.fr,https://nitter.inpt.fr,true,Let's Encrypt,🇫🇷
|
||||
unofficialbird.com,https://unofficialbird.com,true,ZeroSSL,🇺🇸
|
||||
nederland.unofficialbird.com,https://nederland.unofficialbird.com,true,Let's Encrypt,🇳🇱
|
||||
nitter.twei.space,https://nitter.twei.space,true,Let's Encrypt,🇦🇹
|
||||
nitter.rawbit.ninja,https://nitter.rawbit.ninja,true,Cloudflare,🇺🇸
|
||||
nitter.bird.froth.zone,https://nitter.bird.froth.zone,true,Let's Encrypt,🇨🇦
|
||||
twitter.moe.ngo,https://twitter.moe.ngo,true,Google Trust Services LLC,🇮🇩
|
||||
nitter.fdn.fr,https://nitter.fdn.fr,true,Let's Encrypt,🇫🇷
|
||||
nitter.onthescent.xyz,https://nitter.onthescent.xyz,true,Let's Encrypt,🇺🇸
|
||||
uk.unofficialbird.com,https://uk.unofficialbird.com,true,Let's Encrypt,🇬🇧
|
||||
nitter.perennialte.ch,https://nitter.perennialte.ch,true,Google Trust Services LLC + Cloudflare,🇦🇺
|
||||
nitter.eu.projectsegfau.lt,https://nitter.eu.projectsegfau.lt,true,Let's Encrypt,🇱🇺
|
||||
nitter.kylrth.com,https://nitter.kylrth.com,true,Let's Encrypt,🇨🇦
|
||||
nitter.in.projectsegfau.lt,https://nitter.in.projectsegfau.lt,true,Let's Encrypt,🇮🇳
|
||||
nitter.privacy.com.de,https://nitter.privacy.com.de,true,ZeroSSL,🇩🇪
|
||||
nitter.unixfox.eu,https://nitter.unixfox.eu,true,Buypass AS-983163327,🇩🇪
|
||||
nitter.dafriser.be,https://nitter.dafriser.be,true,Let's Encrypt,🇩🇪
|
||||
nt.vern.cc,https://nt.vern.cc,false,Let's Encrypt,🇺🇸
|
||||
nitter.io.lol,https://nitter.io.lol,true,Let's Encrypt,🇺🇸
|
||||
india.unofficialbird.com,https://india.unofficialbird.com,true,Let's Encrypt,🇮🇳
|
||||
t.com.sb,https://t.com.sb,true,ZeroSSL,🇪🇪
|
||||
nitter.tux.pizza,https://nitter.tux.pizza,true,Let's Encrypt,🇺🇸
|
||||
nitter.weiler.rocks,https://nitter.weiler.rocks,true,Let's Encrypt,🇩🇪
|
||||
nitter.nicfab.eu,https://nitter.nicfab.eu,true,Let's Encrypt,🇮🇹
|
||||
nitter.tiekoetter.com,https://nitter.tiekoetter.com,true,Let's Encrypt,🇩🇪
|
||||
nitter.salastil.com,https://nitter.salastil.com,true,Let's Encrypt,🇺🇸
|
||||
nitter.esmailelbob.xyz,https://nitter.esmailelbob.xyz,true,Let's Encrypt,🇨🇦
|
||||
nitter.freedit.eu,https://nitter.freedit.eu,true,Let's Encrypt,🇺🇸
|
||||
nitter.sethforprivacy.com,https://nitter.sethforprivacy.com,true,Let's Encrypt,🇩🇪
|
||||
nitter.kavin.rocks,https://nitter.kavin.rocks,true,Let's Encrypt,🇮🇳
|
||||
nitter.services.woodland.cafe,https://nitter.services.woodland.cafe,true,Let's Encrypt,🇩🇪
|
||||
nitter.hostux.net,https://nitter.hostux.net,true,Let's Encrypt,🇫🇷
|
||||
nitter.1d4.us,https://nitter.1d4.us,true,Let's Encrypt,🇺🇸
|
||||
nitter.cz,https://nitter.cz,true,Let's Encrypt,🇨🇿
|
||||
nitter.nl,https://nitter.nl,true,Let's Encrypt,🇳🇱
|
||||
nitter.altgr.xyz,https://nitter.altgr.xyz,true,Cloudflare,🇨🇱
|
||||
nitter.x86-64-unknown-linux-gnu.zip,https://nitter.x86-64-unknown-linux-gnu.zip,true,Let's Encrypt,🇩🇪
|
||||
tw.artemislena.eu,https://tw.artemislena.eu,true,Let's Encrypt,🇩🇪
|
||||
nitter.private.coffee,https://nitter.private.coffee,true,Let's Encrypt,🇦🇹
|
||||
nitter.simpleprivacy.fr,https://nitter.simpleprivacy.fr,true,Let's Encrypt,🇫🇷
|
||||
nitter.kling.gg,https://nitter.kling.gg,true,Let's Encrypt,🇳🇱
|
||||
nitter.qwik.space,https://nitter.qwik.space,true,Let's Encrypt,🇸🇪
|
||||
nitter.soopy.moe,https://nitter.soopy.moe,true,Let's Encrypt,🇭🇰
|
||||
nitter.caioalonso.com,https://nitter.caioalonso.com,false,Let's Encrypt,🇺🇸
|
||||
nitter.laserdisc.tokyo,https://nitter.laserdisc.tokyo,true,Let's Encrypt,🇸🇪
|
||||
tweet.whateveritworks.org,https://tweet.whateveritworks.org,true,Let's Encrypt + Cloudflare,🇩🇪
|
||||
jote.lile.cl,https://jote.lile.cl,true,Cloudflare,🇨🇱
|
||||
nitter.nixnet.services,https://nitter:[email protected],true,Let's Encrypt,🇩🇪
|
||||
bird.trom.tf,https://bird.trom.tf,false,Let's Encrypt,🇩🇪
|
||||
nitter.at,https://nitter.at,true,Let's Encrypt,🇩🇪
|
||||
nitter.no-logs.com,https://nitter.no-logs.com,true,Let's Encrypt,🇸🇪
|
||||
nitter.it,https://nitter.it,true,Let's Encrypt,🇫🇮
|
||||
canada.unofficialbird.com,https://canada.unofficialbird.com,true,Let's Encrypt,🇨🇦
|
||||
nitter.lacontrevoie.fr,https://nitter.lacontrevoie.fr,true,Let's Encrypt,🇫🇷
|
||||
nitter.moomoo.me,https://nitter.moomoo.me,true,Let's Encrypt,🇺🇸
|
||||
nitter.projectsegfau.lt,https://nitter.projectsegfau.lt,true,Let's Encrypt,GeoDNS
|
||||
nitter.oksocial.net,https://nitter.oksocial.net,true,Let's Encrypt,🇺🇸
|
||||
nitter.catsarch.com,https://nitter.catsarch.com,true,Let's Encrypt,🇺🇸
|
||||
nitter.datura.network,https://nitter.datura.network,true,Let's Encrypt,🇩🇪
|
||||
nitter.d420.de,https://nitter.d420.de,true,Cloudflare,🇩🇪
|
||||
nitter.lunar.icu,https://nitter.lunar.icu,true,Cloudflare,🇩🇪
|
||||
nitter.nohost.network,https://nitter.nohost.network,true,Let's Encrypt,🇲🇽
|
||||
|
@@ -35,7 +35,7 @@
|
||||
<li><code>Points</code> is a weighted instance score based on the availability over the last 3h, 30 and 120 days, together with the version.</li>
|
||||
</ul>
|
||||
|
||||
<p><a href="https://github.com/4cuck/nitter-status">Sourcecode</a></p>
|
||||
<p><a rel="nofollow" href="https://github.com/4cuck/nitter-status">Sourcecode</a></p>
|
||||
<p>License: AGPL3</p>
|
||||
</body>
|
||||
</html>
|
||||
@@ -4,7 +4,6 @@
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<link href="/static/bootstrap.min.css" rel="stylesheet">
|
||||
<meta name="robots" content="noindex,nofollow" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Nitter Instance Health</title>
|
||||
</head>
|
||||
@@ -12,16 +11,17 @@
|
||||
<div class="container">
|
||||
<h1>Nitter Instance Uptime & Health</h1>
|
||||
<p><a href="/about">About</a></p>
|
||||
<p>Please use <a href="/api/v1/instances">/api/v1/instances</a> for bots.</p>
|
||||
<p>Last Updated {{last_updated}} <a href="https://time.is/UTC">UTC</a>.</p>
|
||||
<p>Please use <a rel="nofollow" href="/api/v1/instances">/api/v1/instances</a> for bots.</p>
|
||||
<p>Last Updated {{last_updated}} <a rel="nofollow" href="https://time.is/UTC">UTC</a>.</p>
|
||||
<div class="table-responsive">
|
||||
<table class="table sortable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col">Instance</th>
|
||||
<th scope="col">Country</th>
|
||||
<th scope="col">Healthy</th>
|
||||
<th scope="col">Response Time</th>
|
||||
<th scope="col">Average</th>
|
||||
<th scope="col">Response Times</th>
|
||||
<th scope="col">Average (ms)</th>
|
||||
<th scope="col">RSS</th>
|
||||
<th scope="col">Version</th>
|
||||
<th scope="col">Points</th>
|
||||
@@ -30,7 +30,8 @@
|
||||
<tbody>
|
||||
{% for host in instances -%}
|
||||
<tr>
|
||||
<td><a href="{{host.url}}">{{host.domain}}</a></td>
|
||||
<td><a rel="nofollow external" href="{{host.url}}">{{host.domain}}</a></td>
|
||||
<td>{{host.country}}</td>
|
||||
<td>
|
||||
{%- if host.is_bad_host -%}
|
||||
<div title="Host known for blocking healthchecks">❓</div>
|
||||
@@ -42,7 +43,7 @@
|
||||
<td>
|
||||
{%- if host.recent_pings|length > 0 -%}
|
||||
{%- set height = 57 -%}
|
||||
{%- set width = 182 -%}
|
||||
{%- set width = 100 -%}
|
||||
{%- set amount = host.recent_pings|length -%}
|
||||
{%- set width_bar = width / amount -%}
|
||||
{%- set offset = 5 -%}
|
||||
|
||||
@@ -117,6 +117,8 @@ fn read_scanner_cfg() -> miette::Result<ScannerConfig> {
|
||||
.expect("PROFILE_POSTS_MIN must be a positive number");
|
||||
let rss_content = require_env_str("RSS_CONTENT")?;
|
||||
let additional_hosts: Vec<String> = require_env_vec_str("ADDITIONAL_HOSTS")?;
|
||||
let additional_host_country = require_env_str("ADDITIONAL_HOSTS_COUNTRY")?;
|
||||
let rss_content = require_env_str("RSS_CONTENT")?;
|
||||
let bad_hosts: Vec<String> = require_env_vec_str("BAD_HOSTS")?;
|
||||
let referrer = require_env_str("REFERER")?;
|
||||
let auto_mute = require_env_str("AUTO_MUTE")? == "true";
|
||||
@@ -134,6 +136,7 @@ fn read_scanner_cfg() -> miette::Result<ScannerConfig> {
|
||||
profile_posts_min,
|
||||
rss_content,
|
||||
additional_hosts,
|
||||
additional_host_country,
|
||||
referrer,
|
||||
ping_range: chrono::Duration::hours(ping_range as _),
|
||||
auto_mute,
|
||||
|
||||
Reference in new issue
Block a user