Add nitter.cf instance branding, search dedupe, and safer API errors.

Keep keyword search from repeating the same viral tweet on every retweet, encode pagination cursors, and treat X 404s as not-found instead of 500s. Sessions, live config, and logs stay untracked.

Co-authored-by: Cursor <[email protected]>

.gitignore

@@ -26,6 +26,9 @@
 # Session example
 !sessions.example.jsonl
 
+# Live instance files stay ignored even if allowlists are widened later
+# sessions.jsonl, config/teapot.toml, session-limits.json, ecosystem.config.cjs, *.log
+
 # Source
 !src/
 !src/**

config/teapot.example.toml

@@ -82,7 +82,7 @@ externalDomain = ""
 
 [preferences]
 # Default theme
-theme = "teapot"
+theme = "black"
 
 # URL replacements for privacy
 replaceReddit  = ""

public/css/navbar.css

@@ -124,3 +124,12 @@ nav {
   background-color: var(--bg_hover);
   text-decoration: none;
 }
+
+.nav-icon.sister-site img {
+  display: block;
+  width: 22px;
+  height: 22px;
+  object-fit: cover;
+  border-radius: 4px;
+  image-rendering: pixelated;
+}

public/robots.txt

@@ -1,5 +1,25 @@
+# nitter.cf is a public Nitter replacement (nitter.net shut down).
+# Crawl the instance pages. Do not spider every proxied Twitter profile —
+# those hit an upstream rate limit and are not the indexable site.
+
 User-agent: *
+Allow: /$
+Allow: /about
+Allow: /css/
+Allow: /js/
+Allow: /fonts/
+Allow: /public/
+Allow: /logo
+Allow: /favicon
+Allow: /apple-touch-icon
+Allow: /site.webmanifest
+Allow: /robots.txt
+Allow: /sitemap.xml
 Disallow: /
-Crawl-delay: 1
+Crawl-delay: 2
+
+# Card unfurls when someone shares a nitter.cf link.
 User-agent: Twitterbot
 Disallow:
+
+Sitemap: https://nitter.cf/sitemap.xml

public/sitemap.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <url>
+    <loc>https://nitter.cf/</loc>
+    <changefreq>daily</changefreq>
+    <priority>1.0</priority>
+  </url>
+  <url>
+    <loc>https://nitter.cf/about</loc>
+    <changefreq>weekly</changefreq>
+    <priority>0.8</priority>
+  </url>
+  <url>
+    <loc>https://xitter.cf/</loc>
+    <changefreq>daily</changefreq>
+    <priority>0.6</priority>
+  </url>
+  <url>
+    <loc>https://xitter.cf/about</loc>
+    <changefreq>weekly</changefreq>
+    <priority>0.5</priority>
+  </url>
+</urlset>

src/api/client.rs

@@ -323,6 +323,17 @@ impl ApiClient {
          return Ok(());
       };
 
+      if error.code == 214 {
+         if error.message.contains("Unknown request cursor") {
+            return Err(Error::InvalidUrl(
+               "This page has expired. Go back and try again.".into(),
+            ));
+         }
+         if error.message.contains("denylist") {
+            return Err(Error::InvalidUrl(error.message.clone()));
+         }
+      }
+
       if let Some(twitter_err) = TwitterError::from_code(error.code) {
          return match twitter_err {
             TwitterError::UserNotFound | TwitterError::NoUserMatches => {
@@ -599,13 +610,13 @@ impl ApiClient {
       if !response.status().is_success() {
          let status = response.status();
          let body = response.text().await.unwrap_or_default();
-         tracing::error!(
-            session_id = session.id,
-            session_user = %session.username,
-            "API request failed: {status} - {body}"
-         );
 
          if status.as_u16() == 429 {
+            tracing::warn!(
+               session_id = session.id,
+               session_user = %session.username,
+               "API request failed: {status} - {body}"
+            );
             if !limit_recorded {
                self
                   .sessions
@@ -618,10 +629,40 @@ impl ApiClient {
          // Only 401 means the credentials themselves were refused. A 403 is an
          // authenticated request denied a particular resource.
          if status.as_u16() == 401 {
+            tracing::error!(
+               session_id = session.id,
+               session_user = %session.username,
+               "API request failed: {status} - {body}"
+            );
             self.sessions.mark_rejected(session.id).await;
             return Err(Error::SessionRejected(format!("Status {status}: {body}")));
          }
 
+         // Deleted tweets, missing users, and dead media URLs come back as
+         // HTTP 404. Those are not instance faults.
+         if status.as_u16() == 404 {
+            tracing::debug!(
+               session_id = session.id,
+               session_user = %session.username,
+               "API 404: {body}"
+            );
+            return Err(Error::NotFound("Not found".into()));
+         }
+
+         if status.is_server_error() {
+            tracing::error!(
+               session_id = session.id,
+               session_user = %session.username,
+               "API request failed: {status} - {body}"
+            );
+         } else {
+            tracing::warn!(
+               session_id = session.id,
+               session_user = %session.username,
+               "API request failed: {status} - {body}"
+            );
+         }
+
          return Err(Error::TwitterApi(format!("Status {status}: {body}")));
       }
 

src/api/client_endpoints.rs

@@ -42,6 +42,21 @@ use super::{
    timeout,
 };
 
+fn search_page_is_stuck(timeline: &Timeline) -> bool {
+   let mut ids = timeline.content.iter().flatten().map(Tweet::original_id);
+   let Some(first) = ids.next() else {
+      return false;
+   };
+   let mut count = 1_usize;
+   for id in ids {
+      if id != first {
+         return false;
+      }
+      count += 1;
+   }
+   count >= 3
+}
+
 #[expect(
    clippy::multiple_inherent_impl,
    reason = "endpoint methods are split from transport/authentication internals"
@@ -354,13 +369,11 @@ impl ApiClient {
          .await?;
       let mut timeline = parser::parse_search_timeline(&data);
 
-      // When no more items are available the API returns the last page again.
-      // Detect this by comparing the first 64 chars of the input and output cursors.
+      // X often repeats the last page with a new cursor that still shares a
+      // long prefix. Stop only when the cursor is unchanged or this page is
+      // the same tweet over and over.
       if let Some(after) = cursor
-         && let Some(ref bottom) = timeline.bottom
-         && let Some(after_prefix) = after.get(..64)
-         && let Some(bottom_prefix) = bottom.get(..64)
-         && after_prefix == bottom_prefix
+         && (timeline.bottom.as_deref() == Some(after) || search_page_is_stuck(&timeline))
       {
          timeline.content.clear();
          timeline.bottom = None;

src/api/parser/timeline.rs

@@ -8,10 +8,7 @@ use crate::{
       SearchTimelineData,
       UserTimelineData,
    },
-   error::{
-      Error,
-      Result,
-   },
+   error::Result,
    types::{
       Query,
       Timeline,
@@ -21,14 +18,16 @@ use crate::{
 
 /// Parse a user timeline (tweets, media, tweets-and-replies).
 pub fn parse_timeline(data: &UserTimelineData) -> Result<Timeline> {
-   let instructions = data
+   let Some(instructions) = data
       .user
       .as_ref()
       .or(data.user_result.as_ref())
       .and_then(|nested| nested.result.as_ref())
       .map(super::super::schema::TimelineResultData::instructions)
       .filter(|instr| !instr.is_empty())
-      .ok_or_else(|| Error::Internal("Timeline instructions not found".into()))?;
+   else {
+      return Ok(Timeline::default());
+   };
 
    parse_timeline_instructions(instructions)
 }
@@ -37,7 +36,7 @@ pub fn parse_timeline(data: &UserTimelineData) -> Result<Timeline> {
 pub fn parse_list_timeline(data: &ListTimelineData) -> Result<Timeline> {
    let instructions = data.instructions();
    if instructions.is_empty() {
-      return Err(Error::Internal("Timeline instructions not found".into()));
+      return Ok(Timeline::default());
    }
    parse_timeline_instructions(instructions)
 }

src/config.rs

@@ -200,7 +200,7 @@ const fn default_max_concurrent_reqs() -> u32 {
    2
 }
 fn default_theme() -> String {
-   "teapot".to_owned()
+   "black".to_owned()
 }
 fn default_paid_emoji() -> String {
    "🤝".to_owned()

src/main.rs

@@ -176,6 +176,7 @@ async fn main() -> eyre::Result<()> {
         .route_service("/apple-touch-icon.png", ServeFile::new(format!("{static_dir}/apple-touch-icon.png")))
         .route_service("/site.webmanifest", ServeFile::new(format!("{static_dir}/site.webmanifest")))
         .route_service("/robots.txt", ServeFile::new(format!("{static_dir}/robots.txt")))
+        .route_service("/sitemap.xml", ServeFile::new(format!("{static_dir}/sitemap.xml")))
         .route_service("/opensearch.xml", ServeFile::new(format!("{static_dir}/opensearch.xml")))
         .layer(middleware::from_fn(routes::prefs_middleware))
         .layer(middleware::from_fn_with_state(

src/routes/pages.rs

@@ -11,7 +11,10 @@ use axum::{
    routing::get,
 };
 use axum_extra::extract::CookieJar;
-use maud::html;
+use maud::{
+   PreEscaped,
+   html,
+};
 
 use crate::{
    AppState,
@@ -22,6 +25,10 @@ use crate::{
    },
 };
 
+const HOME_DESCRIPTION: &str = "nitter.cf is a public Nitter replacement — a privacy-focused Twitter/X frontend. nitter.net shut down; this instance is a drop-in successor.";
+const ABOUT_DESCRIPTION: &str = "About nitter.cf, a public Nitter replacement. After nitter.net shut down, this teapot instance lets you browse Twitter/X without JavaScript or tracking.";
+const HOME_JSON_LD: &str = r#"{"@context":"https://schema.org","@type":"WebSite","name":"nitter.cf","alternateName":["Nitter","teapot","xitter.cf"],"url":"https://nitter.cf/","description":"Public Nitter replacement. nitter.net shut down; nitter.cf is a privacy-focused Twitter/X frontend.","sameAs":["https://xitter.cf"]}"#;
+
 /// The pages teapot serves itself rather than fetching from X.
 pub fn router() -> Router<AppState> {
    Router::new()
@@ -34,24 +41,54 @@ pub fn router() -> Router<AppState> {
 async fn home(State(state): State<AppState>, jar: CookieJar) -> impl IntoResponse {
    let prefs = Prefs::from_cookies(&jar, &state.config);
    let content = search_view::render_search_page();
+   let head = html! {
+       link rel="canonical" href="https://nitter.cf/";
+       meta property="og:type" content="website";
+       meta property="og:url" content="https://nitter.cf/";
+       script type="application/ld+json" {
+           (PreEscaped(HOME_JSON_LD))
+       }
+   };
 
-   let markup = PageLayout::new(&state.config, "Home", content)
-      .description("A privacy-focused Twitter/X frontend")
+   let markup = PageLayout::new(&state.config, "nitter.cf — Nitter replacement", content)
+      .description(HOME_DESCRIPTION)
       .prefs(&prefs)
+      .head_extra(&head)
       .render();
    Html(markup.into_string())
 }
 
 async fn about(State(state): State<AppState>, jar: CookieJar) -> impl IntoResponse {
    let prefs = Prefs::from_cookies(&jar, &state.config);
+   let head = html! {
+       link rel="canonical" href="https://nitter.cf/about";
+       meta property="og:type" content="website";
+       meta property="og:url" content="https://nitter.cf/about";
+   };
    let content = html! {
        div class="overlay-panel" {
-           h1 { "About" }
+           h1 { "About nitter.cf" }
 
            p {
-               "teapot is a free and open source alternative Twitter front-end focused on privacy and performance. "
-               "The source is available on GitHub at "
-               a href="https://github.com/amaanq/teapot" { "https://github.com/amaanq/teapot" }
+               strong { "nitter.cf" }
+               " is a public "
+               a href="https://github.com/zedeus/nitter" { "Nitter" }
+               " replacement — a privacy-focused Twitter/X frontend. "
+               "nitter.net shut down; this instance is meant as a drop-in successor. "
+               "Same URL style ("
+               code { "/username" }
+               ", "
+               code { "/username/status/id" }
+               "), no JavaScript required, and Twitter never sees your IP or fingerprint. "
+               "Also available at "
+               a href="https://xitter.cf" { "xitter.cf" }
+               "."
+           }
+
+           p {
+               "This site runs "
+               a href="https://github.com/4cuck/teapot" { "teapot" }
+               ", a free and open source alternative Twitter front-end focused on privacy and performance."
            }
 
            ul {
@@ -103,9 +140,10 @@ async fn about(State(state): State<AppState>, jar: CookieJar) -> impl IntoRespon
        }
    };
 
-   let markup = PageLayout::new(&state.config, "About", content)
-      .description("About teapot")
+   let markup = PageLayout::new(&state.config, "About nitter.cf", content)
+      .description(ABOUT_DESCRIPTION)
       .prefs(&prefs)
+      .head_extra(&head)
       .render();
    Html(markup.into_string())
 }

src/routes/search.rs

@@ -26,6 +26,7 @@ use crate::{
       Prefs,
       Query,
       QueryKind,
+      Tweet,
    },
    views::{
       layout,
@@ -33,6 +34,27 @@ use crate::{
    },
 };
 
+/// One card per original tweet. Latest search otherwise lists every retweet
+/// of a viral post as its own row.
+fn dedup_search_tweets(tweets: Vec<Tweet>) -> Vec<Tweet> {
+   let mut order = Vec::new();
+   let mut best = std::collections::HashMap::new();
+   for tweet in tweets {
+      let id = tweet.original_id();
+      match best.get(&id) {
+         None => {
+            order.push(id);
+            best.insert(id, tweet);
+         },
+         Some(existing) if existing.retweet.is_some() && tweet.retweet.is_none() => {
+            best.insert(id, tweet);
+         },
+         _ => {},
+      }
+   }
+   order.into_iter().filter_map(|id| best.remove(&id)).collect()
+}
+
 #[derive(Debug, Default, Deserialize)]
 pub struct SearchQuery {
    #[serde(rename = "q")]
@@ -323,7 +345,7 @@ async fn search(
 
       match search_result {
          Ok(timeline) => {
-            let tweets = timeline.content.into_iter().flatten().collect::<Vec<_>>();
+            let tweets = dedup_search_tweets(timeline.content.into_iter().flatten().collect());
             let cursor = timeline.bottom.as_deref();
 
             // Display the original user query, not the API query

src/types/query.rs

@@ -90,8 +90,10 @@ impl Query {
          param.push_str("(filter:self_threads OR -filter:replies) ");
       }
 
-      // Add include:nativeretweets unless explicitly excluded
-      if !self.excludes.contains(&"nativeretweets".to_owned()) {
+      // Only force-include retweets for from:user searches (a profile-style
+      // feed). Keyword search would otherwise repeat the same viral tweet
+      // once per retweeter.
+      if !self.from_user.is_empty() && !self.excludes.contains(&"nativeretweets".to_owned()) {
          param.push_str("include:nativeretweets ");
       }
 
@@ -326,4 +328,16 @@ mod tests {
       let query = Query::parse("rust -spam", QueryKind::Posts);
       assert!(query.to_url_params().contains("q=rust%20%2Dspam"));
    }
+
+   #[test]
+   fn keyword_search_does_not_force_nativeretweets() {
+      let query = Query::parse("marefair", QueryKind::Posts);
+      assert!(!query.build().contains("include:nativeretweets"));
+   }
+
+   #[test]
+   fn from_user_search_still_includes_nativeretweets() {
+      let query = Query::parse("from:alice hello", QueryKind::Posts);
+      assert!(query.build().contains("include:nativeretweets"));
+   }
 }

src/types/tweet.rs

@@ -358,4 +358,12 @@ impl Tweet {
          || !self.additional_videos.is_empty()
          || self.gif.is_some()
    }
+
+   /// Status id of the original post. Retweet wrappers have their own id.
+   pub fn original_id(&self) -> i64 {
+      match self.retweet.as_ref() {
+         Some(inner) if inner.id != 0 => inner.id,
+         _ => self.id,
+      }
+   }
 }

src/views/layout.rs

@@ -196,6 +196,10 @@ impl<'a> PageLayout<'a> {
                       title { (self.title) " | " (self.config.server.title) }
                   }
 
+                  @if !self.description.is_empty() {
+                      meta name="description" content=(strip_html(self.description));
+                  }
+
                   // OpenGraph meta tags
                   @if !self.custom_og {
                       meta property="og:site_name" content=(self.config.server.title);
@@ -251,6 +255,22 @@ impl<'a> PageLayout<'a> {
    }
 }
 
+const SOYJAK_ICONS: &[&str] = &[
+   "/public/icons/soyjak-babyjak.png",
+   "/public/icons/soyjak-cobson.png",
+   "/public/icons/soyjak-feraljak.png",
+   "/public/icons/soyjak-flartson.png",
+   "/public/icons/soyjak-gapejak.png",
+   "/public/icons/soyjak-soyjak.png",
+];
+
+fn random_soyjak_icon() -> &'static str {
+   let idx = std::time::SystemTime::now()
+      .duration_since(std::time::UNIX_EPOCH)
+      .map_or(0, |elapsed| elapsed.subsec_nanos() as usize % SOYJAK_ICONS.len());
+   SOYJAK_ICONS[idx]
+}
+
 /// Render navbar with all context.
 pub fn render_navbar_full(config: &Config, rss: &str, canonical: &str, referer: &str) -> Markup {
    let canonical = if canonical.is_empty() {
@@ -278,6 +298,15 @@ pub fn render_navbar_full(config: &Config, rss: &str, canonical: &str, referer:
                    img class="site-logo" src="/logo.svg" alt=(format!("{} logo", config.server.title));
                }
                div class="nav-item right" {
+                   a class="nav-icon sister-site" title="owo.vg — 4chan frontend" href="https://owo.vg" target="_blank" rel="noopener noreferrer" {
+                       img src="/public/icons/4chan.png" alt="4chan" width="22" height="22";
+                   }
+                   a class="nav-icon sister-site" title="soyjak.cf — soyjak.st frontend" href="https://soyjak.cf" target="_blank" rel="noopener noreferrer" {
+                       img src=(random_soyjak_icon()) alt="soyjak" width="22" height="22";
+                   }
+                   a class="nav-icon sister-site" title="awoo.cf — imageboard" href="https://awoo.cf" target="_blank" rel="noopener noreferrer" {
+                       img src="/public/icons/awoo.png" alt="awoo" width="22" height="22";
+                   }
                    a class="nav-icon" title="Search" href="/search" {
                        span class="icon-search" aria-hidden="true" {}
                    }

src/views/search.rs

@@ -18,6 +18,24 @@ use crate::{
    },
 };
 
+/// Search pagination URL. The Twitter cursor is opaque and must be encoded —
+/// raw `+` / `=` in the query string get treated as form syntax and break paging.
+fn search_page_url(query: &str, tab: &str, cursor: Option<&str>) -> String {
+   let mut url = format!("/search?q={}", formatters::url_encode(query));
+   match tab {
+      "users" | "top" | "media" => {
+         url.push_str("&f=");
+         url.push_str(tab);
+      },
+      _ => {},
+   }
+   if let Some(cursor) = cursor {
+      url.push_str("&cursor=");
+      url.push_str(&formatters::url_encode(cursor));
+   }
+   url
+}
+
 /// Render the empty search page (home search bar).
 pub fn render_search_page() -> Markup {
    html! {
@@ -75,7 +93,7 @@ pub fn render_search_results_with_prefs(
 
                    @if let Some(cur) = cursor {
                        div class="show-more" {
-                           a href=(format!("/search?q={}&cursor={}", formatters::url_encode(query), cur)) {
+                           a href=(search_page_url(query, active_tab, Some(cur))) {
                                "Load more"
                            }
                        }
@@ -132,7 +150,7 @@ pub fn render_user_search_results(
 
                    @if let Some(cur) = cursor {
                        div class="show-more" {
-                           a href=(format!("/search?q={}&f=users&cursor={}", formatters::url_encode(query), cur)) {
+                           a href=(search_page_url(query, "users", Some(cur))) {
                                "Load more"
                            }
                        }