Stop dead proxy sockets and the articles check from holding a profile.

Co-authored-by: Cursor <[email protected]>

src/api/client_endpoints.rs

@@ -97,6 +97,35 @@ enum Rail {
    None,
 }
 
+/// How long `UserMedia` may keep going after the tweets beside it are back.
+/// That call is the one that stalls and 503s. The timeline is already
+/// renderable, so a rail that has not landed is dropped instead of holding
+/// the page through another header deadline.
+const RAIL_GRACE: Duration = Duration::from_millis(200);
+
+struct RailTask(tokio::task::JoinHandle<Rail>);
+
+impl Drop for RailTask {
+   fn drop(&mut self) {
+      self.0.abort();
+   }
+}
+
+impl RailTask {
+   fn spawn(api: &ApiClient, user_id: &str, given: Option<PhotoRail>) -> Self {
+      let api = api.clone();
+      let user_id = user_id.to_owned();
+      Self(tokio::spawn(async move { api.rail_for(&user_id, given).await }))
+   }
+
+   async fn finish(mut self) -> Rail {
+      match timeout(RAIL_GRACE, &mut self.0).await {
+         Ok(Ok(rail)) => rail,
+         _ => Rail::None,
+      }
+   }
+}
+
 fn profile_from_parts(user: User, tweets: Timeline, rail: Rail) -> Profile {
    let pinned = tweets
       .content
@@ -582,10 +611,10 @@ impl ApiClient {
 
       if let Some(id) = known_id {
          if cursor.is_none() {
-            let (user_res, tweets_res, rail) = tokio::join!(
+            let rail_task = RailTask::spawn(self, id, rail);
+            let (user_res, tweets_res) = tokio::join!(
                self.get_user(screen_name),
                self.get_user_tweets(id, None),
-               self.rail_for(id, rail)
             );
             let user = match user_res {
                Ok(user) => user,
@@ -600,6 +629,7 @@ impl ApiClient {
                   ..Profile::default()
                });
             }
+            let rail = rail_task.finish().await;
             return Ok(profile_from_parts(user, tweets_res?, rail));
          }
 
@@ -630,11 +660,9 @@ impl ApiClient {
       }
 
       let (tweets, rail) = if cursor.is_none() {
-         let (tweets_result, rail) = tokio::join!(
-            self.get_user_tweets(&user.id, None),
-            self.rail_for(&user.id, rail)
-         );
-         (tweets_result?, rail)
+         let rail_task = RailTask::spawn(self, &user.id, rail);
+         let tweets = self.get_user_tweets(&user.id, None).await?;
+         (tweets, rail_task.finish().await)
       } else {
          (self.get_user_tweets(&user.id, cursor).await?, Rail::None)
       };

src/api/http.rs

@@ -51,9 +51,12 @@ const MEDIA_HEADER_DEADLINE: Duration = Duration::from_secs(2);
 /// connection the proxy has already dropped sits until the gateway gives up
 /// (about five seconds) and then returns 503 "remote refused stream reset".
 /// Abandon headers by now and open another connection.
-const API_HEADER_DEADLINE: Duration = Duration::from_secs(3);
-const KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(45);
-const KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(15);
+const API_HEADER_DEADLINE: Duration = Duration::from_millis(1500);
+/// `scrape-socks.py` reaps a tunnel after a quiet minute. Pings have to land
+/// well inside that window. A 45s ping left every idle session's socket
+/// already closed, and the next visitor waited out the header deadline on it.
+const KEEP_ALIVE_INTERVAL: Duration = Duration::from_secs(10);
+const KEEP_ALIVE_TIMEOUT: Duration = Duration::from_secs(5);
 /// After an HTML 429, skip the direct address this long and send every
 /// request through the fallback exits.
 const DIRECT_BLOCK_FOR: Duration = Duration::from_secs(5 * 60);

src/cache/ttl.rs

@@ -41,6 +41,9 @@ pub const EXPLORE_CURSOR_STALE: u64 = 15 * 60;
 pub const ARTICLES_FLAG: u64 = 60 * 60;
 /// Keep a known answer through a rate-limit blip.
 pub const ARTICLES_FLAG_STALE: u64 = 6 * 60 * 60;
+/// A failed articles check. Long enough that the next view does not call X
+/// again, short enough that a blip does not pin the tab for the hour.
+pub const ARTICLES_FLAG_RETRY: u64 = 10 * 60;
 
 /// Long TTL for immutable mappings like user ID -> username.
 pub const USER_ID_MAPPING: u64 = 24 * 60 * 60; // 1 day

src/routes/helpers.rs

@@ -86,9 +86,10 @@ pub fn remember_user_id(state: &AppState, user: &User) {
 
 /// Whether this account has published an article.
 ///
-/// X does not put that on the profile record, so the articles timeline is
-/// checked once and remembered. A failed check keeps the tab, so a rate limit
-/// cannot hide articles that are there.
+/// X does not put that on the profile record. A stored answer is used as-is.
+/// Anything else shows the tab and checks in the background. Waiting here was
+/// a second GraphQL round trip after the timeline was already in hand, and a
+/// failed check was paid again on the next view because nothing was stored.
 pub async fn user_has_articles(state: &AppState, user_id: &str) -> bool {
    if user_id.is_empty() {
       return false;
@@ -97,16 +98,26 @@ pub async fn user_has_articles(state: &AppState, user_id: &str) -> bool {
    if let Some(Hit::Fresh(has) | Hit::Stale(has)) = state.cache.lookup(&key) {
       return has;
    }
-   match state.api.get_user_articles(user_id, None).await {
-      Ok(timeline) => {
-         let has = timeline.content.iter().flatten().next().is_some();
-         state
-            .cache
-            .set_swr(&key, &has, ttl::ARTICLES_FLAG, ttl::ARTICLES_FLAG_STALE);
-         has
-      },
-      Err(_) => true,
+   if let Some(guard) = state.cache.start_refresh(&key) {
+      let state = state.clone();
+      let user_id = user_id.to_owned();
+      tokio::spawn(async move {
+         let _guard = guard;
+         let key = cache_keys::articles_flag(&user_id);
+         match state.api.get_user_articles(&user_id, None).await {
+            Ok(timeline) => {
+               let has = timeline.content.iter().flatten().next().is_some();
+               state
+                  .cache
+                  .set_swr(&key, &has, ttl::ARTICLES_FLAG, ttl::ARTICLES_FLAG_STALE);
+            },
+            Err(_) => {
+               state.cache.set(&key, &true, ttl::ARTICLES_FLAG_RETRY);
+            },
+         }
+      });
    }
+   true
 }
 
 /// Store a full user record plus id mappings.