From 68f2313fc4a68ae3a96d9a83cfba10c636398ce5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 15:45:42 +0000 Subject: [PATCH 1/4] Retry sbt Docker e2e jar fetch on Central 404s agent_sbt_versions_patch_in_place failed in 0.05s on PR run 37948228245 when Maven Central answered 404 for the pinned commons-text-1.9.jar. fetch_from_central retried only transport errors, 429 and 5xx, so a single stale CDN negative entry failed the cell before docker run. Every URL the helper fetches is a pinned, released artifact Central never deletes, so a 404 is the same CDN blip sbt-warm-seed.sh already retries; treat it the same way. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CfAQ1cTuWfH3kKiCi6pPR4 --- crates/socket-patch-cli/tests/docker_e2e_sbt.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/socket-patch-cli/tests/docker_e2e_sbt.rs b/crates/socket-patch-cli/tests/docker_e2e_sbt.rs index 8028fe0dc..8d7dc45a1 100644 --- a/crates/socket-patch-cli/tests/docker_e2e_sbt.rs +++ b/crates/socket-patch-cli/tests/docker_e2e_sbt.rs @@ -111,8 +111,11 @@ async fn jars() -> (Vec, Vec) { /// GET `url` from Maven Central, retrying what a CI runner sees as a /// transient blip: a transport error (DNS, connect, reset, timeout) or a -/// 429 / 5xx. Any other status fails at once. Without this, one blip +/// 404 / 429 / 5xx. Any other status fails at once. Without this, one blip /// failed every cell of a leg within milliseconds, before `docker run`. +/// A 404 counts as a blip because every `url` here is a pinned, released +/// artifact that Central never deletes: a miss is a CDN edge serving a +/// stale negative entry (the same signature `sbt-warm-seed.sh` retries). async fn fetch_from_central(client: &reqwest::Client, url: &str) -> Vec { const ATTEMPTS: u32 = 5; let mut last = String::new(); @@ -129,7 +132,10 @@ async fn fetch_from_central(client: &reqwest::Client, url: &str) -> Vec { } }; let status = resp.status(); - if status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS { + if status.is_server_error() + || status == reqwest::StatusCode::TOO_MANY_REQUESTS + || status == reqwest::StatusCode::NOT_FOUND + { last = status.to_string(); continue; } From d63015aa0e69fc04c435c9c92f07e6f8f29a5a6a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 16:09:43 +0000 Subject: [PATCH 2/4] Retry sbt e2e Central fetch on 403 too; send UA On this PR's own run (37954180751) the sbt 1.9.9 agent leg failed both tests in 0.07s with 403 Forbidden for the pinned commons-text-1.9.jar, while the other seven agent legs fetched the same jar fine. Like the 404, it is one runner's CDN edge rejecting a public, released artifact, so retry it the same way. Also identify the client with a User-Agent: reqwest sends none by default. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CfAQ1cTuWfH3kKiCi6pPR4 --- .../socket-patch-cli/tests/docker_e2e_sbt.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/socket-patch-cli/tests/docker_e2e_sbt.rs b/crates/socket-patch-cli/tests/docker_e2e_sbt.rs index 8d7dc45a1..11202e2a4 100644 --- a/crates/socket-patch-cli/tests/docker_e2e_sbt.rs +++ b/crates/socket-patch-cli/tests/docker_e2e_sbt.rs @@ -75,8 +75,14 @@ fn hex_of(bytes: &[u8]) -> String { /// The pristine jar from Maven Central (checked against Central's `.sha1`) /// and the patched one: every member copied raw, plus [`MARKER`]. async fn jars() -> (Vec, Vec) { + // reqwest sends no User-Agent by default; identify the client so + // Central's edge does not treat the fetch as anonymous traffic. let client = reqwest::Client::builder() .timeout(std::time::Duration::from_secs(60)) + .user_agent(concat!( + "socket-patch-docker-e2e/", + env!("CARGO_PKG_VERSION") + )) .build() .expect("reqwest client"); let get = |url: String| { @@ -111,11 +117,13 @@ async fn jars() -> (Vec, Vec) { /// GET `url` from Maven Central, retrying what a CI runner sees as a /// transient blip: a transport error (DNS, connect, reset, timeout) or a -/// 404 / 429 / 5xx. Any other status fails at once. Without this, one blip -/// failed every cell of a leg within milliseconds, before `docker run`. -/// A 404 counts as a blip because every `url` here is a pinned, released -/// artifact that Central never deletes: a miss is a CDN edge serving a -/// stale negative entry (the same signature `sbt-warm-seed.sh` retries). +/// 403 / 404 / 429 / 5xx. Any other status fails at once. Without this, one +/// blip failed every cell of a leg within milliseconds, before `docker run`. +/// A 403 or 404 counts as a blip because every `url` here is a pinned, +/// released, public artifact that Central never deletes or restricts: CI +/// saw both from one runner's CDN edge while the other legs of the same +/// run fetched the same jar fine (the 404 is also what `sbt-warm-seed.sh` +/// retries). async fn fetch_from_central(client: &reqwest::Client, url: &str) -> Vec { const ATTEMPTS: u32 = 5; let mut last = String::new(); @@ -135,6 +143,7 @@ async fn fetch_from_central(client: &reqwest::Client, url: &str) -> Vec { if status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS || status == reqwest::StatusCode::NOT_FOUND + || status == reqwest::StatusCode::FORBIDDEN { last = status.to_string(); continue; From f04ae778854a27e940b4659752d4f71e578449d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 16:31:03 +0000 Subject: [PATCH 3/4] Fall back to Google's Central mirror in sbt e2e coverage-docker (sbt) on this PR (run 37957095405) got 403 Forbidden from repo1.maven.org on all five tries in both agent tests, about 15s of backoff each, so retrying the same edge cannot ride it out. Alternate attempts between Central and Google's official Central mirror and check the jar against Central's published sha1, now pinned, instead of fetching the .sha1 from the same origin. Whichever mirror answers, the test patches the exact released bytes. Verified locally: a scratch test fetching through jars() passes, and with the Central URL broken (Central answers 403) the second attempt gets the jar from the mirror and the sha1 matches. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CfAQ1cTuWfH3kKiCi6pPR4 --- .../socket-patch-cli/tests/docker_e2e_sbt.rs | 71 +++++++++++-------- 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/crates/socket-patch-cli/tests/docker_e2e_sbt.rs b/crates/socket-patch-cli/tests/docker_e2e_sbt.rs index 11202e2a4..ea86f4d05 100644 --- a/crates/socket-patch-cli/tests/docker_e2e_sbt.rs +++ b/crates/socket-patch-cli/tests/docker_e2e_sbt.rs @@ -63,8 +63,20 @@ const ORG: &str = "test-org"; const PURL: &str = "pkg:maven/org.apache.commons/commons-text@1.9"; const UUID: &str = "5b7a0000-0000-4000-8000-0000000000a1"; const JAR: &str = "commons-text-1.9.jar"; -const CENTRAL_JAR: &str = - "https://repo1.maven.org/maven2/org/apache/commons/commons-text/1.9/commons-text-1.9.jar"; +/// The jar's path under a Maven Central layout, served by every mirror in +/// [`CENTRAL_MIRRORS`]. +const CENTRAL_JAR_PATH: &str = "org/apache/commons/commons-text/1.9/commons-text-1.9.jar"; +/// Central's published `.sha1` for [`CENTRAL_JAR_PATH`]. Pinned so the jar +/// may come from either mirror and still be the exact released bytes. +const CENTRAL_JAR_SHA1: &str = "ba6ac8c2807490944a0a27f6f8e68fb5ed2e80e2"; +/// Maven Central itself, then Google's official Central mirror. CI saw +/// Central's CDN answer one runner 403 for every retry while other runners +/// fetched the same jar fine, so a second origin is what makes the fetch +/// survive an edge that has turned the runner away. +const CENTRAL_MIRRORS: [&str; 2] = [ + "https://repo1.maven.org/maven2", + "https://maven-central.storage-download.googleapis.com/maven2", +]; /// The marker member the patched jar adds. const MARKER: &str = "SOCKET_PATCHED.txt"; @@ -72,8 +84,9 @@ fn hex_of(bytes: &[u8]) -> String { hex::encode(D::digest(bytes)) } -/// The pristine jar from Maven Central (checked against Central's `.sha1`) -/// and the patched one: every member copied raw, plus [`MARKER`]. +/// The pristine jar from Maven Central (checked against the pinned +/// [`CENTRAL_JAR_SHA1`]) and the patched one: every member copied raw, plus +/// [`MARKER`]. async fn jars() -> (Vec, Vec) { // reqwest sends no User-Agent by default; identify the client so // Central's edge does not treat the fetch as anonymous traffic. @@ -85,13 +98,8 @@ async fn jars() -> (Vec, Vec) { )) .build() .expect("reqwest client"); - let get = |url: String| { - let client = client.clone(); - async move { fetch_from_central(&client, &url).await } - }; - let pristine = get(CENTRAL_JAR.to_string()).await; - let sha1 = String::from_utf8(get(format!("{CENTRAL_JAR}.sha1")).await).unwrap(); - assert_eq!(hex_of::(&pristine), sha1.trim(), "Central sha1"); + let pristine = fetch_from_central(&client, CENTRAL_JAR_PATH).await; + assert_eq!(hex_of::(&pristine), CENTRAL_JAR_SHA1, "Central sha1"); let mut archive = zip::ZipArchive::new(std::io::Cursor::new(pristine.clone())).unwrap(); let mut out = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); @@ -115,27 +123,32 @@ async fn jars() -> (Vec, Vec) { (pristine, patched) } -/// GET `url` from Maven Central, retrying what a CI runner sees as a -/// transient blip: a transport error (DNS, connect, reset, timeout) or a -/// 403 / 404 / 429 / 5xx. Any other status fails at once. Without this, one -/// blip failed every cell of a leg within milliseconds, before `docker run`. -/// A 403 or 404 counts as a blip because every `url` here is a pinned, -/// released, public artifact that Central never deletes or restricts: CI -/// saw both from one runner's CDN edge while the other legs of the same -/// run fetched the same jar fine (the 404 is also what `sbt-warm-seed.sh` -/// retries). -async fn fetch_from_central(client: &reqwest::Client, url: &str) -> Vec { - const ATTEMPTS: u32 = 5; +/// GET `path` from Maven Central, alternating between [`CENTRAL_MIRRORS`] +/// and retrying what a CI runner sees as a transient blip: a transport +/// error (DNS, connect, reset, timeout) or a 403 / 404 / 429 / 5xx. Any +/// other status fails at once. Without this, one blip failed every cell of +/// a leg within milliseconds, before `docker run`. A 403 or 404 counts as a +/// blip because `path` is a pinned, released, public artifact that Central +/// never deletes or restricts: CI saw both from one runner's CDN edge +/// while the other legs of the same run fetched the same jar fine (the 404 +/// is also what `sbt-warm-seed.sh` retries). The caller checks the bytes +/// against a pinned sha1, so which mirror served them does not matter. +async fn fetch_from_central(client: &reqwest::Client, path: &str) -> Vec { + const ATTEMPTS: u32 = 6; let mut last = String::new(); for attempt in 1..=ATTEMPTS { + let url = format!( + "{}/{path}", + CENTRAL_MIRRORS[(attempt as usize - 1) % CENTRAL_MIRRORS.len()] + ); if attempt > 1 { - eprintln!("{url}: attempt {} failed ({last}); retrying", attempt - 1); - tokio::time::sleep(std::time::Duration::from_secs(1 << (attempt - 2))).await; + eprintln!("{path}: attempt {} failed ({last}); retrying", attempt - 1); + tokio::time::sleep(std::time::Duration::from_secs(1 << (attempt - 2).min(3))).await; } - let resp = match client.get(url).send().await { + let resp = match client.get(&url).send().await { Ok(resp) => resp, Err(e) => { - last = format!("{e:?}"); + last = format!("{url}: {e:?}"); continue; } }; @@ -145,16 +158,16 @@ async fn fetch_from_central(client: &reqwest::Client, url: &str) -> Vec { || status == reqwest::StatusCode::NOT_FOUND || status == reqwest::StatusCode::FORBIDDEN { - last = status.to_string(); + last = format!("{url}: {status}"); continue; } assert!(status.is_success(), "{url}: {status}"); match resp.bytes().await { Ok(body) => return body.to_vec(), - Err(e) => last = format!("{e:?}"), + Err(e) => last = format!("{url}: {e:?}"), } } - panic!("{url}: {ATTEMPTS} attempts failed, last: {last}"); + panic!("{path}: {ATTEMPTS} attempts failed, last: {last}"); } /// The authenticated patch API serving one whole-jar patch for From 2f52f2fb53c99b7ebff4620a8acd30712acda231 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 16:32:04 +0000 Subject: [PATCH 4/4] Drop the custom User-Agent from the sbt e2e fetch With the User-Agent from d63015a, every Linux agent, mill and scala-cli leg of run 37957095379 and coverage-docker (sbt) got 403 from repo1.maven.org on all five tries. On the commit before it, without the header, seven of eight agent legs fetched the jar fine. The header is the likeliest trigger, so go back to reqwest's default request; the Google mirror fallback still covers a 403 from Central. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01CfAQ1cTuWfH3kKiCi6pPR4 --- crates/socket-patch-cli/tests/docker_e2e_sbt.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/socket-patch-cli/tests/docker_e2e_sbt.rs b/crates/socket-patch-cli/tests/docker_e2e_sbt.rs index ea86f4d05..62cbdb06e 100644 --- a/crates/socket-patch-cli/tests/docker_e2e_sbt.rs +++ b/crates/socket-patch-cli/tests/docker_e2e_sbt.rs @@ -88,14 +88,8 @@ fn hex_of(bytes: &[u8]) -> String { /// [`CENTRAL_JAR_SHA1`]) and the patched one: every member copied raw, plus /// [`MARKER`]. async fn jars() -> (Vec, Vec) { - // reqwest sends no User-Agent by default; identify the client so - // Central's edge does not treat the fetch as anonymous traffic. let client = reqwest::Client::builder() .timeout(std::time::Duration::from_secs(60)) - .user_agent(concat!( - "socket-patch-docker-e2e/", - env!("CARGO_PKG_VERSION") - )) .build() .expect("reqwest client"); let pristine = fetch_from_central(&client, CENTRAL_JAR_PATH).await;