From 54432af4690b12a8378f0e1f58d4f6dc745eaa63 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 16:56:07 +0000 Subject: [PATCH] Retry sbt docker e2e's Central fetch on blips docker_e2e_sbt fetches commons-text 1.9 and its .sha1 from Maven Central before it starts any container, with no retry. On 2026-10-08 four sbt/Mill compatibility legs failed this way on unrelated PRs and on main, every test in the leg failing within 0.1-10 s (before `docker run`, which takes minutes), while sibling legs on the same commit passed. Retry that fetch up to 5 times (1+2+4+8 s backoff) on transport errors and 429/5xx only; other statuses still fail at once. Add a 60 s request timeout so a stalled connection is retried, not hung. The matrix script uploaded the failing log only as an artifact, so the cause never showed in the job log. report() now prints the log's tail on failure, including the "no test ran" case. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01E35LYStnEzhojF7DU3rP8M --- .../socket-patch-cli/tests/docker_e2e_sbt.rs | 44 ++++++++++++++++--- scripts/sbt-compat-matrix.sh | 37 +++++++++------- 2 files changed, 60 insertions(+), 21 deletions(-) diff --git a/crates/socket-patch-cli/tests/docker_e2e_sbt.rs b/crates/socket-patch-cli/tests/docker_e2e_sbt.rs index 3f502accf..884f6fa42 100644 --- a/crates/socket-patch-cli/tests/docker_e2e_sbt.rs +++ b/crates/socket-patch-cli/tests/docker_e2e_sbt.rs @@ -78,14 +78,13 @@ fn git_sha256(content: &[u8]) -> String { /// The pristine jar from Maven Central (checked against Central's `.sha1`) /// and the patched one: every member copied raw, plus [`MARKER`]. async fn jars() -> (Vec, Vec) { - let client = reqwest::Client::new(); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(60)) + .build() + .expect("reqwest client"); let get = |url: String| { let client = client.clone(); - async move { - let resp = client.get(&url).send().await.expect("fetch from Central"); - assert!(resp.status().is_success(), "{url}: {}", resp.status()); - resp.bytes().await.expect("body").to_vec() - } + async move { fetch_from_central(&client, &url).await } }; let pristine = get(CENTRAL_JAR.to_string()).await; let sha1 = String::from_utf8(get(format!("{CENTRAL_JAR}.sha1")).await).unwrap(); @@ -113,6 +112,39 @@ async fn jars() -> (Vec, Vec) { (pristine, patched) } +/// GET `url` from Maven Central, retrying what a CI runner sees as a +/// transient blip: a transport error (DNS, connect, reset, timeout) or a +/// 429 / 5xx. Any other status fails at once. Without this, one blip +/// failed every cell of a leg within milliseconds, before `docker run`. +async fn fetch_from_central(client: &reqwest::Client, url: &str) -> Vec { + const ATTEMPTS: u32 = 5; + let mut last = String::new(); + for attempt in 1..=ATTEMPTS { + if attempt > 1 { + eprintln!("{url}: attempt {} failed ({last}); retrying", attempt - 1); + tokio::time::sleep(std::time::Duration::from_secs(1 << (attempt - 2))).await; + } + let resp = match client.get(url).send().await { + Ok(resp) => resp, + Err(e) => { + last = format!("{e:?}"); + continue; + } + }; + let status = resp.status(); + if status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS { + last = status.to_string(); + continue; + } + assert!(status.is_success(), "{url}: {status}"); + match resp.bytes().await { + Ok(body) => return body.to_vec(), + Err(e) => last = format!("{e:?}"), + } + } + panic!("{url}: {ATTEMPTS} attempts failed, last: {last}"); +} + /// The authenticated patch API serving one whole-jar patch for /// commons-text 1.9 (the patched blob inline, the pristine one by hash). async fn mock_api(pristine: &[u8], patched: &[u8]) -> MockServer { diff --git a/scripts/sbt-compat-matrix.sh b/scripts/sbt-compat-matrix.sh index ca4f89b8f..79cb1108d 100755 --- a/scripts/sbt-compat-matrix.sh +++ b/scripts/sbt-compat-matrix.sh @@ -181,26 +181,33 @@ STATUS=0 report() { # PASS / FAIL / SKIP per `test ... ` line of log $1. local log="$1" label="$2" - local lines + local lines failed=0 lines=$(grep -E '^test [^ ]+ \.\.\. ' "$log" || true) if [ -z "$lines" ]; then echo "FAIL $label (no test ran; see $log)" - STATUS=1 - return - fi - while IFS= read -r line; do - name=$(echo "$line" | awk '{print $2}') - case "$line" in - *"... ok") echo "PASS $label $name" ;; - *"... ignored"*) echo "SKIP $label $name" ;; - *) echo "FAIL $label $name"; STATUS=1 ;; - esac - done <<< "$lines" - if grep -q '^SKIP ' "$log"; then - grep '^SKIP ' "$log" | sed "s/^/NOTE $label /" + failed=1 + else + while IFS= read -r line; do + name=$(echo "$line" | awk '{print $2}') + case "$line" in + *"... ok") echo "PASS $label $name" ;; + *"... ignored"*) echo "SKIP $label $name" ;; + *) echo "FAIL $label $name"; failed=1 ;; + esac + done <<< "$lines" + if grep -q '^SKIP ' "$log"; then + grep '^SKIP ' "$log" | sed "s/^/NOTE $label /" + fi + if ! grep -qE '^test result: ok\.' "$log"; then + failed=1 + fi fi - if ! grep -qE '^test result: ok\.' "$log"; then + if [ "$failed" = 1 ]; then STATUS=1 + # The log is uploaded only as an artifact; show why in the job log too. + echo "::group::tail of $log" >&2 + tail -n 60 "$log" >&2 + echo "::endgroup::" >&2 fi }