diff --git a/scripts/backtest-bun.py b/scripts/backtest-bun.py index c7f10aa77..71020c4c8 100644 --- a/scripts/backtest-bun.py +++ b/scripts/backtest-bun.py @@ -187,11 +187,12 @@ def save(path, data): BUN_TRANSPORT_FAILURE = re.compile( r'^error: (?:Connection\w+|FailedToOpenSocket|Timeout|TLSHandshakeTimeout) downloading ' r'|^error: GET \S+ - 5\d\d\b', re.M) - # The harness's own fetches (published record, hosted tarball digest) that # still fail after their in-place retries surface as the cell's `error`: -# ``. -HARNESS_TRANSPORT_FAILURE = re.compile(r'^`). +HARNESS_TRANSPORT_FAILURE = re.compile( + r'\bHTTP Error (?:5\d\d|429)\b|^= (1, 2, 23)): diff --git a/scripts/tests/test_backtest_harnesses.py b/scripts/tests/test_backtest_harnesses.py index 5bb56de5c..bc2ee49dc 100644 --- a/scripts/tests/test_backtest_harnesses.py +++ b/scripts/tests/test_backtest_harnesses.py @@ -69,6 +69,45 @@ def test_a_patch_api_5xx_is_a_transport_failure(self): self.assertTrue(bun.has_transport_failure(['API request failed with status 504: error code: 504'])) self.assertFalse(bun.has_transport_failure({'error': 'API request failed with status 404: not found'})) + def test_a_harness_fetch_5xx_is_a_transport_failure(self): + # run 37852649519: hosted_lockb_digest's urlopen of patch.socket.dev + # raised this on all four bun.lockb legs and none of them retried. + self.assertTrue(bun.has_transport_failure({'error': 'HTTP Error 503: Service Unavailable'})) + self.assertTrue(bun.has_transport_failure({'error': 'HTTP Error 429: Too Many Requests'})) + self.assertTrue(bun.has_transport_failure( + {'error': ''})) + self.assertFalse(bun.has_transport_failure({'error': 'HTTP Error 404: Not Found'})) + + def test_a_failed_install_fetch_is_a_transport_failure(self): + output = 'bun install v1.0.0\nerror: GET https://patch.socket.dev/x.tgz - 503\n' + row = dict(passed=False, checks={'frozenPatchedBytes': False}, + frozenInstallTransport=bun.bun_transport_failures(output)) + self.assertTrue(bun.has_transport_failure(row)) + row['frozenInstallTransport'] = bun.bun_transport_failures('bun install v1.0.0\n') + self.assertFalse(bun.has_transport_failure(row)) + + def test_a_harness_fetch_5xx_cell_retries_fresh(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + job = ('0.8.1', 'vendored-then-hosted', 'vendored') + case = root / 'captures' / '-'.join(job) + calls = [] + + def run_case(_job): + case.mkdir(parents=True) + calls.append(True) + row = dict(passed=len(calls) > 1, checks={'frozenPatchedBytes': len(calls) > 1}) + if len(calls) == 1: + row['error'] = 'HTTP Error 503: Service Unavailable' + bun.save(case / 'result.json', row) + return row + + with patch.object(bun.time, 'sleep'): + row = bun.retry_network_cell(run_case, job, root) + self.assertTrue(row['passed']) + self.assertEqual(len(calls), 2) + self.assertEqual(row['networkRetryAttempts'][0]['failedChecks'], ['frozenPatchedBytes']) + def test_functional_failure_is_never_retried(self): with tempfile.TemporaryDirectory() as temp: row = dict(passed=False, checks={'frozenPatchedBytes': False}, error='installed bytes differ')