From 0dbf50e841e837ea821ca1088ea3eba0f1e96a3a Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Fri, 9 Oct 2026 12:44:17 -0400 Subject: [PATCH 1/5] WIP: start v5 blocker fix Co-Authored-By: Claude Opus 5.5 (1M context) From 66b16b128f662b8e6245cee49d2ac37a3fa0a6b8 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Fri, 9 Oct 2026 13:40:07 -0400 Subject: [PATCH 2/5] Pin yarn classic hosted tarballs by sha1 A hosted yarn classic pin wrote `resolved ""` with no `#` fragment when the grant carried only a sha512. Yarn 1 names its cache slot after that fragment, so the hosted tarball shared the slot of any fragmentless upstream copy of the same version: a warm cache installed the unpatched bytes (yarn <= 1.17) or failed every install with "Incorrect integrity" (yarn >= 1.19), while scan reported success. When a classic yarn.lock is targeted and the grant has no sha1, the disk and in-memory hosted flows now download the served tarball, check it against the grant's sha512 and pin the sha1 of those bytes. A tarball that can't be fetched or verified drops the patch as `npm_tarball_unavailable`, and the rewriter itself refuses a dep that still lacks a sha1 (`redirect_yarn_classic_missing_sha1`). Fixes #558 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/commands/scan/hosted.rs | 31 ++ .../tests/covgap_commands_scan_hosted.rs | 4 +- .../tests/hosted_memory_engine.rs | 100 +++++ .../tests/in_process_redirect.rs | 6 +- .../tests/in_process_vendor.rs | 3 +- .../tests/scan/hosted_yarn_classic_sha1.rs | 264 ++++++++++++ crates/socket-patch-cli/tests/scan/main.rs | 1 + crates/socket-patch-core/src/hosted/engine.rs | 54 +++ .../src/hosted/memory/discover.rs | 29 ++ .../src/hosted/memory/mod.rs | 13 +- .../src/hosted/memory/stages.rs | 33 +- .../src/hosted/npm_manifest.rs | 38 ++ .../redirect/lock_index_equivalence_tests.rs | 10 +- .../src/patch/redirect/mod.rs | 100 ++++- .../equivalence/yarn_classic_rewrite.golden | 400 +++++++++--------- docs/ecosystems.md | 8 +- 16 files changed, 867 insertions(+), 227 deletions(-) create mode 100644 crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index eec27aed1..b6cd40702 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1042,6 +1042,34 @@ pub(crate) async fn run_redirect_selected( } } } + // A yarn classic pin needs the served tarball's sha1 as its `resolved` + // fragment: yarn 1 keys its cache slot by it (#558). When the grant + // carries only a sha512, the scan downloads the tarball, checks it + // against that sha512 and pins the sha1 of those bytes. A tarball that + // cannot be fetched or verified drops its patch rather than pin a + // fragmentless URL a warm cache serves stale bytes for. + let sha1_targets: Vec<(String, String, DepOverride)> = + engine::yarn_classic_sha1_targets(&candidates, &read.files) + .into_iter() + .filter_map(|dep| { + let sha512 = dep.integrity.sha512.clone()?; + Some((dep.artifact_url.clone(), sha512, dep.clone())) + }) + .collect(); + for (url, sha512, dep) in sha1_targets { + status.set(format!("Fetching hosted tarball for {}...", dep.name)); + match socket_patch_core::hosted::npm_manifest::fetch_hosted_npm_sha1( + api_client, &url, &sha512, + ) + .await + { + Ok(sha1) => engine::set_derived_sha1(&mut candidates, &url, &sha1), + Err(detail) => { + unavailable_python_artifacts.insert(url.clone()); + skipped.push(engine::npm_tarball_unavailable(&dep, &detail)); + } + } + } status.finish(); candidates.retain(|c| !unavailable_python_artifacts.contains(&c.dep.artifact_url)); // The Pipfile.lock reference shape depends on the installing Pipenv @@ -2034,6 +2062,9 @@ fn describe_skip_reason(reason: &str) -> String { "npm_manifest_unavailable" => { "the hosted tarball's package.json could not be fetched".into() } + "npm_tarball_unavailable" => { + "the hosted tarball could not be fetched or did not match its sha512".into() + } "redirect_bun_lock_unsupported" | "redirect_bun_lockb_invalid" => { "the Bun lockfile blocks the vendored-to-hosted migration (see the warning)".into() } diff --git a/crates/socket-patch-cli/tests/covgap_commands_scan_hosted.rs b/crates/socket-patch-cli/tests/covgap_commands_scan_hosted.rs index 979207e57..385562706 100644 --- a/crates/socket-patch-cli/tests/covgap_commands_scan_hosted.rs +++ b/crates/socket-patch-cli/tests/covgap_commands_scan_hosted.rs @@ -33,6 +33,8 @@ const PURL: &str = "pkg:npm/covgap-hosted@1.0.0"; const UUID: &str = "11111111-1111-4111-8111-111111111111"; const HOSTED_URL: &str = "http://patch.test/patch/npm/covgap-hosted/1.0.0/22222222-2222-4222-8222-222222222222/11111111-1111-4111-8111-111111111111/covgap-hosted-1.0.0.tgz"; const PATCHED_SHA512: &str = "sha512-PATCHEDpatchedPATCHEDpatched0123456789=="; +/// The grant's sha1: yarn classic pins it as `resolved`'s `#` fragment (#558). +const PATCHED_SHA1: &str = "5ba15ba15ba15ba15ba15ba15ba15ba15ba15ba1"; const UPSTREAM_SHA512: &str = "sha512-UPSTREAMupstream=="; const GHSA: &str = "GHSA-cvgp-hstd-aaaa"; @@ -94,7 +96,7 @@ async fn mock_granted_reference(server: &MockServer, uuid: &str, purl: &str, url "artifacts": [{ "kind": "tarball", "url": url, - "integrity": { "sha512": PATCHED_SHA512 } + "integrity": { "sha512": PATCHED_SHA512, "sha1": PATCHED_SHA1 } }], "registryOverride": null } diff --git a/crates/socket-patch-cli/tests/hosted_memory_engine.rs b/crates/socket-patch-cli/tests/hosted_memory_engine.rs index ee3687b1c..39a3fb0cd 100644 --- a/crates/socket-patch-cli/tests/hosted_memory_engine.rs +++ b/crates/socket-patch-cli/tests/hosted_memory_engine.rs @@ -1336,6 +1336,106 @@ async fn yarn_berry_pin_takes_bin_from_the_served_tarball() { } } +/// #558 in the in-memory engine: a yarn classic pin whose grant carries +/// only a sha512 takes its `#` fragment from the served tarball +/// (fetched through the provider and checked against that sha512); a +/// tarball it cannot fetch drops the patch instead of pinning a +/// fragmentless URL yarn 1 would serve stale cached bytes for. +#[tokio::test] +async fn issue_558_yarn_classic_pin_takes_sha1_from_the_served_tarball() { + use base64::Engine as _; + use sha1::Digest as _; + + const UUID: &str = "55858558-5585-4558-8558-558558558558"; + let tarball = { + let manifest = br#"{"name":"left-pad","version":"1.3.0"}"#; + let mut builder = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + let mut header = tar::Header::new_gnu(); + header.set_size(manifest.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "package/package.json", &manifest[..]) + .unwrap(); + builder.into_inner().unwrap().finish().unwrap() + }; + let sha512 = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(&tarball)) + ); + let sha1 = hex::encode(sha1::Sha1::digest(&tarball)); + let mut files = BTreeMap::new(); + files.insert( + "package.json".to_string(), + b"{\n \"name\": \"app\",\n \"version\": \"1.0.0\",\n \"packageManager\": \"yarn@1.22.22\",\n \"dependencies\": {\n \"left-pad\": \"1.3.0\"\n }\n}\n".to_vec(), + ); + files.insert( + "yarn.lock".to_string(), + b"# yarn lockfile v1\n\n\nleft-pad@1.3.0:\n version \"1.3.0\"\n \ + resolved \"https://registry.yarnpkg.com/left-pad/-/left-pad-1.3.0.tgz\"\n \ + integrity sha512-UP==\n" + .to_vec(), + ); + + for served in [true, false] { + let server = MockServer::start().await; + let artifact = format!("/patch/npm/left-pad/1.3.0/tok/{UUID}/left-pad-1.3.0.tgz"); + let url = format!("{}{artifact}", server.uri()); + let patch = common::Patch { + purl: "pkg:npm/left-pad@1.3.0".into(), + uuid: UUID.into(), + reference: serde_json::json!({ + "status": "granted", + "url": url, + "purl": null, + "artifacts": [ + { "kind": "tarball", "url": url, "integrity": { "sha512": sha512 } } + ], + "registryOverride": null, + }), + }; + mount_api(&server, &[patch]).await; + Mock::given(method("GET")) + .and(path(artifact)) + .respond_with(if served { + ResponseTemplate::new(200).set_body_bytes(tarball.clone()) + } else { + ResponseTemplate::new(404) + }) + .mount(&server) + .await; + + let output = run_engine(&server, build_input(&files, &[], options(false))).await; + let project = &output.projects[0]; + assert!(project.error.is_none(), "{:?}", project.error); + let changed = engine_changed(&output); + if served { + assert_eq!(project.redirected.len(), 1, "{:?}", project.skipped); + let lock = String::from_utf8(changed["yarn.lock"].clone()).unwrap(); + assert!( + lock.contains(&format!( + " resolved \"{url}#{sha1}\"\n integrity {sha512}\n" + )), + "{lock}" + ); + } else { + assert!(project.redirected.is_empty(), "{:?}", project.redirected); + assert!( + project + .skipped + .iter() + .any(|s| s.reason == "npm_tarball_unavailable"), + "{:?}", + project.skipped + ); + assert!(!changed.contains_key("yarn.lock"), "{changed:?}"); + } + } +} + /// #734 in memory: with no node_modules in the view, package.json's /// `packageManager` is the only pin. pnpm 9.15.9 gets its lock pinned and /// no root-only pnpm-workspace.yaml; pnpm 11 still gets the trust scaffold. diff --git a/crates/socket-patch-cli/tests/in_process_redirect.rs b/crates/socket-patch-cli/tests/in_process_redirect.rs index 799267eeb..97f69b1b4 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect.rs @@ -41,6 +41,8 @@ const PURL: &str = "pkg:npm/in-proc-redirect@1.0.0"; const UUID: &str = "11111111-1111-4111-8111-111111111111"; const HOSTED_URL: &str = "http://patch.test/patch/npm/in-proc-redirect/1.0.0/22222222-2222-4222-8222-222222222222/11111111-1111-4111-8111-111111111111/in-proc-redirect-1.0.0.tgz"; const PATCHED_SHA512: &str = "sha512-PATCHEDpatchedPATCHEDpatched0123456789=="; +/// The grant's sha1: yarn classic pins it as `resolved`'s `#` fragment (#558). +const PATCHED_SHA1: &str = "5ba15ba15ba15ba15ba15ba15ba15ba15ba15ba1"; const GHSA: &str = "GHSA-rdir-aaaa-bbbb"; fn redirect_args(cwd: &Path, api_url: String) -> ScanArgs { @@ -114,7 +116,7 @@ async fn mock_reference(server: &MockServer) { "artifacts": [{ "kind": "tarball", "url": HOSTED_URL, - "integrity": { "sha512": PATCHED_SHA512 } + "integrity": { "sha512": PATCHED_SHA512, "sha1": PATCHED_SHA1 } }], "registryOverride": null } @@ -1155,7 +1157,7 @@ async fn scan_redirect_rewrites_correct_entry_in_crlf_classic_lock() { "the decoy entry must stay byte-identical: {lock}" ); assert!( - lock.contains(&format!("resolved \"{HOSTED_URL}\"\r\n")) + lock.contains(&format!("resolved \"{HOSTED_URL}#{PATCHED_SHA1}\"\r\n")) && lock.contains(&format!("integrity {PATCHED_SHA512}\r\n")), "the target entry must pin the hosted patch: {lock}" ); diff --git a/crates/socket-patch-cli/tests/in_process_vendor.rs b/crates/socket-patch-cli/tests/in_process_vendor.rs index 93dbf0c24..110150c38 100644 --- a/crates/socket-patch-cli/tests/in_process_vendor.rs +++ b/crates/socket-patch-cli/tests/in_process_vendor.rs @@ -1378,7 +1378,8 @@ async fn mount_berry_hosted_api_opts(server: &wiremock::MockServer, berry_zip: b .mount(server) .await; let mut artifacts = vec![json!({ "kind": "tarball", "url": hosted_url, - "integrity": { "sha512": "sha512-unused-by-berry==" } })]; + "integrity": { "sha512": "sha512-unused-by-berry==", + "sha1": "5ba15ba15ba15ba15ba15ba15ba15ba15ba15ba1" } })]; if berry_zip { artifacts.push(json!({ "kind": "yarn-berry-zip", "url": hosted_url, "integrity": { "yarnBerry10c0": format!("10c0/{}", "7".repeat(128)) } })); diff --git a/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs b/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs new file mode 100644 index 000000000..71962b4aa --- /dev/null +++ b/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs @@ -0,0 +1,264 @@ +//! `scan --mode hosted` over a yarn classic project whose grant carries +//! only a sha512 for the hosted tarball (#558). Yarn classic files a +//! tarball in its cache under `npm---`, the +//! fragment being the `#` of `resolved`. A fragmentless hosted URL +//! shares the slot of every fragmentless upstream copy, so a warm cache +//! installs stale bytes (yarn <= 1.17) or fails (yarn >= 1.19). The scan +//! downloads the served tarball, checks it against the grant's sha512 and +//! pins the sha1 of those bytes. A tarball it cannot fetch or verify drops +//! the patch (`npm_tarball_unavailable`) rather than pin a fragmentless URL. +//! +//! Runs the built binary as a subprocess against a wiremock patch API. + +use std::path::Path; + +use serde_json::{json, Value}; +use wiremock::matchers::{method, path, path_regex}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +use crate::common; + +const ORG: &str = "test-org"; +const PURL: &str = "pkg:npm/left-pad@1.3.0"; +const UUID: &str = "55858558-5585-4558-8558-558558558558"; +const TOKEN: &str = "33333333-3333-4333-8333-333333333333"; + +fn tgz(entries: &[(&str, &[u8])]) -> Vec { + let mut builder = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + for (name, data) in entries { + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder.append_data(&mut header, name, *data).unwrap(); + } + builder.into_inner().unwrap().finish().unwrap() +} + +fn sha512_sri(bytes: &[u8]) -> String { + use base64::Engine as _; + use sha2::Digest as _; + format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(bytes)) + ) +} + +fn sha1_hex(bytes: &[u8]) -> String { + use sha1::Digest as _; + hex::encode(sha1::Sha1::digest(bytes)) +} + +fn patched_tarball() -> Vec { + tgz(&[ + ( + "package/package.json", + br#"{"name":"left-pad","version":"1.3.0","main":"index.js"}"#, + ), + ("package/index.js", b"module.exports = 'patched';\n"), + ]) +} + +/// A classic lock whose `resolved` has no `#sha1` fragment (a private +/// registry, or a lock yarn wrote from such a registry). +const LOCK: &str = "# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.\n\ +# yarn lockfile v1\n\n\n\ +left-pad@1.3.0:\n version \"1.3.0\"\n \ +resolved \"https://registry.yarnpkg.com/left-pad/-/left-pad-1.3.0.tgz\"\n \ +integrity sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA==\n"; + +fn write_classic_project(root: &Path) { + std::fs::create_dir_all(root.join("node_modules/left-pad")).unwrap(); + std::fs::write( + root.join("package.json"), + "{\n \"name\": \"app\",\n \"version\": \"1.0.0\",\n \"private\": true,\n \ + \"packageManager\": \"yarn@1.22.22\",\n \ + \"dependencies\": {\n \"left-pad\": \"1.3.0\"\n }\n}\n", + ) + .unwrap(); + std::fs::write(root.join("yarn.lock"), LOCK).unwrap(); + std::fs::write( + root.join("node_modules/left-pad/package.json"), + r#"{"name":"left-pad","version":"1.3.0"}"#, + ) + .unwrap(); +} + +/// The patch API, granting the hosted tarball with a sha512-only +/// integrity of `grant_bytes`, and serving `served` (or 404). +async fn mock_api(server: &MockServer, grant_bytes: &[u8], served: Option>) -> String { + let artifact = format!("/patch/npm/left-pad/1.3.0/{TOKEN}/{UUID}/left-pad-1.3.0.tgz"); + let url = format!("{}{artifact}", server.uri()); + let sha512 = sha512_sri(grant_bytes); + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/batch"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "packages": [{ + "purl": PURL, + "patches": [{ + "uuid": UUID, "purl": PURL, "tier": "free", + "cveIds": [], "ghsaIds": [], "severity": "high", + "title": "left-pad fixture" + }] + }], + "canAccessPaidPatches": false, + }))) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path_regex(format!( + "^/v0/orgs/{ORG}/patches/by-package/.+$" + ))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "patches": [{ + "uuid": UUID, "purl": PURL, + "publishedAt": "2026-01-01T00:00:00Z", + "description": "x", "license": "MIT", "tier": "free", + "vulnerabilities": {} + }], + "canAccessPaidPatches": false, + }))) + .mount(server) + .await; + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/package"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "results": { + UUID: { + "status": "granted", + "url": url, + "purl": PURL, + "artifacts": [ + { "kind": "tarball", "url": url, "integrity": { "sha512": sha512 } } + ], + "registryOverride": null + } + } + }))) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path(format!("/v0/orgs/{ORG}/patches/view/{UUID}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "uuid": UUID, "purl": PURL, + "publishedAt": "2026-01-01T00:00:00Z", + "files": {}, + "vulnerabilities": {}, + "description": "x", "license": "MIT", "tier": "free" + }))) + .mount(server) + .await; + let response = match served { + Some(bytes) => ResponseTemplate::new(200).set_body_raw(bytes, "application/octet-stream"), + None => ResponseTemplate::new(404), + }; + Mock::given(method("GET")) + .and(path(artifact)) + .respond_with(response) + .mount(server) + .await; + url +} + +fn scan(root: &Path, api: &str) -> (i32, Value, String) { + let cwd = root.to_str().unwrap().to_string(); + let (code, stdout, stderr) = common::run_with_env( + root, + &[ + "scan", + "--mode", + "hosted", + "--json", + "--yes", + "--cwd", + &cwd, + "--api-url", + api, + "--org", + ORG, + "--api-token", + "fake", + ], + &[], + ); + let doc: Value = serde_json::from_str(&stdout) + .unwrap_or_else(|e| panic!("JSON envelope ({e}):\n{stdout}\n{stderr}")); + (code, doc, stderr) +} + +#[tokio::test] +async fn issue_558_sha512_only_grant_pins_the_served_tarballs_sha1() { + let server = MockServer::start().await; + let tarball = patched_tarball(); + let url = mock_api(&server, &tarball, Some(tarball.clone())).await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + write_classic_project(&root); + + let (code, doc, stderr) = scan(&root, &server.uri()); + assert_eq!(code, 0, "{doc:#}\n{stderr}"); + assert_eq!(doc["redirect"]["redirected"], 1, "{doc:#}"); + let lock = std::fs::read_to_string(root.join("yarn.lock")).unwrap(); + assert!( + lock.contains(&format!( + " resolved \"{url}#{}\"\n integrity {}\n", + sha1_hex(&tarball), + sha512_sri(&tarball) + )), + "the hosted pin carries the served tarball's sha1 fragment:\n{lock}" + ); +} + +/// The served bytes are not the ones the grant's sha512 names (a stale CDN +/// copy, a truncated proxy body): no sha1 is derived from them, and nothing +/// is pinned. +#[tokio::test] +async fn issue_558_served_tarball_not_matching_the_grant_skips_the_patch() { + let server = MockServer::start().await; + let tarball = patched_tarball(); + mock_api(&server, &tarball, Some(b"not the granted tarball".to_vec())).await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + write_classic_project(&root); + + let (_, doc, stderr) = scan(&root, &server.uri()); + assert_skipped_untouched(&root, &server, &doc, &stderr); +} + +#[tokio::test] +async fn issue_558_unfetchable_tarball_skips_the_patch() { + let server = MockServer::start().await; + let tarball = patched_tarball(); + mock_api(&server, &tarball, None).await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + write_classic_project(&root); + + let (_, doc, stderr) = scan(&root, &server.uri()); + assert_skipped_untouched(&root, &server, &doc, &stderr); +} + +fn assert_skipped_untouched(root: &Path, server: &MockServer, doc: &Value, stderr: &str) { + let skipped: Vec<&Value> = doc["redirect"]["skipped"] + .as_array() + .unwrap_or_else(|| panic!("redirect.skipped: {doc:#}\n{stderr}")) + .iter() + .filter(|s| s["reason"] == "npm_tarball_unavailable") + .collect(); + assert_eq!(skipped.len(), 1, "{doc:#}"); + assert_eq!(skipped[0]["purl"], PURL, "{doc:#}"); + let detail = skipped[0]["detail"].as_str().unwrap(); + assert!( + !detail.contains(&server.uri()), + "the hosted URL is redacted: {detail}" + ); + assert_eq!(doc["redirect"]["redirected"], 0, "{doc:#}"); + assert_eq!( + std::fs::read_to_string(root.join("yarn.lock")).unwrap(), + LOCK, + "no fragmentless hosted pin is written" + ); +} diff --git a/crates/socket-patch-cli/tests/scan/main.rs b/crates/socket-patch-cli/tests/scan/main.rs index e37f9f9d6..3074772ec 100644 --- a/crates/socket-patch-cli/tests/scan/main.rs +++ b/crates/socket-patch-cli/tests/scan/main.rs @@ -17,6 +17,7 @@ mod hosted_management_refusals; mod hosted_symlinked_files; mod hosted_wheel_metadata_order; mod hosted_yarn_berry_manifest; +mod hosted_yarn_classic_sha1; mod scan_batch_sizing_e2e; mod scan_ecosystems_scope_e2e; mod scan_invariants; diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index e1f023d25..38131a2bc 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -1108,6 +1108,60 @@ pub fn yarn_berry_manifest_targets<'a>( .collect() } +/// The npm deps whose yarn classic pin needs the sha1 of the served +/// tarball (#558): the grant carries a sha512 but no sha1, and the +/// project's `yarn.lock` is a classic lock that names the package. Yarn 1 +/// keys its cache slot by the `resolved` URL's `#` fragment, so the +/// pin must carry one. One per distinct artifact URL. +pub fn yarn_classic_sha1_targets<'a>( + candidates: &'a [Candidate], + files: &BTreeMap, +) -> Vec<&'a DepOverride> { + let Some(lock) = files + .get("yarn.lock") + .filter(|lock| !crate::formats::yarn::is_berry_lock(lock)) + else { + return Vec::new(); + }; + let mut seen = BTreeSet::new(); + candidates + .iter() + .map(|c| &c.dep) + .filter(|dep| dep.ecosystem == "npm") + .filter(|dep| dep.integrity.sha1.is_none() && dep.integrity.sha512.is_some()) + .filter(|dep| lock.contains(crate::patch::redirect::full_name(dep).as_str())) + .filter(|dep| seen.insert(dep.artifact_url.clone())) + .collect() +} + +/// Record the sha1 derived from the served tarball at `url` on every +/// candidate granted that artifact. +pub fn set_derived_sha1(candidates: &mut [Candidate], url: &str, sha1: &str) { + for candidate in candidates + .iter_mut() + .filter(|c| c.dep.artifact_url == url && c.dep.integrity.sha1.is_none()) + { + candidate.dep.integrity.sha1 = Some(sha1.to_string()); + } +} + +/// The skip recorded for an npm dep whose served tarball could not be +/// fetched or did not match its grant's sha512, so no sha1 could be +/// derived for its yarn classic pin (the grant token in `detail` is +/// redacted to ``). +pub fn npm_tarball_unavailable(dep: &DepOverride, detail: &str) -> SkippedPatch { + SkippedPatch { + purl: format!( + "pkg:npm/{}@{}", + crate::patch::redirect::full_name(dep), + dep.version + ), + uuid: dep.patch_uuid.clone(), + reason: "npm_tarball_unavailable".to_string(), + detail: Some(detail.replace(&dep.artifact_url, "")), + } +} + /// The skip recorded for an npm dep whose served `package.json` could not /// be fetched (the grant token in `detail` is redacted to ``). diff --git a/crates/socket-patch-core/src/hosted/memory/discover.rs b/crates/socket-patch-core/src/hosted/memory/discover.rs index 83f2bc039..b9a8cde21 100644 --- a/crates/socket-patch-core/src/hosted/memory/discover.rs +++ b/crates/socket-patch-core/src/hosted/memory/discover.rs @@ -391,6 +391,35 @@ pub(crate) async fn fetch_npm_manifests( .collect() } +/// Served npm tarballs' sha1 once per distinct `(url, sha512)`, for the +/// yarn classic pin's `#` fragment when the grant carries none +/// (#558): the disk flow's `fetch_hosted_npm_sha1` over the provider. +pub(crate) async fn fetch_npm_sha1s( + provider: &Provider, + wanted: &BTreeSet<(String, String)>, + max_bytes: u64, +) -> BTreeMap> { + let ordered: Vec<&(String, String)> = wanted.iter().collect(); + let futures: Vec> = ordered + .iter() + .map(|(url, sha512)| -> BoxFuture<'_, Result> { + Box::pin(async move { + let bytes = provider + .download_artifact(url, max_bytes) + .await + .map_err(|error| format!("cannot fetch the hosted tarball: {error}"))?; + crate::hosted::npm_manifest::decode_hosted_npm_sha1(&bytes, sha512) + }) + }) + .collect(); + let results = join_bounded(futures, provider.concurrency).await; + ordered + .into_iter() + .map(|(url, _)| url.clone()) + .zip(results) + .collect() +} + /// Patch views for every distinct confirmed uuid (wet runs only). pub(crate) async fn fetch_records( provider: &Provider, diff --git a/crates/socket-patch-core/src/hosted/memory/mod.rs b/crates/socket-patch-core/src/hosted/memory/mod.rs index 15286e24b..c752ca33c 100644 --- a/crates/socket-patch-core/src/hosted/memory/mod.rs +++ b/crates/socket-patch-core/src/hosted/memory/mod.rs @@ -1106,6 +1106,15 @@ async fn engine( .await, ); } + let npm_sha1s: BTreeSet<(String, String)> = planned + .iter() + .flat_map(|(_, p)| p.npm_sha1s.iter().cloned()) + .collect(); + let artifact_sha1s = if npm_sha1s.is_empty() { + BTreeMap::new() + } else { + discover::fetch_npm_sha1s(&provider, &npm_sha1s, options.limits.max_artifact_bytes).await + }; phases.mark("plan"); let stage_options = StageOptions { @@ -1121,7 +1130,7 @@ async fn engine( if stage.capped() { first_plans.insert(index, plan.clone()); } - match stages::rewrite(plan, &artifact_metadata, stage_options).await { + match stages::rewrite(plan, &artifact_metadata, &artifact_sha1s, stage_options).await { Ok(done) => rewritten.push((index, done)), Err(RewriteRefused { refusal, skipped }) => { states[index].skipped = skipped; @@ -1208,7 +1217,7 @@ async fn engine( .retain(|c| !root_deferred.contains(&c.dep.patch_uuid)); plan.skipped .extend(states[index].deferred.iter().map(deferred_skip)); - match stages::rewrite(plan, &artifact_metadata, stage_options).await { + match stages::rewrite(plan, &artifact_metadata, &artifact_sha1s, stage_options).await { Ok(done) => again.push((index, done)), Err(RewriteRefused { refusal, skipped }) => { states[index].skipped = skipped; diff --git a/crates/socket-patch-core/src/hosted/memory/stages.rs b/crates/socket-patch-core/src/hosted/memory/stages.rs index 1fa037092..f1d611c47 100644 --- a/crates/socket-patch-core/src/hosted/memory/stages.rs +++ b/crates/socket-patch-core/src/hosted/memory/stages.rs @@ -157,6 +157,9 @@ pub(crate) struct Planned { /// `(artifact url, sha512)` of every npm tarball whose own /// package.json a yarn berry pin needs (#718). pub(crate) npm_manifests: Vec<(String, Option)>, + /// `(artifact url, sha512)` of every npm tarball whose sha1 a yarn + /// classic pin needs because its grant carries none (#558). + pub(crate) npm_sha1s: Vec<(String, String)>, /// The vlt artifact preflight, judged offline (no network here, so /// every in-scope dep is withheld instead of pinned: `--offline` /// parity). @@ -210,6 +213,10 @@ pub(crate) async fn plan( .into_iter() .map(|dep| (dep.artifact_url.clone(), dep.integrity.sha512.clone())) .collect(); + let npm_sha1s = engine::yarn_classic_sha1_targets(&candidates, &read.files) + .into_iter() + .filter_map(|dep| Some((dep.artifact_url.clone(), dep.integrity.sha512.clone()?))) + .collect(); Ok(Planned { project, candidates, @@ -218,6 +225,7 @@ pub(crate) async fn plan( read, wheels, npm_manifests, + npm_sha1s, vlt_preflight, }) } @@ -241,11 +249,12 @@ pub(crate) struct RewriteRefused { pub(crate) skipped: Vec, } -/// Wheel metadata and served npm manifests (keyed by artifact URL) → the -/// engine's rewrite → the guard. +/// Wheel metadata, served npm manifests and served npm tarballs' sha1s +/// (each keyed by artifact URL) → the engine's rewrite → the guard. pub(crate) async fn rewrite( planned: Planned, artifact_metadata: &BTreeMap, String>>, + artifact_sha1s: &BTreeMap>, options: StageOptions, ) -> Result { let Planned { @@ -256,6 +265,7 @@ pub(crate) async fn rewrite( read, wheels, npm_manifests, + npm_sha1s, vlt_preflight, } = planned; let skipped_before = skipped.clone(); @@ -304,6 +314,25 @@ pub(crate) async fn rewrite( } } } + for (url, _) in &npm_sha1s { + match artifact_sha1s.get(url) { + Some(Ok(sha1)) => engine::set_derived_sha1(&mut candidates, url, sha1), + Some(Err(detail)) => { + if unavailable.insert(url.clone()) { + for dep in candidates + .iter() + .map(|c| &c.dep) + .filter(|d| &d.artifact_url == url) + { + skipped.push(engine::npm_tarball_unavailable(dep, detail)); + } + } + } + None => { + unavailable.insert(url.clone()); + } + } + } candidates.retain(|c| !unavailable.contains(&c.dep.artifact_url)); let view = ProjectView::Memory(&project); diff --git a/crates/socket-patch-core/src/hosted/npm_manifest.rs b/crates/socket-patch-core/src/hosted/npm_manifest.rs index 9087631a7..532784c41 100644 --- a/crates/socket-patch-core/src/hosted/npm_manifest.rs +++ b/crates/socket-patch-core/src/hosted/npm_manifest.rs @@ -43,6 +43,32 @@ pub async fn fetch_hosted_npm_manifest( decode_hosted_npm_manifest(&bytes, sha512) } +/// The sha1 (hex) of a served npm tarball, for the yarn classic hosted +/// pin's `resolved "#"` fragment when the grant carries no sha1 +/// (#558). Yarn 1 names its cache slot after that fragment, so a +/// fragmentless URL shares the slot of any fragmentless upstream copy of the +/// same version and installs its bytes. The bytes must match the grant's +/// sha512: that is what the pin's `integrity` line names, and a sha1 taken +/// from any other bytes would pin a tarball yarn then refuses. +pub fn decode_hosted_npm_sha1(bytes: &[u8], sha512: &str) -> Result { + crate::vendor::registry_fetch::verify_sri(bytes, sha512) + .map_err(|_| "hosted tarball does not match its published sha512".to_string())?; + Ok(crate::utils::digest::sha1_hex_of(bytes)) +} + +/// Download the served tarball and take its sha1 ([`decode_hosted_npm_sha1`]). +pub async fn fetch_hosted_npm_sha1( + client: &ApiClient, + url: &str, + sha512: &str, +) -> Result { + let bytes = client + .download_artifact(url) + .await + .map_err(|error| format!("cannot fetch the hosted tarball: {error}"))?; + decode_hosted_npm_sha1(&bytes, sha512) +} + #[cfg(test)] mod tests { use super::*; @@ -63,6 +89,18 @@ mod tests { builder.into_inner().unwrap().finish().unwrap() } + #[test] + fn sha1_is_taken_from_bytes_matching_the_sha512() { + let bytes = tgz(&[("package/package.json", br#"{"name":"left-pad"}"#)]); + let sri = sha512_sri_of(&bytes); + assert_eq!( + decode_hosted_npm_sha1(&bytes, &sri).unwrap(), + crate::utils::digest::sha1_hex_of(&bytes) + ); + let other = sha512_sri_of(b"other bytes"); + assert!(decode_hosted_npm_sha1(&bytes, &other).is_err()); + } + #[test] fn decodes_the_manifest_and_checks_the_sha512() { let manifest = br#"{"name":"uuid","bin":{"uuid":"./dist/bin/uuid"}}"#; diff --git a/crates/socket-patch-core/src/patch/redirect/lock_index_equivalence_tests.rs b/crates/socket-patch-core/src/patch/redirect/lock_index_equivalence_tests.rs index 79225d6e8..9e4b8b52c 100644 --- a/crates/socket-patch-core/src/patch/redirect/lock_index_equivalence_tests.rs +++ b/crates/socket-patch-core/src/patch/redirect/lock_index_equivalence_tests.rs @@ -262,7 +262,14 @@ fn indexed_yarn_classic_rewrite_matches_golden() { _ => {} } let files = BTreeMap::from([("yarn.lock".to_string(), text)]); - let deps = overrides(&pool, &mut rng); + let mut deps = overrides(&pool, &mut rng); + // A yarn classic pin needs a sha1 (#558): most grants carry one + // (or the hosted flow derives it); every seventh dep lacks it. + for (i, dep) in deps.iter_mut().enumerate() { + if dep.integrity.sha1.is_none() && i % 7 != 0 { + dep.integrity.sha1 = Some(format!("{i:04x}")); + } + } let mut got = RewriteResult::default(); rewrite_yarn_classic(&files, &deps, &mut got); record(&(&files, &deps), &got); @@ -272,6 +279,7 @@ fn indexed_yarn_classic_rewrite_matches_golden() { assert!(edits > 400, "edits: {edits}"); for code in [ "redirect_yarn_classic_missing_sha512", + "redirect_yarn_classic_missing_sha1", "redirect_yarn_classic_alias_skipped", "redirect_yarn_classic_entry_not_found", "redirect_yarn_classic_unresolved_entry_skipped", diff --git a/crates/socket-patch-core/src/patch/redirect/mod.rs b/crates/socket-patch-core/src/patch/redirect/mod.rs index 7f007a03f..323ccacaa 100644 --- a/crates/socket-patch-core/src/patch/redirect/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/mod.rs @@ -3762,6 +3762,23 @@ fn rewrite_yarn_classic_with( }); continue; }; + // Yarn 1 files a tarball in its cache under the `resolved` URL's + // `#` fragment; a fragmentless hosted URL shares the slot of + // any fragmentless upstream copy of this version, so a warm cache + // installs those bytes or fails the integrity check (#558). The + // hosted flows derive the sha1 from the served tarball when the + // grant carries none; a dep that still lacks one is never pinned. + let Some(sha1) = dep.integrity.sha1.clone() else { + result.warnings.push(RewriteWarning { + code: "redirect_yarn_classic_missing_sha1".into(), + detail: format!( + "{fname}@{} has no sha1 for the yarn.lock `resolved` fragment, so it \ + is not pinned: yarn 1 would share the cache slot of an unpatched copy", + dep.version + ), + }); + continue; + }; let mut matched_any = false; let mut pinned_any = false; let mut alias_skipped = false; @@ -3962,15 +3979,9 @@ fn rewrite_yarn_classic_with( continue; } pinned_any = true; - let frag = dep - .integrity - .sha1 - .as_ref() - .map(|s| format!("#{s}")) - .unwrap_or_default(); let pinned = repin_classic_block( &block.lines, - &format!("{}{frag}", dep.artifact_url), + &format!("{}#{sha1}", dep.artifact_url), &sha512, ); if pinned != block.lines { @@ -8114,6 +8125,10 @@ fn rewrite_golang( mod tests { use super::*; + /// The sha1 an npm grant carries (or the hosted flow derives from the + /// served tarball, #558): yarn classic pins it as `resolved`'s fragment. + const NPM_SHA1: &str = "5ha1"; + fn npm_override(name: &str, version: &str, url: &str, sha512: &str) -> DepOverride { DepOverride { ecosystem: "npm".into(), @@ -8126,6 +8141,7 @@ mod tests { registry_override: None, integrity: Integrity { sha512: Some(sha512.into()), + sha1: Some(NPM_SHA1.into()), ..Default::default() }, } @@ -10808,7 +10824,7 @@ mod tests { assert!( out.contains( "left-pad@^1.3.0:\r\n version \"1.3.0\"\r\n \ - resolved \"http://p.test/lp.tgz\"\r\n integrity sha512-PATCHED==\r\n" + resolved \"http://p.test/lp.tgz#5ha1\"\r\n integrity sha512-PATCHED==\r\n" ), "the target entry must pin the hosted artifact: {out}" ); @@ -10861,7 +10877,7 @@ mod tests { let want = lock.replace( "resolved \"https://registry.yarnpkg.com/left-pad/-/left-pad-1.3.0.tgz#bbbb\"\n \ integrity sha512-UPSTREAMupstream==", - "resolved \"http://p.test/lp.tgz\"\n integrity sha512-PATCHED==", + "resolved \"http://p.test/lp.tgz#5ha1\"\n integrity sha512-PATCHED==", ); assert_eq!(r.files["yarn.lock"], want); } @@ -10882,7 +10898,7 @@ mod tests { let mut r = RewriteResult::default(); rewrite_yarn_classic(&files, std::slice::from_ref(&ovr), &mut r); assert!( - r.files["yarn.lock"].contains(" resolved \"http://p.test/$1/$name/lp.tgz\"\n"), + r.files["yarn.lock"].contains(" resolved \"http://p.test/$1/$name/lp.tgz#5ha1\"\n"), "{}", r.files["yarn.lock"] ); @@ -11277,7 +11293,10 @@ mod tests { rewrite_yarn_classic(&files, std::slice::from_ref(&ovr), &mut r); assert_eq!(r.edits.len(), 1, "{copy}: {:?}", r.edits); let out = &r.files["yarn.lock"]; - assert!(out.contains("resolved \"http://p.test/lp.tgz\""), "{out}"); + assert!( + out.contains("resolved \"http://p.test/lp.tgz#5ha1\""), + "{out}" + ); assert!(out.ends_with(copy), "{copy}: copy byte-identical:\n{out}"); let codes: Vec<&str> = r.warnings.iter().map(|w| w.code.as_str()).collect(); assert_eq!( @@ -11343,7 +11362,10 @@ mod tests { rewrite_yarn_classic(&files, std::slice::from_ref(&ovr), &mut r); assert_eq!(r.edits.len(), 1, "{:?}", r.edits); let out = &r.files["yarn.lock"]; - assert!(out.contains("resolved \"http://p.test/lp.tgz\""), "{out}"); + assert!( + out.contains("resolved \"http://p.test/lp.tgz#5ha1\""), + "{out}" + ); assert!( out.contains(file_block), "file: block byte-identical:\n{out}" @@ -11406,7 +11428,7 @@ mod tests { let mut r = RewriteResult::default(); rewrite_yarn_classic(&files, std::slice::from_ref(&ovr), &mut r); assert!( - r.files["yarn.lock"].contains("resolved \"http://p.test/lp.tgz\""), + r.files["yarn.lock"].contains("resolved \"http://p.test/lp.tgz#5ha1\""), "the registry block is still pinned: {:?}", r.files ); @@ -11489,7 +11511,10 @@ mod tests { rewrite_yarn_classic(&files, std::slice::from_ref(&ovr), &mut r); assert_eq!(r.edits.len(), 1, "{:?}", r.edits); let out = &r.files["yarn.lock"]; - assert!(out.contains("resolved \"http://p.test/lp.tgz\""), "{out}"); + assert!( + out.contains("resolved \"http://p.test/lp.tgz#5ha1\""), + "{out}" + ); assert!(out.contains(git_block), "git block byte-identical:\n{out}"); assert!(r .warnings @@ -11591,7 +11616,7 @@ mod tests { assert!(r.warnings.is_empty(), "no warnings: {:?}", r.warnings); let out = r.files.get("yarn.lock").expect("must rewrite"); assert!( - out.contains("resolved \"http://p.test/lp.tgz\"") + out.contains("resolved \"http://p.test/lp.tgz#5ha1\"") && out.contains("left-pad@^1.3.0, \"safe-pad@npm:left-pad@^1.3.0\":"), "merged key preserved, resolution repointed: {out}" ); @@ -20815,6 +20840,44 @@ packages: ); } + /// #558: a yarn classic pin without a sha1 would have no `#` + /// fragment, so yarn 1 would file the hosted tarball in the cache slot + /// of a fragmentless upstream copy and install its bytes. The rewriter + /// refuses such a dep and leaves the lock untouched. + #[test] + fn issue_558_yarn_classic_refuses_a_dep_without_sha1() { + let mut ovr = npm_override("left-pad", "1.3.0", "http://p.test/lp.tgz", "sha512-P=="); + ovr.integrity.sha1 = None; + let mut files = BTreeMap::new(); + files.insert( + "yarn.lock".to_string(), + "left-pad@1.3.0:\n version \"1.3.0\"\n \ + resolved \"https://registry.yarnpkg.com/left-pad/-/left-pad-1.3.0.tgz\"\n \ + integrity sha512-UP==\n" + .to_string(), + ); + let mut r = RewriteResult::default(); + rewrite_yarn_classic(&files, std::slice::from_ref(&ovr), &mut r); + assert!( + r.files.is_empty() && r.edits.is_empty(), + "no fragmentless pin: {:?}", + r.files + ); + assert_eq!( + warning_codes(&r), + vec!["redirect_yarn_classic_missing_sha1"] + ); + + ovr.integrity.sha1 = Some("abc123".into()); + let mut r = RewriteResult::default(); + rewrite_yarn_classic(&files, std::slice::from_ref(&ovr), &mut r); + assert!( + r.files["yarn.lock"].contains(" resolved \"http://p.test/lp.tgz#abc123\"\n"), + "{:?}", + r.files + ); + } + /// The pypi twins of the missing-integrity legs: requirements.txt and /// uv.lock each warn for a granted dep with no sha256. #[test] @@ -21913,7 +21976,7 @@ packages: assert!( out.contains( "left-pad@^1.3.0:\n version \"1.3.0\"\n \ - resolved \"http://p.test/lp.tgz\"\n integrity sha512-PATCHED==" + resolved \"http://p.test/lp.tgz#5ha1\"\n integrity sha512-PATCHED==" ), "integrity inserted after the repointed resolved: {out}" ); @@ -23158,7 +23221,10 @@ packages: out.contains("not-a-key-line"), "keyless block preserved: {out}" ); - assert!(out.contains("resolved \"http://p.test/lp.tgz\""), "{out}"); + assert!( + out.contains("resolved \"http://p.test/lp.tgz#5ha1\""), + "{out}" + ); } /// An EXPLICIT `.yarnrc.yml` `compressionLevel: 0` (the supported value, diff --git a/crates/socket-patch-core/tests/equivalence/yarn_classic_rewrite.golden b/crates/socket-patch-core/tests/equivalence/yarn_classic_rewrite.golden index d88a25727..ce0c4ff23 100644 --- a/crates/socket-patch-core/tests/equivalence/yarn_classic_rewrite.golden +++ b/crates/socket-patch-core/tests/equivalence/yarn_classic_rewrite.golden @@ -1,202 +1,202 @@ # One seeded yarn.lock (v1) + overrides. # -0-1 1190d243c3c9a00e 26123a54a217cc6a -2-3 ba98bffaf480d822 462668fcfc8da48c -4-5 d80b15a695b0937f 17c1f49d0da5b64a -6-7 8e178c2cb5e2cb14 7beaacd493dec64a -8-9 e43fd997c62b9e42 7ad679dee1ec8d49 -10-11 edc61bca32cfc9f9 d36e0c7622166846 -12-13 175b8cae66ba4ad3 fd167be2cc4baa0f -14-15 ef91b060692096d2 0e56ab7ae7b308b3 -16-17 6f0831d9cf265f98 a5226cb66c765543 -18-19 751e330a33334e51 cd325152b319dc99 -20-21 7bbcf7e2f9c9c88e 4c166a49cf87d968 -22-23 5e8538d25f0c3d2f ce0a8119ff5d0bd1 -24-25 1b52d945841c8298 1d812d689a293c0a -26-27 85871c4064b2c3e3 ff5fda6a76281fe2 -28-29 352dff2ab375c6a3 ecfbf66dd2c128b4 -30-31 edc5d9ff68527344 e111668f86f256b6 -32-33 ce2c969dad5ecb04 17882ac3447ac356 -34-35 de24932e90101396 6299761ad1e40014 -36-37 33a2e2444574429e 652c5bf017c28bdb -38-39 c57c275705d2a2a7 c10df418bd28ee37 -40-41 b5152432f665aeb8 d48d6b27414664f1 -42-43 eef444c636c55e21 e3b972dea8d7e329 -44-45 5cd2da04fa3dac81 2a097e42a1c82ef2 -46-47 9dec65795e922fa4 f56dda6fbc4a342c -48-49 1d9d1d776f1f7250 0369cdaa0fba4722 -50-51 2d362a1b1674d398 969272f078a3eccb -52-53 be73a4ad6b8e5aee d503591229d2ebbe -54-55 ba9646f86c624aaf 070cebbd84a57962 -56-57 3e96a5f86e6af680 b9d86d938087da55 -58-59 8dcd4ce057935cc3 2faa46d48b275db0 -60-61 6333a5219075a09d 9a223786aae324f4 -62-63 f84be846c0d54615 f5136e8afcf7da29 -64-65 e6f8ad9ceedd2e4c def49b7bcdf8b86a -66-67 c4e236ff81ac9a8d 90885de7399544af -68-69 dadc04add7c7c9e0 1747157119d5b11e -70-71 a5711b822e995dfd 554475f10b1ff3e5 -72-73 c976cf46cd6b6f85 9c7b691c8d6b1390 -74-75 18b9eec756bb47e3 9f12e86f7660befd -76-77 54ce49f58715780c 767eef79260a156b -78-79 e8288b75119434a1 c96af7a042a0e96b -80-81 94ff16825564c958 12b0a29c28c9fcda -82-83 bb940199daee8606 773ce11ce18b2d49 -84-85 2ff605c25684cc72 438eb1fd24089b17 -86-87 157e638a5bba3a3c f3f28575a81fa59c -88-89 f5d52d058b7378f8 644f44f392f76522 -90-91 c756f82012e55c10 a6b2890dcc2c9e8b -92-93 7d2b44d28176a529 61924f9eefcc34f3 -94-95 174fb249b68ff4ad ec5baff76f6e2c99 -96-97 5dbedd5a89add533 67ebf69068f6dc10 -98-99 3078474883707efc d7aa0eb390e30e3e -100-101 85bdf6d5adbbf56d f726d638959e924e -102-103 baa7ffe0727796d9 89a7fcc45903a226 -104-105 292106ad225e1037 9fe9dc9c26d5a7cf -106-107 d08889126f5fe2a3 9d4aadc6ba4e7e2e -108-109 d7c8957c4b25e62a fc7752feb4e46245 -110-111 3a71c785f0a34d60 19ef39a099202dc3 -112-113 cd9963e800c7f246 873028674465efe9 -114-115 e2ec8f5f04db713d ccad19c0a78ba813 -116-117 9028fc942e7550c3 1cd3d2d35c77f8b2 -118-119 8ebdd92eeeef6f36 bc02491aa9c459f1 -120-121 92a4e961365a2f97 545803bf5fb26aee -122-123 5482e22a9366a649 e84a7ea1aa4e7e7d -124-125 1791bc6454f83fd8 192d0065f8820dfa -126-127 3fc012d899a28130 2fbccd0114f6583e -128-129 6d3b3fc00498115f c9a345287353c637 -130-131 27f5cff8b7be114c e21f7ef6afefa1c6 -132-133 caeabae83aeb4228 ab3533e29575b0f1 -134-135 a376820bfb6ab010 fe1aadea7f633d41 -136-137 cfd0efc3c3db5202 6f717d6e639ef601 -138-139 c587b2ef7d70c2aa 15b28c52a55883ba -140-141 e09bae32f9966aca d7d6bdfde0b94271 -142-143 60ce0d4f6fa6794b f09b8a1fd7ea148f -144-145 410e854b4e2efd89 71d3c014b94687ef -146-147 fd493568c3ac3652 47ac1f33a38092c0 -148-149 deb618ecb555fd73 aef35ea15ae64f2a -150-151 d6f51b46647d1d15 80e95c4628a02002 -152-153 d0f497c3393351fc cdf20344f77fb72d -154-155 a5e2964501d49184 7982b4dbe175edf7 -156-157 342978fadea628c9 1c11b1b8cba0de98 -158-159 c068738f7de05532 e25303a2fcc08cbf -160-161 82f66cf7979cc1e3 876b624b0262cf1c -162-163 79bb0f03807f6b44 f904123e265b1c03 -164-165 22981a3f179cfc9a 97689d6bbf14955e -166-167 0aa1b383329a8627 ea72381fa410b037 -168-169 3f162034aef1dee7 947a9d3e170cb3ad -170-171 d270a58f50af5bc2 b08d4b0cc8843b3d -172-173 3de38dd2c44458ee 11cc4dcc7e53b279 -174-175 a1224e468a2a3299 ea6469029021b1bf -176-177 2907e9afcb20dcd1 be6367180453d478 -178-179 33c22e07e587c29c 192a91899a0c5c71 -180-181 e6f1f6e30a93629b 005e4e7141371c79 -182-183 aaa04fba9f217938 998fb75fe23e5e78 -184-185 f74901f680aa0406 48723ce071811b45 -186-187 57f8170e481abfa8 7710e41cc383d389 -188-189 2e20a5ea93aa89f6 00983fc2e7ef6033 -190-191 3f828aa93ae947b7 90fa77ec9101d6ec -192-193 dd2bebf464535865 b8e8e41d6f914f58 -194-195 5d92b24539551fa0 66db9abc47d08fcf -196-197 507866064aec4111 39285d092da8958a -198-199 17c0359ab8b2f84c 2b674796d041828b -200-201 53f4743f0fae2db0 2b100dc05a2e69eb -202-203 6640c39887079e0f 8d6673589dac7ebd -204-205 c08391b643d19e32 1b07d17000ef2731 -206-207 6b805f5639c7107c f0f025204824b532 -208-209 5c5bfc2ad062e253 e86681d615a21831 -210-211 e041d12d9e34a7e4 e5c398731e6218f0 -212-213 8162e7cdf8275290 3cc752c6a920fed7 -214-215 1de934fb8b35e04a ea468ae83cd41e51 -216-217 410197c9dfc7c2f2 e8323ac70d642f6a -218-219 d739a2f19922bb59 b73b9cbee7124679 -220-221 82e62cba865edff5 75d0f607bc7f55da -222-223 87879277b6d6b27a b33f4a8273770e65 -224-225 926079079c0ecb3f c0c847d8035b3a9b -226-227 d7a6db3f2ad44ad0 6da057b1baadd966 -228-229 872bca09ed8ba084 30337baa82fd71b8 -230-231 7d7bf22a0e9cb805 b2f9c543a136d2b4 -232-233 be20fb9e96cb7c53 285c79e26c5de1ef -234-235 acd2ff104a2ac96f 895cadeaaa008c99 -236-237 196b2da9f9488cde d6aa788f91a4481b -238-239 afeb25d31570da3e 02b9e34e47da4a52 -240-241 ca7bab4e4dc37bdf 67b1916a93f5a457 -242-243 0004a191c10ab26d 46d54cf3a7187164 -244-245 d29b7cf3240f3a55 43f2a3301300e012 -246-247 756953a9e086f2d5 f6ff6c6ea51a6673 -248-249 1f0d7cd899a8ef8c e30a21855d1dbc36 -250-251 21c745b4215a4f2b b06230462ee85642 -252-253 6eb261d8c1405b0f aecfca3b9924e8ad -254-255 1ddd81908edf76f1 3c07e1cc234c6929 -256-257 d9bbc8bfea3bad46 475c48d49638e01e -258-259 25a4e018fbb1645a 718dd94601a7ae4b -260-261 62e250bc92ffe414 565a49ea1d97cdbb -262-263 e784b7716f83965a b390cf8ddcc0e2bc -264-265 e458082ea4c109ad 8a5936d112d341a6 -266-267 cac892ea348b5ecc 7c47c66b810a7468 -268-269 85dd46c8a49a55ef 3cd56f9c53cff45f -270-271 506bc333993d7c90 83a127973b41a04f -272-273 e97bb5a51749b02b 0aa45b9ebefb1e8a -274-275 292826999ded1d5e fb34d4991ff9e60d -276-277 d90823b57af9eda7 ed4720c5c0545c2d -278-279 ad19b90a41936767 4d4c6b8279526c7a -280-281 edafe6d07499b8e4 398bd84b1ca365cb -282-283 50064db2df0bd060 a72e4d2f8b1f36c1 -284-285 c8ee70bf288a3dcd a32c08f334f66f3b -286-287 04dfa6be66fb6d83 8bf9cc835fd7450f -288-289 514eb6ba7feedc57 7e791f56ef0aac2c -290-291 e8eaef431b35e2b3 dcc35dd2fb83a73c -292-293 eb6dffaf52bd3be4 e1bf28914e121d02 -294-295 d904ba055623d9f8 7c72b519d61ab05c -296-297 1f2c741d7d420ce6 e65c603d8adf6e27 -298-299 c03582ccddb96cee 63f84810369c7406 -300-301 748a964dddc5b4dd 8bcdd0b91e8108da -302-303 42d640238fbf8361 cbcb4962672a0884 -304-305 228ec73b8d5ae872 e4597ee389c2f3eb -306-307 1655281cb4d8c9ce 6033fb187f35408b -308-309 dd3eb82b3e77ec6f a34e37e47900b3ae -310-311 f811e22101402c2c 55c63b649a76d3ca -312-313 e44293fa25db1eaa 7c774f5ae0d0ec4e -314-315 1e14263df9269f95 d9d25add40395f4e -316-317 686729154bf30a49 d44cf0e0c62f9fbe -318-319 77a4f2ba9a0cf3df ceed8ae56dbf991c -320-321 0330a69284ddd224 5d752e4c141366eb -322-323 441245a0d0613419 12f21c2f3049b467 -324-325 27bfb24e0adaca59 92b144a41b45c2b4 -326-327 dfbfe9addb1ce656 ad2a0ae506d3fb56 -328-329 8d28a268abb3b404 e0344a8938d2be6f -330-331 48f7939e56400f0f 631b6633830571d3 -332-333 581fc9a0c31802f0 cc9968a7b2030ab7 -334-335 b0350490b1ee0793 90b01dba37cd6e67 -336-337 421a82f155863960 100c2e40ed2d12c4 -338-339 e1a31b3355f246bf 3859f4780498b2a7 -340-341 4e99c984d6efa99b 35c48ff807ba1e57 -342-343 93cd9abc548829ff 1d109ecb63aea187 -344-345 4940857b43e7fce5 4fe02418204ef42a -346-347 089a20d53499ef50 b68c39cc0a19918c -348-349 d6cda592cf180789 63d1fcbe4c03e310 -350-351 88cf16f5d1148288 267d4c2db16acc9d -352-353 13fcaa8b580b87c8 ca9764368c465b93 -354-355 0c50455095172a40 fc276d10f198f125 -356-357 4b370e3af0e4f194 7bc98453d92b4267 -358-359 7f1f14a0f8535a2f 7d3220dbde920697 -360-361 a34fbcd774798fac ee5ca5aa6a27eff4 -362-363 e8deea23e015fbce 4d4ce8f0cfee9d1c -364-365 118298c4bd6929b9 791b880312f2c6e0 -366-367 a88c88ad0662add9 1ec9f06182316448 -368-369 7e222e2654d0869d 7f9899a3bc672d5b -370-371 192d435734b4b17f 504d045bac4ed52c -372-373 8b9b5c3c4a391ee0 af6b455822938294 -374-375 30c7c3e962b54688 a974738c309b67f1 -376-377 58fecebbcdbd5a7e 070766d406a13adc -378-379 8347ff535fcf69f8 1188530426a4d631 -380-381 8c4209360acca12c 4501cb987fd0d172 -382-383 8c93f53cc0d92461 418ae47e248ffde2 -384-385 19edb8803a2e153b b0bc8c57a471c371 -386-387 c70af0e28589e6cf 1f72156b060fb3ce -388-389 5602e24e0e9c95ae b3172535a5c3f284 -390-391 defead10329f9dda 7c5de1730928e92a -392-393 24b162373746f101 75c2f66f04c6db8e -394-395 449abdb26f8b082e f0591f59b3e668c3 -396-397 bf06982a6266b8d1 e176180d20ff8061 -398-399 4a73ec47d01832f8 197ba0aeeb7822a1 +0-1 fe9f78c1dfd2facf a1c1cc0ee08f7c03 +2-3 bbbbfdf486e085a9 88799d5df5ebb781 +4-5 9a9c5284ebf2ba8f 20221be6367cacdb +6-7 1fd3b3de9157cf27 be211914e3c7a047 +8-9 1a27b64dffda9474 2978de99b10fa3c6 +10-11 705e115b01db8b62 de3c8d46b1d70f19 +12-13 ea6a8abd66053518 cf3ce7acf8755a45 +14-15 00570d8ec71d475a 897bed4469158af6 +16-17 e229b32134c55626 dbba4172f3b06ea7 +18-19 4430af42d267ccd5 d2e4f41783765855 +20-21 d2dc31e23909c202 d6fc2532c467f15a +22-23 dd22f9187f6e881a d98c07a03a4a0c05 +24-25 a591244e682f348b c8a63fe5d8786529 +26-27 ea007389041436c4 a169da848aceedcc +28-29 7923ab27a188f3e3 a3ee0c0345924573 +30-31 dec69bc8cd929020 2a992f5bd4d5e692 +32-33 ea4ebf15abf9978c 354d9d7dafcd9d44 +34-35 7c106b4b44d79051 11f80c90b30389d6 +36-37 58305fd11b5d6f3a d0109f3b56e8ff8f +38-39 99ebfbdafa7fd7ff 877a3e99689d4290 +40-41 a0bb17192a0a620c 937581ca3125714b +42-43 12883d0234562cd5 dbe18a50a41344ef +44-45 9b43f9d548ea636f e39cc4caff960c9b +46-47 53f01953a7c07dfb 7556e7906ae90be1 +48-49 7ab6d7a36c60619c 4a4cb4e8b265be82 +50-51 5dfcd49c18fcc644 3d9a010e6e2ddc31 +52-53 db1d2c9d124a6674 395264d6e4a48d04 +54-55 da122894c78627c0 cf0ec8e9518353d3 +56-57 8864e2938b79d65c bf2c0f087690a9ea +58-59 b00f85e0c9c0801f 7dfd3e077cff8d0e +60-61 a68ac301f03dd3a4 9bd60e2c734f6df4 +62-63 6888d0545a2a4db1 d6e5af5ddebbc5f9 +64-65 70f2bbccb386b78a f10e3ab5b5b82c85 +66-67 32cfb7657214d5d3 ca0af112566fffe9 +68-69 f54ddcbf8a3f902b 52beb0da91954d06 +70-71 0cce23c597817e4f cf88c3b73dd663ce +72-73 aab086387bc3dfe6 6238b98daa855dac +74-75 c0c0b87095a27938 f289e978188b253d +76-77 f3a9d99bc7718697 4b281d03b26f1c51 +78-79 0e3d15d4dacf0c6d 193d073c255a12ae +80-81 ba2cb43b7239100c aca8df37c6da3da5 +82-83 ddab402a3a960b84 b1cf3240f5a6f64a +84-85 ecc6824d489c2389 378e545847161c27 +86-87 699a44ba969e864e f36b9327bc254160 +88-89 379c50e01e78a465 28cbd8c71f14a14a +90-91 d20092f0cc363029 e1e6f913bee55c31 +92-93 a51fc27952d62b5e 425bb4772dd9db54 +94-95 e5fd6f0f5e606e07 d6e855e522a323e6 +96-97 e2131aa3b26d53b8 606e40a3b2aeeb22 +98-99 294d01580f557b04 b285eac535cd3e6f +100-101 406ce6e5d67792f5 ee77973ce5e36d3f +102-103 5a2faa257c1a6129 582cb411e672959c +104-105 239e31c182b0d86b 544a7a9c808338c0 +106-107 059473cef5484a9d 5cf65af2cbbed22a +108-109 db47eee5fb332e5c ceedf0b44c84b7ca +110-111 421d37fe374cb96b 5ed1d89295cf68e4 +112-113 f8381a08b9273633 f6f9f2710678bbd2 +114-115 a5dbae12ce20654f 5e8b4df58243ce6a +116-117 86caa0678978dce9 5a034fbe15bc44ad +118-119 0c3339b7d6b6287f 014787ee641ae69a +120-121 1e04b3b73a25e281 86556b3584fbc034 +122-123 df93e9e32c1fe73e c8ceaae6b9cbdbd1 +124-125 143bebf42d6e6ba5 f7289cd6a22d7600 +126-127 38d4e1c27964f929 094223eae489711f +128-129 b9febf52a415e00e fd8d1fe1ef1cb174 +130-131 ec7c1c8e571a7201 57501e73364fc47b +132-133 3e8221ba167dab8b 9864f91c910a9483 +134-135 76b29253eb92239c 9d69dd4a1e3da1c0 +136-137 016f53907b5bedb5 2650e830e0d6fdcd +138-139 b895e8fb4431d28e 3cb4976c3c5b2294 +140-141 69a3e30cc6105ed1 0bdf4b721298a194 +142-143 4638a36e50f5f758 5d5fc789c0517b6c +144-145 080263050c143ae6 e885e14930fe7114 +146-147 a284a0f8b1f5e0b1 e9fd82e1800faa59 +148-149 4dc043a3f15e2e24 f4051f4193016ed3 +150-151 414b62e4937e4649 1620c6e0b70314e6 +152-153 ff968c7c2cd15d15 aea5a6d11d83c464 +154-155 22b650a562fcdfa7 8f96902c1b789ae3 +156-157 c2387f78d29e825b dc6eb8f21b80a0fb +158-159 8574b550f8346b5c 20ab25c28e1a47e2 +160-161 fe8a0c59fe117b6b 0b3568e5a39823a1 +162-163 2892c95fefe272db 32031c72999afd72 +164-165 b25d2a376132b587 31ebe652e3d154e6 +166-167 85faa59690dd38c2 d91d0225cbbe55bd +168-169 95c3ff79d35c98dc 7afe0097db848dac +170-171 9abcbee57274b63d 1a309a7cf4bca7c1 +172-173 9de7270be0884737 ed0fabf062d3f82b +174-175 c9af9ae8c7f90981 f8889c3416d09abf +176-177 f266914ad784966b 5be4a264c5407af5 +178-179 b77f3b201d6447e6 066c4fa87184e639 +180-181 08da6182e9c44ce7 199f56c9993a1951 +182-183 14f7f9be2b292809 998fb75fe23e5e78 +184-185 ef12afbf40aa9f5d df07f7ec58d51ff6 +186-187 e23fbd6a57ddb9fe 312d6da9b541dd70 +188-189 01a25a92b62bb721 5a0f794e04e5142f +190-191 3439c81c4f6d8fd0 877b9014b348be29 +192-193 7a0b64e30e401b43 3760f753554702bc +194-195 c35131e42527da41 b0047d8044a25a8b +196-197 ceed2c82e1817b97 101da0923c5ce657 +198-199 ad296d3e74cdafa7 0897a2ea52e57481 +200-201 87e1756708196876 d5a351f98c23b42a +202-203 5a48a3eb3cfe69b6 b6e499157ee82356 +204-205 1d9f40fb1fcc4963 75cc9d518eec50e4 +206-207 d274bae10511adc4 91c8b16809add18e +208-209 776e8f9b96a3dcfb c69464194a0e1abf +210-211 5497990b4a1e1af6 091ef69297e860d0 +212-213 96e477ca287aa142 c825ad0c6f9e9d4d +214-215 6879b27d17777eb5 1e33ed23cbbef56a +216-217 ca7533c50ae6340f 684f5a16358de5ff +218-219 91b56648b43beea8 693c405ed1eef7d2 +220-221 0796099ce4cd3ed6 df95b0c5bf67d3be +222-223 3d7bb7968080586b acd87b86d6d3d056 +224-225 e78aeddb820fac3a f7e13ba3b8bed6a8 +226-227 750a2cd9e1276b99 6fae1e6aa67dbfe3 +228-229 de28949cbe040876 8095c3d649fd0bca +230-231 acb014486eff2fab f0bc8ff50f0c4333 +232-233 86cf596d4255d0f8 12859fdd56b5e81e +234-235 0bb3046ccbde81ad f4ba21a17001b759 +236-237 133a11f1814abb50 f3081a80fbd374f1 +238-239 75a3051e34591fed a084b2a321cc312f +240-241 f0a19a251b21de96 7835f05cbc661cf0 +242-243 b5ce3ed10257d10d c326167670ac0262 +244-245 fb7e90d13e9bf7cd 6282f5b8e9254e02 +246-247 6576f97ea87bb3ee 872e567b2abf2501 +248-249 9d0a5fa6c35d5dca 01e9ae5711eab207 +250-251 a07dfd54df2e8970 5e581b42bc5c80e0 +252-253 023089d365245a90 5349b26cfa8c6925 +254-255 c45562f224cab9c4 2a3d5efc9e03efbc +256-257 f1b8c77972883f83 4a932ee0a768959c +258-259 dededac41eac41e4 546779cbdf479f6b +260-261 a2ff23f7d709fd27 815c13ee7611b832 +262-263 e339ff61812b76d2 06916dab9623c20c +264-265 ebf4508987a6494c 04a4b079acd5f775 +266-267 a2eba7ecb8205014 ff54db163cdc5f1e +268-269 b6a48b7e4a0e9473 05e794446671b15d +270-271 dda06f4ed1d9a139 c96eb0361d702bbf +272-273 6f1bcf87e17b2fd1 f2061d4eb9277428 +274-275 e4cd511e2e055f5e 5e545466ea1a606e +276-277 cd744cd138037dd8 da208be368cfbffe +278-279 9b8af06bcd0f4d54 afcff83a8ef7eb6d +280-281 4a6cadf5171571cf 5407fd6163dae514 +282-283 79a9111885765720 11571bfa7c4b150b +284-285 2f89bc6bf3bfcac0 c307dc8a0f590fbf +286-287 b78d4859f2135e2a 031930d7fcf0d448 +288-289 2ad3d902e78789e5 47ce909c06b78d74 +290-291 70beeeaab183a454 9798c4ee895f24b6 +292-293 44cca66f82dc90e1 7c2b9cdecc04bc28 +294-295 d0f77d14f8f9f222 5f19ed30367d6dfe +296-297 a6726fe3c8b82603 4cb49cd83e89f57c +298-299 2f406ebae87af96a 955ab0b00b11d7f8 +300-301 c16ea012a99abdaa 2712cd69107ccabe +302-303 281b5dca23465534 12461c2b65863123 +304-305 3ddfacfde12702f2 cf0865af069bf3be +306-307 a2d792ed9bb5b7bd cf7610102ccd429a +308-309 5b5276bc3b04254e 409484fe000442c1 +310-311 946b30260acc25ab ea3d9980db5006d9 +312-313 f29765d06ad3f261 626ab854e9c1db81 +314-315 a2db7551ceb64071 f419b51044795f57 +316-317 ae151cda3a820df2 5391f308eee63f60 +318-319 6e3394650aac03c0 48efbfe7537b1dc4 +320-321 1f874c027596b989 32741014f5e759c0 +322-323 b1f6f0f347738cc4 1dc0221eb01ebffb +324-325 bf7a4b4b0416539f dc04c813b1068a7e +326-327 d796c28167ccec32 57a991d9b2b4e4c9 +328-329 f0862801e60443ff b7c27ba4c474fd55 +330-331 b6933433d1c86e68 26e379dfa25c389a +332-333 1d2af9f3dd6ee079 8252306747348cbe +334-335 6831f0fc480161b8 91a3c134f310b865 +336-337 3c9a1047cad5b6c4 a0f2d2a2cd5af2a2 +338-339 f35b76f576f2f5a7 475f04b96516367c +340-341 6b33c8e65c8cb774 011b51f6ded30ca8 +342-343 f2aaf232939f4c85 ba7942649ca9ff3f +344-345 790dc1fe8dcbc593 3b304b408bdc4a7d +346-347 cf2b9571a4b7d09f 630f7b6f711c1e37 +348-349 72724ac0e0305684 1c5d4e8fd4b15d6d +350-351 4bba6d6cbca85d7e 59f8dd8a8e6ab287 +352-353 a36a1398babe7ffe a0e986ce1ff051fa +354-355 b0994f8c4ad1d0a4 e7990c90e1ef0a1c +356-357 299c2c85a5b0a8ce 9f6a02b68f86fe96 +358-359 18ed513871b03115 bb9e5bd9a0e37027 +360-361 f8510bc8727e1bb3 836f4a6c99bda1d2 +362-363 20fc3ec84dd52452 45775eac414a2011 +364-365 9a7c0096e36e9f4c 4b3ab044e3191938 +366-367 558df82888675581 aa4770bda39fad7e +368-369 14e708a8916fd56f dc817ade8852bd60 +370-371 880d2f4a36058d42 ab529c652aa7a975 +372-373 9f2b3882dea1d199 b1d2ea5d2d2153d4 +374-375 4c79e5f0afccbf66 54e8394aa88c12fd +376-377 be6c1768e803ee67 c35d67b56f34baff +378-379 fc0cf081ae6f09d2 291699621baf5882 +380-381 cee0816aebbf706a e03fa6839983da58 +382-383 bd026994f39f61df 85fd24f953442933 +384-385 5f767d8c7f8cbc5e 8f51b972a1fa795d +386-387 cd50769024183e58 d63fa4cdf5db0e8f +388-389 c6df70f1fe7bd125 3d58dd29e299abaf +390-391 f30e290e498db6a6 e888bb7f935b4c41 +392-393 cfbc92a221f2223f 708148f6010f2fda +394-395 ba581a1ed5a8e829 d856caa76fcbd282 +396-397 7724707c1990c870 e286daead3c36bd5 +398-399 63fad3e3018f55ec 3ec936da5426a265 diff --git a/docs/ecosystems.md b/docs/ecosystems.md index 759d7d182..e5f64f1be 100644 --- a/docs/ecosystems.md +++ b/docs/ecosystems.md @@ -133,7 +133,13 @@ The backticked slug in each row is the value `-e`/`--ecosystems` accepts (e.g. a reliable substitute. Run `socket-patch vex` after installation to verify the patched files. See the [compatibility matrix and workflow](testing/pnpm-compatibility.md). - **yarn classic** — the `yarn.lock` entry's `resolved` / `integrity` are - rewritten to the hosted tarball. A project that sets `yarn-offline-mirror` + rewritten to the hosted tarball. `resolved` always carries the tarball's + `#` fragment: yarn 1 names its cache slot after it, so a fragmentless + URL would share the slot of an unpatched copy of the same version and a warm + cache would install those bytes (or fail the integrity check). When the grant + carries no sha1, the scan downloads the served tarball, checks it against the + grant's sha512 and pins the sha1 of those bytes. If that download or check + fails, the patch is skipped as `npm_tarball_unavailable`. A project that sets `yarn-offline-mirror` is refused with `redirect_yarn_classic_offline_mirror`. The mirror is resolved the way yarn 1 resolves it: the project's `.yarnrc` / `.npmrc`, the user's (`~/.yarnrc`, which `yarn config set` writes, and `~/.npmrc`), From 494932e9e98e0358e99fce2672221446fa905c6d Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Fri, 9 Oct 2026 14:23:09 -0400 Subject: [PATCH 3/5] Match classic lock blocks by name for sha1 fetch The served-tarball fetch picked its yarn classic targets with a raw substring test, so `lodash` matched a `lodash.debounce` block and a package locked only as a git or file: copy (which the rewriter never pins) was fetched too, and a failed fetch dropped its patch. Targets are now read by block real name, version and registry copy source, the way the rewriter selects blocks. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/socket-patch-core/src/hosted/engine.rs | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 38131a2bc..d3927dc2a 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -1129,11 +1129,34 @@ pub fn yarn_classic_sha1_targets<'a>( .map(|c| &c.dep) .filter(|dep| dep.ecosystem == "npm") .filter(|dep| dep.integrity.sha1.is_none() && dep.integrity.sha512.is_some()) - .filter(|dep| lock.contains(crate::patch::redirect::full_name(dep).as_str())) + .filter(|dep| { + classic_locks_registry_copy(lock, &crate::patch::redirect::full_name(dep), &dep.version) + }) .filter(|dep| seen.insert(dep.artifact_url.clone())) .collect() } +/// Whether a classic `lock` has a registry block of `name@version`: the +/// only copy a hosted pin rewrites (a git, `file:`, `link:` or remote +/// tarball copy is skipped by name, so it needs no served tarball). Read +/// by the blocks' real names, as the rewriter does, so `lodash` never +/// matches a `lodash.debounce` block. +fn classic_locks_registry_copy(lock: &str, name: &str, version: &str) -> bool { + use crate::formats::yarn::blocks::{classic_field, scan_blocks}; + use crate::formats::yarn::patterns::{classic_key_real_name, split_key_patterns}; + use crate::formats::yarn::source::{classic_copy_source, CopySource}; + if !lock.contains(name) { + return false; + } + scan_blocks(lock).iter().any(|block| { + let patterns = split_key_patterns(&block.key); + classic_key_real_name(&patterns) == Some(name) + && classic_field(&block.lines, "version") == Some(version) + && classic_copy_source(&patterns, classic_field(&block.lines, "resolved")) + == CopySource::Registry + }) +} + /// Record the sha1 derived from the served tarball at `url` on every /// candidate granted that artifact. pub fn set_derived_sha1(candidates: &mut [Candidate], url: &str, sha1: &str) { @@ -2847,6 +2870,30 @@ mod tests { serde_json::from_value(value).unwrap() } + /// #558 review: the served tarball is fetched only for a lock that + /// really locks a registry copy of the package, read by block names + /// (`lodash` is not `lodash.debounce`) and copy source (a git or + /// `file:` copy is never pinned). + #[test] + fn classic_registry_copy_is_matched_by_block_name_and_source() { + let lock = "# yarn lockfile v1\n\n\ + lodash.debounce@^4.0.8:\n version \"4.17.21\"\n \ + resolved \"https://registry.yarnpkg.com/lodash.debounce/-/x.tgz#aa\"\n\n\ + left-pad@git+https://github.com/x/left-pad.git:\n version \"1.3.0\"\n \ + resolved \"git+https://github.com/x/left-pad.git#abc\"\n\n\ + is-odd@^3.0.0:\n version \"3.0.1\"\n \ + resolved \"https://registry.yarnpkg.com/is-odd/-/is-odd-3.0.1.tgz#bb\"\n"; + assert!(!classic_locks_registry_copy(lock, "lodash", "4.17.21")); + assert!(!classic_locks_registry_copy(lock, "left-pad", "1.3.0")); + assert!(!classic_locks_registry_copy(lock, "is-odd", "3.0.0")); + assert!(classic_locks_registry_copy(lock, "is-odd", "3.0.1")); + assert!(classic_locks_registry_copy( + lock, + "lodash.debounce", + "4.17.21" + )); + } + #[test] fn candidates_skip_every_unusable_reference() { let mut refs: HashMap = HashMap::new(); From d13811daebc084c7eab0e0505d30908028ace8ea Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Fri, 9 Oct 2026 17:34:23 -0400 Subject: [PATCH 4/5] Refuse yarn classic pins with unlocked deps (#1363) A patch that adds a dependency to the package's own package.json, or moves one to a new range, left yarn classic locks that yarn could not install reproducibly. Vendored mode recomputed the block's dependencies sub-map but added no block for the new descriptor, so online frozen installs fetched it unpinned, --offline installs failed and every yarn install re-saved the lock. Hosted mode never looked at the patched manifest, so yarn never installed the new dependency and the patched package crashed at runtime, while scan and vex reported success. Both writers now compare the patched package.json with the lock. Vendored mode refuses the patch before any wiring is written (vendor_dep_manifest_unlocked) when a descriptor has no block of its own. Hosted mode reads the served tarball (with the #558 sha1 fetch) for every entry it has not pinned yet, refuses the pin with redirect_yarn_classic_dep_manifest_unlocked, and rewrites the sub-maps when every descriptor is already locked. Each refusal names the descriptors and a remedy (lock them first, e.g. with yarn add). Fixes #591 Co-authored-by: Claude Opus 5.5 (1M context) --- crates/socket-patch-cli/CLI_CONTRACT.md | 5 +- .../src/commands/scan/hosted.rs | 45 ++-- .../tests/covgap_commands_scan_hosted.rs | 61 +++++- .../tests/e2e_redirect_yarn_classic_build.rs | 29 +++ .../tests/hosted_memory_parity.rs | 56 ++++- .../tests/in_process_redirect.rs | 61 +++++- .../tests/scan/hosted_yarn_classic_sha1.rs | 67 ++++++ .../src/formats/yarn/classic_deps.rs | 181 ++++++++++++++++ .../socket-patch-core/src/formats/yarn/mod.rs | 3 + crates/socket-patch-core/src/hosted/engine.rs | 46 ++-- .../src/hosted/memory/discover.rs | 31 +-- .../src/hosted/memory/mod.rs | 18 +- .../src/hosted/memory/stages.rs | 41 ++-- .../src/hosted/npm_manifest.rs | 62 ++++-- .../src/patch/redirect/mod.rs | 194 ++++++++++++++++- .../src/vendor/npm_common.rs | 35 +++- .../src/vendor/yarn_classic_lock.rs | 196 ++++++++++++------ docs/ecosystems.md | 18 +- 18 files changed, 976 insertions(+), 173 deletions(-) create mode 100644 crates/socket-patch-core/src/formats/yarn/classic_deps.rs diff --git a/crates/socket-patch-cli/CLI_CONTRACT.md b/crates/socket-patch-cli/CLI_CONTRACT.md index 863dbcbca..a1af1c694 100644 --- a/crates/socket-patch-cli/CLI_CONTRACT.md +++ b/crates/socket-patch-cli/CLI_CONTRACT.md @@ -738,7 +738,7 @@ to **six flavors**. | eco / flavor | vendored artifact | committed wiring | consumption proof | |---|---|---|---| | npm (package-lock) | deterministic patched tarball `[@scope/]-.tgz`, plus `/.gitignore` (re-includes the tarball against the project's ignores, such as Node.gitignore's `*.tgz`) and `/.gitattributes` (`-text`); every tarball flavor below writes the same pair and refuses `vendor_artifact_gitignored` when git would still drop the tarball | `package-lock.json` only (`npm-shrinkwrap.json` wins when present): every entry matching name+version gets `resolved: "file:…"` + recomputed `integrity`. `package.json` untouched | `npm ci` (integrity-verified). Plain `npm install` preserves the entry; `npm update ` re-resolves and drops it | -| npm / yarn classic | (same tarball) | `yarn.lock` only: matching blocks get `resolved "file:./…#"` + `integrity` (both checksums recomputed; merged-key & `npm:`-alias blocks covered) | `yarn install --frozen-lockfile --offline` (sha1 fragment + sha512 SRI both enforced; byte-stable lock) | +| npm / yarn classic | (same tarball) | `yarn.lock` only: matching blocks get `resolved "file:./…#"` + `integrity` (both checksums recomputed; merged-key & `npm:`-alias blocks covered); a patch that rewrites the package's `package.json` gets the blocks' `dependencies:` / `optionalDependencies:` sub-maps recomputed, and is refused `vendor_dep_manifest_unlocked` before any write when a dependency it adds or a range it changes to has no lock block of its own (#591) | `yarn install --frozen-lockfile --offline` (sha1 fragment + sha512 SRI both enforced; byte-stable lock) | | npm / yarn berry (node-modules linker) | (same tarball) | root `package.json` `resolutions` + `yarn.lock` entry with `checksum: 10c0/` of the berry cache-zip (reproduced from the tarball offline). **PnP is refused** (`.pnp.*` → different artifact pipeline) | `yarn install --immutable --check-cache`, cold cache. Refused if `__metadata.cacheKey ≠ 10c0` or a non-default `compressionLevel`. Both files keep their own layout — a CRLF lock (yarn's output on Windows) is spliced in CRLF, `package.json` is re-serialized with its BOM, indent, line ending and trailing-newline shape — so vendor + `--revert` round-trip byte-exactly; a lock or `package.json` MIXING CRLF and LF is refused before any write (`vendor_yarn_berry_mixed_line_endings`) | | npm / pnpm (lockfileVersion 9) | (same tarball) | root `package.json` `pnpm.overrides` (versioned selector) **+** `pnpm-lock.yaml` surgery (overrides / importer version / packages `resolution.integrity` / snapshots) **+** the same override in `pnpm-workspace.yaml` (pnpm >= 10.5 reads it there; created with a root-only `packages:` scaffold when absent). A project with no `pnpm-workspace.yaml` pinned to pnpm 9.0–10.4 (every pin, read as for the hosted trust config) gets no file: those read package.json, and a root-only workspace makes `pnpm add` fail there (#734); a later vendor on pnpm >= 10.5 adds it. Residual: an unpinned project with no install record still gets the scaffold, so on pnpm 9.0–10.4 it needs `pnpm add -w ` or a `packageManager` pin | `pnpm install --frozen-lockfile --offline`, cold store (integrity-verified; byte-stable on pnpm 9 & 10). Other lockfileVersions: 5.4/6.0 route to the legacy backend below; anything else refused | | npm / pnpm LEGACY (lockfileVersion 5.4 = pnpm 7, 6.0 = pnpm 8; flavor `pnpm-legacy`) | (same tarball) | root `package.json` `pnpm.overrides` **+** legacy lock surgery (overrides / root dep + specifiers / packages rekey to a bare `file:` key with recomputed integrity / in-package dep refs). **No `pnpm-workspace.yaml` is written** (pnpm ≤ 8 reads overrides only from package.json). The lock's SPECIFIER is machine-ABSOLUTE — pnpm ≤ 8 absolutizes `file:` overrides itself — surfaced as `vendor_pnpm_legacy_absolute_specifier`. Legacy WORKSPACE locks (`importers:`) refused | same-path `pnpm install --frozen-lockfile --offline`, cold store (byte-stable on pnpm 7.33.5 / 8.15.9). A checkout at a DIFFERENT path fails the frozen check (path-bound specifier) and must run `pnpm install --offline --no-frozen-lockfile` once (the flag matters on CI, where pnpm defaults frozen on), which installs the vendored tarball and re-resolves only the specifier line | @@ -1350,6 +1350,9 @@ Every `--json` invocation emits a single JSON object that follows the **unified | `vendor_vlt_reinstall_required` | `skipped` (advisory; human: `Warning: …`) | vendor / scan / get `--mode vendored` (vlt), wet and dry runs, and in-sync reruns: (a) the run rewires an optional dependency, or an importer's `node_modules/` of an optional dependency still resolves into `node_modules/.vlt/`: from vlt 0.0.0-30 a plain `vlt install` (1.2.0: also `--force`) keeps that installed upstream copy linked; the detail says to run `vlt ci` (or delete `node_modules` and run `vlt install`) to link the vendored copy, and that vlt 0.0.0-30 … 1.0.4 install no optional dependency from the lock of a project that declares only optional dependencies (upgrade to 1.0.5 or later first); (b) otherwise, an importer's link of the dependency still resolves into `node_modules/.vlt/`: the detail names the links (`node_modules/`, `/node_modules/`) and says `vlt install` (or `vlt ci`) links the vendored copy — on a warm tree after a plain `vlt install` that is true of every vendored direct dependency; (c) an importer's link resolves into the vendored dir of the patch this run replaces (a new patch uuid), which the run removes: the detail names the links and says `vlt install` (or `vlt ci`) links the new vendored copy; (d) a redownload of the payload (vendor, or `repair` after a corrupt or missing payload) could not keep vlt's links to the package's own dependencies (its old `node_modules/` held more than links): the detail says to run `vlt ci` (or delete `node_modules` and run `vlt install`), since a plain `vlt install` does not re-link them. `repair` moves those links back into the downloaded payload when they are only links. The package is vendored either way; a run whose patch fails to apply emits neither. A wet `vendor --revert` (and the revert a vendored → hosted takeover runs, whose advisory joins `redirect.warnings[]`): (a) the revert moves an `optionalDependencies` spec back from the `file:` dir, or an optional importer's `node_modules/` still resolves into the vendored uuid dir: from vlt 0.0.0-30 a plain `vlt install` keeps that link (dangling once the dir is removed), so the detail says to run `vlt ci` (or delete `node_modules` and run `vlt install`) to link the restored copy, with the same vlt 1.0.5 note; (b) otherwise, an importer's link still resolves into the vendored uuid dir: the detail names the links and says `vlt install` (or `vlt ci`) links the restored copy. A dry-run revert emits neither. | | `vendor_bun_reinstall_required` | `skipped` (advisory; human: `Warning: …`); rollback/remove `warnings[]`; `scan --prune` `gc.warnings[]` (human: `GC: …`) | a wet Bun revert (`vendor --revert`, rollback / remove of a vendored entry, `--preserve-state` included) that restored the lock entry while `node_modules/` is a real directory, or the tree has no `node_modules/.bun/` (a hoisted install): Bun's hoisted linker does not re-extract a package whose lock entry moves from the vendored tarball back to the registry record of the same `name@version`, so a plain `bun install` (also `--frozen-lockfile`) reports no changes and keeps the vendored bytes (measured on 1.1.45 … 1.4.2). The detail names `name@version` and says to run `bun install --force` (or delete `node_modules` and run `bun install`); the human revert hint names `bun install --force` too. An isolated install (a link into `node_modules/.bun/`) relinks and a project without `node_modules/` has nothing installed: neither warns, and neither does a dry run, a drift-kept revert, or a revert that restored nothing (`vendor_lockfile_missing`, or `vendor_lock_entry_removed` after `bun remove`, whose copy a plain `bun install` prunes). | | `vendor_flavor_changed` | `failed` | vendor (npm): the purl's vendor ledger entry was written for another lockfile `flavor` than the one the router now detects (for example `npm` → `vlt` after switching package managers). Remedy: `socket-patch vendor --revert` it first, then re-vendor. Refused before any write. | +| `vendor_dep_manifest_unlocked` | refused | vendor (yarn classic): the patch rewrites the package's own `package.json` to depend on a descriptor (`name@range`) no `yarn.lock` block is keyed by — an added dependency, or an existing one moved to a new range. yarn 1 builds its install graph from the lock, so the rewired block would name a dependency it never resolves: online frozen installs fetch it unpinned, `--offline` installs fail and every plain `yarn install` re-saves the lock (#591). Refused after staging and before any wiring is written (the staged uuid dir is removed); the detail names the descriptors. Remedy: lock them first (for example `yarn add `), then re-run. A dry run, which stages nothing, does not foresee it. | +| `redirect_yarn_classic_dep_manifest_unlocked` | `redirect.warnings[]` (warning) | scan/get `--mode hosted` (yarn classic): the served tarball's own `package.json` depends on a descriptor no `yarn.lock` block is keyed by. yarn 1 installs only what the lock names, so a pin would install the patched package without that dependency (#591). The dep is not pinned and never confirmed; the lock is left as it was. Same remedy as `vendor_dep_manifest_unlocked`. | +| `redirect_yarn_classic_dep_manifest_rewritten` | `redirect.warnings[]` (warning) | scan/get `--mode hosted` (yarn classic): the served tarball's own `package.json` declares other dependencies than the pinned block's sub-maps, every descriptor already locked; the block's `dependencies:` / `optionalDependencies:` sub-maps are rewritten to match (#591). | | `vendor_artifact_gitignored` | `failed` | vendor (vlt and the npm-family tarball flavors: npm, pnpm, bun, yarn classic, yarn berry): inside a git work tree, `git check-ignore --no-index` reports the new artifact's uuid directory as ignored by a rule its own `.gitignore` cannot override (such as a root `.socket/` or `vendor/` rule; the detail names the rule). Remedy: drop that rule for `.socket/vendor/`. Refused before any write. A file rule such as `*.tgz` is overridden by the `/.gitignore` vendoring writes; if the written artifact still reads as ignored, the run refuses and removes the uuid dir it created. | | `vendor_artifact_gitignore_unchecked` | warning | vendor (vlt and the npm-family tarball flavors): git is installed but could not answer the ignore check for the written vendored directory (it failed to start, ran past 30 s, or `rev-parse` / `check-ignore` exited with an error); the package is vendored and the detail names what failed. Remedy: make sure no ignore rule covers `.socket/` before committing. Git absent, or a project outside any work tree, raises nothing. | | `vendor_ledger_entry_missing` | `failed` | vendor (vlt): the only installed copy is vlt's link to a committed vendored directory, but the vendor ledger has no entry for the package; restore `.socket/vendor/state.json` from version control (v5.0: `repair` no longer re-synthesizes it). Replaces the `package_not_installed` skip. | diff --git a/crates/socket-patch-cli/src/commands/scan/hosted.rs b/crates/socket-patch-cli/src/commands/scan/hosted.rs index 189f82573..0cbc91971 100644 --- a/crates/socket-patch-cli/src/commands/scan/hosted.rs +++ b/crates/socket-patch-cli/src/commands/scan/hosted.rs @@ -1045,28 +1045,37 @@ pub(crate) async fn run_redirect_selected( } } } - // A yarn classic pin needs the served tarball's sha1 as its `resolved` - // fragment: yarn 1 keys its cache slot by it (#558). When the grant - // carries only a sha512, the scan downloads the tarball, checks it - // against that sha512 and pins the sha1 of those bytes. A tarball that - // cannot be fetched or verified drops its patch rather than pin a - // fragmentless URL a warm cache serves stale bytes for. - let sha1_targets: Vec<(String, String, DepOverride)> = - engine::yarn_classic_sha1_targets(&candidates, &read.files) - .into_iter() - .filter_map(|dep| { - let sha512 = dep.integrity.sha512.clone()?; - Some((dep.artifact_url.clone(), sha512, dep.clone())) - }) - .collect(); - for (url, sha512, dep) in sha1_targets { + // A yarn classic pin reads the served tarball: its sha1 is the + // `resolved` fragment yarn 1 keys its cache slot on when the grant + // carries none (#558), and its package.json's dependencies must match + // the lock block's sub-maps, every new descriptor locked (#591). The + // tarball is checked against the grant's sha512; one that cannot be + // fetched, verified or read drops its patch. + let classic_targets: Vec<(String, String, DepOverride)> = + engine::yarn_classic_artifact_targets( + &candidates, + &read.files, + &resolve_outer_yarn_mirror_for_process(&common.cwd), + ) + .into_iter() + .filter_map(|dep| { + let sha512 = dep.integrity.sha512.clone()?; + Some((dep.artifact_url.clone(), sha512, dep.clone())) + }) + .collect(); + for (url, sha512, dep) in classic_targets { status.set(format!("Fetching hosted tarball for {}...", dep.name)); - match socket_patch_core::hosted::npm_manifest::fetch_hosted_npm_sha1( + match socket_patch_core::hosted::npm_manifest::fetch_hosted_classic_artifact( api_client, &url, &sha512, ) .await { - Ok(sha1) => engine::set_derived_sha1(&mut candidates, &url, &sha1), + Ok(artifact) => engine::record_classic_artifact( + &mut candidates, + &mut python_metadata, + &url, + &artifact, + ), Err(detail) => { unavailable_python_artifacts.insert(url.clone()); skipped.push(engine::npm_tarball_unavailable(&dep, &detail)); @@ -2066,7 +2075,7 @@ fn describe_skip_reason(reason: &str) -> String { "the hosted tarball's package.json could not be fetched".into() } "npm_tarball_unavailable" => { - "the hosted tarball could not be fetched or did not match its sha512".into() + "the hosted tarball could not be fetched, verified or read".into() } "redirect_bun_lock_unsupported" | "redirect_bun_lockb_invalid" => { "the Bun lockfile blocks the vendored-to-hosted migration (see the warning)".into() diff --git a/crates/socket-patch-cli/tests/covgap_commands_scan_hosted.rs b/crates/socket-patch-cli/tests/covgap_commands_scan_hosted.rs index 385562706..46695f131 100644 --- a/crates/socket-patch-cli/tests/covgap_commands_scan_hosted.rs +++ b/crates/socket-patch-cli/tests/covgap_commands_scan_hosted.rs @@ -2719,6 +2719,59 @@ fn write_yarn_classic_project(root: &Path, package_manager: Option<&str>) { .unwrap(); } +/// A granted reference whose tarball the mock serves (a yarn classic pin +/// reads it, #558 / #591), with the grant's hashes matching it; returns its +/// URL. +async fn mock_served_classic_reference(server: &MockServer) -> String { + use base64::Engine as _; + use sha1::Digest as _; + let manifest = format!(r#"{{"name":"{NAME}","version":"{VERSION}"}}"#); + let mut builder = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + let mut header = tar::Header::new_gnu(); + header.set_size(manifest.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "package/package.json", manifest.as_bytes()) + .unwrap(); + let tgz = builder.into_inner().unwrap().finish().unwrap(); + let artifact = format!("/patch/npm/{NAME}/{VERSION}/22222222-2222-4222-8222-222222222222/{UUID}/{NAME}-{VERSION}.tgz"); + let url = format!("{}{artifact}", server.uri()); + mock_reference_results( + server, + json!({ + UUID: { + "status": "granted", + "url": url, + "purl": PURL, + "artifacts": [{ + "kind": "tarball", + "url": url, + "integrity": { + "sha512": format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD + .encode(sha2::Sha512::digest(&tgz)) + ), + "sha1": hex::encode(sha1::Sha1::digest(&tgz)), + } + }], + "registryOverride": null + } + }), + ) + .await; + Mock::given(method("GET")) + .and(path(artifact)) + .respond_with(ResponseTemplate::new(200).set_body_raw(tgz, "application/octet-stream")) + .mount(server) + .await; + url +} + /// #907: a hosted pin in a classic yarn.lock is dropped by the next yarn 2+ /// (berry) install exactly like vendored wiring, so `scan --mode hosted` /// must warn `redirect_yarn_classic_berry_migration_risk` — on a dry run, on @@ -2729,7 +2782,7 @@ async fn hosted_yarn_classic_pin_warns_berry_migration_risk() { for package_manager in [None, Some("yarn@4.18.1")] { let server = MockServer::start().await; mock_discovery(&server, PURL, UUID).await; - mock_granted_reference(&server, UUID, PURL, HOSTED_URL).await; + let hosted_url = mock_served_classic_reference(&server).await; mock_view(&server, UUID, PURL).await; let tmp = tempfile::tempdir().unwrap(); write_yarn_classic_project(tmp.path(), package_manager); @@ -2755,7 +2808,7 @@ async fn hosted_yarn_classic_pin_warns_berry_migration_risk() { } let lock = std::fs::read_to_string(tmp.path().join("yarn.lock")).unwrap(); assert!( - lock.contains(HOSTED_URL), + lock.contains(&hosted_url), "the warning never blocks the pin: {lock}" ); } @@ -2768,7 +2821,7 @@ async fn hosted_yarn_classic_pin_warns_berry_migration_risk() { async fn hosted_yarn_classic_pin_with_yarn1_package_manager_stays_silent() { let server = MockServer::start().await; mock_discovery(&server, PURL, UUID).await; - mock_granted_reference(&server, UUID, PURL, HOSTED_URL).await; + let hosted_url = mock_served_classic_reference(&server).await; mock_view(&server, UUID, PURL).await; let tmp = tempfile::tempdir().unwrap(); write_yarn_classic_project(tmp.path(), Some("yarn@1.22.22")); @@ -2783,5 +2836,5 @@ async fn hosted_yarn_classic_pin_with_yarn1_package_manager_stays_silent() { "a yarn 1 pin suppresses the advisory: {codes:?}" ); let lock = std::fs::read_to_string(tmp.path().join("yarn.lock")).unwrap(); - assert!(lock.contains(HOSTED_URL), "{lock}"); + assert!(lock.contains(&hosted_url), "{lock}"); } diff --git a/crates/socket-patch-cli/tests/e2e_redirect_yarn_classic_build.rs b/crates/socket-patch-cli/tests/e2e_redirect_yarn_classic_build.rs index 611ad484b..bbcc9e1e0 100644 --- a/crates/socket-patch-cli/tests/e2e_redirect_yarn_classic_build.rs +++ b/crates/socket-patch-cli/tests/e2e_redirect_yarn_classic_build.rs @@ -489,6 +489,25 @@ async fn classic_hosted_project( }))) .mount(&server) .await; + if tamper_served_tarball { + // The scan reads the served tarball once (it checks the dependency + // graph against the lock, #591) and verifies it against the grant, + // so it would refuse tampered bytes up front. Serve the real bytes + // to that read and the tampered ones from then on: the tarball is + // swapped after the pin was written, which only yarn's own check of + // the pinned hashes can catch. + Mock::given(method("GET")) + .and(path(format!( + "/patch/npm/{DEP}/{DEP_VERSION}/{TOKEN}/{UUID}/{DEP}-{DEP_VERSION}.tgz" + ))) + .respond_with( + ResponseTemplate::new(200).set_body_raw(tgz.clone(), "application/octet-stream"), + ) + .up_to_n_times(1) + .with_priority(1) + .mount(&server) + .await; + } Mock::given(method("GET")) .and(path(format!( "/patch/npm/{DEP}/{DEP_VERSION}/{TOKEN}/{UUID}/{DEP}-{DEP_VERSION}.tgz" @@ -1538,6 +1557,16 @@ async fn mock_hosted_grant(tgz: &[u8], orig: &[u8], patched: &[u8], title: &str) }))) .mount(&server) .await; + // The hosted tarball itself: the scan reads it before pinning (#591). + Mock::given(method("GET")) + .and(path(format!( + "/patch/npm/{DEP}/{DEP_VERSION}/{TOKEN}/{UUID}/{DEP}-{DEP_VERSION}.tgz" + ))) + .respond_with( + ResponseTemplate::new(200).set_body_raw(tgz.to_vec(), "application/octet-stream"), + ) + .mount(&server) + .await; server } diff --git a/crates/socket-patch-cli/tests/hosted_memory_parity.rs b/crates/socket-patch-cli/tests/hosted_memory_parity.rs index 09245d1c1..563a5c0de 100644 --- a/crates/socket-patch-cli/tests/hosted_memory_parity.rs +++ b/crates/socket-patch-cli/tests/hosted_memory_parity.rs @@ -24,6 +24,10 @@ struct Case { /// nothing for the formats the engine must rewrite). expect_redirect: bool, dry_run: bool, + /// Serve a real tarball for every npm grant, its integrity rewritten + /// to match: a yarn classic pin reads the served tarball (#558, #591), + /// which the fixtures' placeholder hashes could never verify. + serve_npm_tarballs: bool, } fn case(fixture: &'static str) -> Case { @@ -32,6 +36,47 @@ fn case(fixture: &'static str) -> Case { extra: Vec::new(), expect_redirect: true, dry_run: false, + serve_npm_tarballs: false, + } +} + +/// For each npm patch: a minimal tarball (`package/package.json` naming +/// the package) served at its artifact URL, with the grant's sha512 and +/// sha1 set to that tarball's. +async fn serve_npm_tarballs(server: &MockServer, patches: &mut [common::Patch]) { + use sha1::Digest as _; + use wiremock::matchers::{method, path}; + use wiremock::{Mock, ResponseTemplate}; + for patch in patches.iter_mut() { + let Some(rest) = patch.purl.strip_prefix("pkg:npm/") else { + continue; + }; + let (name, version) = rest.rsplit_once('@').unwrap(); + let manifest = format!(r#"{{"name":"{name}","version":"{version}"}}"#); + let mut builder = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + let mut header = tar::Header::new_gnu(); + header.set_size(manifest.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "package/package.json", manifest.as_bytes()) + .unwrap(); + let tgz = builder.into_inner().unwrap().finish().unwrap(); + let url = patch.reference["url"].as_str().unwrap().to_string(); + let artifact = &mut patch.reference["artifacts"][0]; + artifact["integrity"]["sha512"] = Value::String(format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(&tgz)) + )); + artifact["integrity"]["sha1"] = Value::String(hex::encode(sha1::Sha1::digest(&tgz))); + Mock::given(method("GET")) + .and(path(url.strip_prefix(&server.uri()).unwrap().to_string())) + .respond_with(ResponseTemplate::new(200).set_body_raw(tgz, "application/octet-stream")) + .mount(server) + .await; } } @@ -39,7 +84,10 @@ fn case(fixture: &'static str) -> Case { async fn assert_parity(case: Case) -> Value { let dir = fixtures_root().join("redirect").join(case.fixture); let server = MockServer::start().await; - let patches = patches_from_overrides(&dir.join("overrides.json"), Some(&server.uri())); + let mut patches = patches_from_overrides(&dir.join("overrides.json"), Some(&server.uri())); + if case.serve_npm_tarballs { + serve_npm_tarballs(&server, &mut patches).await; + } mount_api(&server, &patches).await; let mut files = fixture_files(&dir.join("input")); for (rel, bytes) in &case.extra { @@ -146,7 +194,11 @@ async fn parity_pnpm_existing_workspace() { #[tokio::test] async fn parity_yarn_classic() { - assert_parity(case("npm/yarn-classic/basic")).await; + assert_parity(Case { + serve_npm_tarballs: true, + ..case("npm/yarn-classic/basic") + }) + .await; } #[tokio::test] diff --git a/crates/socket-patch-cli/tests/in_process_redirect.rs b/crates/socket-patch-cli/tests/in_process_redirect.rs index 97f69b1b4..9040d9fbf 100644 --- a/crates/socket-patch-cli/tests/in_process_redirect.rs +++ b/crates/socket-patch-cli/tests/in_process_redirect.rs @@ -126,6 +126,61 @@ async fn mock_reference(server: &MockServer) { .await; } +/// A granted reference whose tarball the mock serves (a yarn classic pin +/// reads it, #558 / #591), the grant's hashes matching it: `(url, sha512 +/// SRI, sha1 hex)`. +async fn mock_served_reference(server: &MockServer) -> (String, String, String) { + use base64::Engine as _; + use sha1::Digest as _; + let manifest = format!(r#"{{"name":"{NAME}","version":"{VERSION}"}}"#); + let mut builder = tar::Builder::new(flate2::write::GzEncoder::new( + Vec::new(), + flate2::Compression::default(), + )); + let mut header = tar::Header::new_gnu(); + header.set_size(manifest.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data(&mut header, "package/package.json", manifest.as_bytes()) + .unwrap(); + let tgz = builder.into_inner().unwrap().finish().unwrap(); + let sha512 = format!( + "sha512-{}", + base64::engine::general_purpose::STANDARD.encode(sha2::Sha512::digest(&tgz)) + ); + let sha1 = hex::encode(sha1::Sha1::digest(&tgz)); + let artifact = format!( + "/patch/npm/{NAME}/{VERSION}/22222222-2222-4222-8222-222222222222/{UUID}/{NAME}-{VERSION}.tgz" + ); + let url = format!("{}{artifact}", server.uri()); + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/package"))) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "results": { + UUID: { + "status": "granted", + "url": url, + "purl": PURL, + "artifacts": [{ + "kind": "tarball", + "url": url, + "integrity": { "sha512": sha512, "sha1": sha1 } + }], + "registryOverride": null + } + } + }))) + .mount(server) + .await; + Mock::given(method("GET")) + .and(path(artifact)) + .respond_with(ResponseTemplate::new(200).set_body_raw(tgz, "application/octet-stream")) + .mount(server) + .await; + (url, sha512, sha1) +} + /// The `view/{uuid}` endpoint `run_redirect` calls to build the patch record /// (file hashes + vulnerabilities) the in-run VEX attests from. async fn mock_view(server: &MockServer) { @@ -1118,7 +1173,7 @@ async fn scan_redirect_refuses_a_mixed_line_ending_yarn_berry_manifest() { async fn scan_redirect_rewrites_correct_entry_in_crlf_classic_lock() { let server = MockServer::start().await; mock_discovery(&server).await; - mock_reference(&server).await; + let (hosted_url, sha512, sha1) = mock_served_reference(&server).await; let tmp = tempfile::tempdir().unwrap(); std::fs::write( @@ -1157,8 +1212,8 @@ async fn scan_redirect_rewrites_correct_entry_in_crlf_classic_lock() { "the decoy entry must stay byte-identical: {lock}" ); assert!( - lock.contains(&format!("resolved \"{HOSTED_URL}#{PATCHED_SHA1}\"\r\n")) - && lock.contains(&format!("integrity {PATCHED_SHA512}\r\n")), + lock.contains(&format!("resolved \"{hosted_url}#{sha1}\"\r\n")) + && lock.contains(&format!("integrity {sha512}\r\n")), "the target entry must pin the hosted patch: {lock}" ); assert!( diff --git a/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs b/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs index 71962b4aa..88a55cb31 100644 --- a/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs +++ b/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs @@ -262,3 +262,70 @@ fn assert_skipped_untouched(root: &Path, server: &MockServer, doc: &Value, stder "no fragmentless hosted pin is written" ); } + +/// #591: the served tarball's package.json adds a dependency yarn.lock +/// doesn't lock. Yarn 1 installs only what the lock names, so a pin would +/// install the patched package without it (and `vex` would attest it): +/// the scan refuses the pin with an actionable warning and leaves the +/// lock alone. The grant carries a sha1 here, so only the dependency +/// check needs the tarball. +#[tokio::test] +async fn issue_591_served_dependency_the_lock_does_not_lock_is_refused() { + let server = MockServer::start().await; + let tarball = tgz(&[ + ( + "package/package.json", + br#"{"name":"left-pad","version":"1.3.0","dependencies":{"is-odd":"^3.0.0"}}"#, + ), + ("package/index.js", b"module.exports = require('is-odd');\n"), + ]); + mock_api_with_sha1(&server, &tarball).await; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().join("proj"); + write_classic_project(&root); + + let (code, doc, stderr) = scan(&root, &server.uri()); + assert_eq!(code, 0, "{doc:#}\n{stderr}"); + assert_eq!(doc["redirect"]["redirected"], 0, "{doc:#}"); + let warning = doc["redirect"]["warnings"] + .as_array() + .unwrap() + .iter() + .find(|w| w["code"] == "redirect_yarn_classic_dep_manifest_unlocked") + .unwrap_or_else(|| panic!("refusal warning: {doc:#}")); + let detail = warning["detail"].as_str().unwrap(); + assert!( + detail.contains("is-odd@^3.0.0") && detail.contains("yarn add is-odd@^3.0.0"), + "{detail}" + ); + assert_eq!( + std::fs::read_to_string(root.join("yarn.lock")).unwrap(), + LOCK, + "nothing is pinned with an incomplete dependency graph" + ); +} + +/// [`mock_api`] with a grant that carries the tarball's sha1 too. +async fn mock_api_with_sha1(server: &MockServer, tarball: &[u8]) { + let artifact = format!("/patch/npm/left-pad/1.3.0/{TOKEN}/{UUID}/left-pad-1.3.0.tgz"); + let url = format!("{}{artifact}", server.uri()); + Mock::given(method("POST")) + .and(path(format!("/v0/orgs/{ORG}/patches/package"))) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "results": { + UUID: { + "status": "granted", + "url": url, + "purl": PURL, + "artifacts": [{ + "kind": "tarball", "url": url, + "integrity": { "sha512": sha512_sri(tarball), "sha1": sha1_hex(tarball) } + }], + "registryOverride": null + } + } + }))) + .mount(server) + .await; + mock_api(server, tarball, Some(tarball.to_vec())).await; +} diff --git a/crates/socket-patch-core/src/formats/yarn/classic_deps.rs b/crates/socket-patch-core/src/formats/yarn/classic_deps.rs new file mode 100644 index 000000000..77932443e --- /dev/null +++ b/crates/socket-patch-core/src/formats/yarn/classic_deps.rs @@ -0,0 +1,181 @@ +//! A yarn classic block's dependency sub-maps against a patched package's +//! own `package.json` (#591). +//! +//! Yarn 1 builds its install graph from `yarn.lock`: a block's +//! `dependencies:` / `optionalDependencies:` sub-maps name the descriptors +//! (`name@range`) the package needs, and each descriptor must be the key +//! of a block of its own. A patch that adds a dependency or changes a +//! range therefore needs both the sub-map rewritten AND a block for every +//! new descriptor. Without the block, `yarn install --frozen-lockfile` +//! resolves the descriptor from the registry with no pin (and `--offline` +//! fails); without the sub-map, yarn never installs the dependency. +//! Neither writer can resolve a new descriptor itself, so they rewrite +//! the sub-maps only when every descriptor is already locked, and refuse +//! the patch otherwise. + +use serde_json::Value; + +use super::blocks::{body_field_line, LockBlock}; +use super::patterns::split_key_patterns; + +/// The block fields yarn 1 mirrors from a package's manifest, in the +/// order it writes them. +const DEP_FIELDS: [&str; 2] = ["dependencies", "optionalDependencies"]; + +/// `lines` (a block's lines) with its dependency sub-maps replaced by the +/// ones `pkg` declares, written the way yarn 1 writes them: each map's +/// entries sorted by name, after every other field. +pub(crate) fn with_manifest_dep_maps(lines: &[String], pkg: &Value) -> Vec { + let mut out = Vec::with_capacity(lines.len()); + let mut i = 0; + while i < lines.len() { + if i > 0 && body_field_line(&lines[i]).is_some_and(is_dep_map_header) { + // Drop the stale sub-map (header + 4-space entries). + i += 1; + while i < lines.len() && body_field_line(&lines[i]).is_none() { + i += 1; + } + continue; + } + out.push(lines[i].clone()); + i += 1; + } + for (field, deps) in manifest_dep_maps(pkg) { + out.push(format!(" {field}:")); + for (name, range) in deps { + out.push(format!(" {} \"{range}\"", quote_yarn_key(&name))); + } + } + out +} + +/// Every descriptor (`name@range`) `pkg`'s dependency maps declare that no +/// block of `blocks` is keyed by, sorted and deduplicated: the ones a lock +/// rewritten to the patched manifest would leave unresolved. +pub(crate) fn unlocked_descriptors(blocks: &[LockBlock], pkg: &Value) -> Vec { + let locked: std::collections::BTreeSet = blocks + .iter() + .flat_map(|b| split_key_patterns(&b.key)) + .collect(); + let mut missing: Vec = manifest_dep_maps(pkg) + .into_iter() + .flat_map(|(_, deps)| deps) + .map(|(name, range)| format!("{name}@{range}")) + .filter(|descriptor| !locked.contains(descriptor)) + .collect(); + missing.sort(); + missing.dedup(); + missing +} + +/// Whether the sub-maps `lines` carries already are exactly the ones `pkg` +/// declares (so a writer leaves them alone). +pub(crate) fn dep_maps_match(lines: &[String], pkg: &Value) -> bool { + with_manifest_dep_maps(lines, pkg) == lines +} + +fn is_dep_map_header(rest: &str) -> bool { + DEP_FIELDS.iter().any(|f| rest.strip_suffix(':') == Some(f)) +} + +/// `pkg`'s non-empty dependency maps, each sorted by name. A non-string +/// range is skipped, as yarn skips it. +fn manifest_dep_maps(pkg: &Value) -> Vec<(&'static str, Vec<(String, String)>)> { + DEP_FIELDS + .iter() + .filter_map(|&field| { + let map = pkg.get(field).and_then(Value::as_object)?; + let mut deps: Vec<(String, String)> = map + .iter() + .filter_map(|(k, v)| Some((k.clone(), v.as_str()?.to_string()))) + .collect(); + if deps.is_empty() { + return None; + } + deps.sort_unstable(); + Some((field, deps)) + }) + .collect() +} + +/// A sub-map key the way yarn 1's serializer writes it: quoted when it +/// could not be read back bare. +pub(crate) fn quote_yarn_key(key: &str) -> String { + let needs = key.is_empty() + || key.starts_with("true") + || key.starts_with("false") + || !key.chars().next().is_some_and(|c| c.is_ascii_alphabetic()) + || key + .chars() + .any(|c| matches!(c, ':' | ' ' | '\n' | '\t' | '\\' | '"' | ',' | '[' | ']')); + if needs { + format!("\"{key}\"") + } else { + key.to_string() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::formats::yarn::blocks::scan_blocks; + + const LOCK: &str = "# yarn lockfile v1\n\n\ +is-number@^6.0.0:\n version \"6.0.0\"\n\n\ +\"@scope/opt@^2.0.0\":\n version \"2.0.0\"\n\n\ +is-odd@3.0.1:\n version \"3.0.1\"\n dependencies:\n is-number \"^6.0.0\"\n"; + + fn lines(text: &str) -> Vec { + text.lines().map(str::to_string).collect() + } + + #[test] + fn unlocked_descriptors_names_only_the_unresolved_ones() { + let blocks = scan_blocks(LOCK); + let pkg = serde_json::json!({ + "dependencies": {"is-number": "^7.0.0", "wow": "^1.0.0"}, + "optionalDependencies": {"@scope/opt": "^2.0.0"}, + }); + assert_eq!( + unlocked_descriptors(&blocks, &pkg), + vec!["is-number@^7.0.0", "wow@^1.0.0"] + ); + let unchanged = serde_json::json!({"dependencies": {"is-number": "^6.0.0"}}); + assert!(unlocked_descriptors(&blocks, &unchanged).is_empty()); + assert!(unlocked_descriptors(&blocks, &serde_json::json!({})).is_empty()); + } + + #[test] + fn sub_maps_are_rebuilt_in_yarns_order() { + let block = lines("is-odd@3.0.1:\n version \"3.0.1\"\n dependencies:\n is-number \"^6.0.0\"\n integrity sha512-X=="); + let pkg = serde_json::json!({ + "optionalDependencies": {"@scope/opt": "^2.0.0"}, + "dependencies": {"zz": "1", "is-number": "^6.0.0"}, + }); + assert_eq!( + with_manifest_dep_maps(&block, &pkg), + lines( + "is-odd@3.0.1:\n version \"3.0.1\"\n integrity sha512-X==\n dependencies:\n \ + is-number \"^6.0.0\"\n zz \"1\"\n optionalDependencies:\n \"@scope/opt\" \"^2.0.0\"" + ) + ); + let same = + lines("is-odd@3.0.1:\n version \"3.0.1\"\n dependencies:\n is-number \"^6.0.0\""); + assert!(dep_maps_match( + &same, + &serde_json::json!({"dependencies": {"is-number": "^6.0.0"}}) + )); + assert!(!dep_maps_match( + &same, + &serde_json::json!({"dependencies": {"is-number": "^7.0.0"}}) + )); + } + + #[test] + fn quote_yarn_key_quotes_what_yarn_quotes() { + assert_eq!(quote_yarn_key("left-pad"), "left-pad"); + assert_eq!(quote_yarn_key("@scope/x"), "\"@scope/x\""); + assert_eq!(quote_yarn_key("3d-lib"), "\"3d-lib\""); + assert_eq!(quote_yarn_key("true-lib"), "\"true-lib\""); + } +} diff --git a/crates/socket-patch-core/src/formats/yarn/mod.rs b/crates/socket-patch-core/src/formats/yarn/mod.rs index 5e6dba3fd..56b80af0e 100644 --- a/crates/socket-patch-core/src/formats/yarn/mod.rs +++ b/crates/socket-patch-core/src/formats/yarn/mod.rs @@ -3,6 +3,8 @@ //! //! * which grammar a lock is ([`sniff_grammar`], [`is_berry_lock`]); //! * the block walk and field reads ([`blocks`]); +//! * a classic block's dependency sub-maps against a patched manifest +//! ([`classic_deps`]); //! * key, descriptor and locator patterns ([`patterns`]); //! * where yarn 1 installs a block's copy from ([`source`]); //! * the stanza view the hosted berry writers re-key and re-order @@ -17,6 +19,7 @@ pub(crate) mod berry_entry; pub mod berry_gates; pub(crate) mod blocks; +pub(crate) mod classic_deps; pub(crate) mod patterns; pub(crate) mod source; pub(crate) mod stanzas; diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index d3927dc2a..ebad9538d 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -1108,14 +1108,18 @@ pub fn yarn_berry_manifest_targets<'a>( .collect() } -/// The npm deps whose yarn classic pin needs the sha1 of the served -/// tarball (#558): the grant carries a sha512 but no sha1, and the -/// project's `yarn.lock` is a classic lock that names the package. Yarn 1 -/// keys its cache slot by the `resolved` URL's `#` fragment, so the -/// pin must carry one. One per distinct artifact URL. -pub fn yarn_classic_sha1_targets<'a>( +/// The npm deps whose yarn classic pin reads the served tarball, one per +/// distinct artifact URL: the project's `yarn.lock` is a classic lock that +/// names the package, the grant carries a sha512, and either the grant has +/// no sha1 for the `resolved` fragment yarn 1 keys its cache slot on +/// (#558), or the lock doesn't pin this artifact yet, so the tarball's own +/// dependencies must be checked against the lock (#591). +/// A lock the project's offline mirror refuses outright (`yarn_outer`: the +/// mirror settings outside the project files) needs none. +pub fn yarn_classic_artifact_targets<'a>( candidates: &'a [Candidate], files: &BTreeMap, + yarn_outer: &OuterYarnMirror, ) -> Vec<&'a DepOverride> { let Some(lock) = files .get("yarn.lock") @@ -1123,15 +1127,20 @@ pub fn yarn_classic_sha1_targets<'a>( else { return Vec::new(); }; + if crate::patch::redirect::yarn_classic_hosted_refused(files, yarn_outer) { + return Vec::new(); + } let mut seen = BTreeSet::new(); candidates .iter() .map(|c| &c.dep) - .filter(|dep| dep.ecosystem == "npm") - .filter(|dep| dep.integrity.sha1.is_none() && dep.integrity.sha512.is_some()) + .filter(|dep| dep.ecosystem == "npm" && dep.integrity.sha512.is_some()) .filter(|dep| { classic_locks_registry_copy(lock, &crate::patch::redirect::full_name(dep), &dep.version) }) + .filter(|dep| { + dep.integrity.sha1.is_none() || !lock.contains(&format!("\"{}#", dep.artifact_url)) + }) .filter(|dep| seen.insert(dep.artifact_url.clone())) .collect() } @@ -1157,21 +1166,28 @@ fn classic_locks_registry_copy(lock: &str, name: &str, version: &str) -> bool { }) } -/// Record the sha1 derived from the served tarball at `url` on every -/// candidate granted that artifact. -pub fn set_derived_sha1(candidates: &mut [Candidate], url: &str, sha1: &str) { +/// Record what the served tarball at `url` yielded: its sha1 on every +/// candidate granted that artifact without one, and its manifest (keyed by +/// URL) for the rewriter. +pub fn record_classic_artifact( + candidates: &mut [Candidate], + manifests: &mut BTreeMap, + url: &str, + artifact: &crate::hosted::npm_manifest::HostedClassicArtifact, +) { for candidate in candidates .iter_mut() .filter(|c| c.dep.artifact_url == url && c.dep.integrity.sha1.is_none()) { - candidate.dep.integrity.sha1 = Some(sha1.to_string()); + candidate.dep.integrity.sha1 = Some(artifact.sha1.clone()); } + manifests.insert(url.to_string(), artifact.manifest.clone()); } /// The skip recorded for an npm dep whose served tarball could not be -/// fetched or did not match its grant's sha512, so no sha1 could be -/// derived for its yarn classic pin (the grant token in `detail` is -/// redacted to ``). +/// fetched, did not match its grant's sha512 or had no readable +/// package.json, so its yarn classic pin could not be checked (the grant +/// token in `detail` is redacted to ``). pub fn npm_tarball_unavailable(dep: &DepOverride, detail: &str) -> SkippedPatch { SkippedPatch { purl: format!( diff --git a/crates/socket-patch-core/src/hosted/memory/discover.rs b/crates/socket-patch-core/src/hosted/memory/discover.rs index b9a8cde21..563664ba0 100644 --- a/crates/socket-patch-core/src/hosted/memory/discover.rs +++ b/crates/socket-patch-core/src/hosted/memory/discover.rs @@ -15,6 +15,7 @@ use std::time::Duration; use crate::api::client::{ApiError, ApiFuture, PatchApi}; use crate::api::ranking::cmp_search_results; use crate::api::types::{BatchPackagePatches, PackageVendorResult, PatchResponse, SearchResponse}; +use crate::hosted::npm_manifest::HostedClassicArtifact; use crate::utils::purl::{normalize_purl, strip_purl_qualifiers}; use crate::utils::purl_key::PurlKey; @@ -391,26 +392,28 @@ pub(crate) async fn fetch_npm_manifests( .collect() } -/// Served npm tarballs' sha1 once per distinct `(url, sha512)`, for the -/// yarn classic pin's `#` fragment when the grant carries none -/// (#558): the disk flow's `fetch_hosted_npm_sha1` over the provider. -pub(crate) async fn fetch_npm_sha1s( +/// The served npm tarballs a yarn classic pin reads (sha1 and +/// package.json), once per distinct `(url, sha512)`: the disk flow's +/// `fetch_hosted_classic_artifact` over the provider. +pub(crate) async fn fetch_classic_artifacts( provider: &Provider, wanted: &BTreeSet<(String, String)>, max_bytes: u64, -) -> BTreeMap> { +) -> BTreeMap> { let ordered: Vec<&(String, String)> = wanted.iter().collect(); let futures: Vec> = ordered .iter() - .map(|(url, sha512)| -> BoxFuture<'_, Result> { - Box::pin(async move { - let bytes = provider - .download_artifact(url, max_bytes) - .await - .map_err(|error| format!("cannot fetch the hosted tarball: {error}"))?; - crate::hosted::npm_manifest::decode_hosted_npm_sha1(&bytes, sha512) - }) - }) + .map( + |(url, sha512)| -> BoxFuture<'_, Result> { + Box::pin(async move { + let bytes = provider + .download_artifact(url, max_bytes) + .await + .map_err(|error| format!("cannot fetch the hosted tarball: {error}"))?; + crate::hosted::npm_manifest::decode_hosted_classic_artifact(&bytes, sha512) + }) + }, + ) .collect(); let results = join_bounded(futures, provider.concurrency).await; ordered diff --git a/crates/socket-patch-core/src/hosted/memory/mod.rs b/crates/socket-patch-core/src/hosted/memory/mod.rs index c752ca33c..4c40d8845 100644 --- a/crates/socket-patch-core/src/hosted/memory/mod.rs +++ b/crates/socket-patch-core/src/hosted/memory/mod.rs @@ -1106,14 +1106,19 @@ async fn engine( .await, ); } - let npm_sha1s: BTreeSet<(String, String)> = planned + let npm_classic: BTreeSet<(String, String)> = planned .iter() - .flat_map(|(_, p)| p.npm_sha1s.iter().cloned()) + .flat_map(|(_, p)| p.npm_classic.iter().cloned()) .collect(); - let artifact_sha1s = if npm_sha1s.is_empty() { + let artifact_classic = if npm_classic.is_empty() { BTreeMap::new() } else { - discover::fetch_npm_sha1s(&provider, &npm_sha1s, options.limits.max_artifact_bytes).await + discover::fetch_classic_artifacts( + &provider, + &npm_classic, + options.limits.max_artifact_bytes, + ) + .await }; phases.mark("plan"); @@ -1130,7 +1135,7 @@ async fn engine( if stage.capped() { first_plans.insert(index, plan.clone()); } - match stages::rewrite(plan, &artifact_metadata, &artifact_sha1s, stage_options).await { + match stages::rewrite(plan, &artifact_metadata, &artifact_classic, stage_options).await { Ok(done) => rewritten.push((index, done)), Err(RewriteRefused { refusal, skipped }) => { states[index].skipped = skipped; @@ -1217,7 +1222,8 @@ async fn engine( .retain(|c| !root_deferred.contains(&c.dep.patch_uuid)); plan.skipped .extend(states[index].deferred.iter().map(deferred_skip)); - match stages::rewrite(plan, &artifact_metadata, &artifact_sha1s, stage_options).await { + match stages::rewrite(plan, &artifact_metadata, &artifact_classic, stage_options).await + { Ok(done) => again.push((index, done)), Err(RewriteRefused { refusal, skipped }) => { states[index].skipped = skipped; diff --git a/crates/socket-patch-core/src/hosted/memory/stages.rs b/crates/socket-patch-core/src/hosted/memory/stages.rs index f1d611c47..f218262b0 100644 --- a/crates/socket-patch-core/src/hosted/memory/stages.rs +++ b/crates/socket-patch-core/src/hosted/memory/stages.rs @@ -12,6 +12,7 @@ use crate::api::types::PackageVendorResult; use crate::hosted::engine::{ self, Candidate, CandidateFiles, Refusal, RewriteOptions, SkippedPatch, }; +use crate::hosted::npm_manifest::HostedClassicArtifact; use crate::hosted::vlt::Preflight; use crate::patch::redirect::npmrc::OuterAllowRemote; use crate::patch::redirect::yarnrc::OuterYarnMirror; @@ -157,9 +158,9 @@ pub(crate) struct Planned { /// `(artifact url, sha512)` of every npm tarball whose own /// package.json a yarn berry pin needs (#718). pub(crate) npm_manifests: Vec<(String, Option)>, - /// `(artifact url, sha512)` of every npm tarball whose sha1 a yarn - /// classic pin needs because its grant carries none (#558). - pub(crate) npm_sha1s: Vec<(String, String)>, + /// `(artifact url, sha512)` of every npm tarball a yarn classic pin + /// reads (its sha1, #558, and its package.json, #591). + pub(crate) npm_classic: Vec<(String, String)>, /// The vlt artifact preflight, judged offline (no network here, so /// every in-scope dep is withheld instead of pinned: `--offline` /// parity). @@ -213,10 +214,14 @@ pub(crate) async fn plan( .into_iter() .map(|dep| (dep.artifact_url.clone(), dep.integrity.sha512.clone())) .collect(); - let npm_sha1s = engine::yarn_classic_sha1_targets(&candidates, &read.files) - .into_iter() - .filter_map(|dep| Some((dep.artifact_url.clone(), dep.integrity.sha512.clone()?))) - .collect(); + let npm_classic = engine::yarn_classic_artifact_targets( + &candidates, + &read.files, + &OuterYarnMirror::default(), + ) + .into_iter() + .filter_map(|dep| Some((dep.artifact_url.clone(), dep.integrity.sha512.clone()?))) + .collect(); Ok(Planned { project, candidates, @@ -225,7 +230,7 @@ pub(crate) async fn plan( read, wheels, npm_manifests, - npm_sha1s, + npm_classic, vlt_preflight, }) } @@ -249,12 +254,13 @@ pub(crate) struct RewriteRefused { pub(crate) skipped: Vec, } -/// Wheel metadata, served npm manifests and served npm tarballs' sha1s -/// (each keyed by artifact URL) → the engine's rewrite → the guard. +/// Wheel metadata, served npm manifests and the served npm tarballs a +/// yarn classic pin reads (each keyed by artifact URL) → the engine's +/// rewrite → the guard. pub(crate) async fn rewrite( planned: Planned, artifact_metadata: &BTreeMap, String>>, - artifact_sha1s: &BTreeMap>, + artifact_classic: &BTreeMap>, options: StageOptions, ) -> Result { let Planned { @@ -265,7 +271,7 @@ pub(crate) async fn rewrite( read, wheels, npm_manifests, - npm_sha1s, + npm_classic, vlt_preflight, } = planned; let skipped_before = skipped.clone(); @@ -314,9 +320,14 @@ pub(crate) async fn rewrite( } } } - for (url, _) in &npm_sha1s { - match artifact_sha1s.get(url) { - Some(Ok(sha1)) => engine::set_derived_sha1(&mut candidates, url, sha1), + for (url, _) in &npm_classic { + match artifact_classic.get(url) { + Some(Ok(artifact)) => engine::record_classic_artifact( + &mut candidates, + &mut python_metadata, + url, + artifact, + ), Some(Err(detail)) => { if unavailable.insert(url.clone()) { for dep in candidates diff --git a/crates/socket-patch-core/src/hosted/npm_manifest.rs b/crates/socket-patch-core/src/hosted/npm_manifest.rs index 532784c41..8418ba266 100644 --- a/crates/socket-patch-core/src/hosted/npm_manifest.rs +++ b/crates/socket-patch-core/src/hosted/npm_manifest.rs @@ -43,30 +43,48 @@ pub async fn fetch_hosted_npm_manifest( decode_hosted_npm_manifest(&bytes, sha512) } -/// The sha1 (hex) of a served npm tarball, for the yarn classic hosted -/// pin's `resolved "#"` fragment when the grant carries no sha1 -/// (#558). Yarn 1 names its cache slot after that fragment, so a -/// fragmentless URL shares the slot of any fragmentless upstream copy of the -/// same version and installs its bytes. The bytes must match the grant's -/// sha512: that is what the pin's `integrity` line names, and a sha1 taken -/// from any other bytes would pin a tarball yarn then refuses. -pub fn decode_hosted_npm_sha1(bytes: &[u8], sha512: &str) -> Result { +/// What a yarn classic hosted pin reads from the served npm tarball. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostedClassicArtifact { + /// The tarball's sha1 (hex), for the pin's `resolved "#"` + /// fragment when the grant carries none (#558). Yarn 1 names its cache + /// slot after that fragment, so a fragmentless URL shares the slot of + /// any fragmentless upstream copy of the same version. + pub sha1: String, + /// The tarball's own `package.json` text: yarn 1 installs the + /// dependencies the lock block's sub-maps name, so a patch that + /// changes them needs the block rewritten, and every new descriptor + /// locked (#591). + pub manifest: String, +} + +/// [`HostedClassicArtifact`] from the served bytes, which must match the +/// grant's sha512: that is what the pin's `integrity` line names, and a +/// sha1 taken from any other bytes would pin a tarball yarn then refuses. +pub fn decode_hosted_classic_artifact( + bytes: &[u8], + sha512: &str, +) -> Result { crate::vendor::registry_fetch::verify_sri(bytes, sha512) .map_err(|_| "hosted tarball does not match its published sha512".to_string())?; - Ok(crate::utils::digest::sha1_hex_of(bytes)) + Ok(HostedClassicArtifact { + sha1: crate::utils::digest::sha1_hex_of(bytes), + manifest: decode_hosted_npm_manifest(bytes, None)?, + }) } -/// Download the served tarball and take its sha1 ([`decode_hosted_npm_sha1`]). -pub async fn fetch_hosted_npm_sha1( +/// Download the served tarball and decode it +/// ([`decode_hosted_classic_artifact`]). +pub async fn fetch_hosted_classic_artifact( client: &ApiClient, url: &str, sha512: &str, -) -> Result { +) -> Result { let bytes = client .download_artifact(url) .await .map_err(|error| format!("cannot fetch the hosted tarball: {error}"))?; - decode_hosted_npm_sha1(&bytes, sha512) + decode_hosted_classic_artifact(&bytes, sha512) } #[cfg(test)] @@ -90,15 +108,19 @@ mod tests { } #[test] - fn sha1_is_taken_from_bytes_matching_the_sha512() { - let bytes = tgz(&[("package/package.json", br#"{"name":"left-pad"}"#)]); + fn classic_artifact_is_read_from_bytes_matching_the_sha512() { + let manifest = br#"{"name":"left-pad","dependencies":{"is-odd":"^3.0.0"}}"#; + let bytes = tgz(&[("package/package.json", manifest)]); let sri = sha512_sri_of(&bytes); - assert_eq!( - decode_hosted_npm_sha1(&bytes, &sri).unwrap(), - crate::utils::digest::sha1_hex_of(&bytes) - ); + let artifact = decode_hosted_classic_artifact(&bytes, &sri).unwrap(); + assert_eq!(artifact.sha1, crate::utils::digest::sha1_hex_of(&bytes)); + assert_eq!(artifact.manifest.as_bytes(), manifest); let other = sha512_sri_of(b"other bytes"); - assert!(decode_hosted_npm_sha1(&bytes, &other).is_err()); + assert!(decode_hosted_classic_artifact(&bytes, &other).is_err()); + let no_manifest = tgz(&[("package/index.js", b"1")]); + assert!( + decode_hosted_classic_artifact(&no_manifest, &sha512_sri_of(&no_manifest)).is_err() + ); } #[test] diff --git a/crates/socket-patch-core/src/patch/redirect/mod.rs b/crates/socket-patch-core/src/patch/redirect/mod.rs index 635426d1d..a0ec698c7 100644 --- a/crates/socket-patch-core/src/patch/redirect/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/mod.rs @@ -315,9 +315,10 @@ pub struct RewriteResult { serde(skip_serializing_if = "std::collections::BTreeSet::is_empty") )] pub confirmed_yarn_berry_uuids: std::collections::BTreeSet, - /// Patch uuids the yarn classic rewriter refused because the project + /// Patch uuids the yarn classic rewriter refused: the project /// configures a `yarn-offline-mirror` (see - /// [`preflight_yarn_classic_hosted`]). Never confirmed. + /// [`preflight_yarn_classic_hosted`]), or the served tarball depends on + /// a descriptor `yarn.lock` doesn't lock (#591). Never confirmed. #[cfg_attr( test, serde(skip_serializing_if = "std::collections::BTreeSet::is_empty") @@ -730,7 +731,7 @@ fn rewriter_groups<'a>( Box::new(move |result| { rewrite_npm_lock(files, overrides, result); plan_hosted(files, overrides, result); - rewrite_yarn_classic_with(files, overrides, yarn_outer, result); + rewrite_yarn_classic_with(files, overrides, artifact_metadata, yarn_outer, result); rewrite_yarn_berry_with_manifests(files, overrides, artifact_metadata, result); rewrite_bun_lock(files, overrides, result); }), @@ -3569,6 +3570,24 @@ pub fn yarn_classic_offline_mirror( /// mirror, so yarn installs the upstream bytes and fails the patched /// integrity (or, `--offline`, never fetches the patched tarball at all). /// `Ok` for a lock that is not classic (the berry rewriter owns those). +/// Whether the project's yarn offline mirror refuses every hosted yarn +/// classic pin of `files`' `yarn.lock` ([`preflight_yarn_classic_hosted`]), +/// so the hosted flows need not read any served tarball for one. +pub fn yarn_classic_hosted_refused( + files: &BTreeMap, + outer: &yarnrc::OuterYarnMirror, +) -> bool { + files.get("yarn.lock").is_some_and(|lock| { + preflight_yarn_classic_hosted( + lock, + files.get(YARNRC_REL).map(String::as_str), + files.get(npmrc::NPMRC_REL).map(String::as_str), + outer, + ) + .is_err() + }) +} + fn preflight_yarn_classic_hosted( lock: &str, yarnrc: Option<&str>, @@ -3677,14 +3696,19 @@ fn rewrite_yarn_classic( rewrite_yarn_classic_with( files, overrides, + &BTreeMap::new(), &yarnrc::OuterYarnMirror::default(), result, ) } +/// `manifests` holds the served tarballs' own package.json texts, keyed by +/// artifact URL (the hosted flows fetch the ones a classic pin reads; a +/// dep with none keeps its lock block's dependency sub-maps as they are). fn rewrite_yarn_classic_with( files: &BTreeMap, overrides: &[DepOverride], + manifests: &BTreeMap, yarn_outer: &yarnrc::OuterYarnMirror, result: &mut RewriteResult, ) { @@ -3779,6 +3803,38 @@ fn rewrite_yarn_classic_with( }); continue; }; + // The served tarball's own package.json (#591): yarn 1 installs the + // dependencies a block's sub-maps name, each through a block of its + // own. A patch that adds a dependency or changes a range needs the + // sub-maps rewritten and every new descriptor locked; no hosted + // rewrite can resolve one, so such a dep is refused, never pinned + // with a graph yarn would install without it. + let served_manifest: Option = manifests + .get(&dep.artifact_url) + .and_then(|text| serde_json::from_str::(text).ok()) + .filter(Value::is_object); + if let Some(pkg) = &served_manifest { + let missing = crate::formats::yarn::classic_deps::unlocked_descriptors(&blocks, pkg); + if !missing.is_empty() { + result + .refused_yarn_classic_uuids + .insert(dep.patch_uuid.clone()); + result.warnings.push(RewriteWarning { + code: "redirect_yarn_classic_dep_manifest_unlocked".into(), + detail: format!( + "the patched {fname}@{} depends on {}, which yarn.lock does not lock; \ + yarn 1 installs only what the lock names, so it is not pinned and \ + stays unpatched. Lock them first (for example `yarn add {}`), then \ + re-run", + dep.version, + missing.join(", "), + missing.join(" ") + ), + }); + continue; + } + } + let mut manifest_rewritten = false; let mut matched_any = false; let mut pinned_any = false; let mut alias_skipped = false; @@ -3979,11 +4035,18 @@ fn rewrite_yarn_classic_with( continue; } pinned_any = true; - let pinned = repin_classic_block( + let mut pinned = repin_classic_block( &block.lines, &format!("{}#{sha1}", dep.artifact_url), &sha512, ); + if let Some(pkg) = &served_manifest { + if !crate::formats::yarn::classic_deps::dep_maps_match(&pinned, pkg) { + pinned = + crate::formats::yarn::classic_deps::with_manifest_dep_maps(&pinned, pkg); + manifest_rewritten = true; + } + } if pinned != block.lines { // Edits record the block's on-disk bytes (CRLF lines for a // CRLF block), so they match what the file really held. @@ -4003,6 +4066,17 @@ fn rewrite_yarn_classic_with( changed = true; } } + if manifest_rewritten { + result.warnings.push(RewriteWarning { + code: "redirect_yarn_classic_dep_manifest_rewritten".into(), + detail: format!( + "the patched {fname}@{} declares different dependencies; its yarn.lock \ + dependency sub-maps were rewritten to match (every descriptor is already \ + locked)", + dep.version + ), + }); + } if !matched_any && !alias_skipped && !copy_skipped { result.warnings.push(RewriteWarning { code: "redirect_yarn_classic_entry_not_found".into(), @@ -11146,7 +11220,13 @@ mod tests { let mut files = BTreeMap::new(); files.insert("yarn.lock".to_string(), classic_lock_two_entries()); let mut r = RewriteResult::default(); - rewrite_yarn_classic_with(&files, std::slice::from_ref(&ovr), outer, &mut r); + rewrite_yarn_classic_with( + &files, + std::slice::from_ref(&ovr), + &BTreeMap::new(), + outer, + &mut r, + ); assert!( r.files.is_empty() && r.edits.is_empty(), "{outer:?}: {:?}", @@ -11177,7 +11257,13 @@ mod tests { "yarn-offline-mirror false\n".to_string(), ); let mut r = RewriteResult::default(); - rewrite_yarn_classic_with(&files, std::slice::from_ref(&ovr), &refusing[1], &mut r); + rewrite_yarn_classic_with( + &files, + std::slice::from_ref(&ovr), + &BTreeMap::new(), + &refusing[1], + &mut r, + ); assert!(r.warnings.is_empty(), "{:?}", r.warnings); assert!(r.files["yarn.lock"].contains("left-pad-1.3.0.tgz")); // The full chain drives it too. @@ -20881,6 +20967,102 @@ packages: ); } + /// #591: the hosted classic rewriter reads the served tarball's own + /// package.json. A dependency it adds (or a range it changes to) that + /// no block locks would be dropped by yarn 1, which installs only what + /// the lock names: the dep is refused and nothing is written. When + /// every descriptor is locked, the block's sub-maps are rewritten to + /// the served manifest's. + #[test] + fn issue_591_hosted_classic_checks_the_served_manifest_against_the_lock() { + let url = "http://p.test/is-odd-3.0.1.tgz"; + let ovr = npm_override("is-odd", "3.0.1", url, "sha512-P=="); + let lock = "# yarn lockfile v1\n\n\n\ + is-number@^6.0.0:\n version \"6.0.0\"\n \ + resolved \"https://registry.yarnpkg.com/is-number/-/is-number-6.0.0.tgz#aaaa\"\n\n\ + is-odd@3.0.1:\n version \"3.0.1\"\n \ + resolved \"https://registry.yarnpkg.com/is-odd/-/is-odd-3.0.1.tgz#bbbb\"\n \ + integrity sha512-UP==\n dependencies:\n is-number \"^6.0.0\"\n"; + let run = |lock: &str, manifest: &str| { + let files = BTreeMap::from([("yarn.lock".to_string(), lock.to_string())]); + let manifests = BTreeMap::from([(url.to_string(), manifest.to_string())]); + let mut r = RewriteResult::default(); + rewrite_yarn_classic_with( + &files, + std::slice::from_ref(&ovr), + &manifests, + &yarnrc::OuterYarnMirror::default(), + &mut r, + ); + r + }; + + // Added dependency, and a changed range: refused, nothing written. + for (manifest, missing) in [ + ( + r#"{"name":"is-odd","dependencies":{"is-number":"^6.0.0","wow":"^1.0.0"}}"#, + "wow@^1.0.0", + ), + ( + r#"{"name":"is-odd","dependencies":{"is-number":"^7.0.0"}}"#, + "is-number@^7.0.0", + ), + ] { + let r = run(lock, manifest); + assert!( + r.files.is_empty() && r.edits.is_empty(), + "{manifest}: {:?}", + r.files + ); + assert_eq!( + warning_codes(&r), + vec!["redirect_yarn_classic_dep_manifest_unlocked"], + "{manifest}" + ); + assert!(r.warnings[0].detail.contains(missing), "{:?}", r.warnings); + assert!(r.refused_yarn_classic_uuids.contains(&ovr.patch_uuid)); + } + + // The served manifest declares what the lock already says: a plain + // pin, sub-map untouched. + let r = run( + lock, + r#"{"name":"is-odd","dependencies":{"is-number":"^6.0.0"}}"#, + ); + assert!(r.warnings.is_empty(), "{:?}", r.warnings); + assert!( + r.files["yarn.lock"].contains(&format!( + " resolved \"{url}#{NPM_SHA1}\"\n integrity sha512-P==\n \ + dependencies:\n is-number \"^6.0.0\"\n" + )), + "{:?}", + r.files + ); + + // The new range is locked: the sub-map follows the served manifest. + let locked = format!( + "{lock}\nis-number@^7.0.0:\n version \"7.0.0\"\n \ + resolved \"https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#cccc\"\n" + ); + let r = run( + &locked, + r#"{"name":"is-odd","dependencies":{"is-number":"^7.0.0"}}"#, + ); + assert_eq!( + warning_codes(&r), + vec!["redirect_yarn_classic_dep_manifest_rewritten"] + ); + assert!( + r.files["yarn.lock"].contains(&format!( + " resolved \"{url}#{NPM_SHA1}\"\n integrity sha512-P==\n \ + dependencies:\n is-number \"^7.0.0\"\n" + )), + "{:?}", + r.files + ); + assert!(r.refused_yarn_classic_uuids.is_empty()); + } + /// #558: a yarn classic pin without a sha1 would have no `#` /// fragment, so yarn 1 would file the hosted tarball in the cache slot /// of a fragmentless upstream copy and install its bytes. The rewriter diff --git a/crates/socket-patch-core/src/vendor/npm_common.rs b/crates/socket-patch-core/src/vendor/npm_common.rs index 0a82bf702..c1d03a1ff 100644 --- a/crates/socket-patch-core/src/vendor/npm_common.rs +++ b/crates/socket-patch-core/src/vendor/npm_common.rs @@ -726,12 +726,18 @@ pub(super) async fn done_failure_unstage( uuid_dir_rel: &str, uuid_dir_preexisted: bool, ) -> VendorOutcome { + unstage(project_root, uuid_dir_rel, uuid_dir_preexisted).await; + done_failure(purl, error) +} + +/// Remove the uuid dir a run staged, unless it existed before the run (a +/// same-uuid re-vendor's dir may still be referenced by live wiring). +async fn unstage(project_root: &Path, uuid_dir_rel: &str, uuid_dir_preexisted: bool) { if !uuid_dir_preexisted { let uuid_dir = project_root.join(uuid_dir_rel); let _ = remove_tree(&uuid_dir).await; super::common::prune_empty_vendor_levels(&uuid_dir).await; } - done_failure(purl, error) } /// The vendor ledger tail every npm flavor shares once its wiring is on @@ -831,6 +837,19 @@ pub(super) trait NpmLockBackend { warnings: &mut Vec, ) -> Result, String>; + /// A refusal for a patch whose rewritten `package.json` (`staged_pkg`) + /// the flavor's lock can't follow, checked after staging and before any + /// wiring is written (the driver unstages the uuid dir). `None`: wire + /// as usual. + fn manifest_refusal( + &self, + _plan: &Self::Plan, + _staged_pkg: &Value, + _coords: &NpmCoords, + ) -> Option { + None + } + /// The advisory for a patch that rewrites the package's own /// `package.json`, whose mirrors the flavor's lock either recomputes or /// keeps. @@ -904,6 +923,20 @@ pub(super) async fn vendor_npm_family( (coords.name.as_str(), coords.version.as_str()) ); + if let Some(refusal) = staged + .staged_pkg_json + .as_ref() + .and_then(|pkg| backend.manifest_refusal(&plan, pkg, &coords)) + { + unstage( + project_root, + &coords.uuid_dir_rel, + staged.uuid_dir_preexisted, + ) + .await; + return refusal; + } + let cx = WireCx { project_root, coords: &coords, diff --git a/crates/socket-patch-core/src/vendor/yarn_classic_lock.rs b/crates/socket-patch-core/src/vendor/yarn_classic_lock.rs index 12f28accc..203110826 100644 --- a/crates/socket-patch-core/src/vendor/yarn_classic_lock.rs +++ b/crates/socket-patch-core/src/vendor/yarn_classic_lock.rs @@ -28,9 +28,9 @@ use serde_json::Value; use crate::constants::SOCKET_DIR; use crate::formats::yarn::blocks::{ - block_eol, body_field_line, classic_field, repin_classic_block, replace_block, scan_blocks, - LockBlock, + block_eol, classic_field, repin_classic_block, replace_block, scan_blocks, LockBlock, }; +use crate::formats::yarn::classic_deps::{unlocked_descriptors, with_manifest_dep_maps}; use crate::formats::yarn::patterns::{classic_key_real_name, split_key_patterns}; use crate::formats::yarn::source::{classic_copy_source, CopySource}; use crate::manifest::schema::PatchRecord; @@ -222,6 +222,29 @@ impl NpmLockBackend for YarnClassicBackend { })) } + /// #591: yarn 1 installs a package's dependencies from the lock, so a + /// patch whose `package.json` adds a dependency or changes a range + /// needs a lock block for every new descriptor. Vendoring can't + /// resolve one (the sub-map rewrite alone leaves it dangling: online + /// frozen installs fetch it unpinned, offline ones fail, and every + /// plain `yarn install` re-saves the lock), so it refuses instead. + fn manifest_refusal( + &self, + plan: &YarnClassicPlan, + staged_pkg: &Value, + coords: &NpmCoords, + ) -> Option { + let blocks = scan_blocks_shared(&plan.text); + let missing = unlocked_descriptors(&blocks, staged_pkg); + if missing.is_empty() { + return None; + } + Some(refused( + "vendor_dep_manifest_unlocked", + unlocked_detail(&coords.name, &coords.version, &missing), + )) + } + fn manifest_warning(&self, name: &str, version: &str) -> VendorWarning { VendorWarning::new( "vendor_dep_manifest_rewritten", @@ -234,6 +257,19 @@ impl NpmLockBackend for YarnClassicBackend { } } +/// The refusal detail for a patch whose `package.json` declares +/// dependencies `yarn.lock` doesn't lock (#591). +fn unlocked_detail(name: &str, version: &str, missing: &[String]) -> String { + format!( + "the patch rewrites {name}@{version}'s package.json to depend on {}, which \ + {YARN_LOCK} does not lock; yarn 1 would install them unpinned (and fail \ + `--offline` installs), so {name}@{version} was not vendored. Lock them first \ + (for example `yarn add {}`), then re-run", + missing.join(", "), + missing.join(" ") + ) +} + /// [`vendor_yarn_classic`]'s defensive re-sniff: the flavor router already /// separates classic from berry, but rewriting a berry lock with classic /// grammar would corrupt it — never proceed past a `__metadata:` key. @@ -883,44 +919,10 @@ fn rewrite_classic_block( staged_pkg: Option<&Value>, ) -> Vec { let pinned = repin_classic_block(lines, resolved_value, integrity_value); - let Some(pkg) = staged_pkg else { - return pinned; - }; - let mut out = Vec::with_capacity(pinned.len()); - let mut i = 0; - while i < pinned.len() { - if i > 0 - && body_field_line(&pinned[i]) - .is_some_and(|r| r == "dependencies:" || r == "optionalDependencies:") - { - // Drop the stale sub-map (header + 4-space entries); the - // recomputed ones are appended below in yarn's order. - i += 1; - while i < pinned.len() && body_field_line(&pinned[i]).is_none() { - i += 1; - } - continue; - } - out.push(pinned[i].clone()); - i += 1; + match staged_pkg { + Some(pkg) => with_manifest_dep_maps(&pinned, pkg), + None => pinned, } - for field in ["dependencies", "optionalDependencies"] { - let Some(map) = pkg.get(field).and_then(Value::as_object) else { - continue; - }; - if map.is_empty() { - continue; - } - out.push(format!(" {field}:")); - let mut keys: Vec<&String> = map.keys().collect(); - keys.sort_unstable(); - for k in keys { - if let Some(range) = map.get(k).and_then(Value::as_str) { - out.push(format!(" {} \"{range}\"", quote_yarn_key(k))); - } - } - } - out } /// Does this block's `resolved` already point into `.socket/vendor/npm/` @@ -978,23 +980,6 @@ pub(super) fn forget_block_scans() { BLOCK_MEMO.invalidate(); } -/// yarn v1's lockfile key quoting (stringify.js `shouldWrapKey`): wrap when -/// the key would not parse bare. -fn quote_yarn_key(key: &str) -> String { - let needs = key.is_empty() - || key.starts_with("true") - || key.starts_with("false") - || !key.chars().next().is_some_and(|c| c.is_ascii_alphabetic()) - || key - .chars() - .any(|c| matches!(c, ':' | ' ' | '\n' | '\t' | '\\' | '"' | ',' | '[' | ']')); - if needs { - format!("\"{key}\"") - } else { - key.to_string() - } -} - pub(super) fn lines_to_json(lines: &[String]) -> Value { Value::Array(lines.iter().map(|l| Value::String(l.clone())).collect()) } @@ -1429,10 +1414,17 @@ left-pad@^1.3.0: integrity sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA== dependencies: old-dep "^1.0.0" + +wow@^1.0.0: + version "1.0.0" + +"@scope/opt@^2.0.0": + version "2.0.0" "#; let mut fx = fixture_with_lock(lock).await; - // The patch rewrites package.json: new dependency + an optional one. + // The patch rewrites package.json: new dependency + an optional one, + // both already locked (#591: an unlocked one is refused). let before: &[u8] = br#"{"name":"left-pad","version":"1.3.0"}"#; let after: &[u8] = br#"{"name":"left-pad","version":"1.3.0","dependencies":{"wow":"^1.0.0"},"optionalDependencies":{"@scope/opt":"^2.0.0"}}"#; let after_hash = compute_git_sha256_from_bytes(after); @@ -1465,12 +1457,91 @@ left-pad@^1.3.0: ); } + /// A lock block for the `wow@^1.0.0` descriptor the manifest-rewriting + /// fixtures' patches add (#591: vendoring needs it locked). + const WOW_BLOCK: &str = "\nwow@^1.0.0:\n version \"1.0.0\"\n"; + + /// Stage a patch on `fx` that rewrites left-pad's package.json to + /// `after`. + async fn patch_manifest(fx: &mut Fixture, after: &[u8]) { + let before: &[u8] = br#"{"name":"left-pad","version":"1.3.0"}"#; + let after_hash = compute_git_sha256_from_bytes(after); + tokio::fs::write(fx.root().join(".socket/blobs").join(&after_hash), after) + .await + .unwrap(); + fx.record.files.insert( + "package/package.json".to_string(), + PatchFileInfo { + before_hash: compute_git_sha256_from_bytes(before), + after_hash, + }, + ); + } + + /// #591: a patch whose package.json adds a dependency yarn.lock doesn't + /// lock is refused before any wiring: rewriting the sub-map alone would + /// leave a dangling descriptor yarn 1 installs unpinned (and an + /// `--offline` install fails). The lock and the vendor dir are left as + /// they were. + #[tokio::test] + async fn issue_591_added_dependency_without_a_lock_block_is_refused() { + let mut fx = fixture_with_lock(Y2_BEFORE).await; + patch_manifest( + &mut fx, + br#"{"name":"left-pad","version":"1.3.0","dependencies":{"is-odd":"^3.0.0"}}"#, + ) + .await; + let detail = expect_refused(fx.vendor(false).await, "vendor_dep_manifest_unlocked"); + assert!( + detail.contains("is-odd@^3.0.0") && detail.contains("yarn add is-odd@^3.0.0"), + "{detail}" + ); + assert_eq!(fx.lock_text().await, Y2_BEFORE, "lock untouched"); + assert!( + !fx.root().join(".socket/vendor/npm").join(UUID).exists(), + "the staged tarball is unstaged" + ); + } + + /// #591 (range change): the patch moves an existing dependency to a + /// range no block locks. Same refusal: the lock's `is-number@^6.0.0` + /// block can't stand in for `^7.0.0`. + #[tokio::test] + async fn issue_591_changed_range_without_a_lock_block_is_refused() { + let lock = format!( + "{Y2_BEFORE} dependencies:\n is-number \"^6.0.0\"\n\n\ + is-number@^6.0.0:\n version \"6.0.0\"\n" + ); + let mut fx = fixture_with_lock(&lock).await; + patch_manifest( + &mut fx, + br#"{"name":"left-pad","version":"1.3.0","dependencies":{"is-number":"^7.0.0"}}"#, + ) + .await; + let detail = expect_refused(fx.vendor(false).await, "vendor_dep_manifest_unlocked"); + assert!(detail.contains("is-number@^7.0.0"), "{detail}"); + assert!(!detail.contains("^6.0.0"), "{detail}"); + assert_eq!(fx.lock_text().await, lock, "lock untouched"); + + // Once the new range is locked, the patch vendors with the + // sub-map rewritten to it. + let locked = format!("{lock}\nis-number@^7.0.0:\n version \"7.0.0\"\n"); + tokio::fs::write(fx.lock_path(), &locked).await.unwrap(); + let (result, entry, _) = expect_done(fx.vendor(false).await); + assert!(result.success && entry.is_some(), "{:?}", result.error); + let text = fx.lock_text().await; + assert!( + text.contains(" dependencies:\n is-number \"^7.0.0\"\n"), + "{text}" + ); + } + /// #920: the `package.json` advisory is emitted once, by the run that /// wires — an in-sync re-run of a manifest-rewriting patch is a quiet /// AlreadyPatched. #[tokio::test] async fn manifest_rewriting_rerun_is_in_sync_without_the_manifest_warning() { - let mut fx = fixture_with_lock(Y2_BEFORE).await; + let mut fx = fixture_with_lock(&format!("{Y2_BEFORE}{WOW_BLOCK}")).await; let before: &[u8] = br#"{"name":"left-pad","version":"1.3.0"}"#; let after: &[u8] = br#"{"name":"left-pad","version":"1.3.0","dependencies":{"wow":"^1.0.0"}}"#; @@ -1514,6 +1585,9 @@ left-pad@^1.3.0: integrity sha512-XI5MPzVNApjAyhQzphX8BkmKsKUxD4LdyK24iZeQGinBN9yTQT3bFlCBy/aVx2HrNcqQGsdot8ghrjyrvMCoEA== dependencies: old-dep "^1.0.0" + +wow@^1.0.0: + version "1.0.0" "#; let mut fx = fixture_with_lock(lock).await; let before: &[u8] = br#"{"name":"left-pad","version":"1.3.0"}"#; @@ -2449,12 +2523,6 @@ left-pad@^1.3.0: ); assert_eq!(pattern_real_name("alias@npm:left-pad"), Some("left-pad")); assert_eq!(pattern_real_name("no-at-sign"), None); - - // yarn's key quoting rule. - assert_eq!(quote_yarn_key("left-pad"), "left-pad"); - assert_eq!(quote_yarn_key("@scope/x"), "\"@scope/x\""); - assert_eq!(quote_yarn_key("3d-lib"), "\"3d-lib\""); - assert_eq!(quote_yarn_key("true-lib"), "\"true-lib\""); } #[test] diff --git a/docs/ecosystems.md b/docs/ecosystems.md index e5f64f1be..ab023d5a8 100644 --- a/docs/ecosystems.md +++ b/docs/ecosystems.md @@ -136,10 +136,20 @@ The backticked slug in each row is the value `-e`/`--ecosystems` accepts (e.g. rewritten to the hosted tarball. `resolved` always carries the tarball's `#` fragment: yarn 1 names its cache slot after it, so a fragmentless URL would share the slot of an unpatched copy of the same version and a warm - cache would install those bytes (or fail the integrity check). When the grant - carries no sha1, the scan downloads the served tarball, checks it against the - grant's sha512 and pins the sha1 of those bytes. If that download or check - fails, the patch is skipped as `npm_tarball_unavailable`. A project that sets `yarn-offline-mirror` + cache would install those bytes (or fail the integrity check). For an entry it + hasn't pinned yet (or a grant with no sha1), the scan downloads the served + tarball and checks it against the grant's sha512. It pins the sha1 of those + bytes when the grant has none, and it compares the tarball's own + `package.json` with the lock: yarn 1 installs only the dependencies the lock + names, so when the patch adds a dependency (or changes a range) that no + `yarn.lock` block locks, the pin is refused with + `redirect_yarn_classic_dep_manifest_unlocked` (lock the new descriptor first, + for example with `yarn add`, then re-run). When every descriptor is already + locked, the entry's dependency sub-maps are rewritten to match + (`redirect_yarn_classic_dep_manifest_rewritten`). Vendored mode refuses the + same case with `vendor_dep_manifest_unlocked`. If the download, the check or + the `package.json` read fails, the patch is skipped as + `npm_tarball_unavailable`. A project that sets `yarn-offline-mirror` is refused with `redirect_yarn_classic_offline_mirror`. The mirror is resolved the way yarn 1 resolves it: the project's `.yarnrc` / `.npmrc`, the user's (`~/.yarnrc`, which `yarn config set` writes, and `~/.npmrc`), From 224bd9215b77344cf5b8907e1be3c7da47631454 Mon Sep 17 00:00:00 2001 From: Mikola Lysenko Date: Fri, 9 Oct 2026 19:23:09 -0400 Subject: [PATCH 5/5] Adapt yarn classic sha1 pins to main's URL redaction and empty-range test Two interactions with main after merging it into this branch: - #1026 made ApiClient's "artifact not found" errors redact the grant token themselves, so the error no longer contains the raw artifact URL and npm_tarball_unavailable's literal replace of that URL with "" stopped matching. The token was still redacted, but the detail now shows the host and path, which broke issue_558_unfetchable_tarball_skips_the_patch's "server URI absent" check. npm_tarball_unavailable now goes through redact_artifact_text like its sibling skip builders. The test asserts that the grant token never appears, and that the unfetchable detail names the URL with the token redacted. The #558 skip assertions (npm_tarball_unavailable, nothing redirected, yarn.lock untouched) are unchanged. - #1274's yarn_classic_empty_range_key_is_pinned expected a fragmentless resolved. With this PR the pin carries the grant's sha1 fragment (#5ha1), as in the other classic tests this PR already updated. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/scan/hosted_yarn_classic_sha1.rs | 21 ++++++++++++++----- crates/socket-patch-core/src/hosted/engine.rs | 10 ++++++--- .../src/patch/redirect/mod.rs | 2 +- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs b/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs index 88a55cb31..36d7b8bca 100644 --- a/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs +++ b/crates/socket-patch-cli/tests/scan/hosted_yarn_classic_sha1.rs @@ -225,7 +225,7 @@ async fn issue_558_served_tarball_not_matching_the_grant_skips_the_patch() { write_classic_project(&root); let (_, doc, stderr) = scan(&root, &server.uri()); - assert_skipped_untouched(&root, &server, &doc, &stderr); + assert_skipped_untouched(&root, &doc, &stderr); } #[tokio::test] @@ -238,10 +238,18 @@ async fn issue_558_unfetchable_tarball_skips_the_patch() { write_classic_project(&root); let (_, doc, stderr) = scan(&root, &server.uri()); - assert_skipped_untouched(&root, &server, &doc, &stderr); + let detail = assert_skipped_untouched(&root, &doc, &stderr); + // The detail still says where the fetch went, with the token redacted. + assert!( + detail.contains(&format!( + "cannot fetch the hosted tarball: artifact not found: {}/patch/npm/left-pad/1.3.0//{UUID}/left-pad-1.3.0.tgz", + server.uri() + )), + "{detail}" + ); } -fn assert_skipped_untouched(root: &Path, server: &MockServer, doc: &Value, stderr: &str) { +fn assert_skipped_untouched(root: &Path, doc: &Value, stderr: &str) -> String { let skipped: Vec<&Value> = doc["redirect"]["skipped"] .as_array() .unwrap_or_else(|| panic!("redirect.skipped: {doc:#}\n{stderr}")) @@ -250,10 +258,12 @@ fn assert_skipped_untouched(root: &Path, server: &MockServer, doc: &Value, stder .collect(); assert_eq!(skipped.len(), 1, "{doc:#}"); assert_eq!(skipped[0]["purl"], PURL, "{doc:#}"); + // The served URL's grant token authorizes the org's download: never + // shown, whatever the detail says. let detail = skipped[0]["detail"].as_str().unwrap(); assert!( - !detail.contains(&server.uri()), - "the hosted URL is redacted: {detail}" + !detail.contains(TOKEN), + "the grant token is redacted: {detail}" ); assert_eq!(doc["redirect"]["redirected"], 0, "{doc:#}"); assert_eq!( @@ -261,6 +271,7 @@ fn assert_skipped_untouched(root: &Path, server: &MockServer, doc: &Value, stder LOCK, "no fragmentless hosted pin is written" ); + detail.to_string() } /// #591: the served tarball's package.json adds a dependency yarn.lock diff --git a/crates/socket-patch-core/src/hosted/engine.rs b/crates/socket-patch-core/src/hosted/engine.rs index 084ba5d20..b9cab7ce2 100644 --- a/crates/socket-patch-core/src/hosted/engine.rs +++ b/crates/socket-patch-core/src/hosted/engine.rs @@ -1210,8 +1210,8 @@ pub fn record_classic_artifact( /// The skip recorded for an npm dep whose served tarball could not be /// fetched, did not match its grant's sha512 or had no readable -/// package.json, so its yarn classic pin could not be checked (the grant -/// token in `detail` is redacted to ``). +/// package.json, so its yarn classic pin could not be checked (`detail` +/// redacted by [`redact_artifact_text`]). pub fn npm_tarball_unavailable(dep: &DepOverride, detail: &str) -> SkippedPatch { SkippedPatch { purl: format!( @@ -1221,7 +1221,11 @@ pub fn npm_tarball_unavailable(dep: &DepOverride, detail: &str) -> SkippedPatch ), uuid: dep.patch_uuid.clone(), reason: "npm_tarball_unavailable".to_string(), - detail: Some(detail.replace(&dep.artifact_url, "")), + detail: Some(redact_artifact_text( + detail, + &dep.artifact_url, + &dep.patch_uuid, + )), } } diff --git a/crates/socket-patch-core/src/patch/redirect/mod.rs b/crates/socket-patch-core/src/patch/redirect/mod.rs index 681de7fb6..06afda7d9 100644 --- a/crates/socket-patch-core/src/patch/redirect/mod.rs +++ b/crates/socket-patch-core/src/patch/redirect/mod.rs @@ -10819,7 +10819,7 @@ mod tests { .unwrap_or_else(|| panic!("{key}: the block must be pinned: {:?}", r.warnings)); assert!(out.contains(&format!("\n{key}\n")), "{key}: {out}"); assert!( - out.contains("resolved \"http://p.test/lp.tgz\"") + out.contains("resolved \"http://p.test/lp.tgz#5ha1\"") && out.contains("integrity sha512-PATCHED=="), "{key}: {out}" );