Merge pull request #197 from github/feature/release-asset-filtering

Author: lindluni

.gitignore

@@ -2,3 +2,4 @@
 /codeql-action-sync
 /dist/
 /pkged.go
+/release/

README.md

@@ -19,7 +19,7 @@ If your GitHub Enterprise Server instance is on a completely isolated network wh
 From a machine with access to both GitHub.com and GitHub Enterprise Server use the `./codeql-action-sync sync` command to copy the CodeQL Action and bundles.
 
 **Required Arguments:**
-* `--destination-url` - The URL of the GitHub Enterprise Server instance to push the Action to.
+* `--destination-url` - The root URL of the GitHub Enterprise Server instance, for example `https://github.example.com`. Do not include an API path such as `/api/v3`, a query, a fragment, or surrounding whitespace.
 * `--destination-token` - A [Personal Access Token](https://docs.github.com/en/enterprise/user/github/authenticating-to-github/creating-a-personal-access-token) for the destination GitHub Enterprise Server instance. If the destination repository is in an organization that does not yet exist or that you are not an owner of, your token will need to have the `site_admin` scope in order to create the organization or update the repository in it. The organization can also be created manually or an existing organization that you own can be used, in which case the `repo` and `workflow` scopes are sufficient. The token can also be provided by setting the `CODEQL_ACTION_SYNC_TOOL_DESTINATION_TOKEN` environment variable.
 
 **Optional Arguments:**
@@ -29,20 +29,64 @@ From a machine with access to both GitHub.com and GitHub Enterprise Server use t
 * `--actions-admin-user` - The name of the Actions admin user, which will be used if you are updating the bundled CodeQL Action. If not specified `actions-admin` will be used.
 * `--force` - By default the tool will not overwrite existing repositories. Providing this flag will allow it to.
 * `--push-ssh` - Push Git contents over SSH rather than HTTPS. To use this option you must have SSH access to your GitHub Enterprise instance configured.
+* `--include-platforms` - Only download release assets for the listed platforms. Valid values are `linux64`, `linux-arm64`, `osx64`, and `win64`.
+* `--exclude-platforms` - Download release assets for every platform except those listed. This cannot be used with `--include-platforms`.
+* `--bundle-archive-format` - Only download CodeQL bundles in the selected format. Valid values are `tar.gz` and `tar.zst`. Requires explicit `tools` URLs in CodeQL workflows; see [Archive format restrictions](#archive-format-restrictions).
+
+Platform lists are comma-separated. For example, the following syncs Linux x64 and Windows assets, omits the combined all-platform bundle, and downloads CodeQL bundles only as gzip archives. Use it only with workflows configured as described in [Archive format restrictions](#archive-format-restrictions):
+
+```shell
+./codeql-action-sync sync \
+  --destination-url https://github.example.com \
+  --include-platforms linux64,win64 \
+  --bundle-archive-format tar.gz
+```
+
+To sync every platform except macOS:
+
+```shell
+./codeql-action-sync sync \
+  --destination-url https://github.example.com \
+  --exclude-platforms osx64
+```
+
+With none of these flags, the tool continues to copy every release asset. Platform filters also apply to platform-specific checksums, language bundles, and update-job proxies. Non-archive metadata is retained, while the combined all-platform CodeQL bundle is omitted when a platform filter is active. The archive format flag applies only to CodeQL bundles; update-job proxies retain their published format. If a required platform bundle is not published in the requested format, the command fails rather than silently falling back. When an archive format is specified, every explicitly included platform must publish a primary bundle in that format, even if the release has no assets for that platform at all.
+
+These flags limit new downloads and uploads. They do not delete assets copied to GitHub Enterprise Server by an earlier sync.
+
+#### Archive format restrictions
+
+**Warning:** `--bundle-archive-format` controls which bundles are copied, not which archive the CodeQL Action requests. The Action normally chooses gzip on Windows and chooses zstd on Linux/macOS when the CLI and runner support it. If that format was excluded, the Action may try downloading it from GitHub.com, which fails on air-gapped runners. Leave this flag unset to preserve automatic archive selection.
+
+When restricting the format, set the CodeQL init step's `tools` input to an explicit URL for a retained bundle on your destination server. For example, a Linux x64 job using a gzip-only sync:
+
+```yaml
+- uses: github/codeql-action/init@v4
+  with:
+    languages: javascript
+    tools: https://github.example.com/github/codeql-action/releases/download/<bundle-tag>/codeql-bundle-linux64.tar.gz
+```
+
+Replace `<bundle-tag>` with a synced release tag compatible with your Action version, and adjust the host, repository, platform, and archive format for each job. The explicit URL pins the bundle version, so update it when upgrading the bundle. Ensure each runner can extract the selected archive format.
 
 ### I don't have a machine that can access both GitHub.com and GitHub Enterprise Server.
 From a machine with access to GitHub.com use the `./codeql-action-sync pull` command to download a copy of the CodeQL Action and bundles to a local folder.
 
 **Optional Arguments:**
 * `--cache-dir` - The directory in which to store data downloaded from GitHub.com. If not specified a directory next to the sync tool will be used.
 * `--source-token` - A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting. The token does not need to have any scopes.
+* `--include-platforms` - Only download release assets for the listed platforms. Valid values are `linux64`, `linux-arm64`, `osx64`, and `win64`.
+* `--exclude-platforms` - Download release assets for every platform except those listed. This cannot be used with `--include-platforms`.
+* `--bundle-archive-format` - Only download CodeQL bundles in the selected format. Valid values are `tar.gz` and `tar.zst`. Requires explicit `tools` URLs in CodeQL workflows; see [Archive format restrictions](#archive-format-restrictions).
+
+The filtering semantics are the same as for `sync` above. Reusing a cache with different filters removes now-excluded local assets before the cache can be pushed, including assets from historical releases no longer referenced by the current Action defaults. Historical release metadata and matching assets are retained.
 
 Next copy the sync tool and cache directory to another machine which has access to GitHub Enterprise Server.
 
 Now use the `./codeql-action-sync push` command to upload the CodeQL Action and bundles to GitHub Enterprise Server.
 
 **Required Arguments:**
-* `--destination-url` - The URL of the GitHub Enterprise Server instance to push the Action to.
+* `--destination-url` - The root URL of the GitHub Enterprise Server instance, for example `https://github.example.com`. Do not include an API path such as `/api/v3`, a query, a fragment, or surrounding whitespace.
 * `--destination-token` - A [Personal Access Token](https://docs.github.com/en/enterprise/user/github/authenticating-to-github/creating-a-personal-access-token) for the destination GitHub Enterprise Server instance. If the destination repository is in an organization that does not yet exist or that you are not an owner of, your token will need to have the `site_admin` scope in order to create the organization or update the repository in it. The organization can also be created manually or an existing organization that you own can be used, in which case the `repo` and `workflow` scopes are sufficient. The token can also be provided by setting the `CODEQL_ACTION_SYNC_TOOL_DESTINATION_TOKEN` environment variable.
 
 **Optional Arguments:**

cmd/pull.go

@@ -13,13 +13,16 @@ var pullCmd = &cobra.Command{
 	RunE: func(cmd *cobra.Command, args []string) error {
 		version.LogVersion()
 		cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir)
-		return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL)
+		return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL, pullFlags.includePlatforms, pullFlags.excludePlatforms, pullFlags.bundleArchiveFormat)
 	},
 }
 
 type pullFlagFields struct {
-	sourceToken string
-	sourceURL   string
+	sourceToken         string
+	sourceURL           string
+	includePlatforms    []string
+	excludePlatforms    []string
+	bundleArchiveFormat string
 }
 
 var pullFlags = pullFlagFields{}
@@ -28,4 +31,7 @@ func (f *pullFlagFields) Init(cmd *cobra.Command) {
 	cmd.Flags().StringVar(&f.sourceToken, "source-token", "", "A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting.")
 	cmd.Flags().StringVar(&f.sourceURL, "source-url", "", "Use a custom Git URL for fetching the Action repository contents from. The CodeQL bundles will still be fetched from GitHub.com.")
 	cmd.Flags().MarkHidden("source-url")
+	cmd.Flags().StringSliceVar(&f.includePlatforms, "include-platforms", nil, "Only download release assets for these platforms: linux64, linux-arm64, osx64, win64.")
+	cmd.Flags().StringSliceVar(&f.excludePlatforms, "exclude-platforms", nil, "Download release assets for every platform except these: linux64, linux-arm64, osx64, win64.")
+	cmd.Flags().StringVar(&f.bundleArchiveFormat, "bundle-archive-format", "", "Only download CodeQL bundles in this archive format: tar.gz or tar.zst. Requires an explicit tools URL in CodeQL workflows.")
 }

cmd/sync.go

@@ -13,8 +13,12 @@ var syncCmd = &cobra.Command{
 	Short: "Sync the CodeQL Action from GitHub to a GitHub Enterprise Server installation.",
 	RunE: func(cmd *cobra.Command, args []string) error {
 		version.LogVersion()
+		err := push.ValidateArguments(pushFlags.destinationURL, pushFlags.destinationToken, pushFlags.destinationRepository)
+		if err != nil {
+			return err
+		}
 		cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir)
-		err := pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL)
+		err = pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL, pullFlags.includePlatforms, pullFlags.excludePlatforms, pullFlags.bundleArchiveFormat)
 		if err != nil {
 			return err
 		}

internal/pull/pull.go

@@ -42,6 +42,7 @@ type pullService struct {
 	gitCloneURL        string
 	githubDotComClient *github.Client
 	sourceToken        string
+	assetFilter        releaseAssetFilter
 }
 
 func (pullService *pullService) pullGit(fresh bool) error {
@@ -185,6 +186,9 @@ func (pullService *pullService) findRelevantReleases() ([]string, error) {
 
 func (pullService *pullService) pullReleases() error {
 	log.Debug("Pulling CodeQL bundles...")
+	if pullService.assetFilter.bundleArchiveFormat != "" {
+		log.Warnf("--bundle-archive-format %s does not change the CodeQL Action's automatic archive selection. Configure the CodeQL init step's tools input with an explicit URL for a synced bundle; otherwise air-gapped runners may attempt to download an excluded format from GitHub.com.", pullService.assetFilter.bundleArchiveFormat)
+	}
 	relevantReleases, err := pullService.findRelevantReleases()
 	if err != nil {
 		return err
@@ -196,6 +200,11 @@ func (pullService *pullService) pullReleases() error {
 		if err != nil {
 			return githubapiutil.EnrichResponseError(response, err, "Error loading CodeQL release information.")
 		}
+		selectedAssets, skippedAssets, err := pullService.assetFilter.selectAssets(releaseTag, release.Assets)
+		if err != nil {
+			return err
+		}
+		logAssetSelection(releaseTag, len(release.Assets), len(selectedAssets), skippedAssets)
 		err = os.MkdirAll(pullService.cacheDirectory.ReleasePath(releaseTag), 0755)
 		if err != nil {
 			return errors.Wrap(err, "Error creating releases directory.")
@@ -214,7 +223,24 @@ func (pullService *pullService) pullReleases() error {
 		if err != nil {
 			return errors.Wrap(err, "Error creating assets directory.")
 		}
-		for _, asset := range release.Assets {
+		selectedAssetNames := map[string]bool{}
+		for _, asset := range selectedAssets {
+			selectedAssetNames[asset.GetName()] = true
+		}
+		cachedAssets, err := ioutil.ReadDir(assetsPath)
+		if err != nil {
+			return errors.Wrap(err, "Error reading cached release assets.")
+		}
+		for _, cachedAsset := range cachedAssets {
+			if !selectedAssetNames[cachedAsset.Name()] {
+				log.Debugf("Removing filtered cached asset %s...", cachedAsset.Name())
+				err = os.RemoveAll(pullService.cacheDirectory.AssetPath(releaseTag, cachedAsset.Name()))
+				if err != nil {
+					return errors.Wrap(err, "Error removing filtered cached asset.")
+				}
+			}
+		}
+		for _, asset := range selectedAssets {
 			log.Debugf("Downloading asset %s...", asset.GetName())
 			downloadPath := pullService.cacheDirectory.AssetPath(releaseTag, asset.GetName())
 			downloadPathStat, err := os.Stat(downloadPath)
@@ -257,11 +283,46 @@ func (pullService *pullService) pullReleases() error {
 			}
 		}
 	}
+	return pullService.pruneFilteredCachedAssets()
+}
+
+func (pullService *pullService) pruneFilteredCachedAssets() error {
+	if !pullService.assetFilter.filtersPlatforms() && pullService.assetFilter.bundleArchiveFormat == "" {
+		return nil
+	}
+	cachedReleases, err := ioutil.ReadDir(pullService.cacheDirectory.ReleasesPath())
+	if os.IsNotExist(err) {
+		return nil
+	}
+	if err != nil {
+		return errors.Wrap(err, "Error reading cached releases.")
+	}
+	for _, release := range cachedReleases {
+		cachedAssets, err := ioutil.ReadDir(pullService.cacheDirectory.AssetsPath(release.Name()))
+		if err != nil {
+			return errors.Wrapf(err, "Error reading cached release assets for %s.", release.Name())
+		}
+		for _, asset := range cachedAssets {
+			reason := pullService.assetFilter.exclusionReason(classifyReleaseAsset(asset.Name()))
+			if reason == "" {
+				continue
+			}
+			log.Debugf("Removing cached asset %s from release %s: %s.", asset.Name(), release.Name(), reason)
+			err = os.RemoveAll(pullService.cacheDirectory.AssetPath(release.Name(), asset.Name()))
+			if err != nil {
+				return errors.Wrapf(err, "Error removing filtered cached asset %s from release %s.", asset.Name(), release.Name())
+			}
+		}
+	}
 	return nil
 }
 
-func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string, sourceURL string) error {
-	err := cacheDirectory.CheckOrCreateVersionFile(true, version.Version())
+func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string, sourceURL string, includePlatforms []string, excludePlatforms []string, bundleArchiveFormat string) error {
+	assetFilter, err := newReleaseAssetFilter(includePlatforms, excludePlatforms, bundleArchiveFormat)
+	if err != nil {
+		return err
+	}
+	err = cacheDirectory.CheckOrCreateVersionFile(true, version.Version())
 	if err != nil {
 		return err
 	}
@@ -288,6 +349,7 @@ func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sou
 		gitCloneURL:        sourceURL,
 		githubDotComClient: github.NewClient(tokenClient),
 		sourceToken:        sourceToken,
+		assetFilter:        assetFilter,
 	}
 
 	err = pullService.pullGit(false)