diff --git a/.gitignore b/.gitignore index 5c3e848..4087834 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /codeql-action-sync /dist/ /pkged.go +/release/ diff --git a/README.md b/README.md index c4c6ccd..0cf1f42 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ If your GitHub Enterprise Server instance is on a completely isolated network wh From a machine with access to both GitHub.com and GitHub Enterprise Server use the `./codeql-action-sync sync` command to copy the CodeQL Action and bundles. **Required Arguments:** -* `--destination-url` - The URL of the GitHub Enterprise Server instance to push the Action to. +* `--destination-url` - The root URL of the GitHub Enterprise Server instance, for example `https://github.example.com`. Do not include an API path such as `/api/v3`, a query, a fragment, or surrounding whitespace. * `--destination-token` - A [Personal Access Token](https://docs.github.com/en/enterprise/user/github/authenticating-to-github/creating-a-personal-access-token) for the destination GitHub Enterprise Server instance. If the destination repository is in an organization that does not yet exist or that you are not an owner of, your token will need to have the `site_admin` scope in order to create the organization or update the repository in it. The organization can also be created manually or an existing organization that you own can be used, in which case the `repo` and `workflow` scopes are sufficient. The token can also be provided by setting the `CODEQL_ACTION_SYNC_TOOL_DESTINATION_TOKEN` environment variable. **Optional Arguments:** @@ -29,6 +29,45 @@ From a machine with access to both GitHub.com and GitHub Enterprise Server use t * `--actions-admin-user` - The name of the Actions admin user, which will be used if you are updating the bundled CodeQL Action. If not specified `actions-admin` will be used. * `--force` - By default the tool will not overwrite existing repositories. Providing this flag will allow it to. * `--push-ssh` - Push Git contents over SSH rather than HTTPS. To use this option you must have SSH access to your GitHub Enterprise instance configured. +* `--include-platforms` - Only download release assets for the listed platforms. Valid values are `linux64`, `linux-arm64`, `osx64`, and `win64`. +* `--exclude-platforms` - Download release assets for every platform except those listed. This cannot be used with `--include-platforms`. +* `--bundle-archive-format` - Only download CodeQL bundles in the selected format. Valid values are `tar.gz` and `tar.zst`. Requires explicit `tools` URLs in CodeQL workflows; see [Archive format restrictions](#archive-format-restrictions). + +Platform lists are comma-separated. For example, the following syncs Linux x64 and Windows assets, omits the combined all-platform bundle, and downloads CodeQL bundles only as gzip archives. Use it only with workflows configured as described in [Archive format restrictions](#archive-format-restrictions): + +```shell +./codeql-action-sync sync \ + --destination-url https://github.example.com \ + --include-platforms linux64,win64 \ + --bundle-archive-format tar.gz +``` + +To sync every platform except macOS: + +```shell +./codeql-action-sync sync \ + --destination-url https://github.example.com \ + --exclude-platforms osx64 +``` + +With none of these flags, the tool continues to copy every release asset. Platform filters also apply to platform-specific checksums, language bundles, and update-job proxies. Non-archive metadata is retained, while the combined all-platform CodeQL bundle is omitted when a platform filter is active. The archive format flag applies only to CodeQL bundles; update-job proxies retain their published format. If a required platform bundle is not published in the requested format, the command fails rather than silently falling back. When an archive format is specified, every explicitly included platform must publish a primary bundle in that format, even if the release has no assets for that platform at all. + +These flags limit new downloads and uploads. They do not delete assets copied to GitHub Enterprise Server by an earlier sync. + +#### Archive format restrictions + +**Warning:** `--bundle-archive-format` controls which bundles are copied, not which archive the CodeQL Action requests. The Action normally chooses gzip on Windows and chooses zstd on Linux/macOS when the CLI and runner support it. If that format was excluded, the Action may try downloading it from GitHub.com, which fails on air-gapped runners. Leave this flag unset to preserve automatic archive selection. + +When restricting the format, set the CodeQL init step's `tools` input to an explicit URL for a retained bundle on your destination server. For example, a Linux x64 job using a gzip-only sync: + +```yaml +- uses: github/codeql-action/init@v4 + with: + languages: javascript + tools: https://github.example.com/github/codeql-action/releases/download//codeql-bundle-linux64.tar.gz +``` + +Replace `` with a synced release tag compatible with your Action version, and adjust the host, repository, platform, and archive format for each job. The explicit URL pins the bundle version, so update it when upgrading the bundle. Ensure each runner can extract the selected archive format. ### I don't have a machine that can access both GitHub.com and GitHub Enterprise Server. From a machine with access to GitHub.com use the `./codeql-action-sync pull` command to download a copy of the CodeQL Action and bundles to a local folder. @@ -36,13 +75,18 @@ From a machine with access to GitHub.com use the `./codeql-action-sync pull` com **Optional Arguments:** * `--cache-dir` - The directory in which to store data downloaded from GitHub.com. If not specified a directory next to the sync tool will be used. * `--source-token` - A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting. The token does not need to have any scopes. +* `--include-platforms` - Only download release assets for the listed platforms. Valid values are `linux64`, `linux-arm64`, `osx64`, and `win64`. +* `--exclude-platforms` - Download release assets for every platform except those listed. This cannot be used with `--include-platforms`. +* `--bundle-archive-format` - Only download CodeQL bundles in the selected format. Valid values are `tar.gz` and `tar.zst`. Requires explicit `tools` URLs in CodeQL workflows; see [Archive format restrictions](#archive-format-restrictions). + +The filtering semantics are the same as for `sync` above. Reusing a cache with different filters removes now-excluded local assets before the cache can be pushed, including assets from historical releases no longer referenced by the current Action defaults. Historical release metadata and matching assets are retained. Next copy the sync tool and cache directory to another machine which has access to GitHub Enterprise Server. Now use the `./codeql-action-sync push` command to upload the CodeQL Action and bundles to GitHub Enterprise Server. **Required Arguments:** -* `--destination-url` - The URL of the GitHub Enterprise Server instance to push the Action to. +* `--destination-url` - The root URL of the GitHub Enterprise Server instance, for example `https://github.example.com`. Do not include an API path such as `/api/v3`, a query, a fragment, or surrounding whitespace. * `--destination-token` - A [Personal Access Token](https://docs.github.com/en/enterprise/user/github/authenticating-to-github/creating-a-personal-access-token) for the destination GitHub Enterprise Server instance. If the destination repository is in an organization that does not yet exist or that you are not an owner of, your token will need to have the `site_admin` scope in order to create the organization or update the repository in it. The organization can also be created manually or an existing organization that you own can be used, in which case the `repo` and `workflow` scopes are sufficient. The token can also be provided by setting the `CODEQL_ACTION_SYNC_TOOL_DESTINATION_TOKEN` environment variable. **Optional Arguments:** diff --git a/cmd/pull.go b/cmd/pull.go index d69f786..d2305a9 100644 --- a/cmd/pull.go +++ b/cmd/pull.go @@ -13,13 +13,16 @@ var pullCmd = &cobra.Command{ RunE: func(cmd *cobra.Command, args []string) error { version.LogVersion() cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir) - return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL) + return pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL, pullFlags.includePlatforms, pullFlags.excludePlatforms, pullFlags.bundleArchiveFormat) }, } type pullFlagFields struct { - sourceToken string - sourceURL string + sourceToken string + sourceURL string + includePlatforms []string + excludePlatforms []string + bundleArchiveFormat string } var pullFlags = pullFlagFields{} @@ -28,4 +31,7 @@ func (f *pullFlagFields) Init(cmd *cobra.Command) { cmd.Flags().StringVar(&f.sourceToken, "source-token", "", "A token to access the API of GitHub.com. This is normally not required, but can be provided if you have issues with API rate limiting.") cmd.Flags().StringVar(&f.sourceURL, "source-url", "", "Use a custom Git URL for fetching the Action repository contents from. The CodeQL bundles will still be fetched from GitHub.com.") cmd.Flags().MarkHidden("source-url") + cmd.Flags().StringSliceVar(&f.includePlatforms, "include-platforms", nil, "Only download release assets for these platforms: linux64, linux-arm64, osx64, win64.") + cmd.Flags().StringSliceVar(&f.excludePlatforms, "exclude-platforms", nil, "Download release assets for every platform except these: linux64, linux-arm64, osx64, win64.") + cmd.Flags().StringVar(&f.bundleArchiveFormat, "bundle-archive-format", "", "Only download CodeQL bundles in this archive format: tar.gz or tar.zst. Requires an explicit tools URL in CodeQL workflows.") } diff --git a/cmd/sync.go b/cmd/sync.go index 7aec7b7..a8e9d32 100644 --- a/cmd/sync.go +++ b/cmd/sync.go @@ -13,8 +13,12 @@ var syncCmd = &cobra.Command{ Short: "Sync the CodeQL Action from GitHub to a GitHub Enterprise Server installation.", RunE: func(cmd *cobra.Command, args []string) error { version.LogVersion() + err := push.ValidateArguments(pushFlags.destinationURL, pushFlags.destinationToken, pushFlags.destinationRepository) + if err != nil { + return err + } cacheDirectory := cachedirectory.NewCacheDirectory(rootFlags.cacheDir) - err := pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL) + err = pull.Pull(cmd.Context(), cacheDirectory, pullFlags.sourceToken, pullFlags.sourceURL, pullFlags.includePlatforms, pullFlags.excludePlatforms, pullFlags.bundleArchiveFormat) if err != nil { return err } diff --git a/internal/pull/pull.go b/internal/pull/pull.go index 34bd0c6..3cbbf23 100644 --- a/internal/pull/pull.go +++ b/internal/pull/pull.go @@ -42,6 +42,7 @@ type pullService struct { gitCloneURL string githubDotComClient *github.Client sourceToken string + assetFilter releaseAssetFilter } func (pullService *pullService) pullGit(fresh bool) error { @@ -185,6 +186,9 @@ func (pullService *pullService) findRelevantReleases() ([]string, error) { func (pullService *pullService) pullReleases() error { log.Debug("Pulling CodeQL bundles...") + if pullService.assetFilter.bundleArchiveFormat != "" { + log.Warnf("--bundle-archive-format %s does not change the CodeQL Action's automatic archive selection. Configure the CodeQL init step's tools input with an explicit URL for a synced bundle; otherwise air-gapped runners may attempt to download an excluded format from GitHub.com.", pullService.assetFilter.bundleArchiveFormat) + } relevantReleases, err := pullService.findRelevantReleases() if err != nil { return err @@ -196,6 +200,11 @@ func (pullService *pullService) pullReleases() error { if err != nil { return githubapiutil.EnrichResponseError(response, err, "Error loading CodeQL release information.") } + selectedAssets, skippedAssets, err := pullService.assetFilter.selectAssets(releaseTag, release.Assets) + if err != nil { + return err + } + logAssetSelection(releaseTag, len(release.Assets), len(selectedAssets), skippedAssets) err = os.MkdirAll(pullService.cacheDirectory.ReleasePath(releaseTag), 0755) if err != nil { return errors.Wrap(err, "Error creating releases directory.") @@ -214,7 +223,24 @@ func (pullService *pullService) pullReleases() error { if err != nil { return errors.Wrap(err, "Error creating assets directory.") } - for _, asset := range release.Assets { + selectedAssetNames := map[string]bool{} + for _, asset := range selectedAssets { + selectedAssetNames[asset.GetName()] = true + } + cachedAssets, err := ioutil.ReadDir(assetsPath) + if err != nil { + return errors.Wrap(err, "Error reading cached release assets.") + } + for _, cachedAsset := range cachedAssets { + if !selectedAssetNames[cachedAsset.Name()] { + log.Debugf("Removing filtered cached asset %s...", cachedAsset.Name()) + err = os.RemoveAll(pullService.cacheDirectory.AssetPath(releaseTag, cachedAsset.Name())) + if err != nil { + return errors.Wrap(err, "Error removing filtered cached asset.") + } + } + } + for _, asset := range selectedAssets { log.Debugf("Downloading asset %s...", asset.GetName()) downloadPath := pullService.cacheDirectory.AssetPath(releaseTag, asset.GetName()) downloadPathStat, err := os.Stat(downloadPath) @@ -257,11 +283,46 @@ func (pullService *pullService) pullReleases() error { } } } + return pullService.pruneFilteredCachedAssets() +} + +func (pullService *pullService) pruneFilteredCachedAssets() error { + if !pullService.assetFilter.filtersPlatforms() && pullService.assetFilter.bundleArchiveFormat == "" { + return nil + } + cachedReleases, err := ioutil.ReadDir(pullService.cacheDirectory.ReleasesPath()) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return errors.Wrap(err, "Error reading cached releases.") + } + for _, release := range cachedReleases { + cachedAssets, err := ioutil.ReadDir(pullService.cacheDirectory.AssetsPath(release.Name())) + if err != nil { + return errors.Wrapf(err, "Error reading cached release assets for %s.", release.Name()) + } + for _, asset := range cachedAssets { + reason := pullService.assetFilter.exclusionReason(classifyReleaseAsset(asset.Name())) + if reason == "" { + continue + } + log.Debugf("Removing cached asset %s from release %s: %s.", asset.Name(), release.Name(), reason) + err = os.RemoveAll(pullService.cacheDirectory.AssetPath(release.Name(), asset.Name())) + if err != nil { + return errors.Wrapf(err, "Error removing filtered cached asset %s from release %s.", asset.Name(), release.Name()) + } + } + } return nil } -func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string, sourceURL string) error { - err := cacheDirectory.CheckOrCreateVersionFile(true, version.Version()) +func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sourceToken string, sourceURL string, includePlatforms []string, excludePlatforms []string, bundleArchiveFormat string) error { + assetFilter, err := newReleaseAssetFilter(includePlatforms, excludePlatforms, bundleArchiveFormat) + if err != nil { + return err + } + err = cacheDirectory.CheckOrCreateVersionFile(true, version.Version()) if err != nil { return err } @@ -288,6 +349,7 @@ func Pull(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, sou gitCloneURL: sourceURL, githubDotComClient: github.NewClient(tokenClient), sourceToken: sourceToken, + assetFilter: assetFilter, } err = pullService.pullGit(false) diff --git a/internal/pull/pull_test.go b/internal/pull/pull_test.go index 1465078..e1d63be 100644 --- a/internal/pull/pull_test.go +++ b/internal/pull/pull_test.go @@ -1,14 +1,21 @@ package pull import ( + "bytes" "context" + "encoding/json" "io/ioutil" "net/http" + "os" + "path" + "strconv" "testing" "github.com/github/codeql-action-sync/internal/cachedirectory" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "github.com/github/codeql-action-sync/test" @@ -64,6 +71,36 @@ func getTestPullService(t *testing.T, temporaryDirectory string, gitCloneURL str } } +func getTestPullServiceWithAssets(t *testing.T, names ...string) pullService { + t.Helper() + router, githubURL := test.GetTestHTTPServer(t) + assets := releaseAssets(names...) + contents := map[int]string{} + for index, asset := range assets { + id := index + 1 + contents[id] = asset.GetName() + asset.ID = github.Int64(int64(id)) + asset.Size = github.Int(len(contents[id])) + } + for _, tag := range []string{"some-codeql-version-on-main", "some-codeql-version-on-v1-and-v2"} { + release := github.RepositoryRelease{TagName: github.String(tag), Assets: assets} + router.HandleFunc("/api/v3/repos/github/codeql-action/releases/tags/"+tag, func(response http.ResponseWriter, request *http.Request) { + test.ServeHTTPResponseFromObject(t, release, response) + }).Methods("GET") + } + router.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/{id:[0-9]+}", func(response http.ResponseWriter, request *http.Request) { + id, err := strconv.Atoi(mux.Vars(request)["id"]) + require.NoError(t, err) + content, exists := contents[id] + require.True(t, exists) + test.ServeHTTPResponseFromString(t, content, response) + }).Methods("GET").Headers("accept", "application/octet-stream") + + service := getTestPullService(t, test.CreateTemporaryDirectory(t), initialActionRepository, githubURL) + require.NoError(t, service.pullGit(true)) + return service +} + func checkExpectedReferencesInCache(t *testing.T, cacheDirectory cachedirectory.CacheDirectory, expectedReferences []string) { localRepository, err := git.PlainOpen(cacheDirectory.GitPath()) require.NoError(t, err) @@ -190,3 +227,190 @@ func TestPullReleases(t *testing.T) { test.RequireFileHasContent(t, releaseSomeCodeQLVersionOnMainContent, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle.tar.gz")) test.RequireFileHasContent(t, releaseSomeCodeQLVersionOnV1AndV2Content, pullService.cacheDirectory.AssetPath("some-codeql-version-on-v1-and-v2", "codeql-bundle.tar.gz")) } + +func TestPullReleasesFiltersAndPrunesCache(t *testing.T) { + temporaryDirectory := test.CreateTemporaryDirectory(t) + githubTestServer, githubURL := test.GetTestHTTPServer(t) + contents := map[int]string{ + 10: "linux zstd", + 13: "version metadata", + 14: "linux proxy", + 20: "other linux zstd", + } + mainRelease := github.RepositoryRelease{ + TagName: github.String("some-codeql-version-on-main"), + Assets: []*github.ReleaseAsset{ + {ID: github.Int64(10), Name: github.String("codeql-bundle-linux64.tar.zst"), Size: github.Int(len(contents[10]))}, + {ID: github.Int64(11), Name: github.String("codeql-bundle-linux64.tar.gz"), Size: github.Int(1)}, + {ID: github.Int64(12), Name: github.String("codeql-bundle.tar.zst"), Size: github.Int(1)}, + {ID: github.Int64(13), Name: github.String("cli-version-2.27.2.txt"), Size: github.Int(len(contents[13]))}, + {ID: github.Int64(14), Name: github.String("update-job-proxy-linux64.tar.gz"), Size: github.Int(len(contents[14]))}, + {ID: github.Int64(15), Name: github.String("codeql-bundle-win64.tar.zst"), Size: github.Int(1)}, + }, + } + otherRelease := github.RepositoryRelease{ + TagName: github.String("some-codeql-version-on-v1-and-v2"), + Assets: []*github.ReleaseAsset{ + {ID: github.Int64(20), Name: github.String("codeql-bundle-linux64.tar.zst"), Size: github.Int(len(contents[20]))}, + }, + } + githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/tags/some-codeql-version-on-main", func(response http.ResponseWriter, request *http.Request) { + test.ServeHTTPResponseFromObject(t, mainRelease, response) + }).Methods("GET") + githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/tags/some-codeql-version-on-v1-and-v2", func(response http.ResponseWriter, request *http.Request) { + test.ServeHTTPResponseFromObject(t, otherRelease, response) + }).Methods("GET") + githubTestServer.HandleFunc("/api/v3/repos/github/codeql-action/releases/assets/{id:[0-9]+}", func(response http.ResponseWriter, request *http.Request) { + id, err := strconv.Atoi(mux.Vars(request)["id"]) + require.NoError(t, err) + content, expected := contents[id] + require.True(t, expected, "asset %d should have been filtered", id) + test.ServeHTTPResponseFromString(t, content, response) + }).Methods("GET").Headers("accept", "application/octet-stream") + + pullService := getTestPullService(t, temporaryDirectory, initialActionRepository, githubURL) + pullService.assetFilter, _ = newReleaseAssetFilter([]string{"linux64"}, nil, "tar.zst") + err := pullService.pullGit(true) + require.NoError(t, err) + staleAssetsPath := pullService.cacheDirectory.AssetsPath("some-codeql-version-on-main") + require.NoError(t, os.MkdirAll(staleAssetsPath, 0755)) + staleAssetPath := path.Join(staleAssetsPath, "codeql-bundle-osx64.tar.gz") + require.NoError(t, ioutil.WriteFile(staleAssetPath, []byte("stale"), 0644)) + + err = pullService.pullReleases() + require.NoError(t, err) + + require.NoFileExists(t, staleAssetPath) + test.RequireFileHasContent(t, contents[10], pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-linux64.tar.zst")) + test.RequireFileHasContent(t, contents[13], pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "cli-version-2.27.2.txt")) + test.RequireFileHasContent(t, contents[14], pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "update-job-proxy-linux64.tar.gz")) + require.NoFileExists(t, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-linux64.tar.gz")) + require.NoFileExists(t, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle.tar.zst")) + require.NoFileExists(t, pullService.cacheDirectory.AssetPath("some-codeql-version-on-main", "codeql-bundle-win64.tar.zst")) + test.RequireFileHasContent(t, contents[20], pullService.cacheDirectory.AssetPath("some-codeql-version-on-v1-and-v2", "codeql-bundle-linux64.tar.zst")) +} + +func TestPullReleasesFiltersHistoricalCache(t *testing.T) { + names := []string{ + "codeql-bundle-linux64.tar.gz", + "codeql-bundle-linux64.tar.gz.checksum.txt", + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-linux64.tar.zst.checksum.txt", + "codeql-bundle-win64.tar.gz", + "codeql-bundle-win64.tar.zst", + "codeql-bundle.tar.gz", + "codeql-bundle.tar.zst", + "update-job-proxy-linux64.tar.gz", + "update-job-proxy-win64.tar.gz", + "cli-version.txt", + "future-release.tar.gz", + } + cases := []struct { + name string + includePlatforms []string + excludePlatforms []string + bundleArchiveFormat string + expectedNames []string + }{ + { + name: "included platform and gzip", + includePlatforms: []string{"linux64"}, + bundleArchiveFormat: "tar.gz", + expectedNames: []string{ + "codeql-bundle-linux64.tar.gz", + "codeql-bundle-linux64.tar.gz.checksum.txt", + "update-job-proxy-linux64.tar.gz", + "cli-version.txt", + "future-release.tar.gz", + }, + }, + { + name: "excluded platform and zstd", + excludePlatforms: []string{"win64"}, + bundleArchiveFormat: "tar.zst", + expectedNames: []string{ + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-linux64.tar.zst.checksum.txt", + "update-job-proxy-linux64.tar.gz", + "cli-version.txt", + "future-release.tar.gz", + }, + }, + { + name: "format only", + bundleArchiveFormat: "tar.gz", + expectedNames: []string{ + "codeql-bundle-linux64.tar.gz", + "codeql-bundle-linux64.tar.gz.checksum.txt", + "codeql-bundle-win64.tar.gz", + "codeql-bundle.tar.gz", + "update-job-proxy-linux64.tar.gz", + "update-job-proxy-win64.tar.gz", + "cli-version.txt", + "future-release.tar.gz", + }, + }, + { + name: "no filters preserve history", + expectedNames: names, + }, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + service := getTestPullServiceWithAssets(t, names...) + var err error + service.assetFilter, err = newReleaseAssetFilter(testCase.includePlatforms, testCase.excludePlatforms, testCase.bundleArchiveFormat) + require.NoError(t, err) + + historicalTag := "an-ignored-tag-too" + require.NoError(t, os.MkdirAll(service.cacheDirectory.AssetsPath(historicalTag), 0755)) + metadata, err := json.Marshal(github.RepositoryRelease{TagName: github.String(historicalTag)}) + require.NoError(t, err) + require.NoError(t, ioutil.WriteFile(service.cacheDirectory.MetadataPath(historicalTag), metadata, 0644)) + for _, name := range names { + require.NoError(t, ioutil.WriteFile(service.cacheDirectory.AssetPath(historicalTag, name), []byte("historical "+name), 0644)) + } + + require.NoError(t, service.pullReleases()) + cachedAssets, err := ioutil.ReadDir(service.cacheDirectory.AssetsPath(historicalTag)) + require.NoError(t, err) + actualNames := []string{} + for _, asset := range cachedAssets { + actualNames = append(actualNames, asset.Name()) + test.RequireFileHasContent(t, "historical "+asset.Name(), service.cacheDirectory.AssetPath(historicalTag, asset.Name())) + } + require.ElementsMatch(t, testCase.expectedNames, actualNames) + test.RequireFileHasContent(t, string(metadata), service.cacheDirectory.MetadataPath(historicalTag)) + }) + } +} + +func TestPullReleasesWarnsWhenRestrictingBundleFormats(t *testing.T) { + for _, format := range []string{"tar.gz", ""} { + t.Run("format="+format, func(t *testing.T) { + gzipName := "codeql-bundle-linux64.tar.gz" + zstdName := "codeql-bundle-linux64.tar.zst" + service := getTestPullServiceWithAssets(t, gzipName, zstdName) + var err error + service.assetFilter, err = newReleaseAssetFilter([]string{"linux64"}, nil, format) + require.NoError(t, err) + + var logged bytes.Buffer + previousOutput := log.StandardLogger().Out + log.SetOutput(&logged) + t.Cleanup(func() { log.SetOutput(previousOutput) }) + + require.NoError(t, service.pullReleases()) + test.RequireFileHasContent(t, gzipName, service.cacheDirectory.AssetPath("some-codeql-version-on-main", gzipName)) + if format != "" { + require.NoFileExists(t, service.cacheDirectory.AssetPath("some-codeql-version-on-main", zstdName)) + require.Contains(t, logged.String(), "--bundle-archive-format") + require.Contains(t, logged.String(), "tools input with an explicit URL") + require.Contains(t, logged.String(), "GitHub.com") + } else { + test.RequireFileHasContent(t, zstdName, service.cacheDirectory.AssetPath("some-codeql-version-on-main", zstdName)) + require.NotContains(t, logged.String(), "--bundle-archive-format") + } + }) + } +} diff --git a/internal/pull/release_asset_filter.go b/internal/pull/release_asset_filter.go new file mode 100644 index 0000000..1445cee --- /dev/null +++ b/internal/pull/release_asset_filter.go @@ -0,0 +1,214 @@ +package pull + +import ( + "fmt" + "sort" + "strings" + + "github.com/google/go-github/v32/github" + log "github.com/sirupsen/logrus" +) + +var releaseAssetPlatforms = []string{"linux64", "linux-arm64", "osx64", "win64"} + +type releaseAssetFilter struct { + includePlatforms map[string]bool + excludePlatforms map[string]bool + bundleArchiveFormat string +} + +type releaseAsset struct { + name string + platform string + format string + bundle bool + primary bool + combined bool + unclassifiedArchive bool +} + +type skippedReleaseAsset struct { + name string + reason string +} + +func newReleaseAssetFilter(includePlatforms []string, excludePlatforms []string, bundleArchiveFormat string) (releaseAssetFilter, error) { + if len(includePlatforms) > 0 && len(excludePlatforms) > 0 { + return releaseAssetFilter{}, fmt.Errorf("--include-platforms and --exclude-platforms cannot be used together") + } + if bundleArchiveFormat != "" && bundleArchiveFormat != "tar.gz" && bundleArchiveFormat != "tar.zst" { + return releaseAssetFilter{}, fmt.Errorf("invalid bundle archive format %q: expected tar.gz or tar.zst", bundleArchiveFormat) + } + + include, err := platformSet(includePlatforms) + if err != nil { + return releaseAssetFilter{}, err + } + exclude, err := platformSet(excludePlatforms) + if err != nil { + return releaseAssetFilter{}, err + } + return releaseAssetFilter{ + includePlatforms: include, + excludePlatforms: exclude, + bundleArchiveFormat: bundleArchiveFormat, + }, nil +} + +func platformSet(platforms []string) (map[string]bool, error) { + result := map[string]bool{} + for _, platform := range platforms { + if !isReleaseAssetPlatform(platform) { + return nil, fmt.Errorf("invalid platform %q: expected one of %s", platform, strings.Join(releaseAssetPlatforms, ", ")) + } + result[platform] = true + } + return result, nil +} + +func isReleaseAssetPlatform(platform string) bool { + for _, candidate := range releaseAssetPlatforms { + if platform == candidate { + return true + } + } + return false +} + +func classifyReleaseAsset(name string) releaseAsset { + result := releaseAsset{name: name} + archiveName := name + if strings.HasSuffix(archiveName, ".checksum.txt") { + archiveName = strings.TrimSuffix(archiveName, ".checksum.txt") + } + switch { + case strings.HasSuffix(archiveName, ".tar.gz"): + result.format = "tar.gz" + archiveName = strings.TrimSuffix(archiveName, ".tar.gz") + case strings.HasSuffix(archiveName, ".tar.zst"): + result.format = "tar.zst" + archiveName = strings.TrimSuffix(archiveName, ".tar.zst") + default: + return result + } + + result.bundle = strings.HasPrefix(archiveName, "codeql-bundle") + result.combined = archiveName == "codeql-bundle" + for _, platform := range releaseAssetPlatforms { + if strings.HasSuffix(archiveName, "-"+platform) { + result.platform = platform + result.primary = archiveName == "codeql-bundle-"+platform + return result + } + } + result.unclassifiedArchive = !result.combined + return result +} + +func (filter releaseAssetFilter) filtersPlatforms() bool { + return len(filter.includePlatforms) > 0 || len(filter.excludePlatforms) > 0 +} + +func (filter releaseAssetFilter) includesPlatform(platform string) bool { + if len(filter.includePlatforms) > 0 { + return filter.includePlatforms[platform] + } + return !filter.excludePlatforms[platform] +} + +func (filter releaseAssetFilter) exclusionReason(item releaseAsset) string { + switch { + case item.combined && filter.filtersPlatforms(): + return "combined bundle omitted by platform filter" + case item.platform != "" && !filter.includesPlatform(item.platform): + return "platform filtered" + case item.bundle && item.format != "" && filter.bundleArchiveFormat != "" && item.format != filter.bundleArchiveFormat: + return "bundle archive format filtered" + default: + return "" + } +} + +func (filter releaseAssetFilter) selectAssets(releaseTag string, assets []*github.ReleaseAsset) ([]*github.ReleaseAsset, []skippedReleaseAsset, error) { + classified := make([]releaseAsset, len(assets)) + primaryFormats := map[string]map[string]bool{} + for index, asset := range assets { + classified[index] = classifyReleaseAsset(asset.GetName()) + item := classified[index] + if item.bundle && (item.primary || item.combined) && !strings.HasSuffix(item.name, ".checksum.txt") { + if primaryFormats[item.nameWithoutFormat()] == nil { + primaryFormats[item.nameWithoutFormat()] = map[string]bool{} + } + primaryFormats[item.nameWithoutFormat()][item.format] = true + } + } + + if filter.bundleArchiveFormat != "" { + for _, platform := range releaseAssetPlatforms { + if !filter.includePlatforms[platform] { + continue + } + name := "codeql-bundle-" + platform + if !primaryFormats[name][filter.bundleArchiveFormat] { + return nil, nil, fmt.Errorf("release %s does not publish required asset %s.%s", releaseTag, name, filter.bundleArchiveFormat) + } + } + for name, formats := range primaryFormats { + item := classifyReleaseAsset(name + "." + firstFormat(formats)) + if item.combined && filter.filtersPlatforms() { + continue + } + if item.platform != "" && !filter.includesPlatform(item.platform) { + continue + } + if !formats[filter.bundleArchiveFormat] { + return nil, nil, fmt.Errorf("release %s does not publish required asset %s.%s", releaseTag, name, filter.bundleArchiveFormat) + } + } + } + + selected := []*github.ReleaseAsset{} + skipped := []skippedReleaseAsset{} + for index, asset := range assets { + item := classified[index] + reason := filter.exclusionReason(item) + if reason == "" { + if item.unclassifiedArchive { + log.Debugf("Keeping unclassified release archive %s.", asset.GetName()) + } + selected = append(selected, asset) + } else { + skipped = append(skipped, skippedReleaseAsset{name: asset.GetName(), reason: reason}) + } + } + return selected, skipped, nil +} + +func (asset releaseAsset) nameWithoutFormat() string { + name := strings.TrimSuffix(asset.name, ".checksum.txt") + return strings.TrimSuffix(name, "."+asset.format) +} + +func firstFormat(formats map[string]bool) string { + for format := range formats { + return format + } + return "" +} + +func logAssetSelection(releaseTag string, total int, selected int, skipped []skippedReleaseAsset) { + if len(skipped) == 0 { + return + } + reasonCounts := map[string]int{} + for _, asset := range skipped { + reasonCounts[asset.reason]++ + log.Debugf("Skipping release asset %s: %s.", asset.name, asset.reason) + } + reasons := make([]string, 0, len(reasonCounts)) + for reason, count := range reasonCounts { + reasons = append(reasons, fmt.Sprintf("%s=%d", reason, count)) + } + sort.Strings(reasons) + log.Infof("Selected %d of %d assets for release %s; skipped %d (%s).", selected, total, releaseTag, len(skipped), strings.Join(reasons, ", ")) +} diff --git a/internal/pull/release_asset_filter_test.go b/internal/pull/release_asset_filter_test.go new file mode 100644 index 0000000..140d6e4 --- /dev/null +++ b/internal/pull/release_asset_filter_test.go @@ -0,0 +1,172 @@ +package pull + +import ( + "testing" + + "github.com/google/go-github/v32/github" + "github.com/stretchr/testify/require" +) + +func releaseAssets(names ...string) []*github.ReleaseAsset { + assets := make([]*github.ReleaseAsset, len(names)) + for index, name := range names { + assets[index] = &github.ReleaseAsset{Name: github.String(name)} + } + return assets +} + +func releaseAssetNames(assets []*github.ReleaseAsset) []string { + names := make([]string, len(assets)) + for index, asset := range assets { + names[index] = asset.GetName() + } + return names +} + +func TestNewReleaseAssetFilter(t *testing.T) { + _, err := newReleaseAssetFilter([]string{"linux64"}, []string{"osx64"}, "") + require.EqualError(t, err, "--include-platforms and --exclude-platforms cannot be used together") + + _, err = newReleaseAssetFilter([]string{"plan9"}, nil, "") + require.EqualError(t, err, `invalid platform "plan9": expected one of linux64, linux-arm64, osx64, win64`) + + _, err = newReleaseAssetFilter(nil, nil, "zip") + require.EqualError(t, err, `invalid bundle archive format "zip": expected tar.gz or tar.zst`) +} + +func TestReleaseAssetFilterDefaultsToAllAssets(t *testing.T) { + filter, err := newReleaseAssetFilter(nil, nil, "") + require.NoError(t, err) + assets := releaseAssets( + "codeql-bundle.tar.gz", + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-swift-osx64.tar.zst", + "update-job-proxy-win64.tar.gz", + "cli-version-2.27.2.txt", + ) + + selected, skipped, err := filter.selectAssets("release", assets) + require.NoError(t, err) + require.Equal(t, releaseAssetNames(assets), releaseAssetNames(selected)) + require.Empty(t, skipped) +} + +func TestReleaseAssetFilterIncludesExactPlatforms(t *testing.T) { + filter, err := newReleaseAssetFilter([]string{"linux-arm64"}, nil, "") + require.NoError(t, err) + assets := releaseAssets( + "codeql-bundle.tar.gz", + "codeql-bundle-linux64.tar.gz", + "codeql-bundle-linux-arm64.tar.gz", + "codeql-bundle-linux-arm64.tar.gz.checksum.txt", + "update-job-proxy-linux-arm64.tar.gz", + "codeql-bundle-swift-osx64.tar.zst", + "cli-version-2.27.2.txt", + "future-release-metadata.json", + ) + + selected, skipped, err := filter.selectAssets("release", assets) + require.NoError(t, err) + require.Equal(t, []string{ + "codeql-bundle-linux-arm64.tar.gz", + "codeql-bundle-linux-arm64.tar.gz.checksum.txt", + "update-job-proxy-linux-arm64.tar.gz", + "cli-version-2.27.2.txt", + "future-release-metadata.json", + }, releaseAssetNames(selected)) + require.Len(t, skipped, 3) +} + +func TestReleaseAssetFilterExcludesPlatforms(t *testing.T) { + filter, err := newReleaseAssetFilter(nil, []string{"osx64"}, "") + require.NoError(t, err) + assets := releaseAssets( + "codeql-bundle.tar.zst", + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-osx64.tar.zst", + "update-job-proxy-osx64.tar.gz", + "codeql-bundle-win64.tar.zst", + ) + + selected, _, err := filter.selectAssets("release", assets) + require.NoError(t, err) + require.Equal(t, []string{ + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-win64.tar.zst", + }, releaseAssetNames(selected)) +} + +func TestReleaseAssetFilterSelectsBundleFormat(t *testing.T) { + filter, err := newReleaseAssetFilter(nil, nil, "tar.gz") + require.NoError(t, err) + assets := releaseAssets( + "codeql-bundle.tar.gz", + "codeql-bundle.tar.zst", + "codeql-bundle-linux64.tar.gz", + "codeql-bundle-linux64.tar.gz.checksum.txt", + "codeql-bundle-linux64.tar.zst", + "codeql-bundle-actions-linux64.tar.zst", + "update-job-proxy-linux64.tar.gz", + "cli-version-2.27.2.txt", + ) + + selected, _, err := filter.selectAssets("release", assets) + require.NoError(t, err) + require.Equal(t, []string{ + "codeql-bundle.tar.gz", + "codeql-bundle-linux64.tar.gz", + "codeql-bundle-linux64.tar.gz.checksum.txt", + "update-job-proxy-linux64.tar.gz", + "cli-version-2.27.2.txt", + }, releaseAssetNames(selected)) +} + +func TestReleaseAssetFilterFailsWhenPrimaryFormatIsMissing(t *testing.T) { + filter, err := newReleaseAssetFilter([]string{"win64"}, nil, "tar.zst") + require.NoError(t, err) + + _, _, err = filter.selectAssets("codeql-bundle-v1.2.3", releaseAssets( + "codeql-bundle-win64.tar.gz", + "codeql-bundle-actions-linux64.tar.zst", + )) + require.EqualError(t, err, "release codeql-bundle-v1.2.3 does not publish required asset codeql-bundle-win64.tar.zst") +} + +func TestReleaseAssetFilterRequiresEveryIncludedPlatform(t *testing.T) { + cases := []struct { + name string + assets []string + }{ + { + name: "platform absent", + assets: []string{"codeql-bundle-linux64.tar.gz", "codeql-bundle.tar.gz", "cli-version.txt"}, + }, + { + name: "checksum without archive", + assets: []string{"codeql-bundle-linux64.tar.gz", "codeql-bundle-win64.tar.gz.checksum.txt"}, + }, + { + name: "language bundle without primary archive", + assets: []string{"codeql-bundle-linux64.tar.gz", "codeql-bundle-csharp-win64.tar.gz"}, + }, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + filter, err := newReleaseAssetFilter([]string{"linux64", "win64"}, nil, "tar.gz") + require.NoError(t, err) + + _, _, err = filter.selectAssets("codeql-bundle-v1.2.3", releaseAssets(testCase.assets...)) + require.EqualError(t, err, "release codeql-bundle-v1.2.3 does not publish required asset codeql-bundle-win64.tar.gz") + }) + } +} + +func TestReleaseAssetFilterAllowsAbsentPlatformsWithoutFormatRestriction(t *testing.T) { + filter, err := newReleaseAssetFilter([]string{"linux64", "win64"}, nil, "") + require.NoError(t, err) + assets := releaseAssets("codeql-bundle-linux64.tar.gz", "cli-version.txt") + + selected, _, err := filter.selectAssets("release", assets) + require.NoError(t, err) + require.Equal(t, releaseAssetNames(assets), releaseAssetNames(selected)) +} diff --git a/internal/push/push.go b/internal/push/push.go index 05439b7..7a4220b 100644 --- a/internal/push/push.go +++ b/internal/push/push.go @@ -434,7 +434,11 @@ func (pushService *pushService) pushReleases() error { } func Push(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, destinationURL string, destinationToken string, destinationRepository string, actionsAdminUser string, force bool, pushSSH bool, gitURL string) error { - err := cacheDirectory.CheckOrCreateVersionFile(false, version.Version()) + err := ValidateArguments(destinationURL, destinationToken, destinationRepository) + if err != nil { + return err + } + err = cacheDirectory.CheckOrCreateVersionFile(false, version.Version()) if err != nil { return err } @@ -519,3 +523,24 @@ func Push(ctx context.Context, cacheDirectory cachedirectory.CacheDirectory, des log.Infof("Finished pushing CodeQL Action to %s!", destinationRepository) return nil } + +func ValidateArguments(destinationURL string, destinationToken string, destinationRepository string) error { + if destinationURL != strings.TrimSpace(destinationURL) { + return usererrors.New("The destination URL cannot contain surrounding whitespace.") + } + parsedDestinationURL, err := url.Parse(destinationURL) + if err != nil || (parsedDestinationURL.Scheme != "http" && parsedDestinationURL.Scheme != "https") || parsedDestinationURL.Host == "" { + return usererrors.New("The destination URL must be a full HTTP or HTTPS URL, for example `https://github.example.com`.") + } + if strings.TrimRight(parsedDestinationURL.EscapedPath(), "/") != "" || parsedDestinationURL.RawQuery != "" || parsedDestinationURL.Fragment != "" { + return usererrors.New("The destination URL must be the root URL of the GitHub Enterprise instance, without a path, query, or fragment.") + } + if strings.TrimSpace(destinationToken) == "" { + return usererrors.New("The destination token cannot be empty.") + } + destinationRepositorySplit := strings.Split(destinationRepository, "/") + if len(destinationRepositorySplit) != 2 || destinationRepositorySplit[0] == "" || destinationRepositorySplit[1] == "" { + return usererrors.New("The destination repository must be in `owner/repository` format.") + } + return nil +} diff --git a/internal/push/push_test.go b/internal/push/push_test.go index b31d429..01e54c0 100644 --- a/internal/push/push_test.go +++ b/internal/push/push_test.go @@ -33,6 +33,7 @@ func getTestPushService(t *testing.T, cacheDirectoryString string, githubEnterpr } else { githubEnterpriseClient = nil } + token := oauth2.Token{AccessToken: "token"} return pushService{ ctx: context.Background(), @@ -44,6 +45,48 @@ func getTestPushService(t *testing.T, cacheDirectoryString string, githubEnterpr } } +func TestValidateArguments(t *testing.T) { + require.EqualError(t, ValidateArguments("", "token", "owner/repository"), "The destination URL must be a full HTTP or HTTPS URL, for example `https://github.example.com`.") + require.EqualError(t, ValidateArguments("github.example.com", "token", "owner/repository"), "The destination URL must be a full HTTP or HTTPS URL, for example `https://github.example.com`.") + require.EqualError(t, ValidateArguments("https://github.example.com", "", "owner/repository"), "The destination token cannot be empty.") + require.EqualError(t, ValidateArguments("https://github.example.com", "token", "repository"), "The destination repository must be in `owner/repository` format.") + require.NoError(t, ValidateArguments("https://github.example.com", "token", "owner/repository")) +} + +func TestValidateArgumentsRejectsUnsupportedDestinationURLs(t *testing.T) { + whitespaceError := "The destination URL cannot contain surrounding whitespace." + rootURLError := "The destination URL must be the root URL of the GitHub Enterprise instance, without a path, query, or fragment." + cases := []struct { + destinationURL string + expectedError string + }{ + {" https://github.example.com", whitespaceError}, + {"https://github.example.com ", whitespaceError}, + {"https://github.example.com/prefix", rootURLError}, + {"https://github.example.com/api/v3", rootURLError}, + {"https://github.example.com/%2F", rootURLError}, + {"https://github.example.com?tenant=1", rootURLError}, + {"https://github.example.com#fragment", rootURLError}, + } + for _, testCase := range cases { + t.Run(testCase.destinationURL, func(t *testing.T) { + require.EqualError(t, ValidateArguments(testCase.destinationURL, "token", "owner/repository"), testCase.expectedError) + }) + } +} + +func TestValidateArgumentsAcceptsSupportedDestinationURLs(t *testing.T) { + for _, destinationURL := range []string{ + "https://github.example.com/", + "https://github.example.com///", + "http://github.example.com:8080/", + } { + t.Run(destinationURL, func(t *testing.T) { + require.NoError(t, ValidateArguments(destinationURL, "token", "owner/repository")) + }) + } +} + func TestCreateRepositoryWhenUserIsOwner(t *testing.T) { temporaryDirectory := test.CreateTemporaryDirectory(t) githubTestServer, githubEnterpriseURL := test.GetTestHTTPServer(t)