Add age-based retention for cached artifacts

README.md

@@ -718,6 +718,8 @@ base_url: "http://localhost:8080"
 storage:
   url: "file:///var/cache/proxy/artifacts"
   max_size: "10GB"  # Optional: evict LRU when exceeded
+  retention:
+    default: "30d"  # Optional: evict artifacts not downloaded for 30 days
 
 database:
   driver: "sqlite"
@@ -1191,6 +1193,7 @@ The proxy exposes Prometheus metrics at `GET /metrics`. All metric names are pre
 | `proxy_cache_misses_total` | counter | `ecosystem` | Cache misses |
 | `proxy_cache_size_bytes` | gauge | | Total size of cached artifacts |
 | `proxy_cached_artifacts_total` | gauge | | Number of cached artifacts |
+| `proxy_artifacts_evicted_total` | counter | `reason`, `ecosystem` | Artifacts evicted by the size limit (`lru`) or by age (`retention`) |
 | `proxy_upstream_fetch_duration_seconds` | histogram | `ecosystem` | Time spent fetching from upstream |
 | `proxy_upstream_errors_total` | counter | `ecosystem`, `error_type` | Upstream fetch failures |
 | `proxy_storage_operation_duration_seconds` | histogram | `operation` | Storage read/write latency |
@@ -1253,7 +1256,7 @@ The same figures are available as JSON from `GET /stats`, which reports `downloa
 
 `ecosystem` means three slightly different things across `/metrics`, and queries that join across them need to know which.
 
-**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`.
+**From the package record, normalized.** The six `proxy_ecosystem_*` gauges, `proxy_cache_hits_total`, `proxy_cache_misses_total`, `proxy_artifacts_evicted_total`, `proxy_integrity_failures_total` and the scan metrics. Aliases collapse here: `gem` reads as `rubygems`, `composer` as `packagist`, `go` as `golang`.
 
 **From the request path.** `proxy_requests_total`, `proxy_request_duration_seconds` and `proxy_response_bytes_total`. The names mostly coincide with the normalized ones -- these also report `rubygems`, `packagist` and `golang` -- but the Debian route reports `debian` where the package record says `deb`, and any path that is not a package endpoint reports `other`, which corresponds to no ecosystem at all.
 

config.example.yaml

@@ -74,6 +74,21 @@ storage:
   # Empty or "0" means unlimited
   max_size: ""
 
+  # Evict artifacts that have not been served for longer than a duration
+  # ("30d", "72h"), independent of max_size. Resolution order: package rule,
+  # then ecosystem rule, then default. "0" or empty means never evict by age.
+  # The default only applies to ecosystems that support retention; naming an
+  # ecosystem or package that does not is a configuration error. See
+  # docs/configuration.md for the supported ecosystems.
+  retention:
+    default: ""
+    # ecosystems:
+    #   npm: "30d"
+    # packages:
+    #   "pkg:npm/lodash": "0"
+    # How often to look for expired artifacts; at least "1m".
+    sweep_interval: "10m"
+
   # Store fetched artifacts. Set to false to stream every download from
   # upstream without storing it; metadata filtering, cooldown and the
   # denylist still apply. Useful when another cache sits in front of the

docs/configuration.md

@@ -46,6 +46,8 @@ storage:
 | `storage.path` | `PROXY_STORAGE_PATH` | `-storage-path` | Local path (deprecated, use url) |
 | `storage.max_size` | `PROXY_STORAGE_MAX_SIZE` | - | Max cache size (e.g., "10GB") |
 | `storage.cache_artifacts` | `PROXY_STORAGE_CACHE_ARTIFACTS` | - | Store fetched artifacts (default: true); `false` streams them from upstream |
+| `storage.retention.default` | `PROXY_STORAGE_RETENTION_DEFAULT` | - | Evict artifacts not served for this long (e.g. "30d"); see [Retention](#retention) |
+| `storage.retention.sweep_interval` | `PROXY_STORAGE_RETENTION_SWEEP_INTERVAL` | - | How often expired artifacts are looked for (default: "10m", at least "1m") |
 
 `storage.max_size` counts cached artifacts only. An artifact replaced by a refetch stays in storage for at least an hour, or `storage.direct_serve_ttl` if longer, so requests already reading it can finish, and storage use can exceed the limit by what was replaced in that time.
 
@@ -62,6 +64,57 @@ Every download is a fresh upstream fetch, and concurrent requests for the same a
 
 `cache_artifacts: false` cannot be combined with `scanning.enabled`, `storage.direct_serve` or `mirror_api`, which all need stored artifacts, and the `mirror` command refuses to run with it.
 
+### Retention
+
+Retention evicts cached artifacts that nobody has downloaded for a while. It works alongside `storage.max_size`: retention removes what has gone unused, and the size limit still applies to whatever is left.
+
+The `ecosystems` and `packages` entries below only pass validation once the ecosystems they name support retention; see the table further down.
+
+```yaml
+storage:
+  retention:
+    default: "30d"
+    ecosystems:
+      npm: "14d"
+      maven: "0"
+    packages:
+      "pkg:npm/lodash": "0"
+    sweep_interval: "10m"
+```
+
+Each artifact (each version of a package) ages on its own, counted from the last time the proxy served it, or from when it was fetched if it has not been served since. A newer release of the same package does not shorten the life of older versions. Durations take a `d` suffix for days, and `"0"` or an empty value means never evict by age. With no retention configured, artifacts stay until `max_size` evicts them. `sweep_interval` must be at least `1m`.
+
+A package rule wins over an ecosystem rule, which wins over `default`. Package rules are keyed by package PURL without a version. The ecosystems and package rules can only name ecosystems that support retention in the running version; anything else, including a typo, stops the proxy at startup with an error naming the setting. `default` applies only to supported ecosystems. Support is added one ecosystem at a time, so a later release can make an existing `default` cover more ecosystems. On startup the proxy logs which ecosystems retention covers (`covered`) and which ones `default` does not reach yet (`default_not_applied`).
+
+| Ecosystem key | Retention supported | Package rule form |
+|---------------|---------------------|-------------------|
+| `alpine` | no | - |
+| `cargo` | no | - |
+| `composer` | no | - |
+| `conan` | no | - |
+| `conda` | no | - |
+| `cran` | no | - |
+| `deb` | no | - |
+| `gem` | no | - |
+| `golang` | no | - |
+| `helm` | no | - |
+| `hex` | no | - |
+| `julia` | no | - |
+| `maven` | no | - |
+| `npm` | no | - |
+| `nuget` | no | - |
+| `oci` | no | - |
+| `pub` | no | - |
+| `pypi` | no | - |
+| `rpm` | no | - |
+| `swift` | no | - |
+
+An evicted artifact is fetched again from upstream on the next request. Packages preloaded with `proxy mirror` age from the time they were mirrored, so an unused mirrored package is evicted like any other. A registry that removes old releases (Linux distribution archives in particular) cannot serve an evicted version again, so set those ecosystems' rules with that in mind.
+
+Expired artifacts are not deleted right away. Their records are cleared and the files wait in the same queue a replaced artifact goes through, for at least an hour (or `storage.direct_serve_ttl` if longer), so downloads already in progress can finish. `storage.max_size` counts the space as free from the moment the record is cleared. Once that wait is over, each minute the queue is worked through for up to 30 seconds, pausing until the next minute after a delete fails. One sweep clears at most 10,000 artifacts, so the first sweep over a large, old cache spreads its evictions, and the downloads that bring popular artifacts back, over several intervals.
+
+With several proxies sharing one database and `database.hit_flush_interval` set, a proxy's sweep cannot see downloads another proxy has not written yet. Keep retention durations well above the flush interval.
+
 ### Amazon S3
 
 ```yaml
@@ -490,7 +543,7 @@ gradle:
 |--------|-------------|-------------|
 | `gradle.build_cache.read_only` | `PROXY_GRADLE_BUILD_CACHE_READ_ONLY` | Disable PUT uploads and keep GET/HEAD read-only |
 | `gradle.build_cache.max_upload_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_UPLOAD_SIZE` | Maximum accepted PUT body size (must be > 0) |
-| `gradle.build_cache.max_age` | `PROXY_GRADLE_BUILD_CACHE_MAX_AGE` | Delete entries older than this duration (default `168h`, set `0` to disable) |
+| `gradle.build_cache.max_age` | `PROXY_GRADLE_BUILD_CACHE_MAX_AGE` | Delete entries older than this duration (default `168h`, also accepts days such as `7d`; set `0` to disable) |
 | `gradle.build_cache.max_size` | `PROXY_GRADLE_BUILD_CACHE_MAX_SIZE` | Total size cap for `_gradle/http-build-cache`, deleting oldest first (`0` disables) |
 | `gradle.build_cache.sweep_interval` | `PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL` | Frequency for background eviction sweeps |
 

internal/config/config.go

@@ -73,6 +73,7 @@ import (
 	"time"
 
 	"github.com/git-pkgs/proxy/internal/denylist"
+	"github.com/git-pkgs/proxy/internal/retention"
 	"github.com/git-pkgs/purl"
 	"gopkg.in/yaml.v3"
 )
@@ -406,6 +407,33 @@ type StorageConfig struct {
 	// of the proxy. False is incompatible with scanning, direct_serve and
 	// mirror_api, which all depend on stored artifacts. Default: true.
 	CacheArtifacts bool `json:"cache_artifacts" yaml:"cache_artifacts"`
+
+	// Retention evicts cached artifacts that have not been accessed for a
+	// configured time, per ecosystem and per package.
+	Retention RetentionConfig `json:"retention" yaml:"retention"`
+}
+
+// RetentionConfig configures age-based eviction of cached artifacts. An
+// artifact's age is the time since it was last served, or since it was
+// fetched when it has not been served since. Durations accept a "d" suffix
+// for days; "0" or empty means never evict by age.
+type RetentionConfig struct {
+	// Default applies to every ecosystem that supports retention and has no
+	// rule of its own.
+	Default string `json:"default" yaml:"default"`
+
+	// Ecosystems overrides the default per ecosystem key (e.g. "npm", "oci").
+	// Configuring an ecosystem whose retention is not supported yet is an
+	// error.
+	Ecosystems map[string]string `json:"ecosystems" yaml:"ecosystems"`
+
+	// Packages overrides the ecosystem rule for single packages, keyed by
+	// package PURL.
+	Packages map[string]string `json:"packages" yaml:"packages"`
+
+	// SweepInterval is how often expired artifacts are looked for, at least
+	// one minute. Default: "10m".
+	SweepInterval string `json:"sweep_interval" yaml:"sweep_interval"`
 }
 
 // GradleConfig configures Gradle-specific features.
@@ -861,6 +889,9 @@ func Default() *Config {
 			Path:           "./cache/artifacts",
 			MaxSize:        "",
 			CacheArtifacts: true,
+			Retention: RetentionConfig{
+				SweepInterval: defaultRetentionSweepIntervalStr,
+			},
 		},
 		Database: DatabaseConfig{
 			Driver: "sqlite",
@@ -1000,6 +1031,8 @@ func (c *Config) LoadFromEnv() {
 	setEnvString(&c.Storage.DirectServeTTL, "PROXY_STORAGE_DIRECT_SERVE_TTL")
 	setEnvString(&c.Storage.DirectServeBaseURL, "PROXY_STORAGE_DIRECT_SERVE_BASE_URL")
 	setEnvBool(&c.Storage.CacheArtifacts, "PROXY_STORAGE_CACHE_ARTIFACTS")
+	setEnvString(&c.Storage.Retention.Default, "PROXY_STORAGE_RETENTION_DEFAULT")
+	setEnvString(&c.Storage.Retention.SweepInterval, "PROXY_STORAGE_RETENTION_SWEEP_INTERVAL")
 	setEnvString(&c.Database.Driver, "PROXY_DATABASE_DRIVER")
 	setEnvString(&c.Database.Path, "PROXY_DATABASE_PATH")
 	setEnvString(&c.Database.URL, "PROXY_DATABASE_URL")
@@ -1178,6 +1211,10 @@ func (c *Config) validateComponents() error {
 		return err
 	}
 
+	if _, err := c.RetentionRules(retention.Default); err != nil {
+		return err
+	}
+
 	if _, err := denylist.New(c.Denylist.Packages); err != nil {
 		return err
 	}
@@ -1228,7 +1265,7 @@ func (g *GradleBuildCacheConfig) Validate() error {
 	}
 
 	if g.MaxAge != "" && g.MaxAge != "0" {
-		if _, err := time.ParseDuration(g.MaxAge); err != nil {
+		if _, err := parseDayDuration(g.MaxAge); err != nil {
 			return fmt.Errorf("invalid gradle.build_cache.max_age %q: %w", g.MaxAge, err)
 		}
 	}
@@ -1263,6 +1300,10 @@ const (
 	defaultGradleBuildCacheSweepInterval = 10 * time.Minute
 	defaultGradleMaxUploadSizeStr        = "100MB"
 	defaultGradleSweepIntervalStr        = "10m"
+	defaultRetentionSweepInterval        = 10 * time.Minute
+	defaultRetentionSweepIntervalStr     = "10m"
+	minRetentionSweepInterval            = time.Minute
+	maxDurationDays                      = 36500
 	defaultScanningTimeoutStr            = "30s"
 )
 
@@ -1431,7 +1472,7 @@ func (c *Config) ParseGradleBuildCacheMaxAge() time.Duration {
 	if c.Gradle.BuildCache.MaxAge == "" || c.Gradle.BuildCache.MaxAge == "0" {
 		return 0
 	}
-	d, err := time.ParseDuration(c.Gradle.BuildCache.MaxAge)
+	d, err := parseDayDuration(c.Gradle.BuildCache.MaxAge)
 	if err != nil || d <= 0 {
 		return 0
 	}

internal/config/retention.go

@@ -0,0 +1,154 @@
+package config
+
+import (
+	"fmt"
+	"math"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/git-pkgs/cooldown"
+	"github.com/git-pkgs/proxy/internal/retention"
+	"github.com/git-pkgs/purl"
+)
+
+// parseDayDuration parses a duration that may use a "d" suffix for days, as
+// cooldown values do. Day counts that are not finite or exceed
+// maxDurationDays are rejected, since converting them to a time.Duration
+// would overflow.
+func parseDayDuration(s string) (time.Duration, error) {
+	if num, ok := strings.CutSuffix(strings.TrimSpace(s), "d"); ok {
+		days, err := strconv.ParseFloat(num, 64)
+		if err == nil && (math.IsNaN(days) || math.IsInf(days, 0) || math.Abs(days) > maxDurationDays) {
+			return 0, fmt.Errorf("invalid duration %q: day count out of range", s)
+		}
+	}
+	return cooldown.ParseDuration(s)
+}
+
+// parseRetentionDuration parses one retention value. field names the
+// setting in error messages.
+func parseRetentionDuration(field, value string) (time.Duration, error) {
+	d, err := parseDayDuration(value)
+	if err != nil {
+		return 0, fmt.Errorf("invalid %s: %w", field, err)
+	}
+	if d < 0 {
+		return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value)
+	}
+	return d, nil
+}
+
+// RetentionRules validates storage.retention against the ecosystems
+// registered with reg and resolves it. An ecosystem or package that reg
+// does not support is an error, so a rule is never silently ignored.
+func (c *Config) RetentionRules(reg *retention.Registry) (retention.Rules, error) {
+	r := c.Storage.Retention
+	rules := retention.Rules{
+		Ecosystems: map[string]time.Duration{},
+		Packages:   map[string]time.Duration{},
+	}
+
+	var err error
+	if rules.Default, err = parseRetentionDuration("storage.retention.default", r.Default); err != nil {
+		return retention.Rules{}, err
+	}
+
+	if r.SweepInterval != "" {
+		d, err := parseDayDuration(r.SweepInterval)
+		if err != nil {
+			return retention.Rules{}, fmt.Errorf("invalid storage.retention.sweep_interval: %w", err)
+		}
+		if d < minRetentionSweepInterval {
+			return retention.Rules{}, fmt.Errorf("invalid storage.retention.sweep_interval %q: must be at least %s",
+				r.SweepInterval, minRetentionSweepInterval)
+		}
+	}
+
+	for _, key := range sortedKeys(r.Ecosystems) {
+		field := "storage.retention.ecosystems." + key
+		if !retention.IsKnown(key) {
+			return retention.Rules{}, fmt.Errorf("invalid %s: unknown ecosystem %q (valid: %s)",
+				field, key, strings.Join(retention.KnownKeys, ", "))
+		}
+		if _, ok := reg.Lookup(key); !ok {
+			return retention.Rules{}, fmt.Errorf("invalid %s: retention is not supported for %q in this version", field, key)
+		}
+		d, err := parseRetentionDuration(field, r.Ecosystems[key])
+		if err != nil {
+			return retention.Rules{}, err
+		}
+		rules.Ecosystems[key] = d
+	}
+
+	for _, key := range sortedKeys(r.Packages) {
+		field := fmt.Sprintf("storage.retention.packages[%q]", key)
+		canonical, err := retentionPackageKey(reg, key)
+		if err != nil {
+			return retention.Rules{}, fmt.Errorf("invalid %s: %w", field, err)
+		}
+		d, err := parseRetentionDuration(field, r.Packages[key])
+		if err != nil {
+			return retention.Rules{}, err
+		}
+		if prev, ok := rules.Packages[canonical]; ok && prev != d {
+			return retention.Rules{}, fmt.Errorf("invalid %s: conflicts with another key for package %s", field, canonical)
+		}
+		rules.Packages[canonical] = d
+	}
+
+	return rules, nil
+}
+
+// retentionPackageKey returns the stored package PURL a configured package
+// key stands for.
+func retentionPackageKey(reg *retention.Registry, key string) (string, error) {
+	// A julia PURL needs a uuid qualifier to parse at all, and the julia
+	// handler stores none, so no key could ever match one of its packages.
+	if strings.HasPrefix(strings.ToLower(key), "pkg:julia/") {
+		return "", fmt.Errorf("package rules are not supported for julia")
+	}
+	p, err := purl.Parse(key)
+	if err != nil {
+		return "", fmt.Errorf("not a valid package PURL: %w", err)
+	}
+	if p.Version != "" {
+		return "", fmt.Errorf("package PURL must not carry a version")
+	}
+	// A rule names a whole package; qualifiers and subpaths would be
+	// dropped on the way to the stored PURL, so refuse them rather than
+	// let "?type=pom" quietly cover every file of the package.
+	if len(p.Qualifiers) > 0 || p.Subpath != "" {
+		return "", fmt.Errorf("package PURL must not carry qualifiers or a subpath")
+	}
+	canonical, ok := reg.Canonical(p)
+	if ok {
+		return canonical, nil
+	}
+	if key, registered := reg.KeyForDBEcosystem(purl.PURLTypeToEcosystem(p.Type)); registered {
+		return "", fmt.Errorf("does not match how %s packages are stored; see the package rule forms in docs/configuration.md", key)
+	}
+	return "", fmt.Errorf("retention is not supported for %s packages in this version", p.Type)
+}
+
+func sortedKeys(m map[string]string) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// ParseRetentionSweepInterval returns how often the retention sweep runs.
+func (c *Config) ParseRetentionSweepInterval() time.Duration {
+	if c.Storage.Retention.SweepInterval == "" {
+		return defaultRetentionSweepInterval
+	}
+	d, err := parseDayDuration(c.Storage.Retention.SweepInterval)
+	if err != nil || d < minRetentionSweepInterval {
+		return defaultRetentionSweepInterval
+	}
+	return d
+}