diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 3f5f26e..fa67fb3 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -11,21 +11,9 @@ before: - go mod tidy builds: - - id: collector + - id: optimiser main: ./cmd/collector - binary: collector - env: - - CGO_ENABLED=0 - goos: - - linux - goarch: - - amd64 - - arm64 - ldflags: - - -s -w - - id: receiver - main: ./cmd/receiver - binary: receiver + binary: optimiser env: - CGO_ENABLED=0 goos: @@ -49,28 +37,12 @@ snapshot: version_template: "{{ incpatch .Version }}-next" dockers_v2: - - id: collector - ids: - - collector - images: - - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_ORG }}/forgejo-runner-optimiser-collector" + - images: + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_ORG }}/optimiser" tags: - "{{ .Version }}" - latest dockerfile: Dockerfile.goreleaser - build_args: - BINARY: collector - - id: receiver - ids: - - receiver - images: - - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_ORG }}/forgejo-runner-optimiser-receiver" - tags: - - "{{ .Version }}" - - latest - dockerfile: Dockerfile.goreleaser - build_args: - BINARY: receiver changelog: sort: asc diff --git a/CLAUDE.md b/CLAUDE.md index 6b6d7ef..f9d6972 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,7 +28,7 @@ make install-hooks # Install pre-commit and commit-msg hooks ## Architecture Overview -A resource optimiser for CI/CD environments with shared PID namespaces. It consists of two binaries — a **collector** and a **receiver** (which includes the **sizer**): +This is a Go metrics collector designed for CI/CD environments with shared PID namespaces. It consists of two binaries: ### Collector (`cmd/collector`) Runs alongside CI workloads, periodically reads `/proc` filesystem, and pushes a summary to the receiver on shutdown (SIGINT/SIGTERM). @@ -40,12 +40,11 @@ Runs alongside CI workloads, periodically reads `/proc` filesystem, and pushes a 4. On shutdown, `summary.PushClient` sends the summary to the receiver HTTP endpoint ### Receiver (`cmd/receiver`) -HTTP service that stores metric summaries in SQLite (via GORM), provides a query API, and includes the **sizer** — which computes right-sized Kubernetes resource requests and limits from historical data. +HTTP service that stores metric summaries in SQLite (via GORM) and provides a query API. **Key Endpoints:** - `POST /api/v1/metrics` - Receive metrics from collectors - `GET /api/v1/metrics/repo/{org}/{repo}/{workflow}/{job}` - Query stored metrics -- `GET /api/v1/sizing/repo/{org}/{repo}/{workflow}/{job}` - Compute container sizes from historical data ### Internal Packages @@ -56,7 +55,7 @@ HTTP service that stores metric summaries in SQLite (via GORM), provides a query | `internal/proc` | Low-level /proc parsing (stat, status, cgroup) | | `internal/cgroup` | Parses CGROUP_LIMITS and CGROUP_PROCESS_MAP env vars | | `internal/summary` | Accumulates samples, computes stats, pushes to receiver | -| `internal/receiver` | HTTP handlers, SQLite store, and sizer logic | +| `internal/receiver` | HTTP handlers and SQLite store | | `internal/output` | Metrics output formatting (JSON/text) | ### Container Metrics diff --git a/Dockerfile b/Dockerfile index 61ae4e0..a314fb1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,11 @@ FROM builder-base AS builder-collector RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /optimiser ./cmd/collector -# Receiver build +# Receiver build (CGO needed for SQLite) FROM builder-base AS builder-receiver -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /metrics-receiver ./cmd/receiver +RUN apk add --no-cache gcc musl-dev +RUN CGO_ENABLED=1 GOOS=linux go build -ldflags="-s -w" -o /metrics-receiver ./cmd/receiver # Collector image FROM alpine:3.19 AS collector @@ -27,6 +28,8 @@ ENTRYPOINT ["/usr/local/bin/optimiser"] # Receiver image FROM alpine:3.19 AS receiver +RUN apk add --no-cache sqlite-libs + COPY --from=builder-receiver /metrics-receiver /usr/local/bin/metrics-receiver EXPOSE 8080 diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index dc792e1..5810592 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,5 +1,4 @@ FROM gcr.io/distroless/static:nonroot ARG TARGETPLATFORM -ARG BINARY -COPY ${TARGETPLATFORM}/${BINARY} /app -ENTRYPOINT ["/app"] +COPY ${TARGETPLATFORM}/optimiser /optimiser +ENTRYPOINT ["/optimiser"] diff --git a/README.md b/README.md index fc7fb7b..abd58e9 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# Forgejo Runner Optimiser +# Forgejo Runner Resource Collector -A resource optimiser for CI/CD workloads in shared PID namespace environments. The **collector** reads `/proc` to gather CPU and memory metrics grouped by container/cgroup, and pushes run summaries to the **receiver**. The receiver stores metrics and exposes a **sizer** API that computes right-sized Kubernetes resource requests and limits from historical data. +A lightweight metrics collector for CI/CD workloads in shared PID namespace environments. Reads `/proc` to collect CPU and memory metrics, groups them by container/cgroup, and pushes run summaries to a receiver service for storage and querying. ## Architecture -The system has two binaries — a **collector** and a **receiver** (which includes the sizer): +The system has two independent binaries: ``` ┌─────────────────────────────────────────────┐ ┌──────────────────────────┐ @@ -19,9 +19,7 @@ The system has two binaries — a **collector** and a **receiver** (which includ │ └───────────┘ └────────┘ └───────────┘ │ │ │ │ │ │ │ ▼ │ └─────────────────────────────────────────────┘ │ GET /api/v1/metrics/... │ -│ GET /api/v1/sizing/... │ -│ (sizer) │ -└──────────────────────────┘ + └──────────────────────────┘ ``` ### Collector @@ -58,9 +56,9 @@ Runs as a sidecar alongside CI workloads. On a configurable interval, it reads ` CPU supports Kubernetes notation (`"2"` = 2 cores, `"500m"` = 0.5 cores). Memory supports `Ki`, `Mi`, `Gi`, `Ti` (binary) or `K`, `M`, `G`, `T` (decimal). -### Receiver (with sizer) +### Receiver -HTTP service that stores metric summaries in SQLite (via GORM), exposes a query API, and provides a **sizer** endpoint that computes right-sized Kubernetes resource requests and limits from historical run data. +HTTP service that stores metric summaries in SQLite (via GORM) and exposes a query API. ```bash ./receiver --addr=:8080 --db=metrics.db --read-token=my-secret-token --hmac-key=my-hmac-key @@ -80,7 +78,6 @@ HTTP service that stores metric summaries in SQLite (via GORM), exposes a query - `POST /api/v1/metrics` — receive and store a metric summary (requires scoped push token) - `POST /api/v1/token` — generate a scoped push token (requires read token auth) - `GET /api/v1/metrics/repo/{org}/{repo}/{workflow}/{job}` — query stored metrics (requires read token auth) -- `GET /api/v1/sizing/repo/{org}/{repo}/{workflow}/{job}` — compute container sizes from historical data (requires read token auth) **Authentication:** @@ -235,7 +232,7 @@ PUSH_TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/token \ | `internal/cgroup` | Parses `CGROUP_PROCESS_MAP` and `CGROUP_LIMITS` env vars | | `internal/collector` | Orchestrates the collection loop and shutdown | | `internal/summary` | Accumulates samples, computes stats, pushes to receiver | -| `internal/receiver` | HTTP handlers, SQLite store, and sizer logic | +| `internal/receiver` | HTTP handlers and SQLite store | | `internal/output` | Metrics output formatting (JSON/text) | ## Background diff --git a/cmd/receiver/main.go b/cmd/receiver/main.go index c540736..42e688e 100644 --- a/cmd/receiver/main.go +++ b/cmd/receiver/main.go @@ -24,7 +24,6 @@ func main() { dbPath := flag.String("db", defaultDBPath, "SQLite database path") readToken := flag.String("read-token", os.Getenv("RECEIVER_READ_TOKEN"), "Pre-shared token for read endpoints (or set RECEIVER_READ_TOKEN)") hmacKey := flag.String("hmac-key", os.Getenv("RECEIVER_HMAC_KEY"), "Secret key for push token generation/validation (or set RECEIVER_HMAC_KEY)") - tokenTTL := flag.Duration("token-ttl", 2*time.Hour, "Time-to-live for push tokens (default 2h)") flag.Parse() logger := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ @@ -38,7 +37,7 @@ func main() { } defer func() { _ = store.Close() }() - handler := receiver.NewHandler(store, logger, *readToken, *hmacKey, *tokenTTL) + handler := receiver.NewHandler(store, logger, *readToken, *hmacKey) mux := http.NewServeMux() handler.RegisterRoutes(mux) diff --git a/go.mod b/go.mod index 898904b..300d84c 100644 --- a/go.mod +++ b/go.mod @@ -3,22 +3,13 @@ module edp.buildth.ing/DevFW-CICD/forgejo-runner-optimiser go 1.25.6 require ( - github.com/glebarez/sqlite v1.11.0 + gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.1 ) require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/glebarez/go-sqlite v1.21.2 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.7.0 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect golang.org/x/text v0.20.0 // indirect - modernc.org/libc v1.22.5 // indirect - modernc.org/mathutil v1.5.0 // indirect - modernc.org/memory v1.5.0 // indirect - modernc.org/sqlite v1.23.1 // indirect ) diff --git a/go.sum b/go.sum index 95df11c..330dd09 100644 --- a/go.sum +++ b/go.sum @@ -1,34 +1,12 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= -github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= -github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= -github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= +gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= +gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= -modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= -modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= -modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= -modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= -modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index f21fa6f..326e3d5 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -32,7 +32,7 @@ func setupTestReceiver(t *testing.T) (*receiver.Store, *httptest.Server, func()) t.Fatalf("NewStore() error = %v", err) } - handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), testReadToken, testHMACKey, 0) + handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), testReadToken, testHMACKey) mux := http.NewServeMux() handler.RegisterRoutes(mux) @@ -46,9 +46,9 @@ func setupTestReceiver(t *testing.T) (*receiver.Store, *httptest.Server, func()) return store, server, cleanup } -// generatePushToken generates a push token for an execution context +// generatePushToken generates a scoped push token for an execution context func generatePushToken(exec summary.ExecutionContext) string { - return receiver.GenerateToken(testHMACKey, exec.Organization, exec.Repository, exec.Workflow, exec.Job) + return receiver.GenerateScopedToken(testHMACKey, exec.Organization, exec.Repository, exec.Workflow, exec.Job) } func TestPushClientToReceiver(t *testing.T) { @@ -166,8 +166,8 @@ func TestPushClientIntegration(t *testing.T) { t.Setenv("GITHUB_JOB", "push-job") t.Setenv("GITHUB_RUN_ID", "push-run-456") - // Generate push token - pushToken := receiver.GenerateToken(testHMACKey, "push-client-org", "push-client-repo", "push-test.yml", "push-job") + // Generate scoped push token + pushToken := receiver.GenerateScopedToken(testHMACKey, "push-client-org", "push-client-repo", "push-test.yml", "push-job") // Create push client with token - it reads execution context from env vars pushClient := summary.NewPushClient(server.URL+"/api/v1/metrics", pushToken) @@ -371,7 +371,7 @@ func setupTestReceiverWithToken(t *testing.T, readToken, hmacKey string) (*recei t.Fatalf("NewStore() error = %v", err) } - handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), readToken, hmacKey, 0) + handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), readToken, hmacKey) mux := http.NewServeMux() handler.RegisterRoutes(mux) diff --git a/internal/receiver/handler.go b/internal/receiver/handler.go index 57c09b5..d847f62 100644 --- a/internal/receiver/handler.go +++ b/internal/receiver/handler.go @@ -5,11 +5,9 @@ package receiver import ( "crypto/subtle" "encoding/json" - "fmt" "log/slog" "net/http" "strings" - "time" ) // Handler handles HTTP requests for the metrics receiver @@ -18,18 +16,13 @@ type Handler struct { logger *slog.Logger readToken string // Pre-shared token for read endpoint authentication hmacKey string // Separate key for HMAC-based push token generation/validation - tokenTTL time.Duration } // NewHandler creates a new HTTP handler with the given store. // readToken authenticates read endpoints and the token generation endpoint. // hmacKey is the secret used to derive scoped push tokens. -// tokenTTL specifies how long push tokens are valid (0 uses DefaultTokenTTL). -func NewHandler(store *Store, logger *slog.Logger, readToken, hmacKey string, tokenTTL time.Duration) *Handler { - if tokenTTL == 0 { - tokenTTL = DefaultTokenTTL - } - return &Handler{store: store, logger: logger, readToken: readToken, hmacKey: hmacKey, tokenTTL: tokenTTL} +func NewHandler(store *Store, logger *slog.Logger, readToken, hmacKey string) *Handler { + return &Handler{store: store, logger: logger, readToken: readToken, hmacKey: hmacKey} } // RegisterRoutes registers all HTTP routes on the given mux @@ -37,7 +30,6 @@ func (h *Handler) RegisterRoutes(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/metrics", h.handleReceiveMetrics) mux.HandleFunc("POST /api/v1/token", h.handleGenerateToken) mux.HandleFunc("GET /api/v1/metrics/repo/{org}/{repo}/{workflow}/{job}", h.handleGetByWorkflowJob) - mux.HandleFunc("GET /api/v1/sizing/repo/{org}/{repo}/{workflow}/{job}", h.handleGetSizing) mux.HandleFunc("GET /health", h.handleHealth) } @@ -94,7 +86,7 @@ func (h *Handler) handleGenerateToken(w http.ResponseWriter, r *http.Request) { return } - token := GenerateToken(h.hmacKey, req.Organization, req.Repository, req.Workflow, req.Job) + token := GenerateScopedToken(h.hmacKey, req.Organization, req.Repository, req.Workflow, req.Job) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(TokenResponse{Token: token}) @@ -123,7 +115,7 @@ func (h *Handler) validatePushToken(w http.ResponseWriter, r *http.Request, exec } token := strings.TrimPrefix(authHeader, bearerPrefix) - if !ValidateToken(h.hmacKey, token, exec.Organization, exec.Repository, exec.Workflow, exec.Job, h.tokenTTL) { + if !ValidateScopedToken(h.hmacKey, token, exec.Organization, exec.Repository, exec.Workflow, exec.Job) { h.logger.Warn("invalid push token", slog.String("path", r.URL.Path)) http.Error(w, "invalid token", http.StatusUnauthorized) return false @@ -202,71 +194,3 @@ func (h *Handler) handleHealth(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) } - -func (h *Handler) handleGetSizing(w http.ResponseWriter, r *http.Request) { - if !h.validateReadToken(w, r) { - return - } - - org := r.PathValue("org") - repo := r.PathValue("repo") - workflow := r.PathValue("workflow") - job := r.PathValue("job") - if org == "" || repo == "" || workflow == "" || job == "" { - http.Error(w, "org, repo, workflow and job are required", http.StatusBadRequest) - return - } - - // Parse query parameters with defaults - runs := parseIntQueryParam(r, "runs", 5, 1, 100) - buffer := parseIntQueryParam(r, "buffer", 20, 0, 100) - cpuPercentile := r.URL.Query().Get("cpu_percentile") - if cpuPercentile == "" { - cpuPercentile = "p95" - } - if !IsValidPercentile(cpuPercentile) { - http.Error(w, "invalid cpu_percentile: must be one of peak, p99, p95, p75, p50, avg", http.StatusBadRequest) - return - } - - metrics, err := h.store.GetRecentMetricsByWorkflowJob(org, repo, workflow, job, runs) - if err != nil { - h.logger.Error("failed to get metrics", slog.String("error", err.Error())) - http.Error(w, "failed to get metrics", http.StatusInternalServerError) - return - } - - if len(metrics) == 0 { - http.Error(w, "no metrics found for this workflow/job", http.StatusNotFound) - return - } - - response, err := computeSizing(metrics, buffer, cpuPercentile) - if err != nil { - h.logger.Error("failed to compute sizing", slog.String("error", err.Error())) - http.Error(w, "failed to compute sizing", http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - _ = json.NewEncoder(w).Encode(response) -} - -// parseIntQueryParam parses an integer query parameter with default, min, and max values -func parseIntQueryParam(r *http.Request, name string, defaultVal, minVal, maxVal int) int { - strVal := r.URL.Query().Get(name) - if strVal == "" { - return defaultVal - } - var val int - if _, err := fmt.Sscanf(strVal, "%d", &val); err != nil { - return defaultVal - } - if val < minVal { - return minVal - } - if val > maxVal { - return maxVal - } - return val -} diff --git a/internal/receiver/handler_test.go b/internal/receiver/handler_test.go index 12b327e..70d12d9 100644 --- a/internal/receiver/handler_test.go +++ b/internal/receiver/handler_test.go @@ -8,7 +8,6 @@ import ( "net/http" "net/http/httptest" "path/filepath" - "strings" "testing" "edp.buildth.ing/DevFW-CICD/forgejo-runner-optimiser/internal/summary" @@ -26,7 +25,7 @@ func TestHandler_ReceiveMetrics(t *testing.T) { Job: "build", RunID: "run-123", } - pushToken := GenerateToken(readToken, exec.Organization, exec.Repository, exec.Workflow, exec.Job) + pushToken := GenerateScopedToken(readToken, exec.Organization, exec.Repository, exec.Workflow, exec.Job) payload := MetricsPayload{ Execution: exec, @@ -265,13 +264,8 @@ func TestHandler_GenerateToken(t *testing.T) { if resp.Token == "" { t.Error("expected non-empty token") } - // Token format is "timestamp:hmac" where hmac is 64 hex chars - parts := strings.SplitN(resp.Token, ":", 2) - if len(parts) != 2 { - t.Errorf("token should have format 'timestamp:hmac', got %q", resp.Token) - } - if len(parts[1]) != 64 { - t.Errorf("HMAC part length = %d, want 64", len(parts[1])) + if len(resp.Token) != 64 { + t.Errorf("token length = %d, want 64", len(resp.Token)) } } @@ -363,8 +357,8 @@ func TestHandler_ReceiveMetrics_WithPushToken(t *testing.T) { RunID: "run-1", } - validToken := GenerateToken(readToken, exec.Organization, exec.Repository, exec.Workflow, exec.Job) - wrongScopeToken := GenerateToken(readToken, "other-org", "repo", "ci.yml", "build") + validToken := GenerateScopedToken(readToken, exec.Organization, exec.Repository, exec.Workflow, exec.Job) + wrongScopeToken := GenerateScopedToken(readToken, "other-org", "repo", "ci.yml", "build") tests := []struct { name string @@ -454,7 +448,7 @@ func newTestHandler(t *testing.T) (*Handler, func()) { } logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - handler := NewHandler(store, logger, "", "", 0) // no auth — endpoints will reject + handler := NewHandler(store, logger, "", "") // no auth — endpoints will reject return handler, func() { _ = store.Close() } } @@ -473,7 +467,7 @@ func newTestHandlerWithKeys(t *testing.T, readToken, hmacKey string) (*Handler, } logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - handler := NewHandler(store, logger, readToken, hmacKey, 0) // 0 uses DefaultTokenTTL + handler := NewHandler(store, logger, readToken, hmacKey) return handler, func() { _ = store.Close() } } diff --git a/internal/receiver/sizing.go b/internal/receiver/sizing.go deleted file mode 100644 index 32951ab..0000000 --- a/internal/receiver/sizing.go +++ /dev/null @@ -1,221 +0,0 @@ -// ABOUTME: Computes ideal container sizes from historical run data. -// ABOUTME: Provides Kubernetes-style resource sizes. -package receiver - -import ( - "encoding/json" - "fmt" - "math" - "sort" - - "edp.buildth.ing/DevFW-CICD/forgejo-runner-optimiser/internal/summary" -) - -// ResourceSize holds Kubernetes-formatted resource values -type ResourceSize struct { - Request string `json:"request"` - Limit string `json:"limit"` -} - -// ContainerSizing holds computed sizing for a single container -type ContainerSizing struct { - Name string `json:"name"` - CPU ResourceSize `json:"cpu"` - Memory ResourceSize `json:"memory"` -} - -// SizingMeta provides context about the sizing calculation -type SizingMeta struct { - RunsAnalyzed int `json:"runs_analyzed"` - BufferPercent int `json:"buffer_percent"` - CPUPercentile string `json:"cpu_percentile"` -} - -// SizingResponse is the API response for the sizing endpoint -type SizingResponse struct { - Containers []ContainerSizing `json:"containers"` - Total struct { - CPU ResourceSize `json:"cpu"` - Memory ResourceSize `json:"memory"` - } `json:"total"` - Meta SizingMeta `json:"meta"` -} - -// validPercentiles lists the allowed percentile values -var validPercentiles = map[string]bool{ - "peak": true, - "p99": true, - "p95": true, - "p75": true, - "p50": true, - "avg": true, -} - -// IsValidPercentile checks if the given percentile string is valid -func IsValidPercentile(p string) bool { - return validPercentiles[p] -} - -// selectCPUValue extracts the appropriate value from StatSummary based on percentile -func selectCPUValue(stats summary.StatSummary, percentile string) float64 { - switch percentile { - case "peak": - return stats.Peak - case "p99": - return stats.P99 - case "p95": - return stats.P95 - case "p75": - return stats.P75 - case "p50": - return stats.P50 - case "avg": - return stats.Avg - default: - return stats.P95 // default to p95 - } -} - -// formatMemoryK8s converts bytes to Kubernetes memory format (Mi) -func formatMemoryK8s(bytes float64) string { - const Mi = 1024 * 1024 - return fmt.Sprintf("%.0fMi", math.Ceil(bytes/Mi)) -} - -// formatCPUK8s converts cores to Kubernetes CPU format (millicores or whole cores) -func formatCPUK8s(cores float64) string { - millicores := cores * 1000 - if millicores >= 1000 && math.Mod(millicores, 1000) == 0 { - return fmt.Sprintf("%.0f", cores) - } - return fmt.Sprintf("%.0fm", math.Ceil(millicores)) -} - -// roundUpMemoryLimit rounds bytes up to the next power of 2 in Mi -func roundUpMemoryLimit(bytes float64) float64 { - const Mi = 1024 * 1024 - if bytes <= 0 { - return Mi // minimum 1Mi - } - miValue := bytes / Mi - if miValue <= 1 { - return Mi // minimum 1Mi - } - // Find next power of 2 - power := math.Ceil(math.Log2(miValue)) - return math.Pow(2, power) * Mi -} - -// roundUpCPULimit rounds cores up to the next 0.5 increment -func roundUpCPULimit(cores float64) float64 { - if cores <= 0 { - return 0.5 // minimum 0.5 cores - } - return math.Ceil(cores*2) / 2 -} - -// containerAggregation holds accumulated stats for a single container across runs -type containerAggregation struct { - cpuValues []float64 - memoryPeaks []float64 -} - -// computeSizing calculates ideal container sizes from metrics -func computeSizing(metrics []Metric, bufferPercent int, cpuPercentile string) (*SizingResponse, error) { - if len(metrics) == 0 { - return nil, fmt.Errorf("no metrics provided") - } - - // Aggregate container stats across all runs - containerStats := make(map[string]*containerAggregation) - - for _, m := range metrics { - var runSummary summary.RunSummary - if err := json.Unmarshal([]byte(m.Payload), &runSummary); err != nil { - continue // skip invalid payloads - } - - for _, c := range runSummary.Containers { - if _, exists := containerStats[c.Name]; !exists { - containerStats[c.Name] = &containerAggregation{ - cpuValues: make([]float64, 0), - memoryPeaks: make([]float64, 0), - } - } - agg := containerStats[c.Name] - agg.cpuValues = append(agg.cpuValues, selectCPUValue(c.CPUCores, cpuPercentile)) - agg.memoryPeaks = append(agg.memoryPeaks, c.MemoryBytes.Peak) - } - } - - // Calculate sizing for each container - bufferMultiplier := 1.0 + float64(bufferPercent)/100.0 - var containers []ContainerSizing - var totalCPU, totalMemory float64 - - // Sort container names for consistent output - names := make([]string, 0, len(containerStats)) - for name := range containerStats { - names = append(names, name) - } - sort.Strings(names) - - for _, name := range names { - agg := containerStats[name] - - // CPU: max of selected percentile values across runs - maxCPU := 0.0 - for _, v := range agg.cpuValues { - if v > maxCPU { - maxCPU = v - } - } - - // Memory: peak of peaks - maxMemory := 0.0 - for _, v := range agg.memoryPeaks { - if v > maxMemory { - maxMemory = v - } - } - - // Apply buffer - cpuWithBuffer := maxCPU * bufferMultiplier - memoryWithBuffer := maxMemory * bufferMultiplier - - containers = append(containers, ContainerSizing{ - Name: name, - CPU: ResourceSize{ - Request: formatCPUK8s(cpuWithBuffer), - Limit: formatCPUK8s(roundUpCPULimit(cpuWithBuffer)), - }, - Memory: ResourceSize{ - Request: formatMemoryK8s(memoryWithBuffer), - Limit: formatMemoryK8s(roundUpMemoryLimit(memoryWithBuffer)), - }, - }) - - totalCPU += cpuWithBuffer - totalMemory += memoryWithBuffer - } - - response := &SizingResponse{ - Containers: containers, - Meta: SizingMeta{ - RunsAnalyzed: len(metrics), - BufferPercent: bufferPercent, - CPUPercentile: cpuPercentile, - }, - } - - response.Total.CPU = ResourceSize{ - Request: formatCPUK8s(totalCPU), - Limit: formatCPUK8s(roundUpCPULimit(totalCPU)), - } - response.Total.Memory = ResourceSize{ - Request: formatMemoryK8s(totalMemory), - Limit: formatMemoryK8s(roundUpMemoryLimit(totalMemory)), - } - - return response, nil -} diff --git a/internal/receiver/sizing_test.go b/internal/receiver/sizing_test.go deleted file mode 100644 index a1ac3c5..0000000 --- a/internal/receiver/sizing_test.go +++ /dev/null @@ -1,494 +0,0 @@ -package receiver - -import ( - "encoding/json" - "net/http" - "net/http/httptest" - "testing" - - "edp.buildth.ing/DevFW-CICD/forgejo-runner-optimiser/internal/summary" -) - -func TestFormatMemoryK8s(t *testing.T) { - tests := []struct { - bytes float64 - want string - }{ - {0, "0Mi"}, - {1024 * 1024, "1Mi"}, - {256 * 1024 * 1024, "256Mi"}, - {512 * 1024 * 1024, "512Mi"}, - {1024 * 1024 * 1024, "1024Mi"}, - {2 * 1024 * 1024 * 1024, "2048Mi"}, - {1.5 * 1024 * 1024 * 1024, "1536Mi"}, - {100 * 1024 * 1024, "100Mi"}, - } - - for _, tt := range tests { - got := formatMemoryK8s(tt.bytes) - if got != tt.want { - t.Errorf("formatMemoryK8s(%v) = %q, want %q", tt.bytes, got, tt.want) - } - } -} - -func TestFormatCPUK8s(t *testing.T) { - tests := []struct { - cores float64 - want string - }{ - {0, "0m"}, - {0.1, "100m"}, - {0.5, "500m"}, - {1.0, "1"}, - {1.5, "1500m"}, - {2.0, "2"}, - {2.5, "2500m"}, - {0.123, "123m"}, - } - - for _, tt := range tests { - got := formatCPUK8s(tt.cores) - if got != tt.want { - t.Errorf("formatCPUK8s(%v) = %q, want %q", tt.cores, got, tt.want) - } - } -} - -func TestRoundUpMemoryLimit(t *testing.T) { - Mi := float64(1024 * 1024) - tests := []struct { - bytes float64 - want float64 - }{ - {0, Mi}, // minimum 1Mi - {100, Mi}, // rounds up to 1Mi - {Mi, Mi}, // exactly 1Mi stays 1Mi - {1.5 * Mi, 2 * Mi}, - {200 * Mi, 256 * Mi}, - {300 * Mi, 512 * Mi}, - {600 * Mi, 1024 * Mi}, - } - - for _, tt := range tests { - got := roundUpMemoryLimit(tt.bytes) - if got != tt.want { - t.Errorf("roundUpMemoryLimit(%v) = %v, want %v", tt.bytes, got, tt.want) - } - } -} - -func TestRoundUpCPULimit(t *testing.T) { - tests := []struct { - cores float64 - want float64 - }{ - {0, 0.5}, // minimum 0.5 - {0.1, 0.5}, - {0.5, 0.5}, - {0.6, 1.0}, - {1.0, 1.0}, - {1.1, 1.5}, - {1.5, 1.5}, - {2.0, 2.0}, - {2.3, 2.5}, - } - - for _, tt := range tests { - got := roundUpCPULimit(tt.cores) - if got != tt.want { - t.Errorf("roundUpCPULimit(%v) = %v, want %v", tt.cores, got, tt.want) - } - } -} - -func TestSelectCPUValue(t *testing.T) { - stats := summary.StatSummary{ - Peak: 10.0, - P99: 9.0, - P95: 8.0, - P75: 6.0, - P50: 5.0, - Avg: 4.0, - } - - tests := []struct { - percentile string - want float64 - }{ - {"peak", 10.0}, - {"p99", 9.0}, - {"p95", 8.0}, - {"p75", 6.0}, - {"p50", 5.0}, - {"avg", 4.0}, - {"invalid", 8.0}, // defaults to p95 - } - - for _, tt := range tests { - got := selectCPUValue(stats, tt.percentile) - if got != tt.want { - t.Errorf("selectCPUValue(stats, %q) = %v, want %v", tt.percentile, got, tt.want) - } - } -} - -func TestIsValidPercentile(t *testing.T) { - valid := []string{"peak", "p99", "p95", "p75", "p50", "avg"} - for _, p := range valid { - if !IsValidPercentile(p) { - t.Errorf("IsValidPercentile(%q) = false, want true", p) - } - } - - invalid := []string{"p80", "p90", "max", ""} - for _, p := range invalid { - if IsValidPercentile(p) { - t.Errorf("IsValidPercentile(%q) = true, want false", p) - } - } -} - -func TestComputeSizing_SingleRun(t *testing.T) { - runSummary := summary.RunSummary{ - Containers: []summary.ContainerSummary{ - { - Name: "runner", - CPUCores: summary.StatSummary{Peak: 1.0, P99: 0.9, P95: 0.8, P75: 0.6, P50: 0.5, Avg: 0.4}, - MemoryBytes: summary.StatSummary{Peak: 512 * 1024 * 1024}, // 512Mi - }, - }, - } - - payload, _ := json.Marshal(runSummary) - metrics := []Metric{{Payload: string(payload)}} - - resp, err := computeSizing(metrics, 20, "p95") - if err != nil { - t.Fatalf("computeSizing() error = %v", err) - } - - if len(resp.Containers) != 1 { - t.Fatalf("got %d containers, want 1", len(resp.Containers)) - } - - c := resp.Containers[0] - if c.Name != "runner" { - t.Errorf("container name = %q, want %q", c.Name, "runner") - } - - // CPU: 0.8 * 1.2 = 0.96 -> 960m request, 1 limit - if c.CPU.Request != "960m" { - t.Errorf("CPU request = %q, want %q", c.CPU.Request, "960m") - } - if c.CPU.Limit != "1" { - t.Errorf("CPU limit = %q, want %q", c.CPU.Limit, "1") - } - - // Memory: 512Mi * 1.2 = 614.4Mi -> 615Mi request, 1024Mi limit - if c.Memory.Request != "615Mi" { - t.Errorf("Memory request = %q, want %q", c.Memory.Request, "615Mi") - } - if c.Memory.Limit != "1024Mi" { - t.Errorf("Memory limit = %q, want %q", c.Memory.Limit, "1024Mi") - } - - if resp.Meta.RunsAnalyzed != 1 { - t.Errorf("runs_analyzed = %d, want 1", resp.Meta.RunsAnalyzed) - } - if resp.Meta.BufferPercent != 20 { - t.Errorf("buffer_percent = %d, want 20", resp.Meta.BufferPercent) - } - if resp.Meta.CPUPercentile != "p95" { - t.Errorf("cpu_percentile = %q, want %q", resp.Meta.CPUPercentile, "p95") - } -} - -func TestComputeSizing_MultipleRuns(t *testing.T) { - // Run 1: lower values - run1 := summary.RunSummary{ - Containers: []summary.ContainerSummary{ - { - Name: "runner", - CPUCores: summary.StatSummary{Peak: 0.5, P95: 0.4}, - MemoryBytes: summary.StatSummary{Peak: 256 * 1024 * 1024}, - }, - }, - } - // Run 2: higher values (should be used) - run2 := summary.RunSummary{ - Containers: []summary.ContainerSummary{ - { - Name: "runner", - CPUCores: summary.StatSummary{Peak: 1.0, P95: 0.8}, - MemoryBytes: summary.StatSummary{Peak: 512 * 1024 * 1024}, - }, - }, - } - - payload1, _ := json.Marshal(run1) - payload2, _ := json.Marshal(run2) - metrics := []Metric{ - {Payload: string(payload1)}, - {Payload: string(payload2)}, - } - - resp, err := computeSizing(metrics, 0, "p95") // no buffer for easier math - if err != nil { - t.Fatalf("computeSizing() error = %v", err) - } - - c := resp.Containers[0] - - // CPU: max(0.4, 0.8) = 0.8 - if c.CPU.Request != "800m" { - t.Errorf("CPU request = %q, want %q", c.CPU.Request, "800m") - } - - // Memory: max(256, 512) = 512Mi - if c.Memory.Request != "512Mi" { - t.Errorf("Memory request = %q, want %q", c.Memory.Request, "512Mi") - } - - if resp.Meta.RunsAnalyzed != 2 { - t.Errorf("runs_analyzed = %d, want 2", resp.Meta.RunsAnalyzed) - } -} - -func TestComputeSizing_MultipleContainers(t *testing.T) { - runSummary := summary.RunSummary{ - Containers: []summary.ContainerSummary{ - { - Name: "runner", - CPUCores: summary.StatSummary{P95: 1.0}, - MemoryBytes: summary.StatSummary{Peak: 512 * 1024 * 1024}, - }, - { - Name: "dind", - CPUCores: summary.StatSummary{P95: 0.5}, - MemoryBytes: summary.StatSummary{Peak: 256 * 1024 * 1024}, - }, - }, - } - - payload, _ := json.Marshal(runSummary) - metrics := []Metric{{Payload: string(payload)}} - - resp, err := computeSizing(metrics, 0, "p95") - if err != nil { - t.Fatalf("computeSizing() error = %v", err) - } - - if len(resp.Containers) != 2 { - t.Fatalf("got %d containers, want 2", len(resp.Containers)) - } - - // Containers should be sorted alphabetically - if resp.Containers[0].Name != "dind" { - t.Errorf("first container = %q, want %q", resp.Containers[0].Name, "dind") - } - if resp.Containers[1].Name != "runner" { - t.Errorf("second container = %q, want %q", resp.Containers[1].Name, "runner") - } - - // Total should be sum - if resp.Total.CPU.Request != "1500m" { - t.Errorf("total CPU request = %q, want %q", resp.Total.CPU.Request, "1500m") - } - if resp.Total.Memory.Request != "768Mi" { - t.Errorf("total memory request = %q, want %q", resp.Total.Memory.Request, "768Mi") - } -} - -func TestComputeSizing_NoMetrics(t *testing.T) { - _, err := computeSizing([]Metric{}, 20, "p95") - if err == nil { - t.Error("computeSizing() with no metrics should return error") - } -} - -func TestHandler_GetSizing(t *testing.T) { - const readToken = "test-token" - h, cleanup := newTestHandlerWithToken(t, readToken) - defer cleanup() - - // Save metrics with container data - for i := 0; i < 3; i++ { - runSummary := summary.RunSummary{ - Containers: []summary.ContainerSummary{ - { - Name: "runner", - CPUCores: summary.StatSummary{Peak: 1.0, P99: 0.9, P95: 0.8, P75: 0.6, P50: 0.5, Avg: 0.4}, - MemoryBytes: summary.StatSummary{Peak: 512 * 1024 * 1024}, - }, - }, - } - payload := &MetricsPayload{ - Execution: ExecutionContext{ - Organization: "org", - Repository: "repo", - Workflow: "ci.yml", - Job: "build", - RunID: "run-" + string(rune('1'+i)), - }, - Summary: runSummary, - } - if _, err := h.store.SaveMetric(payload); err != nil { - t.Fatalf("SaveMetric() error = %v", err) - } - } - - req := httptest.NewRequest(http.MethodGet, "/api/v1/sizing/repo/org/repo/ci.yml/build", nil) - req.Header.Set("Authorization", "Bearer "+readToken) - rec := httptest.NewRecorder() - - mux := http.NewServeMux() - h.RegisterRoutes(mux) - mux.ServeHTTP(rec, req) - - if rec.Code != http.StatusOK { - t.Errorf("status = %d, want %d", rec.Code, http.StatusOK) - } - - var resp SizingResponse - if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { - t.Fatalf("failed to decode response: %v", err) - } - - if len(resp.Containers) != 1 { - t.Errorf("got %d containers, want 1", len(resp.Containers)) - } - if resp.Meta.RunsAnalyzed != 3 { - t.Errorf("runs_analyzed = %d, want 3", resp.Meta.RunsAnalyzed) - } - if resp.Meta.BufferPercent != 20 { - t.Errorf("buffer_percent = %d, want 20", resp.Meta.BufferPercent) - } - if resp.Meta.CPUPercentile != "p95" { - t.Errorf("cpu_percentile = %q, want %q", resp.Meta.CPUPercentile, "p95") - } -} - -func TestHandler_GetSizing_CustomParams(t *testing.T) { - const readToken = "test-token" - h, cleanup := newTestHandlerWithToken(t, readToken) - defer cleanup() - - // Save one metric - runSummary := summary.RunSummary{ - Containers: []summary.ContainerSummary{ - { - Name: "runner", - CPUCores: summary.StatSummary{Peak: 1.0, P99: 0.9, P95: 0.8, P75: 0.6, P50: 0.5, Avg: 0.4}, - MemoryBytes: summary.StatSummary{Peak: 512 * 1024 * 1024}, - }, - }, - } - payload := &MetricsPayload{ - Execution: ExecutionContext{Organization: "org", Repository: "repo", Workflow: "ci.yml", Job: "build", RunID: "run-1"}, - Summary: runSummary, - } - if _, err := h.store.SaveMetric(payload); err != nil { - t.Fatalf("SaveMetric() error = %v", err) - } - - req := httptest.NewRequest(http.MethodGet, "/api/v1/sizing/repo/org/repo/ci.yml/build?runs=10&buffer=10&cpu_percentile=p75", nil) - req.Header.Set("Authorization", "Bearer "+readToken) - rec := httptest.NewRecorder() - - mux := http.NewServeMux() - h.RegisterRoutes(mux) - mux.ServeHTTP(rec, req) - - if rec.Code != http.StatusOK { - t.Errorf("status = %d, want %d", rec.Code, http.StatusOK) - } - - var resp SizingResponse - if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { - t.Fatalf("failed to decode response: %v", err) - } - - if resp.Meta.BufferPercent != 10 { - t.Errorf("buffer_percent = %d, want 10", resp.Meta.BufferPercent) - } - if resp.Meta.CPUPercentile != "p75" { - t.Errorf("cpu_percentile = %q, want %q", resp.Meta.CPUPercentile, "p75") - } - - // CPU: 0.6 * 1.1 = 0.66 - c := resp.Containers[0] - if c.CPU.Request != "660m" { - t.Errorf("CPU request = %q, want %q", c.CPU.Request, "660m") - } -} - -func TestHandler_GetSizing_NotFound(t *testing.T) { - const readToken = "test-token" - h, cleanup := newTestHandlerWithToken(t, readToken) - defer cleanup() - - req := httptest.NewRequest(http.MethodGet, "/api/v1/sizing/repo/org/repo/ci.yml/build", nil) - req.Header.Set("Authorization", "Bearer "+readToken) - rec := httptest.NewRecorder() - - mux := http.NewServeMux() - h.RegisterRoutes(mux) - mux.ServeHTTP(rec, req) - - if rec.Code != http.StatusNotFound { - t.Errorf("status = %d, want %d", rec.Code, http.StatusNotFound) - } -} - -func TestHandler_GetSizing_InvalidPercentile(t *testing.T) { - const readToken = "test-token" - h, cleanup := newTestHandlerWithToken(t, readToken) - defer cleanup() - - req := httptest.NewRequest(http.MethodGet, "/api/v1/sizing/repo/org/repo/ci.yml/build?cpu_percentile=p80", nil) - req.Header.Set("Authorization", "Bearer "+readToken) - rec := httptest.NewRecorder() - - mux := http.NewServeMux() - h.RegisterRoutes(mux) - mux.ServeHTTP(rec, req) - - if rec.Code != http.StatusBadRequest { - t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest) - } -} - -func TestHandler_GetSizing_AuthRequired(t *testing.T) { - const readToken = "test-token" - h, cleanup := newTestHandlerWithToken(t, readToken) - defer cleanup() - - tests := []struct { - name string - authHeader string - wantCode int - }{ - {"no auth", "", http.StatusUnauthorized}, - {"wrong token", "Bearer wrong-token", http.StatusUnauthorized}, - {"valid token", "Bearer " + readToken, http.StatusNotFound}, // no metrics, but auth works - } - - mux := http.NewServeMux() - h.RegisterRoutes(mux) - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - req := httptest.NewRequest(http.MethodGet, "/api/v1/sizing/repo/org/repo/ci.yml/build", nil) - if tt.authHeader != "" { - req.Header.Set("Authorization", tt.authHeader) - } - rec := httptest.NewRecorder() - mux.ServeHTTP(rec, req) - - if rec.Code != tt.wantCode { - t.Errorf("status = %d, want %d", rec.Code, tt.wantCode) - } - }) - } -} diff --git a/internal/receiver/store.go b/internal/receiver/store.go index 48f853f..1b934de 100644 --- a/internal/receiver/store.go +++ b/internal/receiver/store.go @@ -7,7 +7,7 @@ import ( "fmt" "time" - "github.com/glebarez/sqlite" + "gorm.io/driver/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" ) @@ -103,16 +103,6 @@ func (s *Store) GetMetricsByWorkflowJob(org, repo, workflow, job string) ([]Metr return metrics, result.Error } -// GetRecentMetricsByWorkflowJob retrieves the last N metrics ordered by received_at DESC -func (s *Store) GetRecentMetricsByWorkflowJob(org, repo, workflow, job string, limit int) ([]Metric, error) { - var metrics []Metric - result := s.db.Where( - "organization = ? AND repository = ? AND workflow = ? AND job = ?", - org, repo, workflow, job, - ).Order("received_at DESC").Limit(limit).Find(&metrics) - return metrics, result.Error -} - // Close closes the database connection func (s *Store) Close() error { sqlDB, err := s.db.DB() diff --git a/internal/receiver/token.go b/internal/receiver/token.go index 47721fa..087546c 100644 --- a/internal/receiver/token.go +++ b/internal/receiver/token.go @@ -1,5 +1,5 @@ // ABOUTME: HMAC-SHA256 token generation and validation for scoped push authentication. -// ABOUTME: Tokens are derived from a key + scope + timestamp, enabling stateless validation with expiration. +// ABOUTME: Tokens are derived from a key + scope, enabling stateless validation without DB storage. package receiver import ( @@ -7,71 +7,19 @@ import ( "crypto/sha256" "crypto/subtle" "encoding/hex" - "fmt" - "strconv" - "strings" - "time" ) -// DefaultTokenTTL is the default time-to-live for push tokens. -const DefaultTokenTTL = 2 * time.Hour - -// GenerateToken creates a token with embedded timestamp for expiration support. -// Format: ":" -func GenerateToken(key, org, repo, workflow, job string) string { - return GenerateTokenAt(key, org, repo, workflow, job, time.Now()) -} - -// GenerateTokenAt creates a token with the specified timestamp. -// The HMAC input is "v1\x00\x00\x00\x00\x00". -func GenerateTokenAt(key, org, repo, workflow, job string, timestamp time.Time) string { - ts := strconv.FormatInt(timestamp.Unix(), 10) +// GenerateScopedToken computes an HMAC-SHA256 token scoped to a specific org/repo/workflow/job. +// The canonical input is "v1\x00\x00\x00\x00". +func GenerateScopedToken(key, org, repo, workflow, job string) string { mac := hmac.New(sha256.New, []byte(key)) - mac.Write([]byte("v1\x00" + org + "\x00" + repo + "\x00" + workflow + "\x00" + job + "\x00" + ts)) - return ts + ":" + hex.EncodeToString(mac.Sum(nil)) + mac.Write([]byte("v1\x00" + org + "\x00" + repo + "\x00" + workflow + "\x00" + job)) + return hex.EncodeToString(mac.Sum(nil)) } -// ValidateToken validates a token and checks expiration. -// Returns true if the token is valid and not expired. -func ValidateToken(key, token, org, repo, workflow, job string, ttl time.Duration) bool { - return ValidateTokenAt(key, token, org, repo, workflow, job, ttl, time.Now()) -} - -// ValidateTokenAt validates a token against a specific reference time. -func ValidateTokenAt(key, token, org, repo, workflow, job string, ttl time.Duration, now time.Time) bool { - parts := strings.SplitN(token, ":", 2) - if len(parts) != 2 { - return false - } - - tsStr, hmacHex := parts[0], parts[1] - ts, err := strconv.ParseInt(tsStr, 10, 64) - if err != nil { - return false - } - - tokenTime := time.Unix(ts, 0) - if now.Sub(tokenTime) > ttl { - return false - } - - // Recompute expected HMAC - mac := hmac.New(sha256.New, []byte(key)) - mac.Write([]byte("v1\x00" + org + "\x00" + repo + "\x00" + workflow + "\x00" + job + "\x00" + tsStr)) - expected := hex.EncodeToString(mac.Sum(nil)) - - return subtle.ConstantTimeCompare([]byte(hmacHex), []byte(expected)) == 1 -} - -// ParseTokenTimestamp extracts the timestamp from a timestamped token without validating it. -func ParseTokenTimestamp(token string) (time.Time, error) { - parts := strings.SplitN(token, ":", 2) - if len(parts) != 2 { - return time.Time{}, fmt.Errorf("invalid token format") - } - ts, err := strconv.ParseInt(parts[0], 10, 64) - if err != nil { - return time.Time{}, fmt.Errorf("invalid timestamp: %w", err) - } - return time.Unix(ts, 0), nil +// ValidateScopedToken checks whether a token matches the expected HMAC for the given scope. +// Uses constant-time comparison to prevent timing attacks. +func ValidateScopedToken(key, token, org, repo, workflow, job string) bool { + expected := GenerateScopedToken(key, org, repo, workflow, job) + return subtle.ConstantTimeCompare([]byte(token), []byte(expected)) == 1 } diff --git a/internal/receiver/token_test.go b/internal/receiver/token_test.go index 897ab1a..2140ecd 100644 --- a/internal/receiver/token_test.go +++ b/internal/receiver/token_test.go @@ -1,35 +1,20 @@ package receiver import ( - "strconv" - "strings" + "encoding/hex" "testing" - "time" ) -func TestGenerateToken_Format(t *testing.T) { - token := GenerateToken("key", "org", "repo", "wf", "job") - parts := strings.SplitN(token, ":", 2) - if len(parts) != 2 { - t.Fatalf("token should have format 'timestamp:hmac', got %q", token) - } - if len(parts[1]) != 64 { - t.Errorf("HMAC part length = %d, want 64", len(parts[1])) - } -} - -func TestGenerateTokenAt_Deterministic(t *testing.T) { - ts := time.Unix(1700000000, 0) - token1 := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) - token2 := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) +func TestGenerateScopedToken_Deterministic(t *testing.T) { + token1 := GenerateScopedToken("key", "org", "repo", "wf", "job") + token2 := GenerateScopedToken("key", "org", "repo", "wf", "job") if token1 != token2 { t.Errorf("tokens differ: %q vs %q", token1, token2) } } -func TestGenerateTokenAt_ScopePinning(t *testing.T) { - ts := time.Unix(1700000000, 0) - base := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) +func TestGenerateScopedToken_ScopePinning(t *testing.T) { + base := GenerateScopedToken("key", "org", "repo", "wf", "job") variants := []struct { name string @@ -46,7 +31,7 @@ func TestGenerateTokenAt_ScopePinning(t *testing.T) { for _, v := range variants { t.Run(v.name, func(t *testing.T) { - token := GenerateTokenAt("key", v.org, v.repo, v.wf, v.job, ts) + token := GenerateScopedToken("key", v.org, v.repo, v.wf, v.job) if token == base { t.Errorf("token for %s should differ from base", v.name) } @@ -54,127 +39,40 @@ func TestGenerateTokenAt_ScopePinning(t *testing.T) { } } -func TestGenerateTokenAt_DifferentKeys(t *testing.T) { - ts := time.Unix(1700000000, 0) - token1 := GenerateTokenAt("key-a", "org", "repo", "wf", "job", ts) - token2 := GenerateTokenAt("key-b", "org", "repo", "wf", "job", ts) +func TestGenerateScopedToken_DifferentKeys(t *testing.T) { + token1 := GenerateScopedToken("key-a", "org", "repo", "wf", "job") + token2 := GenerateScopedToken("key-b", "org", "repo", "wf", "job") if token1 == token2 { t.Error("different keys should produce different tokens") } } -func TestGenerateTokenAt_DifferentTimestamps(t *testing.T) { - ts1 := time.Unix(1700000000, 0) - ts2 := time.Unix(1700000001, 0) - token1 := GenerateTokenAt("key", "org", "repo", "wf", "job", ts1) - token2 := GenerateTokenAt("key", "org", "repo", "wf", "job", ts2) - if token1 == token2 { - t.Error("different timestamps should produce different tokens") +func TestGenerateScopedToken_ValidHex(t *testing.T) { + token := GenerateScopedToken("key", "org", "repo", "wf", "job") + if len(token) != 64 { + t.Errorf("token length = %d, want 64", len(token)) + } + if _, err := hex.DecodeString(token); err != nil { + t.Errorf("token is not valid hex: %v", err) } } -func TestValidateToken_Correct(t *testing.T) { - ts := time.Now() - token := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) - if !ValidateToken("key", token, "org", "repo", "wf", "job", 5*time.Minute) { - t.Error("ValidateToken should accept correct token") +func TestValidateScopedToken_Correct(t *testing.T) { + token := GenerateScopedToken("key", "org", "repo", "wf", "job") + if !ValidateScopedToken("key", token, "org", "repo", "wf", "job") { + t.Error("ValidateScopedToken should accept correct token") } } -func TestValidateToken_WrongToken(t *testing.T) { - if ValidateToken("key", "12345:deadbeef", "org", "repo", "wf", "job", 5*time.Minute) { - t.Error("ValidateToken should reject wrong token") +func TestValidateScopedToken_WrongToken(t *testing.T) { + if ValidateScopedToken("key", "deadbeef", "org", "repo", "wf", "job") { + t.Error("ValidateScopedToken should reject wrong token") } } -func TestValidateToken_WrongScope(t *testing.T) { - ts := time.Now() - token := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) - if ValidateToken("key", token, "org", "repo", "wf", "other-job", 5*time.Minute) { - t.Error("ValidateToken should reject token for different scope") - } -} - -func TestValidateToken_Expired(t *testing.T) { - ts := time.Now().Add(-10 * time.Minute) - token := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) - if ValidateToken("key", token, "org", "repo", "wf", "job", 5*time.Minute) { - t.Error("ValidateToken should reject expired token") - } -} - -func TestValidateTokenAt_NotExpired(t *testing.T) { - tokenTime := time.Unix(1700000000, 0) - token := GenerateTokenAt("key", "org", "repo", "wf", "job", tokenTime) - - // Validate at 4 minutes later (within 5 minute TTL) - now := tokenTime.Add(4 * time.Minute) - if !ValidateTokenAt("key", token, "org", "repo", "wf", "job", 5*time.Minute, now) { - t.Error("ValidateTokenAt should accept token within TTL") - } -} - -func TestValidateTokenAt_JustExpired(t *testing.T) { - tokenTime := time.Unix(1700000000, 0) - token := GenerateTokenAt("key", "org", "repo", "wf", "job", tokenTime) - - // Validate at 6 minutes later (beyond 5 minute TTL) - now := tokenTime.Add(6 * time.Minute) - if ValidateTokenAt("key", token, "org", "repo", "wf", "job", 5*time.Minute, now) { - t.Error("ValidateTokenAt should reject token beyond TTL") - } -} - -func TestValidateToken_InvalidFormat(t *testing.T) { - if ValidateToken("key", "no-colon-here", "org", "repo", "wf", "job", 5*time.Minute) { - t.Error("ValidateToken should reject token without colon") - } - if ValidateToken("key", "not-a-number:abc123", "org", "repo", "wf", "job", 5*time.Minute) { - t.Error("ValidateToken should reject token with invalid timestamp") - } -} - -func TestParseTokenTimestamp(t *testing.T) { - ts := time.Unix(1700000000, 0) - token := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) - - parsed, err := ParseTokenTimestamp(token) - if err != nil { - t.Fatalf("ParseTokenTimestamp failed: %v", err) - } - if !parsed.Equal(ts) { - t.Errorf("parsed timestamp = %v, want %v", parsed, ts) - } -} - -func TestParseTokenTimestamp_Invalid(t *testing.T) { - _, err := ParseTokenTimestamp("no-colon") - if err == nil { - t.Error("ParseTokenTimestamp should fail on missing colon") - } - - _, err = ParseTokenTimestamp("not-a-number:abc123") - if err == nil { - t.Error("ParseTokenTimestamp should fail on invalid timestamp") - } -} - -func TestValidateToken_TamperedTimestamp(t *testing.T) { - // Generate a valid token - ts := time.Now() - token := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) - - parts := strings.SplitN(token, ":", 2) - if len(parts) != 2 { - t.Fatalf("unexpected token format: %q", token) - } - hmacPart := parts[1] - - // Tamper with timestamp (e.g., attacker tries to extend token lifetime) - tamperedTimestamp := strconv.FormatInt(time.Now().Add(1*time.Hour).Unix(), 10) - tamperedToken := tamperedTimestamp + ":" + hmacPart - - if ValidateToken("key", tamperedToken, "org", "repo", "wf", "job", 5*time.Minute) { - t.Error("ValidateToken should reject token with tampered timestamp") +func TestValidateScopedToken_WrongScope(t *testing.T) { + token := GenerateScopedToken("key", "org", "repo", "wf", "job") + if ValidateScopedToken("key", token, "org", "repo", "wf", "other-job") { + t.Error("ValidateScopedToken should reject token for different scope") } }