From 7bb0819e48555ce55f3cec8ac9f98d2d34e59708 Mon Sep 17 00:00:00 2001 From: Martin McCaffery Date: Thu, 12 Feb 2026 11:47:51 +0100 Subject: [PATCH 1/6] ci: generate two separate binaries --- .goreleaser.yaml | 38 ++++++++++++++++++++++++++---- Dockerfile | 13 ++++------ Dockerfile.goreleaser | 5 ++-- Makefile | 4 ++-- go.mod | 13 ++++++++-- go.sum | 30 +++++++++++++++++++---- internal/receiver/store.go | 2 +- test/k8s/test-cgroup-grouping.yaml | 4 ++-- 8 files changed, 83 insertions(+), 26 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 7e27b2c..3f5f26e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,6 +1,6 @@ version: 2 -project_name: resource-collector +project_name: optimiser gitea_urls: api: "{{ .Env.GITHUB_SERVER_URL }}/api/v1" @@ -11,9 +11,21 @@ before: - go mod tidy builds: - - id: resource-collector + - id: collector main: ./cmd/collector - binary: resource-collector + binary: collector + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - id: receiver + main: ./cmd/receiver + binary: receiver env: - CGO_ENABLED=0 goos: @@ -37,12 +49,28 @@ snapshot: version_template: "{{ incpatch .Version }}-next" dockers_v2: - - images: - - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_ORG }}/resource-collector" + - id: collector + ids: + - collector + images: + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_ORG }}/forgejo-runner-optimiser-collector" tags: - "{{ .Version }}" - latest dockerfile: Dockerfile.goreleaser + build_args: + BINARY: collector + - id: receiver + ids: + - receiver + images: + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_ORG }}/forgejo-runner-optimiser-receiver" + tags: + - "{{ .Version }}" + - latest + dockerfile: Dockerfile.goreleaser + build_args: + BINARY: receiver changelog: sort: asc diff --git a/Dockerfile b/Dockerfile index 75f7b7f..61ae4e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,26 +10,23 @@ COPY . . # Collector build (no CGO needed) FROM builder-base AS builder-collector -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /resource-collector ./cmd/collector +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /optimiser ./cmd/collector -# Receiver build (CGO needed for SQLite) +# Receiver build FROM builder-base AS builder-receiver -RUN apk add --no-cache gcc musl-dev -RUN CGO_ENABLED=1 GOOS=linux go build -ldflags="-s -w" -o /metrics-receiver ./cmd/receiver +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /metrics-receiver ./cmd/receiver # Collector image FROM alpine:3.19 AS collector -COPY --from=builder-collector /resource-collector /usr/local/bin/resource-collector +COPY --from=builder-collector /optimiser /usr/local/bin/optimiser -ENTRYPOINT ["/usr/local/bin/resource-collector"] +ENTRYPOINT ["/usr/local/bin/optimiser"] # Receiver image FROM alpine:3.19 AS receiver -RUN apk add --no-cache sqlite-libs - COPY --from=builder-receiver /metrics-receiver /usr/local/bin/metrics-receiver EXPOSE 8080 diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index 69c2616..dc792e1 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,4 +1,5 @@ FROM gcr.io/distroless/static:nonroot ARG TARGETPLATFORM -COPY ${TARGETPLATFORM}/resource-collector /resource-collector -ENTRYPOINT ["/resource-collector"] +ARG BINARY +COPY ${TARGETPLATFORM}/${BINARY} /app +ENTRYPOINT ["/app"] diff --git a/Makefile b/Makefile index cb32d30..8bb918a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -# ABOUTME: Makefile for forgejo-runner-resource-collector project. +# ABOUTME: Makefile for forgejo-runner-optimiser project. # ABOUTME: Provides targets for building, formatting, linting, and testing. -BINARY_NAME := resource-collector +BINARY_NAME := optimiser CMD_PATH := ./cmd/collector GO := go GOLANGCI_LINT := $(GO) run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.6.2 diff --git a/go.mod b/go.mod index 300d84c..898904b 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,22 @@ module edp.buildth.ing/DevFW-CICD/forgejo-runner-optimiser go 1.25.6 require ( - gorm.io/driver/sqlite v1.6.0 + github.com/glebarez/sqlite v1.11.0 gorm.io/gorm v1.31.1 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/glebarez/go-sqlite v1.21.2 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.20.0 // indirect + modernc.org/libc v1.22.5 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/sqlite v1.23.1 // indirect ) diff --git a/go.sum b/go.sum index 330dd09..95df11c 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,34 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= +github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= +github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= +github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= -gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= -gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= +modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= +modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= diff --git a/internal/receiver/store.go b/internal/receiver/store.go index 1b934de..7b78487 100644 --- a/internal/receiver/store.go +++ b/internal/receiver/store.go @@ -7,7 +7,7 @@ import ( "fmt" "time" - "gorm.io/driver/sqlite" + "github.com/glebarez/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" ) diff --git a/test/k8s/test-cgroup-grouping.yaml b/test/k8s/test-cgroup-grouping.yaml index e46545b..4b2b1c1 100644 --- a/test/k8s/test-cgroup-grouping.yaml +++ b/test/k8s/test-cgroup-grouping.yaml @@ -55,7 +55,7 @@ spec: # Resource collector sidecar - name: collector - image: ghcr.io/your-org/forgejo-runner-resource-collector:latest # Replace with your image + image: ghcr.io/your-org/forgejo-runner-optimiser:latest # Replace with your image args: - --interval=5s - --top=3 @@ -121,7 +121,7 @@ spec: # Collector - name: collector - image: ghcr.io/your-org/forgejo-runner-resource-collector:latest # Replace with your image + image: ghcr.io/your-org/forgejo-runner-optimiser:latest # Replace with your image args: - --interval=2s - --top=5 From 8101e9b20e926c42157852a69d6722f41c3294bf Mon Sep 17 00:00:00 2001 From: Manuel Ganter Date: Fri, 13 Feb 2026 12:02:53 +0100 Subject: [PATCH 2/6] feat: added first iteration sizes not recomender --- internal/receiver/handler.go | 70 +++++ internal/receiver/sizing.go | 228 ++++++++++++++ internal/receiver/sizing_test.go | 494 +++++++++++++++++++++++++++++++ internal/receiver/store.go | 10 + 4 files changed, 802 insertions(+) create mode 100644 internal/receiver/sizing.go create mode 100644 internal/receiver/sizing_test.go diff --git a/internal/receiver/handler.go b/internal/receiver/handler.go index d847f62..eb1069d 100644 --- a/internal/receiver/handler.go +++ b/internal/receiver/handler.go @@ -5,6 +5,7 @@ package receiver import ( "crypto/subtle" "encoding/json" + "fmt" "log/slog" "net/http" "strings" @@ -30,6 +31,7 @@ func (h *Handler) RegisterRoutes(mux *http.ServeMux) { mux.HandleFunc("POST /api/v1/metrics", h.handleReceiveMetrics) mux.HandleFunc("POST /api/v1/token", h.handleGenerateToken) mux.HandleFunc("GET /api/v1/metrics/repo/{org}/{repo}/{workflow}/{job}", h.handleGetByWorkflowJob) + mux.HandleFunc("GET /api/v1/sizing/repo/{org}/{repo}/{workflow}/{job}", h.handleGetSizing) mux.HandleFunc("GET /health", h.handleHealth) } @@ -194,3 +196,71 @@ func (h *Handler) handleHealth(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{"status": "ok"}) } + +func (h *Handler) handleGetSizing(w http.ResponseWriter, r *http.Request) { + if !h.validateReadToken(w, r) { + return + } + + org := r.PathValue("org") + repo := r.PathValue("repo") + workflow := r.PathValue("workflow") + job := r.PathValue("job") + if org == "" || repo == "" || workflow == "" || job == "" { + http.Error(w, "org, repo, workflow and job are required", http.StatusBadRequest) + return + } + + // Parse query parameters with defaults + runs := parseIntQueryParam(r, "runs", 5, 1, 100) + buffer := parseIntQueryParam(r, "buffer", 20, 0, 100) + cpuPercentile := r.URL.Query().Get("cpu_percentile") + if cpuPercentile == "" { + cpuPercentile = "p95" + } + if !IsValidPercentile(cpuPercentile) { + http.Error(w, "invalid cpu_percentile: must be one of peak, p99, p95, p75, p50, avg", http.StatusBadRequest) + return + } + + metrics, err := h.store.GetRecentMetricsByWorkflowJob(org, repo, workflow, job, runs) + if err != nil { + h.logger.Error("failed to get metrics", slog.String("error", err.Error())) + http.Error(w, "failed to get metrics", http.StatusInternalServerError) + return + } + + if len(metrics) == 0 { + http.Error(w, "no metrics found for this workflow/job", http.StatusNotFound) + return + } + + response, err := computeSizing(metrics, buffer, cpuPercentile) + if err != nil { + h.logger.Error("failed to compute sizing", slog.String("error", err.Error())) + http.Error(w, "failed to compute sizing", http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(response) +} + +// parseIntQueryParam parses an integer query parameter with default, min, and max values +func parseIntQueryParam(r *http.Request, name string, defaultVal, minVal, maxVal int) int { + strVal := r.URL.Query().Get(name) + if strVal == "" { + return defaultVal + } + var val int + if _, err := fmt.Sscanf(strVal, "%d", &val); err != nil { + return defaultVal + } + if val < minVal { + return minVal + } + if val > maxVal { + return maxVal + } + return val +} diff --git a/internal/receiver/sizing.go b/internal/receiver/sizing.go new file mode 100644 index 0000000..f037585 --- /dev/null +++ b/internal/receiver/sizing.go @@ -0,0 +1,228 @@ +// ABOUTME: Computes ideal container sizes from historical run data. +// ABOUTME: Provides Kubernetes-style resource recommendations. +package receiver + +import ( + "encoding/json" + "fmt" + "math" + "sort" + + "edp.buildth.ing/DevFW-CICD/forgejo-runner-optimiser/internal/summary" +) + +// ResourceSize holds Kubernetes-formatted resource values +type ResourceSize struct { + Request string `json:"request"` + Limit string `json:"limit"` +} + +// ContainerSizing holds computed sizing for a single container +type ContainerSizing struct { + Name string `json:"name"` + CPU ResourceSize `json:"cpu"` + Memory ResourceSize `json:"memory"` +} + +// SizingMeta provides context about the sizing calculation +type SizingMeta struct { + RunsAnalyzed int `json:"runs_analyzed"` + BufferPercent int `json:"buffer_percent"` + CPUPercentile string `json:"cpu_percentile"` +} + +// SizingResponse is the API response for the sizing endpoint +type SizingResponse struct { + Containers []ContainerSizing `json:"containers"` + Total struct { + CPU ResourceSize `json:"cpu"` + Memory ResourceSize `json:"memory"` + } `json:"total"` + Meta SizingMeta `json:"meta"` +} + +// validPercentiles lists the allowed percentile values +var validPercentiles = map[string]bool{ + "peak": true, + "p99": true, + "p95": true, + "p75": true, + "p50": true, + "avg": true, +} + +// IsValidPercentile checks if the given percentile string is valid +func IsValidPercentile(p string) bool { + return validPercentiles[p] +} + +// selectCPUValue extracts the appropriate value from StatSummary based on percentile +func selectCPUValue(stats summary.StatSummary, percentile string) float64 { + switch percentile { + case "peak": + return stats.Peak + case "p99": + return stats.P99 + case "p95": + return stats.P95 + case "p75": + return stats.P75 + case "p50": + return stats.P50 + case "avg": + return stats.Avg + default: + return stats.P95 // default to p95 + } +} + +// formatMemoryK8s converts bytes to Kubernetes memory format (Mi or Gi) +func formatMemoryK8s(bytes float64) string { + const ( + Mi = 1024 * 1024 + Gi = 1024 * 1024 * 1024 + ) + + if bytes >= Gi { + return fmt.Sprintf("%.0fGi", math.Ceil(bytes/Gi)) + } + return fmt.Sprintf("%.0fMi", math.Ceil(bytes/Mi)) +} + +// formatCPUK8s converts cores to Kubernetes CPU format (millicores or whole cores) +func formatCPUK8s(cores float64) string { + millicores := cores * 1000 + if millicores >= 1000 && math.Mod(millicores, 1000) == 0 { + return fmt.Sprintf("%.0f", cores) + } + return fmt.Sprintf("%.0fm", math.Ceil(millicores)) +} + +// roundUpMemoryLimit rounds bytes up to the next power of 2 in Mi +func roundUpMemoryLimit(bytes float64) float64 { + const Mi = 1024 * 1024 + if bytes <= 0 { + return Mi // minimum 1Mi + } + miValue := bytes / Mi + if miValue <= 1 { + return Mi // minimum 1Mi + } + // Find next power of 2 + power := math.Ceil(math.Log2(miValue)) + return math.Pow(2, power) * Mi +} + +// roundUpCPULimit rounds cores up to the next 0.5 increment +func roundUpCPULimit(cores float64) float64 { + if cores <= 0 { + return 0.5 // minimum 0.5 cores + } + return math.Ceil(cores*2) / 2 +} + +// containerAggregation holds accumulated stats for a single container across runs +type containerAggregation struct { + cpuValues []float64 + memoryPeaks []float64 +} + +// computeSizing calculates ideal container sizes from metrics +func computeSizing(metrics []Metric, bufferPercent int, cpuPercentile string) (*SizingResponse, error) { + if len(metrics) == 0 { + return nil, fmt.Errorf("no metrics provided") + } + + // Aggregate container stats across all runs + containerStats := make(map[string]*containerAggregation) + + for _, m := range metrics { + var runSummary summary.RunSummary + if err := json.Unmarshal([]byte(m.Payload), &runSummary); err != nil { + continue // skip invalid payloads + } + + for _, c := range runSummary.Containers { + if _, exists := containerStats[c.Name]; !exists { + containerStats[c.Name] = &containerAggregation{ + cpuValues: make([]float64, 0), + memoryPeaks: make([]float64, 0), + } + } + agg := containerStats[c.Name] + agg.cpuValues = append(agg.cpuValues, selectCPUValue(c.CPUCores, cpuPercentile)) + agg.memoryPeaks = append(agg.memoryPeaks, c.MemoryBytes.Peak) + } + } + + // Calculate sizing for each container + bufferMultiplier := 1.0 + float64(bufferPercent)/100.0 + var containers []ContainerSizing + var totalCPU, totalMemory float64 + + // Sort container names for consistent output + names := make([]string, 0, len(containerStats)) + for name := range containerStats { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + agg := containerStats[name] + + // CPU: max of selected percentile values across runs + maxCPU := 0.0 + for _, v := range agg.cpuValues { + if v > maxCPU { + maxCPU = v + } + } + + // Memory: peak of peaks + maxMemory := 0.0 + for _, v := range agg.memoryPeaks { + if v > maxMemory { + maxMemory = v + } + } + + // Apply buffer + cpuWithBuffer := maxCPU * bufferMultiplier + memoryWithBuffer := maxMemory * bufferMultiplier + + containers = append(containers, ContainerSizing{ + Name: name, + CPU: ResourceSize{ + Request: formatCPUK8s(cpuWithBuffer), + Limit: formatCPUK8s(roundUpCPULimit(cpuWithBuffer)), + }, + Memory: ResourceSize{ + Request: formatMemoryK8s(memoryWithBuffer), + Limit: formatMemoryK8s(roundUpMemoryLimit(memoryWithBuffer)), + }, + }) + + totalCPU += cpuWithBuffer + totalMemory += memoryWithBuffer + } + + response := &SizingResponse{ + Containers: containers, + Meta: SizingMeta{ + RunsAnalyzed: len(metrics), + BufferPercent: bufferPercent, + CPUPercentile: cpuPercentile, + }, + } + + response.Total.CPU = ResourceSize{ + Request: formatCPUK8s(totalCPU), + Limit: formatCPUK8s(roundUpCPULimit(totalCPU)), + } + response.Total.Memory = ResourceSize{ + Request: formatMemoryK8s(totalMemory), + Limit: formatMemoryK8s(roundUpMemoryLimit(totalMemory)), + } + + return response, nil +} diff --git a/internal/receiver/sizing_test.go b/internal/receiver/sizing_test.go new file mode 100644 index 0000000..fc16129 --- /dev/null +++ b/internal/receiver/sizing_test.go @@ -0,0 +1,494 @@ +package receiver + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "edp.buildth.ing/DevFW-CICD/forgejo-runner-optimiser/internal/summary" +) + +func TestFormatMemoryK8s(t *testing.T) { + tests := []struct { + bytes float64 + want string + }{ + {0, "0Mi"}, + {1024 * 1024, "1Mi"}, + {256 * 1024 * 1024, "256Mi"}, + {512 * 1024 * 1024, "512Mi"}, + {1024 * 1024 * 1024, "1Gi"}, + {2 * 1024 * 1024 * 1024, "2Gi"}, + {1.5 * 1024 * 1024 * 1024, "2Gi"}, // rounds up + {100 * 1024 * 1024, "100Mi"}, + } + + for _, tt := range tests { + got := formatMemoryK8s(tt.bytes) + if got != tt.want { + t.Errorf("formatMemoryK8s(%v) = %q, want %q", tt.bytes, got, tt.want) + } + } +} + +func TestFormatCPUK8s(t *testing.T) { + tests := []struct { + cores float64 + want string + }{ + {0, "0m"}, + {0.1, "100m"}, + {0.5, "500m"}, + {1.0, "1"}, + {1.5, "1500m"}, + {2.0, "2"}, + {2.5, "2500m"}, + {0.123, "123m"}, + } + + for _, tt := range tests { + got := formatCPUK8s(tt.cores) + if got != tt.want { + t.Errorf("formatCPUK8s(%v) = %q, want %q", tt.cores, got, tt.want) + } + } +} + +func TestRoundUpMemoryLimit(t *testing.T) { + Mi := float64(1024 * 1024) + tests := []struct { + bytes float64 + want float64 + }{ + {0, Mi}, // minimum 1Mi + {100, Mi}, // rounds up to 1Mi + {Mi, Mi}, // exactly 1Mi stays 1Mi + {1.5 * Mi, 2 * Mi}, + {200 * Mi, 256 * Mi}, + {300 * Mi, 512 * Mi}, + {600 * Mi, 1024 * Mi}, + } + + for _, tt := range tests { + got := roundUpMemoryLimit(tt.bytes) + if got != tt.want { + t.Errorf("roundUpMemoryLimit(%v) = %v, want %v", tt.bytes, got, tt.want) + } + } +} + +func TestRoundUpCPULimit(t *testing.T) { + tests := []struct { + cores float64 + want float64 + }{ + {0, 0.5}, // minimum 0.5 + {0.1, 0.5}, + {0.5, 0.5}, + {0.6, 1.0}, + {1.0, 1.0}, + {1.1, 1.5}, + {1.5, 1.5}, + {2.0, 2.0}, + {2.3, 2.5}, + } + + for _, tt := range tests { + got := roundUpCPULimit(tt.cores) + if got != tt.want { + t.Errorf("roundUpCPULimit(%v) = %v, want %v", tt.cores, got, tt.want) + } + } +} + +func TestSelectCPUValue(t *testing.T) { + stats := summary.StatSummary{ + Peak: 10.0, + P99: 9.0, + P95: 8.0, + P75: 6.0, + P50: 5.0, + Avg: 4.0, + } + + tests := []struct { + percentile string + want float64 + }{ + {"peak", 10.0}, + {"p99", 9.0}, + {"p95", 8.0}, + {"p75", 6.0}, + {"p50", 5.0}, + {"avg", 4.0}, + {"invalid", 8.0}, // defaults to p95 + } + + for _, tt := range tests { + got := selectCPUValue(stats, tt.percentile) + if got != tt.want { + t.Errorf("selectCPUValue(stats, %q) = %v, want %v", tt.percentile, got, tt.want) + } + } +} + +func TestIsValidPercentile(t *testing.T) { + valid := []string{"peak", "p99", "p95", "p75", "p50", "avg"} + for _, p := range valid { + if !IsValidPercentile(p) { + t.Errorf("IsValidPercentile(%q) = false, want true", p) + } + } + + invalid := []string{"p80", "p90", "max", ""} + for _, p := range invalid { + if IsValidPercentile(p) { + t.Errorf("IsValidPercentile(%q) = true, want false", p) + } + } +} + +func TestComputeSizing_SingleRun(t *testing.T) { + runSummary := summary.RunSummary{ + Containers: []summary.ContainerSummary{ + { + Name: "runner", + CPUCores: summary.StatSummary{Peak: 1.0, P99: 0.9, P95: 0.8, P75: 0.6, P50: 0.5, Avg: 0.4}, + MemoryBytes: summary.StatSummary{Peak: 512 * 1024 * 1024}, // 512Mi + }, + }, + } + + payload, _ := json.Marshal(runSummary) + metrics := []Metric{{Payload: string(payload)}} + + resp, err := computeSizing(metrics, 20, "p95") + if err != nil { + t.Fatalf("computeSizing() error = %v", err) + } + + if len(resp.Containers) != 1 { + t.Fatalf("got %d containers, want 1", len(resp.Containers)) + } + + c := resp.Containers[0] + if c.Name != "runner" { + t.Errorf("container name = %q, want %q", c.Name, "runner") + } + + // CPU: 0.8 * 1.2 = 0.96 -> 960m request, 1 limit + if c.CPU.Request != "960m" { + t.Errorf("CPU request = %q, want %q", c.CPU.Request, "960m") + } + if c.CPU.Limit != "1" { + t.Errorf("CPU limit = %q, want %q", c.CPU.Limit, "1") + } + + // Memory: 512Mi * 1.2 = 614.4Mi -> 615Mi request, 1Gi limit (1024Mi = 1Gi) + if c.Memory.Request != "615Mi" { + t.Errorf("Memory request = %q, want %q", c.Memory.Request, "615Mi") + } + if c.Memory.Limit != "1Gi" { + t.Errorf("Memory limit = %q, want %q", c.Memory.Limit, "1Gi") + } + + if resp.Meta.RunsAnalyzed != 1 { + t.Errorf("runs_analyzed = %d, want 1", resp.Meta.RunsAnalyzed) + } + if resp.Meta.BufferPercent != 20 { + t.Errorf("buffer_percent = %d, want 20", resp.Meta.BufferPercent) + } + if resp.Meta.CPUPercentile != "p95" { + t.Errorf("cpu_percentile = %q, want %q", resp.Meta.CPUPercentile, "p95") + } +} + +func TestComputeSizing_MultipleRuns(t *testing.T) { + // Run 1: lower values + run1 := summary.RunSummary{ + Containers: []summary.ContainerSummary{ + { + Name: "runner", + CPUCores: summary.StatSummary{Peak: 0.5, P95: 0.4}, + MemoryBytes: summary.StatSummary{Peak: 256 * 1024 * 1024}, + }, + }, + } + // Run 2: higher values (should be used) + run2 := summary.RunSummary{ + Containers: []summary.ContainerSummary{ + { + Name: "runner", + CPUCores: summary.StatSummary{Peak: 1.0, P95: 0.8}, + MemoryBytes: summary.StatSummary{Peak: 512 * 1024 * 1024}, + }, + }, + } + + payload1, _ := json.Marshal(run1) + payload2, _ := json.Marshal(run2) + metrics := []Metric{ + {Payload: string(payload1)}, + {Payload: string(payload2)}, + } + + resp, err := computeSizing(metrics, 0, "p95") // no buffer for easier math + if err != nil { + t.Fatalf("computeSizing() error = %v", err) + } + + c := resp.Containers[0] + + // CPU: max(0.4, 0.8) = 0.8 + if c.CPU.Request != "800m" { + t.Errorf("CPU request = %q, want %q", c.CPU.Request, "800m") + } + + // Memory: max(256, 512) = 512Mi + if c.Memory.Request != "512Mi" { + t.Errorf("Memory request = %q, want %q", c.Memory.Request, "512Mi") + } + + if resp.Meta.RunsAnalyzed != 2 { + t.Errorf("runs_analyzed = %d, want 2", resp.Meta.RunsAnalyzed) + } +} + +func TestComputeSizing_MultipleContainers(t *testing.T) { + runSummary := summary.RunSummary{ + Containers: []summary.ContainerSummary{ + { + Name: "runner", + CPUCores: summary.StatSummary{P95: 1.0}, + MemoryBytes: summary.StatSummary{Peak: 512 * 1024 * 1024}, + }, + { + Name: "dind", + CPUCores: summary.StatSummary{P95: 0.5}, + MemoryBytes: summary.StatSummary{Peak: 256 * 1024 * 1024}, + }, + }, + } + + payload, _ := json.Marshal(runSummary) + metrics := []Metric{{Payload: string(payload)}} + + resp, err := computeSizing(metrics, 0, "p95") + if err != nil { + t.Fatalf("computeSizing() error = %v", err) + } + + if len(resp.Containers) != 2 { + t.Fatalf("got %d containers, want 2", len(resp.Containers)) + } + + // Containers should be sorted alphabetically + if resp.Containers[0].Name != "dind" { + t.Errorf("first container = %q, want %q", resp.Containers[0].Name, "dind") + } + if resp.Containers[1].Name != "runner" { + t.Errorf("second container = %q, want %q", resp.Containers[1].Name, "runner") + } + + // Total should be sum + if resp.Total.CPU.Request != "1500m" { + t.Errorf("total CPU request = %q, want %q", resp.Total.CPU.Request, "1500m") + } + if resp.Total.Memory.Request != "768Mi" { + t.Errorf("total memory request = %q, want %q", resp.Total.Memory.Request, "768Mi") + } +} + +func TestComputeSizing_NoMetrics(t *testing.T) { + _, err := computeSizing([]Metric{}, 20, "p95") + if err == nil { + t.Error("computeSizing() with no metrics should return error") + } +} + +func TestHandler_GetSizing(t *testing.T) { + const readToken = "test-token" + h, cleanup := newTestHandlerWithToken(t, readToken) + defer cleanup() + + // Save metrics with container data + for i := 0; i < 3; i++ { + runSummary := summary.RunSummary{ + Containers: []summary.ContainerSummary{ + { + Name: "runner", + CPUCores: summary.StatSummary{Peak: 1.0, P99: 0.9, P95: 0.8, P75: 0.6, P50: 0.5, Avg: 0.4}, + MemoryBytes: summary.StatSummary{Peak: 512 * 1024 * 1024}, + }, + }, + } + payload := &MetricsPayload{ + Execution: ExecutionContext{ + Organization: "org", + Repository: "repo", + Workflow: "ci.yml", + Job: "build", + RunID: "run-" + string(rune('1'+i)), + }, + Summary: runSummary, + } + if _, err := h.store.SaveMetric(payload); err != nil { + t.Fatalf("SaveMetric() error = %v", err) + } + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sizing/repo/org/repo/ci.yml/build", nil) + req.Header.Set("Authorization", "Bearer "+readToken) + rec := httptest.NewRecorder() + + mux := http.NewServeMux() + h.RegisterRoutes(mux) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d", rec.Code, http.StatusOK) + } + + var resp SizingResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + + if len(resp.Containers) != 1 { + t.Errorf("got %d containers, want 1", len(resp.Containers)) + } + if resp.Meta.RunsAnalyzed != 3 { + t.Errorf("runs_analyzed = %d, want 3", resp.Meta.RunsAnalyzed) + } + if resp.Meta.BufferPercent != 20 { + t.Errorf("buffer_percent = %d, want 20", resp.Meta.BufferPercent) + } + if resp.Meta.CPUPercentile != "p95" { + t.Errorf("cpu_percentile = %q, want %q", resp.Meta.CPUPercentile, "p95") + } +} + +func TestHandler_GetSizing_CustomParams(t *testing.T) { + const readToken = "test-token" + h, cleanup := newTestHandlerWithToken(t, readToken) + defer cleanup() + + // Save one metric + runSummary := summary.RunSummary{ + Containers: []summary.ContainerSummary{ + { + Name: "runner", + CPUCores: summary.StatSummary{Peak: 1.0, P99: 0.9, P95: 0.8, P75: 0.6, P50: 0.5, Avg: 0.4}, + MemoryBytes: summary.StatSummary{Peak: 512 * 1024 * 1024}, + }, + }, + } + payload := &MetricsPayload{ + Execution: ExecutionContext{Organization: "org", Repository: "repo", Workflow: "ci.yml", Job: "build", RunID: "run-1"}, + Summary: runSummary, + } + if _, err := h.store.SaveMetric(payload); err != nil { + t.Fatalf("SaveMetric() error = %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sizing/repo/org/repo/ci.yml/build?runs=10&buffer=10&cpu_percentile=p75", nil) + req.Header.Set("Authorization", "Bearer "+readToken) + rec := httptest.NewRecorder() + + mux := http.NewServeMux() + h.RegisterRoutes(mux) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Errorf("status = %d, want %d", rec.Code, http.StatusOK) + } + + var resp SizingResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("failed to decode response: %v", err) + } + + if resp.Meta.BufferPercent != 10 { + t.Errorf("buffer_percent = %d, want 10", resp.Meta.BufferPercent) + } + if resp.Meta.CPUPercentile != "p75" { + t.Errorf("cpu_percentile = %q, want %q", resp.Meta.CPUPercentile, "p75") + } + + // CPU: 0.6 * 1.1 = 0.66 + c := resp.Containers[0] + if c.CPU.Request != "660m" { + t.Errorf("CPU request = %q, want %q", c.CPU.Request, "660m") + } +} + +func TestHandler_GetSizing_NotFound(t *testing.T) { + const readToken = "test-token" + h, cleanup := newTestHandlerWithToken(t, readToken) + defer cleanup() + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sizing/repo/org/repo/ci.yml/build", nil) + req.Header.Set("Authorization", "Bearer "+readToken) + rec := httptest.NewRecorder() + + mux := http.NewServeMux() + h.RegisterRoutes(mux) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusNotFound { + t.Errorf("status = %d, want %d", rec.Code, http.StatusNotFound) + } +} + +func TestHandler_GetSizing_InvalidPercentile(t *testing.T) { + const readToken = "test-token" + h, cleanup := newTestHandlerWithToken(t, readToken) + defer cleanup() + + req := httptest.NewRequest(http.MethodGet, "/api/v1/sizing/repo/org/repo/ci.yml/build?cpu_percentile=p80", nil) + req.Header.Set("Authorization", "Bearer "+readToken) + rec := httptest.NewRecorder() + + mux := http.NewServeMux() + h.RegisterRoutes(mux) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Errorf("status = %d, want %d", rec.Code, http.StatusBadRequest) + } +} + +func TestHandler_GetSizing_AuthRequired(t *testing.T) { + const readToken = "test-token" + h, cleanup := newTestHandlerWithToken(t, readToken) + defer cleanup() + + tests := []struct { + name string + authHeader string + wantCode int + }{ + {"no auth", "", http.StatusUnauthorized}, + {"wrong token", "Bearer wrong-token", http.StatusUnauthorized}, + {"valid token", "Bearer " + readToken, http.StatusNotFound}, // no metrics, but auth works + } + + mux := http.NewServeMux() + h.RegisterRoutes(mux) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/api/v1/sizing/repo/org/repo/ci.yml/build", nil) + if tt.authHeader != "" { + req.Header.Set("Authorization", tt.authHeader) + } + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + + if rec.Code != tt.wantCode { + t.Errorf("status = %d, want %d", rec.Code, tt.wantCode) + } + }) + } +} diff --git a/internal/receiver/store.go b/internal/receiver/store.go index 1b934de..7d81959 100644 --- a/internal/receiver/store.go +++ b/internal/receiver/store.go @@ -103,6 +103,16 @@ func (s *Store) GetMetricsByWorkflowJob(org, repo, workflow, job string) ([]Metr return metrics, result.Error } +// GetRecentMetricsByWorkflowJob retrieves the last N metrics ordered by received_at DESC +func (s *Store) GetRecentMetricsByWorkflowJob(org, repo, workflow, job string, limit int) ([]Metric, error) { + var metrics []Metric + result := s.db.Where( + "organization = ? AND repository = ? AND workflow = ? AND job = ?", + org, repo, workflow, job, + ).Order("received_at DESC").Limit(limit).Find(&metrics) + return metrics, result.Error +} + // Close closes the database connection func (s *Store) Close() error { sqlDB, err := s.db.DB() From a96a1079eb98c1f236a0def49b8343ac1a7d965b Mon Sep 17 00:00:00 2001 From: Manuel Ganter Date: Fri, 13 Feb 2026 12:30:32 +0100 Subject: [PATCH 3/6] fix: now sizer does not round up to the next Gi when in between two --- internal/receiver/sizing.go | 11 ++--------- internal/receiver/sizing_test.go | 12 ++++++------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/internal/receiver/sizing.go b/internal/receiver/sizing.go index f037585..928a2f5 100644 --- a/internal/receiver/sizing.go +++ b/internal/receiver/sizing.go @@ -76,16 +76,9 @@ func selectCPUValue(stats summary.StatSummary, percentile string) float64 { } } -// formatMemoryK8s converts bytes to Kubernetes memory format (Mi or Gi) +// formatMemoryK8s converts bytes to Kubernetes memory format (Mi) func formatMemoryK8s(bytes float64) string { - const ( - Mi = 1024 * 1024 - Gi = 1024 * 1024 * 1024 - ) - - if bytes >= Gi { - return fmt.Sprintf("%.0fGi", math.Ceil(bytes/Gi)) - } + const Mi = 1024 * 1024 return fmt.Sprintf("%.0fMi", math.Ceil(bytes/Mi)) } diff --git a/internal/receiver/sizing_test.go b/internal/receiver/sizing_test.go index fc16129..a1ac3c5 100644 --- a/internal/receiver/sizing_test.go +++ b/internal/receiver/sizing_test.go @@ -18,9 +18,9 @@ func TestFormatMemoryK8s(t *testing.T) { {1024 * 1024, "1Mi"}, {256 * 1024 * 1024, "256Mi"}, {512 * 1024 * 1024, "512Mi"}, - {1024 * 1024 * 1024, "1Gi"}, - {2 * 1024 * 1024 * 1024, "2Gi"}, - {1.5 * 1024 * 1024 * 1024, "2Gi"}, // rounds up + {1024 * 1024 * 1024, "1024Mi"}, + {2 * 1024 * 1024 * 1024, "2048Mi"}, + {1.5 * 1024 * 1024 * 1024, "1536Mi"}, {100 * 1024 * 1024, "100Mi"}, } @@ -185,12 +185,12 @@ func TestComputeSizing_SingleRun(t *testing.T) { t.Errorf("CPU limit = %q, want %q", c.CPU.Limit, "1") } - // Memory: 512Mi * 1.2 = 614.4Mi -> 615Mi request, 1Gi limit (1024Mi = 1Gi) + // Memory: 512Mi * 1.2 = 614.4Mi -> 615Mi request, 1024Mi limit if c.Memory.Request != "615Mi" { t.Errorf("Memory request = %q, want %q", c.Memory.Request, "615Mi") } - if c.Memory.Limit != "1Gi" { - t.Errorf("Memory limit = %q, want %q", c.Memory.Limit, "1Gi") + if c.Memory.Limit != "1024Mi" { + t.Errorf("Memory limit = %q, want %q", c.Memory.Limit, "1024Mi") } if resp.Meta.RunsAnalyzed != 1 { From 7e3a4efb2dca1f6085ba2a4201404a19430212c1 Mon Sep 17 00:00:00 2001 From: Manuel Ganter Date: Fri, 13 Feb 2026 12:48:57 +0100 Subject: [PATCH 4/6] feat: added timestamp to HMAC to allow a TTL for the token --- cmd/receiver/main.go | 3 +- internal/integration/integration_test.go | 12 +- internal/receiver/handler.go | 14 +- internal/receiver/handler_test.go | 20 ++- internal/receiver/token.go | 76 +++++++++-- internal/receiver/token_test.go | 158 +++++++++++++++++++---- 6 files changed, 225 insertions(+), 58 deletions(-) diff --git a/cmd/receiver/main.go b/cmd/receiver/main.go index 42e688e..c540736 100644 --- a/cmd/receiver/main.go +++ b/cmd/receiver/main.go @@ -24,6 +24,7 @@ func main() { dbPath := flag.String("db", defaultDBPath, "SQLite database path") readToken := flag.String("read-token", os.Getenv("RECEIVER_READ_TOKEN"), "Pre-shared token for read endpoints (or set RECEIVER_READ_TOKEN)") hmacKey := flag.String("hmac-key", os.Getenv("RECEIVER_HMAC_KEY"), "Secret key for push token generation/validation (or set RECEIVER_HMAC_KEY)") + tokenTTL := flag.Duration("token-ttl", 2*time.Hour, "Time-to-live for push tokens (default 2h)") flag.Parse() logger := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{ @@ -37,7 +38,7 @@ func main() { } defer func() { _ = store.Close() }() - handler := receiver.NewHandler(store, logger, *readToken, *hmacKey) + handler := receiver.NewHandler(store, logger, *readToken, *hmacKey, *tokenTTL) mux := http.NewServeMux() handler.RegisterRoutes(mux) diff --git a/internal/integration/integration_test.go b/internal/integration/integration_test.go index 326e3d5..f21fa6f 100644 --- a/internal/integration/integration_test.go +++ b/internal/integration/integration_test.go @@ -32,7 +32,7 @@ func setupTestReceiver(t *testing.T) (*receiver.Store, *httptest.Server, func()) t.Fatalf("NewStore() error = %v", err) } - handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), testReadToken, testHMACKey) + handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), testReadToken, testHMACKey, 0) mux := http.NewServeMux() handler.RegisterRoutes(mux) @@ -46,9 +46,9 @@ func setupTestReceiver(t *testing.T) (*receiver.Store, *httptest.Server, func()) return store, server, cleanup } -// generatePushToken generates a scoped push token for an execution context +// generatePushToken generates a push token for an execution context func generatePushToken(exec summary.ExecutionContext) string { - return receiver.GenerateScopedToken(testHMACKey, exec.Organization, exec.Repository, exec.Workflow, exec.Job) + return receiver.GenerateToken(testHMACKey, exec.Organization, exec.Repository, exec.Workflow, exec.Job) } func TestPushClientToReceiver(t *testing.T) { @@ -166,8 +166,8 @@ func TestPushClientIntegration(t *testing.T) { t.Setenv("GITHUB_JOB", "push-job") t.Setenv("GITHUB_RUN_ID", "push-run-456") - // Generate scoped push token - pushToken := receiver.GenerateScopedToken(testHMACKey, "push-client-org", "push-client-repo", "push-test.yml", "push-job") + // Generate push token + pushToken := receiver.GenerateToken(testHMACKey, "push-client-org", "push-client-repo", "push-test.yml", "push-job") // Create push client with token - it reads execution context from env vars pushClient := summary.NewPushClient(server.URL+"/api/v1/metrics", pushToken) @@ -371,7 +371,7 @@ func setupTestReceiverWithToken(t *testing.T, readToken, hmacKey string) (*recei t.Fatalf("NewStore() error = %v", err) } - handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), readToken, hmacKey) + handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), readToken, hmacKey, 0) mux := http.NewServeMux() handler.RegisterRoutes(mux) diff --git a/internal/receiver/handler.go b/internal/receiver/handler.go index eb1069d..57c09b5 100644 --- a/internal/receiver/handler.go +++ b/internal/receiver/handler.go @@ -9,6 +9,7 @@ import ( "log/slog" "net/http" "strings" + "time" ) // Handler handles HTTP requests for the metrics receiver @@ -17,13 +18,18 @@ type Handler struct { logger *slog.Logger readToken string // Pre-shared token for read endpoint authentication hmacKey string // Separate key for HMAC-based push token generation/validation + tokenTTL time.Duration } // NewHandler creates a new HTTP handler with the given store. // readToken authenticates read endpoints and the token generation endpoint. // hmacKey is the secret used to derive scoped push tokens. -func NewHandler(store *Store, logger *slog.Logger, readToken, hmacKey string) *Handler { - return &Handler{store: store, logger: logger, readToken: readToken, hmacKey: hmacKey} +// tokenTTL specifies how long push tokens are valid (0 uses DefaultTokenTTL). +func NewHandler(store *Store, logger *slog.Logger, readToken, hmacKey string, tokenTTL time.Duration) *Handler { + if tokenTTL == 0 { + tokenTTL = DefaultTokenTTL + } + return &Handler{store: store, logger: logger, readToken: readToken, hmacKey: hmacKey, tokenTTL: tokenTTL} } // RegisterRoutes registers all HTTP routes on the given mux @@ -88,7 +94,7 @@ func (h *Handler) handleGenerateToken(w http.ResponseWriter, r *http.Request) { return } - token := GenerateScopedToken(h.hmacKey, req.Organization, req.Repository, req.Workflow, req.Job) + token := GenerateToken(h.hmacKey, req.Organization, req.Repository, req.Workflow, req.Job) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(TokenResponse{Token: token}) @@ -117,7 +123,7 @@ func (h *Handler) validatePushToken(w http.ResponseWriter, r *http.Request, exec } token := strings.TrimPrefix(authHeader, bearerPrefix) - if !ValidateScopedToken(h.hmacKey, token, exec.Organization, exec.Repository, exec.Workflow, exec.Job) { + if !ValidateToken(h.hmacKey, token, exec.Organization, exec.Repository, exec.Workflow, exec.Job, h.tokenTTL) { h.logger.Warn("invalid push token", slog.String("path", r.URL.Path)) http.Error(w, "invalid token", http.StatusUnauthorized) return false diff --git a/internal/receiver/handler_test.go b/internal/receiver/handler_test.go index 70d12d9..12b327e 100644 --- a/internal/receiver/handler_test.go +++ b/internal/receiver/handler_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "path/filepath" + "strings" "testing" "edp.buildth.ing/DevFW-CICD/forgejo-runner-optimiser/internal/summary" @@ -25,7 +26,7 @@ func TestHandler_ReceiveMetrics(t *testing.T) { Job: "build", RunID: "run-123", } - pushToken := GenerateScopedToken(readToken, exec.Organization, exec.Repository, exec.Workflow, exec.Job) + pushToken := GenerateToken(readToken, exec.Organization, exec.Repository, exec.Workflow, exec.Job) payload := MetricsPayload{ Execution: exec, @@ -264,8 +265,13 @@ func TestHandler_GenerateToken(t *testing.T) { if resp.Token == "" { t.Error("expected non-empty token") } - if len(resp.Token) != 64 { - t.Errorf("token length = %d, want 64", len(resp.Token)) + // Token format is "timestamp:hmac" where hmac is 64 hex chars + parts := strings.SplitN(resp.Token, ":", 2) + if len(parts) != 2 { + t.Errorf("token should have format 'timestamp:hmac', got %q", resp.Token) + } + if len(parts[1]) != 64 { + t.Errorf("HMAC part length = %d, want 64", len(parts[1])) } } @@ -357,8 +363,8 @@ func TestHandler_ReceiveMetrics_WithPushToken(t *testing.T) { RunID: "run-1", } - validToken := GenerateScopedToken(readToken, exec.Organization, exec.Repository, exec.Workflow, exec.Job) - wrongScopeToken := GenerateScopedToken(readToken, "other-org", "repo", "ci.yml", "build") + validToken := GenerateToken(readToken, exec.Organization, exec.Repository, exec.Workflow, exec.Job) + wrongScopeToken := GenerateToken(readToken, "other-org", "repo", "ci.yml", "build") tests := []struct { name string @@ -448,7 +454,7 @@ func newTestHandler(t *testing.T) (*Handler, func()) { } logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - handler := NewHandler(store, logger, "", "") // no auth — endpoints will reject + handler := NewHandler(store, logger, "", "", 0) // no auth — endpoints will reject return handler, func() { _ = store.Close() } } @@ -467,7 +473,7 @@ func newTestHandlerWithKeys(t *testing.T, readToken, hmacKey string) (*Handler, } logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - handler := NewHandler(store, logger, readToken, hmacKey) + handler := NewHandler(store, logger, readToken, hmacKey, 0) // 0 uses DefaultTokenTTL return handler, func() { _ = store.Close() } } diff --git a/internal/receiver/token.go b/internal/receiver/token.go index 087546c..47721fa 100644 --- a/internal/receiver/token.go +++ b/internal/receiver/token.go @@ -1,5 +1,5 @@ // ABOUTME: HMAC-SHA256 token generation and validation for scoped push authentication. -// ABOUTME: Tokens are derived from a key + scope, enabling stateless validation without DB storage. +// ABOUTME: Tokens are derived from a key + scope + timestamp, enabling stateless validation with expiration. package receiver import ( @@ -7,19 +7,71 @@ import ( "crypto/sha256" "crypto/subtle" "encoding/hex" + "fmt" + "strconv" + "strings" + "time" ) -// GenerateScopedToken computes an HMAC-SHA256 token scoped to a specific org/repo/workflow/job. -// The canonical input is "v1\x00\x00\x00\x00". -func GenerateScopedToken(key, org, repo, workflow, job string) string { - mac := hmac.New(sha256.New, []byte(key)) - mac.Write([]byte("v1\x00" + org + "\x00" + repo + "\x00" + workflow + "\x00" + job)) - return hex.EncodeToString(mac.Sum(nil)) +// DefaultTokenTTL is the default time-to-live for push tokens. +const DefaultTokenTTL = 2 * time.Hour + +// GenerateToken creates a token with embedded timestamp for expiration support. +// Format: ":" +func GenerateToken(key, org, repo, workflow, job string) string { + return GenerateTokenAt(key, org, repo, workflow, job, time.Now()) } -// ValidateScopedToken checks whether a token matches the expected HMAC for the given scope. -// Uses constant-time comparison to prevent timing attacks. -func ValidateScopedToken(key, token, org, repo, workflow, job string) bool { - expected := GenerateScopedToken(key, org, repo, workflow, job) - return subtle.ConstantTimeCompare([]byte(token), []byte(expected)) == 1 +// GenerateTokenAt creates a token with the specified timestamp. +// The HMAC input is "v1\x00\x00\x00\x00\x00". +func GenerateTokenAt(key, org, repo, workflow, job string, timestamp time.Time) string { + ts := strconv.FormatInt(timestamp.Unix(), 10) + mac := hmac.New(sha256.New, []byte(key)) + mac.Write([]byte("v1\x00" + org + "\x00" + repo + "\x00" + workflow + "\x00" + job + "\x00" + ts)) + return ts + ":" + hex.EncodeToString(mac.Sum(nil)) +} + +// ValidateToken validates a token and checks expiration. +// Returns true if the token is valid and not expired. +func ValidateToken(key, token, org, repo, workflow, job string, ttl time.Duration) bool { + return ValidateTokenAt(key, token, org, repo, workflow, job, ttl, time.Now()) +} + +// ValidateTokenAt validates a token against a specific reference time. +func ValidateTokenAt(key, token, org, repo, workflow, job string, ttl time.Duration, now time.Time) bool { + parts := strings.SplitN(token, ":", 2) + if len(parts) != 2 { + return false + } + + tsStr, hmacHex := parts[0], parts[1] + ts, err := strconv.ParseInt(tsStr, 10, 64) + if err != nil { + return false + } + + tokenTime := time.Unix(ts, 0) + if now.Sub(tokenTime) > ttl { + return false + } + + // Recompute expected HMAC + mac := hmac.New(sha256.New, []byte(key)) + mac.Write([]byte("v1\x00" + org + "\x00" + repo + "\x00" + workflow + "\x00" + job + "\x00" + tsStr)) + expected := hex.EncodeToString(mac.Sum(nil)) + + return subtle.ConstantTimeCompare([]byte(hmacHex), []byte(expected)) == 1 +} + +// ParseTokenTimestamp extracts the timestamp from a timestamped token without validating it. +func ParseTokenTimestamp(token string) (time.Time, error) { + parts := strings.SplitN(token, ":", 2) + if len(parts) != 2 { + return time.Time{}, fmt.Errorf("invalid token format") + } + ts, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return time.Time{}, fmt.Errorf("invalid timestamp: %w", err) + } + return time.Unix(ts, 0), nil } diff --git a/internal/receiver/token_test.go b/internal/receiver/token_test.go index 2140ecd..897ab1a 100644 --- a/internal/receiver/token_test.go +++ b/internal/receiver/token_test.go @@ -1,20 +1,35 @@ package receiver import ( - "encoding/hex" + "strconv" + "strings" "testing" + "time" ) -func TestGenerateScopedToken_Deterministic(t *testing.T) { - token1 := GenerateScopedToken("key", "org", "repo", "wf", "job") - token2 := GenerateScopedToken("key", "org", "repo", "wf", "job") +func TestGenerateToken_Format(t *testing.T) { + token := GenerateToken("key", "org", "repo", "wf", "job") + parts := strings.SplitN(token, ":", 2) + if len(parts) != 2 { + t.Fatalf("token should have format 'timestamp:hmac', got %q", token) + } + if len(parts[1]) != 64 { + t.Errorf("HMAC part length = %d, want 64", len(parts[1])) + } +} + +func TestGenerateTokenAt_Deterministic(t *testing.T) { + ts := time.Unix(1700000000, 0) + token1 := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) + token2 := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) if token1 != token2 { t.Errorf("tokens differ: %q vs %q", token1, token2) } } -func TestGenerateScopedToken_ScopePinning(t *testing.T) { - base := GenerateScopedToken("key", "org", "repo", "wf", "job") +func TestGenerateTokenAt_ScopePinning(t *testing.T) { + ts := time.Unix(1700000000, 0) + base := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) variants := []struct { name string @@ -31,7 +46,7 @@ func TestGenerateScopedToken_ScopePinning(t *testing.T) { for _, v := range variants { t.Run(v.name, func(t *testing.T) { - token := GenerateScopedToken("key", v.org, v.repo, v.wf, v.job) + token := GenerateTokenAt("key", v.org, v.repo, v.wf, v.job, ts) if token == base { t.Errorf("token for %s should differ from base", v.name) } @@ -39,40 +54,127 @@ func TestGenerateScopedToken_ScopePinning(t *testing.T) { } } -func TestGenerateScopedToken_DifferentKeys(t *testing.T) { - token1 := GenerateScopedToken("key-a", "org", "repo", "wf", "job") - token2 := GenerateScopedToken("key-b", "org", "repo", "wf", "job") +func TestGenerateTokenAt_DifferentKeys(t *testing.T) { + ts := time.Unix(1700000000, 0) + token1 := GenerateTokenAt("key-a", "org", "repo", "wf", "job", ts) + token2 := GenerateTokenAt("key-b", "org", "repo", "wf", "job", ts) if token1 == token2 { t.Error("different keys should produce different tokens") } } -func TestGenerateScopedToken_ValidHex(t *testing.T) { - token := GenerateScopedToken("key", "org", "repo", "wf", "job") - if len(token) != 64 { - t.Errorf("token length = %d, want 64", len(token)) - } - if _, err := hex.DecodeString(token); err != nil { - t.Errorf("token is not valid hex: %v", err) +func TestGenerateTokenAt_DifferentTimestamps(t *testing.T) { + ts1 := time.Unix(1700000000, 0) + ts2 := time.Unix(1700000001, 0) + token1 := GenerateTokenAt("key", "org", "repo", "wf", "job", ts1) + token2 := GenerateTokenAt("key", "org", "repo", "wf", "job", ts2) + if token1 == token2 { + t.Error("different timestamps should produce different tokens") } } -func TestValidateScopedToken_Correct(t *testing.T) { - token := GenerateScopedToken("key", "org", "repo", "wf", "job") - if !ValidateScopedToken("key", token, "org", "repo", "wf", "job") { - t.Error("ValidateScopedToken should accept correct token") +func TestValidateToken_Correct(t *testing.T) { + ts := time.Now() + token := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) + if !ValidateToken("key", token, "org", "repo", "wf", "job", 5*time.Minute) { + t.Error("ValidateToken should accept correct token") } } -func TestValidateScopedToken_WrongToken(t *testing.T) { - if ValidateScopedToken("key", "deadbeef", "org", "repo", "wf", "job") { - t.Error("ValidateScopedToken should reject wrong token") +func TestValidateToken_WrongToken(t *testing.T) { + if ValidateToken("key", "12345:deadbeef", "org", "repo", "wf", "job", 5*time.Minute) { + t.Error("ValidateToken should reject wrong token") } } -func TestValidateScopedToken_WrongScope(t *testing.T) { - token := GenerateScopedToken("key", "org", "repo", "wf", "job") - if ValidateScopedToken("key", token, "org", "repo", "wf", "other-job") { - t.Error("ValidateScopedToken should reject token for different scope") +func TestValidateToken_WrongScope(t *testing.T) { + ts := time.Now() + token := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) + if ValidateToken("key", token, "org", "repo", "wf", "other-job", 5*time.Minute) { + t.Error("ValidateToken should reject token for different scope") + } +} + +func TestValidateToken_Expired(t *testing.T) { + ts := time.Now().Add(-10 * time.Minute) + token := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) + if ValidateToken("key", token, "org", "repo", "wf", "job", 5*time.Minute) { + t.Error("ValidateToken should reject expired token") + } +} + +func TestValidateTokenAt_NotExpired(t *testing.T) { + tokenTime := time.Unix(1700000000, 0) + token := GenerateTokenAt("key", "org", "repo", "wf", "job", tokenTime) + + // Validate at 4 minutes later (within 5 minute TTL) + now := tokenTime.Add(4 * time.Minute) + if !ValidateTokenAt("key", token, "org", "repo", "wf", "job", 5*time.Minute, now) { + t.Error("ValidateTokenAt should accept token within TTL") + } +} + +func TestValidateTokenAt_JustExpired(t *testing.T) { + tokenTime := time.Unix(1700000000, 0) + token := GenerateTokenAt("key", "org", "repo", "wf", "job", tokenTime) + + // Validate at 6 minutes later (beyond 5 minute TTL) + now := tokenTime.Add(6 * time.Minute) + if ValidateTokenAt("key", token, "org", "repo", "wf", "job", 5*time.Minute, now) { + t.Error("ValidateTokenAt should reject token beyond TTL") + } +} + +func TestValidateToken_InvalidFormat(t *testing.T) { + if ValidateToken("key", "no-colon-here", "org", "repo", "wf", "job", 5*time.Minute) { + t.Error("ValidateToken should reject token without colon") + } + if ValidateToken("key", "not-a-number:abc123", "org", "repo", "wf", "job", 5*time.Minute) { + t.Error("ValidateToken should reject token with invalid timestamp") + } +} + +func TestParseTokenTimestamp(t *testing.T) { + ts := time.Unix(1700000000, 0) + token := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) + + parsed, err := ParseTokenTimestamp(token) + if err != nil { + t.Fatalf("ParseTokenTimestamp failed: %v", err) + } + if !parsed.Equal(ts) { + t.Errorf("parsed timestamp = %v, want %v", parsed, ts) + } +} + +func TestParseTokenTimestamp_Invalid(t *testing.T) { + _, err := ParseTokenTimestamp("no-colon") + if err == nil { + t.Error("ParseTokenTimestamp should fail on missing colon") + } + + _, err = ParseTokenTimestamp("not-a-number:abc123") + if err == nil { + t.Error("ParseTokenTimestamp should fail on invalid timestamp") + } +} + +func TestValidateToken_TamperedTimestamp(t *testing.T) { + // Generate a valid token + ts := time.Now() + token := GenerateTokenAt("key", "org", "repo", "wf", "job", ts) + + parts := strings.SplitN(token, ":", 2) + if len(parts) != 2 { + t.Fatalf("unexpected token format: %q", token) + } + hmacPart := parts[1] + + // Tamper with timestamp (e.g., attacker tries to extend token lifetime) + tamperedTimestamp := strconv.FormatInt(time.Now().Add(1*time.Hour).Unix(), 10) + tamperedToken := tamperedTimestamp + ":" + hmacPart + + if ValidateToken("key", tamperedToken, "org", "repo", "wf", "job", 5*time.Minute) { + t.Error("ValidateToken should reject token with tampered timestamp") } } From 862fc073284cd3febec3877b28c102cffdf1df54 Mon Sep 17 00:00:00 2001 From: Martin McCaffery Date: Thu, 12 Feb 2026 11:47:51 +0100 Subject: [PATCH 5/6] ci: generate two separate binaries --- .goreleaser.yaml | 38 ++++++++++++++++++++++++++---- Dockerfile | 13 ++++------ Dockerfile.goreleaser | 5 ++-- Makefile | 4 ++-- go.mod | 13 ++++++++-- go.sum | 30 +++++++++++++++++++---- internal/receiver/store.go | 2 +- test/k8s/test-cgroup-grouping.yaml | 4 ++-- 8 files changed, 83 insertions(+), 26 deletions(-) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 7e27b2c..3f5f26e 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,6 +1,6 @@ version: 2 -project_name: resource-collector +project_name: optimiser gitea_urls: api: "{{ .Env.GITHUB_SERVER_URL }}/api/v1" @@ -11,9 +11,21 @@ before: - go mod tidy builds: - - id: resource-collector + - id: collector main: ./cmd/collector - binary: resource-collector + binary: collector + env: + - CGO_ENABLED=0 + goos: + - linux + goarch: + - amd64 + - arm64 + ldflags: + - -s -w + - id: receiver + main: ./cmd/receiver + binary: receiver env: - CGO_ENABLED=0 goos: @@ -37,12 +49,28 @@ snapshot: version_template: "{{ incpatch .Version }}-next" dockers_v2: - - images: - - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_ORG }}/resource-collector" + - id: collector + ids: + - collector + images: + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_ORG }}/forgejo-runner-optimiser-collector" tags: - "{{ .Version }}" - latest dockerfile: Dockerfile.goreleaser + build_args: + BINARY: collector + - id: receiver + ids: + - receiver + images: + - "{{ .Env.DOCKER_REGISTRY }}/{{ .Env.DOCKER_ORG }}/forgejo-runner-optimiser-receiver" + tags: + - "{{ .Version }}" + - latest + dockerfile: Dockerfile.goreleaser + build_args: + BINARY: receiver changelog: sort: asc diff --git a/Dockerfile b/Dockerfile index 75f7b7f..61ae4e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,26 +10,23 @@ COPY . . # Collector build (no CGO needed) FROM builder-base AS builder-collector -RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /resource-collector ./cmd/collector +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /optimiser ./cmd/collector -# Receiver build (CGO needed for SQLite) +# Receiver build FROM builder-base AS builder-receiver -RUN apk add --no-cache gcc musl-dev -RUN CGO_ENABLED=1 GOOS=linux go build -ldflags="-s -w" -o /metrics-receiver ./cmd/receiver +RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w" -o /metrics-receiver ./cmd/receiver # Collector image FROM alpine:3.19 AS collector -COPY --from=builder-collector /resource-collector /usr/local/bin/resource-collector +COPY --from=builder-collector /optimiser /usr/local/bin/optimiser -ENTRYPOINT ["/usr/local/bin/resource-collector"] +ENTRYPOINT ["/usr/local/bin/optimiser"] # Receiver image FROM alpine:3.19 AS receiver -RUN apk add --no-cache sqlite-libs - COPY --from=builder-receiver /metrics-receiver /usr/local/bin/metrics-receiver EXPOSE 8080 diff --git a/Dockerfile.goreleaser b/Dockerfile.goreleaser index 69c2616..dc792e1 100644 --- a/Dockerfile.goreleaser +++ b/Dockerfile.goreleaser @@ -1,4 +1,5 @@ FROM gcr.io/distroless/static:nonroot ARG TARGETPLATFORM -COPY ${TARGETPLATFORM}/resource-collector /resource-collector -ENTRYPOINT ["/resource-collector"] +ARG BINARY +COPY ${TARGETPLATFORM}/${BINARY} /app +ENTRYPOINT ["/app"] diff --git a/Makefile b/Makefile index cb32d30..8bb918a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ -# ABOUTME: Makefile for forgejo-runner-resource-collector project. +# ABOUTME: Makefile for forgejo-runner-optimiser project. # ABOUTME: Provides targets for building, formatting, linting, and testing. -BINARY_NAME := resource-collector +BINARY_NAME := optimiser CMD_PATH := ./cmd/collector GO := go GOLANGCI_LINT := $(GO) run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.6.2 diff --git a/go.mod b/go.mod index 300d84c..898904b 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,22 @@ module edp.buildth.ing/DevFW-CICD/forgejo-runner-optimiser go 1.25.6 require ( - gorm.io/driver/sqlite v1.6.0 + github.com/glebarez/sqlite v1.11.0 gorm.io/gorm v1.31.1 ) require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/glebarez/go-sqlite v1.21.2 // indirect + github.com/google/uuid v1.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.20.0 // indirect + modernc.org/libc v1.22.5 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/sqlite v1.23.1 // indirect ) diff --git a/go.sum b/go.sum index 330dd09..95df11c 100644 --- a/go.sum +++ b/go.sum @@ -1,12 +1,34 @@ +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo= +github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k= +github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw= +github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= -github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug= golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4= -gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ= -gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwyT8= gorm.io/gorm v1.31.1 h1:7CA8FTFz/gRfgqgpeKIBcervUn3xSyPUmr6B2WXJ7kg= gorm.io/gorm v1.31.1/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs= +modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= +modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= +modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= +modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= +modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM= +modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= diff --git a/internal/receiver/store.go b/internal/receiver/store.go index 7d81959..48f853f 100644 --- a/internal/receiver/store.go +++ b/internal/receiver/store.go @@ -7,7 +7,7 @@ import ( "fmt" "time" - "gorm.io/driver/sqlite" + "github.com/glebarez/sqlite" "gorm.io/gorm" "gorm.io/gorm/logger" ) diff --git a/test/k8s/test-cgroup-grouping.yaml b/test/k8s/test-cgroup-grouping.yaml index e46545b..4b2b1c1 100644 --- a/test/k8s/test-cgroup-grouping.yaml +++ b/test/k8s/test-cgroup-grouping.yaml @@ -55,7 +55,7 @@ spec: # Resource collector sidecar - name: collector - image: ghcr.io/your-org/forgejo-runner-resource-collector:latest # Replace with your image + image: ghcr.io/your-org/forgejo-runner-optimiser:latest # Replace with your image args: - --interval=5s - --top=3 @@ -121,7 +121,7 @@ spec: # Collector - name: collector - image: ghcr.io/your-org/forgejo-runner-resource-collector:latest # Replace with your image + image: ghcr.io/your-org/forgejo-runner-optimiser:latest # Replace with your image args: - --interval=2s - --top=5 From d0aea88a5b4345a72b893e50c7673d4f136ecec0 Mon Sep 17 00:00:00 2001 From: Martin McCaffery Date: Fri, 13 Feb 2026 16:42:37 +0100 Subject: [PATCH 6/6] refactor: Rename recommender to sizer --- CLAUDE.md | 7 ++++--- README.md | 17 ++++++++++------- internal/receiver/sizing.go | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f9d6972..6b6d7ef 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,7 +28,7 @@ make install-hooks # Install pre-commit and commit-msg hooks ## Architecture Overview -This is a Go metrics collector designed for CI/CD environments with shared PID namespaces. It consists of two binaries: +A resource optimiser for CI/CD environments with shared PID namespaces. It consists of two binaries — a **collector** and a **receiver** (which includes the **sizer**): ### Collector (`cmd/collector`) Runs alongside CI workloads, periodically reads `/proc` filesystem, and pushes a summary to the receiver on shutdown (SIGINT/SIGTERM). @@ -40,11 +40,12 @@ Runs alongside CI workloads, periodically reads `/proc` filesystem, and pushes a 4. On shutdown, `summary.PushClient` sends the summary to the receiver HTTP endpoint ### Receiver (`cmd/receiver`) -HTTP service that stores metric summaries in SQLite (via GORM) and provides a query API. +HTTP service that stores metric summaries in SQLite (via GORM), provides a query API, and includes the **sizer** — which computes right-sized Kubernetes resource requests and limits from historical data. **Key Endpoints:** - `POST /api/v1/metrics` - Receive metrics from collectors - `GET /api/v1/metrics/repo/{org}/{repo}/{workflow}/{job}` - Query stored metrics +- `GET /api/v1/sizing/repo/{org}/{repo}/{workflow}/{job}` - Compute container sizes from historical data ### Internal Packages @@ -55,7 +56,7 @@ HTTP service that stores metric summaries in SQLite (via GORM) and provides a qu | `internal/proc` | Low-level /proc parsing (stat, status, cgroup) | | `internal/cgroup` | Parses CGROUP_LIMITS and CGROUP_PROCESS_MAP env vars | | `internal/summary` | Accumulates samples, computes stats, pushes to receiver | -| `internal/receiver` | HTTP handlers and SQLite store | +| `internal/receiver` | HTTP handlers, SQLite store, and sizer logic | | `internal/output` | Metrics output formatting (JSON/text) | ### Container Metrics diff --git a/README.md b/README.md index abd58e9..fc7fb7b 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# Forgejo Runner Resource Collector +# Forgejo Runner Optimiser -A lightweight metrics collector for CI/CD workloads in shared PID namespace environments. Reads `/proc` to collect CPU and memory metrics, groups them by container/cgroup, and pushes run summaries to a receiver service for storage and querying. +A resource optimiser for CI/CD workloads in shared PID namespace environments. The **collector** reads `/proc` to gather CPU and memory metrics grouped by container/cgroup, and pushes run summaries to the **receiver**. The receiver stores metrics and exposes a **sizer** API that computes right-sized Kubernetes resource requests and limits from historical data. ## Architecture -The system has two independent binaries: +The system has two binaries — a **collector** and a **receiver** (which includes the sizer): ``` ┌─────────────────────────────────────────────┐ ┌──────────────────────────┐ @@ -19,7 +19,9 @@ The system has two independent binaries: │ └───────────┘ └────────┘ └───────────┘ │ │ │ │ │ │ │ ▼ │ └─────────────────────────────────────────────┘ │ GET /api/v1/metrics/... │ - └──────────────────────────┘ +│ GET /api/v1/sizing/... │ +│ (sizer) │ +└──────────────────────────┘ ``` ### Collector @@ -56,9 +58,9 @@ Runs as a sidecar alongside CI workloads. On a configurable interval, it reads ` CPU supports Kubernetes notation (`"2"` = 2 cores, `"500m"` = 0.5 cores). Memory supports `Ki`, `Mi`, `Gi`, `Ti` (binary) or `K`, `M`, `G`, `T` (decimal). -### Receiver +### Receiver (with sizer) -HTTP service that stores metric summaries in SQLite (via GORM) and exposes a query API. +HTTP service that stores metric summaries in SQLite (via GORM), exposes a query API, and provides a **sizer** endpoint that computes right-sized Kubernetes resource requests and limits from historical run data. ```bash ./receiver --addr=:8080 --db=metrics.db --read-token=my-secret-token --hmac-key=my-hmac-key @@ -78,6 +80,7 @@ HTTP service that stores metric summaries in SQLite (via GORM) and exposes a que - `POST /api/v1/metrics` — receive and store a metric summary (requires scoped push token) - `POST /api/v1/token` — generate a scoped push token (requires read token auth) - `GET /api/v1/metrics/repo/{org}/{repo}/{workflow}/{job}` — query stored metrics (requires read token auth) +- `GET /api/v1/sizing/repo/{org}/{repo}/{workflow}/{job}` — compute container sizes from historical data (requires read token auth) **Authentication:** @@ -232,7 +235,7 @@ PUSH_TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/token \ | `internal/cgroup` | Parses `CGROUP_PROCESS_MAP` and `CGROUP_LIMITS` env vars | | `internal/collector` | Orchestrates the collection loop and shutdown | | `internal/summary` | Accumulates samples, computes stats, pushes to receiver | -| `internal/receiver` | HTTP handlers and SQLite store | +| `internal/receiver` | HTTP handlers, SQLite store, and sizer logic | | `internal/output` | Metrics output formatting (JSON/text) | ## Background diff --git a/internal/receiver/sizing.go b/internal/receiver/sizing.go index 928a2f5..32951ab 100644 --- a/internal/receiver/sizing.go +++ b/internal/receiver/sizing.go @@ -1,5 +1,5 @@ // ABOUTME: Computes ideal container sizes from historical run data. -// ABOUTME: Provides Kubernetes-style resource recommendations. +// ABOUTME: Provides Kubernetes-style resource sizes. package receiver import (