This commit is contained in:
parent
aa3e8cddf9
commit
6e3242bbd8
GPG key ID:
7C4D0F375BCEE533
6 changed files with 91 additions and 41 deletions
57
README.md
57
README.md
|
|
@ -61,16 +61,17 @@ CPU supports Kubernetes notation (`"2"` = 2 cores, `"500m"` = 0.5 cores). Memory
|
|||
HTTP service that stores metric summaries in SQLite (via GORM) and exposes a query API.
|
||||
|
||||
```bash
|
||||
./receiver --addr=:8080 --db=metrics.db --read-token=my-secret-token
|
||||
./receiver --addr=:8080 --db=metrics.db --read-token=my-secret-token --hmac-key=my-hmac-key
|
||||
```
|
||||
|
||||
**Flags:**
|
||||
|
||||
| Flag | Environment Variable | Description | Default |
|
||||
| -------------- | --------------------- | ---------------------------------------------- | ------------ |
|
||||
| `--addr` | — | HTTP listen address | `:8080` |
|
||||
| `--db` | — | SQLite database path | `metrics.db` |
|
||||
| `--read-token` | `RECEIVER_READ_TOKEN` | Pre-shared token for authentication (required) | — |
|
||||
| Flag | Environment Variable | Description | Default |
|
||||
| -------------- | --------------------- | ----------------------------------------------------- | ------------ |
|
||||
| `--addr` | — | HTTP listen address | `:8080` |
|
||||
| `--db` | — | SQLite database path | `metrics.db` |
|
||||
| `--read-token` | `RECEIVER_READ_TOKEN` | Pre-shared token for read/admin endpoints (required) | — |
|
||||
| `--hmac-key` | `RECEIVER_HMAC_KEY` | Secret key for push token generation/validation (required) | — |
|
||||
|
||||
**Endpoints:**
|
||||
|
||||
|
|
@ -105,7 +106,7 @@ curl -H "Authorization: Bearer my-secret-token" \ #gitleaks:allow
|
|||
http://localhost:8080/api/v1/metrics/repo/my-org/my-repo/ci.yml/build
|
||||
```
|
||||
|
||||
Push tokens are HMAC-SHA256 digests derived from the read token and the scope (org/repo/workflow/job). They are stateless — no database storage is needed.
|
||||
Push tokens are HMAC-SHA256 digests derived from `--hmac-key` and the scope (org/repo/workflow/job). They are stateless — no database storage is needed. The HMAC key is separate from the read token so that compromising a push token does not expose the admin credential.
|
||||
|
||||
## How Metrics Are Collected
|
||||
|
||||
|
|
@ -175,11 +176,28 @@ All memory values are in **bytes**.
|
|||
### Docker Compose
|
||||
|
||||
```bash
|
||||
docker compose -f test/docker/docker-compose-stress.yaml up -d
|
||||
# Wait for collection, then trigger shutdown summary:
|
||||
# Start the receiver (builds image if needed):
|
||||
docker compose -f test/docker/docker-compose-stress.yaml up -d --build receiver
|
||||
|
||||
# Generate a scoped push token for the collector:
|
||||
PUSH_TOKEN=$(curl -s -X POST http://localhost:9080/api/v1/token \
|
||||
-H "Authorization: Bearer dummyreadtoken" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"organization":"test-org","repository":"test-org/stress-test","workflow":"stress-test-workflow","job":"heavy-workload"}' \
|
||||
| jq -r .token)
|
||||
|
||||
# Start the collector and stress workloads with the push token:
|
||||
COLLECTOR_PUSH_TOKEN=$PUSH_TOKEN \
|
||||
docker compose -f test/docker/docker-compose-stress.yaml up -d --build collector
|
||||
|
||||
# ... Wait for data collection ...
|
||||
|
||||
# Trigger shutdown summary:
|
||||
docker compose -f test/docker/docker-compose-stress.yaml stop collector
|
||||
# Query results:
|
||||
curl http://localhost:9080/api/v1/metrics/repo/test-org/test-org%2Fstress-test/stress-test-workflow/heavy-workload
|
||||
|
||||
# Query results with the read token:
|
||||
curl -H "Authorization: Bearer dummyreadtoken" \
|
||||
http://localhost:9080/api/v1/metrics/repo/test-org/test-org%2Fstress-test/stress-test-workflow/heavy-workload
|
||||
```
|
||||
|
||||
### Local
|
||||
|
|
@ -188,8 +206,21 @@ curl http://localhost:9080/api/v1/metrics/repo/test-org/test-org%2Fstress-test/s
|
|||
go build -o collector ./cmd/collector
|
||||
go build -o receiver ./cmd/receiver
|
||||
|
||||
./receiver --addr=:8080 --db=metrics.db
|
||||
./collector --interval=2s --top=10 --push-endpoint=http://localhost:8080/api/v1/metrics
|
||||
# Start receiver with both keys:
|
||||
./receiver --addr=:8080 --db=metrics.db \
|
||||
--read-token=my-secret-token --hmac-key=my-hmac-key
|
||||
|
||||
# Generate a scoped push token:
|
||||
PUSH_TOKEN=$(curl -s -X POST http://localhost:8080/api/v1/token \
|
||||
-H "Authorization: Bearer my-secret-token" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"organization":"my-org","repository":"my-repo","workflow":"ci.yml","job":"build"}' \
|
||||
| jq -r .token)
|
||||
|
||||
# Run collector with the push token:
|
||||
./collector --interval=2s --top=10 \
|
||||
--push-endpoint=http://localhost:8080/api/v1/metrics \
|
||||
--push-token=$PUSH_TOKEN
|
||||
```
|
||||
|
||||
## Internal Packages
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ func main() {
|
|||
addr := flag.String("addr", defaultAddr, "HTTP listen address")
|
||||
dbPath := flag.String("db", defaultDBPath, "SQLite database path")
|
||||
readToken := flag.String("read-token", os.Getenv("RECEIVER_READ_TOKEN"), "Pre-shared token for read endpoints (or set RECEIVER_READ_TOKEN)")
|
||||
hmacKey := flag.String("hmac-key", os.Getenv("RECEIVER_HMAC_KEY"), "Secret key for push token generation/validation (or set RECEIVER_HMAC_KEY)")
|
||||
flag.Parse()
|
||||
|
||||
logger := slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{
|
||||
|
|
@ -36,7 +37,7 @@ func main() {
|
|||
}
|
||||
defer func() { _ = store.Close() }()
|
||||
|
||||
handler := receiver.NewHandler(store, logger, *readToken)
|
||||
handler := receiver.NewHandler(store, logger, *readToken, *hmacKey)
|
||||
mux := http.NewServeMux()
|
||||
handler.RegisterRoutes(mux)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,10 @@ import (
|
|||
"edp.buildth.ing/DevFW-CICD/forgejo-runner-resource-collector/internal/summary"
|
||||
)
|
||||
|
||||
const testReadToken = "integration-test-token"
|
||||
const (
|
||||
testReadToken = "integration-test-token"
|
||||
testHMACKey = "integration-hmac-key"
|
||||
)
|
||||
|
||||
// setupTestReceiver creates a test receiver with SQLite storage, auth, and HTTP server
|
||||
func setupTestReceiver(t *testing.T) (*receiver.Store, *httptest.Server, func()) {
|
||||
|
|
@ -29,7 +32,7 @@ func setupTestReceiver(t *testing.T) (*receiver.Store, *httptest.Server, func())
|
|||
t.Fatalf("NewStore() error = %v", err)
|
||||
}
|
||||
|
||||
handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), testReadToken)
|
||||
handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), testReadToken, testHMACKey)
|
||||
mux := http.NewServeMux()
|
||||
handler.RegisterRoutes(mux)
|
||||
|
||||
|
|
@ -45,7 +48,7 @@ func setupTestReceiver(t *testing.T) (*receiver.Store, *httptest.Server, func())
|
|||
|
||||
// generatePushToken generates a scoped push token for an execution context
|
||||
func generatePushToken(exec summary.ExecutionContext) string {
|
||||
return receiver.GenerateScopedToken(testReadToken, exec.Organization, exec.Repository, exec.Workflow, exec.Job)
|
||||
return receiver.GenerateScopedToken(testHMACKey, exec.Organization, exec.Repository, exec.Workflow, exec.Job)
|
||||
}
|
||||
|
||||
func TestPushClientToReceiver(t *testing.T) {
|
||||
|
|
@ -164,7 +167,7 @@ func TestPushClientIntegration(t *testing.T) {
|
|||
t.Setenv("GITHUB_RUN_ID", "push-run-456")
|
||||
|
||||
// Generate scoped push token
|
||||
pushToken := receiver.GenerateScopedToken(testReadToken, "push-client-org", "push-client-repo", "push-test.yml", "push-job")
|
||||
pushToken := receiver.GenerateScopedToken(testHMACKey, "push-client-org", "push-client-repo", "push-test.yml", "push-job")
|
||||
|
||||
// Create push client with token - it reads execution context from env vars
|
||||
pushClient := summary.NewPushClient(server.URL+"/api/v1/metrics", pushToken)
|
||||
|
|
@ -274,8 +277,9 @@ func TestMultiplePushes(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestPushClientWithTokenIntegration(t *testing.T) {
|
||||
readToken := "integration-secret"
|
||||
store, server, cleanup := setupTestReceiverWithToken(t, readToken)
|
||||
readToken := "integration-read-secret"
|
||||
hmacKey := "integration-hmac-secret"
|
||||
store, server, cleanup := setupTestReceiverWithToken(t, readToken, hmacKey)
|
||||
defer cleanup()
|
||||
|
||||
// Generate a scoped token via the API
|
||||
|
|
@ -340,8 +344,9 @@ func TestPushClientWithTokenIntegration(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestPushClientWithWrongTokenIntegration(t *testing.T) {
|
||||
readToken := "integration-secret"
|
||||
_, server, cleanup := setupTestReceiverWithToken(t, readToken)
|
||||
readToken := "integration-read-secret"
|
||||
hmacKey := "integration-hmac-secret"
|
||||
_, server, cleanup := setupTestReceiverWithToken(t, readToken, hmacKey)
|
||||
defer cleanup()
|
||||
|
||||
t.Setenv("GITHUB_REPOSITORY_OWNER", "token-org")
|
||||
|
|
@ -358,7 +363,7 @@ func TestPushClientWithWrongTokenIntegration(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func setupTestReceiverWithToken(t *testing.T, token string) (*receiver.Store, *httptest.Server, func()) {
|
||||
func setupTestReceiverWithToken(t *testing.T, readToken, hmacKey string) (*receiver.Store, *httptest.Server, func()) {
|
||||
t.Helper()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
store, err := receiver.NewStore(dbPath)
|
||||
|
|
@ -366,7 +371,7 @@ func setupTestReceiverWithToken(t *testing.T, token string) (*receiver.Store, *h
|
|||
t.Fatalf("NewStore() error = %v", err)
|
||||
}
|
||||
|
||||
handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), token)
|
||||
handler := receiver.NewHandler(store, slog.New(slog.NewTextHandler(io.Discard, nil)), readToken, hmacKey)
|
||||
mux := http.NewServeMux()
|
||||
handler.RegisterRoutes(mux)
|
||||
|
||||
|
|
|
|||
|
|
@ -14,13 +14,15 @@ import (
|
|||
type Handler struct {
|
||||
store *Store
|
||||
logger *slog.Logger
|
||||
readToken string // Pre-shared token for authentication (required)
|
||||
readToken string // Pre-shared token for read endpoint authentication
|
||||
hmacKey string // Separate key for HMAC-based push token generation/validation
|
||||
}
|
||||
|
||||
// NewHandler creates a new HTTP handler with the given store.
|
||||
// readToken is required for authenticating all metrics endpoints.
|
||||
func NewHandler(store *Store, logger *slog.Logger, readToken string) *Handler {
|
||||
return &Handler{store: store, logger: logger, readToken: readToken}
|
||||
// readToken authenticates read endpoints and the token generation endpoint.
|
||||
// hmacKey is the secret used to derive scoped push tokens.
|
||||
func NewHandler(store *Store, logger *slog.Logger, readToken, hmacKey string) *Handler {
|
||||
return &Handler{store: store, logger: logger, readToken: readToken, hmacKey: hmacKey}
|
||||
}
|
||||
|
||||
// RegisterRoutes registers all HTTP routes on the given mux
|
||||
|
|
@ -64,8 +66,8 @@ func (h *Handler) validateReadToken(w http.ResponseWriter, r *http.Request) bool
|
|||
}
|
||||
|
||||
func (h *Handler) handleGenerateToken(w http.ResponseWriter, r *http.Request) {
|
||||
if h.readToken == "" {
|
||||
http.Error(w, "token generation requires a configured read-token", http.StatusBadRequest)
|
||||
if h.hmacKey == "" {
|
||||
http.Error(w, "token generation requires a configured HMAC key", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -84,7 +86,7 @@ func (h *Handler) handleGenerateToken(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
token := GenerateScopedToken(h.readToken, req.Organization, req.Repository, req.Workflow, req.Job)
|
||||
token := GenerateScopedToken(h.hmacKey, req.Organization, req.Repository, req.Workflow, req.Job)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(TokenResponse{Token: token})
|
||||
|
|
@ -92,8 +94,8 @@ func (h *Handler) handleGenerateToken(w http.ResponseWriter, r *http.Request) {
|
|||
|
||||
// validatePushToken checks push authentication via scoped HMAC token.
|
||||
func (h *Handler) validatePushToken(w http.ResponseWriter, r *http.Request, exec ExecutionContext) bool {
|
||||
if h.readToken == "" {
|
||||
h.logger.Warn("no read-token configured, rejecting push", slog.String("path", r.URL.Path))
|
||||
if h.hmacKey == "" {
|
||||
h.logger.Warn("no HMAC key configured, rejecting push", slog.String("path", r.URL.Path))
|
||||
http.Error(w, "authorization required", http.StatusUnauthorized)
|
||||
return false
|
||||
}
|
||||
|
|
@ -113,7 +115,7 @@ func (h *Handler) validatePushToken(w http.ResponseWriter, r *http.Request, exec
|
|||
}
|
||||
|
||||
token := strings.TrimPrefix(authHeader, bearerPrefix)
|
||||
if !ValidateScopedToken(h.readToken, token, exec.Organization, exec.Repository, exec.Workflow, exec.Job) {
|
||||
if !ValidateScopedToken(h.hmacKey, token, exec.Organization, exec.Repository, exec.Workflow, exec.Job) {
|
||||
h.logger.Warn("invalid push token", slog.String("path", r.URL.Path))
|
||||
http.Error(w, "invalid token", http.StatusUnauthorized)
|
||||
return false
|
||||
|
|
|
|||
|
|
@ -448,12 +448,17 @@ func newTestHandler(t *testing.T) (*Handler, func()) {
|
|||
}
|
||||
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
handler := NewHandler(store, logger, "") // no auth token — endpoints will reject
|
||||
handler := NewHandler(store, logger, "", "") // no auth — endpoints will reject
|
||||
|
||||
return handler, func() { _ = store.Close() }
|
||||
}
|
||||
|
||||
func newTestHandlerWithToken(t *testing.T, token string) (*Handler, func()) {
|
||||
func newTestHandlerWithToken(t *testing.T, readToken string) (*Handler, func()) {
|
||||
t.Helper()
|
||||
return newTestHandlerWithKeys(t, readToken, readToken)
|
||||
}
|
||||
|
||||
func newTestHandlerWithKeys(t *testing.T, readToken, hmacKey string) (*Handler, func()) {
|
||||
t.Helper()
|
||||
dbPath := filepath.Join(t.TempDir(), "test.db")
|
||||
store, err := NewStore(dbPath)
|
||||
|
|
@ -462,7 +467,7 @@ func newTestHandlerWithToken(t *testing.T, token string) (*Handler, func()) {
|
|||
}
|
||||
|
||||
logger := slog.New(slog.NewTextHandler(io.Discard, nil))
|
||||
handler := NewHandler(store, logger, token)
|
||||
handler := NewHandler(store, logger, readToken, hmacKey)
|
||||
|
||||
return handler, func() { _ = store.Close() }
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
# Docker Compose stress test with receiver
|
||||
# Run with: docker compose -f test/docker/docker-compose-stress.yaml up
|
||||
# See README.md "Docker Compose" section for the full token workflow.
|
||||
#
|
||||
# This test:
|
||||
# 1. Starts the metrics receiver
|
||||
# 2. Runs heavy CPU/memory workloads in multiple containers with shared PID namespace
|
||||
# 3. Collector gathers metrics and pushes summary to receiver on shutdown
|
||||
# 1. Starts the metrics receiver (with read-token and hmac-key)
|
||||
# 2. You generate a scoped push token via POST /api/v1/token
|
||||
# 3. Start the collector with COLLECTOR_PUSH_TOKEN set
|
||||
# 4. Runs heavy CPU/memory workloads in multiple containers with shared PID namespace
|
||||
# 5. Collector gathers metrics and pushes summary to receiver on shutdown
|
||||
#
|
||||
# To trigger the push, stop the collector gracefully:
|
||||
# docker compose -f test/docker/docker-compose-stress.yaml stop collector
|
||||
|
|
@ -20,6 +22,8 @@ services:
|
|||
- "9080:8080"
|
||||
environment:
|
||||
- DB_PATH=/data/metrics.db
|
||||
- RECEIVER_READ_TOKEN=dummyreadtoken
|
||||
- RECEIVER_HMAC_KEY=dummyhmackey
|
||||
volumes:
|
||||
- receiver-data:/data
|
||||
healthcheck:
|
||||
|
|
@ -98,6 +102,8 @@ services:
|
|||
- --log-format=json
|
||||
- --push-endpoint=http://receiver:8080/api/v1/metrics
|
||||
environment:
|
||||
# Push token — pass via COLLECTOR_PUSH_TOKEN from host env
|
||||
COLLECTOR_PUSH_TOKEN: "${COLLECTOR_PUSH_TOKEN}"
|
||||
# Execution context for the receiver
|
||||
GITHUB_REPOSITORY_OWNER: "test-org"
|
||||
GITHUB_REPOSITORY: "test-org/stress-test"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue