From 1268507ce6fa0e1e3176203e6952a84d814e984c Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Tue, 22 Aug 2023 11:58:52 +0000 Subject: [PATCH] Add jit config routes Signed-off-by: Gabriel Adrian Samfira --- apiserver/controllers/metadata.go | 69 +++++++++++++ apiserver/routers/routers.go | 8 ++ go.mod | 5 +- go.sum | 2 - runner/metadata.go | 160 ++++++++++++++++++++++++++++++ runner/pool/pool.go | 18 ++-- runner/runner.go | 46 --------- vendor/modules.txt | 3 +- 8 files changed, 255 insertions(+), 56 deletions(-) diff --git a/apiserver/controllers/metadata.go b/apiserver/controllers/metadata.go index c88e68cb..0f2f7b77 100644 --- a/apiserver/controllers/metadata.go +++ b/apiserver/controllers/metadata.go @@ -16,8 +16,12 @@ package controllers import ( "encoding/json" + "fmt" "log" "net/http" + + "github.com/cloudbase/garm/apiserver/params" + "github.com/gorilla/mux" ) func (a *APIController) InstanceGithubRegistrationTokenHandler(w http.ResponseWriter, r *http.Request) { @@ -36,6 +40,71 @@ func (a *APIController) InstanceGithubRegistrationTokenHandler(w http.ResponseWr } } +func (a *APIController) JITCredentialsFileHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + vars := mux.Vars(r) + fileName, ok := vars["fileName"] + if !ok { + w.WriteHeader(http.StatusNotFound) + if err := json.NewEncoder(w).Encode(params.APIErrorResponse{ + Error: "Not Found", + Details: "Not Found", + }); err != nil { + log.Printf("failed to encode response: %q", err) + } + return + } + + data, err := a.r.GetJITConfigFile(ctx, fileName) + if err != nil { + handleError(w, err) + return + } + + // Note the leading dot in the filename + name := fmt.Sprintf("attachment; filename=.%s", fileName) + w.Header().Set("Content-Disposition", name) + w.Header().Set("Content-Type", "octet-stream") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(data))) + w.WriteHeader(http.StatusOK) + if _, err := w.Write(data); err != nil { + log.Printf("failed to encode response: %q", err) + } +} + +func (a *APIController) SystemdServiceNameHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + serviceName, err := a.r.GetRunnerServiceName(ctx) + if err != nil { + handleError(w, err) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + if _, err := w.Write([]byte(serviceName)); err != nil { + log.Printf("failed to encode response: %q", err) + } +} + +func (a *APIController) SystemdUnitFileHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + runAsUser := r.URL.Query().Get("runAsUser") + + data, err := a.r.GenerateSystemdUnitFile(ctx, runAsUser) + if err != nil { + handleError(w, err) + return + } + + w.Header().Set("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + if _, err := w.Write(data); err != nil { + log.Printf("failed to encode response: %q", err) + } +} + func (a *APIController) RootCertificateBundleHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() diff --git a/apiserver/routers/routers.go b/apiserver/routers/routers.go index 244992fe..88c2e753 100644 --- a/apiserver/routers/routers.go +++ b/apiserver/routers/routers.go @@ -117,6 +117,14 @@ func NewAPIRouter(han *controllers.APIController, logWriter io.Writer, authMiddl // Registration token metadataRouter.Handle("/runner-registration-token/", http.HandlerFunc(han.InstanceGithubRegistrationTokenHandler)).Methods("GET", "OPTIONS") metadataRouter.Handle("/runner-registration-token", http.HandlerFunc(han.InstanceGithubRegistrationTokenHandler)).Methods("GET", "OPTIONS") + // JIT credential files + metadataRouter.Handle("/credentials/{fileName}/", http.HandlerFunc(han.JITCredentialsFileHandler)).Methods("GET", "OPTIONS") + metadataRouter.Handle("/credentials/{fileName}", http.HandlerFunc(han.JITCredentialsFileHandler)).Methods("GET", "OPTIONS") + // Systemd files + metadataRouter.Handle("/systemd/service-name/", http.HandlerFunc(han.SystemdServiceNameHandler)).Methods("GET", "OPTIONS") + metadataRouter.Handle("/systemd/service-name/", http.HandlerFunc(han.SystemdServiceNameHandler)).Methods("GET", "OPTIONS") + metadataRouter.Handle("/systemd/runner-service/", http.HandlerFunc(han.SystemdUnitFileHandler)).Methods("GET", "OPTIONS") + metadataRouter.Handle("/systemd/runner-service", http.HandlerFunc(han.SystemdUnitFileHandler)).Methods("GET", "OPTIONS") metadataRouter.Handle("/system/cert-bundle/", http.HandlerFunc(han.RootCertificateBundleHandler)).Methods("GET", "OPTIONS") metadataRouter.Handle("/system/cert-bundle", http.HandlerFunc(han.RootCertificateBundleHandler)).Methods("GET", "OPTIONS") diff --git a/go.mod b/go.mod index 8c5abdf6..9709b78e 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,10 @@ require ( gorm.io/gorm v1.24.6 ) -replace github.com/google/go-github/v54 => github.com/gabriel-samfira/go-github/v54 v54.0.0-20230821112832-bbb536ee5a3a +replace ( + github.com/cloudbase/garm-provider-common => /home/ubuntu/garm-provider-common + github.com/google/go-github/v54 => github.com/gabriel-samfira/go-github/v54 v54.0.0-20230821112832-bbb536ee5a3a +) require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect diff --git a/go.sum b/go.sum index 0a3bf142..815edd0a 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudbase/garm-provider-common v0.0.0-20230924074517-dd3e26769a05 h1:V9TQBCnwTeKX+gpmlEtZAQ5gNbYrdGelAA3jgnGde1c= -github.com/cloudbase/garm-provider-common v0.0.0-20230924074517-dd3e26769a05/go.mod h1:NgR629o2NYWTffZt3uURmu3orjMDQ2vh6KJ5ytwh7Qw= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/runner/metadata.go b/runner/metadata.go index 9d0a2769..bbbc0e47 100644 --- a/runner/metadata.go +++ b/runner/metadata.go @@ -1,15 +1,175 @@ package runner import ( + "bytes" "context" + "encoding/base64" + "fmt" + "html/template" "log" + "strings" + "github.com/cloudbase/garm-provider-common/defaults" runnerErrors "github.com/cloudbase/garm-provider-common/errors" "github.com/cloudbase/garm/auth" "github.com/cloudbase/garm/params" "github.com/pkg/errors" ) +var systemdUnitTemplate = `[Unit] +Description=GitHub Actions Runner ({{.ServiceName}}) +After=network.target + +[Service] +ExecStart=/home/{{.RunAsUser}}/actions-runner/runsvc.sh +User=runner +WorkingDirectory=/home/{{.RunAsUser}}/actions-runner +KillMode=process +KillSignal=SIGTERM +TimeoutStopSec=5min + +[Install] +WantedBy=multi-user.target +` + +func validateInstanceState(ctx context.Context) (params.Instance, error) { + if !auth.InstanceHasJITConfig(ctx) { + return params.Instance{}, fmt.Errorf("instance not configured for JIT: %w", runnerErrors.ErrNotFound) + } + + status := auth.InstanceRunnerStatus(ctx) + if status != params.RunnerPending && status != params.RunnerInstalling { + return params.Instance{}, runnerErrors.ErrUnauthorized + } + + instance, err := auth.InstanceParams(ctx) + if err != nil { + log.Printf("failed to get instance params: %s", err) + return params.Instance{}, runnerErrors.ErrUnauthorized + } + return instance, nil +} + +func (r *Runner) GetRunnerServiceName(ctx context.Context) (string, error) { + instance, err := validateInstanceState(ctx) + if err != nil { + return "", runnerErrors.ErrUnauthorized + } + + pool, err := r.store.GetPoolByID(r.ctx, instance.PoolID) + if err != nil { + return "", errors.Wrap(err, "fetching pool") + } + + tpl := "actions.runner.%s.%s" + var serviceName string + switch pool.PoolType() { + case params.EnterprisePool: + serviceName = fmt.Sprintf(tpl, pool.EnterpriseName, instance.Name) + case params.OrganizationPool: + serviceName = fmt.Sprintf(tpl, pool.OrgName, instance.Name) + case params.RepositoryPool: + serviceName = fmt.Sprintf(tpl, strings.Replace(pool.RepoName, "/", "-", -1), instance.Name) + } + return serviceName, nil +} + +func (r *Runner) GenerateSystemdUnitFile(ctx context.Context, runAsUser string) ([]byte, error) { + serviceName, err := r.GetRunnerServiceName(ctx) + if err != nil { + return nil, errors.Wrap(err, "fetching runner service name") + } + + unitTemplate, err := template.New("").Parse(systemdUnitTemplate) + if err != nil { + return nil, errors.Wrap(err, "parsing template") + } + + if runAsUser == "" { + runAsUser = defaults.DefaultUser + } + + data := struct { + ServiceName string + RunAsUser string + }{ + ServiceName: serviceName, + RunAsUser: runAsUser, + } + + var unitFile bytes.Buffer + if err := unitTemplate.Execute(&unitFile, data); err != nil { + return nil, errors.Wrap(err, "executing template") + } + return unitFile.Bytes(), nil +} + +func (r *Runner) GetJITConfigFile(ctx context.Context, file string) ([]byte, error) { + instance, err := validateInstanceState(ctx) + if err != nil { + log.Printf("failed to get instance params: %s", err) + return nil, runnerErrors.ErrUnauthorized + } + jitConfig := instance.JitConfiguration + contents, ok := jitConfig[file] + if !ok { + return nil, fmt.Errorf("file not found: %w", runnerErrors.ErrNotFound) + } + + decoded, err := base64.StdEncoding.DecodeString(contents) + if err != nil { + return nil, errors.Wrap(err, "decoding file contents") + } + + return decoded, nil +} + +func (r *Runner) GetInstanceGithubRegistrationToken(ctx context.Context) (string, error) { + // Check if this instance already fetched a registration token or if it was configured using + // the new Just In Time runner feature. If we're still using the old way of configuring a runner, + // we only allow an instance to fetch one token. If the instance fails to bootstrap after a token + // is fetched, we reset the token fetched field when re-queueing the instance. + if auth.InstanceTokenFetched(ctx) || auth.InstanceHasJITConfig(ctx) { + return "", runnerErrors.ErrUnauthorized + } + + status := auth.InstanceRunnerStatus(ctx) + if status != params.RunnerPending && status != params.RunnerInstalling { + return "", runnerErrors.ErrUnauthorized + } + + instance, err := auth.InstanceParams(ctx) + if err != nil { + log.Printf("failed to get instance params: %s", err) + return "", runnerErrors.ErrUnauthorized + } + + poolMgr, err := r.getPoolManagerFromInstance(ctx, instance) + if err != nil { + return "", errors.Wrap(err, "fetching pool manager for instance") + } + + token, err := poolMgr.GithubRunnerRegistrationToken() + if err != nil { + return "", errors.Wrap(err, "fetching runner token") + } + + tokenFetched := true + updateParams := params.UpdateInstanceParams{ + TokenFetched: &tokenFetched, + } + + if _, err := r.store.UpdateInstance(r.ctx, instance.ID, updateParams); err != nil { + return "", errors.Wrap(err, "setting token_fetched for instance") + } + + if err := r.store.AddInstanceEvent(ctx, instance.ID, params.FetchTokenEvent, params.EventInfo, "runner registration token was retrieved"); err != nil { + return "", errors.Wrap(err, "recording event") + } + + return token, nil +} + func (r *Runner) GetRootCertificateBundle(ctx context.Context) (params.CertificateBundle, error) { instance, err := auth.InstanceParams(ctx) if err != nil { diff --git a/runner/pool/pool.go b/runner/pool/pool.go index 0730f07b..842747d4 100644 --- a/runner/pool/pool.go +++ b/runner/pool/pool.go @@ -800,9 +800,6 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error return fmt.Errorf("unknown provider %s for pool %s", pool.ProviderName, pool.ID) } - // We still need the labels here for situations where we don't have a JIT config generated. - // This can happen if GARM is used against an instance of GHES older than version 3.10. - labels := r.getLabelsForInstance(pool) jwtValidity := pool.RunnerTimeout() entity := r.helper.String() @@ -811,6 +808,8 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error return errors.Wrap(err, "fetching instance jwt token") } + hasJITConfig := len(instance.JitConfiguration) > 0 + bootstrapArgs := commonParams.BootstrapInstance{ Name: instance.Name, Tools: r.tools, @@ -823,11 +822,17 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error Flavor: pool.Flavor, Image: pool.Image, ExtraSpecs: pool.ExtraSpecs, - Labels: labels, PoolID: instance.PoolID, CACertBundle: r.credsDetails.CABundle, GitHubRunnerGroup: instance.GitHubRunnerGroup, - // JitConfigEnabled: len(instance.JitConfiguration) > 0, + JitConfigEnabled: hasJITConfig, + } + + if !hasJITConfig { + // We still need the labels here for situations where we don't have a JIT config generated. + // This can happen if GARM is used against an instance of GHES older than version 3.10. + // The labels field should be ignored by providers if JIT config is enabled. + bootstrapArgs.Labels = r.getLabelsForInstance(pool) } var instanceIDToDelete string @@ -1162,11 +1167,12 @@ func (r *basePoolManager) retryFailedInstancesForOnePool(ctx context.Context, po // TODO(gabriel-samfira): Incrementing CreateAttempt should be done within a transaction. // It's fairly safe to do here (for now), as there should be no other code path that updates // an instance in this state. - var tokenFetched bool = false + var tokenFetched bool = len(instance.JitConfiguration) > 0 updateParams := params.UpdateInstanceParams{ CreateAttempt: instance.CreateAttempt + 1, TokenFetched: &tokenFetched, Status: commonParams.InstancePendingCreate, + RunnerStatus: params.RunnerPending, } r.log("queueing previously failed instance %s for retry", instance.Name) // Set instance to pending create and wait for retry. diff --git a/runner/runner.go b/runner/runner.go index 0aba8823..aa4aedbe 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -858,52 +858,6 @@ func (r *Runner) AddInstanceStatusMessage(ctx context.Context, param params.Inst return nil } -func (r *Runner) GetInstanceGithubRegistrationToken(ctx context.Context) (string, error) { - // Check if this instance already fetched a registration token or if it was configured using - // the new Just In Time runner feature. If we're still using the old way of configuring a runner, - // we only allow an instance to fetch one token. If the instance fails to bootstrap after a token - // is fetched, we reset the token fetched field when re-queueing the instance. - if auth.InstanceTokenFetched(ctx) || auth.InstanceHasJITConfig(ctx) { - return "", runnerErrors.ErrUnauthorized - } - - status := auth.InstanceRunnerStatus(ctx) - if status != params.RunnerPending && status != params.RunnerInstalling { - return "", runnerErrors.ErrUnauthorized - } - - instance, err := auth.InstanceParams(ctx) - if err != nil { - log.Printf("failed to get instance params: %s", err) - return "", runnerErrors.ErrUnauthorized - } - - poolMgr, err := r.getPoolManagerFromInstance(ctx, instance) - if err != nil { - return "", errors.Wrap(err, "fetching pool manager for instance") - } - - token, err := poolMgr.GithubRunnerRegistrationToken() - if err != nil { - return "", errors.Wrap(err, "fetching runner token") - } - - tokenFetched := true - updateParams := params.UpdateInstanceParams{ - TokenFetched: &tokenFetched, - } - - if _, err := r.store.UpdateInstance(r.ctx, instance.ID, updateParams); err != nil { - return "", errors.Wrap(err, "setting token_fetched for instance") - } - - if err := r.store.AddInstanceEvent(ctx, instance.ID, params.FetchTokenEvent, params.EventInfo, "runner registration token was retrieved"); err != nil { - return "", errors.Wrap(err, "recording event") - } - - return token, nil -} - func (r *Runner) getPoolManagerFromInstance(ctx context.Context, instance params.Instance) (common.PoolManager, error) { pool, err := r.store.GetPoolByID(ctx, instance.PoolID) if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index 24d7e7ab..7789a712 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -14,7 +14,7 @@ github.com/cespare/xxhash/v2 # github.com/chzyer/readline v1.5.1 ## explicit; go 1.15 github.com/chzyer/readline -# github.com/cloudbase/garm-provider-common v0.0.0-20230924074517-dd3e26769a05 +# github.com/cloudbase/garm-provider-common v0.0.0-20230924074517-dd3e26769a05 => /home/ubuntu/garm-provider-common ## explicit; go 1.20 github.com/cloudbase/garm-provider-common/cloudconfig github.com/cloudbase/garm-provider-common/defaults @@ -439,4 +439,5 @@ gorm.io/gorm/logger gorm.io/gorm/migrator gorm.io/gorm/schema gorm.io/gorm/utils +# github.com/cloudbase/garm-provider-common => /home/ubuntu/garm-provider-common # github.com/google/go-github/v54 => github.com/gabriel-samfira/go-github/v54 v54.0.0-20230821112832-bbb536ee5a3a