From 970ffb608e92f631846e7020a82ab28ebeca8d3d Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Fri, 21 Oct 2022 00:23:04 +0300 Subject: [PATCH] Mark pool manager as offline in case of 403 Signed-off-by: Gabriel Adrian Samfira --- runner/pool/enterprise.go | 10 +++++++-- runner/pool/interfaces.go | 9 ++++---- runner/pool/organization.go | 11 ++++++++-- runner/pool/pool.go | 43 +++++++++++++++++++++++++++++++------ runner/pool/repository.go | 10 +++++++-- 5 files changed, 67 insertions(+), 16 deletions(-) diff --git a/runner/pool/enterprise.go b/runner/pool/enterprise.go index 6bf8f789..0cc99d97 100644 --- a/runner/pool/enterprise.go +++ b/runner/pool/enterprise.go @@ -62,8 +62,11 @@ type enterprise struct { } func (r *enterprise) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) { - workflow, _, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID) + workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID) if err != nil { + if ghResp.StatusCode == http.StatusUnauthorized { + return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runners") + } return "", errors.Wrap(err, "fetching workflow info") } if workflow.RunnerName != nil { @@ -153,9 +156,12 @@ func (r *enterprise) JwtToken() string { } func (r *enterprise) GetGithubRegistrationToken() (string, error) { - tk, _, err := r.ghcEnterpriseCli.CreateRegistrationToken(r.ctx, r.cfg.Name) + tk, ghResp, err := r.ghcEnterpriseCli.CreateRegistrationToken(r.ctx, r.cfg.Name) if err != nil { + if ghResp.StatusCode == http.StatusUnauthorized { + return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching registration token") + } return "", errors.Wrap(err, "creating runner token") } return *tk.Token, nil diff --git a/runner/pool/interfaces.go b/runner/pool/interfaces.go index 4ccece01..9f1f7b85 100644 --- a/runner/pool/interfaces.go +++ b/runner/pool/interfaces.go @@ -23,13 +23,15 @@ import ( type poolHelper interface { GetGithubToken() string GetGithubRunners() ([]*github.Runner, error) - FetchTools() ([]*github.RunnerApplicationDownload, error) - FetchDbInstances() ([]params.Instance, error) + GetGithubRegistrationToken() (string, error) + GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) RemoveGithubRunner(runnerID int64) (*github.Response, error) + FetchTools() ([]*github.RunnerApplicationDownload, error) + + FetchDbInstances() ([]params.Instance, error) ListPools() ([]params.Pool, error) GithubURL() string JwtToken() string - GetGithubRegistrationToken() (string, error) String() string GetCallbackURL() string FindPoolByTags(labels []string) (params.Pool, error) @@ -37,6 +39,5 @@ type poolHelper interface { ValidateOwner(job params.WorkflowJob) error UpdateState(param params.UpdatePoolStateParams) error WebhookSecret() string - GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) ID() string } diff --git a/runner/pool/organization.go b/runner/pool/organization.go index e3cd2ebd..b8efba72 100644 --- a/runner/pool/organization.go +++ b/runner/pool/organization.go @@ -74,8 +74,11 @@ type organization struct { } func (r *organization) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) { - workflow, _, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Organization.Login, job.Repository.Name, job.WorkflowJob.ID) + workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Organization.Login, job.Repository.Name, job.WorkflowJob.ID) if err != nil { + if ghResp.StatusCode == http.StatusUnauthorized { + return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runner name") + } return "", errors.Wrap(err, "fetching workflow info") } if workflow.RunnerName != nil { @@ -165,9 +168,13 @@ func (r *organization) JwtToken() string { } func (r *organization) GetGithubRegistrationToken() (string, error) { - tk, _, err := r.ghcli.CreateOrganizationRegistrationToken(r.ctx, r.cfg.Name) + tk, ghResp, err := r.ghcli.CreateOrganizationRegistrationToken(r.ctx, r.cfg.Name) if err != nil { + if ghResp.StatusCode == http.StatusUnauthorized { + return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching token") + } + return "", errors.Wrap(err, "creating runner token") } return *tk.Token, nil diff --git a/runner/pool/pool.go b/runner/pool/pool.go index 385908b0..1eac96c2 100644 --- a/runner/pool/pool.go +++ b/runner/pool/pool.go @@ -183,6 +183,13 @@ func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner) if resp != nil && resp.StatusCode == http.StatusNotFound { continue } + + if errors.Is(err, runnerErrors.ErrUnauthorized) { + failureReason := fmt.Sprintf("failed to remove github runner: %q", err) + r.setPoolRunningState(false, failureReason) + log.Print(failureReason) + } + return errors.Wrap(err, "removing runner") } continue @@ -220,6 +227,12 @@ func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner) if resp != nil && resp.StatusCode == http.StatusNotFound { log.Printf("runner dissapeared from github") } else { + if errors.Is(err, runnerErrors.ErrUnauthorized) { + failureReason := fmt.Sprintf("failed to remove github runner: %q", err) + r.setPoolRunningState(false, failureReason) + log.Print(failureReason) + } + return errors.Wrap(err, "removing runner from github") } } @@ -385,9 +398,7 @@ func (r *basePoolManager) loop() { r.setPoolRunningState(false, failureReason) log.Print(failureReason) if errors.Is(err, runnerErrors.ErrUnauthorized) { - r.waitForTimeoutOrCanceled(common.UnauthorizedBackoffTimer) - } else { - r.waitForTimeoutOrCanceled(60 * time.Second) + break } continue } @@ -409,9 +420,7 @@ func (r *basePoolManager) loop() { r.setPoolRunningState(false, failureReason) log.Print(failureReason) if errors.Is(err, runnerErrors.ErrUnauthorized) { - r.waitForTimeoutOrCanceled(common.UnauthorizedBackoffTimer) - } else { - r.waitForTimeoutOrCanceled(60 * time.Second) + break } continue } @@ -505,6 +514,11 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error tk, err := r.helper.GetGithubRegistrationToken() if err != nil { + if errors.Is(err, runnerErrors.ErrUnauthorized) { + failureReason := fmt.Sprintf("failed to fetch registration token: %q", err) + r.setPoolRunningState(false, failureReason) + log.Print(failureReason) + } return errors.Wrap(err, "fetching registration token") } @@ -577,6 +591,11 @@ func (r *basePoolManager) getRunnerNameFromJob(job params.WorkflowJob) (string, log.Printf("runner name not found in workflow job, attempting to fetch from API") runnerName, err := r.helper.GetRunnerNameFromWorkflow(job) if err != nil { + if errors.Is(err, runnerErrors.ErrUnauthorized) { + failureReason := fmt.Sprintf("failed to fetch runner name from API: %q", err) + r.setPoolRunningState(false, failureReason) + log.Print(failureReason) + } return "", errors.Wrap(err, "fetching runner name from API") } @@ -915,6 +934,11 @@ func (r *basePoolManager) Wait() error { func (r *basePoolManager) runnerCleanup() error { runners, err := r.helper.GetGithubRunners() if err != nil { + if errors.Is(err, runnerErrors.ErrUnauthorized) { + failureReason := fmt.Sprintf("failed to fetch runners: %q", err) + r.setPoolRunningState(false, failureReason) + log.Print(failureReason) + } return errors.Wrap(err, "fetching github runners") } if err := r.cleanupOrphanedProviderRunners(runners); err != nil { @@ -963,6 +987,13 @@ func (r *basePoolManager) ForceDeleteRunner(runner params.Instance) error { case http.StatusNotFound: // Runner may have been deleted by a finished job, or manually by the user. log.Printf("runner with agent id %d was not found in github", runner.AgentID) + case http.StatusUnauthorized: + // Mark the pool as offline from this point forward + failureReason := fmt.Sprintf("failed to remove runner: %q", err) + r.setPoolRunningState(false, failureReason) + log.Print(failureReason) + // evaluate the next switch case. + fallthrough default: return errors.Wrap(err, "removing runner") } diff --git a/runner/pool/repository.go b/runner/pool/repository.go index 597bbf05..598ac939 100644 --- a/runner/pool/repository.go +++ b/runner/pool/repository.go @@ -76,8 +76,11 @@ type repository struct { } func (r *repository) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) { - workflow, _, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID) + workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID) if err != nil { + if ghResp.StatusCode == http.StatusUnauthorized { + return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runner name") + } return "", errors.Wrap(err, "fetching workflow info") } if workflow.RunnerName != nil { @@ -167,9 +170,12 @@ func (r *repository) JwtToken() string { } func (r *repository) GetGithubRegistrationToken() (string, error) { - tk, _, err := r.ghcli.CreateRegistrationToken(r.ctx, r.cfg.Owner, r.cfg.Name) + tk, ghResp, err := r.ghcli.CreateRegistrationToken(r.ctx, r.cfg.Owner, r.cfg.Name) if err != nil { + if ghResp.StatusCode == http.StatusUnauthorized { + return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching token") + } return "", errors.Wrap(err, "creating runner token") } return *tk.Token, nil