DevFW-CICD/garm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Mark pool manager as offline in case of 403

Browse source 
Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2022-10-21 00:23:04 +03:00
parent 05057e37fd
commit 970ffb608e
No known key found for this signature in database
GPG key ID: 7D073DCC2C074CB5
5 changed files with 67 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

10
runner/pool/enterprise.go
View file

@ -62,8 +62,11 @@ type enterprise struct {
}
func (r *enterprise) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) {
workflow, _, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID)
workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID)
if err != nil {
if ghResp.StatusCode == http.StatusUnauthorized {
return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runners")
}
return "", errors.Wrap(err, "fetching workflow info")
}
if workflow.RunnerName != nil {
@ -153,9 +156,12 @@ func (r *enterprise) JwtToken() string {
}
func (r *enterprise) GetGithubRegistrationToken() (string, error) {
tk, _, err := r.ghcEnterpriseCli.CreateRegistrationToken(r.ctx, r.cfg.Name)
tk, ghResp, err := r.ghcEnterpriseCli.CreateRegistrationToken(r.ctx, r.cfg.Name)
if err != nil {
if ghResp.StatusCode == http.StatusUnauthorized {
return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching registration token")
}
return "", errors.Wrap(err, "creating runner token")
}
return *tk.Token, nil

9
runner/pool/interfaces.go
View file

@ -23,13 +23,15 @@ import (
type poolHelper interface {
GetGithubToken() string
GetGithubRunners() ([]*github.Runner, error)
FetchTools() ([]*github.RunnerApplicationDownload, error)
FetchDbInstances() ([]params.Instance, error)
GetGithubRegistrationToken() (string, error)
GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error)
RemoveGithubRunner(runnerID int64) (*github.Response, error)
FetchTools() ([]*github.RunnerApplicationDownload, error)
FetchDbInstances() ([]params.Instance, error)
ListPools() ([]params.Pool, error)
GithubURL() string
JwtToken() string
GetGithubRegistrationToken() (string, error)
String() string
GetCallbackURL() string
FindPoolByTags(labels []string) (params.Pool, error)
@ -37,6 +39,5 @@ type poolHelper interface {
ValidateOwner(job params.WorkflowJob) error
UpdateState(param params.UpdatePoolStateParams) error
WebhookSecret() string
GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error)
ID() string
}

11
runner/pool/organization.go
View file

@ -74,8 +74,11 @@ type organization struct {
}
func (r *organization) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) {
workflow, _, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Organization.Login, job.Repository.Name, job.WorkflowJob.ID)
workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Organization.Login, job.Repository.Name, job.WorkflowJob.ID)
if err != nil {
if ghResp.StatusCode == http.StatusUnauthorized {
return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runner name")
}
return "", errors.Wrap(err, "fetching workflow info")
}
if workflow.RunnerName != nil {
@ -165,9 +168,13 @@ func (r *organization) JwtToken() string {
}
func (r *organization) GetGithubRegistrationToken() (string, error) {
tk, _, err := r.ghcli.CreateOrganizationRegistrationToken(r.ctx, r.cfg.Name)
tk, ghResp, err := r.ghcli.CreateOrganizationRegistrationToken(r.ctx, r.cfg.Name)
if err != nil {
if ghResp.StatusCode == http.StatusUnauthorized {
return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching token")
}
return "", errors.Wrap(err, "creating runner token")
}
return *tk.Token, nil

43
runner/pool/pool.go
View file

@ -183,6 +183,13 @@ func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner)
if resp != nil && resp.StatusCode == http.StatusNotFound {
continue
}
if errors.Is(err, runnerErrors.ErrUnauthorized) {
failureReason := fmt.Sprintf("failed to remove github runner: %q", err)
r.setPoolRunningState(false, failureReason)
log.Print(failureReason)
}
return errors.Wrap(err, "removing runner")
}
continue
@ -220,6 +227,12 @@ func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner)
if resp != nil && resp.StatusCode == http.StatusNotFound {
log.Printf("runner dissapeared from github")
} else {
if errors.Is(err, runnerErrors.ErrUnauthorized) {
failureReason := fmt.Sprintf("failed to remove github runner: %q", err)
r.setPoolRunningState(false, failureReason)
log.Print(failureReason)
}
return errors.Wrap(err, "removing runner from github")
}
}
@ -385,9 +398,7 @@ func (r *basePoolManager) loop() {
r.setPoolRunningState(false, failureReason)
log.Print(failureReason)
if errors.Is(err, runnerErrors.ErrUnauthorized) {
r.waitForTimeoutOrCanceled(common.UnauthorizedBackoffTimer)
} else {
r.waitForTimeoutOrCanceled(60 * time.Second)
break
}
continue
}
@ -409,9 +420,7 @@ func (r *basePoolManager) loop() {
r.setPoolRunningState(false, failureReason)
log.Print(failureReason)
if errors.Is(err, runnerErrors.ErrUnauthorized) {
r.waitForTimeoutOrCanceled(common.UnauthorizedBackoffTimer)
} else {
r.waitForTimeoutOrCanceled(60 * time.Second)
break
}
continue
}
@ -505,6 +514,11 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error
tk, err := r.helper.GetGithubRegistrationToken()
if err != nil {
if errors.Is(err, runnerErrors.ErrUnauthorized) {
failureReason := fmt.Sprintf("failed to fetch registration token: %q", err)
r.setPoolRunningState(false, failureReason)
log.Print(failureReason)
}
return errors.Wrap(err, "fetching registration token")
}
@ -577,6 +591,11 @@ func (r *basePoolManager) getRunnerNameFromJob(job params.WorkflowJob) (string,
log.Printf("runner name not found in workflow job, attempting to fetch from API")
runnerName, err := r.helper.GetRunnerNameFromWorkflow(job)
if err != nil {
if errors.Is(err, runnerErrors.ErrUnauthorized) {
failureReason := fmt.Sprintf("failed to fetch runner name from API: %q", err)
r.setPoolRunningState(false, failureReason)
log.Print(failureReason)
}
return "", errors.Wrap(err, "fetching runner name from API")
}
@ -915,6 +934,11 @@ func (r *basePoolManager) Wait() error {
func (r *basePoolManager) runnerCleanup() error {
runners, err := r.helper.GetGithubRunners()
if err != nil {
if errors.Is(err, runnerErrors.ErrUnauthorized) {
failureReason := fmt.Sprintf("failed to fetch runners: %q", err)
r.setPoolRunningState(false, failureReason)
log.Print(failureReason)
}
return errors.Wrap(err, "fetching github runners")
}
if err := r.cleanupOrphanedProviderRunners(runners); err != nil {
@ -963,6 +987,13 @@ func (r *basePoolManager) ForceDeleteRunner(runner params.Instance) error {
case http.StatusNotFound:
// Runner may have been deleted by a finished job, or manually by the user.
log.Printf("runner with agent id %d was not found in github", runner.AgentID)
case http.StatusUnauthorized:
// Mark the pool as offline from this point forward
failureReason := fmt.Sprintf("failed to remove runner: %q", err)
r.setPoolRunningState(false, failureReason)
log.Print(failureReason)
// evaluate the next switch case.
fallthrough
default:
return errors.Wrap(err, "removing runner")
}

10
runner/pool/repository.go
View file

@ -76,8 +76,11 @@ type repository struct {
}
func (r *repository) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) {
workflow, _, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID)
workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID)
if err != nil {
if ghResp.StatusCode == http.StatusUnauthorized {
return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runner name")
}
return "", errors.Wrap(err, "fetching workflow info")
}
if workflow.RunnerName != nil {
@ -167,9 +170,12 @@ func (r *repository) JwtToken() string {
}
func (r *repository) GetGithubRegistrationToken() (string, error) {
tk, _, err := r.ghcli.CreateRegistrationToken(r.ctx, r.cfg.Owner, r.cfg.Name)
tk, ghResp, err := r.ghcli.CreateRegistrationToken(r.ctx, r.cfg.Owner, r.cfg.Name)
if err != nil {
if ghResp.StatusCode == http.StatusUnauthorized {
return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching token")
}
return "", errors.Wrap(err, "creating runner token")
}
return *tk.Token, nil