Some cleanup and safety checks

* Add logging middleware
  * Remove some noise from logs
  * Add some safety checks when managing runners
This commit is contained in:
Gabriel Adrian Samfira 2022-12-29 16:49:50 +00:00
parent 5bc2bec991
commit 3a92a5be0e
15 changed files with 384 additions and 198 deletions

View file

@ -18,42 +18,43 @@ import (
"io"
"net/http"
gorillaHandlers "github.com/gorilla/handlers"
"github.com/gorilla/mux"
"garm/apiserver/controllers"
"garm/auth"
"garm/util"
)
func NewAPIRouter(han *controllers.APIController, logWriter io.Writer, authMiddleware, initMiddleware, instanceMiddleware auth.Middleware) *mux.Router {
router := mux.NewRouter()
log := gorillaHandlers.CombinedLoggingHandler
logMiddleware := util.NewLoggingMiddleware(logWriter)
router.Use(logMiddleware)
// Handles github webhooks
webhookRouter := router.PathPrefix("/webhooks").Subrouter()
webhookRouter.PathPrefix("/").Handler(log(logWriter, http.HandlerFunc(han.CatchAll)))
webhookRouter.PathPrefix("").Handler(log(logWriter, http.HandlerFunc(han.CatchAll)))
webhookRouter.PathPrefix("/").Handler(http.HandlerFunc(han.CatchAll))
webhookRouter.PathPrefix("").Handler(http.HandlerFunc(han.CatchAll))
// Handles API calls
apiSubRouter := router.PathPrefix("/api/v1").Subrouter()
// FirstRunHandler
firstRunRouter := apiSubRouter.PathPrefix("/first-run").Subrouter()
firstRunRouter.Handle("/", log(logWriter, http.HandlerFunc(han.FirstRunHandler))).Methods("POST", "OPTIONS")
firstRunRouter.Handle("/", http.HandlerFunc(han.FirstRunHandler)).Methods("POST", "OPTIONS")
// Instance URLs
callbackRouter := apiSubRouter.PathPrefix("/callbacks").Subrouter()
callbackRouter.Handle("/status/", log(logWriter, http.HandlerFunc(han.InstanceStatusMessageHandler))).Methods("POST", "OPTIONS")
callbackRouter.Handle("/status", log(logWriter, http.HandlerFunc(han.InstanceStatusMessageHandler))).Methods("POST", "OPTIONS")
callbackRouter.Handle("/status/", http.HandlerFunc(han.InstanceStatusMessageHandler)).Methods("POST", "OPTIONS")
callbackRouter.Handle("/status", http.HandlerFunc(han.InstanceStatusMessageHandler)).Methods("POST", "OPTIONS")
callbackRouter.Use(instanceMiddleware.Middleware)
metadataRouter := apiSubRouter.PathPrefix("/metadata").Subrouter()
metadataRouter.Handle("/runner-registration-token/", log(logWriter, http.HandlerFunc(han.InstanceGithubRegistrationTokenHandler))).Methods("GET", "OPTIONS")
metadataRouter.Handle("/runner-registration-token", log(logWriter, http.HandlerFunc(han.InstanceGithubRegistrationTokenHandler))).Methods("GET", "OPTIONS")
metadataRouter.Handle("/runner-registration-token/", http.HandlerFunc(han.InstanceGithubRegistrationTokenHandler)).Methods("GET", "OPTIONS")
metadataRouter.Handle("/runner-registration-token", http.HandlerFunc(han.InstanceGithubRegistrationTokenHandler)).Methods("GET", "OPTIONS")
metadataRouter.Use(instanceMiddleware.Middleware)
// Login
authRouter := apiSubRouter.PathPrefix("/auth").Subrouter()
authRouter.Handle("/{login:login\\/?}", log(logWriter, http.HandlerFunc(han.LoginHandler))).Methods("POST", "OPTIONS")
authRouter.Handle("/{login:login\\/?}", http.HandlerFunc(han.LoginHandler)).Methods("POST", "OPTIONS")
authRouter.Use(initMiddleware.Middleware)
apiRouter := apiSubRouter.PathPrefix("").Subrouter()
@ -64,158 +65,158 @@ func NewAPIRouter(han *controllers.APIController, logWriter io.Writer, authMiddl
// Pools //
///////////
// List all pools
apiRouter.Handle("/pools/", log(logWriter, http.HandlerFunc(han.ListAllPoolsHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/pools", log(logWriter, http.HandlerFunc(han.ListAllPoolsHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/pools/", http.HandlerFunc(han.ListAllPoolsHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/pools", http.HandlerFunc(han.ListAllPoolsHandler)).Methods("GET", "OPTIONS")
// Get one pool
apiRouter.Handle("/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.GetPoolByIDHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/pools/{poolID}", log(logWriter, http.HandlerFunc(han.GetPoolByIDHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/pools/{poolID}/", http.HandlerFunc(han.GetPoolByIDHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/pools/{poolID}", http.HandlerFunc(han.GetPoolByIDHandler)).Methods("GET", "OPTIONS")
// Delete one pool
apiRouter.Handle("/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.DeletePoolByIDHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/pools/{poolID}", log(logWriter, http.HandlerFunc(han.DeletePoolByIDHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/pools/{poolID}/", http.HandlerFunc(han.DeletePoolByIDHandler)).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/pools/{poolID}", http.HandlerFunc(han.DeletePoolByIDHandler)).Methods("DELETE", "OPTIONS")
// Update one pool
apiRouter.Handle("/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.UpdatePoolByIDHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/pools/{poolID}", log(logWriter, http.HandlerFunc(han.UpdatePoolByIDHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/pools/{poolID}/", http.HandlerFunc(han.UpdatePoolByIDHandler)).Methods("PUT", "OPTIONS")
apiRouter.Handle("/pools/{poolID}", http.HandlerFunc(han.UpdatePoolByIDHandler)).Methods("PUT", "OPTIONS")
// List pool instances
apiRouter.Handle("/pools/{poolID}/instances/", log(logWriter, http.HandlerFunc(han.ListPoolInstancesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/pools/{poolID}/instances", log(logWriter, http.HandlerFunc(han.ListPoolInstancesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/pools/{poolID}/instances/", http.HandlerFunc(han.ListPoolInstancesHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/pools/{poolID}/instances", http.HandlerFunc(han.ListPoolInstancesHandler)).Methods("GET", "OPTIONS")
/////////////
// Runners //
/////////////
// Get instance
apiRouter.Handle("/instances/{instanceName}/", log(logWriter, http.HandlerFunc(han.GetInstanceHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/instances/{instanceName}", log(logWriter, http.HandlerFunc(han.GetInstanceHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/instances/{instanceName}/", http.HandlerFunc(han.GetInstanceHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/instances/{instanceName}", http.HandlerFunc(han.GetInstanceHandler)).Methods("GET", "OPTIONS")
// Delete runner
apiRouter.Handle("/instances/{instanceName}/", log(logWriter, http.HandlerFunc(han.DeleteInstanceHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/instances/{instanceName}", log(logWriter, http.HandlerFunc(han.DeleteInstanceHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/instances/{instanceName}/", http.HandlerFunc(han.DeleteInstanceHandler)).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/instances/{instanceName}", http.HandlerFunc(han.DeleteInstanceHandler)).Methods("DELETE", "OPTIONS")
// List runners
apiRouter.Handle("/instances/", log(logWriter, http.HandlerFunc(han.ListAllInstancesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/instances", log(logWriter, http.HandlerFunc(han.ListAllInstancesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/instances/", http.HandlerFunc(han.ListAllInstancesHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/instances", http.HandlerFunc(han.ListAllInstancesHandler)).Methods("GET", "OPTIONS")
/////////////////////
// Repos and pools //
/////////////////////
// Get pool
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.GetRepoPoolHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.GetRepoPoolHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", http.HandlerFunc(han.GetRepoPoolHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", http.HandlerFunc(han.GetRepoPoolHandler)).Methods("GET", "OPTIONS")
// Delete pool
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.DeleteRepoPoolHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.DeleteRepoPoolHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", http.HandlerFunc(han.DeleteRepoPoolHandler)).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", http.HandlerFunc(han.DeleteRepoPoolHandler)).Methods("DELETE", "OPTIONS")
// Update pool
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.UpdateRepoPoolHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.UpdateRepoPoolHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", http.HandlerFunc(han.UpdateRepoPoolHandler)).Methods("PUT", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", http.HandlerFunc(han.UpdateRepoPoolHandler)).Methods("PUT", "OPTIONS")
// List pools
apiRouter.Handle("/repositories/{repoID}/pools/", log(logWriter, http.HandlerFunc(han.ListRepoPoolsHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools", log(logWriter, http.HandlerFunc(han.ListRepoPoolsHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/", http.HandlerFunc(han.ListRepoPoolsHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools", http.HandlerFunc(han.ListRepoPoolsHandler)).Methods("GET", "OPTIONS")
// Create pool
apiRouter.Handle("/repositories/{repoID}/pools/", log(logWriter, http.HandlerFunc(han.CreateRepoPoolHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools", log(logWriter, http.HandlerFunc(han.CreateRepoPoolHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools/", http.HandlerFunc(han.CreateRepoPoolHandler)).Methods("POST", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/pools", http.HandlerFunc(han.CreateRepoPoolHandler)).Methods("POST", "OPTIONS")
// Repo instances list
apiRouter.Handle("/repositories/{repoID}/instances/", log(logWriter, http.HandlerFunc(han.ListRepoInstancesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/instances", log(logWriter, http.HandlerFunc(han.ListRepoInstancesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/instances/", http.HandlerFunc(han.ListRepoInstancesHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/instances", http.HandlerFunc(han.ListRepoInstancesHandler)).Methods("GET", "OPTIONS")
// Get repo
apiRouter.Handle("/repositories/{repoID}/", log(logWriter, http.HandlerFunc(han.GetRepoByIDHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}", log(logWriter, http.HandlerFunc(han.GetRepoByIDHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/", http.HandlerFunc(han.GetRepoByIDHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}", http.HandlerFunc(han.GetRepoByIDHandler)).Methods("GET", "OPTIONS")
// Update repo
apiRouter.Handle("/repositories/{repoID}/", log(logWriter, http.HandlerFunc(han.UpdateRepoHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}", log(logWriter, http.HandlerFunc(han.UpdateRepoHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/", http.HandlerFunc(han.UpdateRepoHandler)).Methods("PUT", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}", http.HandlerFunc(han.UpdateRepoHandler)).Methods("PUT", "OPTIONS")
// Delete repo
apiRouter.Handle("/repositories/{repoID}/", log(logWriter, http.HandlerFunc(han.DeleteRepoHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}", log(logWriter, http.HandlerFunc(han.DeleteRepoHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}/", http.HandlerFunc(han.DeleteRepoHandler)).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/repositories/{repoID}", http.HandlerFunc(han.DeleteRepoHandler)).Methods("DELETE", "OPTIONS")
// List repos
apiRouter.Handle("/repositories/", log(logWriter, http.HandlerFunc(han.ListReposHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories", log(logWriter, http.HandlerFunc(han.ListReposHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories/", http.HandlerFunc(han.ListReposHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/repositories", http.HandlerFunc(han.ListReposHandler)).Methods("GET", "OPTIONS")
// Create repo
apiRouter.Handle("/repositories/", log(logWriter, http.HandlerFunc(han.CreateRepoHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/repositories", log(logWriter, http.HandlerFunc(han.CreateRepoHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/repositories/", http.HandlerFunc(han.CreateRepoHandler)).Methods("POST", "OPTIONS")
apiRouter.Handle("/repositories", http.HandlerFunc(han.CreateRepoHandler)).Methods("POST", "OPTIONS")
/////////////////////////////
// Organizations and pools //
/////////////////////////////
// Get pool
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.GetOrgPoolHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.GetOrgPoolHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", http.HandlerFunc(han.GetOrgPoolHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", http.HandlerFunc(han.GetOrgPoolHandler)).Methods("GET", "OPTIONS")
// Delete pool
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.DeleteOrgPoolHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.DeleteOrgPoolHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", http.HandlerFunc(han.DeleteOrgPoolHandler)).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", http.HandlerFunc(han.DeleteOrgPoolHandler)).Methods("DELETE", "OPTIONS")
// Update pool
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.UpdateOrgPoolHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.UpdateOrgPoolHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", http.HandlerFunc(han.UpdateOrgPoolHandler)).Methods("PUT", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", http.HandlerFunc(han.UpdateOrgPoolHandler)).Methods("PUT", "OPTIONS")
// List pools
apiRouter.Handle("/organizations/{orgID}/pools/", log(logWriter, http.HandlerFunc(han.ListOrgPoolsHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools", log(logWriter, http.HandlerFunc(han.ListOrgPoolsHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/", http.HandlerFunc(han.ListOrgPoolsHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools", http.HandlerFunc(han.ListOrgPoolsHandler)).Methods("GET", "OPTIONS")
// Create pool
apiRouter.Handle("/organizations/{orgID}/pools/", log(logWriter, http.HandlerFunc(han.CreateOrgPoolHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools", log(logWriter, http.HandlerFunc(han.CreateOrgPoolHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools/", http.HandlerFunc(han.CreateOrgPoolHandler)).Methods("POST", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/pools", http.HandlerFunc(han.CreateOrgPoolHandler)).Methods("POST", "OPTIONS")
// Repo instances list
apiRouter.Handle("/organizations/{orgID}/instances/", log(logWriter, http.HandlerFunc(han.ListOrgInstancesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/instances", log(logWriter, http.HandlerFunc(han.ListOrgInstancesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/instances/", http.HandlerFunc(han.ListOrgInstancesHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/instances", http.HandlerFunc(han.ListOrgInstancesHandler)).Methods("GET", "OPTIONS")
// Get org
apiRouter.Handle("/organizations/{orgID}/", log(logWriter, http.HandlerFunc(han.GetOrgByIDHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}", log(logWriter, http.HandlerFunc(han.GetOrgByIDHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/", http.HandlerFunc(han.GetOrgByIDHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}", http.HandlerFunc(han.GetOrgByIDHandler)).Methods("GET", "OPTIONS")
// Update org
apiRouter.Handle("/organizations/{orgID}/", log(logWriter, http.HandlerFunc(han.UpdateOrgHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}", log(logWriter, http.HandlerFunc(han.UpdateOrgHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/", http.HandlerFunc(han.UpdateOrgHandler)).Methods("PUT", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}", http.HandlerFunc(han.UpdateOrgHandler)).Methods("PUT", "OPTIONS")
// Delete org
apiRouter.Handle("/organizations/{orgID}/", log(logWriter, http.HandlerFunc(han.DeleteOrgHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}", log(logWriter, http.HandlerFunc(han.DeleteOrgHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}/", http.HandlerFunc(han.DeleteOrgHandler)).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/organizations/{orgID}", http.HandlerFunc(han.DeleteOrgHandler)).Methods("DELETE", "OPTIONS")
// List orgs
apiRouter.Handle("/organizations/", log(logWriter, http.HandlerFunc(han.ListOrgsHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations", log(logWriter, http.HandlerFunc(han.ListOrgsHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations/", http.HandlerFunc(han.ListOrgsHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/organizations", http.HandlerFunc(han.ListOrgsHandler)).Methods("GET", "OPTIONS")
// Create org
apiRouter.Handle("/organizations/", log(logWriter, http.HandlerFunc(han.CreateOrgHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/organizations", log(logWriter, http.HandlerFunc(han.CreateOrgHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/organizations/", http.HandlerFunc(han.CreateOrgHandler)).Methods("POST", "OPTIONS")
apiRouter.Handle("/organizations", http.HandlerFunc(han.CreateOrgHandler)).Methods("POST", "OPTIONS")
/////////////////////////////
// Enterprises and pools //
/////////////////////////////
// Get pool
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.GetEnterprisePoolHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.GetEnterprisePoolHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", http.HandlerFunc(han.GetEnterprisePoolHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", http.HandlerFunc(han.GetEnterprisePoolHandler)).Methods("GET", "OPTIONS")
// Delete pool
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.DeleteEnterprisePoolHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.DeleteEnterprisePoolHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", http.HandlerFunc(han.DeleteEnterprisePoolHandler)).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", http.HandlerFunc(han.DeleteEnterprisePoolHandler)).Methods("DELETE", "OPTIONS")
// Update pool
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.UpdateEnterprisePoolHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.UpdateEnterprisePoolHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", http.HandlerFunc(han.UpdateEnterprisePoolHandler)).Methods("PUT", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", http.HandlerFunc(han.UpdateEnterprisePoolHandler)).Methods("PUT", "OPTIONS")
// List pools
apiRouter.Handle("/enterprises/{enterpriseID}/pools/", log(logWriter, http.HandlerFunc(han.ListEnterprisePoolsHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools", log(logWriter, http.HandlerFunc(han.ListEnterprisePoolsHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/", http.HandlerFunc(han.ListEnterprisePoolsHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools", http.HandlerFunc(han.ListEnterprisePoolsHandler)).Methods("GET", "OPTIONS")
// Create pool
apiRouter.Handle("/enterprises/{enterpriseID}/pools/", log(logWriter, http.HandlerFunc(han.CreateEnterprisePoolHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools", log(logWriter, http.HandlerFunc(han.CreateEnterprisePoolHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools/", http.HandlerFunc(han.CreateEnterprisePoolHandler)).Methods("POST", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/pools", http.HandlerFunc(han.CreateEnterprisePoolHandler)).Methods("POST", "OPTIONS")
// Repo instances list
apiRouter.Handle("/enterprises/{enterpriseID}/instances/", log(logWriter, http.HandlerFunc(han.ListEnterpriseInstancesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/instances", log(logWriter, http.HandlerFunc(han.ListEnterpriseInstancesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/instances/", http.HandlerFunc(han.ListEnterpriseInstancesHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/instances", http.HandlerFunc(han.ListEnterpriseInstancesHandler)).Methods("GET", "OPTIONS")
// Get org
apiRouter.Handle("/enterprises/{enterpriseID}/", log(logWriter, http.HandlerFunc(han.GetEnterpriseByIDHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}", log(logWriter, http.HandlerFunc(han.GetEnterpriseByIDHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/", http.HandlerFunc(han.GetEnterpriseByIDHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}", http.HandlerFunc(han.GetEnterpriseByIDHandler)).Methods("GET", "OPTIONS")
// Update org
apiRouter.Handle("/enterprises/{enterpriseID}/", log(logWriter, http.HandlerFunc(han.UpdateEnterpriseHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}", log(logWriter, http.HandlerFunc(han.UpdateEnterpriseHandler))).Methods("PUT", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/", http.HandlerFunc(han.UpdateEnterpriseHandler)).Methods("PUT", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}", http.HandlerFunc(han.UpdateEnterpriseHandler)).Methods("PUT", "OPTIONS")
// Delete org
apiRouter.Handle("/enterprises/{enterpriseID}/", log(logWriter, http.HandlerFunc(han.DeleteEnterpriseHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}", log(logWriter, http.HandlerFunc(han.DeleteEnterpriseHandler))).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}/", http.HandlerFunc(han.DeleteEnterpriseHandler)).Methods("DELETE", "OPTIONS")
apiRouter.Handle("/enterprises/{enterpriseID}", http.HandlerFunc(han.DeleteEnterpriseHandler)).Methods("DELETE", "OPTIONS")
// List orgs
apiRouter.Handle("/enterprises/", log(logWriter, http.HandlerFunc(han.ListEnterprisesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises", log(logWriter, http.HandlerFunc(han.ListEnterprisesHandler))).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises/", http.HandlerFunc(han.ListEnterprisesHandler)).Methods("GET", "OPTIONS")
apiRouter.Handle("/enterprises", http.HandlerFunc(han.ListEnterprisesHandler)).Methods("GET", "OPTIONS")
// Create org
apiRouter.Handle("/enterprises/", log(logWriter, http.HandlerFunc(han.CreateEnterpriseHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/enterprises", log(logWriter, http.HandlerFunc(han.CreateEnterpriseHandler))).Methods("POST", "OPTIONS")
apiRouter.Handle("/enterprises/", http.HandlerFunc(han.CreateEnterpriseHandler)).Methods("POST", "OPTIONS")
apiRouter.Handle("/enterprises", http.HandlerFunc(han.CreateEnterpriseHandler)).Methods("POST", "OPTIONS")
// Credentials and providers
apiRouter.Handle("/credentials/", log(logWriter, http.HandlerFunc(han.ListCredentials))).Methods("GET", "OPTIONS")
apiRouter.Handle("/credentials", log(logWriter, http.HandlerFunc(han.ListCredentials))).Methods("GET", "OPTIONS")
apiRouter.Handle("/providers/", log(logWriter, http.HandlerFunc(han.ListProviders))).Methods("GET", "OPTIONS")
apiRouter.Handle("/providers", log(logWriter, http.HandlerFunc(han.ListProviders))).Methods("GET", "OPTIONS")
apiRouter.Handle("/credentials/", http.HandlerFunc(han.ListCredentials)).Methods("GET", "OPTIONS")
apiRouter.Handle("/credentials", http.HandlerFunc(han.ListCredentials)).Methods("GET", "OPTIONS")
apiRouter.Handle("/providers/", http.HandlerFunc(han.ListProviders)).Methods("GET", "OPTIONS")
apiRouter.Handle("/providers", http.HandlerFunc(han.ListProviders)).Methods("GET", "OPTIONS")
// Websocket log writer
apiRouter.Handle("/{ws:ws\\/?}", log(logWriter, http.HandlerFunc(han.WSHandler))).Methods("GET")
apiRouter.Handle("/{ws:ws\\/?}", http.HandlerFunc(han.WSHandler)).Methods("GET")
return router
}

View file

@ -106,7 +106,8 @@ type InstanceStore interface {
ListAllInstances(ctx context.Context) ([]params.Instance, error)
GetInstanceByName(ctx context.Context, instanceName string) (params.Instance, error)
AddInstanceStatusMessage(ctx context.Context, instanceID string, statusMessage string) error
AddInstanceEvent(ctx context.Context, instanceID string, event params.EventType, eventLevel params.EventLevel, eventMessage string) error
ListInstanceEvents(ctx context.Context, instanceID string, eventType params.EventType, eventLevel params.EventLevel) ([]params.StatusMessage, error)
}
//go:generate mockery --name=Store

View file

@ -14,13 +14,13 @@ type Store struct {
mock.Mock
}
// AddInstanceStatusMessage provides a mock function with given fields: ctx, instanceID, statusMessage
func (_m *Store) AddInstanceStatusMessage(ctx context.Context, instanceID string, statusMessage string) error {
ret := _m.Called(ctx, instanceID, statusMessage)
// AddInstanceEvent provides a mock function with given fields: ctx, instanceID, event, statusMessage
func (_m *Store) AddInstanceEvent(ctx context.Context, instanceID string, event params.EventType, statusMessage string) error {
ret := _m.Called(ctx, instanceID, event, statusMessage)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok {
r0 = rf(ctx, instanceID, statusMessage)
if rf, ok := ret.Get(0).(func(context.Context, string, params.EventType, string) error); ok {
r0 = rf(ctx, instanceID, event, statusMessage)
} else {
r0 = ret.Error(0)
}

View file

@ -141,14 +141,42 @@ func (s *sqlDatabase) DeleteInstance(ctx context.Context, poolID string, instanc
return nil
}
func (s *sqlDatabase) AddInstanceStatusMessage(ctx context.Context, instanceID string, statusMessage string) error {
func (s *sqlDatabase) ListInstanceEvents(ctx context.Context, instanceID string, eventType params.EventType, eventLevel params.EventLevel) ([]params.StatusMessage, error) {
var events []InstanceStatusUpdate
query := s.conn.Model(&InstanceStatusUpdate{}).Where("instance_id = ?", instanceID)
if eventLevel != "" {
query = query.Where("event_level = ?", eventLevel)
}
if eventType != "" {
query = query.Where("event_type = ?", eventType)
}
if result := query.Find(&events); result.Error != nil {
return nil, errors.Wrap(result.Error, "fetching events")
}
eventParams := make([]params.StatusMessage, len(events))
for idx, val := range events {
eventParams[idx] = params.StatusMessage{
Message: val.Message,
EventType: val.EventType,
EventLevel: val.EventLevel,
}
}
return eventParams, nil
}
func (s *sqlDatabase) AddInstanceEvent(ctx context.Context, instanceID string, event params.EventType, eventLevel params.EventLevel, statusMessage string) error {
instance, err := s.getInstanceByID(ctx, instanceID)
if err != nil {
return errors.Wrap(err, "updating instance")
}
msg := InstanceStatusUpdate{
Message: statusMessage,
Message: statusMessage,
EventType: event,
EventLevel: eventLevel,
}
if err := s.conn.Model(&instance).Association("StatusMessages").Append(&msg); err != nil {
@ -214,13 +242,20 @@ func (s *sqlDatabase) UpdateInstance(ctx context.Context, instanceID string, par
}
func (s *sqlDatabase) ListPoolInstances(ctx context.Context, poolID string) ([]params.Instance, error) {
pool, err := s.getPoolByID(ctx, poolID, "Tags", "Instances")
u, err := uuid.FromString(poolID)
if err != nil {
return nil, errors.Wrap(err, "fetching pool")
return nil, errors.Wrap(runnerErrors.ErrBadRequest, "parsing id")
}
ret := make([]params.Instance, len(pool.Instances))
for idx, inst := range pool.Instances {
var instances []Instance
query := s.conn.Model(&Instance{}).Where("pool_id = ?", u)
if err := query.Find(&instances); err.Error != nil {
return nil, errors.Wrap(err.Error, "fetching instances")
}
ret := make([]params.Instance, len(instances))
for idx, inst := range instances {
ret[idx] = s.sqlToParamsInstance(inst)
}
return ret, nil

View file

@ -343,11 +343,11 @@ func (s *InstancesTestSuite) TestDeleteInstanceDBDeleteErr() {
s.Require().Equal("deleting instance: mocked delete instance error", err.Error())
}
func (s *InstancesTestSuite) TestAddInstanceStatusMessage() {
func (s *InstancesTestSuite) TestAddInstanceEvent() {
storeInstance := s.Fixtures.Instances[0]
statusMsg := "test-status-message"
err := s.Store.AddInstanceStatusMessage(context.Background(), storeInstance.ID, statusMsg)
err := s.Store.AddInstanceEvent(context.Background(), storeInstance.ID, params.StatusEvent, params.EventInfo, statusMsg)
s.Require().Nil(err)
instance, err := s.Store.GetInstanceByName(context.Background(), storeInstance.Name)
@ -358,13 +358,13 @@ func (s *InstancesTestSuite) TestAddInstanceStatusMessage() {
s.Require().Equal(statusMsg, instance.StatusMessages[0].Message)
}
func (s *InstancesTestSuite) TestAddInstanceStatusMessageInvalidPoolID() {
err := s.Store.AddInstanceStatusMessage(context.Background(), "dummy-id", "dummy-message")
func (s *InstancesTestSuite) TestAddInstanceEventInvalidPoolID() {
err := s.Store.AddInstanceEvent(context.Background(), "dummy-id", params.StatusEvent, params.EventInfo, "dummy-message")
s.Require().Equal("updating instance: parsing id: invalid request", err.Error())
}
func (s *InstancesTestSuite) TestAddInstanceStatusMessageDBUpdateErr() {
func (s *InstancesTestSuite) TestAddInstanceEventDBUpdateErr() {
instance := s.Fixtures.Instances[0]
statusMsg := "test-status-message"
@ -390,7 +390,7 @@ func (s *InstancesTestSuite) TestAddInstanceStatusMessageDBUpdateErr() {
WillReturnError(fmt.Errorf("mocked add status message error"))
s.Fixtures.SQLMock.ExpectRollback()
err := s.StoreSQLMocked.AddInstanceStatusMessage(context.Background(), instance.ID, statusMsg)
err := s.StoreSQLMocked.AddInstanceEvent(context.Background(), instance.ID, params.StatusEvent, params.EventInfo, statusMsg)
s.assertSQLMockExpectations()
s.Require().NotNil(err)
@ -496,7 +496,7 @@ func (s *InstancesTestSuite) TestListPoolInstances() {
func (s *InstancesTestSuite) TestListPoolInstancesInvalidPoolID() {
_, err := s.Store.ListPoolInstances(context.Background(), "dummy-pool-id")
s.Require().Equal("fetching pool: parsing id: invalid request", err.Error())
s.Require().Equal("parsing id: invalid request", err.Error())
}
func (s *InstancesTestSuite) TestListAllInstances() {

View file

@ -16,6 +16,7 @@ package sql
import (
"garm/config"
"garm/params"
"garm/runner/providers/common"
"time"
@ -118,7 +119,9 @@ type Address struct {
type InstanceStatusUpdate struct {
Base
Message string `gorm:"type:text"`
EventType params.EventType `gorm:"index:eventType"`
EventLevel params.EventLevel
Message string `gorm:"type:text"`
InstanceID uuid.UUID
Instance Instance `gorm:"foreignKey:InstanceID"`

View file

@ -57,8 +57,10 @@ func (s *sqlDatabase) sqlToParamsInstance(instance Instance) params.Instance {
for _, msg := range instance.StatusMessages {
ret.StatusMessages = append(ret.StatusMessages, params.StatusMessage{
CreatedAt: msg.CreatedAt,
Message: msg.Message,
CreatedAt: msg.CreatedAt,
Message: msg.Message,
EventType: msg.EventType,
EventLevel: msg.EventLevel,
})
}
return ret

View file

@ -24,20 +24,35 @@ import (
)
type AddressType string
type EventType string
type EventLevel string
const (
PublicAddress AddressType = "public"
PrivateAddress AddressType = "private"
)
const (
StatusEvent EventType = "status"
FetchTokenEvent EventType = "fetchToken"
)
const (
EventInfo EventLevel = "info"
EventWarning EventLevel = "warning"
EventError EventLevel = "error"
)
type Address struct {
Address string `json:"address"`
Type AddressType `json:"type"`
}
type StatusMessage struct {
CreatedAt time.Time `json:"created_at"`
Message string `json:"message"`
CreatedAt time.Time `json:"created_at"`
Message string `json:"message"`
EventType EventType `json:"event_type"`
EventLevel EventLevel `json:"event_level"`
}
type Instance struct {
@ -264,3 +279,8 @@ type PoolManagerStatus struct {
IsRunning bool `json:"running"`
FailureReason string `json:"failure_reason,omitempty"`
}
type RunnerInfo struct {
Name string
Labels []string
}

View file

@ -61,18 +61,25 @@ type enterprise struct {
mux sync.Mutex
}
func (r *enterprise) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) {
func (r *enterprise) GetRunnerInfoFromWorkflow(job params.WorkflowJob) (params.RunnerInfo, error) {
if err := r.ValidateOwner(job); err != nil {
return params.RunnerInfo{}, errors.Wrap(err, "validating owner")
}
workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID)
if err != nil {
if ghResp.StatusCode == http.StatusUnauthorized {
return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runners")
return params.RunnerInfo{}, errors.Wrap(runnerErrors.ErrUnauthorized, "fetching workflow info")
}
return "", errors.Wrap(err, "fetching workflow info")
return params.RunnerInfo{}, errors.Wrap(err, "fetching workflow info")
}
if workflow.RunnerName != nil {
return *workflow.RunnerName, nil
return params.RunnerInfo{
Name: *workflow.RunnerName,
Labels: workflow.Labels,
}, nil
}
return "", fmt.Errorf("failed to find runner name from workflow")
return params.RunnerInfo{}, fmt.Errorf("failed to find runner name from workflow")
}
func (r *enterprise) UpdateState(param params.UpdatePoolStateParams) error {

View file

@ -24,7 +24,7 @@ type poolHelper interface {
GetGithubToken() string
GetGithubRunners() ([]*github.Runner, error)
GetGithubRegistrationToken() (string, error)
GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error)
GetRunnerInfoFromWorkflow(job params.WorkflowJob) (params.RunnerInfo, error)
RemoveGithubRunner(runnerID int64) (*github.Response, error)
FetchTools() ([]*github.RunnerApplicationDownload, error)

View file

@ -73,18 +73,25 @@ type organization struct {
mux sync.Mutex
}
func (r *organization) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) {
func (r *organization) GetRunnerInfoFromWorkflow(job params.WorkflowJob) (params.RunnerInfo, error) {
if err := r.ValidateOwner(job); err != nil {
return params.RunnerInfo{}, errors.Wrap(err, "validating owner")
}
workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Organization.Login, job.Repository.Name, job.WorkflowJob.ID)
if err != nil {
if ghResp.StatusCode == http.StatusUnauthorized {
return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runner name")
return params.RunnerInfo{}, errors.Wrap(runnerErrors.ErrUnauthorized, "fetching workflow info")
}
return "", errors.Wrap(err, "fetching workflow info")
return params.RunnerInfo{}, errors.Wrap(err, "fetching workflow info")
}
if workflow.RunnerName != nil {
return *workflow.RunnerName, nil
return params.RunnerInfo{
Name: *workflow.RunnerName,
Labels: workflow.Labels,
}, nil
}
return "", fmt.Errorf("failed to find runner name from workflow")
return params.RunnerInfo{}, fmt.Errorf("failed to find runner name from workflow")
}
func (r *organization) UpdateState(param params.UpdatePoolStateParams) error {

View file

@ -67,16 +67,57 @@ type basePoolManager struct {
mux sync.Mutex
}
func controllerIDFromLabels(labels []*github.RunnerLabels) string {
func controllerIDFromLabels(labels []string) string {
for _, lbl := range labels {
if lbl.Name != nil && strings.HasPrefix(*lbl.Name, controllerLabelPrefix) {
labelName := *lbl.Name
return labelName[len(controllerLabelPrefix):]
if strings.HasPrefix(lbl, controllerLabelPrefix) {
return lbl[len(controllerLabelPrefix):]
}
}
return ""
}
func poolIDFromLabels(labels []string) string {
for _, lbl := range labels {
if strings.HasPrefix(lbl, poolIDLabelprefix) {
return lbl[len(poolIDLabelprefix):]
}
}
return ""
}
func poolIsOwnedBy(pool params.Pool, ownerID string) bool {
return pool.RepoID == ownerID || pool.OrgID == ownerID || pool.EnterpriseID == ownerID
}
func labelsFromRunner(runner *github.Runner) []string {
if runner == nil || runner.Labels == nil {
return []string{}
}
var labels []string
for _, val := range runner.Labels {
if val == nil {
continue
}
labels = append(labels, val.GetName())
}
return labels
}
func (r *basePoolManager) isControllerRunner(labels []string) bool {
runnerControllerID := controllerIDFromLabels(labels)
if runnerControllerID != r.controllerID {
return false
}
poolID := poolIDFromLabels(labels)
if poolID == "" {
return false
}
_, err := r.helper.GetPoolByID(poolID)
return err == nil
}
// cleanupOrphanedProviderRunners compares runners in github with local runners and removes
// any local runners that are not present in Github. Runners that are "idle" in our
// provider, but do not exist in github, will be removed. This can happen if the
@ -92,6 +133,9 @@ func (r *basePoolManager) cleanupOrphanedProviderRunners(runners []*github.Runne
runnerNames := map[string]bool{}
for _, run := range runners {
if !r.isControllerRunner(labelsFromRunner(run)) {
continue
}
runnerNames[*run.Name] = true
}
@ -127,24 +171,25 @@ func (r *basePoolManager) reapTimedOutRunners(runners []*github.Runner) error {
runnerNames := map[string]bool{}
for _, run := range runners {
if !r.isControllerRunner(labelsFromRunner(run)) {
continue
}
runnerNames[*run.Name] = true
}
for _, instance := range dbInstances {
if ok := runnerNames[instance.Name]; !ok {
if instance.Status == providerCommon.InstanceRunning {
pool, err := r.store.GetPoolByID(r.ctx, instance.PoolID)
if err != nil {
return errors.Wrap(err, "fetching instance pool info")
}
if time.Since(instance.UpdatedAt).Minutes() < float64(pool.RunnerTimeout()) {
continue
}
log.Printf("reaping instance %s due to timeout", instance.Name)
if err := r.setInstanceStatus(instance.Name, providerCommon.InstancePendingDelete, nil); err != nil {
log.Printf("failed to update runner %s status", instance.Name)
return errors.Wrap(err, "updating runner")
}
pool, err := r.store.GetPoolByID(r.ctx, instance.PoolID)
if err != nil {
return errors.Wrap(err, "fetching instance pool info")
}
if time.Since(instance.UpdatedAt).Minutes() < float64(pool.RunnerTimeout()) {
continue
}
log.Printf("reaping instance %s due to timeout", instance.Name)
if err := r.setInstanceStatus(instance.Name, providerCommon.InstancePendingDelete, nil); err != nil {
log.Printf("failed to update runner %s status", instance.Name)
return errors.Wrap(err, "updating runner")
}
}
}
@ -157,9 +202,7 @@ func (r *basePoolManager) reapTimedOutRunners(runners []*github.Runner) error {
// first remove the instance from github, and then from our database.
func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner) error {
for _, runner := range runners {
runnerControllerID := controllerIDFromLabels(runner.Labels)
if runnerControllerID != r.controllerID {
// Not a runner we manage. Do not remove foreign runner.
if !r.isControllerRunner(labelsFromRunner(runner)) {
continue
}
@ -205,6 +248,9 @@ func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner)
if err != nil {
return errors.Wrap(err, "fetching pool")
}
if !poolIsOwnedBy(pool, r.ID()) {
continue
}
// check if the provider still has the instance.
provider, ok := r.providers[pool.ProviderName]
@ -265,6 +311,11 @@ func (r *basePoolManager) fetchInstance(runnerName string) (params.Instance, err
return params.Instance{}, errors.Wrap(err, "fetching instance")
}
_, err = r.helper.GetPoolByID(runner.PoolID)
if err != nil {
return params.Instance{}, errors.Wrap(err, "fetching pool")
}
return runner, nil
}
@ -312,6 +363,9 @@ func (r *basePoolManager) acquireNewInstance(job params.WorkflowJob) error {
pool, err := r.helper.FindPoolByTags(requestedLabels)
if err != nil {
if errors.Is(err, runnerErrors.ErrNotFound) {
return nil
}
return errors.Wrap(err, "fetching suitable pool")
}
log.Printf("adding new runner with requested tags %s in pool %s", strings.Join(job.WorkflowJob.Labels, ", "), pool.ID)
@ -572,25 +626,28 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error
return nil
}
func (r *basePoolManager) getRunnerNameFromJob(job params.WorkflowJob) (string, error) {
func (r *basePoolManager) getRunnerDetailsFromJob(job params.WorkflowJob) (params.RunnerInfo, error) {
if job.WorkflowJob.RunnerName != "" {
return job.WorkflowJob.RunnerName, nil
return params.RunnerInfo{
Name: job.WorkflowJob.RunnerName,
Labels: job.WorkflowJob.Labels,
}, nil
}
// Runner name was not set in WorkflowJob by github. We can still attempt to
// fetch the info we need, using the workflow run ID, from the API.
log.Printf("runner name not found in workflow job, attempting to fetch from API")
runnerName, err := r.helper.GetRunnerNameFromWorkflow(job)
runnerInfo, err := r.helper.GetRunnerInfoFromWorkflow(job)
if err != nil {
if errors.Is(err, runnerErrors.ErrUnauthorized) {
failureReason := fmt.Sprintf("failed to fetch runner name from API: %q", err)
r.setPoolRunningState(false, failureReason)
log.Print(failureReason)
}
return "", errors.Wrap(err, "fetching runner name from API")
return params.RunnerInfo{}, errors.Wrap(err, "fetching runner name from API")
}
return runnerName, nil
return runnerInfo, nil
}
func (r *basePoolManager) HandleWorkflowJob(job params.WorkflowJob) error {
@ -612,34 +669,52 @@ func (r *basePoolManager) HandleWorkflowJob(job params.WorkflowJob) error {
case "completed":
// ignore the error here. A completed job may not have a runner name set
// if it was never assigned to a runner, and was canceled.
runnerName, _ := r.getRunnerNameFromJob(job)
// Set instance in database to pending delete.
if runnerName == "" {
runnerInfo, err := r.getRunnerDetailsFromJob(job)
if err != nil {
// Unassigned jobs will have an empty runner_name.
// There is nothing to to in this case.
log.Printf("no runner was assigned. Skipping.")
return nil
}
if !r.isControllerRunner(runnerInfo.Labels) {
return nil
}
// update instance workload state.
if err := r.setInstanceRunnerStatus(runnerName, providerCommon.RunnerTerminated); err != nil {
log.Printf("failed to update runner %s status", runnerName)
if err := r.setInstanceRunnerStatus(runnerInfo.Name, providerCommon.RunnerTerminated); err != nil {
if errors.Is(err, runnerErrors.ErrNotFound) {
return nil
}
log.Printf("failed to update runner %s status", runnerInfo.Name)
return errors.Wrap(err, "updating runner")
}
log.Printf("marking instance %s as pending_delete", runnerName)
if err := r.setInstanceStatus(runnerName, providerCommon.InstancePendingDelete, nil); err != nil {
log.Printf("failed to update runner %s status", runnerName)
log.Printf("marking instance %s as pending_delete", runnerInfo.Name)
if err := r.setInstanceStatus(runnerInfo.Name, providerCommon.InstancePendingDelete, nil); err != nil {
if errors.Is(err, runnerErrors.ErrNotFound) {
return nil
}
log.Printf("failed to update runner %s status", runnerInfo.Name)
return errors.Wrap(err, "updating runner")
}
case "in_progress":
// in_progress jobs must have a runner name/ID assigned. Sometimes github will send a hook without
// a runner set. In such cases, we attemt to fetch it from the API.
runnerName, err := r.getRunnerNameFromJob(job)
runnerInfo, err := r.getRunnerDetailsFromJob(job)
if err != nil {
return errors.Wrap(err, "determining runner name")
}
if !r.isControllerRunner(runnerInfo.Labels) {
return nil
}
// update instance workload state.
if err := r.setInstanceRunnerStatus(runnerName, providerCommon.RunnerActive); err != nil {
log.Printf("failed to update runner %s status", job.WorkflowJob.RunnerName)
if err := r.setInstanceRunnerStatus(runnerInfo.Name, providerCommon.RunnerActive); err != nil {
if errors.Is(err, runnerErrors.ErrNotFound) {
return nil
}
log.Printf("failed to update runner %s status", runnerInfo.Name)
return errors.Wrap(err, "updating runner")
}
}
@ -716,10 +791,11 @@ func (r *basePoolManager) retryFailedInstancesForOnePool(pool params.Pool) {
existingInstances, err := r.store.ListPoolInstances(r.ctx, pool.ID)
if err != nil {
log.Printf("failed to ensure minimum idle workers for pool %s: %s", pool.ID, err)
log.Printf("retrying failed instances: failed to list instances for pool %s: %s", pool.ID, err)
return
}
wg := sync.WaitGroup{}
for _, instance := range existingInstances {
if instance.Status != providerCommon.InstanceError {
continue
@ -727,28 +803,31 @@ func (r *basePoolManager) retryFailedInstancesForOnePool(pool params.Pool) {
if instance.CreateAttempt >= maxCreateAttempts {
continue
}
wg.Add(1)
go func(inst params.Instance) {
defer wg.Done()
// NOTE(gabriel-samfira): this is done in parallel. If there are many failed instances
// this has the potential to create many API requests to the target provider.
// TODO(gabriel-samfira): implement request throttling.
if err := r.deleteInstanceFromProvider(inst); err != nil {
log.Printf("failed to delete instance %s from provider: %s", inst.Name, err)
}
// NOTE(gabriel-samfira): this is done in parallel. If there are many failed instances
// this has the potential to create many API requests to the target provider.
// TODO(gabriel-samfira): implement request throttling.
if instance.ProviderID == "" && instance.Name == "" {
// This really should not happen, but no harm in being extra paranoid. The name is set
// when creating a db entity for the runner, so we should at least have a name.
return
}
// TODO(gabriel-samfira): Incrementing CreateAttempt should be done within a transaction.
// It's fairly safe to do here (for now), as there should be no other code path that updates
// an instance in this state.
updateParams := params.UpdateInstanceParams{
CreateAttempt: instance.CreateAttempt + 1,
Status: providerCommon.InstancePendingCreate,
}
log.Printf("queueing previously failed instance %s for retry", instance.Name)
// Set instance to pending create and wait for retry.
if err := r.updateInstance(instance.Name, updateParams); err != nil {
log.Printf("failed to update runner %s status", instance.Name)
}
// TODO(gabriel-samfira): Incrementing CreateAttempt should be done within a transaction.
// It's fairly safe to do here (for now), as there should be no other code path that updates
// an instance in this state.
updateParams := params.UpdateInstanceParams{
CreateAttempt: inst.CreateAttempt + 1,
Status: providerCommon.InstancePendingCreate,
}
log.Printf("queueing previously failed instance %s for retry", inst.Name)
// Set instance to pending create and wait for retry.
if err := r.updateInstance(inst.Name, updateParams); err != nil {
log.Printf("failed to update runner %s status", inst.Name)
}
}(instance)
}
wg.Wait()
}
func (r *basePoolManager) retryFailedInstances() {
@ -807,9 +886,6 @@ func (r *basePoolManager) deleteInstanceFromProvider(instance params.Instance) e
return errors.Wrap(err, "removing instance")
}
if err := r.store.DeleteInstance(r.ctx, pool.ID, instance.Name); err != nil {
return errors.Wrap(err, "deleting instance from database")
}
return nil
}
@ -846,6 +922,10 @@ func (r *basePoolManager) deletePendingInstances() {
if err != nil {
log.Printf("failed to delete instance from provider: %+v", err)
}
if err := r.store.DeleteInstance(r.ctx, instance.PoolID, instance.Name); err != nil {
return errors.Wrap(err, "deleting instance from database")
}
return
}(instance)
}

View file

@ -75,18 +75,25 @@ type repository struct {
mux sync.Mutex
}
func (r *repository) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) {
func (r *repository) GetRunnerInfoFromWorkflow(job params.WorkflowJob) (params.RunnerInfo, error) {
if err := r.ValidateOwner(job); err != nil {
return params.RunnerInfo{}, errors.Wrap(err, "validating owner")
}
workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID)
if err != nil {
if ghResp.StatusCode == http.StatusUnauthorized {
return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runner name")
return params.RunnerInfo{}, errors.Wrap(runnerErrors.ErrUnauthorized, "fetching workflow info")
}
return "", errors.Wrap(err, "fetching workflow info")
return params.RunnerInfo{}, errors.Wrap(err, "fetching workflow info")
}
if workflow.RunnerName != nil {
return *workflow.RunnerName, nil
return params.RunnerInfo{
Name: *workflow.RunnerName,
Labels: workflow.Labels,
}, nil
}
return "", fmt.Errorf("failed to find runner name from workflow")
return params.RunnerInfo{}, fmt.Errorf("failed to find runner name from workflow")
}
func (r *repository) UpdateState(param params.UpdatePoolStateParams) error {

View file

@ -584,8 +584,10 @@ func (r *Runner) DispatchWorkflowJob(hookTargetType, signature string, jobData [
switch HookTargetType(hookTargetType) {
case RepoHook:
log.Printf("got hook for repo %s/%s", job.Repository.Owner.Login, job.Repository.Name)
poolManager, err = r.findRepoPoolManager(job.Repository.Owner.Login, job.Repository.Name)
case OrganizationHook:
log.Printf("got hook for org %s", job.Organization.Login)
poolManager, err = r.findOrgPoolManager(job.Organization.Login)
case EnterpriseHook:
poolManager, err = r.findEnterprisePoolManager(job.Enterprise.Slug)
@ -704,7 +706,7 @@ func (r *Runner) AddInstanceStatusMessage(ctx context.Context, param params.Inst
return runnerErrors.ErrUnauthorized
}
if err := r.store.AddInstanceStatusMessage(ctx, instanceID, param.Message); err != nil {
if err := r.store.AddInstanceEvent(ctx, instanceID, params.StatusEvent, params.EventInfo, param.Message); err != nil {
return errors.Wrap(err, "adding status update")
}
@ -744,11 +746,25 @@ func (r *Runner) GetInstanceGithubRegistrationToken(ctx context.Context) (string
return "", errors.Wrap(err, "fetching pool manager for instance")
}
tokenEvents, err := r.store.ListInstanceEvents(ctx, instance.ID, params.FetchTokenEvent, "")
if err != nil {
return "", errors.Wrap(err, "fetching instance events")
}
if len(tokenEvents) > 0 {
// Token already retrieved
return "", runnerErrors.ErrUnauthorized
}
token, err := poolMgr.GithubRunnerRegistrationToken()
if err != nil {
return "", errors.Wrap(err, "fetching runner token")
}
if err := r.store.AddInstanceEvent(ctx, instance.ID, params.FetchTokenEvent, params.EventInfo, "runner registration token was retrieved"); err != nil {
return "", errors.Wrap(err, "recording event")
}
return token, nil
}

View file

@ -38,6 +38,7 @@ import (
"garm/runner/common"
"github.com/google/go-github/v48/github"
gorillaHandlers "github.com/gorilla/handlers"
"github.com/pkg/errors"
"golang.org/x/crypto/bcrypt"
"golang.org/x/oauth2"
@ -320,3 +321,9 @@ func PaswsordToBcrypt(password string) (string, error) {
}
return string(hashedPassword), nil
}
func NewLoggingMiddleware(writer io.Writer) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return gorillaHandlers.CombinedLoggingHandler(writer, next)
}
}