From 3a92a5be0e4cc90bd8f6b690eb1072afe56db981 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Thu, 29 Dec 2022 16:49:50 +0000 Subject: [PATCH] Some cleanup and safety checks * Add logging middleware * Remove some noise from logs * Add some safety checks when managing runners --- apiserver/routers/routers.go | 195 ++++++++++++++++---------------- database/common/common.go | 3 +- database/common/mocks/Store.go | 10 +- database/sql/instances.go | 47 +++++++- database/sql/instances_test.go | 14 +-- database/sql/models.go | 5 +- database/sql/util.go | 6 +- params/params.go | 24 +++- runner/pool/enterprise.go | 17 ++- runner/pool/interfaces.go | 2 +- runner/pool/organization.go | 17 ++- runner/pool/pool.go | 200 +++++++++++++++++++++++---------- runner/pool/repository.go | 17 ++- runner/runner.go | 18 ++- util/util.go | 7 ++ 15 files changed, 384 insertions(+), 198 deletions(-) diff --git a/apiserver/routers/routers.go b/apiserver/routers/routers.go index eefc3ac6..c7db9aaa 100644 --- a/apiserver/routers/routers.go +++ b/apiserver/routers/routers.go @@ -18,42 +18,43 @@ import ( "io" "net/http" - gorillaHandlers "github.com/gorilla/handlers" "github.com/gorilla/mux" "garm/apiserver/controllers" "garm/auth" + "garm/util" ) func NewAPIRouter(han *controllers.APIController, logWriter io.Writer, authMiddleware, initMiddleware, instanceMiddleware auth.Middleware) *mux.Router { router := mux.NewRouter() - log := gorillaHandlers.CombinedLoggingHandler + logMiddleware := util.NewLoggingMiddleware(logWriter) + router.Use(logMiddleware) // Handles github webhooks webhookRouter := router.PathPrefix("/webhooks").Subrouter() - webhookRouter.PathPrefix("/").Handler(log(logWriter, http.HandlerFunc(han.CatchAll))) - webhookRouter.PathPrefix("").Handler(log(logWriter, http.HandlerFunc(han.CatchAll))) + webhookRouter.PathPrefix("/").Handler(http.HandlerFunc(han.CatchAll)) + webhookRouter.PathPrefix("").Handler(http.HandlerFunc(han.CatchAll)) // Handles API calls apiSubRouter := router.PathPrefix("/api/v1").Subrouter() // FirstRunHandler firstRunRouter := apiSubRouter.PathPrefix("/first-run").Subrouter() - firstRunRouter.Handle("/", log(logWriter, http.HandlerFunc(han.FirstRunHandler))).Methods("POST", "OPTIONS") + firstRunRouter.Handle("/", http.HandlerFunc(han.FirstRunHandler)).Methods("POST", "OPTIONS") // Instance URLs callbackRouter := apiSubRouter.PathPrefix("/callbacks").Subrouter() - callbackRouter.Handle("/status/", log(logWriter, http.HandlerFunc(han.InstanceStatusMessageHandler))).Methods("POST", "OPTIONS") - callbackRouter.Handle("/status", log(logWriter, http.HandlerFunc(han.InstanceStatusMessageHandler))).Methods("POST", "OPTIONS") + callbackRouter.Handle("/status/", http.HandlerFunc(han.InstanceStatusMessageHandler)).Methods("POST", "OPTIONS") + callbackRouter.Handle("/status", http.HandlerFunc(han.InstanceStatusMessageHandler)).Methods("POST", "OPTIONS") callbackRouter.Use(instanceMiddleware.Middleware) metadataRouter := apiSubRouter.PathPrefix("/metadata").Subrouter() - metadataRouter.Handle("/runner-registration-token/", log(logWriter, http.HandlerFunc(han.InstanceGithubRegistrationTokenHandler))).Methods("GET", "OPTIONS") - metadataRouter.Handle("/runner-registration-token", log(logWriter, http.HandlerFunc(han.InstanceGithubRegistrationTokenHandler))).Methods("GET", "OPTIONS") + metadataRouter.Handle("/runner-registration-token/", http.HandlerFunc(han.InstanceGithubRegistrationTokenHandler)).Methods("GET", "OPTIONS") + metadataRouter.Handle("/runner-registration-token", http.HandlerFunc(han.InstanceGithubRegistrationTokenHandler)).Methods("GET", "OPTIONS") metadataRouter.Use(instanceMiddleware.Middleware) // Login authRouter := apiSubRouter.PathPrefix("/auth").Subrouter() - authRouter.Handle("/{login:login\\/?}", log(logWriter, http.HandlerFunc(han.LoginHandler))).Methods("POST", "OPTIONS") + authRouter.Handle("/{login:login\\/?}", http.HandlerFunc(han.LoginHandler)).Methods("POST", "OPTIONS") authRouter.Use(initMiddleware.Middleware) apiRouter := apiSubRouter.PathPrefix("").Subrouter() @@ -64,158 +65,158 @@ func NewAPIRouter(han *controllers.APIController, logWriter io.Writer, authMiddl // Pools // /////////// // List all pools - apiRouter.Handle("/pools/", log(logWriter, http.HandlerFunc(han.ListAllPoolsHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/pools", log(logWriter, http.HandlerFunc(han.ListAllPoolsHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/pools/", http.HandlerFunc(han.ListAllPoolsHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/pools", http.HandlerFunc(han.ListAllPoolsHandler)).Methods("GET", "OPTIONS") // Get one pool - apiRouter.Handle("/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.GetPoolByIDHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/pools/{poolID}", log(logWriter, http.HandlerFunc(han.GetPoolByIDHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/pools/{poolID}/", http.HandlerFunc(han.GetPoolByIDHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/pools/{poolID}", http.HandlerFunc(han.GetPoolByIDHandler)).Methods("GET", "OPTIONS") // Delete one pool - apiRouter.Handle("/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.DeletePoolByIDHandler))).Methods("DELETE", "OPTIONS") - apiRouter.Handle("/pools/{poolID}", log(logWriter, http.HandlerFunc(han.DeletePoolByIDHandler))).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/pools/{poolID}/", http.HandlerFunc(han.DeletePoolByIDHandler)).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/pools/{poolID}", http.HandlerFunc(han.DeletePoolByIDHandler)).Methods("DELETE", "OPTIONS") // Update one pool - apiRouter.Handle("/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.UpdatePoolByIDHandler))).Methods("PUT", "OPTIONS") - apiRouter.Handle("/pools/{poolID}", log(logWriter, http.HandlerFunc(han.UpdatePoolByIDHandler))).Methods("PUT", "OPTIONS") + apiRouter.Handle("/pools/{poolID}/", http.HandlerFunc(han.UpdatePoolByIDHandler)).Methods("PUT", "OPTIONS") + apiRouter.Handle("/pools/{poolID}", http.HandlerFunc(han.UpdatePoolByIDHandler)).Methods("PUT", "OPTIONS") // List pool instances - apiRouter.Handle("/pools/{poolID}/instances/", log(logWriter, http.HandlerFunc(han.ListPoolInstancesHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/pools/{poolID}/instances", log(logWriter, http.HandlerFunc(han.ListPoolInstancesHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/pools/{poolID}/instances/", http.HandlerFunc(han.ListPoolInstancesHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/pools/{poolID}/instances", http.HandlerFunc(han.ListPoolInstancesHandler)).Methods("GET", "OPTIONS") ///////////// // Runners // ///////////// // Get instance - apiRouter.Handle("/instances/{instanceName}/", log(logWriter, http.HandlerFunc(han.GetInstanceHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/instances/{instanceName}", log(logWriter, http.HandlerFunc(han.GetInstanceHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/instances/{instanceName}/", http.HandlerFunc(han.GetInstanceHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/instances/{instanceName}", http.HandlerFunc(han.GetInstanceHandler)).Methods("GET", "OPTIONS") // Delete runner - apiRouter.Handle("/instances/{instanceName}/", log(logWriter, http.HandlerFunc(han.DeleteInstanceHandler))).Methods("DELETE", "OPTIONS") - apiRouter.Handle("/instances/{instanceName}", log(logWriter, http.HandlerFunc(han.DeleteInstanceHandler))).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/instances/{instanceName}/", http.HandlerFunc(han.DeleteInstanceHandler)).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/instances/{instanceName}", http.HandlerFunc(han.DeleteInstanceHandler)).Methods("DELETE", "OPTIONS") // List runners - apiRouter.Handle("/instances/", log(logWriter, http.HandlerFunc(han.ListAllInstancesHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/instances", log(logWriter, http.HandlerFunc(han.ListAllInstancesHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/instances/", http.HandlerFunc(han.ListAllInstancesHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/instances", http.HandlerFunc(han.ListAllInstancesHandler)).Methods("GET", "OPTIONS") ///////////////////// // Repos and pools // ///////////////////// // Get pool - apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.GetRepoPoolHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.GetRepoPoolHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", http.HandlerFunc(han.GetRepoPoolHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", http.HandlerFunc(han.GetRepoPoolHandler)).Methods("GET", "OPTIONS") // Delete pool - apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.DeleteRepoPoolHandler))).Methods("DELETE", "OPTIONS") - apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.DeleteRepoPoolHandler))).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", http.HandlerFunc(han.DeleteRepoPoolHandler)).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", http.HandlerFunc(han.DeleteRepoPoolHandler)).Methods("DELETE", "OPTIONS") // Update pool - apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.UpdateRepoPoolHandler))).Methods("PUT", "OPTIONS") - apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.UpdateRepoPoolHandler))).Methods("PUT", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/pools/{poolID}/", http.HandlerFunc(han.UpdateRepoPoolHandler)).Methods("PUT", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/pools/{poolID}", http.HandlerFunc(han.UpdateRepoPoolHandler)).Methods("PUT", "OPTIONS") // List pools - apiRouter.Handle("/repositories/{repoID}/pools/", log(logWriter, http.HandlerFunc(han.ListRepoPoolsHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/repositories/{repoID}/pools", log(logWriter, http.HandlerFunc(han.ListRepoPoolsHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/pools/", http.HandlerFunc(han.ListRepoPoolsHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/pools", http.HandlerFunc(han.ListRepoPoolsHandler)).Methods("GET", "OPTIONS") // Create pool - apiRouter.Handle("/repositories/{repoID}/pools/", log(logWriter, http.HandlerFunc(han.CreateRepoPoolHandler))).Methods("POST", "OPTIONS") - apiRouter.Handle("/repositories/{repoID}/pools", log(logWriter, http.HandlerFunc(han.CreateRepoPoolHandler))).Methods("POST", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/pools/", http.HandlerFunc(han.CreateRepoPoolHandler)).Methods("POST", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/pools", http.HandlerFunc(han.CreateRepoPoolHandler)).Methods("POST", "OPTIONS") // Repo instances list - apiRouter.Handle("/repositories/{repoID}/instances/", log(logWriter, http.HandlerFunc(han.ListRepoInstancesHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/repositories/{repoID}/instances", log(logWriter, http.HandlerFunc(han.ListRepoInstancesHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/instances/", http.HandlerFunc(han.ListRepoInstancesHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/instances", http.HandlerFunc(han.ListRepoInstancesHandler)).Methods("GET", "OPTIONS") // Get repo - apiRouter.Handle("/repositories/{repoID}/", log(logWriter, http.HandlerFunc(han.GetRepoByIDHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/repositories/{repoID}", log(logWriter, http.HandlerFunc(han.GetRepoByIDHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/", http.HandlerFunc(han.GetRepoByIDHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}", http.HandlerFunc(han.GetRepoByIDHandler)).Methods("GET", "OPTIONS") // Update repo - apiRouter.Handle("/repositories/{repoID}/", log(logWriter, http.HandlerFunc(han.UpdateRepoHandler))).Methods("PUT", "OPTIONS") - apiRouter.Handle("/repositories/{repoID}", log(logWriter, http.HandlerFunc(han.UpdateRepoHandler))).Methods("PUT", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/", http.HandlerFunc(han.UpdateRepoHandler)).Methods("PUT", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}", http.HandlerFunc(han.UpdateRepoHandler)).Methods("PUT", "OPTIONS") // Delete repo - apiRouter.Handle("/repositories/{repoID}/", log(logWriter, http.HandlerFunc(han.DeleteRepoHandler))).Methods("DELETE", "OPTIONS") - apiRouter.Handle("/repositories/{repoID}", log(logWriter, http.HandlerFunc(han.DeleteRepoHandler))).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}/", http.HandlerFunc(han.DeleteRepoHandler)).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/repositories/{repoID}", http.HandlerFunc(han.DeleteRepoHandler)).Methods("DELETE", "OPTIONS") // List repos - apiRouter.Handle("/repositories/", log(logWriter, http.HandlerFunc(han.ListReposHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/repositories", log(logWriter, http.HandlerFunc(han.ListReposHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/repositories/", http.HandlerFunc(han.ListReposHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/repositories", http.HandlerFunc(han.ListReposHandler)).Methods("GET", "OPTIONS") // Create repo - apiRouter.Handle("/repositories/", log(logWriter, http.HandlerFunc(han.CreateRepoHandler))).Methods("POST", "OPTIONS") - apiRouter.Handle("/repositories", log(logWriter, http.HandlerFunc(han.CreateRepoHandler))).Methods("POST", "OPTIONS") + apiRouter.Handle("/repositories/", http.HandlerFunc(han.CreateRepoHandler)).Methods("POST", "OPTIONS") + apiRouter.Handle("/repositories", http.HandlerFunc(han.CreateRepoHandler)).Methods("POST", "OPTIONS") ///////////////////////////// // Organizations and pools // ///////////////////////////// // Get pool - apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.GetOrgPoolHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.GetOrgPoolHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", http.HandlerFunc(han.GetOrgPoolHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", http.HandlerFunc(han.GetOrgPoolHandler)).Methods("GET", "OPTIONS") // Delete pool - apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.DeleteOrgPoolHandler))).Methods("DELETE", "OPTIONS") - apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.DeleteOrgPoolHandler))).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", http.HandlerFunc(han.DeleteOrgPoolHandler)).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", http.HandlerFunc(han.DeleteOrgPoolHandler)).Methods("DELETE", "OPTIONS") // Update pool - apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.UpdateOrgPoolHandler))).Methods("PUT", "OPTIONS") - apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.UpdateOrgPoolHandler))).Methods("PUT", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/pools/{poolID}/", http.HandlerFunc(han.UpdateOrgPoolHandler)).Methods("PUT", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/pools/{poolID}", http.HandlerFunc(han.UpdateOrgPoolHandler)).Methods("PUT", "OPTIONS") // List pools - apiRouter.Handle("/organizations/{orgID}/pools/", log(logWriter, http.HandlerFunc(han.ListOrgPoolsHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/organizations/{orgID}/pools", log(logWriter, http.HandlerFunc(han.ListOrgPoolsHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/pools/", http.HandlerFunc(han.ListOrgPoolsHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/pools", http.HandlerFunc(han.ListOrgPoolsHandler)).Methods("GET", "OPTIONS") // Create pool - apiRouter.Handle("/organizations/{orgID}/pools/", log(logWriter, http.HandlerFunc(han.CreateOrgPoolHandler))).Methods("POST", "OPTIONS") - apiRouter.Handle("/organizations/{orgID}/pools", log(logWriter, http.HandlerFunc(han.CreateOrgPoolHandler))).Methods("POST", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/pools/", http.HandlerFunc(han.CreateOrgPoolHandler)).Methods("POST", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/pools", http.HandlerFunc(han.CreateOrgPoolHandler)).Methods("POST", "OPTIONS") // Repo instances list - apiRouter.Handle("/organizations/{orgID}/instances/", log(logWriter, http.HandlerFunc(han.ListOrgInstancesHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/organizations/{orgID}/instances", log(logWriter, http.HandlerFunc(han.ListOrgInstancesHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/instances/", http.HandlerFunc(han.ListOrgInstancesHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/instances", http.HandlerFunc(han.ListOrgInstancesHandler)).Methods("GET", "OPTIONS") // Get org - apiRouter.Handle("/organizations/{orgID}/", log(logWriter, http.HandlerFunc(han.GetOrgByIDHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/organizations/{orgID}", log(logWriter, http.HandlerFunc(han.GetOrgByIDHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/", http.HandlerFunc(han.GetOrgByIDHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}", http.HandlerFunc(han.GetOrgByIDHandler)).Methods("GET", "OPTIONS") // Update org - apiRouter.Handle("/organizations/{orgID}/", log(logWriter, http.HandlerFunc(han.UpdateOrgHandler))).Methods("PUT", "OPTIONS") - apiRouter.Handle("/organizations/{orgID}", log(logWriter, http.HandlerFunc(han.UpdateOrgHandler))).Methods("PUT", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/", http.HandlerFunc(han.UpdateOrgHandler)).Methods("PUT", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}", http.HandlerFunc(han.UpdateOrgHandler)).Methods("PUT", "OPTIONS") // Delete org - apiRouter.Handle("/organizations/{orgID}/", log(logWriter, http.HandlerFunc(han.DeleteOrgHandler))).Methods("DELETE", "OPTIONS") - apiRouter.Handle("/organizations/{orgID}", log(logWriter, http.HandlerFunc(han.DeleteOrgHandler))).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}/", http.HandlerFunc(han.DeleteOrgHandler)).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/organizations/{orgID}", http.HandlerFunc(han.DeleteOrgHandler)).Methods("DELETE", "OPTIONS") // List orgs - apiRouter.Handle("/organizations/", log(logWriter, http.HandlerFunc(han.ListOrgsHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/organizations", log(logWriter, http.HandlerFunc(han.ListOrgsHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/organizations/", http.HandlerFunc(han.ListOrgsHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/organizations", http.HandlerFunc(han.ListOrgsHandler)).Methods("GET", "OPTIONS") // Create org - apiRouter.Handle("/organizations/", log(logWriter, http.HandlerFunc(han.CreateOrgHandler))).Methods("POST", "OPTIONS") - apiRouter.Handle("/organizations", log(logWriter, http.HandlerFunc(han.CreateOrgHandler))).Methods("POST", "OPTIONS") + apiRouter.Handle("/organizations/", http.HandlerFunc(han.CreateOrgHandler)).Methods("POST", "OPTIONS") + apiRouter.Handle("/organizations", http.HandlerFunc(han.CreateOrgHandler)).Methods("POST", "OPTIONS") ///////////////////////////// // Enterprises and pools // ///////////////////////////// // Get pool - apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.GetEnterprisePoolHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.GetEnterprisePoolHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", http.HandlerFunc(han.GetEnterprisePoolHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", http.HandlerFunc(han.GetEnterprisePoolHandler)).Methods("GET", "OPTIONS") // Delete pool - apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.DeleteEnterprisePoolHandler))).Methods("DELETE", "OPTIONS") - apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.DeleteEnterprisePoolHandler))).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", http.HandlerFunc(han.DeleteEnterprisePoolHandler)).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", http.HandlerFunc(han.DeleteEnterprisePoolHandler)).Methods("DELETE", "OPTIONS") // Update pool - apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", log(logWriter, http.HandlerFunc(han.UpdateEnterprisePoolHandler))).Methods("PUT", "OPTIONS") - apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", log(logWriter, http.HandlerFunc(han.UpdateEnterprisePoolHandler))).Methods("PUT", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}/", http.HandlerFunc(han.UpdateEnterprisePoolHandler)).Methods("PUT", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/pools/{poolID}", http.HandlerFunc(han.UpdateEnterprisePoolHandler)).Methods("PUT", "OPTIONS") // List pools - apiRouter.Handle("/enterprises/{enterpriseID}/pools/", log(logWriter, http.HandlerFunc(han.ListEnterprisePoolsHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/enterprises/{enterpriseID}/pools", log(logWriter, http.HandlerFunc(han.ListEnterprisePoolsHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/pools/", http.HandlerFunc(han.ListEnterprisePoolsHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/pools", http.HandlerFunc(han.ListEnterprisePoolsHandler)).Methods("GET", "OPTIONS") // Create pool - apiRouter.Handle("/enterprises/{enterpriseID}/pools/", log(logWriter, http.HandlerFunc(han.CreateEnterprisePoolHandler))).Methods("POST", "OPTIONS") - apiRouter.Handle("/enterprises/{enterpriseID}/pools", log(logWriter, http.HandlerFunc(han.CreateEnterprisePoolHandler))).Methods("POST", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/pools/", http.HandlerFunc(han.CreateEnterprisePoolHandler)).Methods("POST", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/pools", http.HandlerFunc(han.CreateEnterprisePoolHandler)).Methods("POST", "OPTIONS") // Repo instances list - apiRouter.Handle("/enterprises/{enterpriseID}/instances/", log(logWriter, http.HandlerFunc(han.ListEnterpriseInstancesHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/enterprises/{enterpriseID}/instances", log(logWriter, http.HandlerFunc(han.ListEnterpriseInstancesHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/instances/", http.HandlerFunc(han.ListEnterpriseInstancesHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/instances", http.HandlerFunc(han.ListEnterpriseInstancesHandler)).Methods("GET", "OPTIONS") // Get org - apiRouter.Handle("/enterprises/{enterpriseID}/", log(logWriter, http.HandlerFunc(han.GetEnterpriseByIDHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/enterprises/{enterpriseID}", log(logWriter, http.HandlerFunc(han.GetEnterpriseByIDHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/", http.HandlerFunc(han.GetEnterpriseByIDHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}", http.HandlerFunc(han.GetEnterpriseByIDHandler)).Methods("GET", "OPTIONS") // Update org - apiRouter.Handle("/enterprises/{enterpriseID}/", log(logWriter, http.HandlerFunc(han.UpdateEnterpriseHandler))).Methods("PUT", "OPTIONS") - apiRouter.Handle("/enterprises/{enterpriseID}", log(logWriter, http.HandlerFunc(han.UpdateEnterpriseHandler))).Methods("PUT", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/", http.HandlerFunc(han.UpdateEnterpriseHandler)).Methods("PUT", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}", http.HandlerFunc(han.UpdateEnterpriseHandler)).Methods("PUT", "OPTIONS") // Delete org - apiRouter.Handle("/enterprises/{enterpriseID}/", log(logWriter, http.HandlerFunc(han.DeleteEnterpriseHandler))).Methods("DELETE", "OPTIONS") - apiRouter.Handle("/enterprises/{enterpriseID}", log(logWriter, http.HandlerFunc(han.DeleteEnterpriseHandler))).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}/", http.HandlerFunc(han.DeleteEnterpriseHandler)).Methods("DELETE", "OPTIONS") + apiRouter.Handle("/enterprises/{enterpriseID}", http.HandlerFunc(han.DeleteEnterpriseHandler)).Methods("DELETE", "OPTIONS") // List orgs - apiRouter.Handle("/enterprises/", log(logWriter, http.HandlerFunc(han.ListEnterprisesHandler))).Methods("GET", "OPTIONS") - apiRouter.Handle("/enterprises", log(logWriter, http.HandlerFunc(han.ListEnterprisesHandler))).Methods("GET", "OPTIONS") + apiRouter.Handle("/enterprises/", http.HandlerFunc(han.ListEnterprisesHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/enterprises", http.HandlerFunc(han.ListEnterprisesHandler)).Methods("GET", "OPTIONS") // Create org - apiRouter.Handle("/enterprises/", log(logWriter, http.HandlerFunc(han.CreateEnterpriseHandler))).Methods("POST", "OPTIONS") - apiRouter.Handle("/enterprises", log(logWriter, http.HandlerFunc(han.CreateEnterpriseHandler))).Methods("POST", "OPTIONS") + apiRouter.Handle("/enterprises/", http.HandlerFunc(han.CreateEnterpriseHandler)).Methods("POST", "OPTIONS") + apiRouter.Handle("/enterprises", http.HandlerFunc(han.CreateEnterpriseHandler)).Methods("POST", "OPTIONS") // Credentials and providers - apiRouter.Handle("/credentials/", log(logWriter, http.HandlerFunc(han.ListCredentials))).Methods("GET", "OPTIONS") - apiRouter.Handle("/credentials", log(logWriter, http.HandlerFunc(han.ListCredentials))).Methods("GET", "OPTIONS") - apiRouter.Handle("/providers/", log(logWriter, http.HandlerFunc(han.ListProviders))).Methods("GET", "OPTIONS") - apiRouter.Handle("/providers", log(logWriter, http.HandlerFunc(han.ListProviders))).Methods("GET", "OPTIONS") + apiRouter.Handle("/credentials/", http.HandlerFunc(han.ListCredentials)).Methods("GET", "OPTIONS") + apiRouter.Handle("/credentials", http.HandlerFunc(han.ListCredentials)).Methods("GET", "OPTIONS") + apiRouter.Handle("/providers/", http.HandlerFunc(han.ListProviders)).Methods("GET", "OPTIONS") + apiRouter.Handle("/providers", http.HandlerFunc(han.ListProviders)).Methods("GET", "OPTIONS") // Websocket log writer - apiRouter.Handle("/{ws:ws\\/?}", log(logWriter, http.HandlerFunc(han.WSHandler))).Methods("GET") + apiRouter.Handle("/{ws:ws\\/?}", http.HandlerFunc(han.WSHandler)).Methods("GET") return router } diff --git a/database/common/common.go b/database/common/common.go index d5955f54..1b4153d4 100644 --- a/database/common/common.go +++ b/database/common/common.go @@ -106,7 +106,8 @@ type InstanceStore interface { ListAllInstances(ctx context.Context) ([]params.Instance, error) GetInstanceByName(ctx context.Context, instanceName string) (params.Instance, error) - AddInstanceStatusMessage(ctx context.Context, instanceID string, statusMessage string) error + AddInstanceEvent(ctx context.Context, instanceID string, event params.EventType, eventLevel params.EventLevel, eventMessage string) error + ListInstanceEvents(ctx context.Context, instanceID string, eventType params.EventType, eventLevel params.EventLevel) ([]params.StatusMessage, error) } //go:generate mockery --name=Store diff --git a/database/common/mocks/Store.go b/database/common/mocks/Store.go index d5bc37bf..31ed2617 100644 --- a/database/common/mocks/Store.go +++ b/database/common/mocks/Store.go @@ -14,13 +14,13 @@ type Store struct { mock.Mock } -// AddInstanceStatusMessage provides a mock function with given fields: ctx, instanceID, statusMessage -func (_m *Store) AddInstanceStatusMessage(ctx context.Context, instanceID string, statusMessage string) error { - ret := _m.Called(ctx, instanceID, statusMessage) +// AddInstanceEvent provides a mock function with given fields: ctx, instanceID, event, statusMessage +func (_m *Store) AddInstanceEvent(ctx context.Context, instanceID string, event params.EventType, statusMessage string) error { + ret := _m.Called(ctx, instanceID, event, statusMessage) var r0 error - if rf, ok := ret.Get(0).(func(context.Context, string, string) error); ok { - r0 = rf(ctx, instanceID, statusMessage) + if rf, ok := ret.Get(0).(func(context.Context, string, params.EventType, string) error); ok { + r0 = rf(ctx, instanceID, event, statusMessage) } else { r0 = ret.Error(0) } diff --git a/database/sql/instances.go b/database/sql/instances.go index 9696cba5..3e32e745 100644 --- a/database/sql/instances.go +++ b/database/sql/instances.go @@ -141,14 +141,42 @@ func (s *sqlDatabase) DeleteInstance(ctx context.Context, poolID string, instanc return nil } -func (s *sqlDatabase) AddInstanceStatusMessage(ctx context.Context, instanceID string, statusMessage string) error { +func (s *sqlDatabase) ListInstanceEvents(ctx context.Context, instanceID string, eventType params.EventType, eventLevel params.EventLevel) ([]params.StatusMessage, error) { + var events []InstanceStatusUpdate + query := s.conn.Model(&InstanceStatusUpdate{}).Where("instance_id = ?", instanceID) + if eventLevel != "" { + query = query.Where("event_level = ?", eventLevel) + } + + if eventType != "" { + query = query.Where("event_type = ?", eventType) + } + + if result := query.Find(&events); result.Error != nil { + return nil, errors.Wrap(result.Error, "fetching events") + } + + eventParams := make([]params.StatusMessage, len(events)) + for idx, val := range events { + eventParams[idx] = params.StatusMessage{ + Message: val.Message, + EventType: val.EventType, + EventLevel: val.EventLevel, + } + } + return eventParams, nil +} + +func (s *sqlDatabase) AddInstanceEvent(ctx context.Context, instanceID string, event params.EventType, eventLevel params.EventLevel, statusMessage string) error { instance, err := s.getInstanceByID(ctx, instanceID) if err != nil { return errors.Wrap(err, "updating instance") } msg := InstanceStatusUpdate{ - Message: statusMessage, + Message: statusMessage, + EventType: event, + EventLevel: eventLevel, } if err := s.conn.Model(&instance).Association("StatusMessages").Append(&msg); err != nil { @@ -214,13 +242,20 @@ func (s *sqlDatabase) UpdateInstance(ctx context.Context, instanceID string, par } func (s *sqlDatabase) ListPoolInstances(ctx context.Context, poolID string) ([]params.Instance, error) { - pool, err := s.getPoolByID(ctx, poolID, "Tags", "Instances") + u, err := uuid.FromString(poolID) if err != nil { - return nil, errors.Wrap(err, "fetching pool") + return nil, errors.Wrap(runnerErrors.ErrBadRequest, "parsing id") } - ret := make([]params.Instance, len(pool.Instances)) - for idx, inst := range pool.Instances { + var instances []Instance + query := s.conn.Model(&Instance{}).Where("pool_id = ?", u) + + if err := query.Find(&instances); err.Error != nil { + return nil, errors.Wrap(err.Error, "fetching instances") + } + + ret := make([]params.Instance, len(instances)) + for idx, inst := range instances { ret[idx] = s.sqlToParamsInstance(inst) } return ret, nil diff --git a/database/sql/instances_test.go b/database/sql/instances_test.go index 88278774..d1831b31 100644 --- a/database/sql/instances_test.go +++ b/database/sql/instances_test.go @@ -343,11 +343,11 @@ func (s *InstancesTestSuite) TestDeleteInstanceDBDeleteErr() { s.Require().Equal("deleting instance: mocked delete instance error", err.Error()) } -func (s *InstancesTestSuite) TestAddInstanceStatusMessage() { +func (s *InstancesTestSuite) TestAddInstanceEvent() { storeInstance := s.Fixtures.Instances[0] statusMsg := "test-status-message" - err := s.Store.AddInstanceStatusMessage(context.Background(), storeInstance.ID, statusMsg) + err := s.Store.AddInstanceEvent(context.Background(), storeInstance.ID, params.StatusEvent, params.EventInfo, statusMsg) s.Require().Nil(err) instance, err := s.Store.GetInstanceByName(context.Background(), storeInstance.Name) @@ -358,13 +358,13 @@ func (s *InstancesTestSuite) TestAddInstanceStatusMessage() { s.Require().Equal(statusMsg, instance.StatusMessages[0].Message) } -func (s *InstancesTestSuite) TestAddInstanceStatusMessageInvalidPoolID() { - err := s.Store.AddInstanceStatusMessage(context.Background(), "dummy-id", "dummy-message") +func (s *InstancesTestSuite) TestAddInstanceEventInvalidPoolID() { + err := s.Store.AddInstanceEvent(context.Background(), "dummy-id", params.StatusEvent, params.EventInfo, "dummy-message") s.Require().Equal("updating instance: parsing id: invalid request", err.Error()) } -func (s *InstancesTestSuite) TestAddInstanceStatusMessageDBUpdateErr() { +func (s *InstancesTestSuite) TestAddInstanceEventDBUpdateErr() { instance := s.Fixtures.Instances[0] statusMsg := "test-status-message" @@ -390,7 +390,7 @@ func (s *InstancesTestSuite) TestAddInstanceStatusMessageDBUpdateErr() { WillReturnError(fmt.Errorf("mocked add status message error")) s.Fixtures.SQLMock.ExpectRollback() - err := s.StoreSQLMocked.AddInstanceStatusMessage(context.Background(), instance.ID, statusMsg) + err := s.StoreSQLMocked.AddInstanceEvent(context.Background(), instance.ID, params.StatusEvent, params.EventInfo, statusMsg) s.assertSQLMockExpectations() s.Require().NotNil(err) @@ -496,7 +496,7 @@ func (s *InstancesTestSuite) TestListPoolInstances() { func (s *InstancesTestSuite) TestListPoolInstancesInvalidPoolID() { _, err := s.Store.ListPoolInstances(context.Background(), "dummy-pool-id") - s.Require().Equal("fetching pool: parsing id: invalid request", err.Error()) + s.Require().Equal("parsing id: invalid request", err.Error()) } func (s *InstancesTestSuite) TestListAllInstances() { diff --git a/database/sql/models.go b/database/sql/models.go index 64eaf4d7..66a6dc3d 100644 --- a/database/sql/models.go +++ b/database/sql/models.go @@ -16,6 +16,7 @@ package sql import ( "garm/config" + "garm/params" "garm/runner/providers/common" "time" @@ -118,7 +119,9 @@ type Address struct { type InstanceStatusUpdate struct { Base - Message string `gorm:"type:text"` + EventType params.EventType `gorm:"index:eventType"` + EventLevel params.EventLevel + Message string `gorm:"type:text"` InstanceID uuid.UUID Instance Instance `gorm:"foreignKey:InstanceID"` diff --git a/database/sql/util.go b/database/sql/util.go index dfcb74fd..a5aa15c2 100644 --- a/database/sql/util.go +++ b/database/sql/util.go @@ -57,8 +57,10 @@ func (s *sqlDatabase) sqlToParamsInstance(instance Instance) params.Instance { for _, msg := range instance.StatusMessages { ret.StatusMessages = append(ret.StatusMessages, params.StatusMessage{ - CreatedAt: msg.CreatedAt, - Message: msg.Message, + CreatedAt: msg.CreatedAt, + Message: msg.Message, + EventType: msg.EventType, + EventLevel: msg.EventLevel, }) } return ret diff --git a/params/params.go b/params/params.go index 9eb09878..0e4f067e 100644 --- a/params/params.go +++ b/params/params.go @@ -24,20 +24,35 @@ import ( ) type AddressType string +type EventType string +type EventLevel string const ( PublicAddress AddressType = "public" PrivateAddress AddressType = "private" ) +const ( + StatusEvent EventType = "status" + FetchTokenEvent EventType = "fetchToken" +) + +const ( + EventInfo EventLevel = "info" + EventWarning EventLevel = "warning" + EventError EventLevel = "error" +) + type Address struct { Address string `json:"address"` Type AddressType `json:"type"` } type StatusMessage struct { - CreatedAt time.Time `json:"created_at"` - Message string `json:"message"` + CreatedAt time.Time `json:"created_at"` + Message string `json:"message"` + EventType EventType `json:"event_type"` + EventLevel EventLevel `json:"event_level"` } type Instance struct { @@ -264,3 +279,8 @@ type PoolManagerStatus struct { IsRunning bool `json:"running"` FailureReason string `json:"failure_reason,omitempty"` } + +type RunnerInfo struct { + Name string + Labels []string +} diff --git a/runner/pool/enterprise.go b/runner/pool/enterprise.go index fa0b25b4..45931842 100644 --- a/runner/pool/enterprise.go +++ b/runner/pool/enterprise.go @@ -61,18 +61,25 @@ type enterprise struct { mux sync.Mutex } -func (r *enterprise) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) { +func (r *enterprise) GetRunnerInfoFromWorkflow(job params.WorkflowJob) (params.RunnerInfo, error) { + if err := r.ValidateOwner(job); err != nil { + return params.RunnerInfo{}, errors.Wrap(err, "validating owner") + } workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID) if err != nil { if ghResp.StatusCode == http.StatusUnauthorized { - return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runners") + return params.RunnerInfo{}, errors.Wrap(runnerErrors.ErrUnauthorized, "fetching workflow info") } - return "", errors.Wrap(err, "fetching workflow info") + return params.RunnerInfo{}, errors.Wrap(err, "fetching workflow info") } + if workflow.RunnerName != nil { - return *workflow.RunnerName, nil + return params.RunnerInfo{ + Name: *workflow.RunnerName, + Labels: workflow.Labels, + }, nil } - return "", fmt.Errorf("failed to find runner name from workflow") + return params.RunnerInfo{}, fmt.Errorf("failed to find runner name from workflow") } func (r *enterprise) UpdateState(param params.UpdatePoolStateParams) error { diff --git a/runner/pool/interfaces.go b/runner/pool/interfaces.go index 1e07ab52..9e69dec3 100644 --- a/runner/pool/interfaces.go +++ b/runner/pool/interfaces.go @@ -24,7 +24,7 @@ type poolHelper interface { GetGithubToken() string GetGithubRunners() ([]*github.Runner, error) GetGithubRegistrationToken() (string, error) - GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) + GetRunnerInfoFromWorkflow(job params.WorkflowJob) (params.RunnerInfo, error) RemoveGithubRunner(runnerID int64) (*github.Response, error) FetchTools() ([]*github.RunnerApplicationDownload, error) diff --git a/runner/pool/organization.go b/runner/pool/organization.go index a5985f8d..9371b4d5 100644 --- a/runner/pool/organization.go +++ b/runner/pool/organization.go @@ -73,18 +73,25 @@ type organization struct { mux sync.Mutex } -func (r *organization) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) { +func (r *organization) GetRunnerInfoFromWorkflow(job params.WorkflowJob) (params.RunnerInfo, error) { + if err := r.ValidateOwner(job); err != nil { + return params.RunnerInfo{}, errors.Wrap(err, "validating owner") + } workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Organization.Login, job.Repository.Name, job.WorkflowJob.ID) if err != nil { if ghResp.StatusCode == http.StatusUnauthorized { - return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runner name") + return params.RunnerInfo{}, errors.Wrap(runnerErrors.ErrUnauthorized, "fetching workflow info") } - return "", errors.Wrap(err, "fetching workflow info") + return params.RunnerInfo{}, errors.Wrap(err, "fetching workflow info") } + if workflow.RunnerName != nil { - return *workflow.RunnerName, nil + return params.RunnerInfo{ + Name: *workflow.RunnerName, + Labels: workflow.Labels, + }, nil } - return "", fmt.Errorf("failed to find runner name from workflow") + return params.RunnerInfo{}, fmt.Errorf("failed to find runner name from workflow") } func (r *organization) UpdateState(param params.UpdatePoolStateParams) error { diff --git a/runner/pool/pool.go b/runner/pool/pool.go index 97755024..e5fecf67 100644 --- a/runner/pool/pool.go +++ b/runner/pool/pool.go @@ -67,16 +67,57 @@ type basePoolManager struct { mux sync.Mutex } -func controllerIDFromLabels(labels []*github.RunnerLabels) string { +func controllerIDFromLabels(labels []string) string { for _, lbl := range labels { - if lbl.Name != nil && strings.HasPrefix(*lbl.Name, controllerLabelPrefix) { - labelName := *lbl.Name - return labelName[len(controllerLabelPrefix):] + if strings.HasPrefix(lbl, controllerLabelPrefix) { + return lbl[len(controllerLabelPrefix):] } } return "" } +func poolIDFromLabels(labels []string) string { + for _, lbl := range labels { + if strings.HasPrefix(lbl, poolIDLabelprefix) { + return lbl[len(poolIDLabelprefix):] + } + } + return "" +} + +func poolIsOwnedBy(pool params.Pool, ownerID string) bool { + return pool.RepoID == ownerID || pool.OrgID == ownerID || pool.EnterpriseID == ownerID +} + +func labelsFromRunner(runner *github.Runner) []string { + if runner == nil || runner.Labels == nil { + return []string{} + } + + var labels []string + for _, val := range runner.Labels { + if val == nil { + continue + } + labels = append(labels, val.GetName()) + } + return labels +} + +func (r *basePoolManager) isControllerRunner(labels []string) bool { + runnerControllerID := controllerIDFromLabels(labels) + if runnerControllerID != r.controllerID { + return false + } + + poolID := poolIDFromLabels(labels) + if poolID == "" { + return false + } + _, err := r.helper.GetPoolByID(poolID) + return err == nil +} + // cleanupOrphanedProviderRunners compares runners in github with local runners and removes // any local runners that are not present in Github. Runners that are "idle" in our // provider, but do not exist in github, will be removed. This can happen if the @@ -92,6 +133,9 @@ func (r *basePoolManager) cleanupOrphanedProviderRunners(runners []*github.Runne runnerNames := map[string]bool{} for _, run := range runners { + if !r.isControllerRunner(labelsFromRunner(run)) { + continue + } runnerNames[*run.Name] = true } @@ -127,24 +171,25 @@ func (r *basePoolManager) reapTimedOutRunners(runners []*github.Runner) error { runnerNames := map[string]bool{} for _, run := range runners { + if !r.isControllerRunner(labelsFromRunner(run)) { + continue + } runnerNames[*run.Name] = true } for _, instance := range dbInstances { if ok := runnerNames[instance.Name]; !ok { - if instance.Status == providerCommon.InstanceRunning { - pool, err := r.store.GetPoolByID(r.ctx, instance.PoolID) - if err != nil { - return errors.Wrap(err, "fetching instance pool info") - } - if time.Since(instance.UpdatedAt).Minutes() < float64(pool.RunnerTimeout()) { - continue - } - log.Printf("reaping instance %s due to timeout", instance.Name) - if err := r.setInstanceStatus(instance.Name, providerCommon.InstancePendingDelete, nil); err != nil { - log.Printf("failed to update runner %s status", instance.Name) - return errors.Wrap(err, "updating runner") - } + pool, err := r.store.GetPoolByID(r.ctx, instance.PoolID) + if err != nil { + return errors.Wrap(err, "fetching instance pool info") + } + if time.Since(instance.UpdatedAt).Minutes() < float64(pool.RunnerTimeout()) { + continue + } + log.Printf("reaping instance %s due to timeout", instance.Name) + if err := r.setInstanceStatus(instance.Name, providerCommon.InstancePendingDelete, nil); err != nil { + log.Printf("failed to update runner %s status", instance.Name) + return errors.Wrap(err, "updating runner") } } } @@ -157,9 +202,7 @@ func (r *basePoolManager) reapTimedOutRunners(runners []*github.Runner) error { // first remove the instance from github, and then from our database. func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner) error { for _, runner := range runners { - runnerControllerID := controllerIDFromLabels(runner.Labels) - if runnerControllerID != r.controllerID { - // Not a runner we manage. Do not remove foreign runner. + if !r.isControllerRunner(labelsFromRunner(runner)) { continue } @@ -205,6 +248,9 @@ func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner) if err != nil { return errors.Wrap(err, "fetching pool") } + if !poolIsOwnedBy(pool, r.ID()) { + continue + } // check if the provider still has the instance. provider, ok := r.providers[pool.ProviderName] @@ -265,6 +311,11 @@ func (r *basePoolManager) fetchInstance(runnerName string) (params.Instance, err return params.Instance{}, errors.Wrap(err, "fetching instance") } + _, err = r.helper.GetPoolByID(runner.PoolID) + if err != nil { + return params.Instance{}, errors.Wrap(err, "fetching pool") + } + return runner, nil } @@ -312,6 +363,9 @@ func (r *basePoolManager) acquireNewInstance(job params.WorkflowJob) error { pool, err := r.helper.FindPoolByTags(requestedLabels) if err != nil { + if errors.Is(err, runnerErrors.ErrNotFound) { + return nil + } return errors.Wrap(err, "fetching suitable pool") } log.Printf("adding new runner with requested tags %s in pool %s", strings.Join(job.WorkflowJob.Labels, ", "), pool.ID) @@ -572,25 +626,28 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error return nil } -func (r *basePoolManager) getRunnerNameFromJob(job params.WorkflowJob) (string, error) { +func (r *basePoolManager) getRunnerDetailsFromJob(job params.WorkflowJob) (params.RunnerInfo, error) { if job.WorkflowJob.RunnerName != "" { - return job.WorkflowJob.RunnerName, nil + return params.RunnerInfo{ + Name: job.WorkflowJob.RunnerName, + Labels: job.WorkflowJob.Labels, + }, nil } // Runner name was not set in WorkflowJob by github. We can still attempt to // fetch the info we need, using the workflow run ID, from the API. log.Printf("runner name not found in workflow job, attempting to fetch from API") - runnerName, err := r.helper.GetRunnerNameFromWorkflow(job) + runnerInfo, err := r.helper.GetRunnerInfoFromWorkflow(job) if err != nil { if errors.Is(err, runnerErrors.ErrUnauthorized) { failureReason := fmt.Sprintf("failed to fetch runner name from API: %q", err) r.setPoolRunningState(false, failureReason) log.Print(failureReason) } - return "", errors.Wrap(err, "fetching runner name from API") + return params.RunnerInfo{}, errors.Wrap(err, "fetching runner name from API") } - return runnerName, nil + return runnerInfo, nil } func (r *basePoolManager) HandleWorkflowJob(job params.WorkflowJob) error { @@ -612,34 +669,52 @@ func (r *basePoolManager) HandleWorkflowJob(job params.WorkflowJob) error { case "completed": // ignore the error here. A completed job may not have a runner name set // if it was never assigned to a runner, and was canceled. - runnerName, _ := r.getRunnerNameFromJob(job) - // Set instance in database to pending delete. - if runnerName == "" { + runnerInfo, err := r.getRunnerDetailsFromJob(job) + if err != nil { // Unassigned jobs will have an empty runner_name. // There is nothing to to in this case. log.Printf("no runner was assigned. Skipping.") return nil } + + if !r.isControllerRunner(runnerInfo.Labels) { + return nil + } + // update instance workload state. - if err := r.setInstanceRunnerStatus(runnerName, providerCommon.RunnerTerminated); err != nil { - log.Printf("failed to update runner %s status", runnerName) + if err := r.setInstanceRunnerStatus(runnerInfo.Name, providerCommon.RunnerTerminated); err != nil { + if errors.Is(err, runnerErrors.ErrNotFound) { + return nil + } + log.Printf("failed to update runner %s status", runnerInfo.Name) return errors.Wrap(err, "updating runner") } - log.Printf("marking instance %s as pending_delete", runnerName) - if err := r.setInstanceStatus(runnerName, providerCommon.InstancePendingDelete, nil); err != nil { - log.Printf("failed to update runner %s status", runnerName) + log.Printf("marking instance %s as pending_delete", runnerInfo.Name) + if err := r.setInstanceStatus(runnerInfo.Name, providerCommon.InstancePendingDelete, nil); err != nil { + if errors.Is(err, runnerErrors.ErrNotFound) { + return nil + } + log.Printf("failed to update runner %s status", runnerInfo.Name) return errors.Wrap(err, "updating runner") } case "in_progress": // in_progress jobs must have a runner name/ID assigned. Sometimes github will send a hook without // a runner set. In such cases, we attemt to fetch it from the API. - runnerName, err := r.getRunnerNameFromJob(job) + runnerInfo, err := r.getRunnerDetailsFromJob(job) if err != nil { return errors.Wrap(err, "determining runner name") } + + if !r.isControllerRunner(runnerInfo.Labels) { + return nil + } + // update instance workload state. - if err := r.setInstanceRunnerStatus(runnerName, providerCommon.RunnerActive); err != nil { - log.Printf("failed to update runner %s status", job.WorkflowJob.RunnerName) + if err := r.setInstanceRunnerStatus(runnerInfo.Name, providerCommon.RunnerActive); err != nil { + if errors.Is(err, runnerErrors.ErrNotFound) { + return nil + } + log.Printf("failed to update runner %s status", runnerInfo.Name) return errors.Wrap(err, "updating runner") } } @@ -716,10 +791,11 @@ func (r *basePoolManager) retryFailedInstancesForOnePool(pool params.Pool) { existingInstances, err := r.store.ListPoolInstances(r.ctx, pool.ID) if err != nil { - log.Printf("failed to ensure minimum idle workers for pool %s: %s", pool.ID, err) + log.Printf("retrying failed instances: failed to list instances for pool %s: %s", pool.ID, err) return } + wg := sync.WaitGroup{} for _, instance := range existingInstances { if instance.Status != providerCommon.InstanceError { continue @@ -727,28 +803,31 @@ func (r *basePoolManager) retryFailedInstancesForOnePool(pool params.Pool) { if instance.CreateAttempt >= maxCreateAttempts { continue } + wg.Add(1) + go func(inst params.Instance) { + defer wg.Done() + // NOTE(gabriel-samfira): this is done in parallel. If there are many failed instances + // this has the potential to create many API requests to the target provider. + // TODO(gabriel-samfira): implement request throttling. + if err := r.deleteInstanceFromProvider(inst); err != nil { + log.Printf("failed to delete instance %s from provider: %s", inst.Name, err) + } - // NOTE(gabriel-samfira): this is done in parallel. If there are many failed instances - // this has the potential to create many API requests to the target provider. - // TODO(gabriel-samfira): implement request throttling. - if instance.ProviderID == "" && instance.Name == "" { - // This really should not happen, but no harm in being extra paranoid. The name is set - // when creating a db entity for the runner, so we should at least have a name. - return - } - // TODO(gabriel-samfira): Incrementing CreateAttempt should be done within a transaction. - // It's fairly safe to do here (for now), as there should be no other code path that updates - // an instance in this state. - updateParams := params.UpdateInstanceParams{ - CreateAttempt: instance.CreateAttempt + 1, - Status: providerCommon.InstancePendingCreate, - } - log.Printf("queueing previously failed instance %s for retry", instance.Name) - // Set instance to pending create and wait for retry. - if err := r.updateInstance(instance.Name, updateParams); err != nil { - log.Printf("failed to update runner %s status", instance.Name) - } + // TODO(gabriel-samfira): Incrementing CreateAttempt should be done within a transaction. + // It's fairly safe to do here (for now), as there should be no other code path that updates + // an instance in this state. + updateParams := params.UpdateInstanceParams{ + CreateAttempt: inst.CreateAttempt + 1, + Status: providerCommon.InstancePendingCreate, + } + log.Printf("queueing previously failed instance %s for retry", inst.Name) + // Set instance to pending create and wait for retry. + if err := r.updateInstance(inst.Name, updateParams); err != nil { + log.Printf("failed to update runner %s status", inst.Name) + } + }(instance) } + wg.Wait() } func (r *basePoolManager) retryFailedInstances() { @@ -807,9 +886,6 @@ func (r *basePoolManager) deleteInstanceFromProvider(instance params.Instance) e return errors.Wrap(err, "removing instance") } - if err := r.store.DeleteInstance(r.ctx, pool.ID, instance.Name); err != nil { - return errors.Wrap(err, "deleting instance from database") - } return nil } @@ -846,6 +922,10 @@ func (r *basePoolManager) deletePendingInstances() { if err != nil { log.Printf("failed to delete instance from provider: %+v", err) } + + if err := r.store.DeleteInstance(r.ctx, instance.PoolID, instance.Name); err != nil { + return errors.Wrap(err, "deleting instance from database") + } return }(instance) } diff --git a/runner/pool/repository.go b/runner/pool/repository.go index f5415d7a..9c69915a 100644 --- a/runner/pool/repository.go +++ b/runner/pool/repository.go @@ -75,18 +75,25 @@ type repository struct { mux sync.Mutex } -func (r *repository) GetRunnerNameFromWorkflow(job params.WorkflowJob) (string, error) { +func (r *repository) GetRunnerInfoFromWorkflow(job params.WorkflowJob) (params.RunnerInfo, error) { + if err := r.ValidateOwner(job); err != nil { + return params.RunnerInfo{}, errors.Wrap(err, "validating owner") + } workflow, ghResp, err := r.ghcli.GetWorkflowJobByID(r.ctx, job.Repository.Owner.Login, job.Repository.Name, job.WorkflowJob.ID) if err != nil { if ghResp.StatusCode == http.StatusUnauthorized { - return "", errors.Wrap(runnerErrors.ErrUnauthorized, "fetching runner name") + return params.RunnerInfo{}, errors.Wrap(runnerErrors.ErrUnauthorized, "fetching workflow info") } - return "", errors.Wrap(err, "fetching workflow info") + return params.RunnerInfo{}, errors.Wrap(err, "fetching workflow info") } + if workflow.RunnerName != nil { - return *workflow.RunnerName, nil + return params.RunnerInfo{ + Name: *workflow.RunnerName, + Labels: workflow.Labels, + }, nil } - return "", fmt.Errorf("failed to find runner name from workflow") + return params.RunnerInfo{}, fmt.Errorf("failed to find runner name from workflow") } func (r *repository) UpdateState(param params.UpdatePoolStateParams) error { diff --git a/runner/runner.go b/runner/runner.go index c363daec..3c5a7b70 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -584,8 +584,10 @@ func (r *Runner) DispatchWorkflowJob(hookTargetType, signature string, jobData [ switch HookTargetType(hookTargetType) { case RepoHook: + log.Printf("got hook for repo %s/%s", job.Repository.Owner.Login, job.Repository.Name) poolManager, err = r.findRepoPoolManager(job.Repository.Owner.Login, job.Repository.Name) case OrganizationHook: + log.Printf("got hook for org %s", job.Organization.Login) poolManager, err = r.findOrgPoolManager(job.Organization.Login) case EnterpriseHook: poolManager, err = r.findEnterprisePoolManager(job.Enterprise.Slug) @@ -704,7 +706,7 @@ func (r *Runner) AddInstanceStatusMessage(ctx context.Context, param params.Inst return runnerErrors.ErrUnauthorized } - if err := r.store.AddInstanceStatusMessage(ctx, instanceID, param.Message); err != nil { + if err := r.store.AddInstanceEvent(ctx, instanceID, params.StatusEvent, params.EventInfo, param.Message); err != nil { return errors.Wrap(err, "adding status update") } @@ -744,11 +746,25 @@ func (r *Runner) GetInstanceGithubRegistrationToken(ctx context.Context) (string return "", errors.Wrap(err, "fetching pool manager for instance") } + tokenEvents, err := r.store.ListInstanceEvents(ctx, instance.ID, params.FetchTokenEvent, "") + if err != nil { + return "", errors.Wrap(err, "fetching instance events") + } + + if len(tokenEvents) > 0 { + // Token already retrieved + return "", runnerErrors.ErrUnauthorized + } + token, err := poolMgr.GithubRunnerRegistrationToken() if err != nil { return "", errors.Wrap(err, "fetching runner token") } + if err := r.store.AddInstanceEvent(ctx, instance.ID, params.FetchTokenEvent, params.EventInfo, "runner registration token was retrieved"); err != nil { + return "", errors.Wrap(err, "recording event") + } + return token, nil } diff --git a/util/util.go b/util/util.go index eba4f188..4834be32 100644 --- a/util/util.go +++ b/util/util.go @@ -38,6 +38,7 @@ import ( "garm/runner/common" "github.com/google/go-github/v48/github" + gorillaHandlers "github.com/gorilla/handlers" "github.com/pkg/errors" "golang.org/x/crypto/bcrypt" "golang.org/x/oauth2" @@ -320,3 +321,9 @@ func PaswsordToBcrypt(password string) (string, error) { } return string(hashedPassword), nil } + +func NewLoggingMiddleware(writer io.Writer) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return gorillaHandlers.CombinedLoggingHandler(writer, next) + } +}