From 80e042ee88072e71066c3fc44f40e84e642bd801 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Sun, 8 Feb 2026 23:48:57 +0200 Subject: [PATCH] Add runner rotate ability to CLI This change adds a new "generation" field to pools, scalesets and runners. The generation field is inherited by runners from scale sets or pools at the time of creation. The generation field on scalesets and pools is incremented when the pool or scale set is updated in a way that might influence how runners are created (flavor, image, specs, runner groups, etc). Using this new field, we can determine if existing runners have diverged from the settings of the pool/scale set that spawned them. In the CLI we now have a new set of commands available for both pools and scalesets that lists runners, with an optional --outdated flag and a new "rotate" flag that removes all idle runners. Optionally the --outdated flag can be passed to the rotate command to only remove outdated runners. Signed-off-by: Gabriel Adrian Samfira --- apiserver/controllers/instances.go | 18 ++- apiserver/swagger.yaml | 8 + .../list_pool_instances_parameters.go | 35 +++++ .../list_scale_set_instances_parameters.go | 35 +++++ cmd/garm-cli/cmd/pool.go | 138 ++++++++++++++++++ cmd/garm-cli/cmd/runner.go | 42 ++++-- cmd/garm-cli/cmd/scalesets.go | 112 ++++++++++++++ database/common/mocks/Store.go | 58 ++++---- database/common/store.go | 4 +- database/sql/instances.go | 11 +- database/sql/instances_test.go | 4 +- database/sql/models.go | 21 +++ database/sql/pools_test.go | 2 +- database/sql/scaleset_instances.go | 8 +- database/sql/scalesets.go | 12 +- database/sql/scalesets_test.go | 2 +- database/sql/util.go | 15 ++ params/params.go | 23 +++ params/requests.go | 1 + runner/garm_tools_test.go | 1 - runner/pool/pool.go | 7 +- runner/repositories.go | 4 +- runner/repositories_test.go | 4 +- runner/scalesets.go | 4 +- webapp/src/lib/api/generated/api.ts | 106 ++++++++++---- webapp/swagger.yaml | 64 +++++++- workers/scaleset/scaleset.go | 2 +- 27 files changed, 648 insertions(+), 93 deletions(-) diff --git a/apiserver/controllers/instances.go b/apiserver/controllers/instances.go index 3209a5c2..e41be49d 100644 --- a/apiserver/controllers/instances.go +++ b/apiserver/controllers/instances.go @@ -38,6 +38,12 @@ import ( // in: path // required: true // +// + name: outdatedOnly +// description: List only instances that were created prior to a pool update that changed a setting which influences how instances are created (image, flavor, runner group, etc). +// type: boolean +// in: query +// required: false +// // Responses: // 200: Instances // default: APIErrorResponse @@ -56,7 +62,8 @@ func (a *APIController) ListPoolInstancesHandler(w http.ResponseWriter, r *http. return } - instances, err := a.r.ListPoolInstances(ctx, poolID) + filterByOutdated, _ := strconv.ParseBool(r.URL.Query().Get("outdatedOnly")) + instances, err := a.r.ListPoolInstances(ctx, poolID, filterByOutdated) if err != nil { slog.With(slog.Any("error", err)).ErrorContext(ctx, "listing pool instances") handleError(ctx, w, err) @@ -80,6 +87,12 @@ func (a *APIController) ListPoolInstancesHandler(w http.ResponseWriter, r *http. // in: path // required: true // +// + name: outdatedOnly +// description: List only instances that were created prior to a scaleset update that changed a setting which influences how instances are created (image, flavor, runner group, etc). +// type: boolean +// in: query +// required: false +// // Responses: // 200: Instances // default: APIErrorResponse @@ -104,7 +117,8 @@ func (a *APIController) ListScaleSetInstancesHandler(w http.ResponseWriter, r *h return } - instances, err := a.r.ListScaleSetInstances(ctx, uint(id)) + filterByOutdated, _ := strconv.ParseBool(r.URL.Query().Get("outdatedOnly")) + instances, err := a.r.ListScaleSetInstances(ctx, uint(id), filterByOutdated) if err != nil { slog.With(slog.Any("error", err)).ErrorContext(ctx, "listing pool instances") handleError(ctx, w, err) diff --git a/apiserver/swagger.yaml b/apiserver/swagger.yaml index e7575b4b..ba7ca1c9 100644 --- a/apiserver/swagger.yaml +++ b/apiserver/swagger.yaml @@ -1940,6 +1940,10 @@ paths: name: poolID required: true type: string + - description: List only instances that were created prior to a pool update that changed a setting which influences how instances are created (image, flavor, runner group, etc). + in: query + name: outdatedOnly + type: boolean responses: "200": description: Instances @@ -2453,6 +2457,10 @@ paths: name: scalesetID required: true type: string + - description: List only instances that were created prior to a scaleset update that changed a setting which influences how instances are created (image, flavor, runner group, etc). + in: query + name: outdatedOnly + type: boolean responses: "200": description: Instances diff --git a/client/instances/list_pool_instances_parameters.go b/client/instances/list_pool_instances_parameters.go index 622010de..38ca3d7a 100644 --- a/client/instances/list_pool_instances_parameters.go +++ b/client/instances/list_pool_instances_parameters.go @@ -14,6 +14,7 @@ import ( "github.com/go-openapi/runtime" cr "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" ) // NewListPoolInstancesParams creates a new ListPoolInstancesParams object, @@ -61,6 +62,12 @@ ListPoolInstancesParams contains all the parameters to send to the API endpoint */ type ListPoolInstancesParams struct { + /* OutdatedOnly. + + List only instances that were created prior to a pool update that changed a setting which influences how instances are created (image, flavor, runner group, etc). + */ + OutdatedOnly *bool + /* PoolID. Runner pool ID. @@ -120,6 +127,17 @@ func (o *ListPoolInstancesParams) SetHTTPClient(client *http.Client) { o.HTTPClient = client } +// WithOutdatedOnly adds the outdatedOnly to the list pool instances params +func (o *ListPoolInstancesParams) WithOutdatedOnly(outdatedOnly *bool) *ListPoolInstancesParams { + o.SetOutdatedOnly(outdatedOnly) + return o +} + +// SetOutdatedOnly adds the outdatedOnly to the list pool instances params +func (o *ListPoolInstancesParams) SetOutdatedOnly(outdatedOnly *bool) { + o.OutdatedOnly = outdatedOnly +} + // WithPoolID adds the poolID to the list pool instances params func (o *ListPoolInstancesParams) WithPoolID(poolID string) *ListPoolInstancesParams { o.SetPoolID(poolID) @@ -139,6 +157,23 @@ func (o *ListPoolInstancesParams) WriteToRequest(r runtime.ClientRequest, reg st } var res []error + if o.OutdatedOnly != nil { + + // query param outdatedOnly + var qrOutdatedOnly bool + + if o.OutdatedOnly != nil { + qrOutdatedOnly = *o.OutdatedOnly + } + qOutdatedOnly := swag.FormatBool(qrOutdatedOnly) + if qOutdatedOnly != "" { + + if err := r.SetQueryParam("outdatedOnly", qOutdatedOnly); err != nil { + return err + } + } + } + // path param poolID if err := r.SetPathParam("poolID", o.PoolID); err != nil { return err diff --git a/client/instances/list_scale_set_instances_parameters.go b/client/instances/list_scale_set_instances_parameters.go index 7b38ef82..4db41224 100644 --- a/client/instances/list_scale_set_instances_parameters.go +++ b/client/instances/list_scale_set_instances_parameters.go @@ -14,6 +14,7 @@ import ( "github.com/go-openapi/runtime" cr "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" + "github.com/go-openapi/swag" ) // NewListScaleSetInstancesParams creates a new ListScaleSetInstancesParams object, @@ -61,6 +62,12 @@ ListScaleSetInstancesParams contains all the parameters to send to the API endpo */ type ListScaleSetInstancesParams struct { + /* OutdatedOnly. + + List only instances that were created prior to a scaleset update that changed a setting which influences how instances are created (image, flavor, runner group, etc). + */ + OutdatedOnly *bool + /* ScalesetID. Runner scale set ID. @@ -120,6 +127,17 @@ func (o *ListScaleSetInstancesParams) SetHTTPClient(client *http.Client) { o.HTTPClient = client } +// WithOutdatedOnly adds the outdatedOnly to the list scale set instances params +func (o *ListScaleSetInstancesParams) WithOutdatedOnly(outdatedOnly *bool) *ListScaleSetInstancesParams { + o.SetOutdatedOnly(outdatedOnly) + return o +} + +// SetOutdatedOnly adds the outdatedOnly to the list scale set instances params +func (o *ListScaleSetInstancesParams) SetOutdatedOnly(outdatedOnly *bool) { + o.OutdatedOnly = outdatedOnly +} + // WithScalesetID adds the scalesetID to the list scale set instances params func (o *ListScaleSetInstancesParams) WithScalesetID(scalesetID string) *ListScaleSetInstancesParams { o.SetScalesetID(scalesetID) @@ -139,6 +157,23 @@ func (o *ListScaleSetInstancesParams) WriteToRequest(r runtime.ClientRequest, re } var res []error + if o.OutdatedOnly != nil { + + // query param outdatedOnly + var qrOutdatedOnly bool + + if o.OutdatedOnly != nil { + qrOutdatedOnly = *o.OutdatedOnly + } + qOutdatedOnly := swag.FormatBool(qrOutdatedOnly) + if qOutdatedOnly != "" { + + if err := r.SetQueryParam("outdatedOnly", qOutdatedOnly); err != nil { + return err + } + } + } + // path param scalesetID if err := r.SetPathParam("scalesetID", o.ScalesetID); err != nil { return err diff --git a/cmd/garm-cli/cmd/pool.go b/cmd/garm-cli/cmd/pool.go index 2dc73d17..abcf3e55 100644 --- a/cmd/garm-cli/cmd/pool.go +++ b/cmd/garm-cli/cmd/pool.go @@ -15,6 +15,7 @@ package cmd import ( + "bufio" "encoding/json" "fmt" "os" @@ -25,6 +26,7 @@ import ( commonParams "github.com/cloudbase/garm-provider-common/params" apiClientEnterprises "github.com/cloudbase/garm/client/enterprises" + apiClientInstances "github.com/cloudbase/garm/client/instances" apiClientOrgs "github.com/cloudbase/garm/client/organizations" apiClientPools "github.com/cloudbase/garm/client/pools" apiClientRepos "github.com/cloudbase/garm/client/repositories" @@ -424,6 +426,128 @@ explicitly remove them using the runner delete command. }, } +var ( + poolRunnerOutdated bool + poolRunnerDryRun bool + poolRunnerForce bool + poolRunnerBypass bool + poolRunnerConfirmSkip bool +) + +var poolRunnerCmd = &cobra.Command{ + Use: "runner", + Short: "Manage runners in a pool", + Long: `List or rotate runners in a pool.`, + SilenceUsage: true, + Run: nil, +} + +var poolRunnerListCmd = &cobra.Command{ + Use: "list ", + Aliases: []string{"ls"}, + Short: "List runners in a pool", + Long: `List all runners belonging to a pool. Use --outdated to show only runners with a generation older than the pool.`, + Args: cobra.ExactArgs(1), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if needsInit { + return errNeedsInitError + } + + listReq := apiClientInstances.NewListPoolInstancesParams() + listReq.PoolID = args[0] + if cmd.Flags().Changed("outdated") { + listReq.OutdatedOnly = &poolRunnerOutdated + } + + response, err := apiCli.Instances.ListPoolInstances(listReq, authToken) + if err != nil { + return err + } + + formatInstances(response.Payload, false, false) + return nil + }, +} + +var poolRunnerRotateCmd = &cobra.Command{ + Use: "rotate ", + Short: "Rotate idle runners in a pool", + Long: `Remove idle runners in a pool so they get replaced with fresh ones. Use --outdated to only rotate runners with a generation older than the pool.`, + Args: cobra.ExactArgs(1), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if needsInit { + return errNeedsInitError + } + + listReq := apiClientInstances.NewListPoolInstancesParams() + listReq.PoolID = args[0] + if cmd.Flags().Changed("outdated") { + listReq.OutdatedOnly = &poolRunnerOutdated + } + + response, err := apiCli.Instances.ListPoolInstances(listReq, authToken) + if err != nil { + return err + } + + idle := filterIdleRunners(response.Payload) + if len(idle) == 0 { + fmt.Println("No idle runners to rotate.") + return nil + } + + if poolRunnerDryRun { + fmt.Printf("Would remove %d idle runner(s):\n", len(idle)) + formatInstances(idle, false, false) + return nil + } + + if !poolRunnerConfirmSkip { + fmt.Printf("About to remove %d idle runner(s). Continue? [y/N] ", len(idle)) + reader := bufio.NewReader(os.Stdin) + answer, _ := reader.ReadString('\n') + answer = strings.TrimSpace(strings.ToLower(answer)) + if answer != "y" && answer != "yes" { + fmt.Println("Aborted.") + return nil + } + } + + removed, skipped := rotateRunners(idle, poolRunnerForce, poolRunnerBypass) + fmt.Printf("Removed %d runner(s), skipped %d.\n", removed, skipped) + return nil + }, +} + +func filterIdleRunners(instances []params.Instance) []params.Instance { + var idle []params.Instance + for _, inst := range instances { + if inst.Status == commonParams.InstanceRunning && inst.RunnerStatus == params.RunnerIdle { + idle = append(idle, inst) + } + } + return idle +} + +func rotateRunners(instances []params.Instance, forceRemove, bypassGH bool) (removed, skipped int) { + for _, inst := range instances { + deleteReq := apiClientInstances.NewDeleteInstanceParams() + deleteReq.InstanceName = inst.Name + deleteReq.ForceRemove = &forceRemove + deleteReq.BypassGHUnauthorized = &bypassGH + if err := apiCli.Instances.DeleteInstance(deleteReq, authToken); err != nil { + fmt.Printf(" Warning: failed to remove %s: %v\n", inst.Name, err) + skipped++ + continue + } + fmt.Printf(" Removed %s\n", inst.Name) + removed++ + } + return +} + func init() { poolListCmd.Flags().StringVarP(&poolRepository, "repo", "r", "", "List all pools within this repository.") poolListCmd.Flags().StringVarP(&poolOrganization, "org", "o", "", "List all pools within this organization.") @@ -483,12 +607,26 @@ func init() { poolAddCmd.MarkFlagsMutuallyExclusive("repo", "org", "enterprise") poolAddCmd.MarkFlagsMutuallyExclusive("extra-specs-file", "extra-specs") + poolRunnerListCmd.Flags().BoolVar(&poolRunnerOutdated, "outdated", false, "List only runners with a generation older than the pool.") + + poolRunnerRotateCmd.Flags().BoolVar(&poolRunnerOutdated, "outdated", false, "Only rotate runners with a generation older than the pool.") + poolRunnerRotateCmd.Flags().BoolVar(&poolRunnerDryRun, "dry-run", false, "Show what would be removed without actually removing.") + poolRunnerRotateCmd.Flags().BoolVarP(&poolRunnerForce, "force-remove-runner", "f", false, "Ignore provider errors when removing runners.") + poolRunnerRotateCmd.Flags().BoolVarP(&poolRunnerBypass, "bypass-github-unauthorized", "b", false, "Ignore GitHub unauthorized errors when removing runners.") + poolRunnerRotateCmd.Flags().BoolVar(&poolRunnerConfirmSkip, "yes-i-really-mean-it", false, "Skip the confirmation prompt.") + + poolRunnerCmd.AddCommand( + poolRunnerListCmd, + poolRunnerRotateCmd, + ) + poolCmd.AddCommand( poolListCmd, poolShowCmd, poolDeleteCmd, poolUpdateCmd, poolAddCmd, + poolRunnerCmd, ) rootCmd.AddCommand(poolCmd) diff --git a/cmd/garm-cli/cmd/runner.go b/cmd/garm-cli/cmd/runner.go index aa885adf..a40d5e40 100644 --- a/cmd/garm-cli/cmd/runner.go +++ b/cmd/garm-cli/cmd/runner.go @@ -226,16 +226,20 @@ var runnerListCmd = &cobra.Command{ Use: "list", Aliases: []string{"ls"}, Short: "List runners", - Long: `List runners of pools, repositories, orgs or all of the above. + Long: `List runners of pools, scale sets, repositories, orgs or all of the above. -This command expects to get either a pool ID as a positional parameter, or it expects -that one of the supported switches be used to fetch runners of --repo, --org or --all +This command expects to get either a pool ID (UUID) or scale set ID (integer) as a +positional parameter, or it expects that one of the supported switches be used to +fetch runners of --repo, --org or --all Example: List runners from one pool: garm-cli runner list e87e70bd-3d0d-4b25-be9a-86b85e114bcb + List runners from one scale set: + garm-cli runner list 42 + List runners from one repo: garm-cli runner list --repo=05e7eac6-4705-486d-89c9-0170bbb576af @@ -265,11 +269,17 @@ Example: cmd.Flags().Changed("enterprise") || cmd.Flags().Changed("all") { - return fmt.Errorf("specifying a pool ID and any of [all org repo enterprise] are mutually exclusive") + return fmt.Errorf("specifying a pool/scaleset ID and any of [all org repo enterprise] are mutually exclusive") + } + if _, parseErr := uuid.Parse(args[0]); parseErr == nil { + listPoolInstancesReq := apiClientInstances.NewListPoolInstancesParams() + listPoolInstancesReq.PoolID = args[0] + response, err = apiCli.Instances.ListPoolInstances(listPoolInstancesReq, authToken) + } else { + listScaleSetReq := apiClientInstances.NewListScaleSetInstancesParams() + listScaleSetReq.ScalesetID = args[0] + response, err = apiCli.Instances.ListScaleSetInstances(listScaleSetReq, authToken) } - listPoolInstancesReq := apiClientInstances.NewListPoolInstancesParams() - listPoolInstancesReq.PoolID = args[0] - response, err = apiCli.Instances.ListPoolInstances(listPoolInstancesReq, authToken) case 0: if cmd.Flags().Changed("repo") { runnerRepo, resErr := resolveRepository(runnerRepository, endpointName) @@ -309,7 +319,7 @@ Example: } instances := response.GetPayload() - formatInstances(instances, long) + formatInstances(instances, long, true) return nil }, } @@ -403,20 +413,30 @@ func init() { rootCmd.AddCommand(runnerCmd) } -func formatInstances(param []params.Instance, detailed bool) { +func formatInstances(param []params.Instance, detailed bool, includeParent bool) { if outputFormat == common.OutputFormatJSON { printAsJSON(param) return } t := table.NewWriter() - header := table.Row{"Nr", "Name", "Status", "Runner Status", "Pool ID", "Scalse Set ID"} + header := table.Row{"Nr", "Name", "Status", "Runner Status"} + if includeParent { + header = append(header, "Pool / Scale Set") + } if detailed { header = append(header, "Created At", "Updated At", "Job Name", "Started At", "Run ID", "Repository") } t.AppendHeader(header) for idx, inst := range param { - row := table.Row{idx + 1, inst.Name, inst.Status, inst.RunnerStatus, inst.PoolID, inst.ScaleSetID} + row := table.Row{idx + 1, inst.Name, inst.Status, inst.RunnerStatus} + if includeParent { + poolOrScaleSet := fmt.Sprintf("Pool: %v", inst.PoolID) + if inst.ScaleSetID > 0 { + poolOrScaleSet = fmt.Sprintf("Scale Set: %d", inst.ScaleSetID) + } + row = append(row, poolOrScaleSet) + } if detailed { row = append(row, inst.CreatedAt, inst.UpdatedAt) if inst.Job != nil { diff --git a/cmd/garm-cli/cmd/scalesets.go b/cmd/garm-cli/cmd/scalesets.go index ca45ab90..41d4945c 100644 --- a/cmd/garm-cli/cmd/scalesets.go +++ b/cmd/garm-cli/cmd/scalesets.go @@ -15,14 +15,17 @@ package cmd import ( + "bufio" "fmt" "os" + "strings" "github.com/jedib0t/go-pretty/v6/table" "github.com/spf13/cobra" commonParams "github.com/cloudbase/garm-provider-common/params" apiClientEnterprises "github.com/cloudbase/garm/client/enterprises" + apiClientInstances "github.com/cloudbase/garm/client/instances" apiClientOrgs "github.com/cloudbase/garm/client/organizations" apiClientRepos "github.com/cloudbase/garm/client/repositories" apiClientScaleSets "github.com/cloudbase/garm/client/scalesets" @@ -415,6 +418,101 @@ explicitly remove them using the runner delete command. }, } +var ( + scalesetRunnerOutdated bool + scalesetRunnerDryRun bool + scalesetRunnerForce bool + scalesetRunnerBypass bool + scalesetRunnerConfirmSkip bool +) + +var scalesetRunnerCmd = &cobra.Command{ + Use: "runner", + Short: "Manage runners in a scale set", + Long: `List or rotate runners in a scale set.`, + SilenceUsage: true, + Run: nil, +} + +var scalesetRunnerListCmd = &cobra.Command{ + Use: "list ", + Aliases: []string{"ls"}, + Short: "List runners in a scale set", + Long: `List all runners belonging to a scale set. Use --outdated to show only runners with a generation older than the scale set.`, + Args: cobra.ExactArgs(1), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if needsInit { + return errNeedsInitError + } + + listReq := apiClientInstances.NewListScaleSetInstancesParams() + listReq.ScalesetID = args[0] + if cmd.Flags().Changed("outdated") { + listReq.OutdatedOnly = &scalesetRunnerOutdated + } + + response, err := apiCli.Instances.ListScaleSetInstances(listReq, authToken) + if err != nil { + return err + } + + formatInstances(response.Payload, false, false) + return nil + }, +} + +var scalesetRunnerRotateCmd = &cobra.Command{ + Use: "rotate ", + Short: "Rotate idle runners in a scale set", + Long: `Remove idle runners in a scale set so they get replaced with fresh ones. Use --outdated to only rotate runners with a generation older than the scale set.`, + Args: cobra.ExactArgs(1), + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + if needsInit { + return errNeedsInitError + } + + listReq := apiClientInstances.NewListScaleSetInstancesParams() + listReq.ScalesetID = args[0] + if cmd.Flags().Changed("outdated") { + listReq.OutdatedOnly = &scalesetRunnerOutdated + } + + response, err := apiCli.Instances.ListScaleSetInstances(listReq, authToken) + if err != nil { + return err + } + + idle := filterIdleRunners(response.Payload) + if len(idle) == 0 { + fmt.Println("No idle runners to rotate.") + return nil + } + + if scalesetRunnerDryRun { + fmt.Printf("Would remove %d idle runner(s):\n", len(idle)) + formatInstances(idle, false, false) + return nil + } + + if !scalesetRunnerConfirmSkip { + fmt.Printf("About to remove %d idle runner(s). Continue? [y/N] ", len(idle)) + reader := bufio.NewReader(os.Stdin) + answer, _ := reader.ReadString('\n') + answer = strings.TrimSpace(strings.ToLower(answer)) + if answer != "y" && answer != "yes" { + fmt.Println("Aborted.") + return nil + } + } + + removed, skipped := rotateRunners(idle, scalesetRunnerForce, scalesetRunnerBypass) + fmt.Printf("Removed %d runner(s), skipped %d.\n", removed, skipped) + return nil + }, +} + func init() { scalesetListCmd.Flags().StringVarP(&scalesetRepository, "repo", "r", "", "List all scale sets within this repository.") scalesetListCmd.Flags().StringVarP(&scalesetOrganization, "org", "o", "", "List all scale sets within this organization.") @@ -467,12 +565,26 @@ func init() { scaleSetAddCmd.MarkFlagsMutuallyExclusive("repo", "org", "enterprise") scaleSetAddCmd.MarkFlagsMutuallyExclusive("extra-specs-file", "extra-specs") + scalesetRunnerListCmd.Flags().BoolVar(&scalesetRunnerOutdated, "outdated", false, "List only runners with a generation older than the scale set.") + + scalesetRunnerRotateCmd.Flags().BoolVar(&scalesetRunnerOutdated, "outdated", false, "Only rotate runners with a generation older than the scale set.") + scalesetRunnerRotateCmd.Flags().BoolVar(&scalesetRunnerDryRun, "dry-run", false, "Show what would be removed without actually removing.") + scalesetRunnerRotateCmd.Flags().BoolVarP(&scalesetRunnerForce, "force-remove-runner", "f", false, "Ignore provider errors when removing runners.") + scalesetRunnerRotateCmd.Flags().BoolVarP(&scalesetRunnerBypass, "bypass-github-unauthorized", "b", false, "Ignore GitHub unauthorized errors when removing runners.") + scalesetRunnerRotateCmd.Flags().BoolVar(&scalesetRunnerConfirmSkip, "yes-i-really-mean-it", false, "Skip the confirmation prompt.") + + scalesetRunnerCmd.AddCommand( + scalesetRunnerListCmd, + scalesetRunnerRotateCmd, + ) + scalesetCmd.AddCommand( scalesetListCmd, scaleSetShowCmd, scaleSetDeleteCmd, scaleSetUpdateCmd, scaleSetAddCmd, + scalesetRunnerCmd, ) rootCmd.AddCommand(scalesetCmd) diff --git a/database/common/mocks/Store.go b/database/common/mocks/Store.go index 472cec44..353afc70 100644 --- a/database/common/mocks/Store.go +++ b/database/common/mocks/Store.go @@ -4445,9 +4445,9 @@ func (_c *Store_ListOrganizations_Call) RunAndReturn(run func(context.Context, p return _c } -// ListPoolInstances provides a mock function with given fields: ctx, poolID -func (_m *Store) ListPoolInstances(ctx context.Context, poolID string) ([]params.Instance, error) { - ret := _m.Called(ctx, poolID) +// ListPoolInstances provides a mock function with given fields: ctx, poolID, oudatedOnly +func (_m *Store) ListPoolInstances(ctx context.Context, poolID string, oudatedOnly bool) ([]params.Instance, error) { + ret := _m.Called(ctx, poolID, oudatedOnly) if len(ret) == 0 { panic("no return value specified for ListPoolInstances") @@ -4455,19 +4455,19 @@ func (_m *Store) ListPoolInstances(ctx context.Context, poolID string) ([]params var r0 []params.Instance var r1 error - if rf, ok := ret.Get(0).(func(context.Context, string) ([]params.Instance, error)); ok { - return rf(ctx, poolID) + if rf, ok := ret.Get(0).(func(context.Context, string, bool) ([]params.Instance, error)); ok { + return rf(ctx, poolID, oudatedOnly) } - if rf, ok := ret.Get(0).(func(context.Context, string) []params.Instance); ok { - r0 = rf(ctx, poolID) + if rf, ok := ret.Get(0).(func(context.Context, string, bool) []params.Instance); ok { + r0 = rf(ctx, poolID, oudatedOnly) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]params.Instance) } } - if rf, ok := ret.Get(1).(func(context.Context, string) error); ok { - r1 = rf(ctx, poolID) + if rf, ok := ret.Get(1).(func(context.Context, string, bool) error); ok { + r1 = rf(ctx, poolID, oudatedOnly) } else { r1 = ret.Error(1) } @@ -4483,13 +4483,14 @@ type Store_ListPoolInstances_Call struct { // ListPoolInstances is a helper method to define mock.On call // - ctx context.Context // - poolID string -func (_e *Store_Expecter) ListPoolInstances(ctx interface{}, poolID interface{}) *Store_ListPoolInstances_Call { - return &Store_ListPoolInstances_Call{Call: _e.mock.On("ListPoolInstances", ctx, poolID)} +// - oudatedOnly bool +func (_e *Store_Expecter) ListPoolInstances(ctx interface{}, poolID interface{}, oudatedOnly interface{}) *Store_ListPoolInstances_Call { + return &Store_ListPoolInstances_Call{Call: _e.mock.On("ListPoolInstances", ctx, poolID, oudatedOnly)} } -func (_c *Store_ListPoolInstances_Call) Run(run func(ctx context.Context, poolID string)) *Store_ListPoolInstances_Call { +func (_c *Store_ListPoolInstances_Call) Run(run func(ctx context.Context, poolID string, oudatedOnly bool)) *Store_ListPoolInstances_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(string)) + run(args[0].(context.Context), args[1].(string), args[2].(bool)) }) return _c } @@ -4499,7 +4500,7 @@ func (_c *Store_ListPoolInstances_Call) Return(_a0 []params.Instance, _a1 error) return _c } -func (_c *Store_ListPoolInstances_Call) RunAndReturn(run func(context.Context, string) ([]params.Instance, error)) *Store_ListPoolInstances_Call { +func (_c *Store_ListPoolInstances_Call) RunAndReturn(run func(context.Context, string, bool) ([]params.Instance, error)) *Store_ListPoolInstances_Call { _c.Call.Return(run) return _c } @@ -4563,9 +4564,9 @@ func (_c *Store_ListRepositories_Call) RunAndReturn(run func(context.Context, pa return _c } -// ListScaleSetInstances provides a mock function with given fields: _a0, scalesetID -func (_m *Store) ListScaleSetInstances(_a0 context.Context, scalesetID uint) ([]params.Instance, error) { - ret := _m.Called(_a0, scalesetID) +// ListScaleSetInstances provides a mock function with given fields: _a0, scalesetID, outdatedOnly +func (_m *Store) ListScaleSetInstances(_a0 context.Context, scalesetID uint, outdatedOnly bool) ([]params.Instance, error) { + ret := _m.Called(_a0, scalesetID, outdatedOnly) if len(ret) == 0 { panic("no return value specified for ListScaleSetInstances") @@ -4573,19 +4574,19 @@ func (_m *Store) ListScaleSetInstances(_a0 context.Context, scalesetID uint) ([] var r0 []params.Instance var r1 error - if rf, ok := ret.Get(0).(func(context.Context, uint) ([]params.Instance, error)); ok { - return rf(_a0, scalesetID) + if rf, ok := ret.Get(0).(func(context.Context, uint, bool) ([]params.Instance, error)); ok { + return rf(_a0, scalesetID, outdatedOnly) } - if rf, ok := ret.Get(0).(func(context.Context, uint) []params.Instance); ok { - r0 = rf(_a0, scalesetID) + if rf, ok := ret.Get(0).(func(context.Context, uint, bool) []params.Instance); ok { + r0 = rf(_a0, scalesetID, outdatedOnly) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]params.Instance) } } - if rf, ok := ret.Get(1).(func(context.Context, uint) error); ok { - r1 = rf(_a0, scalesetID) + if rf, ok := ret.Get(1).(func(context.Context, uint, bool) error); ok { + r1 = rf(_a0, scalesetID, outdatedOnly) } else { r1 = ret.Error(1) } @@ -4601,13 +4602,14 @@ type Store_ListScaleSetInstances_Call struct { // ListScaleSetInstances is a helper method to define mock.On call // - _a0 context.Context // - scalesetID uint -func (_e *Store_Expecter) ListScaleSetInstances(_a0 interface{}, scalesetID interface{}) *Store_ListScaleSetInstances_Call { - return &Store_ListScaleSetInstances_Call{Call: _e.mock.On("ListScaleSetInstances", _a0, scalesetID)} +// - outdatedOnly bool +func (_e *Store_Expecter) ListScaleSetInstances(_a0 interface{}, scalesetID interface{}, outdatedOnly interface{}) *Store_ListScaleSetInstances_Call { + return &Store_ListScaleSetInstances_Call{Call: _e.mock.On("ListScaleSetInstances", _a0, scalesetID, outdatedOnly)} } -func (_c *Store_ListScaleSetInstances_Call) Run(run func(_a0 context.Context, scalesetID uint)) *Store_ListScaleSetInstances_Call { +func (_c *Store_ListScaleSetInstances_Call) Run(run func(_a0 context.Context, scalesetID uint, outdatedOnly bool)) *Store_ListScaleSetInstances_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(uint)) + run(args[0].(context.Context), args[1].(uint), args[2].(bool)) }) return _c } @@ -4617,7 +4619,7 @@ func (_c *Store_ListScaleSetInstances_Call) Return(_a0 []params.Instance, _a1 er return _c } -func (_c *Store_ListScaleSetInstances_Call) RunAndReturn(run func(context.Context, uint) ([]params.Instance, error)) *Store_ListScaleSetInstances_Call { +func (_c *Store_ListScaleSetInstances_Call) RunAndReturn(run func(context.Context, uint, bool) ([]params.Instance, error)) *Store_ListScaleSetInstances_Call { _c.Call.Return(run) return _c } diff --git a/database/common/store.go b/database/common/store.go index f22ce146..f5bf73c2 100644 --- a/database/common/store.go +++ b/database/common/store.go @@ -75,7 +75,7 @@ type PoolStore interface { GetPoolByID(ctx context.Context, poolID string) (params.Pool, error) DeletePoolByID(ctx context.Context, poolID string) error - ListPoolInstances(ctx context.Context, poolID string) ([]params.Instance, error) + ListPoolInstances(ctx context.Context, poolID string, oudatedOnly bool) ([]params.Instance, error) PoolInstanceCount(ctx context.Context, poolID string) (int64, error) FindPoolsMatchingAllTags(ctx context.Context, entityType params.ForgeEntityType, entityID string, tags []string) ([]params.Pool, error) @@ -152,7 +152,7 @@ type ScaleSetsStore interface { } type ScaleSetInstanceStore interface { - ListScaleSetInstances(_ context.Context, scalesetID uint) ([]params.Instance, error) + ListScaleSetInstances(_ context.Context, scalesetID uint, outdatedOnly bool) ([]params.Instance, error) CreateScaleSetInstance(_ context.Context, scaleSetID uint, param params.CreateInstanceParams) (instance params.Instance, err error) } diff --git a/database/sql/instances.go b/database/sql/instances.go index a5e9949e..f3d6720e 100644 --- a/database/sql/instances.go +++ b/database/sql/instances.go @@ -90,6 +90,7 @@ func (s *sqlDatabase) CreateInstance(ctx context.Context, poolID string, param p JitConfiguration: secret, AditionalLabels: labels, AgentID: param.AgentID, + Generation: param.Generation, } q = tx.Create(&newInstance) if q.Error != nil { @@ -511,14 +512,18 @@ func (s *sqlDatabase) listInstancesBatched(queryModifier func(*gorm.DB) *gorm.DB return ret, err } -func (s *sqlDatabase) ListPoolInstances(_ context.Context, poolID string) ([]params.Instance, error) { +func (s *sqlDatabase) ListPoolInstances(_ context.Context, poolID string, outdatedOnly bool) ([]params.Instance, error) { u, err := uuid.Parse(poolID) if err != nil { return nil, fmt.Errorf("error parsing id: %w", runnerErrors.ErrBadRequest) } ret, err := s.listInstancesBatched(func(query *gorm.DB) *gorm.DB { - return query.Where("pool_id = ?", u) + q := query.Where("pool_id = ?", u) + if outdatedOnly { + q = q.Where("instances.generation < (SELECT pools.generation FROM pools WHERE pools.id = instances.pool_id)") + } + return q }) if err != nil { return nil, fmt.Errorf("failed to list pool instances: %w", err) @@ -527,7 +532,7 @@ func (s *sqlDatabase) ListPoolInstances(_ context.Context, poolID string) ([]par } func (s *sqlDatabase) ListAllInstances(_ context.Context) ([]params.Instance, error) { - ret, err := s.listInstancesBatched(nil) // No query modifier for all instances + ret, err := s.listInstancesBatched(nil) if err != nil { return nil, fmt.Errorf("failed to list all instances: %w", err) } diff --git a/database/sql/instances_test.go b/database/sql/instances_test.go index 8c29fe9b..a183f917 100644 --- a/database/sql/instances_test.go +++ b/database/sql/instances_test.go @@ -668,14 +668,14 @@ func (s *InstancesTestSuite) TestUpdateInstanceDBUpdateAddressErr() { } func (s *InstancesTestSuite) TestListPoolInstances() { - instances, err := s.Store.ListPoolInstances(s.adminCtx, s.Fixtures.Pool.ID) + instances, err := s.Store.ListPoolInstances(s.adminCtx, s.Fixtures.Pool.ID, false) s.Require().Nil(err) s.equalInstancesByName(s.Fixtures.Instances, instances) } func (s *InstancesTestSuite) TestListPoolInstancesInvalidPoolID() { - _, err := s.Store.ListPoolInstances(s.adminCtx, "dummy-pool-id") + _, err := s.Store.ListPoolInstances(s.adminCtx, "dummy-pool-id", false) s.Require().Equal("error parsing id: invalid request", err.Error()) } diff --git a/database/sql/models.go b/database/sql/models.go index efbe8797..2a5313d7 100644 --- a/database/sql/models.go +++ b/database/sql/models.go @@ -121,6 +121,14 @@ type Pool struct { GitHubRunnerGroup string EnableShell bool + // Generation holds the numeric generation of the pool. This number + // will be incremented, every time certain settings of the pool, which + // may influence how runners are created (flavor, specs, image) are changed. + // When a runner is created, this generation will be copied to the runners as + // well. That way if some settings diverge, we can target those runners + // to be recreated. + Generation uint64 + RepoID *uuid.UUID `gorm:"index"` Repository Repository `gorm:"foreignKey:RepoID;"` @@ -178,6 +186,13 @@ type ScaleSet struct { ExtraSpecs datatypes.JSON EnableShell bool + // Generation is the scaleset generation at the time of creating this instance. + // This field is to track a divergence between when the instance was created + // and the settings currently set on a scaleset. We can then use this field to know + // if the instance is out of date with the scaleset, allowing us to remove it if we + // need to. + Generation uint64 + RepoID *uuid.UUID `gorm:"index"` Repository Repository `gorm:"foreignKey:RepoID;"` @@ -336,6 +351,12 @@ type Instance struct { GitHubRunnerGroup string AditionalLabels datatypes.JSON Capabilities datatypes.JSON + // Generation is the pool generation at the time of creating this instance. + // This field is to track a divergence between when the instance was created + // and the settings currently set on a pool. We can then use this field to know + // if the instance is out of date with the pool, allowing us to remove it if we + // need to. + Generation uint64 PoolID *uuid.UUID Pool Pool `gorm:"foreignKey:PoolID"` diff --git a/database/sql/pools_test.go b/database/sql/pools_test.go index bf9101db..04f365cb 100644 --- a/database/sql/pools_test.go +++ b/database/sql/pools_test.go @@ -150,7 +150,7 @@ func (s *PoolsTestSuite) TestListAllPools() { func (s *PoolsTestSuite) TestListAllPoolsDBFetchErr() { s.Fixtures.SQLMock. - ExpectQuery(regexp.QuoteMeta("SELECT `pools`.`id`,`pools`.`created_at`,`pools`.`updated_at`,`pools`.`deleted_at`,`pools`.`provider_name`,`pools`.`runner_prefix`,`pools`.`max_runners`,`pools`.`min_idle_runners`,`pools`.`runner_bootstrap_timeout`,`pools`.`image`,`pools`.`flavor`,`pools`.`os_type`,`pools`.`os_arch`,`pools`.`enabled`,`pools`.`git_hub_runner_group`,`pools`.`enable_shell`,`pools`.`repo_id`,`pools`.`org_id`,`pools`.`enterprise_id`,`pools`.`template_id`,`pools`.`priority` FROM `pools` WHERE `pools`.`deleted_at` IS NULL")). + ExpectQuery(regexp.QuoteMeta("SELECT `pools`.`id`,`pools`.`created_at`,`pools`.`updated_at`,`pools`.`deleted_at`,`pools`.`provider_name`,`pools`.`runner_prefix`,`pools`.`max_runners`,`pools`.`min_idle_runners`,`pools`.`runner_bootstrap_timeout`,`pools`.`image`,`pools`.`flavor`,`pools`.`os_type`,`pools`.`os_arch`,`pools`.`enabled`,`pools`.`git_hub_runner_group`,`pools`.`enable_shell`,`pools`.`generation`,`pools`.`repo_id`,`pools`.`org_id`,`pools`.`enterprise_id`,`pools`.`template_id`,`pools`.`priority` FROM `pools` WHERE `pools`.`deleted_at` IS NULL")). WillReturnError(fmt.Errorf("mocked fetching all pools error")) _, err := s.StoreSQLMocked.ListAllPools(s.adminCtx) diff --git a/database/sql/scaleset_instances.go b/database/sql/scaleset_instances.go index 6c7e9167..543ec0d7 100644 --- a/database/sql/scaleset_instances.go +++ b/database/sql/scaleset_instances.go @@ -65,9 +65,13 @@ func (s *sqlDatabase) CreateScaleSetInstance(_ context.Context, scaleSetID uint, return s.sqlToParamsInstance(newInstance) } -func (s *sqlDatabase) ListScaleSetInstances(_ context.Context, scalesetID uint) ([]params.Instance, error) { +func (s *sqlDatabase) ListScaleSetInstances(_ context.Context, scalesetID uint, outdatedOnly bool) ([]params.Instance, error) { ret, err := s.listInstancesBatched(func(query *gorm.DB) *gorm.DB { - return query.Where("scale_set_fk_id = ?", scalesetID) + q := query.Where("scale_set_fk_id = ?", scalesetID) + if outdatedOnly { + q = q.Where("instances.generation < (SELECT scale_sets.generation FROM scale_sets WHERE scale_sets.id = instances.scale_set_fk_id)") + } + return q }) if err != nil { return nil, fmt.Errorf("failed to list scaleset instances: %w", err) diff --git a/database/sql/scalesets.go b/database/sql/scalesets.go index 19d7c282..44080d26 100644 --- a/database/sql/scalesets.go +++ b/database/sql/scalesets.go @@ -284,6 +284,7 @@ func (s *sqlDatabase) getEntityScaleSet(tx *gorm.DB, entityType params.ForgeEnti } func (s *sqlDatabase) updateScaleSet(tx *gorm.DB, scaleSet ScaleSet, param params.UpdateScaleSetParams) (params.ScaleSet, error) { + incrementGeneration := false if param.Enabled != nil && scaleSet.Enabled != *param.Enabled { scaleSet.Enabled = *param.Enabled } @@ -306,6 +307,7 @@ func (s *sqlDatabase) updateScaleSet(tx *gorm.DB, scaleSet ScaleSet, param param if param.EnableShell != nil { scaleSet.EnableShell = *param.EnableShell + incrementGeneration = true } if param.Name != "" { @@ -314,14 +316,17 @@ func (s *sqlDatabase) updateScaleSet(tx *gorm.DB, scaleSet ScaleSet, param param if param.GitHubRunnerGroup != nil && *param.GitHubRunnerGroup != "" { scaleSet.GitHubRunnerGroup = *param.GitHubRunnerGroup + incrementGeneration = true } if param.Flavor != "" { scaleSet.Flavor = param.Flavor + incrementGeneration = true } if param.Image != "" { scaleSet.Image = param.Image + incrementGeneration = true } if param.Prefix != "" { @@ -338,22 +343,25 @@ func (s *sqlDatabase) updateScaleSet(tx *gorm.DB, scaleSet ScaleSet, param param if param.OSArch != "" { scaleSet.OSArch = param.OSArch + incrementGeneration = true } if param.OSType != "" { scaleSet.OSType = param.OSType + incrementGeneration = true } if param.ExtraSpecs != nil { scaleSet.ExtraSpecs = datatypes.JSON(param.ExtraSpecs) + incrementGeneration = true } if param.RunnerBootstrapTimeout != nil && *param.RunnerBootstrapTimeout > 0 { scaleSet.RunnerBootstrapTimeout = *param.RunnerBootstrapTimeout } - if param.GitHubRunnerGroup != nil { - scaleSet.GitHubRunnerGroup = *param.GitHubRunnerGroup + if incrementGeneration { + scaleSet.Generation++ } if q := tx.Save(&scaleSet); q.Error != nil { diff --git a/database/sql/scalesets_test.go b/database/sql/scalesets_test.go index 527cd89a..74e57253 100644 --- a/database/sql/scalesets_test.go +++ b/database/sql/scalesets_test.go @@ -356,7 +356,7 @@ func (s *ScaleSetsTestSuite) TestScaleSetOperations() { }) s.T().Run("List repo scale set instances", func(_ *testing.T) { - instances, err := s.Store.ListScaleSetInstances(s.adminCtx, repoScaleSet.ID) + instances, err := s.Store.ListScaleSetInstances(s.adminCtx, repoScaleSet.ID, false) s.Require().NoError(err) s.Require().NotEmpty(instances) s.Require().Len(instances, 1) diff --git a/database/sql/util.go b/database/sql/util.go index 3220d844..bc793518 100644 --- a/database/sql/util.go +++ b/database/sql/util.go @@ -75,6 +75,7 @@ func (s *sqlDatabase) sqlToParamsInstance(instance Instance) (params.Instance, e GitHubRunnerGroup: instance.GitHubRunnerGroup, AditionalLabels: labels, Heartbeat: instance.Heartbeat, + Generation: instance.Generation, } if len(instance.Capabilities) > 0 { @@ -299,6 +300,7 @@ func (s *sqlDatabase) sqlToCommonPool(pool Pool) (params.Pool, error) { CreatedAt: pool.CreatedAt, UpdatedAt: pool.UpdatedAt, EnableShell: pool.EnableShell, + Generation: pool.Generation, } if pool.TemplateID != nil && *pool.TemplateID != 0 { @@ -376,6 +378,7 @@ func (s *sqlDatabase) sqlToCommonScaleSet(scaleSet ScaleSet) (params.ScaleSet, e LastMessageID: scaleSet.LastMessageID, DesiredRunnerCount: scaleSet.DesiredRunnerCount, EnableShell: scaleSet.EnableShell, + Generation: scaleSet.Generation, } if scaleSet.TemplateID != nil && *scaleSet.TemplateID != 0 { @@ -539,20 +542,24 @@ func (s *sqlDatabase) getOrCreateTag(tx *gorm.DB, tagName string) (Tag, error) { } func (s *sqlDatabase) updatePool(tx *gorm.DB, pool Pool, param params.UpdatePoolParams) (params.Pool, error) { + incrementGeneration := false if param.Enabled != nil && pool.Enabled != *param.Enabled { pool.Enabled = *param.Enabled } if param.Flavor != "" { pool.Flavor = param.Flavor + incrementGeneration = true } if param.EnableShell != nil { pool.EnableShell = *param.EnableShell + incrementGeneration = true } if param.Image != "" { pool.Image = param.Image + incrementGeneration = true } if param.Prefix != "" { @@ -573,14 +580,17 @@ func (s *sqlDatabase) updatePool(tx *gorm.DB, pool Pool, param params.UpdatePool if param.OSArch != "" { pool.OSArch = param.OSArch + incrementGeneration = true } if param.OSType != "" { pool.OSType = param.OSType + incrementGeneration = true } if param.ExtraSpecs != nil { pool.ExtraSpecs = datatypes.JSON(param.ExtraSpecs) + incrementGeneration = true } if param.RunnerBootstrapTimeout != nil && *param.RunnerBootstrapTimeout > 0 { @@ -589,12 +599,17 @@ func (s *sqlDatabase) updatePool(tx *gorm.DB, pool Pool, param params.UpdatePool if param.GitHubRunnerGroup != nil { pool.GitHubRunnerGroup = *param.GitHubRunnerGroup + incrementGeneration = true } if param.Priority != nil { pool.Priority = *param.Priority } + if incrementGeneration { + pool.Generation++ + } + if q := tx.Save(&pool); q.Error != nil { return params.Pool{}, fmt.Errorf("error saving database entry: %w", q.Error) } diff --git a/params/params.go b/params/params.go index 3cae573b..427fbb36 100644 --- a/params/params.go +++ b/params/params.go @@ -364,6 +364,13 @@ type Instance struct { Heartbeat time.Time `json:"heartbeat"` Capabilities AgentCapabilities `json:"capabilities"` + // Generation is the pool generation at the time of creating this instance. + // This field is to track a divergence between when the instance was created + // and the settings currently set on a pool. We can then use this field to know + // if the instance is out of date with the pool, allowing us to remove it if we + // need to. + Generation uint64 `json:"generation"` + // Do not serialize sensitive info. CallbackURL string `json:"-"` MetadataURL string `json:"-"` @@ -474,6 +481,14 @@ type Pool struct { Instances []Instance `json:"instances,omitempty"` EnableShell bool `json:"enable_shell"` + // Generation holds the numeric generation of the pool. This number + // will be incremented, every time certain settings of the pool, which + // may influence how runners are created (flavor, specs, image) are changed. + // When a runner is created, this generation will be copied to the runners as + // well. That way if some settings diverge, we can target those runners + // to be recreated. + Generation uint64 `json:"generation"` + RepoID string `json:"repo_id,omitempty"` RepoName string `json:"repo_name,omitempty"` @@ -630,6 +645,14 @@ type ScaleSet struct { DesiredRunnerCount int `json:"desired_runner_count,omitempty"` EnableShell bool `json:"enable_shell"` + // Generation holds the numeric generation of the scaleset. This number + // will be incremented, every time certain settings of the scaleset, which + // may influence how runners are created (flavor, specs, image) are changed. + // When a runner is created, this generation will be copied to the runners as + // well. That way if some settings diverge, we can target those runners + // to be recreated. + Generation uint64 `json:"generation"` + Endpoint ForgeEndpoint `json:"endpoint,omitempty"` RunnerBootstrapTimeout uint `json:"runner_bootstrap_timeout,omitempty"` diff --git a/params/requests.go b/params/requests.go index cd756e99..ad639147 100644 --- a/params/requests.go +++ b/params/requests.go @@ -196,6 +196,7 @@ type CreateInstanceParams struct { AgentID int64 `json:"-"` AditionalLabels []string `json:"aditional_labels,omitempty"` JitConfiguration map[string]string `json:"jit_configuration,omitempty"` + Generation uint64 `json:"generation"` } // swagger:model CreatePoolParams diff --git a/runner/garm_tools_test.go b/runner/garm_tools_test.go index ca742795..4d6205e1 100644 --- a/runner/garm_tools_test.go +++ b/runner/garm_tools_test.go @@ -166,7 +166,6 @@ func (s *GARMToolsTestSuite) TestCreateGARMToolSuccess() { s.Equal(param.Description, tool.Description) s.Equal(int64(len(content)), tool.Size) - // Verify tags (should include origin=manual) expectedTags := []string{ "category=garm-agent", "os_type=linux", diff --git a/runner/pool/pool.go b/runner/pool/pool.go index 6b2a95db..4722f0d6 100644 --- a/runner/pool/pool.go +++ b/runner/pool/pool.go @@ -798,6 +798,7 @@ func (r *basePoolManager) AddRunner(ctx context.Context, poolID string, aditiona GitHubRunnerGroup: pool.GitHubRunnerGroup, AditionalLabels: aditionalLabels, JitConfiguration: jitConfig, + Generation: pool.Generation, } if runner != nil { @@ -1050,7 +1051,7 @@ func (r *basePoolManager) scaleDownOnePool(ctx context.Context, pool params.Pool return nil } - existingInstances, err := r.store.ListPoolInstances(r.ctx, pool.ID) + existingInstances, err := r.store.ListPoolInstances(r.ctx, pool.ID, false) if err != nil { return fmt.Errorf("failed to ensure minimum idle workers for pool %s: %w", pool.ID, err) } @@ -1156,7 +1157,7 @@ func (r *basePoolManager) ensureIdleRunnersForOnePool(pool params.Pool) error { return nil } - existingInstances, err := r.store.ListPoolInstances(r.ctx, pool.ID) + existingInstances, err := r.store.ListPoolInstances(r.ctx, pool.ID, false) if err != nil { return fmt.Errorf("failed to ensure minimum idle workers for pool %s: %w", pool.ID, err) } @@ -1213,7 +1214,7 @@ func (r *basePoolManager) retryFailedInstancesForOnePool(ctx context.Context, po ctx, "running retry failed instances for pool", "pool_id", pool.ID) - existingInstances, err := r.store.ListPoolInstances(r.ctx, pool.ID) + existingInstances, err := r.store.ListPoolInstances(r.ctx, pool.ID, false) if err != nil { return fmt.Errorf("failed to list instances for pool %s: %w", pool.ID, err) } diff --git a/runner/repositories.go b/runner/repositories.go index 1eb7580c..9a39f523 100644 --- a/runner/repositories.go +++ b/runner/repositories.go @@ -364,12 +364,12 @@ func (r *Runner) ListRepoPools(ctx context.Context, repoID string) ([]params.Poo return pools, nil } -func (r *Runner) ListPoolInstances(ctx context.Context, poolID string) ([]params.Instance, error) { +func (r *Runner) ListPoolInstances(ctx context.Context, poolID string, outdatedOnly bool) ([]params.Instance, error) { if !auth.IsAdmin(ctx) { return nil, runnerErrors.ErrUnauthorized } - instances, err := r.store.ListPoolInstances(ctx, poolID) + instances, err := r.store.ListPoolInstances(ctx, poolID, outdatedOnly) if err != nil { return []params.Instance{}, fmt.Errorf("error fetching instances: %w", err) } diff --git a/runner/repositories_test.go b/runner/repositories_test.go index 05a52b60..14c7c559 100644 --- a/runner/repositories_test.go +++ b/runner/repositories_test.go @@ -605,14 +605,14 @@ func (s *RepoTestSuite) TestListPoolInstances() { poolInstances = append(poolInstances, instance) } - instances, err := s.Runner.ListPoolInstances(s.Fixtures.AdminContext, pool.ID) + instances, err := s.Runner.ListPoolInstances(s.Fixtures.AdminContext, pool.ID, false) s.Require().Nil(err) garmTesting.EqualDBEntityID(s.T(), poolInstances, instances) } func (s *RepoTestSuite) TestListPoolInstancesErrUnauthorized() { - _, err := s.Runner.ListPoolInstances(context.Background(), "dummy-pool-id") + _, err := s.Runner.ListPoolInstances(context.Background(), "dummy-pool-id", false) s.Require().Equal(runnerErrors.ErrUnauthorized, err) } diff --git a/runner/scalesets.go b/runner/scalesets.go index a9a2258d..20d2fa3d 100644 --- a/runner/scalesets.go +++ b/runner/scalesets.go @@ -288,12 +288,12 @@ func (r *Runner) CreateEntityScaleSet(ctx context.Context, entityType params.For return scaleSet, nil } -func (r *Runner) ListScaleSetInstances(ctx context.Context, scalesetID uint) ([]params.Instance, error) { +func (r *Runner) ListScaleSetInstances(ctx context.Context, scalesetID uint, outdatedOnly bool) ([]params.Instance, error) { if !auth.IsAdmin(ctx) { return nil, runnerErrors.ErrUnauthorized } - instances, err := r.store.ListScaleSetInstances(ctx, scalesetID) + instances, err := r.store.ListScaleSetInstances(ctx, scalesetID, outdatedOnly) if err != nil { return []params.Instance{}, fmt.Errorf("error fetching instances: %w", err) } diff --git a/webapp/src/lib/api/generated/api.ts b/webapp/src/lib/api/generated/api.ts index bb413dc1..c8fe1246 100644 --- a/webapp/src/lib/api/generated/api.ts +++ b/webapp/src/lib/api/generated/api.ts @@ -1391,6 +1391,12 @@ export interface GARMAgentTool { * @memberof GARMAgentTool */ 'size'?: number; + /** + * Source indicates where this tool is currently stored. \"local\" means the tool is stored in the internal object store. \"upstream\" means the tool is only available from the upstream cached release and has not been downloaded locally. + * @type {string} + * @memberof GARMAgentTool + */ + 'source'?: string; /** * * @type {string} @@ -1519,6 +1525,12 @@ export interface GARMAgentToolsPaginatedResponseResultsInner { * @memberof GARMAgentToolsPaginatedResponseResultsInner */ 'size'?: number; + /** + * Source indicates where this tool is currently stored. \"local\" means the tool is stored in the internal object store. \"upstream\" means the tool is only available from the upstream cached release and has not been downloaded locally. + * @type {string} + * @memberof GARMAgentToolsPaginatedResponseResultsInner + */ + 'source'?: string; /** * * @type {string} @@ -1687,6 +1699,12 @@ export interface Instance { * @memberof Instance */ 'created_at'?: string; + /** + * Generation is the pool generation at the time of creating this instance. This field is to track a divergence between when the instance was created and the settings currently set on a pool. We can then use this field to know if the instance is out of date with the pool, allowing us to remove it if we need to. + * @type {number} + * @memberof Instance + */ + 'generation'?: number; /** * GithubRunnerGroup is the github runner group to which the runner belongs. The runner group must be created by someone with access to the enterprise. * @type {string} @@ -2253,6 +2271,12 @@ export interface Pool { * @memberof Pool */ 'flavor'?: string; + /** + * Generation holds the numeric generation of the pool. This number will be incremented, every time certain settings of the pool, which may influence how runners are created (flavor, specs, image) are changed. When a runner is created, this generation will be copied to the runners as well. That way if some settings diverge, we can target those runners to be recreated. + * @type {number} + * @memberof Pool + */ + 'generation'?: number; /** * GithubRunnerGroup is the github runner group in which the runners will be added. The runner group must be created by someone with access to the enterprise. * @type {string} @@ -2662,6 +2686,12 @@ export interface ScaleSet { * @memberof ScaleSet */ 'flavor'?: string; + /** + * Generation holds the numeric generation of the scaleset. This number will be incremented, every time certain settings of the scaleset, which may influence how runners are created (flavor, specs, image) are changed. When a runner is created, this generation will be copied to the runners as well. That way if some settings diverge, we can target those runners to be recreated. + * @type {number} + * @memberof ScaleSet + */ + 'generation'?: number; /** * GithubRunnerGroup is the github runner group in which the runners will be added. The runner group must be created by someone with access to the enterprise. * @type {string} @@ -7082,10 +7112,11 @@ export const InstancesApiAxiosParamCreator = function (configuration?: Configura * * @summary List runner instances in a pool. * @param {string} poolID Runner pool ID. + * @param {boolean} [outdatedOnly] List only instances that were created prior to a pool update that changed a setting which influences how instances are created (image, flavor, runner group, etc). * @param {*} [options] Override http request option. * @throws {RequiredError} */ - listPoolInstances: async (poolID: string, options: RawAxiosRequestConfig = {}): Promise => { + listPoolInstances: async (poolID: string, outdatedOnly?: boolean, options: RawAxiosRequestConfig = {}): Promise => { // verify required parameter 'poolID' is not null or undefined assertParamExists('listPoolInstances', 'poolID', poolID) const localVarPath = `/pools/{poolID}/instances` @@ -7104,6 +7135,10 @@ export const InstancesApiAxiosParamCreator = function (configuration?: Configura // authentication Bearer required await setApiKeyToObject(localVarHeaderParameter, "Authorization", configuration) + if (outdatedOnly !== undefined) { + localVarQueryParameter['outdatedOnly'] = outdatedOnly; + } + setSearchParams(localVarUrlObj, localVarQueryParameter); @@ -7156,10 +7191,11 @@ export const InstancesApiAxiosParamCreator = function (configuration?: Configura * * @summary List runner instances in a scale set. * @param {string} scalesetID Runner scale set ID. + * @param {boolean} [outdatedOnly] List only instances that were created prior to a scaleset update that changed a setting which influences how instances are created (image, flavor, runner group, etc). * @param {*} [options] Override http request option. * @throws {RequiredError} */ - listScaleSetInstances: async (scalesetID: string, options: RawAxiosRequestConfig = {}): Promise => { + listScaleSetInstances: async (scalesetID: string, outdatedOnly?: boolean, options: RawAxiosRequestConfig = {}): Promise => { // verify required parameter 'scalesetID' is not null or undefined assertParamExists('listScaleSetInstances', 'scalesetID', scalesetID) const localVarPath = `/scalesets/{scalesetID}/instances` @@ -7178,6 +7214,10 @@ export const InstancesApiAxiosParamCreator = function (configuration?: Configura // authentication Bearer required await setApiKeyToObject(localVarHeaderParameter, "Authorization", configuration) + if (outdatedOnly !== undefined) { + localVarQueryParameter['outdatedOnly'] = outdatedOnly; + } + setSearchParams(localVarUrlObj, localVarQueryParameter); @@ -7269,11 +7309,12 @@ export const InstancesApiFp = function(configuration?: Configuration) { * * @summary List runner instances in a pool. * @param {string} poolID Runner pool ID. + * @param {boolean} [outdatedOnly] List only instances that were created prior to a pool update that changed a setting which influences how instances are created (image, flavor, runner group, etc). * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async listPoolInstances(poolID: string, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { - const localVarAxiosArgs = await localVarAxiosParamCreator.listPoolInstances(poolID, options); + async listPoolInstances(poolID: string, outdatedOnly?: boolean, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { + const localVarAxiosArgs = await localVarAxiosParamCreator.listPoolInstances(poolID, outdatedOnly, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; const localVarOperationServerBasePath = operationServerMap['InstancesApi.listPoolInstances']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); @@ -7295,11 +7336,12 @@ export const InstancesApiFp = function(configuration?: Configuration) { * * @summary List runner instances in a scale set. * @param {string} scalesetID Runner scale set ID. + * @param {boolean} [outdatedOnly] List only instances that were created prior to a scaleset update that changed a setting which influences how instances are created (image, flavor, runner group, etc). * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async listScaleSetInstances(scalesetID: string, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { - const localVarAxiosArgs = await localVarAxiosParamCreator.listScaleSetInstances(scalesetID, options); + async listScaleSetInstances(scalesetID: string, outdatedOnly?: boolean, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { + const localVarAxiosArgs = await localVarAxiosParamCreator.listScaleSetInstances(scalesetID, outdatedOnly, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; const localVarOperationServerBasePath = operationServerMap['InstancesApi.listScaleSetInstances']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); @@ -7369,11 +7411,12 @@ export const InstancesApiFactory = function (configuration?: Configuration, base * * @summary List runner instances in a pool. * @param {string} poolID Runner pool ID. + * @param {boolean} [outdatedOnly] List only instances that were created prior to a pool update that changed a setting which influences how instances are created (image, flavor, runner group, etc). * @param {*} [options] Override http request option. * @throws {RequiredError} */ - listPoolInstances(poolID: string, options?: RawAxiosRequestConfig): AxiosPromise> { - return localVarFp.listPoolInstances(poolID, options).then((request) => request(axios, basePath)); + listPoolInstances(poolID: string, outdatedOnly?: boolean, options?: RawAxiosRequestConfig): AxiosPromise> { + return localVarFp.listPoolInstances(poolID, outdatedOnly, options).then((request) => request(axios, basePath)); }, /** * @@ -7389,11 +7432,12 @@ export const InstancesApiFactory = function (configuration?: Configuration, base * * @summary List runner instances in a scale set. * @param {string} scalesetID Runner scale set ID. + * @param {boolean} [outdatedOnly] List only instances that were created prior to a scaleset update that changed a setting which influences how instances are created (image, flavor, runner group, etc). * @param {*} [options] Override http request option. * @throws {RequiredError} */ - listScaleSetInstances(scalesetID: string, options?: RawAxiosRequestConfig): AxiosPromise> { - return localVarFp.listScaleSetInstances(scalesetID, options).then((request) => request(axios, basePath)); + listScaleSetInstances(scalesetID: string, outdatedOnly?: boolean, options?: RawAxiosRequestConfig): AxiosPromise> { + return localVarFp.listScaleSetInstances(scalesetID, outdatedOnly, options).then((request) => request(axios, basePath)); }, }; }; @@ -7470,12 +7514,13 @@ export class InstancesApi extends BaseAPI { * * @summary List runner instances in a pool. * @param {string} poolID Runner pool ID. + * @param {boolean} [outdatedOnly] List only instances that were created prior to a pool update that changed a setting which influences how instances are created (image, flavor, runner group, etc). * @param {*} [options] Override http request option. * @throws {RequiredError} * @memberof InstancesApi */ - public listPoolInstances(poolID: string, options?: RawAxiosRequestConfig) { - return InstancesApiFp(this.configuration).listPoolInstances(poolID, options).then((request) => request(this.axios, this.basePath)); + public listPoolInstances(poolID: string, outdatedOnly?: boolean, options?: RawAxiosRequestConfig) { + return InstancesApiFp(this.configuration).listPoolInstances(poolID, outdatedOnly, options).then((request) => request(this.axios, this.basePath)); } /** @@ -7494,12 +7539,13 @@ export class InstancesApi extends BaseAPI { * * @summary List runner instances in a scale set. * @param {string} scalesetID Runner scale set ID. + * @param {boolean} [outdatedOnly] List only instances that were created prior to a scaleset update that changed a setting which influences how instances are created (image, flavor, runner group, etc). * @param {*} [options] Override http request option. * @throws {RequiredError} * @memberof InstancesApi */ - public listScaleSetInstances(scalesetID: string, options?: RawAxiosRequestConfig) { - return InstancesApiFp(this.configuration).listScaleSetInstances(scalesetID, options).then((request) => request(this.axios, this.basePath)); + public listScaleSetInstances(scalesetID: string, outdatedOnly?: boolean, options?: RawAxiosRequestConfig) { + return InstancesApiFp(this.configuration).listScaleSetInstances(scalesetID, outdatedOnly, options).then((request) => request(this.axios, this.basePath)); } } @@ -13658,13 +13704,14 @@ export const ToolsApiAxiosParamCreator = function (configuration?: Configuration return { /** * - * @summary List GARM agent tools. + * @summary List GARM agent tools for admin users. * @param {number} [page] The page at which to list. * @param {number} [pageSize] Number of items per page. + * @param {boolean} [upstream] If true, list tools from the upstream cached release instead of the local object store. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - garmAgentList: async (page?: number, pageSize?: number, options: RawAxiosRequestConfig = {}): Promise => { + adminGarmAgentList: async (page?: number, pageSize?: number, upstream?: boolean, options: RawAxiosRequestConfig = {}): Promise => { const localVarPath = `/tools/garm-agent`; // use dummy base URL string because the URL constructor only accepts absolute URLs. const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); @@ -13688,6 +13735,10 @@ export const ToolsApiAxiosParamCreator = function (configuration?: Configuration localVarQueryParameter['pageSize'] = pageSize; } + if (upstream !== undefined) { + localVarQueryParameter['upstream'] = upstream; + } + setSearchParams(localVarUrlObj, localVarQueryParameter); @@ -13744,16 +13795,17 @@ export const ToolsApiFp = function(configuration?: Configuration) { return { /** * - * @summary List GARM agent tools. + * @summary List GARM agent tools for admin users. * @param {number} [page] The page at which to list. * @param {number} [pageSize] Number of items per page. + * @param {boolean} [upstream] If true, list tools from the upstream cached release instead of the local object store. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - async garmAgentList(page?: number, pageSize?: number, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { - const localVarAxiosArgs = await localVarAxiosParamCreator.garmAgentList(page, pageSize, options); + async adminGarmAgentList(page?: number, pageSize?: number, upstream?: boolean, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.adminGarmAgentList(page, pageSize, upstream, options); const localVarOperationServerIndex = configuration?.serverIndex ?? 0; - const localVarOperationServerBasePath = operationServerMap['ToolsApi.garmAgentList']?.[localVarOperationServerIndex]?.url; + const localVarOperationServerBasePath = operationServerMap['ToolsApi.adminGarmAgentList']?.[localVarOperationServerIndex]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, localVarOperationServerBasePath || basePath); }, /** @@ -13780,14 +13832,15 @@ export const ToolsApiFactory = function (configuration?: Configuration, basePath return { /** * - * @summary List GARM agent tools. + * @summary List GARM agent tools for admin users. * @param {number} [page] The page at which to list. * @param {number} [pageSize] Number of items per page. + * @param {boolean} [upstream] If true, list tools from the upstream cached release instead of the local object store. * @param {*} [options] Override http request option. * @throws {RequiredError} */ - garmAgentList(page?: number, pageSize?: number, options?: RawAxiosRequestConfig): AxiosPromise { - return localVarFp.garmAgentList(page, pageSize, options).then((request) => request(axios, basePath)); + adminGarmAgentList(page?: number, pageSize?: number, upstream?: boolean, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.adminGarmAgentList(page, pageSize, upstream, options).then((request) => request(axios, basePath)); }, /** * Uploads a GARM agent tool for a specific OS and architecture. This will automatically replace any existing tool for the same OS/architecture combination. Uses custom headers for metadata: X-Tool-Name: Name of the tool X-Tool-Description: Description X-Tool-OS-Type: OS type (linux or windows) X-Tool-OS-Arch: Architecture (amd64 or arm64) X-Tool-Version: Version string @@ -13810,15 +13863,16 @@ export const ToolsApiFactory = function (configuration?: Configuration, basePath export class ToolsApi extends BaseAPI { /** * - * @summary List GARM agent tools. + * @summary List GARM agent tools for admin users. * @param {number} [page] The page at which to list. * @param {number} [pageSize] Number of items per page. + * @param {boolean} [upstream] If true, list tools from the upstream cached release instead of the local object store. * @param {*} [options] Override http request option. * @throws {RequiredError} * @memberof ToolsApi */ - public garmAgentList(page?: number, pageSize?: number, options?: RawAxiosRequestConfig) { - return ToolsApiFp(this.configuration).garmAgentList(page, pageSize, options).then((request) => request(this.axios, this.basePath)); + public adminGarmAgentList(page?: number, pageSize?: number, upstream?: boolean, options?: RawAxiosRequestConfig) { + return ToolsApiFp(this.configuration).adminGarmAgentList(page, pageSize, upstream, options).then((request) => request(this.axios, this.basePath)); } /** diff --git a/webapp/swagger.yaml b/webapp/swagger.yaml index 7b8de5c3..82f064b9 100644 --- a/webapp/swagger.yaml +++ b/webapp/swagger.yaml @@ -840,6 +840,14 @@ definitions: format: int64 type: integer x-go-name: Size + source: + description: |- + Source indicates where this tool is currently stored. + "local" means the tool is stored in the internal object store. + "upstream" means the tool is only available from the upstream + cached release and has not been downloaded locally. + type: string + x-go-name: Source updated_at: format: date-time type: string @@ -911,6 +919,14 @@ definitions: format: int64 type: integer x-go-name: Size + source: + description: |- + Source indicates where this tool is currently stored. + "local" means the tool is stored in the internal object store. + "upstream" means the tool is only available from the upstream + cached release and has not been downloaded locally. + type: string + x-go-name: Source updated_at: format: date-time type: string @@ -1025,6 +1041,16 @@ definitions: format: date-time type: string x-go-name: CreatedAt + generation: + description: |- + Generation is the pool generation at the time of creating this instance. + This field is to track a divergence between when the instance was created + and the settings currently set on a pool. We can then use this field to know + if the instance is out of date with the pool, allowing us to remove it if we + need to. + format: uint64 + type: integer + x-go-name: Generation github-runner-group: description: |- GithubRunnerGroup is the github runner group to which the runner belongs. @@ -1441,6 +1467,17 @@ definitions: flavor: type: string x-go-name: Flavor + generation: + description: |- + Generation holds the numeric generation of the pool. This number + will be incremented, every time certain settings of the pool, which + may influence how runners are created (flavor, specs, image) are changed. + When a runner is created, this generation will be copied to the runners as + well. That way if some settings diverge, we can target those runners + to be recreated. + format: uint64 + type: integer + x-go-name: Generation github-runner-group: description: |- GithubRunnerGroup is the github runner group in which the runners will be added. @@ -1710,6 +1747,17 @@ definitions: flavor: type: string x-go-name: Flavor + generation: + description: |- + Generation holds the numeric generation of the scaleset. This number + will be incremented, every time certain settings of the scaleset, which + may influence how runners are created (flavor, specs, image) are changed. + When a runner is created, this generation will be copied to the runners as + well. That way if some settings diverge, we can target those runners + to be recreated. + format: uint64 + type: integer + x-go-name: Generation github-runner-group: description: |- GithubRunnerGroup is the github runner group in which the runners will be added. @@ -3675,6 +3723,10 @@ paths: name: poolID required: true type: string + - description: List only instances that were created prior to a pool update that changed a setting which influences how instances are created (image, flavor, runner group, etc). + in: query + name: outdatedOnly + type: boolean responses: "200": description: Instances @@ -4188,6 +4240,10 @@ paths: name: scalesetID required: true type: string + - description: List only instances that were created prior to a scaleset update that changed a setting which influences how instances are created (image, flavor, runner group, etc). + in: query + name: outdatedOnly + type: boolean responses: "200": description: Instances @@ -4338,7 +4394,7 @@ paths: - templates /tools/garm-agent: get: - operationId: GarmAgentList + operationId: AdminGarmAgentList parameters: - description: The page at which to list. in: query @@ -4348,6 +4404,10 @@ paths: in: query name: pageSize type: integer + - description: If true, list tools from the upstream cached release instead of the local object store. + in: query + name: upstream + type: boolean responses: "200": description: GARMAgentToolsPaginatedResponse @@ -4357,7 +4417,7 @@ paths: description: APIErrorResponse schema: $ref: '#/definitions/APIErrorResponse' - summary: List GARM agent tools. + summary: List GARM agent tools for admin users. tags: - tools post: diff --git a/workers/scaleset/scaleset.go b/workers/scaleset/scaleset.go index 526aee0e..d8942aa9 100644 --- a/workers/scaleset/scaleset.go +++ b/workers/scaleset/scaleset.go @@ -176,7 +176,7 @@ func (w *Worker) Start() (err error) { return nil } - instances, err := w.store.ListScaleSetInstances(w.ctx, w.scaleSet.ID) + instances, err := w.store.ListScaleSetInstances(w.ctx, w.scaleSet.ID, false) if err != nil { return fmt.Errorf("listing scale set instances: %w", err) }