Add force delete runner
This branch adds the ability to forcefully remove a runner from GARM.
When the operator wishes to manually remove a runner, the workflow is as
follows:
* Check that the runner exists in GitHub. If it does, attempt to
remove it. An error here indicates that the runner may be processing
a job. In this case, we don't continue and the operator gets immediate
feedback from the API.
* Mark the runner in the database as pending_delete
* Allow the consolidate loop to reap it from the provider and remove it
from the database.
Removing the instance from the provider is async. If the provider errs out,
GARM will keep trying to remove it in perpetuity until the provider succedes.
In situations where the provider is misconfigured, this will never happen, leaving
the instance in a permanent state of pending_delete.
A provider may fail for various reasons. Either credentials have expired, the
API endpoint has changed, the provider is misconfigured or the operator may just
have removed it from the config before cleaning up the runners. While some cases
are recoverable, some are not. We cannot have a situation in which we cannot clean
resources in garm because of a misconfiguration.
This change adds the pending_force_delete instance status. Instances marked with
this status, will be removed from GARM even if the provider reports an error.
The GARM cli has been modified to give new meaning to the --force-remove-runner
option. This option in the CLI is no longer mandatory. Instead, setting it will mark
the runner with the new pending_force_delete status. Omitting it will mark the runner
with the old status of pending_delete.
Fixes: #160
Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
2023-10-12 06:15:36 +00:00
// Code generated by mockery v2.28.1. DO NOT EDIT.
2022-07-10 14:52:15 +00:00
package mocks
import (
context "context"
2023-07-22 22:39:17 +00:00
garm_provider_commonparams "github.com/cloudbase/garm-provider-common/params"
2022-07-10 14:52:15 +00:00
mock "github.com/stretchr/testify/mock"
2023-07-22 22:39:17 +00:00
params "github.com/cloudbase/garm/params"
2022-07-10 14:52:15 +00:00
)
// Provider is an autogenerated mock type for the Provider type
type Provider struct {
mock . Mock
}
// AsParams provides a mock function with given fields:
func ( _m * Provider ) AsParams ( ) params . Provider {
ret := _m . Called ( )
var r0 params . Provider
if rf , ok := ret . Get ( 0 ) . ( func ( ) params . Provider ) ; ok {
r0 = rf ( )
} else {
r0 = ret . Get ( 0 ) . ( params . Provider )
}
return r0
}
// CreateInstance provides a mock function with given fields: ctx, bootstrapParams
2023-07-23 12:47:56 +00:00
func ( _m * Provider ) CreateInstance ( ctx context . Context , bootstrapParams garm_provider_commonparams . BootstrapInstance ) ( garm_provider_commonparams . ProviderInstance , error ) {
2022-07-10 14:52:15 +00:00
ret := _m . Called ( ctx , bootstrapParams )
2023-07-23 12:47:56 +00:00
var r0 garm_provider_commonparams . ProviderInstance
2023-03-12 16:01:49 +02:00
var r1 error
2023-07-23 12:47:56 +00:00
if rf , ok := ret . Get ( 0 ) . ( func ( context . Context , garm_provider_commonparams . BootstrapInstance ) ( garm_provider_commonparams . ProviderInstance , error ) ) ; ok {
2023-03-12 16:01:49 +02:00
return rf ( ctx , bootstrapParams )
}
2023-07-23 12:47:56 +00:00
if rf , ok := ret . Get ( 0 ) . ( func ( context . Context , garm_provider_commonparams . BootstrapInstance ) garm_provider_commonparams . ProviderInstance ) ; ok {
2022-07-10 14:52:15 +00:00
r0 = rf ( ctx , bootstrapParams )
} else {
2023-07-23 12:47:56 +00:00
r0 = ret . Get ( 0 ) . ( garm_provider_commonparams . ProviderInstance )
2022-07-10 14:52:15 +00:00
}
2023-07-22 22:39:17 +00:00
if rf , ok := ret . Get ( 1 ) . ( func ( context . Context , garm_provider_commonparams . BootstrapInstance ) error ) ; ok {
2022-07-10 14:52:15 +00:00
r1 = rf ( ctx , bootstrapParams )
} else {
r1 = ret . Error ( 1 )
}
return r0 , r1
}
// DeleteInstance provides a mock function with given fields: ctx, instance
func ( _m * Provider ) DeleteInstance ( ctx context . Context , instance string ) error {
ret := _m . Called ( ctx , instance )
var r0 error
if rf , ok := ret . Get ( 0 ) . ( func ( context . Context , string ) error ) ; ok {
r0 = rf ( ctx , instance )
} else {
r0 = ret . Error ( 0 )
}
return r0
}
2023-12-11 12:37:33 +00:00
// DisableJITConfig provides a mock function with given fields:
func ( _m * Provider ) DisableJITConfig ( ) bool {
ret := _m . Called ( )
var r0 bool
if rf , ok := ret . Get ( 0 ) . ( func ( ) bool ) ; ok {
r0 = rf ( )
} else {
r0 = ret . Get ( 0 ) . ( bool )
}
return r0
}
2022-07-10 14:52:15 +00:00
// GetInstance provides a mock function with given fields: ctx, instance
2023-07-23 12:47:56 +00:00
func ( _m * Provider ) GetInstance ( ctx context . Context , instance string ) ( garm_provider_commonparams . ProviderInstance , error ) {
2022-07-10 14:52:15 +00:00
ret := _m . Called ( ctx , instance )
2023-07-23 12:47:56 +00:00
var r0 garm_provider_commonparams . ProviderInstance
2023-03-12 16:01:49 +02:00
var r1 error
2023-07-23 12:47:56 +00:00
if rf , ok := ret . Get ( 0 ) . ( func ( context . Context , string ) ( garm_provider_commonparams . ProviderInstance , error ) ) ; ok {
2023-03-12 16:01:49 +02:00
return rf ( ctx , instance )
}
2023-07-23 12:47:56 +00:00
if rf , ok := ret . Get ( 0 ) . ( func ( context . Context , string ) garm_provider_commonparams . ProviderInstance ) ; ok {
2022-07-10 14:52:15 +00:00
r0 = rf ( ctx , instance )
} else {
2023-07-23 12:47:56 +00:00
r0 = ret . Get ( 0 ) . ( garm_provider_commonparams . ProviderInstance )
2022-07-10 14:52:15 +00:00
}
if rf , ok := ret . Get ( 1 ) . ( func ( context . Context , string ) error ) ; ok {
r1 = rf ( ctx , instance )
} else {
r1 = ret . Error ( 1 )
}
return r0 , r1
}
// ListInstances provides a mock function with given fields: ctx, poolID
2023-07-23 12:47:56 +00:00
func ( _m * Provider ) ListInstances ( ctx context . Context , poolID string ) ( [ ] garm_provider_commonparams . ProviderInstance , error ) {
2022-07-10 14:52:15 +00:00
ret := _m . Called ( ctx , poolID )
2023-07-23 12:47:56 +00:00
var r0 [ ] garm_provider_commonparams . ProviderInstance
2023-03-12 16:01:49 +02:00
var r1 error
2023-07-23 12:47:56 +00:00
if rf , ok := ret . Get ( 0 ) . ( func ( context . Context , string ) ( [ ] garm_provider_commonparams . ProviderInstance , error ) ) ; ok {
2023-03-12 16:01:49 +02:00
return rf ( ctx , poolID )
}
2023-07-23 12:47:56 +00:00
if rf , ok := ret . Get ( 0 ) . ( func ( context . Context , string ) [ ] garm_provider_commonparams . ProviderInstance ) ; ok {
2022-07-10 14:52:15 +00:00
r0 = rf ( ctx , poolID )
} else {
if ret . Get ( 0 ) != nil {
2023-07-23 12:47:56 +00:00
r0 = ret . Get ( 0 ) . ( [ ] garm_provider_commonparams . ProviderInstance )
2022-07-10 14:52:15 +00:00
}
}
if rf , ok := ret . Get ( 1 ) . ( func ( context . Context , string ) error ) ; ok {
r1 = rf ( ctx , poolID )
} else {
r1 = ret . Error ( 1 )
}
return r0 , r1
}
// RemoveAllInstances provides a mock function with given fields: ctx
func ( _m * Provider ) RemoveAllInstances ( ctx context . Context ) error {
ret := _m . Called ( ctx )
var r0 error
if rf , ok := ret . Get ( 0 ) . ( func ( context . Context ) error ) ; ok {
r0 = rf ( ctx )
} else {
r0 = ret . Error ( 0 )
}
return r0
}
// Start provides a mock function with given fields: ctx, instance
func ( _m * Provider ) Start ( ctx context . Context , instance string ) error {
ret := _m . Called ( ctx , instance )
var r0 error
if rf , ok := ret . Get ( 0 ) . ( func ( context . Context , string ) error ) ; ok {
r0 = rf ( ctx , instance )
} else {
r0 = ret . Error ( 0 )
}
return r0
}
// Stop provides a mock function with given fields: ctx, instance, force
2024-02-22 08:38:44 +01:00
func ( _m * Provider ) Stop ( ctx context . Context , instance string ) error {
ret := _m . Called ( ctx , instance )
2022-07-10 14:52:15 +00:00
var r0 error
2024-02-22 08:38:44 +01:00
if rf , ok := ret . Get ( 0 ) . ( func ( context . Context , string ) error ) ; ok {
r0 = rf ( ctx , instance )
2022-07-10 14:52:15 +00:00
} else {
r0 = ret . Error ( 0 )
}
return r0
}
Add force delete runner
This branch adds the ability to forcefully remove a runner from GARM.
When the operator wishes to manually remove a runner, the workflow is as
follows:
* Check that the runner exists in GitHub. If it does, attempt to
remove it. An error here indicates that the runner may be processing
a job. In this case, we don't continue and the operator gets immediate
feedback from the API.
* Mark the runner in the database as pending_delete
* Allow the consolidate loop to reap it from the provider and remove it
from the database.
Removing the instance from the provider is async. If the provider errs out,
GARM will keep trying to remove it in perpetuity until the provider succedes.
In situations where the provider is misconfigured, this will never happen, leaving
the instance in a permanent state of pending_delete.
A provider may fail for various reasons. Either credentials have expired, the
API endpoint has changed, the provider is misconfigured or the operator may just
have removed it from the config before cleaning up the runners. While some cases
are recoverable, some are not. We cannot have a situation in which we cannot clean
resources in garm because of a misconfiguration.
This change adds the pending_force_delete instance status. Instances marked with
this status, will be removed from GARM even if the provider reports an error.
The GARM cli has been modified to give new meaning to the --force-remove-runner
option. This option in the CLI is no longer mandatory. Instead, setting it will mark
the runner with the new pending_force_delete status. Omitting it will mark the runner
with the old status of pending_delete.
Fixes: #160
Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
2023-10-12 06:15:36 +00:00
type mockConstructorTestingTNewProvider interface {
2022-07-10 14:52:15 +00:00
mock . TestingT
Cleanup ( func ( ) )
Add force delete runner
This branch adds the ability to forcefully remove a runner from GARM.
When the operator wishes to manually remove a runner, the workflow is as
follows:
* Check that the runner exists in GitHub. If it does, attempt to
remove it. An error here indicates that the runner may be processing
a job. In this case, we don't continue and the operator gets immediate
feedback from the API.
* Mark the runner in the database as pending_delete
* Allow the consolidate loop to reap it from the provider and remove it
from the database.
Removing the instance from the provider is async. If the provider errs out,
GARM will keep trying to remove it in perpetuity until the provider succedes.
In situations where the provider is misconfigured, this will never happen, leaving
the instance in a permanent state of pending_delete.
A provider may fail for various reasons. Either credentials have expired, the
API endpoint has changed, the provider is misconfigured or the operator may just
have removed it from the config before cleaning up the runners. While some cases
are recoverable, some are not. We cannot have a situation in which we cannot clean
resources in garm because of a misconfiguration.
This change adds the pending_force_delete instance status. Instances marked with
this status, will be removed from GARM even if the provider reports an error.
The GARM cli has been modified to give new meaning to the --force-remove-runner
option. This option in the CLI is no longer mandatory. Instead, setting it will mark
the runner with the new pending_force_delete status. Omitting it will mark the runner
with the old status of pending_delete.
Fixes: #160
Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
2023-10-12 06:15:36 +00:00
}
// NewProvider creates a new instance of Provider. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
func NewProvider ( t mockConstructorTestingTNewProvider ) * Provider {
2022-07-10 14:52:15 +00:00
mock := & Provider { }
mock . Mock . Test ( t )
t . Cleanup ( func ( ) { mock . AssertExpectations ( t ) } )
return mock
}