DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Add grace period to scale-down

Browse source 
Add a grace period for idle runners of 5 minutes. A new idle runner will
not be taken into consideration for scale-down unless it's older than 5
minutes. This should prevent situations where the scaleDown() routine
that runs every minute will evaluate candidates for reaping and
erroneously count the new one as well. The in_progress hooks that
transitiones an idle runner to "active" may arive a long while after the
"queued" hook has spun up a runner.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2023-02-06 15:29:01 +02:00
parent fc1012d4a1
commit 43d2fd8c2d
No known key found for this signature in database
GPG key ID: 7D073DCC2C074CB5
1 changed files with 9 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

11
runner/pool/pool.go
View file

@ -394,7 +394,9 @@ func (r *basePoolManager) acquireNewInstance(job params.WorkflowJob) error {
}
}
if int64(idleWorkers) >= int64(pool.MinIdleRunners) {
// Skip creating a new runner if we have at least one idle runner and the minimum is already satisfied.
// This should work even for pools that define a MinIdleRunner of 0.
if int64(idleWorkers) > 0 && int64(idleWorkers) >= int64(pool.MinIdleRunners) {
log.Printf("we have enough min_idle_runners (%d) for pool %s, skipping...", pool.MinIdleRunners, pool.ID)
return nil
}
@ -786,8 +788,13 @@ func (r *basePoolManager) scaleDownOnePool(pool params.Pool) {
idleWorkers := []params.Instance{}
for _, inst := range existingInstances {
// Idle runners that have been spawned and are still idle after 5 minutes, are take into
// consideration for scale-down. The 5 minute grace period prevents a situation where a
// "queued" workflow triggers the creation of a new idle runner, and this routine reaps
// an idle runner before they have a chance to pick up a job.
if providerCommon.RunnerStatus(inst.RunnerStatus) == providerCommon.RunnerIdle &&
providerCommon.InstanceStatus(inst.Status) == providerCommon.InstanceRunning {
providerCommon.InstanceStatus(inst.Status) == providerCommon.InstanceRunning &&
time.Since(inst.UpdatedAt).Minutes() > 5 {
idleWorkers = append(idleWorkers, inst)
}
}