From d5f8cf079ef018f79c4936b0459b62611e7adc7c Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Sun, 20 Aug 2023 14:27:58 +0000 Subject: [PATCH] Ignore instances that are still being created from reaping When using JIT runners, we register the runner on GitHub before we get a chance to spin up the instance in the provider. In such cases, we end up with a runner in "offline" state while we're creating the actual resource that will embody the runner. This change will give runners a chance to come online before garm tries to clean them up. Signed-off-by: Gabriel Adrian Samfira --- runner/pool/pool.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/runner/pool/pool.go b/runner/pool/pool.go index a74c44c4..8e1e661a 100644 --- a/runner/pool/pool.go +++ b/runner/pool/pool.go @@ -527,6 +527,16 @@ func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner) // already marked for deletion or is in the process of being deleted. // Let consolidate take care of it. continue + case commonParams.InstancePendingCreate, commonParams.InstanceCreating: + // instance is still being created. We give it a chance to finish. + r.log("instance %s is still being created, give it a chance to finish", dbInstance.Name) + continue + case commonParams.InstanceRunning: + if time.Since(dbInstance.UpdatedAt).Minutes() < 5 { + // instance was updated recently. We give it a chance to register itself in github. + r.log("instance %s was updated recently, skipping check", dbInstance.Name) + continue + } } pool, err := r.helper.GetPoolByID(dbInstance.PoolID)