Fix findEndpointForJob

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>

apiserver/controllers/controllers.go

@@ -107,8 +107,15 @@ func (a *APIController) handleWorkflowJobEvent(ctx context.Context, w http.Respo
 
 	signature := r.Header.Get("X-Hub-Signature-256")
 	hookType := r.Header.Get("X-Github-Hook-Installation-Target-Type")
+	giteaTargetType := r.Header.Get("X-Gitea-Hook-Installation-Target-Type")
 
-	if err := a.r.DispatchWorkflowJob(hookType, signature, body); err != nil {
+	forgeType := runnerParams.GithubEndpointType
+	if giteaTargetType != "" {
+		forgeType = runnerParams.GiteaEndpointType
+		hookType = giteaTargetType
+	}
+
+	if err := a.r.DispatchWorkflowJob(hookType, signature, forgeType, body); err != nil {
 		switch {
 		case errors.Is(err, gErrors.ErrNotFound):
 			metrics.WebhooksReceived.WithLabelValues(

cache/cache_test.go

@@ -55,8 +55,8 @@ func (c *CacheTestSuite) TestSetCacheWorks() {
 	c.Require().Len(githubToolsCache.entities, 0)
 	SetGithubToolsCache(c.entity, tools)
 	c.Require().Len(githubToolsCache.entities, 1)
-	cachedTools, ok := GetGithubToolsCache(c.entity.ID)
+	cachedTools, err := GetGithubToolsCache(c.entity.ID)
-	c.Require().True(ok)
+	c.Require().NoError(err)
 	c.Require().Len(cachedTools, 1)
 	c.Require().Equal(tools[0].GetDownloadURL(), cachedTools[0].GetDownloadURL())
 }
@@ -76,16 +76,16 @@ func (c *CacheTestSuite) TestTimedOutToolsCache() {
 	entity.updatedAt = entity.updatedAt.Add(-2 * time.Hour)
 	githubToolsCache.entities[c.entity.ID] = entity
 
-	cachedTools, ok := GetGithubToolsCache(c.entity.ID)
+	cachedTools, err := GetGithubToolsCache(c.entity.ID)
-	c.Require().False(ok)
+	c.Require().NoError(err)
 	c.Require().Nil(cachedTools)
 }
 
 func (c *CacheTestSuite) TestGetInexistentCache() {
 	c.Require().NotNil(githubToolsCache)
 	c.Require().Len(githubToolsCache.entities, 0)
-	cachedTools, ok := GetGithubToolsCache(c.entity.ID)
+	cachedTools, err := GetGithubToolsCache(c.entity.ID)
-	c.Require().False(ok)
+	c.Require().NoError(err)
 	c.Require().Nil(cachedTools)
 }
 

cache/tools_cache.go

@@ -1,6 +1,7 @@
 package cache
 
 import (
+	"fmt"
 	"sync"
 	"time"
 
@@ -20,17 +21,25 @@ func init() {
 type GithubEntityTools struct {
 	updatedAt time.Time
 	expiresAt time.Time
+	err       error
 	entity    params.ForgeEntity
 	tools     []commonParams.RunnerApplicationDownload
 }
 
+func (g GithubEntityTools) Error() string {
+	if g.err != nil {
+		return g.err.Error()
+	}
+	return ""
+}
+
 type GithubToolsCache struct {
 	mux sync.Mutex
 	// entity IDs are UUID4s. It is highly unlikely they will collide (🤞).
 	entities map[string]GithubEntityTools
 }
 
-func (g *GithubToolsCache) Get(entityID string) ([]commonParams.RunnerApplicationDownload, bool) {
+func (g *GithubToolsCache) Get(entityID string) ([]commonParams.RunnerApplicationDownload, error) {
 	g.mux.Lock()
 	defer g.mux.Unlock()
 
@@ -39,12 +48,12 @@ func (g *GithubToolsCache) Get(entityID string) ([]commonParams.RunnerApplicatio
 			if time.Now().UTC().After(cache.expiresAt.Add(-5 * time.Minute)) {
 				// Stale cache, remove it.
 				delete(g.entities, entityID)
-				return nil, false
+				return nil, fmt.Errorf("cache expired for entity %s", entityID)
 			}
 		}
-		return cache.tools, true
+		return cache.tools, cache.err
 	}
-	return nil, false
+	return nil, fmt.Errorf("no cache found for entity %s", entityID)
 }
 
 func (g *GithubToolsCache) Set(entity params.ForgeEntity, tools []commonParams.RunnerApplicationDownload) {
@@ -55,6 +64,7 @@ func (g *GithubToolsCache) Set(entity params.ForgeEntity, tools []commonParams.R
 		updatedAt: time.Now(),
 		entity:    entity,
 		tools:     tools,
+		err:       nil,
 	}
 
 	if entity.Credentials.ForgeType == params.GithubEndpointType {
@@ -64,10 +74,30 @@ func (g *GithubToolsCache) Set(entity params.ForgeEntity, tools []commonParams.R
 	g.entities[entity.ID] = forgeTools
 }
 
+func (g *GithubToolsCache) SetToolsError(entity params.ForgeEntity, err error) {
+	g.mux.Lock()
+	defer g.mux.Unlock()
+
+	// If the entity is not in the cache, add it with the error.
+	cache, ok := g.entities[entity.ID]
+	if !ok {
+		g.entities[entity.ID] = GithubEntityTools{
+			updatedAt: time.Now(),
+			entity:    entity,
+			err:       err,
+		}
+		return
+	}
+
+	// Update the error for the existing entity.
+	cache.err = err
+	g.entities[entity.ID] = cache
+}
+
 func SetGithubToolsCache(entity params.ForgeEntity, tools []commonParams.RunnerApplicationDownload) {
 	githubToolsCache.Set(entity, tools)
 }
 
-func GetGithubToolsCache(entityID string) ([]commonParams.RunnerApplicationDownload, bool) {
+func GetGithubToolsCache(entityID string) ([]commonParams.RunnerApplicationDownload, error) {
 	return githubToolsCache.Get(entityID)
 }

params/params.go

@@ -1099,13 +1099,18 @@ func (g ForgeEntity) GetCreatedAt() time.Time {
 }
 
 func (g ForgeEntity) ForgeURL() string {
-	switch g.EntityType {
+	switch g.Credentials.ForgeType {
-	case ForgeEntityTypeRepository:
+	case GiteaEndpointType:
-		return fmt.Sprintf("%s/%s/%s", g.Credentials.BaseURL, g.Owner, g.Name)
+		return g.Credentials.Endpoint.APIBaseURL
-	case ForgeEntityTypeOrganization:
+	default:
-		return fmt.Sprintf("%s/%s", g.Credentials.BaseURL, g.Owner)
+		switch g.EntityType {
-	case ForgeEntityTypeEnterprise:
+		case ForgeEntityTypeRepository:
-		return fmt.Sprintf("%s/enterprises/%s", g.Credentials.BaseURL, g.Owner)
+			return fmt.Sprintf("%s/%s/%s", g.Credentials.BaseURL, g.Owner, g.Name)
+		case ForgeEntityTypeOrganization:
+			return fmt.Sprintf("%s/%s", g.Credentials.BaseURL, g.Owner)
+		case ForgeEntityTypeEnterprise:
+			return fmt.Sprintf("%s/enterprises/%s", g.Credentials.BaseURL, g.Owner)
+		}
 	}
 	return ""
 }

runner/metadata.go

@@ -16,7 +16,7 @@ import (
 	"github.com/cloudbase/garm/params"
 )
 
-var systemdUnitTemplate = `[Unit]
+var githubSystemdUnitTemplate = `[Unit]
 Description=GitHub Actions Runner ({{.ServiceName}})
 After=network.target
 
@@ -32,11 +32,24 @@ TimeoutStopSec=5min
 WantedBy=multi-user.target
 `
 
-func validateInstanceState(ctx context.Context) (params.Instance, error) {
+var giteaSystemdUnitTemplate = `[Unit]
-	if !auth.InstanceHasJITConfig(ctx) {
+Description=Act Runner ({{.ServiceName}})
-		return params.Instance{}, fmt.Errorf("instance not configured for JIT: %w", runnerErrors.ErrNotFound)
+After=network.target
-	}
 
+[Service]
+ExecStart=/home/{{.RunAsUser}}/act-runner/act_runner daemon --once
+User={{.RunAsUser}}
+WorkingDirectory=/home/{{.RunAsUser}}/act-runner
+KillMode=process
+KillSignal=SIGTERM
+TimeoutStopSec=5min
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+`
+
+func validateInstanceState(ctx context.Context) (params.Instance, error) {
 	status := auth.InstanceRunnerStatus(ctx)
 	if status != params.RunnerPending && status != params.RunnerInstalling {
 		return params.Instance{}, runnerErrors.ErrUnauthorized
@@ -49,6 +62,56 @@ func validateInstanceState(ctx context.Context) (params.Instance, error) {
 	return instance, nil
 }
 
+func (r *Runner) getForgeEntityFromInstance(ctx context.Context, instance params.Instance) (params.ForgeEntity, error) {
+	var entityGetter params.EntityGetter
+	var err error
+	switch {
+	case instance.PoolID != "":
+		entityGetter, err = r.store.GetPoolByID(r.ctx, instance.PoolID)
+	case instance.ScaleSetID != 0:
+		entityGetter, err = r.store.GetScaleSetByID(r.ctx, instance.ScaleSetID)
+	default:
+		return params.ForgeEntity{}, errors.New("instance not associated with a pool or scale set")
+	}
+
+	if err != nil {
+		slog.With(slog.Any("error", err)).ErrorContext(
+			ctx, "failed to get entity getter",
+			"instance", instance.Name)
+		return params.ForgeEntity{}, errors.Wrap(err, "fetching entity getter")
+	}
+
+	poolEntity, err := entityGetter.GetEntity()
+	if err != nil {
+		slog.With(slog.Any("error", err)).ErrorContext(
+			ctx, "failed to get entity",
+			"instance", instance.Name)
+		return params.ForgeEntity{}, errors.Wrap(err, "fetching entity")
+	}
+
+	entity, err := r.store.GetForgeEntity(r.ctx, poolEntity.EntityType, poolEntity.ID)
+	if err != nil {
+		slog.With(slog.Any("error", err)).ErrorContext(
+			ctx, "failed to get entity",
+			"instance", instance.Name)
+		return params.ForgeEntity{}, errors.Wrap(err, "fetching entity")
+	}
+	return entity, nil
+}
+
+func (r *Runner) getServiceNameForEntity(entity params.ForgeEntity) (string, error) {
+	switch entity.EntityType {
+	case params.ForgeEntityTypeEnterprise:
+		return fmt.Sprintf("actions.runner.%s.%s", entity.Owner, entity.Name), nil
+	case params.ForgeEntityTypeOrganization:
+		return fmt.Sprintf("actions.runner.%s.%s", entity.Owner, entity.Name), nil
+	case params.ForgeEntityTypeRepository:
+		return fmt.Sprintf("actions.runner.%s-%s.%s", entity.Owner, entity.Name, entity.Name), nil
+	default:
+		return "", errors.New("unknown entity type")
+	}
+}
+
 func (r *Runner) GetRunnerServiceName(ctx context.Context) (string, error) {
 	instance, err := validateInstanceState(ctx)
 	if err != nil {
@@ -56,64 +119,51 @@ func (r *Runner) GetRunnerServiceName(ctx context.Context) (string, error) {
 			ctx, "failed to get instance params")
 		return "", runnerErrors.ErrUnauthorized
 	}
-	var entity params.ForgeEntity
+	entity, err := r.getForgeEntityFromInstance(ctx, instance)
-
+	if err != nil {
-	switch {
+		slog.ErrorContext(r.ctx, "failed to get entity", "error", err)
-	case instance.PoolID != "":
+		return "", errors.Wrap(err, "fetching entity")
-		pool, err := r.store.GetPoolByID(r.ctx, instance.PoolID)
-		if err != nil {
-			slog.With(slog.Any("error", err)).ErrorContext(
-				ctx, "failed to get pool",
-				"pool_id", instance.PoolID)
-			return "", errors.Wrap(err, "fetching pool")
-		}
-		entity, err = pool.GetEntity()
-		if err != nil {
-			slog.With(slog.Any("error", err)).ErrorContext(
-				ctx, "failed to get pool entity",
-				"pool_id", instance.PoolID)
-			return "", errors.Wrap(err, "fetching pool entity")
-		}
-	case instance.ScaleSetID != 0:
-		scaleSet, err := r.store.GetScaleSetByID(r.ctx, instance.ScaleSetID)
-		if err != nil {
-			slog.With(slog.Any("error", err)).ErrorContext(
-				ctx, "failed to get scale set",
-				"scale_set_id", instance.ScaleSetID)
-			return "", errors.Wrap(err, "fetching scale set")
-		}
-		entity, err = scaleSet.GetEntity()
-		if err != nil {
-			slog.With(slog.Any("error", err)).ErrorContext(
-				ctx, "failed to get scale set entity",
-				"scale_set_id", instance.ScaleSetID)
-			return "", errors.Wrap(err, "fetching scale set entity")
-		}
-	default:
-		return "", errors.New("instance not associated with a pool or scale set")
 	}
 
-	tpl := "actions.runner.%s.%s"
+	serviceName, err := r.getServiceNameForEntity(entity)
-	var serviceName string
+	if err != nil {
-	switch entity.EntityType {
+		slog.ErrorContext(r.ctx, "failed to get service name", "error", err)
-	case params.ForgeEntityTypeEnterprise:
+		return "", errors.Wrap(err, "fetching service name")
-		serviceName = fmt.Sprintf(tpl, entity.Owner, instance.Name)
-	case params.ForgeEntityTypeOrganization:
-		serviceName = fmt.Sprintf(tpl, entity.Owner, instance.Name)
-	case params.ForgeEntityTypeRepository:
-		serviceName = fmt.Sprintf(tpl, fmt.Sprintf("%s-%s", entity.Owner, entity.Name), instance.Name)
 	}
 	return serviceName, nil
 }
 
 func (r *Runner) GenerateSystemdUnitFile(ctx context.Context, runAsUser string) ([]byte, error) {
-	serviceName, err := r.GetRunnerServiceName(ctx)
+	instance, err := validateInstanceState(ctx)
 	if err != nil {
-		return nil, errors.Wrap(err, "fetching runner service name")
+		slog.With(slog.Any("error", err)).ErrorContext(
+			ctx, "failed to get instance params")
+		return nil, runnerErrors.ErrUnauthorized
+	}
+	entity, err := r.getForgeEntityFromInstance(ctx, instance)
+	if err != nil {
+		slog.ErrorContext(r.ctx, "failed to get entity", "error", err)
+		return nil, errors.Wrap(err, "fetching entity")
 	}
 
-	unitTemplate, err := template.New("").Parse(systemdUnitTemplate)
+	serviceName, err := r.getServiceNameForEntity(entity)
 	if err != nil {
+		slog.ErrorContext(r.ctx, "failed to get service name", "error", err)
+		return nil, errors.Wrap(err, "fetching service name")
+	}
+
+	var unitTemplate *template.Template
+	switch entity.Credentials.ForgeType {
+	case params.GithubEndpointType:
+		unitTemplate, err = template.New("").Parse(githubSystemdUnitTemplate)
+	case params.GiteaEndpointType:
+		unitTemplate, err = template.New("").Parse(giteaSystemdUnitTemplate)
+	default:
+		slog.ErrorContext(r.ctx, "unknown forge type", "forge_type", entity.Credentials.ForgeType)
+		return nil, errors.New("unknown forge type")
+	}
+	if err != nil {
+		slog.ErrorContext(r.ctx, "failed to parse template", "error", err)
 		return nil, errors.Wrap(err, "parsing template")
 	}
 
@@ -131,12 +181,17 @@ func (r *Runner) GenerateSystemdUnitFile(ctx context.Context, runAsUser string)
 
 	var unitFile bytes.Buffer
 	if err := unitTemplate.Execute(&unitFile, data); err != nil {
+		slog.ErrorContext(r.ctx, "failed to execute template", "error", err)
 		return nil, errors.Wrap(err, "executing template")
 	}
 	return unitFile.Bytes(), nil
 }
 
 func (r *Runner) GetJITConfigFile(ctx context.Context, file string) ([]byte, error) {
+	if !auth.InstanceHasJITConfig(ctx) {
+		return nil, fmt.Errorf("instance not configured for JIT: %w", runnerErrors.ErrNotFound)
+	}
+
 	instance, err := validateInstanceState(ctx)
 	if err != nil {
 		slog.With(slog.Any("error", err)).ErrorContext(

runner/pool/pool.go

@@ -47,15 +47,15 @@ import (
 )
 
 var (
-	poolIDLabelprefix     = "runner-pool-id:"
+	poolIDLabelprefix     = "runner-pool-id"
-	controllerLabelPrefix = "runner-controller-id:"
+	controllerLabelPrefix = "runner-controller-id"
 	// We tag runners that have been spawned as a result of a queued job with the job ID
 	// that spawned them. There is no way to guarantee that the runner spawned in response to a particular
 	// job, will be picked up by that job. We mark them so as in the very likely event that the runner
 	// has picked up a different job, we can clear the lock on the job that spaned it.
 	// The job it picked up would already be transitioned to in_progress so it will be ignored by the
 	// consume loop.
-	jobLabelPrefix = "in_response_to_job:"
+	jobLabelPrefix = "in_response_to_job"
 )
 
 const (
@@ -296,7 +296,8 @@ func (r *basePoolManager) HandleWorkflowJob(job params.WorkflowJob) error {
 func jobIDFromLabels(labels []string) int64 {
 	for _, lbl := range labels {
 		if strings.HasPrefix(lbl, jobLabelPrefix) {
-			jobID, err := strconv.ParseInt(lbl[len(jobLabelPrefix):], 10, 64)
+			trimLength := min(len(jobLabelPrefix)+1, len(lbl))
+			jobID, err := strconv.ParseInt(lbl[trimLength:], 10, 64)
 			if err != nil {
 				return 0
 			}
@@ -361,21 +362,21 @@ func (r *basePoolManager) startLoopForFunction(f func() error, interval time.Dur
 }
 
 func (r *basePoolManager) updateTools() error {
-	// Update tools cache.
+	tools, err := cache.GetGithubToolsCache(r.entity.ID)
-	tools, err := r.FetchTools()
 	if err != nil {
 		slog.With(slog.Any("error", err)).ErrorContext(
 			r.ctx, "failed to update tools for entity", "entity", r.entity.String())
 		r.SetPoolRunningState(false, err.Error())
 		return fmt.Errorf("failed to update tools for entity %s: %w", r.entity.String(), err)
 	}
+
 	r.mux.Lock()
 	r.tools = tools
 	r.mux.Unlock()
 
 	slog.DebugContext(r.ctx, "successfully updated tools")
 	r.SetPoolRunningState(true, "")
-	return err
+	return nil
 }
 
 // cleanupOrphanedProviderRunners compares runners in github with local runners and removes
@@ -995,11 +996,11 @@ func (r *basePoolManager) paramsWorkflowJobToParamsJob(job params.WorkflowJob) (
 }
 
 func (r *basePoolManager) poolLabel(poolID string) string {
-	return fmt.Sprintf("%s%s", poolIDLabelprefix, poolID)
+	return fmt.Sprintf("%s=%s", poolIDLabelprefix, poolID)
 }
 
 func (r *basePoolManager) controllerLabel() string {
-	return fmt.Sprintf("%s%s", controllerLabelPrefix, r.controllerInfo.ControllerID.String())
+	return fmt.Sprintf("%s=%s", controllerLabelPrefix, r.controllerInfo.ControllerID.String())
 }
 
 func (r *basePoolManager) updateArgsFromProviderInstance(providerInstance commonParams.ProviderInstance) params.UpdateInstanceParams {
@@ -1613,6 +1614,16 @@ func (r *basePoolManager) Start() error {
 	initialToolUpdate := make(chan struct{}, 1)
 	go func() {
 		slog.Info("running initial tool update")
+		for {
+			slog.DebugContext(r.ctx, "waiting for tools to be available")
+			hasTools, stopped := r.waitForToolsOrCancel()
+			if stopped {
+				return
+			}
+			if hasTools {
+				break
+			}
+		}
 		if err := r.updateTools(); err != nil {
 			slog.With(slog.Any("error", err)).Error("failed to update tools")
 		}
@@ -1804,7 +1815,7 @@ func (r *basePoolManager) consumeQueuedJobs() error {
 		}
 
 		jobLabels := []string{
-			fmt.Sprintf("%s%d", jobLabelPrefix, job.ID),
+			fmt.Sprintf("%s=%d", jobLabelPrefix, job.ID),
 		}
 		for i := 0; i < poolRR.Len(); i++ {
 			pool, err := poolRR.Next()

runner/pool/util.go

@@ -5,11 +5,13 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	"github.com/google/go-github/v71/github"
 
 	runnerErrors "github.com/cloudbase/garm-provider-common/errors"
 	commonParams "github.com/cloudbase/garm-provider-common/params"
+	"github.com/cloudbase/garm/cache"
 	dbCommon "github.com/cloudbase/garm/database/common"
 	"github.com/cloudbase/garm/database/watcher"
 	"github.com/cloudbase/garm/params"
@@ -91,7 +93,8 @@ func instanceInList(instanceName string, instances []commonParams.ProviderInstan
 func controllerIDFromLabels(labels []string) string {
 	for _, lbl := range labels {
 		if strings.HasPrefix(lbl, controllerLabelPrefix) {
-			return lbl[len(controllerLabelPrefix):]
+			trimLength := min(len(controllerLabelPrefix)+1, len(lbl))
+			return lbl[trimLength:]
 		}
 	}
 	return ""
@@ -134,3 +137,19 @@ func composeWatcherFilters(entity params.ForgeEntity) dbCommon.PayloadFilterFunc
 		watcher.WithForgeCredentialsFilter(entity.Credentials),
 	)
 }
+
+func (r *basePoolManager) waitForToolsOrCancel() (hasTools, stopped bool) {
+	ticker := time.NewTicker(1 * time.Second)
+	defer ticker.Stop()
+	select {
+	case <-ticker.C:
+		if _, err := cache.GetGithubToolsCache(r.entity.ID); err != nil {
+			return false, false
+		}
+		return true, false
+	case <-r.quit:
+		return false, true
+	case <-r.ctx.Done():
+		return false, true
+	}
+}

runner/runner.go

@@ -602,7 +602,7 @@ func (r *Runner) validateHookBody(signature, secret string, body []byte) error {
 	return nil
 }
 
-func (r *Runner) findEndpointForJob(job params.WorkflowJob) (params.ForgeEndpoint, error) {
+func (r *Runner) findEndpointForJob(job params.WorkflowJob, forgeType params.EndpointType) (params.ForgeEndpoint, error) {
 	uri, err := url.ParseRequestURI(job.WorkflowJob.HTMLURL)
 	if err != nil {
 		return params.ForgeEndpoint{}, errors.Wrap(err, "parsing job URL")
@@ -614,12 +614,23 @@ func (r *Runner) findEndpointForJob(job params.WorkflowJob) (params.ForgeEndpoin
 	// a GHES involved, those users will have just one extra endpoint or 2 (if they also have a
 	// test env). But there should be a relatively small number, regardless. So we don't really care
 	// that much about the performance of this function.
-	endpoints, err := r.store.ListGithubEndpoints(r.ctx)
+	var endpoints []params.ForgeEndpoint
+	switch forgeType {
+	case params.GithubEndpointType:
+		endpoints, err = r.store.ListGithubEndpoints(r.ctx)
+	case params.GiteaEndpointType:
+		endpoints, err = r.store.ListGiteaEndpoints(r.ctx)
+	default:
+		return params.ForgeEndpoint{}, runnerErrors.NewBadRequestError("unknown forge type %s", forgeType)
+	}
+
 	if err != nil {
 		return params.ForgeEndpoint{}, errors.Wrap(err, "fetching github endpoints")
 	}
 	for _, ep := range endpoints {
-		if ep.BaseURL == baseURI {
+		slog.DebugContext(r.ctx, "checking endpoint", "base_uri", baseURI, "endpoint", ep.BaseURL)
+		epBaseURI := strings.TrimSuffix(ep.BaseURL, "/")
+		if epBaseURI == baseURI {
 			return ep, nil
 		}
 	}
@@ -627,18 +638,21 @@ func (r *Runner) findEndpointForJob(job params.WorkflowJob) (params.ForgeEndpoin
 	return params.ForgeEndpoint{}, runnerErrors.NewNotFoundError("no endpoint found for job")
 }
 
-func (r *Runner) DispatchWorkflowJob(hookTargetType, signature string, jobData []byte) error {
+func (r *Runner) DispatchWorkflowJob(hookTargetType, signature string, forgeType params.EndpointType, jobData []byte) error {
 	if len(jobData) == 0 {
+		slog.ErrorContext(r.ctx, "missing job data")
 		return runnerErrors.NewBadRequestError("missing job data")
 	}
 
 	var job params.WorkflowJob
 	if err := json.Unmarshal(jobData, &job); err != nil {
+		slog.ErrorContext(r.ctx, "failed to unmarshal job data", "error", err)
 		return errors.Wrapf(runnerErrors.ErrBadRequest, "invalid job data: %s", err)
 	}
 
-	endpoint, err := r.findEndpointForJob(job)
+	endpoint, err := r.findEndpointForJob(job, forgeType)
 	if err != nil {
+		slog.ErrorContext(r.ctx, "failed to find endpoint for job", "error", err)
 		return errors.Wrap(err, "finding endpoint for job")
 	}
 
@@ -867,15 +881,17 @@ func (r *Runner) DeleteRunner(ctx context.Context, instanceName string, forceDel
 		}
 
 		if err != nil {
-			if errors.Is(err, runnerErrors.ErrUnauthorized) && instance.PoolID != "" {
+			if !errors.Is(err, runnerErrors.ErrNotFound) {
-				poolMgr, err := r.getPoolManagerFromInstance(ctx, instance)
+				if errors.Is(err, runnerErrors.ErrUnauthorized) && instance.PoolID != "" {
-				if err != nil {
+					poolMgr, err := r.getPoolManagerFromInstance(ctx, instance)
-					return errors.Wrap(err, "fetching pool manager for instance")
+					if err != nil {
+						return errors.Wrap(err, "fetching pool manager for instance")
+					}
+					poolMgr.SetPoolRunningState(false, fmt.Sprintf("failed to remove runner: %q", err))
+				}
+				if !bypassGithubUnauthorized {
+					return errors.Wrap(err, "removing runner from github")
 				}
-				poolMgr.SetPoolRunningState(false, fmt.Sprintf("failed to remove runner: %q", err))
-			}
-			if !bypassGithubUnauthorized {
-				return errors.Wrap(err, "removing runner from github")
 			}
 		}
 	}

util/github/client.go

@@ -229,6 +229,7 @@ func (g *githubClient) ListEntityRunnerApplicationDownloads(ctx context.Context)
 }
 
 func parseError(response *github.Response, err error) error {
+	slog.Debug("parsing error", "status_code", response.StatusCode, "response", response, "error", err)
 	switch response.StatusCode {
 	case http.StatusNotFound:
 		return runnerErrors.ErrNotFound
@@ -251,6 +252,10 @@ func parseError(response *github.Response, err error) error {
 				case http.StatusUnprocessableEntity:
 					return runnerErrors.ErrBadRequest
 				default:
+					// ugly hack. Gitea returns 500 if we try to remove a runner that does not exist.
+					if strings.Contains(err.Error(), "does not exist") {
+						return runnerErrors.ErrNotFound
+					}
 					return err
 				}
 			}

workers/cache/gitea_tools.go

@@ -20,6 +20,18 @@ const (
 	GiteaRunnerMinimumVersion = "v0.2.12"
 )
 
+var (
+	githubArchMapping map[string]string = map[string]string{
+		"x86_64":  "x64",
+		"amd64":   "x64",
+		"armv7l":  "arm",
+		"aarch64": "arm64",
+		"x64":     "x64",
+		"arm":     "arm",
+		"arm64":   "arm64",
+	}
+)
+
 var nightlyActRunner = GiteaEntityTool{
 	TagName:    "nightly",
 	Name:       "nightly",
@@ -50,36 +62,39 @@ type GiteaToolsAssets struct {
 	DownloadURL   string    `json:"browser_download_url"`
 }
 
-func (g GiteaToolsAssets) GetOS() *string {
+func (g GiteaToolsAssets) GetOS() (*string, error) {
 	if g.Name == "" {
-		return nil
+		return nil, fmt.Errorf("gitea tools name is empty")
 	}
 
 	parts := strings.SplitN(g.Name, "-", 4)
 	if len(parts) != 4 {
-		return nil
+		return nil, fmt.Errorf("could not parse asset name")
 	}
 
 	os := parts[2]
-	return &os
+	return &os, nil
 }
 
-func (g GiteaToolsAssets) GetArch() *string {
+func (g GiteaToolsAssets) GetArch() (*string, error) {
 	if g.Name == "" {
-		return nil
+		return nil, fmt.Errorf("gitea tools name is empty")
 	}
 
 	parts := strings.SplitN(g.Name, "-", 4)
 	if len(parts) != 4 {
-		return nil
+		return nil, fmt.Errorf("could not parse asset name")
 	}
 
 	archParts := strings.SplitN(parts[3], ".", 2)
 	if len(archParts) == 0 {
-		return nil
+		return nil, fmt.Errorf("unexpected asset name format")
 	}
-	arch := archParts[0]
+	arch := githubArchMapping[archParts[0]]
-	return &arch
+	if arch == "" {
+		return nil, fmt.Errorf("could not find arch for %s", archParts[0])
+	}
+	return &arch, nil
 }
 
 type GiteaEntityTool struct {
@@ -140,9 +155,17 @@ func getTools() ([]commonParams.RunnerApplicationDownload, error) {
 	ret := []commonParams.RunnerApplicationDownload{}
 
 	for _, asset := range latest.Assets {
+		arch, err := asset.GetArch()
+		if err != nil {
+			return nil, fmt.Errorf("getting arch: %w", err)
+		}
+		os, err := asset.GetOS()
+		if err != nil {
+			return nil, fmt.Errorf("getting os: %w", err)
+		}
 		ret = append(ret, commonParams.RunnerApplicationDownload{
-			OS:           asset.GetOS(),
+			OS:           os,
-			Architecture: asset.GetArch(),
+			Architecture: arch,
 			DownloadURL:  &asset.DownloadURL,
 			Filename:     &asset.Name,
 		})

workers/cache/tool_cache.go

@@ -49,7 +49,7 @@ func (t *toolsUpdater) Start() error {
 	t.running = true
 	t.quit = make(chan struct{})
 
-	slog.DebugContext(t.ctx, "starting tools updater", "entity", t.entity.String(), "forge_type", t.entity.Credentials)
+	slog.DebugContext(t.ctx, "starting tools updater", "entity", t.entity.String(), "forge_type", t.entity.Credentials.ForgeType)
 
 	switch t.entity.Credentials.ForgeType {
 	case params.GithubEndpointType:

workers/provider/instance_manager.go

@@ -148,9 +148,9 @@ func (i *instanceManager) handleCreateInstanceInProvider(instance params.Instanc
 	if err != nil {
 		return fmt.Errorf("creating instance token: %w", err)
 	}
-	tools, ok := cache.GetGithubToolsCache(entity.ID)
+	tools, err := cache.GetGithubToolsCache(entity.ID)
-	if !ok {
+	if err != nil {
-		return fmt.Errorf("tools not found in cache for entity %s", entity.String())
+		return fmt.Errorf("tools not found in cache for entity %s: %w", entity.String(), err)
 	}
 
 	bootstrapArgs := commonParams.BootstrapInstance{

workers/scaleset/scaleset.go

@@ -776,8 +776,11 @@ func (w *Worker) waitForToolsOrCancel() (hasTools, stopped bool) {
 		if err != nil {
 			slog.ErrorContext(w.ctx, "error getting entity", "error", err)
 		}
-		_, ok := cache.GetGithubToolsCache(entity.ID)
+		if _, err := cache.GetGithubToolsCache(entity.ID); err != nil {
-		return ok, false
+			slog.DebugContext(w.ctx, "tools not found in cache; waiting for tools")
+			return false, false
+		}
+		return true, false
 	case <-w.quit:
 		return false, true
 	case <-w.ctx.Done():