Add agent mode

This change adds a new "agent mode" to GARM. The agent enables GARM to
set up a persistent websocket connection between the garm server and the
runners it spawns. The goal is to be able to easier keep track of state,
even without subsequent webhooks from the forge.

The Agent will report via websockets when the runner is actually online,
when it started a job and when it finished a job.

Additionally, the agent allows us to enable optional remote shell between
the user and any runner that is spun up using agent mode. The remote shell
is multiplexed over the same persistent websocket connection the agent
sets up with the server (the agent never listens on a port).

Enablement has also been done in the web UI for this functionality.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>

apiserver/controllers/agent.go

@@ -0,0 +1,116 @@
+package controllers
+
+import (
+	"encoding/json"
+	"fmt"
+	"log/slog"
+	"net/http"
+
+	"github.com/google/uuid"
+	"github.com/gorilla/mux"
+
+	gErrors "github.com/cloudbase/garm-provider-common/errors"
+	"github.com/cloudbase/garm/apiserver/params"
+	"github.com/cloudbase/garm/auth"
+	"github.com/cloudbase/garm/workers/websocket/agent"
+)
+
+func (a *APIController) AgentHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	instance, err := auth.InstanceParams(ctx)
+	if err != nil {
+		w.WriteHeader(http.StatusForbidden)
+		slog.ErrorContext(ctx, "failed to authenticate instance")
+		return
+	}
+
+	conn, err := a.upgrader.Upgrade(w, r, nil)
+	if err != nil {
+		slog.With(slog.Any("error", err)).ErrorContext(ctx, "error upgrading to websockets")
+		return
+	}
+
+	slog.DebugContext(ctx, "new agent connected", "agent_name", instance.Name)
+	agent, err := agent.NewAgent(ctx, conn, instance, a.r)
+	if err != nil {
+		slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to create agent")
+		return
+	}
+	defer func() {
+		slog.DebugContext(ctx, "stopping agent", "agent_name", instance.Name)
+		agent.Stop()
+	}()
+
+	if err := agent.Start(); err != nil {
+		slog.ErrorContext(ctx, "failed to start agent loop", "error", err, "agent_name", instance.Name)
+		handleError(ctx, w, err)
+		return
+	}
+
+	if err := a.agentHub.RegisterAgent(agent); err != nil {
+		handleError(ctx, w, err)
+		return
+	}
+	defer a.agentHub.UnregisterAgent(instance.Name)
+
+	select {
+	case <-agent.Done():
+	case <-ctx.Done():
+	}
+	slog.InfoContext(ctx, "connection closed", "agent_name", instance.Name)
+}
+
+func (a *APIController) AgentShellHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	if !auth.IsAdmin(ctx) {
+		handleError(ctx, w, gErrors.ErrUnauthorized)
+		return
+	}
+
+	vars := mux.Vars(r)
+	agentName, ok := vars["agentName"]
+	if !ok {
+		w.WriteHeader(http.StatusBadRequest)
+		if err := json.NewEncoder(w).Encode(params.APIErrorResponse{
+			Error:   "Bad Request",
+			Details: "No agent name specified",
+		}); err != nil {
+			slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
+		}
+		return
+	}
+
+	sessionID, err := uuid.NewRandom()
+	if err != nil {
+		handleError(ctx, w, fmt.Errorf("failed to generate UUID: %w", err))
+		return
+	}
+
+	agent, err := a.agentHub.GetAgent(agentName)
+	if err != nil {
+		slog.InfoContext(ctx, "session for agent not found", "agent_name", agentName)
+		handleError(ctx, w, err)
+		return
+	}
+
+	conn, err := a.upgrader.Upgrade(w, r, nil)
+	if err != nil {
+		slog.With(slog.Any("error", err)).ErrorContext(ctx, "error upgrading to websockets")
+		return
+	}
+
+	sess, err := agent.CreateShellSession(ctx, sessionID, conn)
+	if err != nil {
+		slog.ErrorContext(ctx, "failed to create client session", "error", err)
+		return
+	}
+	slog.InfoContext(ctx, "shell session created", "session_id", sessionID, "agent_name", agentName)
+	defer agent.RemoveClientSession(sessionID, false)
+
+	select {
+	case <-sess.Done():
+	case <-agent.Done():
+	case <-ctx.Done():
+	}
+	slog.InfoContext(ctx, "connection closed", "session_id", sessionID, "agent_name", agentName)
+}

apiserver/controllers/controllers.go

@@ -38,10 +38,11 @@ import (
 	"github.com/cloudbase/garm/runner" //nolint:typecheck
 	garmUtil "github.com/cloudbase/garm/util"
 	wsWriter "github.com/cloudbase/garm/websocket"
+	"github.com/cloudbase/garm/workers/websocket/agent"
 	"github.com/cloudbase/garm/workers/websocket/events"
 )
 
-func NewAPIController(r *runner.Runner, authenticator *auth.Authenticator, hub *wsWriter.Hub, apiCfg config.APIServer) (*APIController, error) {
+func NewAPIController(r *runner.Runner, authenticator *auth.Authenticator, hub *wsWriter.Hub, agentHub *agent.Hub, apiCfg config.APIServer) (*APIController, error) {
 	controllerInfo, err := r.GetControllerInfo(auth.GetAdminContext(context.Background()))
 	if err != nil {
 		return nil, fmt.Errorf("failed to get controller info: %w", err)
@@ -74,9 +75,10 @@ func NewAPIController(r *runner.Runner, authenticator *auth.Authenticator, hub *
 		}
 	}
 	return &APIController{
-		r:    r,
-		auth: authenticator,
-		hub:  hub,
+		r:        r,
+		auth:     authenticator,
+		hub:      hub,
+		agentHub: agentHub,
 		upgrader: websocket.Upgrader{
 			ReadBufferSize:  1024,
 			WriteBufferSize: 16384,
@@ -90,6 +92,7 @@ type APIController struct {
 	r            *runner.Runner
 	auth         *auth.Authenticator
 	hub          *wsWriter.Hub
+	agentHub     *agent.Hub
 	upgrader     websocket.Upgrader
 	controllerID string
 }

apiserver/controllers/metadata.go

@@ -17,12 +17,14 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/http"
 	"strconv"
 
 	"github.com/gorilla/mux"
 
+	gErrors "github.com/cloudbase/garm-provider-common/errors"
 	"github.com/cloudbase/garm/apiserver/params"
 )
 
@@ -41,6 +43,25 @@ func (a *APIController) InstanceMetadataHandler(w http.ResponseWriter, r *http.R
 	}
 }
 
+// swagger:route GET /tools/garm-agent tools GarmAgentList
+//
+// List GARM agent tools.
+//
+//	Parameters:
+//	  + name: page
+//	    description: The page at which to list.
+//	    type: integer
+//	    in: query
+//	    required: false
+//	  + name: pageSize
+//	    description: Number of items per page.
+//	    type: integer
+//	    in: query
+//	    required: false
+//
+//	Responses:
+//	  200: GARMAgentToolsPaginatedResponse
+//	  400: APIErrorResponse
 func (a *APIController) InstanceGARMToolsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
@@ -74,6 +95,50 @@ func (a *APIController) InstanceGARMToolsHandler(w http.ResponseWriter, r *http.
 	}
 }
 
+func (a *APIController) InstanceShowGARMToolHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	vars := mux.Vars(r)
+	objectID, err := getObjectIDFromVars(vars)
+	if err != nil {
+		slog.ErrorContext(ctx, "failed to get object ID", "error", err)
+		handleError(ctx, w, gErrors.NewBadRequestError("invalid objectID: %s", err))
+		return
+	}
+	tools, err := a.r.ShowGARMTools(ctx, objectID)
+	if err != nil {
+		slog.ErrorContext(ctx, "failed to get garm tools", "error", err)
+		handleError(ctx, w, err)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if err := json.NewEncoder(w).Encode(tools); err != nil {
+		slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response")
+	}
+}
+
+func (a *APIController) InstanceGARMToolDownloadHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	vars := mux.Vars(r)
+	objectID, err := getObjectIDFromVars(vars)
+	if err != nil {
+		slog.ErrorContext(ctx, "failed to get object ID", "error", err)
+		handleError(ctx, w, gErrors.NewBadRequestError("invalid objectID: %s", err))
+		return
+	}
+
+	reader, err := a.r.GetGARMToolsReadHandler(ctx, objectID)
+	if err != nil {
+		handleError(ctx, w, err)
+		return
+	}
+	defer reader.Close()
+	if _, err := io.Copy(w, reader); err != nil {
+		slog.ErrorContext(ctx, "failed to stream data", "error", err)
+	}
+}
+
 func (a *APIController) InstanceGithubRegistrationTokenHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 

apiserver/controllers/templates.go

@@ -193,6 +193,38 @@ func (a *APIController) CreateTemplateHandler(w http.ResponseWriter, r *http.Req
 	}
 }
 
+// swagger:route POST /templates/restore templates RestoreTemplates
+//
+// Create template with the parameters given.
+//
+//	Parameters:
+//	  + name: Body
+//	    description: Parameters used when restoring the templates.
+//	    type: RestoreTemplateRequest
+//	    in: body
+//	    required: true
+//
+//	Responses:
+//	  default: APIErrorResponse
+func (a *APIController) RestoreTemplatesHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	var templateData runnerParams.RestoreTemplateRequest
+	if err := json.NewDecoder(r.Body).Decode(&templateData); err != nil {
+		handleError(ctx, w, gErrors.ErrBadRequest)
+		return
+	}
+
+	if err := a.r.RestoreTemplate(ctx, templateData); err != nil {
+		slog.ErrorContext(ctx, "failed to restore system templates", "error", err)
+		handleError(ctx, w, err)
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK)
+}
+
 // swagger:route PUT /templates/{templateID} templates UpdateTemplate
 //
 // Update template with the parameters given.

apiserver/routers/routers.go

@@ -75,6 +75,18 @@ func WithMetricsRouter(parentRouter *mux.Router, disableAuth bool, metricsMiddle
 	return parentRouter
 }
 
+func WithAgentRouter(parentRouter *mux.Router, han *controllers.APIController, middleware auth.Middleware) *mux.Router {
+	if parentRouter == nil {
+		return nil
+	}
+
+	agentRouter := parentRouter.PathPrefix("/agent").Subrouter()
+	agentRouter.Use(middleware.Middleware)
+	agentRouter.Handle("/", http.HandlerFunc(han.AgentHandler)).Methods("GET")
+	agentRouter.Handle("", http.HandlerFunc(han.AgentHandler)).Methods("GET")
+	return parentRouter
+}
+
 func WithDebugServer(parentRouter *mux.Router) *mux.Router {
 	if parentRouter == nil {
 		return nil
@@ -139,7 +151,6 @@ func NewAPIRouter(han *controllers.APIController, authMiddleware, initMiddleware
 
 	// Handles API calls
 	apiSubRouter := router.PathPrefix("/api/v1").Subrouter()
-
 	// FirstRunHandler
 	firstRunRouter := apiSubRouter.PathPrefix("/first-run").Subrouter()
 	firstRunRouter.Handle("/", http.HandlerFunc(han.FirstRunHandler)).Methods("POST", "OPTIONS")
@@ -175,9 +186,15 @@ func NewAPIRouter(han *controllers.APIController, authMiddleware, initMiddleware
 	metadataRouter.Handle("/systemd/unit-file", http.HandlerFunc(han.SystemdUnitFileHandler)).Methods("GET", "OPTIONS")
 	metadataRouter.Handle("/system/cert-bundle/", http.HandlerFunc(han.RootCertificateBundleHandler)).Methods("GET", "OPTIONS")
 	metadataRouter.Handle("/system/cert-bundle", http.HandlerFunc(han.RootCertificateBundleHandler)).Methods("GET", "OPTIONS")
-	// Tools
-	metadataRouter.Handle("/tools/garm/", http.HandlerFunc(han.InstanceGARMToolsHandler)).Methods("GET", "OPTIONS")
-	metadataRouter.Handle("/tools/garm", http.HandlerFunc(han.InstanceGARMToolsHandler)).Methods("GET", "OPTIONS")
+	// List garm agent downloads
+	metadataRouter.Handle("/tools/garm-agent/", http.HandlerFunc(han.InstanceGARMToolsHandler)).Methods("GET", "OPTIONS")
+	metadataRouter.Handle("/tools/garm-agent", http.HandlerFunc(han.InstanceGARMToolsHandler)).Methods("GET", "OPTIONS")
+	// Show details of a particular garm agent
+	metadataRouter.Handle("/tools/garm-agent/{objectID}/", http.HandlerFunc(han.InstanceShowGARMToolHandler)).Methods("GET", "OPTIONS")
+	metadataRouter.Handle("/tools/garm-agent/{objectID}", http.HandlerFunc(han.InstanceShowGARMToolHandler)).Methods("GET", "OPTIONS")
+	// Download garm agent
+	metadataRouter.Handle("/tools/garm-agent/{objectID}/download/", http.HandlerFunc(han.InstanceGARMToolDownloadHandler)).Methods("GET", "OPTIONS")
+	metadataRouter.Handle("/tools/garm-agent/{objectID}/download", http.HandlerFunc(han.InstanceGARMToolDownloadHandler)).Methods("GET", "OPTIONS")
 	// install script
 	metadataRouter.Handle("/install-script/", http.HandlerFunc(han.RunnerInstallScriptHandler)).Methods("GET", "OPTIONS")
 	metadataRouter.Handle("/install-script", http.HandlerFunc(han.RunnerInstallScriptHandler)).Methods("GET", "OPTIONS")
@@ -249,6 +266,9 @@ func NewAPIRouter(han *controllers.APIController, authMiddleware, initMiddleware
 	///////////////////////////////////////////////////////
 	apiRouter.Handle("/tools/garm-agent/", http.HandlerFunc(han.InstanceGARMToolsHandler)).Methods("GET", "OPTIONS")
 	apiRouter.Handle("/tools/garm-agent", http.HandlerFunc(han.InstanceGARMToolsHandler)).Methods("GET", "OPTIONS")
+	// Download garm agent
+	apiRouter.Handle("/tools/garm-agent/{objectID}/download/", http.HandlerFunc(han.InstanceGARMToolDownloadHandler)).Methods("GET", "OPTIONS")
+	apiRouter.Handle("/tools/garm-agent/{objectID}/download", http.HandlerFunc(han.InstanceGARMToolDownloadHandler)).Methods("GET", "OPTIONS")
 
 	//////////
 	// Jobs //
@@ -571,7 +591,9 @@ func NewAPIRouter(han *controllers.APIController, authMiddleware, initMiddleware
 	// Update template
 	apiRouter.Handle("/templates/{templateID}/", http.HandlerFunc(han.UpdateTemplateHandler)).Methods("PUT", "OPTIONS")
 	apiRouter.Handle("/templates/{templateID}", http.HandlerFunc(han.UpdateTemplateHandler)).Methods("PUT", "OPTIONS")
-
+	// Restore templates
+	apiRouter.Handle("/templates/restore/", http.HandlerFunc(han.RestoreTemplatesHandler)).Methods("POST", "OPTIONS")
+	apiRouter.Handle("/templates/restore", http.HandlerFunc(han.RestoreTemplatesHandler)).Methods("POST", "OPTIONS")
 	/////////////////////////
 	// Websocket endpoints //
 	/////////////////////////
@@ -584,6 +606,7 @@ func NewAPIRouter(han *controllers.APIController, authMiddleware, initMiddleware
 	// DB watcher websocket endpoint
 	apiRouter.Handle("/ws/events/", http.HandlerFunc(han.EventsHandler)).Methods("GET")
 	apiRouter.Handle("/ws/events", http.HandlerFunc(han.EventsHandler)).Methods("GET")
+	apiRouter.Handle("/ws/agent/{agentName}/shell", http.HandlerFunc(han.AgentShellHandler)).Methods("GET")
 
 	// NotFound handler - this should be last
 	apiRouter.PathPrefix("/").HandlerFunc(han.NotFoundHandler).Methods("GET", "POST", "PUT", "DELETE", "OPTIONS")

apiserver/swagger-models.yaml

@@ -401,3 +401,24 @@ definitions:
             alias: garm_params
     items:
         $ref: '#/definitions/Template'
+  RestoreTemplateRequest:
+    type: object
+    x-go-type:
+        type: RestoreTemplateRequest
+        import:
+            package: github.com/cloudbase/garm/params
+            alias: garm_params
+  GARMAgentToolsPaginatedResponse:
+    type: object
+    x-go-type:
+        type: GARMAgentToolsPaginatedResponse
+        import:
+            package: github.com/cloudbase/garm/params
+            alias: garm_params
+  GARMAgentTool:
+    type: object
+    x-go-type:
+        type: GARMAgentTool
+        import:
+            package: github.com/cloudbase/garm/params
+            alias: garm_params

apiserver/swagger.yaml

@@ -162,6 +162,20 @@ definitions:
                 alias: garm_params
                 package: github.com/cloudbase/garm/params
             type: ForgeEndpoints
+    GARMAgentTool:
+        type: object
+        x-go-type:
+            import:
+                alias: garm_params
+                package: github.com/cloudbase/garm/params
+            type: GARMAgentTool
+    GARMAgentToolsPaginatedResponse:
+        type: object
+        x-go-type:
+            import:
+                alias: garm_params
+                package: github.com/cloudbase/garm/params
+            type: GARMAgentToolsPaginatedResponse
     HookInfo:
         type: object
         x-go-type:
@@ -293,6 +307,13 @@ definitions:
                 alias: garm_params
                 package: github.com/cloudbase/garm/params
             type: Repository
+    RestoreTemplateRequest:
+        type: object
+        x-go-type:
+            import:
+                alias: garm_params
+                package: github.com/cloudbase/garm/params
+            type: RestoreTemplateRequest
     ScaleSet:
         type: object
         x-go-type:
@@ -2534,6 +2555,50 @@ paths:
             summary: Update template with the parameters given.
             tags:
                 - templates
+    /templates/restore:
+        post:
+            operationId: RestoreTemplates
+            parameters:
+                - description: Parameters used when restoring the templates.
+                  in: body
+                  name: Body
+                  required: true
+                  schema:
+                    $ref: '#/definitions/RestoreTemplateRequest'
+                    description: Parameters used when restoring the templates.
+                    type: object
+            responses:
+                default:
+                    description: APIErrorResponse
+                    schema:
+                        $ref: '#/definitions/APIErrorResponse'
+            summary: Create template with the parameters given.
+            tags:
+                - templates
+    /tools/garm-agent:
+        get:
+            operationId: GarmAgentList
+            parameters:
+                - description: The page at which to list.
+                  in: query
+                  name: page
+                  type: integer
+                - description: Number of items per page.
+                  in: query
+                  name: pageSize
+                  type: integer
+            responses:
+                "200":
+                    description: GARMAgentToolsPaginatedResponse
+                    schema:
+                        $ref: '#/definitions/GARMAgentToolsPaginatedResponse'
+                "400":
+                    description: APIErrorResponse
+                    schema:
+                        $ref: '#/definitions/APIErrorResponse'
+            summary: List GARM agent tools.
+            tags:
+                - tools
 produces:
     - application/json
 security:

auth/agent_middleware.go

@@ -0,0 +1,190 @@
+// Copyright 2022 Cloudbase Solutions SRL
+//
+//    Licensed under the Apache License, Version 2.0 (the "License"); you may
+//    not use this file except in compliance with the License. You may obtain
+//    a copy of the License at
+//
+//         http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+//    License for the specific language governing permissions and limitations
+//    under the License.
+
+package auth
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"net/http"
+	"strings"
+
+	jwt "github.com/golang-jwt/jwt/v5"
+
+	runnerErrors "github.com/cloudbase/garm-provider-common/errors"
+	commonParams "github.com/cloudbase/garm-provider-common/params"
+	"github.com/cloudbase/garm/config"
+	dbCommon "github.com/cloudbase/garm/database/common"
+	"github.com/cloudbase/garm/params"
+)
+
+func (i *instanceToken) NewAgentJWTToken(instance params.Instance, entity params.ForgeEntity) (string, error) {
+	claims := InstanceJWTClaims{
+		RegisteredClaims: jwt.RegisteredClaims{
+			Issuer: "garm",
+		},
+		ID:            instance.ID,
+		Name:          instance.Name,
+		PoolID:        instance.PoolID,
+		Scope:         entity.EntityType,
+		Entity:        entity.ID,
+		IsAgent:       true,
+		ForgeType:     string(entity.Credentials.ForgeType),
+		CreateAttempt: instance.CreateAttempt,
+	}
+	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
+	tokenString, err := token.SignedString([]byte(i.jwtSecret))
+	if err != nil {
+		return "", fmt.Errorf("error signing token: %w", err)
+	}
+
+	return tokenString, nil
+}
+
+// agentMiddleware is the authentication middleware
+// used with gorilla
+type agentMiddleware struct {
+	store dbCommon.Store
+	cfg   config.JWTAuth
+}
+
+// NewjwtMiddleware returns a populated jwtMiddleware
+func AgentMiddleware(store dbCommon.Store, cfg config.JWTAuth) (Middleware, error) {
+	return &agentMiddleware{
+		store: store,
+		cfg:   cfg,
+	}, nil
+}
+
+func (amw *agentMiddleware) claimsToContext(ctx context.Context, claims *InstanceJWTClaims) (context.Context, error) {
+	if claims == nil {
+		return ctx, runnerErrors.ErrUnauthorized
+	}
+
+	if claims.Name == "" {
+		return nil, runnerErrors.ErrUnauthorized
+	}
+
+	instanceInfo, err := amw.store.GetInstance(ctx, claims.Name)
+	if err != nil {
+		return ctx, runnerErrors.ErrUnauthorized
+	}
+
+	entity, err := getForgeEntityFromInstance(ctx, amw.store, instanceInfo)
+	if err != nil {
+		slog.ErrorContext(ctx, "failed to get entity from instance", "error", err)
+		return ctx, runnerErrors.ErrUnauthorized
+	}
+
+	ctx = PopulateInstanceContext(ctx, instanceInfo, entity, claims)
+	return ctx, nil
+}
+
+// Middleware implements the middleware interface
+func (amw *agentMiddleware) Middleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		authorizationHeader := r.Header.Get("authorization")
+		if authorizationHeader == "" {
+			slog.InfoContext(ctx, "authorization header was empty")
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		bearerToken := strings.Split(authorizationHeader, " ")
+		if len(bearerToken) != 2 {
+			slog.InfoContext(ctx, "invalid authorization header")
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		claims := &InstanceJWTClaims{}
+		token, err := jwt.ParseWithClaims(bearerToken[1], claims, func(token *jwt.Token) (interface{}, error) {
+			if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
+				return nil, fmt.Errorf("invalid signing method")
+			}
+			return []byte(amw.cfg.Secret), nil
+		})
+		if err != nil {
+			slog.InfoContext(ctx, "failed to validate JWT token", "error", err)
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		if !claims.IsAgent {
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		if !token.Valid {
+			slog.InfoContext(ctx, "JWT token is invalid")
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		ctx, err = amw.claimsToContext(ctx, claims)
+		if err != nil {
+			slog.InfoContext(ctx, "failed to populate context", "error", err)
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		if InstanceID(ctx) == "" {
+			slog.InfoContext(ctx, "failed to find instance ID in context")
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		runnerStatus := InstanceRunnerStatus(ctx)
+		switch runnerStatus {
+		case params.RunnerActive, params.RunnerTerminated, params.RunnerFailed:
+			// Once a job starts to run, we can no longer trust that the JWT token was not compromised.
+			// Any new auth requests using that token are not to be allowed.
+			slog.InfoContext(ctx, "invalid runner status", "status", runnerStatus)
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		instanceParams, err := InstanceParams(ctx)
+		if err != nil {
+			slog.InfoContext(
+				ctx, "could not find instance params",
+				"runner_name", InstanceName(ctx))
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		// Token was generated for a previous attempt at creating this instance.
+		if claims.CreateAttempt != instanceParams.CreateAttempt {
+			slog.InfoContext(
+				ctx, "invalid token create attempt",
+				"runner_name", InstanceName(ctx),
+				"token_create_attempt", claims.CreateAttempt,
+				"instance_create_attempt", instanceParams.CreateAttempt)
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		// instance must be running. Anything else is either still creating or in the process
+		// of being deleted and shouldn't be trying to authenticate.
+		if instanceParams.Status != commonParams.InstanceRunning {
+			slog.InfoContext(ctx, "invalid instance status", "status", instanceParams.Status)
+			invalidAuthResponse(ctx, w)
+			return
+		}
+
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
+}

auth/context.go

@@ -45,8 +45,21 @@ const (
 	instanceHasJITConfig contextFlags = "hasJITConfig"
 	instanceParams       contextFlags = "instanceParams"
 	instanceForgeTypeKey contextFlags = "forge_type"
+	instanceIsAgent      contextFlags = "is_agent"
 )
 
+func SetInstanceIsAgent(ctx context.Context, val bool) context.Context {
+	return context.WithValue(ctx, instanceIsAgent, val)
+}
+
+func InstanceIsAgent(ctx context.Context) bool {
+	elem := ctx.Value(instanceIsAgent)
+	if elem == nil {
+		return false
+	}
+	return elem.(bool)
+}
+
 func SetInstanceForgeType(ctx context.Context, val string) context.Context {
 	return context.WithValue(ctx, instanceForgeTypeKey, val)
 }
@@ -194,6 +207,7 @@ func PopulateInstanceContext(ctx context.Context, instance params.Instance, enti
 	ctx = SetInstanceParams(ctx, instance)
 	ctx = SetInstanceForgeType(ctx, claims.ForgeType)
 	ctx = SetInstanceEntity(ctx, entity)
+	ctx = SetInstanceIsAgent(ctx, claims.IsAgent)
 	return ctx
 }
 

auth/init_required.go

@@ -66,7 +66,7 @@ func (u *urlsRequired) Middleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := r.Context()
 		ctrlInfo, err := u.store.ControllerInfo()
-		if err != nil || ctrlInfo.MetadataURL == "" || ctrlInfo.CallbackURL == "" {
+		if err != nil || ctrlInfo.MetadataURL == "" || ctrlInfo.CallbackURL == "" || ctrlInfo.AgentURL == "" {
 			w.Header().Add("Content-Type", "application/json")
 			w.WriteHeader(http.StatusConflict)
 			if err := json.NewEncoder(w).Encode(params.URLsRequired); err != nil {

auth/instance_middleware.go

@@ -45,6 +45,7 @@ type InstanceJWTClaims struct {
 	Entity        string `json:"entity"`
 	CreateAttempt int    `json:"create_attempt"`
 	ForgeType     string `json:"forge_type"`
+	IsAgent       bool   `json:"is_agent"`
 	jwt.RegisteredClaims
 }
 
@@ -86,6 +87,7 @@ func (i *instanceToken) NewInstanceJWTToken(instance params.Instance, entity par
 		Scope:         entity.EntityType,
 		Entity:        entity.String(),
 		ForgeType:     string(entity.Credentials.ForgeType),
+		IsAgent:       false,
 		CreateAttempt: instance.CreateAttempt,
 	}
 	token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
@@ -112,14 +114,14 @@ func NewInstanceMiddleware(store dbCommon.Store, cfg config.JWTAuth) (Middleware
 	}, nil
 }
 
-func (amw *instanceMiddleware) getForgeEntityFromInstance(ctx context.Context, instance params.Instance) (params.ForgeEntity, error) {
+func getForgeEntityFromInstance(ctx context.Context, store dbCommon.Store, instance params.Instance) (params.ForgeEntity, error) {
 	var entityGetter params.EntityGetter
 	var err error
 	switch {
 	case instance.PoolID != "":
-		entityGetter, err = amw.store.GetPoolByID(ctx, instance.PoolID)
+		entityGetter, err = store.GetPoolByID(ctx, instance.PoolID)
 	case instance.ScaleSetID != 0:
-		entityGetter, err = amw.store.GetScaleSetByID(ctx, instance.ScaleSetID)
+		entityGetter, err = store.GetScaleSetByID(ctx, instance.ScaleSetID)
 	default:
 		return params.ForgeEntity{}, errors.New("instance not associated with a pool or scale set")
 	}
@@ -139,7 +141,7 @@ func (amw *instanceMiddleware) getForgeEntityFromInstance(ctx context.Context, i
 		return params.ForgeEntity{}, fmt.Errorf("error fetching entity: %w", err)
 	}
 
-	entity, err := amw.store.GetForgeEntity(ctx, poolEntity.EntityType, poolEntity.ID)
+	entity, err := store.GetForgeEntity(ctx, poolEntity.EntityType, poolEntity.ID)
 	if err != nil {
 		slog.With(slog.Any("error", err)).ErrorContext(
 			ctx, "failed to get entity",
@@ -166,7 +168,7 @@ func (amw *instanceMiddleware) claimsToContext(ctx context.Context, claims *Inst
 		return ctx, runnerErrors.ErrUnauthorized
 	}
 
-	entity, err := amw.getForgeEntityFromInstance(ctx, instanceInfo)
+	entity, err := getForgeEntityFromInstance(ctx, amw.store, instanceInfo)
 	if err != nil {
 		slog.ErrorContext(ctx, "failed to get entity from instance", "error", err)
 		return ctx, runnerErrors.ErrUnauthorized
@@ -210,6 +212,10 @@ func (amw *instanceMiddleware) Middleware(next http.Handler) http.Handler {
 			invalidAuthResponse(ctx, w)
 			return
 		}
+		if claims.IsAgent {
+			invalidAuthResponse(ctx, w)
+			return
+		}
 
 		ctx, err = amw.claimsToContext(ctx, claims)
 		if err != nil {

auth/interfaces.go

@@ -27,4 +27,5 @@ type Middleware interface {
 
 type InstanceTokenGetter interface {
 	NewInstanceJWTToken(instance params.Instance, entity params.ForgeEntity, ttlMinutes uint) (string, error)
+	NewAgentJWTToken(instance params.Instance, entity params.ForgeEntity) (string, error)
 }

auth/jwt.go

@@ -73,7 +73,7 @@ func (amw *jwtMiddleware) claimsToContext(ctx context.Context, claims *JWTClaims
 
 	var expiresAt *time.Time
 	if claims.ExpiresAt != nil {
-		expires := claims.ExpiresAt.Time.UTC()
+		expires := claims.ExpiresAt.UTC()
 		expiresAt = &expires
 	}
 

client/garm_api_client.go

@@ -27,6 +27,7 @@ import (
 	"github.com/cloudbase/garm/client/repositories"
 	"github.com/cloudbase/garm/client/scalesets"
 	"github.com/cloudbase/garm/client/templates"
+	"github.com/cloudbase/garm/client/tools"
 )
 
 // Default garm API HTTP client.
@@ -88,6 +89,7 @@ func New(transport runtime.ClientTransport, formats strfmt.Registry) *GarmAPI {
 	cli.Repositories = repositories.New(transport, formats)
 	cli.Scalesets = scalesets.New(transport, formats)
 	cli.Templates = templates.New(transport, formats)
+	cli.Tools = tools.New(transport, formats)
 	return cli
 }
 
@@ -166,6 +168,8 @@ type GarmAPI struct {
 
 	Templates templates.ClientService
 
+	Tools tools.ClientService
+
 	Transport runtime.ClientTransport
 }
 
@@ -189,4 +193,5 @@ func (c *GarmAPI) SetTransport(transport runtime.ClientTransport) {
 	c.Repositories.SetTransport(transport)
 	c.Scalesets.SetTransport(transport)
 	c.Templates.SetTransport(transport)
+	c.Tools.SetTransport(transport)
 }

client/templates/restore_templates_parameters.go

@@ -0,0 +1,151 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package templates
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/runtime"
+	cr "github.com/go-openapi/runtime/client"
+	"github.com/go-openapi/strfmt"
+
+	garm_params "github.com/cloudbase/garm/params"
+)
+
+// NewRestoreTemplatesParams creates a new RestoreTemplatesParams object,
+// with the default timeout for this client.
+//
+// Default values are not hydrated, since defaults are normally applied by the API server side.
+//
+// To enforce default values in parameter, use SetDefaults or WithDefaults.
+func NewRestoreTemplatesParams() *RestoreTemplatesParams {
+	return &RestoreTemplatesParams{
+		timeout: cr.DefaultTimeout,
+	}
+}
+
+// NewRestoreTemplatesParamsWithTimeout creates a new RestoreTemplatesParams object
+// with the ability to set a timeout on a request.
+func NewRestoreTemplatesParamsWithTimeout(timeout time.Duration) *RestoreTemplatesParams {
+	return &RestoreTemplatesParams{
+		timeout: timeout,
+	}
+}
+
+// NewRestoreTemplatesParamsWithContext creates a new RestoreTemplatesParams object
+// with the ability to set a context for a request.
+func NewRestoreTemplatesParamsWithContext(ctx context.Context) *RestoreTemplatesParams {
+	return &RestoreTemplatesParams{
+		Context: ctx,
+	}
+}
+
+// NewRestoreTemplatesParamsWithHTTPClient creates a new RestoreTemplatesParams object
+// with the ability to set a custom HTTPClient for a request.
+func NewRestoreTemplatesParamsWithHTTPClient(client *http.Client) *RestoreTemplatesParams {
+	return &RestoreTemplatesParams{
+		HTTPClient: client,
+	}
+}
+
+/*
+RestoreTemplatesParams contains all the parameters to send to the API endpoint
+
+	for the restore templates operation.
+
+	Typically these are written to a http.Request.
+*/
+type RestoreTemplatesParams struct {
+
+	/* Body.
+
+	   Parameters used when restoring the templates.
+	*/
+	Body garm_params.RestoreTemplateRequest
+
+	timeout    time.Duration
+	Context    context.Context
+	HTTPClient *http.Client
+}
+
+// WithDefaults hydrates default values in the restore templates params (not the query body).
+//
+// All values with no default are reset to their zero value.
+func (o *RestoreTemplatesParams) WithDefaults() *RestoreTemplatesParams {
+	o.SetDefaults()
+	return o
+}
+
+// SetDefaults hydrates default values in the restore templates params (not the query body).
+//
+// All values with no default are reset to their zero value.
+func (o *RestoreTemplatesParams) SetDefaults() {
+	// no default values defined for this parameter
+}
+
+// WithTimeout adds the timeout to the restore templates params
+func (o *RestoreTemplatesParams) WithTimeout(timeout time.Duration) *RestoreTemplatesParams {
+	o.SetTimeout(timeout)
+	return o
+}
+
+// SetTimeout adds the timeout to the restore templates params
+func (o *RestoreTemplatesParams) SetTimeout(timeout time.Duration) {
+	o.timeout = timeout
+}
+
+// WithContext adds the context to the restore templates params
+func (o *RestoreTemplatesParams) WithContext(ctx context.Context) *RestoreTemplatesParams {
+	o.SetContext(ctx)
+	return o
+}
+
+// SetContext adds the context to the restore templates params
+func (o *RestoreTemplatesParams) SetContext(ctx context.Context) {
+	o.Context = ctx
+}
+
+// WithHTTPClient adds the HTTPClient to the restore templates params
+func (o *RestoreTemplatesParams) WithHTTPClient(client *http.Client) *RestoreTemplatesParams {
+	o.SetHTTPClient(client)
+	return o
+}
+
+// SetHTTPClient adds the HTTPClient to the restore templates params
+func (o *RestoreTemplatesParams) SetHTTPClient(client *http.Client) {
+	o.HTTPClient = client
+}
+
+// WithBody adds the body to the restore templates params
+func (o *RestoreTemplatesParams) WithBody(body garm_params.RestoreTemplateRequest) *RestoreTemplatesParams {
+	o.SetBody(body)
+	return o
+}
+
+// SetBody adds the body to the restore templates params
+func (o *RestoreTemplatesParams) SetBody(body garm_params.RestoreTemplateRequest) {
+	o.Body = body
+}
+
+// WriteToRequest writes these params to a swagger request
+func (o *RestoreTemplatesParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error {
+
+	if err := r.SetTimeout(o.timeout); err != nil {
+		return err
+	}
+	var res []error
+	if err := r.SetBodyParam(o.Body); err != nil {
+		return err
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}

client/templates/restore_templates_responses.go

@@ -0,0 +1,106 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package templates
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/go-openapi/runtime"
+	"github.com/go-openapi/strfmt"
+
+	apiserver_params "github.com/cloudbase/garm/apiserver/params"
+)
+
+// RestoreTemplatesReader is a Reader for the RestoreTemplates structure.
+type RestoreTemplatesReader struct {
+	formats strfmt.Registry
+}
+
+// ReadResponse reads a server response into the received o.
+func (o *RestoreTemplatesReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) {
+	result := NewRestoreTemplatesDefault(response.Code())
+	if err := result.readResponse(response, consumer, o.formats); err != nil {
+		return nil, err
+	}
+	if response.Code()/100 == 2 {
+		return result, nil
+	}
+	return nil, result
+}
+
+// NewRestoreTemplatesDefault creates a RestoreTemplatesDefault with default headers values
+func NewRestoreTemplatesDefault(code int) *RestoreTemplatesDefault {
+	return &RestoreTemplatesDefault{
+		_statusCode: code,
+	}
+}
+
+/*
+RestoreTemplatesDefault describes a response with status code -1, with default header values.
+
+APIErrorResponse
+*/
+type RestoreTemplatesDefault struct {
+	_statusCode int
+
+	Payload apiserver_params.APIErrorResponse
+}
+
+// IsSuccess returns true when this restore templates default response has a 2xx status code
+func (o *RestoreTemplatesDefault) IsSuccess() bool {
+	return o._statusCode/100 == 2
+}
+
+// IsRedirect returns true when this restore templates default response has a 3xx status code
+func (o *RestoreTemplatesDefault) IsRedirect() bool {
+	return o._statusCode/100 == 3
+}
+
+// IsClientError returns true when this restore templates default response has a 4xx status code
+func (o *RestoreTemplatesDefault) IsClientError() bool {
+	return o._statusCode/100 == 4
+}
+
+// IsServerError returns true when this restore templates default response has a 5xx status code
+func (o *RestoreTemplatesDefault) IsServerError() bool {
+	return o._statusCode/100 == 5
+}
+
+// IsCode returns true when this restore templates default response a status code equal to that given
+func (o *RestoreTemplatesDefault) IsCode(code int) bool {
+	return o._statusCode == code
+}
+
+// Code gets the status code for the restore templates default response
+func (o *RestoreTemplatesDefault) Code() int {
+	return o._statusCode
+}
+
+func (o *RestoreTemplatesDefault) Error() string {
+	payload, _ := json.Marshal(o.Payload)
+	return fmt.Sprintf("[POST /templates/restore][%d] RestoreTemplates default %s", o._statusCode, payload)
+}
+
+func (o *RestoreTemplatesDefault) String() string {
+	payload, _ := json.Marshal(o.Payload)
+	return fmt.Sprintf("[POST /templates/restore][%d] RestoreTemplates default %s", o._statusCode, payload)
+}
+
+func (o *RestoreTemplatesDefault) GetPayload() apiserver_params.APIErrorResponse {
+	return o.Payload
+}
+
+func (o *RestoreTemplatesDefault) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	// response payload
+	if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}

client/templates/templates_client.go

@@ -62,6 +62,8 @@ type ClientService interface {
 
 	ListTemplates(params *ListTemplatesParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*ListTemplatesOK, error)
 
+	RestoreTemplates(params *RestoreTemplatesParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) error
+
 	UpdateTemplate(params *UpdateTemplateParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*UpdateTemplateOK, error)
 
 	SetTransport(transport runtime.ClientTransport)
@@ -213,6 +215,38 @@ func (a *Client) ListTemplates(params *ListTemplatesParams, authInfo runtime.Cli
 	return nil, runtime.NewAPIError("unexpected success response: content available as default response in error", unexpectedSuccess, unexpectedSuccess.Code())
 }
 
+/*
+RestoreTemplates creates template with the parameters given
+*/
+func (a *Client) RestoreTemplates(params *RestoreTemplatesParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) error {
+	// TODO: Validate the params before sending
+	if params == nil {
+		params = NewRestoreTemplatesParams()
+	}
+	op := &runtime.ClientOperation{
+		ID:                 "RestoreTemplates",
+		Method:             "POST",
+		PathPattern:        "/templates/restore",
+		ProducesMediaTypes: []string{"application/json"},
+		ConsumesMediaTypes: []string{"application/json"},
+		Schemes:            []string{"http"},
+		Params:             params,
+		Reader:             &RestoreTemplatesReader{formats: a.formats},
+		AuthInfo:           authInfo,
+		Context:            params.Context,
+		Client:             params.HTTPClient,
+	}
+	for _, opt := range opts {
+		opt(op)
+	}
+
+	_, err := a.transport.Submit(op)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 /*
 UpdateTemplate updates template with the parameters given
 */

client/tools/garm_agent_list_parameters.go

@@ -0,0 +1,198 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package tools
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"github.com/go-openapi/errors"
+	"github.com/go-openapi/runtime"
+	cr "github.com/go-openapi/runtime/client"
+	"github.com/go-openapi/strfmt"
+	"github.com/go-openapi/swag"
+)
+
+// NewGarmAgentListParams creates a new GarmAgentListParams object,
+// with the default timeout for this client.
+//
+// Default values are not hydrated, since defaults are normally applied by the API server side.
+//
+// To enforce default values in parameter, use SetDefaults or WithDefaults.
+func NewGarmAgentListParams() *GarmAgentListParams {
+	return &GarmAgentListParams{
+		timeout: cr.DefaultTimeout,
+	}
+}
+
+// NewGarmAgentListParamsWithTimeout creates a new GarmAgentListParams object
+// with the ability to set a timeout on a request.
+func NewGarmAgentListParamsWithTimeout(timeout time.Duration) *GarmAgentListParams {
+	return &GarmAgentListParams{
+		timeout: timeout,
+	}
+}
+
+// NewGarmAgentListParamsWithContext creates a new GarmAgentListParams object
+// with the ability to set a context for a request.
+func NewGarmAgentListParamsWithContext(ctx context.Context) *GarmAgentListParams {
+	return &GarmAgentListParams{
+		Context: ctx,
+	}
+}
+
+// NewGarmAgentListParamsWithHTTPClient creates a new GarmAgentListParams object
+// with the ability to set a custom HTTPClient for a request.
+func NewGarmAgentListParamsWithHTTPClient(client *http.Client) *GarmAgentListParams {
+	return &GarmAgentListParams{
+		HTTPClient: client,
+	}
+}
+
+/*
+GarmAgentListParams contains all the parameters to send to the API endpoint
+
+	for the garm agent list operation.
+
+	Typically these are written to a http.Request.
+*/
+type GarmAgentListParams struct {
+
+	/* Page.
+
+	   The page at which to list.
+	*/
+	Page *int64
+
+	/* PageSize.
+
+	   Number of items per page.
+	*/
+	PageSize *int64
+
+	timeout    time.Duration
+	Context    context.Context
+	HTTPClient *http.Client
+}
+
+// WithDefaults hydrates default values in the garm agent list params (not the query body).
+//
+// All values with no default are reset to their zero value.
+func (o *GarmAgentListParams) WithDefaults() *GarmAgentListParams {
+	o.SetDefaults()
+	return o
+}
+
+// SetDefaults hydrates default values in the garm agent list params (not the query body).
+//
+// All values with no default are reset to their zero value.
+func (o *GarmAgentListParams) SetDefaults() {
+	// no default values defined for this parameter
+}
+
+// WithTimeout adds the timeout to the garm agent list params
+func (o *GarmAgentListParams) WithTimeout(timeout time.Duration) *GarmAgentListParams {
+	o.SetTimeout(timeout)
+	return o
+}
+
+// SetTimeout adds the timeout to the garm agent list params
+func (o *GarmAgentListParams) SetTimeout(timeout time.Duration) {
+	o.timeout = timeout
+}
+
+// WithContext adds the context to the garm agent list params
+func (o *GarmAgentListParams) WithContext(ctx context.Context) *GarmAgentListParams {
+	o.SetContext(ctx)
+	return o
+}
+
+// SetContext adds the context to the garm agent list params
+func (o *GarmAgentListParams) SetContext(ctx context.Context) {
+	o.Context = ctx
+}
+
+// WithHTTPClient adds the HTTPClient to the garm agent list params
+func (o *GarmAgentListParams) WithHTTPClient(client *http.Client) *GarmAgentListParams {
+	o.SetHTTPClient(client)
+	return o
+}
+
+// SetHTTPClient adds the HTTPClient to the garm agent list params
+func (o *GarmAgentListParams) SetHTTPClient(client *http.Client) {
+	o.HTTPClient = client
+}
+
+// WithPage adds the page to the garm agent list params
+func (o *GarmAgentListParams) WithPage(page *int64) *GarmAgentListParams {
+	o.SetPage(page)
+	return o
+}
+
+// SetPage adds the page to the garm agent list params
+func (o *GarmAgentListParams) SetPage(page *int64) {
+	o.Page = page
+}
+
+// WithPageSize adds the pageSize to the garm agent list params
+func (o *GarmAgentListParams) WithPageSize(pageSize *int64) *GarmAgentListParams {
+	o.SetPageSize(pageSize)
+	return o
+}
+
+// SetPageSize adds the pageSize to the garm agent list params
+func (o *GarmAgentListParams) SetPageSize(pageSize *int64) {
+	o.PageSize = pageSize
+}
+
+// WriteToRequest writes these params to a swagger request
+func (o *GarmAgentListParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error {
+
+	if err := r.SetTimeout(o.timeout); err != nil {
+		return err
+	}
+	var res []error
+
+	if o.Page != nil {
+
+		// query param page
+		var qrPage int64
+
+		if o.Page != nil {
+			qrPage = *o.Page
+		}
+		qPage := swag.FormatInt64(qrPage)
+		if qPage != "" {
+
+			if err := r.SetQueryParam("page", qPage); err != nil {
+				return err
+			}
+		}
+	}
+
+	if o.PageSize != nil {
+
+		// query param pageSize
+		var qrPageSize int64
+
+		if o.PageSize != nil {
+			qrPageSize = *o.PageSize
+		}
+		qPageSize := swag.FormatInt64(qrPageSize)
+		if qPageSize != "" {
+
+			if err := r.SetQueryParam("pageSize", qPageSize); err != nil {
+				return err
+			}
+		}
+	}
+
+	if len(res) > 0 {
+		return errors.CompositeValidationError(res...)
+	}
+	return nil
+}

client/tools/garm_agent_list_responses.go

@@ -0,0 +1,179 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package tools
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/go-openapi/runtime"
+	"github.com/go-openapi/strfmt"
+
+	apiserver_params "github.com/cloudbase/garm/apiserver/params"
+	garm_params "github.com/cloudbase/garm/params"
+)
+
+// GarmAgentListReader is a Reader for the GarmAgentList structure.
+type GarmAgentListReader struct {
+	formats strfmt.Registry
+}
+
+// ReadResponse reads a server response into the received o.
+func (o *GarmAgentListReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) {
+	switch response.Code() {
+	case 200:
+		result := NewGarmAgentListOK()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return result, nil
+	case 400:
+		result := NewGarmAgentListBadRequest()
+		if err := result.readResponse(response, consumer, o.formats); err != nil {
+			return nil, err
+		}
+		return nil, result
+	default:
+		return nil, runtime.NewAPIError("[GET /tools/garm-agent] GarmAgentList", response, response.Code())
+	}
+}
+
+// NewGarmAgentListOK creates a GarmAgentListOK with default headers values
+func NewGarmAgentListOK() *GarmAgentListOK {
+	return &GarmAgentListOK{}
+}
+
+/*
+GarmAgentListOK describes a response with status code 200, with default header values.
+
+GARMAgentToolsPaginatedResponse
+*/
+type GarmAgentListOK struct {
+	Payload garm_params.GARMAgentToolsPaginatedResponse
+}
+
+// IsSuccess returns true when this garm agent list o k response has a 2xx status code
+func (o *GarmAgentListOK) IsSuccess() bool {
+	return true
+}
+
+// IsRedirect returns true when this garm agent list o k response has a 3xx status code
+func (o *GarmAgentListOK) IsRedirect() bool {
+	return false
+}
+
+// IsClientError returns true when this garm agent list o k response has a 4xx status code
+func (o *GarmAgentListOK) IsClientError() bool {
+	return false
+}
+
+// IsServerError returns true when this garm agent list o k response has a 5xx status code
+func (o *GarmAgentListOK) IsServerError() bool {
+	return false
+}
+
+// IsCode returns true when this garm agent list o k response a status code equal to that given
+func (o *GarmAgentListOK) IsCode(code int) bool {
+	return code == 200
+}
+
+// Code gets the status code for the garm agent list o k response
+func (o *GarmAgentListOK) Code() int {
+	return 200
+}
+
+func (o *GarmAgentListOK) Error() string {
+	payload, _ := json.Marshal(o.Payload)
+	return fmt.Sprintf("[GET /tools/garm-agent][%d] garmAgentListOK %s", 200, payload)
+}
+
+func (o *GarmAgentListOK) String() string {
+	payload, _ := json.Marshal(o.Payload)
+	return fmt.Sprintf("[GET /tools/garm-agent][%d] garmAgentListOK %s", 200, payload)
+}
+
+func (o *GarmAgentListOK) GetPayload() garm_params.GARMAgentToolsPaginatedResponse {
+	return o.Payload
+}
+
+func (o *GarmAgentListOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	// response payload
+	if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}
+
+// NewGarmAgentListBadRequest creates a GarmAgentListBadRequest with default headers values
+func NewGarmAgentListBadRequest() *GarmAgentListBadRequest {
+	return &GarmAgentListBadRequest{}
+}
+
+/*
+GarmAgentListBadRequest describes a response with status code 400, with default header values.
+
+APIErrorResponse
+*/
+type GarmAgentListBadRequest struct {
+	Payload apiserver_params.APIErrorResponse
+}
+
+// IsSuccess returns true when this garm agent list bad request response has a 2xx status code
+func (o *GarmAgentListBadRequest) IsSuccess() bool {
+	return false
+}
+
+// IsRedirect returns true when this garm agent list bad request response has a 3xx status code
+func (o *GarmAgentListBadRequest) IsRedirect() bool {
+	return false
+}
+
+// IsClientError returns true when this garm agent list bad request response has a 4xx status code
+func (o *GarmAgentListBadRequest) IsClientError() bool {
+	return true
+}
+
+// IsServerError returns true when this garm agent list bad request response has a 5xx status code
+func (o *GarmAgentListBadRequest) IsServerError() bool {
+	return false
+}
+
+// IsCode returns true when this garm agent list bad request response a status code equal to that given
+func (o *GarmAgentListBadRequest) IsCode(code int) bool {
+	return code == 400
+}
+
+// Code gets the status code for the garm agent list bad request response
+func (o *GarmAgentListBadRequest) Code() int {
+	return 400
+}
+
+func (o *GarmAgentListBadRequest) Error() string {
+	payload, _ := json.Marshal(o.Payload)
+	return fmt.Sprintf("[GET /tools/garm-agent][%d] garmAgentListBadRequest %s", 400, payload)
+}
+
+func (o *GarmAgentListBadRequest) String() string {
+	payload, _ := json.Marshal(o.Payload)
+	return fmt.Sprintf("[GET /tools/garm-agent][%d] garmAgentListBadRequest %s", 400, payload)
+}
+
+func (o *GarmAgentListBadRequest) GetPayload() apiserver_params.APIErrorResponse {
+	return o.Payload
+}
+
+func (o *GarmAgentListBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error {
+
+	// response payload
+	if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF {
+		return err
+	}
+
+	return nil
+}

client/tools/tools_client.go

@@ -0,0 +1,106 @@
+// Code generated by go-swagger; DO NOT EDIT.
+
+package tools
+
+// This file was generated by the swagger tool.
+// Editing this file might prove futile when you re-run the swagger generate command
+
+import (
+	"fmt"
+
+	"github.com/go-openapi/runtime"
+	httptransport "github.com/go-openapi/runtime/client"
+	"github.com/go-openapi/strfmt"
+)
+
+// New creates a new tools API client.
+func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService {
+	return &Client{transport: transport, formats: formats}
+}
+
+// New creates a new tools API client with basic auth credentials.
+// It takes the following parameters:
+// - host: http host (github.com).
+// - basePath: any base path for the API client ("/v1", "/v3").
+// - scheme: http scheme ("http", "https").
+// - user: user for basic authentication header.
+// - password: password for basic authentication header.
+func NewClientWithBasicAuth(host, basePath, scheme, user, password string) ClientService {
+	transport := httptransport.New(host, basePath, []string{scheme})
+	transport.DefaultAuthentication = httptransport.BasicAuth(user, password)
+	return &Client{transport: transport, formats: strfmt.Default}
+}
+
+// New creates a new tools API client with a bearer token for authentication.
+// It takes the following parameters:
+// - host: http host (github.com).
+// - basePath: any base path for the API client ("/v1", "/v3").
+// - scheme: http scheme ("http", "https").
+// - bearerToken: bearer token for Bearer authentication header.
+func NewClientWithBearerToken(host, basePath, scheme, bearerToken string) ClientService {
+	transport := httptransport.New(host, basePath, []string{scheme})
+	transport.DefaultAuthentication = httptransport.BearerToken(bearerToken)
+	return &Client{transport: transport, formats: strfmt.Default}
+}
+
+/*
+Client for tools API
+*/
+type Client struct {
+	transport runtime.ClientTransport
+	formats   strfmt.Registry
+}
+
+// ClientOption may be used to customize the behavior of Client methods.
+type ClientOption func(*runtime.ClientOperation)
+
+// ClientService is the interface for Client methods
+type ClientService interface {
+	GarmAgentList(params *GarmAgentListParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*GarmAgentListOK, error)
+
+	SetTransport(transport runtime.ClientTransport)
+}
+
+/*
+GarmAgentList lists g a r m agent tools
+*/
+func (a *Client) GarmAgentList(params *GarmAgentListParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*GarmAgentListOK, error) {
+	// TODO: Validate the params before sending
+	if params == nil {
+		params = NewGarmAgentListParams()
+	}
+	op := &runtime.ClientOperation{
+		ID:                 "GarmAgentList",
+		Method:             "GET",
+		PathPattern:        "/tools/garm-agent",
+		ProducesMediaTypes: []string{"application/json"},
+		ConsumesMediaTypes: []string{"application/json"},
+		Schemes:            []string{"http"},
+		Params:             params,
+		Reader:             &GarmAgentListReader{formats: a.formats},
+		AuthInfo:           authInfo,
+		Context:            params.Context,
+		Client:             params.HTTPClient,
+	}
+	for _, opt := range opts {
+		opt(op)
+	}
+
+	result, err := a.transport.Submit(op)
+	if err != nil {
+		return nil, err
+	}
+	success, ok := result.(*GarmAgentListOK)
+	if ok {
+		return success, nil
+	}
+	// unexpected success response
+	// safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue
+	msg := fmt.Sprintf("unexpected success response for GarmAgentList: API contract not enforced by server. Client expected to get an error, but got: %T", result)
+	panic(msg)
+}
+
+// SetTransport changes the transport on the client
+func (a *Client) SetTransport(transport runtime.ClientTransport) {
+	a.transport = transport
+}

cmd/garm-cli/cmd/agent_nix.go

@@ -0,0 +1,32 @@
+//go:build !windows
+// +build !windows
+
+package cmd
+
+import (
+	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/google/uuid"
+	"golang.org/x/term"
+)
+
+var sigs = make(chan os.Signal, 1)
+
+func watchTermResize(ctx context.Context, resizeCh chan [2]int, sessionID uuid.UUID) {
+	signal.Notify(sigs, syscall.SIGWINCH)
+
+	for {
+		select {
+		case <-sigs:
+			w, h, err := term.GetSize(int(os.Stdin.Fd()))
+			if err == nil && sessionID != uuid.Nil {
+				resizeCh <- [2]int{w, h}
+			}
+		case <-ctx.Done():
+			return
+		}
+	}
+}

cmd/garm-cli/cmd/agent_windows.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	"context"
+	"os"
+	"time"
+
+	"github.com/google/uuid"
+	"golang.org/x/term"
+)
+
+func watchTermResize(ctx context.Context, resizeCh chan [2]int, sessionID uuid.UUID) {
+	var lastW, lastH int
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			w, h, err := term.GetSize(int(os.Stdin.Fd()))
+			if err == nil && (w != lastW || h != lastH) && sessionID != uuid.Nil {
+				lastW, lastH = w, h
+				resizeCh <- [2]int{w, h}
+			}
+			time.Sleep(500 * time.Millisecond)
+		}
+	}
+}

cmd/garm-cli/cmd/controller.go

@@ -18,10 +18,12 @@ import (
 	"fmt"
 
 	"github.com/jedib0t/go-pretty/v6/table"
+	"github.com/jedib0t/go-pretty/v6/text"
 	"github.com/spf13/cobra"
 
 	apiClientController "github.com/cloudbase/garm/client/controller"
 	apiClientControllerInfo "github.com/cloudbase/garm/client/controller_info"
+	apiClientTools "github.com/cloudbase/garm/client/tools"
 	"github.com/cloudbase/garm/cmd/garm-cli/common"
 	"github.com/cloudbase/garm/params"
 )
@@ -72,6 +74,7 @@ as follows:
   * /webhooks - the base URL for the webhooks. Github needs to reach this URL.
   * /api/v1/metadata - the metadata URL. Your runners need to be able to reach this URL.
   * /api/v1/callbacks - the callback URL. Your runners need to be able to reach this URL.
+  * /agent - the agent URL. Your runners need to be able to reach this URL, when agent mode is used.
 
 You need to expose these endpoints to the interested parties (github or
 your runners), then you need to update the controller with the URLs you set up.
@@ -95,7 +98,15 @@ up the GARM controller URLs as:
   garm-cli controller update \
     --webhook-url=https://garm.example.com/webhooks \
     --metadata-url=https://garm.example.com/api/v1/metadata \
-    --callback-url=https://garm.example.com/api/v1/callbacks
+    --callback-url=https://garm.example.com/api/v1/callbacks \
+    --agent-url=https://garm.example.com/agent
+
+Additionally, there is one URL that is not meant to expose any service on the GARM server,
+but is needed if you wish GARM to automatically sync the garm-agent tooling needed for agent
+mode. This url is called garm-tools-url:
+
+garm-cli controller update \
+	--garm-tools-url=https://api.github.com/repos/cloudbase/garm-agent/releases
 `,
 	SilenceUsage: true,
 	RunE: func(cmd *cobra.Command, _ []string) error {
@@ -113,14 +124,23 @@ up the GARM controller URLs as:
 		if cmd.Flags().Changed("webhook-url") {
 			params.WebhookURL = &webhookURL
 		}
+		if cmd.Flags().Changed("agent-url") {
+			params.AgentURL = &agentURL
+		}
+		if cmd.Flags().Changed("garm-tools-url") {
+			params.GARMAgentReleasesURL = &garmToolsReleasesURL
+		}
+		if cmd.Flags().Changed("enable-tools-sync") {
+			params.SyncGARMAgentTools = &enableToolsSync
+		}
 
 		if cmd.Flags().Changed("minimum-job-age-backoff") {
 			params.MinimumJobAgeBackoff = &minimumJobAgeBackoff
 		}
 
-		if params.WebhookURL == nil && params.MetadataURL == nil && params.CallbackURL == nil && params.MinimumJobAgeBackoff == nil {
+		if params.WebhookURL == nil && params.MetadataURL == nil && params.CallbackURL == nil && params.MinimumJobAgeBackoff == nil && params.GARMAgentReleasesURL == nil && params.SyncGARMAgentTools == nil {
 			cmd.Help()
-			return fmt.Errorf("at least one of minimum-job-age-backoff, metadata-url, callback-url or webhook-url must be provided")
+			return fmt.Errorf("at least one of minimum-job-age-backoff, metadata-url, callback-url, enable-tools-sync, garm-tools-url  or webhook-url must be provided")
 		}
 
 		updateUrlsReq := apiClientController.NewUpdateControllerParams()
@@ -135,6 +155,55 @@ up the GARM controller URLs as:
 	},
 }
 
+var controllerToolsCmd = &cobra.Command{
+	Use:   "tools",
+	Short: "Show information about garm tools",
+	Long: `Show information about GARM tools available in this controller.
+
+GARM has two modes by which we deploy runners:
+
+  * Black box mode
+  * Agent mode
+
+In black box mode, we are completely agentless on the runners. The only software we really
+have to install besides standrd tools like jq, curl, etc is the runner software (github/gitea).
+We rely on information we get from the API of GitHub/Gitea and the APIs of the various providers
+to understand the state of our runner. We care both about the lifecycle of the VM/container/Bare metal
+and the lifecycle state of the runner itself (idle, active, terminated, etc). In black box mode,
+we do not get any status update from the instance.
+
+In Agent mode, we install the garm-agent on the runner, which in turn starts the actual runner. The agent
+also connects back to the garm server over websockets and sends back periodic heartbeats as well as the
+current state of the runner. We are able to immediately know when a job is picked up, when the job is done
+and whether or not the user forcefully deleted the BM/VM/container the runner was running on or the
+runner registered in github/gitea. At that point we can clean up the runner without having to thech the
+github/gitea API or the API of the provider in which the runner was spawned.
+
+This command lists the available tools in the controller. Tools can either sync automatically or be
+manually uploaded. As long as the controller has access to the tools, agent mode can be enabled.
+`,
+	SilenceUsage: true,
+	RunE: func(cmd *cobra.Command, _ []string) error {
+		if needsInit {
+			return errNeedsInitError
+		}
+
+		showTools := apiClientTools.NewGarmAgentListParams()
+		if cmd.Flags().Changed("page") {
+			showTools.Page = &fileObjPage
+		}
+		if cmd.Flags().Changed("page-size") {
+			showTools.PageSize = &fileObjPageSize
+		}
+		response, err := apiCli.Tools.GarmAgentList(showTools, authToken)
+		if err != nil {
+			return err
+		}
+		formatGARMToolsList(response.Payload)
+		return nil
+	},
+}
+
 func renderControllerInfoTable(info params.ControllerInfo) string {
 	t := table.NewWriter()
 	header := table.Row{"Field", "Value"}
@@ -159,6 +228,9 @@ func renderControllerInfoTable(info params.ControllerInfo) string {
 	t.AppendRow(table.Row{"Callback URL", info.CallbackURL})
 	t.AppendRow(table.Row{"Webhook Base URL", info.WebhookURL})
 	t.AppendRow(table.Row{"Controller Webhook URL", info.ControllerWebhookURL})
+	t.AppendRow(table.Row{"Agent URL", info.AgentURL})
+	t.AppendRow(table.Row{"GARM agent tools sync URL", info.GARMAgentReleasesURL})
+	t.AppendRow(table.Row{"Tools sync enabled", info.SyncGARMAgentTools})
 	t.AppendRow(table.Row{"Minimum Job Age Backoff", info.MinimumJobAgeBackoff})
 	t.AppendRow(table.Row{"Version", serverVersion})
 	return t.Render()
@@ -177,12 +249,55 @@ func init() {
 	controllerUpdateCmd.Flags().StringVarP(&metadataURL, "metadata-url", "m", "", "The metadata URL for the controller (ie. https://garm.example.com/api/v1/metadata)")
 	controllerUpdateCmd.Flags().StringVarP(&callbackURL, "callback-url", "c", "", "The callback URL for the controller (ie. https://garm.example.com/api/v1/callbacks)")
 	controllerUpdateCmd.Flags().StringVarP(&webhookURL, "webhook-url", "w", "", "The webhook URL for the controller (ie. https://garm.example.com/webhooks)")
+	controllerUpdateCmd.Flags().StringVarP(&agentURL, "agent-url", "g", "", "The agent URL for the controller (ie. https://garm.example.com/agent)")
+	controllerUpdateCmd.Flags().StringVarP(&garmToolsReleasesURL, "garm-tools-url", "t", "", "The URL for the garm-agent releases page (ie. https://api.github.com/repos/cloudbase/garm-agent/releases)")
+	controllerUpdateCmd.Flags().BoolVarP(&enableToolsSync, "enable-tools-sync", "s", false, "Enable or disable automatic garm tools sync.")
 	controllerUpdateCmd.Flags().UintVarP(&minimumJobAgeBackoff, "minimum-job-age-backoff", "b", 0, "The minimum job age backoff for the controller")
 
+	controllerToolsCmd.Flags().Int64Var(&fileObjPage, "page", 0, "The tools page to display")
+	controllerToolsCmd.Flags().Int64Var(&fileObjPageSize, "page-size", 25, "Total number of results per page")
 	controllerCmd.AddCommand(
 		controllerShowCmd,
 		controllerUpdateCmd,
+		controllerToolsCmd,
 	)
 
 	rootCmd.AddCommand(controllerCmd)
 }
+
+func formatGARMToolsList(files params.GARMAgentToolsPaginatedResponse) {
+	if outputFormat == common.OutputFormatJSON {
+		printAsJSON(files)
+		return
+	}
+	t := table.NewWriter()
+	// Define column count
+	numCols := 8
+	t.Style().Options.SeparateHeader = true
+	t.Style().Options.SeparateRows = true
+
+	// Page header - fill all columns with the same text
+	pageHeaderText := fmt.Sprintf("Page %d of %d", files.CurrentPage, files.Pages)
+	pageHeader := make(table.Row, numCols)
+	for i := range pageHeader {
+		pageHeader[i] = pageHeaderText
+	}
+	t.AppendHeader(pageHeader, table.RowConfig{
+		AutoMerge:      true,
+		AutoMergeAlign: text.AlignCenter,
+	})
+	// Column headers
+	header := table.Row{"ID", "Name", "Size", "Version", "OS Type", "OS Architecture", "Created", "Updated"}
+	t.AppendHeader(header)
+	// Right-align numeric columns
+	t.SetColumnConfigs([]table.ColumnConfig{
+		{Number: 1, Align: text.AlignRight},
+		{Number: 3, Align: text.AlignRight},
+	})
+
+	for _, val := range files.Results {
+		row := table.Row{val.ID, val.Name, formatSize(val.Size), val.Version, val.OSType, val.OSArch, val.CreatedAt.Format("2006-01-02 15:04:05"), val.UpdatedAt.Format("2006-01-02 15:04:05")}
+		t.AppendRow(row)
+	}
+	fmt.Println(t.Render())
+}

cmd/garm-cli/cmd/enterprise.go

@@ -31,6 +31,7 @@ var (
 	enterpriseEndpoint      string
 	enterpriseWebhookSecret string
 	enterpriseCreds         string
+	enterpriseAgentMode     bool
 )
 
 // enterpriseCmd represents the enterprise command
@@ -64,6 +65,7 @@ var enterpriseAddCmd = &cobra.Command{
 			WebhookSecret:    enterpriseWebhookSecret,
 			CredentialsName:  enterpriseCreds,
 			PoolBalancerType: params.PoolBalancerType(poolBalancerType),
+			AgentMode:        enterpriseAgentMode,
 		}
 		response, err := apiCli.Enterprises.CreateEnterprise(newEnterpriseReq, authToken)
 		if err != nil {
@@ -165,7 +167,7 @@ var enterpriseUpdateCmd = &cobra.Command{
 	Short:        "Update enterprise",
 	Long:         `Update enterprise credentials or webhook secret.`,
 	SilenceUsage: true,
-	RunE: func(_ *cobra.Command, args []string) error {
+	RunE: func(cmd *cobra.Command, args []string) error {
 		if needsInit {
 			return errNeedsInitError
 		}
@@ -184,10 +186,13 @@ var enterpriseUpdateCmd = &cobra.Command{
 
 		updateEnterpriseReq := apiClientEnterprises.NewUpdateEnterpriseParams()
 		updateEnterpriseReq.Body = params.UpdateEntityParams{
-			WebhookSecret:    repoWebhookSecret,
-			CredentialsName:  repoCreds,
+			WebhookSecret:    enterpriseWebhookSecret,
+			CredentialsName:  enterpriseCreds,
 			PoolBalancerType: params.PoolBalancerType(poolBalancerType),
 		}
+		if cmd.Flags().Changed("agent-mode") {
+			updateEnterpriseReq.Body.AgentMode = &enterpriseAgentMode
+		}
 		updateEnterpriseReq.EnterpriseID = enterpriseID
 		response, err := apiCli.Enterprises.UpdateEnterprise(updateEnterpriseReq, authToken)
 		if err != nil {
@@ -203,6 +208,7 @@ func init() {
 	enterpriseAddCmd.Flags().StringVar(&enterpriseWebhookSecret, "webhook-secret", "", "The webhook secret for this enterprise")
 	enterpriseAddCmd.Flags().StringVar(&enterpriseCreds, "credentials", "", "Credentials name. See credentials list.")
 	enterpriseAddCmd.Flags().StringVar(&poolBalancerType, "pool-balancer-type", string(params.PoolBalancerTypeRoundRobin), "The balancing strategy to use when creating runners in pools matching requested labels.")
+	enterpriseAddCmd.Flags().BoolVar(&enterpriseAgentMode, "agent-mode", false, "Enable agent mode for runners in this enterprise.")
 
 	enterpriseListCmd.Flags().BoolVarP(&long, "long", "l", false, "Include additional info.")
 	enterpriseListCmd.Flags().StringVarP(&enterpriseName, "name", "n", "", "Exact enterprise name to filter by.")
@@ -213,6 +219,7 @@ func init() {
 	enterpriseUpdateCmd.Flags().StringVar(&enterpriseWebhookSecret, "webhook-secret", "", "The webhook secret for this enterprise")
 	enterpriseUpdateCmd.Flags().StringVar(&enterpriseCreds, "credentials", "", "Credentials name. See credentials list.")
 	enterpriseUpdateCmd.Flags().StringVar(&poolBalancerType, "pool-balancer-type", "", "The balancing strategy to use when creating runners in pools matching requested labels.")
+	enterpriseUpdateCmd.Flags().BoolVar(&enterpriseAgentMode, "agent-mode", false, "Enable agent mode for runners in this enterprise.")
 	enterpriseUpdateCmd.Flags().StringVar(&enterpriseEndpoint, "endpoint", "", "When using the name of the enterprise, the endpoint must be specified when multiple enterprises with the same name exist.")
 
 	enterpriseDeleteCmd.Flags().StringVar(&enterpriseEndpoint, "endpoint", "", "When using the name of the enterprise, the endpoint must be specified when multiple enterprises with the same name exist.")

cmd/garm-cli/cmd/init.go

@@ -35,6 +35,9 @@ var (
 	callbackURL          string
 	metadataURL          string
 	webhookURL           string
+	agentURL             string
+	garmToolsReleasesURL string
+	enableToolsSync      bool
 	minimumJobAgeBackoff uint
 )
 
@@ -140,6 +143,13 @@ func ensureDefaultEndpoints(loginURL string) (err error) {
 			return err
 		}
 	}
+
+	if agentURL == "" {
+		agentURL, err = url.JoinPath(loginURL, "agent")
+		if err != nil {
+			return err
+		}
+	}
 	return nil
 }
 
@@ -185,6 +195,7 @@ func init() {
 	initCmd.Flags().StringVarP(&metadataURL, "metadata-url", "m", "", "The metadata URL for the controller (ie. https://garm.example.com/api/v1/metadata)")
 	initCmd.Flags().StringVarP(&callbackURL, "callback-url", "c", "", "The callback URL for the controller (ie. https://garm.example.com/api/v1/callbacks)")
 	initCmd.Flags().StringVarP(&webhookURL, "webhook-url", "w", "", "The webhook URL for the controller (ie. https://garm.example.com/webhooks)")
+	initCmd.Flags().StringVarP(&agentURL, "agent-url", "g", "", "The agent URL for the controller (ie. https://garm.example.com/agent)")
 	initCmd.Flags().StringVarP(&loginFullName, "full-name", "f", "", "Full name of the user")
 	initCmd.Flags().StringVarP(&loginPassword, "password", "p", "", "The admin password")
 	initCmd.MarkFlagRequired("name") //nolint
@@ -220,7 +231,7 @@ Admin user information:
 Make sure that the URLs in the table above are reachable by the relevant parties.
 
 The metadata and callback URLs *must* be accessible by the runners that GARM spins up.
-The base webhook and the controller webhook URLs must be accessible by GitHub or GHES. 
+The base webhook and the controller webhook URLs must be accessible by GitHub or GHES.
 `
 
 	controllerErrorMsg := `WARNING: Failed to set the required controller URLs with error: %q
@@ -228,7 +239,7 @@ The base webhook and the controller webhook URLs must be accessible by GitHub or
 Please run:
 
   garm-cli controller show
-  
+
 To make sure that the callback, metadata and webhook URLs are set correctly. If not,
 you must set them up by running:
 

cmd/garm-cli/cmd/organization.go

@@ -36,6 +36,7 @@ var (
 	insecureOrgWebhook     bool
 	keepOrgWebhook         bool
 	installOrgWebhook      bool
+	orgAgentMode           bool
 )
 
 // organizationCmd represents the organization command
@@ -184,6 +185,7 @@ var orgAddCmd = &cobra.Command{
 			CredentialsName:  orgCreds,
 			ForgeType:        params.EndpointType(forgeType),
 			PoolBalancerType: params.PoolBalancerType(poolBalancerType),
+			AgentMode:        orgAgentMode,
 		}
 		response, err := apiCli.Organizations.CreateOrg(newOrgReq, authToken)
 		if err != nil {
@@ -217,7 +219,7 @@ var orgUpdateCmd = &cobra.Command{
 	Short:        "Update organization",
 	Long:         `Update organization credentials or webhook secret.`,
 	SilenceUsage: true,
-	RunE: func(_ *cobra.Command, args []string) error {
+	RunE: func(cmd *cobra.Command, args []string) error {
 		if needsInit {
 			return errNeedsInitError
 		}
@@ -241,6 +243,9 @@ var orgUpdateCmd = &cobra.Command{
 			CredentialsName:  orgCreds,
 			PoolBalancerType: params.PoolBalancerType(poolBalancerType),
 		}
+		if cmd.Flags().Changed("agent-mode") {
+			updateOrgReq.Body.AgentMode = &orgAgentMode
+		}
 		updateOrgReq.OrgID = orgID
 		response, err := apiCli.Organizations.UpdateOrg(updateOrgReq, authToken)
 		if err != nil {
@@ -346,6 +351,7 @@ func init() {
 	orgAddCmd.Flags().StringVar(&orgCreds, "credentials", "", "Credentials name. See credentials list.")
 	orgAddCmd.Flags().BoolVar(&orgRandomWebhookSecret, "random-webhook-secret", false, "Generate a random webhook secret for this organization.")
 	orgAddCmd.Flags().BoolVar(&installOrgWebhook, "install-webhook", false, "Install the webhook as part of the add operation.")
+	orgAddCmd.Flags().BoolVar(&orgAgentMode, "agent-mode", false, "Enable agent mode for runners in this organization.")
 	orgAddCmd.MarkFlagsMutuallyExclusive("webhook-secret", "random-webhook-secret")
 	orgAddCmd.MarkFlagsOneRequired("webhook-secret", "random-webhook-secret")
 
@@ -364,6 +370,7 @@ func init() {
 	orgUpdateCmd.Flags().StringVar(&orgWebhookSecret, "webhook-secret", "", "The webhook secret for this organization")
 	orgUpdateCmd.Flags().StringVar(&orgCreds, "credentials", "", "Credentials name. See credentials list.")
 	orgUpdateCmd.Flags().StringVar(&poolBalancerType, "pool-balancer-type", "", "The balancing strategy to use when creating runners in pools matching requested labels.")
+	orgUpdateCmd.Flags().BoolVar(&orgAgentMode, "agent-mode", false, "Enable agent mode for runners in this organization.")
 	orgUpdateCmd.Flags().StringVar(&orgEndpoint, "endpoint", "", "When using the name of the org, the endpoint must be specified when multiple organizations with the same name exist.")
 
 	orgWebhookInstallCmd.Flags().BoolVar(&insecureOrgWebhook, "insecure", false, "Ignore self signed certificate errors.")

cmd/garm-cli/cmd/pool.go

@@ -53,6 +53,7 @@ var (
 	poolGitHubRunnerGroup      string
 	priority                   uint
 	poolTemplateNameOrID       string
+	poolEnableShell            bool
 )
 
 type poolsPayloadGetter interface {
@@ -234,6 +235,7 @@ var poolAddCmd = &cobra.Command{
 			RunnerBootstrapTimeout: poolRunnerBootstrapTimeout,
 			GitHubRunnerGroup:      poolGitHubRunnerGroup,
 			Priority:               priority,
+			EnableShell:            poolEnableShell,
 		}
 
 		if cmd.Flags().Changed("extra-specs") {
@@ -390,6 +392,10 @@ explicitly remove them using the runner delete command.
 			poolUpdateParams.RunnerBootstrapTimeout = &poolRunnerBootstrapTimeout
 		}
 
+		if cmd.Flags().Changed("enable-shell") {
+			poolUpdateParams.EnableShell = &poolEnableShell
+		}
+
 		if cmd.Flags().Changed("extra-specs") {
 			data, err := asRawMessage([]byte(poolExtraSpecs))
 			if err != nil {
@@ -443,6 +449,7 @@ func init() {
 	poolUpdateCmd.Flags().UintVar(&poolRunnerBootstrapTimeout, "runner-bootstrap-timeout", 20, "Duration in minutes after which a runner is considered failed if it does not join Github.")
 	poolUpdateCmd.Flags().StringVar(&poolExtraSpecsFile, "extra-specs-file", "", "A file containing a valid json which will be passed to the IaaS provider managing the pool.")
 	poolUpdateCmd.Flags().StringVar(&poolExtraSpecs, "extra-specs", "", "A valid json which will be passed to the IaaS provider managing the pool.")
+	poolUpdateCmd.Flags().BoolVar(&poolEnableShell, "enable-shell", false, "Enable shell access for runners in this pool.")
 	poolUpdateCmd.MarkFlagsMutuallyExclusive("extra-specs-file", "extra-specs")
 	poolUpdateCmd.Flags().StringVar(&poolTemplateNameOrID, "runner-install-template", "", "The runner install template name or ID to use for this pool.")
 
@@ -461,6 +468,7 @@ func init() {
 	poolAddCmd.Flags().UintVar(&poolRunnerBootstrapTimeout, "runner-bootstrap-timeout", 20, "Duration in minutes after which a runner is considered failed if it does not join Github.")
 	poolAddCmd.Flags().UintVar(&poolMinIdleRunners, "min-idle-runners", 1, "Attempt to maintain a minimum of idle self-hosted runners of this type.")
 	poolAddCmd.Flags().BoolVar(&poolEnabled, "enabled", false, "Enable this pool.")
+	poolAddCmd.Flags().BoolVar(&poolEnableShell, "enable-shell", false, "Enable shell access for runners in this pool.")
 	poolAddCmd.Flags().StringVar(&poolTemplateNameOrID, "runner-install-template", "", "The runner install template name or ID to use for this pool.")
 	poolAddCmd.Flags().StringVar(&endpointName, "endpoint", "", "When using the name of an entity, the endpoint must be specified when multiple entities with the same name exist.")
 

cmd/garm-cli/cmd/repository.go

@@ -38,6 +38,7 @@ var (
 	insecureRepoWebhook bool
 	keepRepoWebhook     bool
 	installRepoWebhook  bool
+	repoAgentMode       bool
 )
 
 // repositoryCmd represents the repository command
@@ -189,6 +190,7 @@ var repoAddCmd = &cobra.Command{
 			CredentialsName:  repoCreds,
 			ForgeType:        params.EndpointType(forgeType),
 			PoolBalancerType: params.PoolBalancerType(poolBalancerType),
+			AgentMode:        repoAgentMode,
 		}
 		response, err := apiCli.Repositories.CreateRepo(newRepoReq, authToken)
 		if err != nil {
@@ -246,7 +248,7 @@ var repoUpdateCmd = &cobra.Command{
 	Short:        "Update repository",
 	Long:         `Update repository credentials or webhook secret.`,
 	SilenceUsage: true,
-	RunE: func(_ *cobra.Command, args []string) error {
+	RunE: func(cmd *cobra.Command, args []string) error {
 		if needsInit {
 			return errNeedsInitError
 		}
@@ -270,6 +272,9 @@ var repoUpdateCmd = &cobra.Command{
 			CredentialsName:  repoCreds,
 			PoolBalancerType: params.PoolBalancerType(poolBalancerType),
 		}
+		if cmd.Flags().Changed("agent-mode") {
+			updateReposReq.Body.AgentMode = &repoAgentMode
+		}
 		updateReposReq.RepoID = repoID
 
 		response, err := apiCli.Repositories.UpdateRepo(updateReposReq, authToken)
@@ -354,6 +359,7 @@ func init() {
 	repoAddCmd.Flags().StringVar(&repoCreds, "credentials", "", "Credentials name. See credentials list.")
 	repoAddCmd.Flags().BoolVar(&randomWebhookSecret, "random-webhook-secret", false, "Generate a random webhook secret for this repository.")
 	repoAddCmd.Flags().BoolVar(&installRepoWebhook, "install-webhook", false, "Install the webhook as part of the add operation.")
+	repoAddCmd.Flags().BoolVar(&repoAgentMode, "agent-mode", false, "Enable agent mode for runners in this repository.")
 	repoAddCmd.MarkFlagsMutuallyExclusive("webhook-secret", "random-webhook-secret")
 	repoAddCmd.MarkFlagsOneRequired("webhook-secret", "random-webhook-secret")
 
@@ -374,6 +380,7 @@ func init() {
 	repoUpdateCmd.Flags().StringVar(&repoWebhookSecret, "webhook-secret", "", "The webhook secret for this repository. If you update this secret, you will have to manually update the secret in GitHub as well.")
 	repoUpdateCmd.Flags().StringVar(&repoCreds, "credentials", "", "Credentials name. See credentials list.")
 	repoUpdateCmd.Flags().StringVar(&poolBalancerType, "pool-balancer-type", "", "The balancing strategy to use when creating runners in pools matching requested labels.")
+	repoUpdateCmd.Flags().BoolVar(&repoAgentMode, "agent-mode", false, "Enable agent mode for runners in this repository.")
 	repoUpdateCmd.Flags().StringVar(&repoEndpoint, "endpoint", "", "When using the name of the repo, the endpoint must be specified when multiple repositories with the same name exist.")
 
 	repoWebhookInstallCmd.Flags().BoolVar(&insecureRepoWebhook, "insecure", false, "Ignore self signed certificate errors.")

cmd/garm-cli/cmd/runner.go

@@ -15,18 +15,27 @@
 package cmd
 
 import (
+	"context"
 	"fmt"
 	"os"
+	"os/signal"
+	"sync"
+	"syscall"
 
+	"github.com/google/uuid"
+	"github.com/gorilla/websocket"
 	"github.com/jedib0t/go-pretty/v6/table"
 	"github.com/spf13/cobra"
+	"golang.org/x/term"
 
+	garmWs "github.com/cloudbase/garm-provider-common/util/websocket"
 	apiClientEnterprises "github.com/cloudbase/garm/client/enterprises"
 	apiClientInstances "github.com/cloudbase/garm/client/instances"
 	apiClientOrgs "github.com/cloudbase/garm/client/organizations"
 	apiClientRepos "github.com/cloudbase/garm/client/repositories"
 	"github.com/cloudbase/garm/cmd/garm-cli/common"
 	"github.com/cloudbase/garm/params"
+	"github.com/cloudbase/garm/workers/websocket/agent/messaging"
 )
 
 var (
@@ -50,6 +59,165 @@ list all instances.`,
 	Run: nil,
 }
 
+type handlerErr struct {
+	done chan struct{}
+	once sync.Once
+}
+
+func (h *handlerErr) Close() {
+	h.once.Do(func() { close(h.done) })
+}
+
+var agentShellCmd = &cobra.Command{
+	Use:          "shell",
+	Short:        "Execute an interactive shell",
+	Long:         `Execute an interactive shell on the runner.`,
+	SilenceUsage: true,
+	RunE: func(_ *cobra.Command, args []string) error {
+		if needsInit {
+			return errNeedsInitError
+		}
+
+		if len(args) != 1 {
+			return fmt.Errorf("requires a runner name")
+		}
+
+		var sessionID uuid.UUID
+
+		handlerErr := handlerErr{
+			done: make(chan struct{}),
+		}
+		resizeCh := make(chan [2]int, 1)
+		defer close(resizeCh)
+		handler := func(msgType int, msg []byte) error {
+			switch msgType {
+			case websocket.CloseAbnormalClosure, websocket.CloseGoingAway, websocket.CloseMessage:
+				os.Stderr.Write([]byte("remote server closed the connection"))
+				handlerErr.Close()
+			case websocket.BinaryMessage, websocket.TextMessage:
+				agentMsg, err := messaging.UnmarshalAgentMessage(msg)
+				if err != nil {
+					os.Stderr.Write([]byte("failed to unmarshal message"))
+					handlerErr.Close()
+				}
+				switch agentMsg.Type {
+				case messaging.MessageTypeShellReady:
+					shellReady, err := messaging.Unmarshal[messaging.ShellReadyMessage](agentMsg)
+					if err != nil {
+						os.Stderr.Write(fmt.Appendf(nil, "failed to unmarshal shell ready: %q", err))
+						handlerErr.Close()
+					}
+					sessionID = shellReady.SessionID
+					if shellReady.IsError == 1 {
+						if len(shellReady.Message) > 0 {
+							os.Stderr.Write(fmt.Appendf(shellReady.Message, "\r\n"))
+						}
+						handlerErr.Close()
+						return nil
+					}
+					if w, h, err := term.GetSize(int(os.Stdin.Fd())); err == nil {
+						resizeCh <- [2]int{w, h}
+					}
+				case messaging.MessageTypeShellExit:
+					handlerErr.Close()
+				case messaging.MessageTypeShellData:
+					shellData, err := messaging.Unmarshal[messaging.ShellDataMessage](agentMsg)
+					if err != nil {
+						os.Stderr.Write([]byte("failed to unmarshal shell data message"))
+						handlerErr.Close()
+					}
+					os.Stdout.Write(shellData.Data)
+				default:
+					os.Stdout.Write(fmt.Appendf(nil, "invalid agentMsg.Type: %v", agentMsg.Type))
+				}
+			default:
+				os.Stdout.Write(fmt.Appendf(nil, "invalid message type: %v", msgType))
+			}
+			return nil
+		}
+
+		// Put terminal in raw mode
+		oldState, err := term.MakeRaw(int(os.Stdin.Fd()))
+		if err != nil {
+			return err
+		}
+		defer term.Restore(int(os.Stdin.Fd()), oldState)
+		// Channel to stop on Ctrl+C
+		sigch := make(chan os.Signal, 1)
+		signal.Notify(sigch, os.Interrupt, syscall.SIGTERM)
+
+		ctx, stop := signal.NotifyContext(context.Background(), signals...)
+		defer stop()
+
+		reader, err := garmWs.NewReader(ctx, mgr.BaseURL, fmt.Sprintf("/api/v1/ws/agent/%s/shell", args[0]), mgr.Token, handler)
+		if err != nil {
+			return err
+		}
+
+		if err := reader.Start(); err != nil {
+			return err
+		}
+
+		go func() {
+			buf := make([]byte, 1024)
+			for {
+				n, err := os.Stdin.Read(buf)
+				if err != nil {
+					os.Stderr.Write(fmt.Appendf(nil, "failed to write message: %q", err))
+					handlerErr.Close()
+					return
+				}
+
+				if n > 0 && sessionID != uuid.Nil {
+					msg := messaging.ShellDataMessage{
+						SessionID: sessionID,
+						Data:      buf[:n],
+					}
+					if err := reader.WriteMessage(websocket.BinaryMessage, msg.Marshal()); err != nil {
+						os.Stderr.Write(fmt.Appendf(nil, "failed to write message: %q", err))
+						handlerErr.Close()
+						return
+					}
+				}
+			}
+		}()
+
+		// ---- Watch terminal resize ----
+		go watchTermResize(ctx, resizeCh, sessionID)
+
+		// ---- Send resize messages ----
+		go func() {
+			for {
+				select {
+				case size := <-resizeCh:
+					if sessionID == uuid.Nil {
+						continue
+					}
+					msg := messaging.ShellResizeMessage{
+						SessionID: sessionID,
+						Cols:      uint16(size[0]),
+						Rows:      uint16(size[1]),
+					}
+					reader.WriteMessage(websocket.BinaryMessage, msg.Marshal())
+				case <-ctx.Done():
+					return
+				case <-reader.Done():
+					return
+				case <-handlerErr.done:
+					return
+				}
+			}
+		}()
+
+		select {
+		case <-ctx.Done():
+		case <-reader.Done():
+		case <-handlerErr.done:
+		}
+		return nil
+	},
+}
+
 type instancesPayloadGetter interface {
 	GetPayload() params.Instances
 }
@@ -59,7 +227,7 @@ var runnerListCmd = &cobra.Command{
 	Aliases: []string{"ls"},
 	Short:   "List runners",
 	Long: `List runners of pools, repositories, orgs or all of the above.
-	
+
 This command expects to get either a pool ID as a positional parameter, or it expects
 that one of the supported switches be used to fetch runners of --repo, --org or --all
 
@@ -229,6 +397,7 @@ func init() {
 		runnerListCmd,
 		runnerShowCmd,
 		runnerDeleteCmd,
+		agentShellCmd,
 	)
 
 	rootCmd.AddCommand(runnerCmd)
@@ -282,6 +451,8 @@ func formatSingleInstance(instance params.Instance) {
 	t.AppendRow(table.Row{"OS Version", instance.OSVersion}, table.RowConfig{AutoMerge: false})
 	t.AppendRow(table.Row{"Status", instance.Status}, table.RowConfig{AutoMerge: false})
 	t.AppendRow(table.Row{"Runner Status", instance.RunnerStatus}, table.RowConfig{AutoMerge: false})
+	t.AppendRow(table.Row{"Capabilities", fmt.Sprintf("Shell: %v", instance.Capabilities.Shell)}, table.RowConfig{AutoMerge: true})
+
 	if instance.PoolID != "" {
 		t.AppendRow(table.Row{"Pool ID", instance.PoolID}, table.RowConfig{AutoMerge: false})
 	} else if instance.ScaleSetID != 0 {

cmd/garm-cli/cmd/scalesets.go

@@ -49,6 +49,7 @@ var (
 	scalesetExtraSpecs             string
 	scalesetGitHubRunnerGroup      string
 	scaleSetTemplateNameOrID       string
+	scalesetEnableShell            bool
 )
 
 type scalesetPayloadGetter interface {
@@ -228,6 +229,7 @@ var scaleSetAddCmd = &cobra.Command{
 			Enabled:                scalesetEnabled,
 			RunnerBootstrapTimeout: scalesetRunnerBootstrapTimeout,
 			GitHubRunnerGroup:      scalesetGitHubRunnerGroup,
+			EnableShell:            scalesetEnableShell,
 		}
 
 		if cmd.Flags().Changed("extra-specs") {
@@ -381,6 +383,10 @@ explicitly remove them using the runner delete command.
 			scaleSetUpdateParams.RunnerBootstrapTimeout = &scalesetRunnerBootstrapTimeout
 		}
 
+		if cmd.Flags().Changed("enable-shell") {
+			scaleSetUpdateParams.EnableShell = &scalesetEnableShell
+		}
+
 		if cmd.Flags().Changed("extra-specs") {
 			data, err := asRawMessage([]byte(scalesetExtraSpecs))
 			if err != nil {
@@ -429,6 +435,7 @@ func init() {
 	scaleSetUpdateCmd.Flags().UintVar(&scalesetRunnerBootstrapTimeout, "runner-bootstrap-timeout", 20, "Duration in minutes after which a runner is considered failed if it does not join Github.")
 	scaleSetUpdateCmd.Flags().StringVar(&scalesetExtraSpecsFile, "extra-specs-file", "", "A file containing a valid json which will be passed to the IaaS provider managing the scale set.")
 	scaleSetUpdateCmd.Flags().StringVar(&scalesetExtraSpecs, "extra-specs", "", "A valid json which will be passed to the IaaS provider managing the scale set.")
+	scaleSetUpdateCmd.Flags().BoolVar(&scalesetEnableShell, "enable-shell", false, "Enable shell access for runners in this scale set.")
 	scaleSetUpdateCmd.Flags().StringVar(&scaleSetTemplateNameOrID, "runner-install-template", "", "The runner install template name or ID to use for this scale set.")
 	scaleSetUpdateCmd.MarkFlagsMutuallyExclusive("extra-specs-file", "extra-specs")
 
@@ -446,6 +453,7 @@ func init() {
 	scaleSetAddCmd.Flags().UintVar(&scalesetRunnerBootstrapTimeout, "runner-bootstrap-timeout", 20, "Duration in minutes after which a runner is considered failed if it does not join Github.")
 	scaleSetAddCmd.Flags().UintVar(&scalesetMinIdleRunners, "min-idle-runners", 1, "Attempt to maintain a minimum of idle self-hosted runners of this type.")
 	scaleSetAddCmd.Flags().BoolVar(&scalesetEnabled, "enabled", false, "Enable this scale set.")
+	scaleSetAddCmd.Flags().BoolVar(&scalesetEnableShell, "enable-shell", false, "Enable shell access for runners in this scale set.")
 	scaleSetAddCmd.Flags().StringVar(&endpointName, "endpoint", "", "When using the name of an entity, the endpoint must be specified when multiple entities with the same name exist.")
 	scaleSetAddCmd.Flags().StringVar(&scaleSetTemplateNameOrID, "runner-install-template", "", "The runner install template name or ID to use for this scale set.")
 	scaleSetAddCmd.MarkFlagRequired("provider-name") //nolint

cmd/garm/main.go

@@ -52,6 +52,7 @@ import (
 	"github.com/cloudbase/garm/workers/cache"
 	"github.com/cloudbase/garm/workers/entity"
 	"github.com/cloudbase/garm/workers/provider"
+	"github.com/cloudbase/garm/workers/websocket/agent"
 )
 
 var (
@@ -220,6 +221,15 @@ func main() {
 		log.Fatal(err)
 	}
 
+	agentHub, err := agent.NewHub(ctx)
+	if err != nil {
+		log.Fatalf("failed to create agent hub: %q", err)
+	}
+
+	if err := agentHub.Start(); err != nil {
+		log.Fatalf("failed to start agent hub: %q", err)
+	}
+
 	// Local locker for now. Will be configurable in the future,
 	// as we add scale-out capability to GARM.
 	lock, err := locking.NewLocalLocker(ctx, db)
@@ -277,7 +287,7 @@ func main() {
 	}
 
 	authenticator := auth.NewAuthenticator(cfg.JWTAuth, db)
-	controller, err := controllers.NewAPIController(runner, authenticator, hub, cfg.APIServer)
+	controller, err := controllers.NewAPIController(runner, authenticator, hub, agentHub, cfg.APIServer)
 	if err != nil {
 		log.Fatalf("failed to create controller: %+v", err)
 	}
@@ -306,11 +316,16 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+	agentMiddleware, err := auth.AgentMiddleware(db, cfg.JWTAuth)
+	if err != nil {
+		log.Fatal(err)
+	}
 
 	router := routers.NewAPIRouter(controller, jwtMiddleware, initMiddleware, urlsRequiredMiddleware, instanceMiddleware, cfg.Default.EnableWebhookManagement)
 
 	// Add WebUI routes
 	router = routers.WithWebUI(router, cfg.APIServer)
+	router = routers.WithAgentRouter(router, controller, agentMiddleware)
 
 	// start the metrics collector
 	if cfg.Metrics.Enable {

database/common/mocks/Store.go

@@ -229,9 +229,9 @@ func (_c *Store_ControllerInfo_Call) RunAndReturn(run func() (params.ControllerI
 	return _c
 }
 
-// CreateEnterprise provides a mock function with given fields: ctx, name, credentialsName, webhookSecret, poolBalancerType
-func (_m *Store) CreateEnterprise(ctx context.Context, name string, credentialsName params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Enterprise, error) {
-	ret := _m.Called(ctx, name, credentialsName, webhookSecret, poolBalancerType)
+// CreateEnterprise provides a mock function with given fields: ctx, name, credentialsName, webhookSecret, poolBalancerType, agentMode
+func (_m *Store) CreateEnterprise(ctx context.Context, name string, credentialsName params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool) (params.Enterprise, error) {
+	ret := _m.Called(ctx, name, credentialsName, webhookSecret, poolBalancerType, agentMode)
 
 	if len(ret) == 0 {
 		panic("no return value specified for CreateEnterprise")
@@ -239,17 +239,17 @@ func (_m *Store) CreateEnterprise(ctx context.Context, name string, credentialsN
 
 	var r0 params.Enterprise
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) (params.Enterprise, error)); ok {
-		return rf(ctx, name, credentialsName, webhookSecret, poolBalancerType)
+	if rf, ok := ret.Get(0).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) (params.Enterprise, error)); ok {
+		return rf(ctx, name, credentialsName, webhookSecret, poolBalancerType, agentMode)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) params.Enterprise); ok {
-		r0 = rf(ctx, name, credentialsName, webhookSecret, poolBalancerType)
+	if rf, ok := ret.Get(0).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) params.Enterprise); ok {
+		r0 = rf(ctx, name, credentialsName, webhookSecret, poolBalancerType, agentMode)
 	} else {
 		r0 = ret.Get(0).(params.Enterprise)
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) error); ok {
-		r1 = rf(ctx, name, credentialsName, webhookSecret, poolBalancerType)
+	if rf, ok := ret.Get(1).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) error); ok {
+		r1 = rf(ctx, name, credentialsName, webhookSecret, poolBalancerType, agentMode)
 	} else {
 		r1 = ret.Error(1)
 	}
@@ -268,13 +268,14 @@ type Store_CreateEnterprise_Call struct {
 //   - credentialsName params.ForgeCredentials
 //   - webhookSecret string
 //   - poolBalancerType params.PoolBalancerType
-func (_e *Store_Expecter) CreateEnterprise(ctx interface{}, name interface{}, credentialsName interface{}, webhookSecret interface{}, poolBalancerType interface{}) *Store_CreateEnterprise_Call {
-	return &Store_CreateEnterprise_Call{Call: _e.mock.On("CreateEnterprise", ctx, name, credentialsName, webhookSecret, poolBalancerType)}
+//   - agentMode bool
+func (_e *Store_Expecter) CreateEnterprise(ctx interface{}, name interface{}, credentialsName interface{}, webhookSecret interface{}, poolBalancerType interface{}, agentMode interface{}) *Store_CreateEnterprise_Call {
+	return &Store_CreateEnterprise_Call{Call: _e.mock.On("CreateEnterprise", ctx, name, credentialsName, webhookSecret, poolBalancerType, agentMode)}
 }
 
-func (_c *Store_CreateEnterprise_Call) Run(run func(ctx context.Context, name string, credentialsName params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType)) *Store_CreateEnterprise_Call {
+func (_c *Store_CreateEnterprise_Call) Run(run func(ctx context.Context, name string, credentialsName params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool)) *Store_CreateEnterprise_Call {
 	_c.Call.Run(func(args mock.Arguments) {
-		run(args[0].(context.Context), args[1].(string), args[2].(params.ForgeCredentials), args[3].(string), args[4].(params.PoolBalancerType))
+		run(args[0].(context.Context), args[1].(string), args[2].(params.ForgeCredentials), args[3].(string), args[4].(params.PoolBalancerType), args[5].(bool))
 	})
 	return _c
 }
@@ -284,7 +285,7 @@ func (_c *Store_CreateEnterprise_Call) Return(_a0 params.Enterprise, _a1 error)
 	return _c
 }
 
-func (_c *Store_CreateEnterprise_Call) RunAndReturn(run func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) (params.Enterprise, error)) *Store_CreateEnterprise_Call {
+func (_c *Store_CreateEnterprise_Call) RunAndReturn(run func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) (params.Enterprise, error)) *Store_CreateEnterprise_Call {
 	_c.Call.Return(run)
 	return _c
 }
@@ -806,9 +807,9 @@ func (_c *Store_CreateOrUpdateJob_Call) RunAndReturn(run func(context.Context, p
 	return _c
 }
 
-// CreateOrganization provides a mock function with given fields: ctx, name, credentials, webhookSecret, poolBalancerType
-func (_m *Store) CreateOrganization(ctx context.Context, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Organization, error) {
-	ret := _m.Called(ctx, name, credentials, webhookSecret, poolBalancerType)
+// CreateOrganization provides a mock function with given fields: ctx, name, credentials, webhookSecret, poolBalancerType, agentMode
+func (_m *Store) CreateOrganization(ctx context.Context, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool) (params.Organization, error) {
+	ret := _m.Called(ctx, name, credentials, webhookSecret, poolBalancerType, agentMode)
 
 	if len(ret) == 0 {
 		panic("no return value specified for CreateOrganization")
@@ -816,17 +817,17 @@ func (_m *Store) CreateOrganization(ctx context.Context, name string, credential
 
 	var r0 params.Organization
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) (params.Organization, error)); ok {
-		return rf(ctx, name, credentials, webhookSecret, poolBalancerType)
+	if rf, ok := ret.Get(0).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) (params.Organization, error)); ok {
+		return rf(ctx, name, credentials, webhookSecret, poolBalancerType, agentMode)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) params.Organization); ok {
-		r0 = rf(ctx, name, credentials, webhookSecret, poolBalancerType)
+	if rf, ok := ret.Get(0).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) params.Organization); ok {
+		r0 = rf(ctx, name, credentials, webhookSecret, poolBalancerType, agentMode)
 	} else {
 		r0 = ret.Get(0).(params.Organization)
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) error); ok {
-		r1 = rf(ctx, name, credentials, webhookSecret, poolBalancerType)
+	if rf, ok := ret.Get(1).(func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) error); ok {
+		r1 = rf(ctx, name, credentials, webhookSecret, poolBalancerType, agentMode)
 	} else {
 		r1 = ret.Error(1)
 	}
@@ -845,13 +846,14 @@ type Store_CreateOrganization_Call struct {
 //   - credentials params.ForgeCredentials
 //   - webhookSecret string
 //   - poolBalancerType params.PoolBalancerType
-func (_e *Store_Expecter) CreateOrganization(ctx interface{}, name interface{}, credentials interface{}, webhookSecret interface{}, poolBalancerType interface{}) *Store_CreateOrganization_Call {
-	return &Store_CreateOrganization_Call{Call: _e.mock.On("CreateOrganization", ctx, name, credentials, webhookSecret, poolBalancerType)}
+//   - agentMode bool
+func (_e *Store_Expecter) CreateOrganization(ctx interface{}, name interface{}, credentials interface{}, webhookSecret interface{}, poolBalancerType interface{}, agentMode interface{}) *Store_CreateOrganization_Call {
+	return &Store_CreateOrganization_Call{Call: _e.mock.On("CreateOrganization", ctx, name, credentials, webhookSecret, poolBalancerType, agentMode)}
 }
 
-func (_c *Store_CreateOrganization_Call) Run(run func(ctx context.Context, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType)) *Store_CreateOrganization_Call {
+func (_c *Store_CreateOrganization_Call) Run(run func(ctx context.Context, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool)) *Store_CreateOrganization_Call {
 	_c.Call.Run(func(args mock.Arguments) {
-		run(args[0].(context.Context), args[1].(string), args[2].(params.ForgeCredentials), args[3].(string), args[4].(params.PoolBalancerType))
+		run(args[0].(context.Context), args[1].(string), args[2].(params.ForgeCredentials), args[3].(string), args[4].(params.PoolBalancerType), args[5].(bool))
 	})
 	return _c
 }
@@ -861,14 +863,14 @@ func (_c *Store_CreateOrganization_Call) Return(org params.Organization, err err
 	return _c
 }
 
-func (_c *Store_CreateOrganization_Call) RunAndReturn(run func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType) (params.Organization, error)) *Store_CreateOrganization_Call {
+func (_c *Store_CreateOrganization_Call) RunAndReturn(run func(context.Context, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) (params.Organization, error)) *Store_CreateOrganization_Call {
 	_c.Call.Return(run)
 	return _c
 }
 
-// CreateRepository provides a mock function with given fields: ctx, owner, name, credentials, webhookSecret, poolBalancerType
-func (_m *Store) CreateRepository(ctx context.Context, owner string, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Repository, error) {
-	ret := _m.Called(ctx, owner, name, credentials, webhookSecret, poolBalancerType)
+// CreateRepository provides a mock function with given fields: ctx, owner, name, credentials, webhookSecret, poolBalancerType, agentMode
+func (_m *Store) CreateRepository(ctx context.Context, owner string, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool) (params.Repository, error) {
+	ret := _m.Called(ctx, owner, name, credentials, webhookSecret, poolBalancerType, agentMode)
 
 	if len(ret) == 0 {
 		panic("no return value specified for CreateRepository")
@@ -876,17 +878,17 @@ func (_m *Store) CreateRepository(ctx context.Context, owner string, name string
 
 	var r0 params.Repository
 	var r1 error
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, params.ForgeCredentials, string, params.PoolBalancerType) (params.Repository, error)); ok {
-		return rf(ctx, owner, name, credentials, webhookSecret, poolBalancerType)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) (params.Repository, error)); ok {
+		return rf(ctx, owner, name, credentials, webhookSecret, poolBalancerType, agentMode)
 	}
-	if rf, ok := ret.Get(0).(func(context.Context, string, string, params.ForgeCredentials, string, params.PoolBalancerType) params.Repository); ok {
-		r0 = rf(ctx, owner, name, credentials, webhookSecret, poolBalancerType)
+	if rf, ok := ret.Get(0).(func(context.Context, string, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) params.Repository); ok {
+		r0 = rf(ctx, owner, name, credentials, webhookSecret, poolBalancerType, agentMode)
 	} else {
 		r0 = ret.Get(0).(params.Repository)
 	}
 
-	if rf, ok := ret.Get(1).(func(context.Context, string, string, params.ForgeCredentials, string, params.PoolBalancerType) error); ok {
-		r1 = rf(ctx, owner, name, credentials, webhookSecret, poolBalancerType)
+	if rf, ok := ret.Get(1).(func(context.Context, string, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) error); ok {
+		r1 = rf(ctx, owner, name, credentials, webhookSecret, poolBalancerType, agentMode)
 	} else {
 		r1 = ret.Error(1)
 	}
@@ -906,13 +908,14 @@ type Store_CreateRepository_Call struct {
 //   - credentials params.ForgeCredentials
 //   - webhookSecret string
 //   - poolBalancerType params.PoolBalancerType
-func (_e *Store_Expecter) CreateRepository(ctx interface{}, owner interface{}, name interface{}, credentials interface{}, webhookSecret interface{}, poolBalancerType interface{}) *Store_CreateRepository_Call {
-	return &Store_CreateRepository_Call{Call: _e.mock.On("CreateRepository", ctx, owner, name, credentials, webhookSecret, poolBalancerType)}
+//   - agentMode bool
+func (_e *Store_Expecter) CreateRepository(ctx interface{}, owner interface{}, name interface{}, credentials interface{}, webhookSecret interface{}, poolBalancerType interface{}, agentMode interface{}) *Store_CreateRepository_Call {
+	return &Store_CreateRepository_Call{Call: _e.mock.On("CreateRepository", ctx, owner, name, credentials, webhookSecret, poolBalancerType, agentMode)}
 }
 
-func (_c *Store_CreateRepository_Call) Run(run func(ctx context.Context, owner string, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType)) *Store_CreateRepository_Call {
+func (_c *Store_CreateRepository_Call) Run(run func(ctx context.Context, owner string, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool)) *Store_CreateRepository_Call {
 	_c.Call.Run(func(args mock.Arguments) {
-		run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(params.ForgeCredentials), args[4].(string), args[5].(params.PoolBalancerType))
+		run(args[0].(context.Context), args[1].(string), args[2].(string), args[3].(params.ForgeCredentials), args[4].(string), args[5].(params.PoolBalancerType), args[6].(bool))
 	})
 	return _c
 }
@@ -922,7 +925,7 @@ func (_c *Store_CreateRepository_Call) Return(param params.Repository, err error
 	return _c
 }
 
-func (_c *Store_CreateRepository_Call) RunAndReturn(run func(context.Context, string, string, params.ForgeCredentials, string, params.PoolBalancerType) (params.Repository, error)) *Store_CreateRepository_Call {
+func (_c *Store_CreateRepository_Call) RunAndReturn(run func(context.Context, string, string, params.ForgeCredentials, string, params.PoolBalancerType, bool) (params.Repository, error)) *Store_CreateRepository_Call {
 	_c.Call.Return(run)
 	return _c
 }

database/common/store.go

@@ -40,7 +40,7 @@ type GithubCredentialsStore interface {
 }
 
 type RepoStore interface {
-	CreateRepository(ctx context.Context, owner, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType) (param params.Repository, err error)
+	CreateRepository(ctx context.Context, owner, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool) (param params.Repository, err error)
 	GetRepository(ctx context.Context, owner, name, endpointName string) (params.Repository, error)
 	GetRepositoryByID(ctx context.Context, repoID string) (params.Repository, error)
 	ListRepositories(ctx context.Context, filter params.RepositoryFilter) ([]params.Repository, error)
@@ -49,7 +49,7 @@ type RepoStore interface {
 }
 
 type OrgStore interface {
-	CreateOrganization(ctx context.Context, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType) (org params.Organization, err error)
+	CreateOrganization(ctx context.Context, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool) (org params.Organization, err error)
 	GetOrganization(ctx context.Context, name, endpointName string) (params.Organization, error)
 	GetOrganizationByID(ctx context.Context, orgID string) (params.Organization, error)
 	ListOrganizations(ctx context.Context, filter params.OrganizationFilter) ([]params.Organization, error)
@@ -58,7 +58,7 @@ type OrgStore interface {
 }
 
 type EnterpriseStore interface {
-	CreateEnterprise(ctx context.Context, name string, credentialsName params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType) (params.Enterprise, error)
+	CreateEnterprise(ctx context.Context, name string, credentialsName params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool) (params.Enterprise, error)
 	GetEnterprise(ctx context.Context, name, endpointName string) (params.Enterprise, error)
 	GetEnterpriseByID(ctx context.Context, enterpriseID string) (params.Enterprise, error)
 	ListEnterprises(ctx context.Context, filter params.EnterpriseFilter) ([]params.Enterprise, error)
@@ -135,6 +135,7 @@ type ControllerStore interface {
 	ControllerInfo() (params.ControllerInfo, error)
 	InitController() (params.ControllerInfo, error)
 	UpdateController(info params.UpdateControllerParams) (params.ControllerInfo, error)
+	HasEntitiesWithAgentModeEnabled() (bool, error)
 }
 
 type ScaleSetsStore interface {
@@ -186,6 +187,7 @@ type FileObjectStore interface {
 	CreateFileObject(ctx context.Context, param params.CreateFileObjectParams, reader io.Reader) (fileObjParam params.FileObject, err error)
 	UpdateFileObject(ctx context.Context, objID uint, param params.UpdateFileObjectParams) (params.FileObject, error)
 	DeleteFileObject(ctx context.Context, objID uint) error
+	DeleteFileObjectsByTags(ctx context.Context, tags []string) (int64, error)
 	OpenFileObjectContent(ctx context.Context, objID uint) (io.ReadCloser, error)
 }
 

database/sql/controller.go

@@ -34,15 +34,23 @@ func dbControllerToCommonController(dbInfo ControllerInfo) (params.ControllerInf
 		return params.ControllerInfo{}, fmt.Errorf("error joining webhook URL: %w", err)
 	}
 
-	return params.ControllerInfo{
+	if dbInfo.GARMAgentReleasesURL == "" {
+		dbInfo.GARMAgentReleasesURL = appdefaults.GARMAgentDefaultReleasesURL
+	}
+
+	ret := params.ControllerInfo{
 		ControllerID:         dbInfo.ControllerID,
 		MetadataURL:          dbInfo.MetadataURL,
 		WebhookURL:           dbInfo.WebhookBaseURL,
 		ControllerWebhookURL: url,
 		CallbackURL:          dbInfo.CallbackURL,
+		AgentURL:             dbInfo.AgentURL,
 		MinimumJobAgeBackoff: dbInfo.MinimumJobAgeBackoff,
 		Version:              appdefaults.GetVersion(),
-	}, nil
+		GARMAgentReleasesURL: dbInfo.GARMAgentReleasesURL,
+		SyncGARMAgentTools:   dbInfo.SyncGARMAgentTools,
+	}
+	return ret, nil
 }
 
 func (s *sqlDatabase) ControllerInfo() (params.ControllerInfo, error) {
@@ -63,6 +71,24 @@ func (s *sqlDatabase) ControllerInfo() (params.ControllerInfo, error) {
 	return paramInfo, nil
 }
 
+func (s *sqlDatabase) HasEntitiesWithAgentModeEnabled() (bool, error) {
+	var reposCnt int64
+	if err := s.conn.Model(&Repository{}).Where("agent_mode = ?", true).Count(&reposCnt).Error; err != nil {
+		return false, fmt.Errorf("error fetching repo count: %w", err)
+	}
+
+	var orgCount int64
+	if err := s.conn.Model(&Organization{}).Where("agent_mode = ?", true).Count(&orgCount).Error; err != nil {
+		return false, fmt.Errorf("error fetching repo count: %w", err)
+	}
+
+	var enterpriseCount int64
+	if err := s.conn.Model(&Enterprise{}).Where("agent_mode = ?", true).Count(&enterpriseCount).Error; err != nil {
+		return false, fmt.Errorf("error fetching repo count: %w", err)
+	}
+	return reposCnt+orgCount+enterpriseCount > 0, nil
+}
+
 func (s *sqlDatabase) InitController() (params.ControllerInfo, error) {
 	if _, err := s.ControllerInfo(); err == nil {
 		return params.ControllerInfo{}, runnerErrors.NewConflictError("controller already initialized")
@@ -76,6 +102,7 @@ func (s *sqlDatabase) InitController() (params.ControllerInfo, error) {
 	newInfo := ControllerInfo{
 		ControllerID:         newID,
 		MinimumJobAgeBackoff: 30,
+		GARMAgentReleasesURL: appdefaults.GARMAgentDefaultReleasesURL,
 	}
 
 	q := s.conn.Save(&newInfo)
@@ -120,6 +147,22 @@ func (s *sqlDatabase) UpdateController(info params.UpdateControllerParams) (para
 			dbInfo.WebhookBaseURL = *info.WebhookURL
 		}
 
+		if info.AgentURL != nil {
+			dbInfo.AgentURL = *info.AgentURL
+		}
+
+		if info.GARMAgentReleasesURL != nil {
+			agentToolsURL := *info.GARMAgentReleasesURL
+			if agentToolsURL == "" {
+				agentToolsURL = appdefaults.GARMAgentDefaultReleasesURL
+			}
+			dbInfo.GARMAgentReleasesURL = agentToolsURL
+		}
+
+		if info.SyncGARMAgentTools != nil {
+			dbInfo.SyncGARMAgentTools = *info.SyncGARMAgentTools
+		}
+
 		if info.MinimumJobAgeBackoff != nil {
 			dbInfo.MinimumJobAgeBackoff = *info.MinimumJobAgeBackoff
 		}

database/sql/enterprise.go

@@ -29,7 +29,7 @@ import (
 	"github.com/cloudbase/garm/params"
 )
 
-func (s *sqlDatabase) CreateEnterprise(ctx context.Context, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType) (paramEnt params.Enterprise, err error) {
+func (s *sqlDatabase) CreateEnterprise(ctx context.Context, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool) (paramEnt params.Enterprise, err error) {
 	if webhookSecret == "" {
 		return params.Enterprise{}, errors.New("creating enterprise: missing secret")
 	}
@@ -51,6 +51,7 @@ func (s *sqlDatabase) CreateEnterprise(ctx context.Context, name string, credent
 		Name:             name,
 		WebhookSecret:    secret,
 		PoolBalancerType: poolBalancerType,
+		AgentMode:        agentMode,
 	}
 	err = s.conn.Transaction(func(tx *gorm.DB) error {
 		newEnterprise.CredentialsID = &credentials.ID
@@ -211,6 +212,10 @@ func (s *sqlDatabase) UpdateEnterprise(ctx context.Context, enterpriseID string,
 			enterprise.PoolBalancerType = param.PoolBalancerType
 		}
 
+		if param.AgentMode != nil {
+			enterprise.AgentMode = *param.AgentMode
+		}
+
 		q := tx.Save(&enterprise)
 		if q.Error != nil {
 			return fmt.Errorf("error saving enterprise: %w", q.Error)

database/sql/enterprise_test.go

@@ -113,6 +113,7 @@ func (s *EnterpriseTestSuite) SetupTest() {
 			s.testCreds,
 			fmt.Sprintf("test-webhook-secret-%d", i),
 			params.PoolBalancerTypeRoundRobin,
+			false,
 		)
 		if err != nil {
 			s.FailNow(fmt.Sprintf("failed to create database object (test-enterprise-%d): %q", i, err))
@@ -191,7 +192,9 @@ func (s *EnterpriseTestSuite) TestCreateEnterprise() {
 		s.Fixtures.CreateEnterpriseParams.Name,
 		s.testCreds,
 		s.Fixtures.CreateEnterpriseParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 
 	// assertions
 	s.Require().Nil(err)
@@ -222,7 +225,9 @@ func (s *EnterpriseTestSuite) TestCreateEnterpriseInvalidDBPassphrase() {
 		s.Fixtures.CreateEnterpriseParams.Name,
 		s.testCreds,
 		s.Fixtures.CreateEnterpriseParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 
 	s.Require().NotNil(err)
 	s.Require().Equal("error encoding secret: invalid passphrase length (expected length 32 characters)", err.Error())
@@ -240,7 +245,9 @@ func (s *EnterpriseTestSuite) TestCreateEnterpriseDBCreateErr() {
 		s.Fixtures.CreateEnterpriseParams.Name,
 		s.testCreds,
 		s.Fixtures.CreateEnterpriseParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 
 	s.Require().NotNil(err)
 	s.Require().Equal("error creating enterprise: error creating enterprise: creating enterprise mock error", err.Error())
@@ -296,6 +303,7 @@ func (s *EnterpriseTestSuite) TestListEnterprisesWithFilter() {
 		s.ghesCreds,
 		"test-secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 
@@ -305,6 +313,7 @@ func (s *EnterpriseTestSuite) TestListEnterprisesWithFilter() {
 		s.testCreds,
 		"test-secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 
@@ -314,6 +323,7 @@ func (s *EnterpriseTestSuite) TestListEnterprisesWithFilter() {
 		s.testCreds,
 		"test-secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 	enterprises, err := s.Store.ListEnterprises(s.adminCtx, params.EnterpriseFilter{
@@ -844,7 +854,9 @@ func (s *EnterpriseTestSuite) TestAddRepoEntityEvent() {
 		s.Fixtures.CreateEnterpriseParams.Name,
 		s.testCreds,
 		s.Fixtures.CreateEnterpriseParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 
 	s.Require().Nil(err)
 	entity, err := enterprise.GetEntity()

database/sql/file_store.go

@@ -253,6 +253,63 @@ func (s *sqlDatabase) DeleteFileObject(_ context.Context, objID uint) (err error
 	return nil
 }
 
+func (s *sqlDatabase) DeleteFileObjectsByTags(_ context.Context, tags []string) (int64, error) {
+	if len(tags) == 0 {
+		return 0, fmt.Errorf("no tags provided")
+	}
+
+	var deletedCount int64
+
+	err := s.objectsConn.Transaction(func(tx *gorm.DB) error {
+		// Build query to find all file objects matching ALL tags
+		query := tx.Model(&FileObject{}).Preload("TagsList").Omit("content")
+		for _, tag := range tags {
+			query = query.Where("EXISTS (SELECT 1 FROM file_object_tags WHERE file_object_tags.file_object_id = file_objects.id AND file_object_tags.tag = ?)", tag)
+		}
+
+		// Get matching objects with their full details (except content blob)
+		var fileObjects []FileObject
+		if err := query.Find(&fileObjects).Error; err != nil {
+			return fmt.Errorf("failed to find matching objects: %w", err)
+		}
+
+		if len(fileObjects) == 0 {
+			// No objects match - not an error, just nothing to delete
+			return nil
+		}
+
+		// Extract IDs for deletion
+		fileObjIDs := make([]uint, len(fileObjects))
+		for i, obj := range fileObjects {
+			fileObjIDs[i] = obj.ID
+		}
+
+		// Delete all matching objects (hard delete with Unscoped)
+		result := tx.Unscoped().Where("id IN ?", fileObjIDs).Delete(&FileObject{})
+		if result.Error != nil {
+			return fmt.Errorf("failed to delete objects: %w", result.Error)
+		}
+
+		deletedCount = result.RowsAffected
+
+		// Send notifications with full object details for each deleted object
+		for _, obj := range fileObjects {
+			s.sendNotify(common.FileObjectEntityType, common.DeleteOperation, s.sqlFileObjectToCommonParams(obj))
+		}
+
+		return nil
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	// NOTE: Same as DeleteFileObject - deleted file objects leave empty space
+	// in the database. Users should run VACUUM manually to reclaim space.
+	// See DeleteFileObject for performance details.
+
+	return deletedCount, nil
+}
+
 func (s *sqlDatabase) GetFileObject(_ context.Context, objID uint) (params.FileObject, error) {
 	var fileObj FileObject
 	if err := s.objectsConn.Preload("TagsList").Where("id = ?", objID).Omit("content").First(&fileObj).Error; err != nil {
@@ -304,7 +361,7 @@ func (s *sqlDatabase) SearchFileObjectByTags(_ context.Context, tags []string, p
 	if err := query.
 		Limit(queryPageSize).
 		Offset(queryOffset).
-		Order("created_at DESC").
+		Order("id DESC").
 		Omit("content").
 		Find(&fileObjectRes).Error; err != nil {
 		return params.FileObjectPaginatedResponse{}, fmt.Errorf("failed to query database: %w", err)
@@ -426,7 +483,7 @@ func (s *sqlDatabase) ListFileObjects(_ context.Context, page, pageSize uint64)
 	if err := s.objectsConn.Preload("TagsList").Omit("content").
 		Limit(queryPageSize).
 		Offset(queryOffset).
-		Order("created_at DESC").
+		Order("id DESC").
 		Find(&fileObjs).Error; err != nil {
 		return params.FileObjectPaginatedResponse{}, fmt.Errorf("failed to list file objects: %w", err)
 	}

database/sql/gitea_test.go

@@ -529,7 +529,7 @@ func (s *GiteaTestSuite) TestDeleteCredentialsFailsIfReposOrgsOrEntitiesUseIt()
 	s.Require().NoError(err)
 	s.Require().NotNil(creds)
 
-	repo, err := s.db.CreateRepository(ctx, "test-owner", "test-repo", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.db.CreateRepository(ctx, "test-owner", "test-repo", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(repo)
 
@@ -540,7 +540,7 @@ func (s *GiteaTestSuite) TestDeleteCredentialsFailsIfReposOrgsOrEntitiesUseIt()
 	err = s.db.DeleteRepository(ctx, repo.ID)
 	s.Require().NoError(err)
 
-	org, err := s.db.CreateOrganization(ctx, "test-org", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	org, err := s.db.CreateOrganization(ctx, "test-org", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(org)
 
@@ -551,7 +551,7 @@ func (s *GiteaTestSuite) TestDeleteCredentialsFailsIfReposOrgsOrEntitiesUseIt()
 	err = s.db.DeleteOrganization(ctx, org.ID)
 	s.Require().NoError(err)
 
-	enterprise, err := s.db.CreateEnterprise(ctx, "test-enterprise", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	enterprise, err := s.db.CreateEnterprise(ctx, "test-enterprise", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().ErrorIs(err, runnerErrors.ErrBadRequest)
 	s.Require().Equal(params.Enterprise{}, enterprise)
 
@@ -685,7 +685,7 @@ func (s *GiteaTestSuite) TestDeleteCredentialsWithOrgsOrReposFails() {
 	s.Require().NoError(err)
 	s.Require().NotNil(creds)
 
-	repo, err := s.db.CreateRepository(ctx, "test-owner", "test-repo", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.db.CreateRepository(ctx, "test-owner", "test-repo", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(repo)
 
@@ -696,7 +696,7 @@ func (s *GiteaTestSuite) TestDeleteCredentialsWithOrgsOrReposFails() {
 	err = s.db.DeleteRepository(ctx, repo.ID)
 	s.Require().NoError(err)
 
-	org, err := s.db.CreateOrganization(ctx, "test-org", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	org, err := s.db.CreateOrganization(ctx, "test-org", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(org)
 
@@ -743,7 +743,7 @@ func (s *GiteaTestSuite) TestDeleteGiteaEndpointFailsWithOrgsReposOrCredentials(
 	s.Require().NoError(err)
 	s.Require().NotNil(creds)
 
-	repo, err := s.db.CreateRepository(ctx, "test-owner", "test-repo", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.db.CreateRepository(ctx, "test-owner", "test-repo", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(repo)
 
@@ -755,7 +755,7 @@ func (s *GiteaTestSuite) TestDeleteGiteaEndpointFailsWithOrgsReposOrCredentials(
 	err = s.db.DeleteRepository(ctx, repo.ID)
 	s.Require().NoError(err)
 
-	org, err := s.db.CreateOrganization(ctx, "test-org", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	org, err := s.db.CreateOrganization(ctx, "test-org", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(org)
 

database/sql/github_test.go

@@ -640,7 +640,7 @@ func (s *GithubTestSuite) TestDeleteCredentialsFailsIfReposOrgsOrEntitiesUseIt()
 	s.Require().NoError(err)
 	s.Require().NotNil(creds)
 
-	repo, err := s.db.CreateRepository(ctx, "test-owner", "test-repo", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.db.CreateRepository(ctx, "test-owner", "test-repo", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(repo)
 
@@ -651,7 +651,7 @@ func (s *GithubTestSuite) TestDeleteCredentialsFailsIfReposOrgsOrEntitiesUseIt()
 	err = s.db.DeleteRepository(ctx, repo.ID)
 	s.Require().NoError(err)
 
-	org, err := s.db.CreateOrganization(ctx, "test-org", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	org, err := s.db.CreateOrganization(ctx, "test-org", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(org)
 
@@ -662,7 +662,7 @@ func (s *GithubTestSuite) TestDeleteCredentialsFailsIfReposOrgsOrEntitiesUseIt()
 	err = s.db.DeleteOrganization(ctx, org.ID)
 	s.Require().NoError(err)
 
-	enterprise, err := s.db.CreateEnterprise(ctx, "test-enterprise", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	enterprise, err := s.db.CreateEnterprise(ctx, "test-enterprise", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(enterprise)
 
@@ -872,7 +872,7 @@ func (s *GithubTestSuite) TestDeleteGithubEndpointFailsWithOrgsReposOrCredential
 	s.Require().NoError(err)
 	s.Require().NotNil(creds)
 
-	repo, err := s.db.CreateRepository(ctx, "test-owner", "test-repo", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.db.CreateRepository(ctx, "test-owner", "test-repo", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(repo)
 
@@ -884,7 +884,7 @@ func (s *GithubTestSuite) TestDeleteGithubEndpointFailsWithOrgsReposOrCredential
 	err = s.db.DeleteRepository(ctx, repo.ID)
 	s.Require().NoError(err)
 
-	org, err := s.db.CreateOrganization(ctx, "test-org", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin)
+	org, err := s.db.CreateOrganization(ctx, "test-org", creds, "superSecret@123BlaBla", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotNil(org)
 

database/sql/instances.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"log/slog"
 	"math"
+	"slices"
 
 	"github.com/google/uuid"
 	"gorm.io/datatypes"
@@ -28,6 +29,7 @@ import (
 	"gorm.io/gorm/clause"
 
 	runnerErrors "github.com/cloudbase/garm-provider-common/errors"
+	commonParams "github.com/cloudbase/garm-provider-common/params"
 	"github.com/cloudbase/garm/database/common"
 	"github.com/cloudbase/garm/params"
 )
@@ -124,7 +126,7 @@ func (s *sqlDatabase) getPoolInstanceByName(poolID string, instanceName string)
 	return instance, nil
 }
 
-func (s *sqlDatabase) getInstance(_ context.Context, instanceNameOrID string, preload ...string) (Instance, error) {
+func (s *sqlDatabase) getInstance(_ context.Context, tx *gorm.DB, instanceNameOrID string, preload ...string) (Instance, error) {
 	var instance Instance
 
 	var whereArg any = instanceNameOrID
@@ -134,7 +136,7 @@ func (s *sqlDatabase) getInstance(_ context.Context, instanceNameOrID string, pr
 		whereArg = id
 		whereClause = "id = ?"
 	}
-	q := s.conn
+	q := tx
 
 	if len(preload) > 0 {
 		for _, item := range preload {
@@ -156,7 +158,7 @@ func (s *sqlDatabase) getInstance(_ context.Context, instanceNameOrID string, pr
 }
 
 func (s *sqlDatabase) GetInstance(ctx context.Context, instanceName string) (params.Instance, error) {
-	instance, err := s.getInstance(ctx, instanceName, "StatusMessages", "Pool", "ScaleSet")
+	instance, err := s.getInstance(ctx, s.conn, instanceName, "StatusMessages", "Pool", "ScaleSet")
 	if err != nil {
 		return params.Instance{}, fmt.Errorf("error fetching instance: %w", err)
 	}
@@ -208,7 +210,7 @@ func (s *sqlDatabase) DeleteInstance(_ context.Context, poolID string, instanceN
 }
 
 func (s *sqlDatabase) DeleteInstanceByName(ctx context.Context, instanceName string) error {
-	instance, err := s.getInstance(ctx, instanceName, "Pool", "ScaleSet")
+	instance, err := s.getInstance(ctx, s.conn, instanceName, "Pool", "ScaleSet")
 	if err != nil {
 		if errors.Is(err, runnerErrors.ErrNotFound) {
 			return nil
@@ -250,7 +252,7 @@ func (s *sqlDatabase) DeleteInstanceByName(ctx context.Context, instanceName str
 }
 
 func (s *sqlDatabase) AddInstanceEvent(ctx context.Context, instanceName string, event params.EventType, eventLevel params.EventLevel, statusMessage string) error {
-	instance, err := s.getInstance(ctx, instanceName)
+	instance, err := s.getInstance(ctx, s.conn, instanceName)
 	if err != nil {
 		return fmt.Errorf("error updating instance: %w", err)
 	}
@@ -261,81 +263,199 @@ func (s *sqlDatabase) AddInstanceEvent(ctx context.Context, instanceName string,
 		EventLevel: eventLevel,
 	}
 
-	if err := s.conn.Model(&instance).Association("StatusMessages").Append(&msg); err != nil {
+	// Use Create instead of Association.Append to avoid loading all existing messages
+	msg.InstanceID = instance.ID
+	if err := s.conn.Create(&msg).Error; err != nil {
 		return fmt.Errorf("error adding status message: %w", err)
 	}
+
+	// Keep only the latest 30 status messages to prevent database bloat
+	const maxStatusMessages = 30
+	var count int64
+	if err := s.conn.Model(&InstanceStatusUpdate{}).Where("instance_id = ?", instance.ID).Count(&count).Error; err != nil {
+		return fmt.Errorf("error counting status messages: %w", err)
+	}
+
+	if count > maxStatusMessages {
+		// Get the ID of the 30th most recent message
+		var cutoffMsg InstanceStatusUpdate
+		if err := s.conn.Model(&InstanceStatusUpdate{}).
+			Select("id").
+			Where("instance_id = ?", instance.ID).
+			Order("id desc").
+			Offset(maxStatusMessages - 1).
+			Limit(1).
+			First(&cutoffMsg).Error; err != nil {
+			return fmt.Errorf("error finding cutoff message: %w", err)
+		}
+
+		// Delete all messages older than the cutoff
+		if err := s.conn.Where("instance_id = ? and id < ?", instance.ID, cutoffMsg.ID).Unscoped().Delete(&InstanceStatusUpdate{}).Error; err != nil {
+			return fmt.Errorf("error deleting old status messages: %w", err)
+		}
+	}
+
+	return nil
+}
+
+// validateAgentID checks agent ID consistency
+func (s *sqlDatabase) validateAgentID(currentAgentID, newAgentID int64) error {
+	if currentAgentID != 0 && newAgentID != 0 && currentAgentID != newAgentID {
+		return runnerErrors.NewBadRequestError("agent ID mismatch")
+	}
 	return nil
 }
 
-func (s *sqlDatabase) UpdateInstance(ctx context.Context, instanceName string, param params.UpdateInstanceParams) (params.Instance, error) {
-	instance, err := s.getInstance(ctx, instanceName, "Pool", "ScaleSet")
-	if err != nil {
-		return params.Instance{}, fmt.Errorf("error updating instance: %w", err)
+// validateRunnerStatusTransition validates runner status state transition
+func (s *sqlDatabase) validateRunnerStatusTransition(current, newStatus params.RunnerStatus) error {
+	if newStatus == "" || newStatus == current {
+		return nil
 	}
 
+	allowedTransitions, ok := params.RunnerStatusTransitions[current]
+	if !ok {
+		return fmt.Errorf("Instance is in invalid state: %s", current)
+	}
+
+	if !slices.Contains(allowedTransitions, newStatus) {
+		return runnerErrors.NewBadRequestError("invalid runner status transition from %s to %s", current, newStatus)
+	}
+	return nil
+}
+
+// validateInstanceStatusTransition validates instance status state transition
+func (s *sqlDatabase) validateInstanceStatusTransition(current, newStatus commonParams.InstanceStatus) error {
+	if newStatus == "" || newStatus == current {
+		return nil
+	}
+
+	allowedTransitions, ok := params.InstanceStatusTransitions[current]
+	if !ok {
+		// we need a better way to handle this. Because if we err out here, we cannot recover
+		// unless the user manually updates the instance.
+		return fmt.Errorf("Instance is in invalid state: %s", current)
+	}
+
+	if !slices.Contains(allowedTransitions, newStatus) {
+		return runnerErrors.NewBadRequestError("invalid instance status transition from %s to %s", current, newStatus)
+	}
+	return nil
+}
+
+// applyInstanceUpdates applies parameter updates to the instance
+func (s *sqlDatabase) applyInstanceUpdates(instance *Instance, param params.UpdateInstanceParams) error {
+	// Simple field updates
 	if param.AgentID != 0 {
 		instance.AgentID = param.AgentID
 	}
-
 	if param.ProviderID != "" {
 		instance.ProviderID = &param.ProviderID
 	}
-
 	if param.OSName != "" {
 		instance.OSName = param.OSName
 	}
-
 	if param.OSVersion != "" {
 		instance.OSVersion = param.OSVersion
 	}
-
 	if string(param.RunnerStatus) != "" {
 		instance.RunnerStatus = param.RunnerStatus
 	}
-
-	if string(param.Status) != "" {
+	if param.Heartbeat != nil {
+		instance.Heartbeat = *param.Heartbeat
+	}
+	if param.Status != "" {
 		instance.Status = param.Status
 	}
 	if param.CreateAttempt != 0 {
 		instance.CreateAttempt = param.CreateAttempt
 	}
-
 	if param.TokenFetched != nil {
 		instance.TokenFetched = *param.TokenFetched
 	}
 
+	// Complex field updates
+	if param.Capabilities != nil {
+		asJs, err := json.Marshal(*param.Capabilities)
+		if err != nil {
+			return runnerErrors.NewBadRequestError("invalid capabilities: %s", err)
+		}
+		instance.Capabilities = asJs
+	}
+
 	if param.JitConfiguration != nil {
 		secret, err := s.marshalAndSeal(param.JitConfiguration)
 		if err != nil {
-			return params.Instance{}, fmt.Errorf("error marshalling jit config: %w", err)
+			return fmt.Errorf("error marshalling jit config: %w", err)
 		}
 		instance.JitConfiguration = secret
 	}
 
 	instance.ProviderFault = param.ProviderFault
+	return nil
+}
 
-	q := s.conn.Save(&instance)
-	if q.Error != nil {
-		return params.Instance{}, fmt.Errorf("error updating instance: %w", q.Error)
+func (s *sqlDatabase) UpdateInstance(ctx context.Context, instanceName string, param params.UpdateInstanceParams) (params.Instance, error) {
+	var rowsAffected int64
+	err := s.conn.Transaction(func(tx *gorm.DB) error {
+		instance, err := s.getInstance(ctx, tx, instanceName, "Pool", "ScaleSet")
+		if err != nil {
+			return fmt.Errorf("error updating instance: %w", err)
+		}
+
+		// Validate transitions
+		if err := s.validateAgentID(instance.AgentID, param.AgentID); err != nil {
+			return err
+		}
+		if err := s.validateRunnerStatusTransition(instance.RunnerStatus, param.RunnerStatus); err != nil {
+			return err
+		}
+		if err := s.validateInstanceStatusTransition(instance.Status, param.Status); err != nil {
+			return err
+		}
+
+		// Apply updates
+		if err := s.applyInstanceUpdates(&instance, param); err != nil {
+			return err
+		}
+
+		// Save instance
+		result := tx.Save(&instance)
+		if result.Error != nil {
+			return fmt.Errorf("error updating instance: %w", result.Error)
+		}
+		rowsAffected = result.RowsAffected
+
+		// Update addresses if provided
+		if len(param.Addresses) > 0 {
+			addrs := make([]Address, 0, len(param.Addresses))
+			for _, addr := range param.Addresses {
+				addrs = append(addrs, Address{
+					Address: addr.Address,
+					Type:    string(addr.Type),
+				})
+			}
+			if err := tx.Model(&instance).Association("Addresses").Replace(addrs); err != nil {
+				return fmt.Errorf("error updating addresses: %w", err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return params.Instance{}, fmt.Errorf("error updating instance: %w", err)
 	}
 
-	if len(param.Addresses) > 0 {
-		addrs := []Address{}
-		for _, addr := range param.Addresses {
-			addrs = append(addrs, Address{
-				Address: addr.Address,
-				Type:    string(addr.Type),
-			})
-		}
-		if err := s.conn.Model(&instance).Association("Addresses").Replace(addrs); err != nil {
-			return params.Instance{}, fmt.Errorf("error updating addresses: %w", err)
-		}
+	instance, err := s.getInstance(ctx, s.conn, instanceName, "Pool", "ScaleSet")
+	if err != nil {
+		return params.Instance{}, fmt.Errorf("error updating instance: %w", err)
 	}
+
 	inst, err := s.sqlToParamsInstance(instance)
 	if err != nil {
 		return params.Instance{}, fmt.Errorf("error converting instance: %w", err)
 	}
-	s.sendNotify(common.InstanceEntityType, common.UpdateOperation, inst)
+	if rowsAffected > 0 {
+		s.sendNotify(common.InstanceEntityType, common.UpdateOperation, inst)
+	}
 	return inst, nil
 }
 

database/sql/instances_test.go

@@ -92,7 +92,7 @@ func (s *InstancesTestSuite) SetupTest() {
 	creds := garmTesting.CreateTestGithubCredentials(adminCtx, "new-creds", db, s.T(), githubEndpoint)
 
 	// create an organization for testing purposes
-	org, err := s.Store.CreateOrganization(s.adminCtx, "test-org", creds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin)
+	org, err := s.Store.CreateOrganization(s.adminCtx, "test-org", creds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin, false)
 	if err != nil {
 		s.FailNow(fmt.Sprintf("failed to create org: %s", err))
 	}
@@ -573,10 +573,6 @@ func (s *InstancesTestSuite) TestAddInstanceEventDBUpdateErr() {
 		WithArgs(instance.ID).
 		WillReturnRows(sqlmock.NewRows([]string{"message", "instance_id"}).AddRow("instance sample message", instance.ID))
 	s.Fixtures.SQLMock.ExpectBegin()
-	s.Fixtures.SQLMock.
-		ExpectExec(regexp.QuoteMeta("UPDATE `instances` SET `updated_at`=? WHERE `instances`.`deleted_at` IS NULL AND `id` = ?")).
-		WithArgs(sqlmock.AnyArg(), instance.ID).
-		WillReturnResult(sqlmock.NewResult(1, 1))
 	s.Fixtures.SQLMock.
 		ExpectExec(regexp.QuoteMeta("INSERT INTO `instance_status_updates`")).
 		WillReturnError(fmt.Errorf("mocked add status message error"))
@@ -605,10 +601,12 @@ func (s *InstancesTestSuite) TestUpdateInstance() {
 func (s *InstancesTestSuite) TestUpdateInstanceDBUpdateInstanceErr() {
 	instance := s.Fixtures.Instances[0]
 
+	s.Fixtures.SQLMock.ExpectBegin()
 	s.Fixtures.SQLMock.
 		ExpectQuery(regexp.QuoteMeta("SELECT * FROM `instances` WHERE name = ? AND `instances`.`deleted_at` IS NULL ORDER BY `instances`.`id` LIMIT ?")).
 		WithArgs(instance.Name, 1).
-		WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(instance.ID))
+		WillReturnRows(sqlmock.NewRows([]string{"id", "created_at", "updated_at", "deleted_at", "provider_id", "name", "agent_id", "os_type", "os_arch", "os_name", "os_version", "status", "runner_status", "heartbeat", "callback_url", "metadata_url", "provider_fault", "create_attempt", "token_fetched", "jit_configuration", "git_hub_runner_group", "aditional_labels", "capabilities", "pool_id", "scale_set_fk_id"}).
+			AddRow(instance.ID, instance.CreatedAt, instance.UpdatedAt, nil, nil, instance.Name, 0, "linux", "amd64", "", "", "running", "idle", instance.Heartbeat, "", "", nil, 0, false, nil, "", nil, nil, nil, nil))
 	s.Fixtures.SQLMock.
 		ExpectQuery(regexp.QuoteMeta("SELECT * FROM `addresses` WHERE `addresses`.`instance_id` = ? AND `addresses`.`deleted_at` IS NULL")).
 		WithArgs(instance.ID).
@@ -621,7 +619,6 @@ func (s *InstancesTestSuite) TestUpdateInstanceDBUpdateInstanceErr() {
 		ExpectQuery(regexp.QuoteMeta("SELECT * FROM `instance_status_updates` WHERE `instance_status_updates`.`instance_id` = ? AND `instance_status_updates`.`deleted_at` IS NULL")).
 		WithArgs(instance.ID).
 		WillReturnRows(sqlmock.NewRows([]string{"message", "instance_id"}).AddRow("instance sample message", instance.ID))
-	s.Fixtures.SQLMock.ExpectBegin()
 	s.Fixtures.SQLMock.
 		ExpectExec(("UPDATE `instances`")).
 		WillReturnError(fmt.Errorf("mocked update instance error"))
@@ -630,17 +627,19 @@ func (s *InstancesTestSuite) TestUpdateInstanceDBUpdateInstanceErr() {
 	_, err := s.StoreSQLMocked.UpdateInstance(s.adminCtx, instance.Name, s.Fixtures.UpdateInstanceParams)
 
 	s.Require().NotNil(err)
-	s.Require().Equal("error updating instance: mocked update instance error", err.Error())
+	s.Require().Equal("error updating instance: error updating instance: mocked update instance error", err.Error())
 	s.assertSQLMockExpectations()
 }
 
 func (s *InstancesTestSuite) TestUpdateInstanceDBUpdateAddressErr() {
 	instance := s.Fixtures.Instances[0]
 
+	s.Fixtures.SQLMock.ExpectBegin()
 	s.Fixtures.SQLMock.
 		ExpectQuery(regexp.QuoteMeta("SELECT * FROM `instances` WHERE name = ? AND `instances`.`deleted_at` IS NULL ORDER BY `instances`.`id` LIMIT ?")).
 		WithArgs(instance.Name, 1).
-		WillReturnRows(sqlmock.NewRows([]string{"id"}).AddRow(instance.ID))
+		WillReturnRows(sqlmock.NewRows([]string{"id", "created_at", "updated_at", "deleted_at", "provider_id", "name", "agent_id", "os_type", "os_arch", "os_name", "os_version", "status", "runner_status", "heartbeat", "callback_url", "metadata_url", "provider_fault", "create_attempt", "token_fetched", "jit_configuration", "git_hub_runner_group", "aditional_labels", "capabilities", "pool_id", "scale_set_fk_id"}).
+			AddRow(instance.ID, instance.CreatedAt, instance.UpdatedAt, nil, nil, instance.Name, 0, "linux", "amd64", "", "", "running", "idle", instance.Heartbeat, "", "", nil, 0, false, nil, "", nil, nil, nil, nil))
 	s.Fixtures.SQLMock.
 		ExpectQuery(regexp.QuoteMeta("SELECT * FROM `addresses` WHERE `addresses`.`instance_id` = ? AND `addresses`.`deleted_at` IS NULL")).
 		WithArgs(instance.ID).
@@ -653,18 +652,6 @@ func (s *InstancesTestSuite) TestUpdateInstanceDBUpdateAddressErr() {
 		ExpectQuery(regexp.QuoteMeta("SELECT * FROM `instance_status_updates` WHERE `instance_status_updates`.`instance_id` = ? AND `instance_status_updates`.`deleted_at` IS NULL")).
 		WithArgs(instance.ID).
 		WillReturnRows(sqlmock.NewRows([]string{"message", "instance_id"}).AddRow("instance sample message", instance.ID))
-	s.Fixtures.SQLMock.ExpectBegin()
-	s.Fixtures.SQLMock.
-		ExpectExec(regexp.QuoteMeta("UPDATE `instances` SET")).
-		WillReturnResult(sqlmock.NewResult(1, 1))
-	s.Fixtures.SQLMock.
-		ExpectExec(regexp.QuoteMeta("INSERT INTO `addresses`")).
-		WillReturnResult(sqlmock.NewResult(1, 1))
-	s.Fixtures.SQLMock.
-		ExpectExec(regexp.QuoteMeta("INSERT INTO `instance_status_updates`")).
-		WillReturnResult(sqlmock.NewResult(1, 1))
-	s.Fixtures.SQLMock.ExpectCommit()
-	s.Fixtures.SQLMock.ExpectBegin()
 	s.Fixtures.SQLMock.
 		ExpectExec(regexp.QuoteMeta("UPDATE `instances` SET")).
 		WillReturnResult(sqlmock.NewResult(1, 1))
@@ -676,7 +663,7 @@ func (s *InstancesTestSuite) TestUpdateInstanceDBUpdateAddressErr() {
 	_, err := s.StoreSQLMocked.UpdateInstance(s.adminCtx, instance.Name, s.Fixtures.UpdateInstanceParams)
 
 	s.Require().NotNil(err)
-	s.Require().Equal("error updating addresses: update addresses mock error", err.Error())
+	s.Require().Equal("error updating instance: error updating instance: update addresses mock error; update addresses mock error", err.Error())
 	s.assertSQLMockExpectations()
 }
 

database/sql/jobs.go

@@ -100,7 +100,7 @@ func (s *sqlDatabase) paramsJobToWorkflowJob(ctx context.Context, job params.Job
 	}
 
 	if job.RunnerName != "" {
-		instance, err := s.getInstance(s.ctx, job.RunnerName)
+		instance, err := s.getInstance(s.ctx, s.conn, job.RunnerName)
 		if err != nil {
 			// This usually is very normal as not all jobs run on our runners.
 			slog.DebugContext(ctx, "failed to get instance by name", "instance_name", job.RunnerName)
@@ -282,7 +282,7 @@ func (s *sqlDatabase) CreateOrUpdateJob(ctx context.Context, job params.Job) (pa
 		}
 
 		if job.RunnerName != "" {
-			instance, err := s.getInstance(ctx, job.RunnerName)
+			instance, err := s.getInstance(ctx, s.conn, job.RunnerName)
 			if err == nil {
 				workflowJob.InstanceID = &instance.ID
 			} else {

database/sql/models.go

@@ -51,9 +51,20 @@ type ControllerInfo struct {
 
 	ControllerID uuid.UUID
 
-	CallbackURL    string
-	MetadataURL    string
+	// CallbackURL is the URL where userdata scripts call back into, to send status updates
+	// and installation progress.
+	CallbackURL string
+	// MetadataURL is the base URL from which runners can get their installation metadata.
+	MetadataURL string
+	// WebhookBaseURL is the base URL used to construct the controller webhook URL.
 	WebhookBaseURL string
+	// AgentURL is the websocket enabled URL whenre garm agents connect to.
+	AgentURL string
+	// GARMAgentReleasesURL is the URL from which GARM can sync garm-agent binaries. Alternatively
+	// the user can manually upload binaries.
+	GARMAgentReleasesURL string
+	// SyncGARMAgentTools enables or disables automatic sync of garm-agent tools.
+	SyncGARMAgentTools bool
 	// MinimumJobAgeBackoff is the minimum time that a job must be in the queue
 	// before GARM will attempt to allocate a runner to service it. This backoff
 	// is useful if you have idle runners in various pools that could potentially
@@ -104,6 +115,7 @@ type Pool struct {
 	// any kind of data needed by providers.
 	ExtraSpecs        datatypes.JSON
 	GitHubRunnerGroup string
+	EnableShell       bool
 
 	RepoID     *uuid.UUID `gorm:"index"`
 	Repository Repository `gorm:"foreignKey:RepoID;"`
@@ -159,7 +171,8 @@ type ScaleSet struct {
 	// ExtraSpecs is an opaque json that gets sent to the provider
 	// as part of the bootstrap params for instances. It can contain
 	// any kind of data needed by providers.
-	ExtraSpecs datatypes.JSON
+	ExtraSpecs  datatypes.JSON
+	EnableShell bool
 
 	RepoID     *uuid.UUID `gorm:"index"`
 	Repository Repository `gorm:"foreignKey:RepoID;"`
@@ -203,6 +216,7 @@ type Repository struct {
 	ScaleSets        []ScaleSet              `gorm:"foreignKey:RepoID"`
 	Jobs             []WorkflowJob           `gorm:"foreignKey:RepoID;constraint:OnDelete:SET NULL"`
 	PoolBalancerType params.PoolBalancerType `gorm:"type:varchar(64)"`
+	AgentMode        bool                    `gorm:"index:repo_agent_idx"`
 
 	EndpointName *string        `gorm:"index:idx_owner_nocase,unique,collate:nocase"`
 	Endpoint     GithubEndpoint `gorm:"foreignKey:EndpointName;constraint:OnDelete:SET NULL"`
@@ -235,6 +249,7 @@ type Organization struct {
 	ScaleSet         []ScaleSet              `gorm:"foreignKey:OrgID"`
 	Jobs             []WorkflowJob           `gorm:"foreignKey:OrgID;constraint:OnDelete:SET NULL"`
 	PoolBalancerType params.PoolBalancerType `gorm:"type:varchar(64)"`
+	AgentMode        bool                    `gorm:"index:org_agent_idx"`
 
 	EndpointName *string        `gorm:"index:idx_org_name_nocase,collate:nocase"`
 	Endpoint     GithubEndpoint `gorm:"foreignKey:EndpointName;constraint:OnDelete:SET NULL"`
@@ -265,6 +280,7 @@ type Enterprise struct {
 	ScaleSet         []ScaleSet              `gorm:"foreignKey:EnterpriseID"`
 	Jobs             []WorkflowJob           `gorm:"foreignKey:EnterpriseID;constraint:OnDelete:SET NULL"`
 	PoolBalancerType params.PoolBalancerType `gorm:"type:varchar(64)"`
+	AgentMode        bool                    `gorm:"index:enterprise_agent_idx"`
 
 	EndpointName *string        `gorm:"index:idx_ent_name_nocase,collate:nocase"`
 	Endpoint     GithubEndpoint `gorm:"foreignKey:EndpointName;constraint:OnDelete:SET NULL"`
@@ -306,6 +322,7 @@ type Instance struct {
 	Addresses         []Address `gorm:"foreignKey:InstanceID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"`
 	Status            commonParams.InstanceStatus
 	RunnerStatus      params.RunnerStatus
+	Heartbeat         time.Time
 	CallbackURL       string
 	MetadataURL       string
 	ProviderFault     []byte `gorm:"type:longblob"`
@@ -314,6 +331,7 @@ type Instance struct {
 	JitConfiguration  []byte `gorm:"type:longblob"`
 	GitHubRunnerGroup string
 	AditionalLabels   datatypes.JSON
+	Capabilities      datatypes.JSON
 
 	PoolID *uuid.UUID
 	Pool   Pool `gorm:"foreignKey:PoolID"`

database/sql/organizations.go

@@ -29,7 +29,7 @@ import (
 	"github.com/cloudbase/garm/params"
 )
 
-func (s *sqlDatabase) CreateOrganization(ctx context.Context, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType) (param params.Organization, err error) {
+func (s *sqlDatabase) CreateOrganization(ctx context.Context, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool) (param params.Organization, err error) {
 	if webhookSecret == "" {
 		return params.Organization{}, errors.New("creating org: missing secret")
 	}
@@ -47,6 +47,7 @@ func (s *sqlDatabase) CreateOrganization(ctx context.Context, name string, crede
 		Name:             name,
 		WebhookSecret:    secret,
 		PoolBalancerType: poolBalancerType,
+		AgentMode:        agentMode,
 	}
 
 	err = s.conn.Transaction(func(tx *gorm.DB) error {
@@ -195,6 +196,10 @@ func (s *sqlDatabase) UpdateOrganization(ctx context.Context, orgID string, para
 			org.PoolBalancerType = param.PoolBalancerType
 		}
 
+		if param.AgentMode != nil {
+			org.AgentMode = *param.AgentMode
+		}
+
 		q := tx.Save(&org)
 		if q.Error != nil {
 			return fmt.Errorf("error saving org: %w", q.Error)

database/sql/organizations_test.go

@@ -114,6 +114,7 @@ func (s *OrgTestSuite) SetupTest() {
 			s.testCreds,
 			fmt.Sprintf("test-webhook-secret-%d", i),
 			params.PoolBalancerTypeRoundRobin,
+			false,
 		)
 		if err != nil {
 			s.FailNow(fmt.Sprintf("failed to create database object (test-org-%d): %q", i, err))
@@ -192,7 +193,9 @@ func (s *OrgTestSuite) TestCreateOrganization() {
 		s.Fixtures.CreateOrgParams.Name,
 		s.testCreds,
 		s.Fixtures.CreateOrgParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 
 	// assertions
 	s.Require().Nil(err)
@@ -221,7 +224,9 @@ func (s *OrgTestSuite) TestCreateOrgForGitea() {
 		s.Fixtures.CreateOrgParams.Name,
 		s.testCredsGitea,
 		s.Fixtures.CreateOrgParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 
 	// assertions
 	s.Require().Nil(err)
@@ -256,7 +261,9 @@ func (s *OrgTestSuite) TestCreateOrganizationInvalidForgeType() {
 		s.Fixtures.CreateOrgParams.Name,
 		credentials,
 		s.Fixtures.CreateOrgParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 	s.Require().NotNil(err)
 	s.Require().Equal("error creating org: unsupported credentials type: invalid request", err.Error())
 }
@@ -279,7 +286,9 @@ func (s *OrgTestSuite) TestCreateOrganizationInvalidDBPassphrase() {
 		s.Fixtures.CreateOrgParams.Name,
 		s.testCreds,
 		s.Fixtures.CreateOrgParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 
 	s.Require().NotNil(err)
 	s.Require().Equal("error encoding secret: invalid passphrase length (expected length 32 characters)", err.Error())
@@ -297,7 +306,8 @@ func (s *OrgTestSuite) TestCreateOrganizationDBCreateErr() {
 		s.Fixtures.CreateOrgParams.Name,
 		s.testCreds,
 		s.Fixtures.CreateOrgParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false)
 
 	s.Require().NotNil(err)
 	s.Require().Equal("error creating org: error creating org: creating org mock error", err.Error())
@@ -353,6 +363,7 @@ func (s *OrgTestSuite) TestListOrganizationsWithFilters() {
 		s.testCreds,
 		"super secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 
@@ -362,6 +373,7 @@ func (s *OrgTestSuite) TestListOrganizationsWithFilters() {
 		s.testCredsGitea,
 		"super secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 
@@ -371,6 +383,7 @@ func (s *OrgTestSuite) TestListOrganizationsWithFilters() {
 		s.testCreds,
 		"super secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 	orgs, err := s.Store.ListOrganizations(
@@ -899,7 +912,8 @@ func (s *OrgTestSuite) TestAddOrgEntityEvent() {
 		s.Fixtures.CreateOrgParams.Name,
 		s.testCreds,
 		s.Fixtures.CreateOrgParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false)
 
 	s.Require().Nil(err)
 	entity, err := org.GetEntity()

database/sql/pools.go

@@ -293,6 +293,7 @@ func (s *sqlDatabase) CreateEntityPool(ctx context.Context, entity params.ForgeE
 		GitHubRunnerGroup:      param.GitHubRunnerGroup,
 		Priority:               param.Priority,
 		TemplateID:             param.TemplateID,
+		EnableShell:            param.EnableShell,
 	}
 	if len(param.ExtraSpecs) > 0 {
 		newPool.ExtraSpecs = datatypes.JSON(param.ExtraSpecs)
@@ -316,13 +317,13 @@ func (s *sqlDatabase) CreateEntityPool(ctx context.Context, entity params.ForgeE
 			return fmt.Errorf("error checking entity existence: %w", err)
 		}
 
-		tags := []Tag{}
+		var tags []*Tag
 		for _, val := range param.Tags {
 			t, err := s.getOrCreateTag(tx, val)
 			if err != nil {
 				return fmt.Errorf("error creating tag: %w", err)
 			}
-			tags = append(tags, t)
+			tags = append(tags, &t)
 		}
 
 		q := tx.Create(&newPool)
@@ -330,8 +331,9 @@ func (s *sqlDatabase) CreateEntityPool(ctx context.Context, entity params.ForgeE
 			return fmt.Errorf("error creating pool: %w", q.Error)
 		}
 
-		for i := range tags {
-			if err := tx.Model(&newPool).Association("Tags").Append(&tags[i]); err != nil {
+		// Append all tags at once instead of one by one for better performance
+		if len(tags) > 0 {
+			if err := tx.Model(&newPool).Association("Tags").Append(tags); err != nil {
 				return fmt.Errorf("error associating tags: %w", err)
 			}
 		}

database/sql/pools_test.go

@@ -81,7 +81,7 @@ func (s *PoolsTestSuite) SetupTest() {
 	creds := garmTesting.CreateTestGithubCredentials(adminCtx, "new-creds", db, s.T(), githubEndpoint)
 
 	// create an organization for testing purposes
-	org, err := s.Store.CreateOrganization(s.adminCtx, "test-org", creds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin)
+	org, err := s.Store.CreateOrganization(s.adminCtx, "test-org", creds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin, false)
 	if err != nil {
 		s.FailNow(fmt.Sprintf("failed to create org: %s", err))
 	}
@@ -211,7 +211,7 @@ func (s *PoolsTestSuite) TestEntityPoolOperations() {
 	ep := garmTesting.CreateDefaultGithubEndpoint(s.ctx, s.Store, s.T())
 	creds := garmTesting.CreateTestGithubCredentials(s.ctx, "test-creds", s.Store, s.T(), ep)
 	s.T().Cleanup(func() { s.Store.DeleteGithubCredentials(s.ctx, creds.ID) })
-	repo, err := s.Store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.Store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotEmpty(repo.ID)
 	s.T().Cleanup(func() { s.Store.DeleteRepository(s.ctx, repo.ID) })
@@ -261,7 +261,7 @@ func (s *PoolsTestSuite) TestEntityPoolOperations() {
 	s.Require().Equal(*updatePoolParams.Enabled, pool.Enabled)
 	s.Require().Equal(updatePoolParams.Flavor, pool.Flavor)
 	s.Require().Equal(updatePoolParams.Image, pool.Image)
-	s.Require().Equal(updatePoolParams.RunnerPrefix.Prefix, pool.RunnerPrefix.Prefix)
+	s.Require().Equal(updatePoolParams.Prefix, pool.Prefix)
 	s.Require().Equal(*updatePoolParams.MaxRunners, pool.MaxRunners)
 	s.Require().Equal(*updatePoolParams.MinIdleRunners, pool.MinIdleRunners)
 	s.Require().Equal(updatePoolParams.OSType, pool.OSType)
@@ -292,7 +292,7 @@ func (s *PoolsTestSuite) TestListEntityInstances() {
 	ep := garmTesting.CreateDefaultGithubEndpoint(s.ctx, s.Store, s.T())
 	creds := garmTesting.CreateTestGithubCredentials(s.ctx, "test-creds", s.Store, s.T(), ep)
 	s.T().Cleanup(func() { s.Store.DeleteGithubCredentials(s.ctx, creds.ID) })
-	repo, err := s.Store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.Store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotEmpty(repo.ID)
 	s.T().Cleanup(func() { s.Store.DeleteRepository(s.ctx, repo.ID) })

database/sql/repositories.go

@@ -29,7 +29,7 @@ import (
 	"github.com/cloudbase/garm/params"
 )
 
-func (s *sqlDatabase) CreateRepository(ctx context.Context, owner, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType) (param params.Repository, err error) {
+func (s *sqlDatabase) CreateRepository(ctx context.Context, owner, name string, credentials params.ForgeCredentials, webhookSecret string, poolBalancerType params.PoolBalancerType, agentMode bool) (param params.Repository, err error) {
 	defer func() {
 		if err == nil {
 			s.sendNotify(common.RepositoryEntityType, common.CreateOperation, param)
@@ -49,6 +49,7 @@ func (s *sqlDatabase) CreateRepository(ctx context.Context, owner, name string,
 		Owner:            owner,
 		WebhookSecret:    secret,
 		PoolBalancerType: poolBalancerType,
+		AgentMode:        agentMode,
 	}
 	err = s.conn.Transaction(func(tx *gorm.DB) error {
 		switch credentials.ForgeType {
@@ -196,6 +197,9 @@ func (s *sqlDatabase) UpdateRepository(ctx context.Context, repoID string, param
 		if param.PoolBalancerType != "" {
 			repo.PoolBalancerType = param.PoolBalancerType
 		}
+		if param.AgentMode != nil {
+			repo.AgentMode = *param.AgentMode
+		}
 
 		q := tx.Save(&repo)
 		if q.Error != nil {

database/sql/repositories_test.go

@@ -126,6 +126,7 @@ func (s *RepoTestSuite) SetupTest() {
 			s.testCreds,
 			fmt.Sprintf("test-webhook-secret-%d", i),
 			params.PoolBalancerTypeRoundRobin,
+			false,
 		)
 		if err != nil {
 			s.FailNow(fmt.Sprintf("failed to create database object (test-repo-%d): %v", i, err))
@@ -211,6 +212,7 @@ func (s *RepoTestSuite) TestCreateRepository() {
 		s.testCreds,
 		s.Fixtures.CreateRepoParams.WebhookSecret,
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 
 	// assertions
@@ -243,6 +245,7 @@ func (s *RepoTestSuite) TestCreateRepositoryGitea() {
 		s.testCredsGitea,
 		s.Fixtures.CreateRepoParams.WebhookSecret,
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 
 	// assertions
@@ -281,6 +284,7 @@ func (s *RepoTestSuite) TestCreateRepositoryInvalidForgeType() {
 		},
 		s.Fixtures.CreateRepoParams.WebhookSecret,
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 
 	s.Require().NotNil(err)
@@ -307,6 +311,7 @@ func (s *RepoTestSuite) TestCreateRepositoryInvalidDBPassphrase() {
 		s.testCreds,
 		s.Fixtures.CreateRepoParams.WebhookSecret,
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 
 	s.Require().NotNil(err)
@@ -327,6 +332,7 @@ func (s *RepoTestSuite) TestCreateRepositoryInvalidDBCreateErr() {
 		s.testCreds,
 		s.Fixtures.CreateRepoParams.WebhookSecret,
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 
 	s.Require().NotNil(err)
@@ -390,6 +396,7 @@ func (s *RepoTestSuite) TestListRepositoriesWithFilters() {
 		s.testCreds,
 		"super secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 
@@ -400,6 +407,7 @@ func (s *RepoTestSuite) TestListRepositoriesWithFilters() {
 		s.testCredsGitea,
 		"super secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 
@@ -410,6 +418,7 @@ func (s *RepoTestSuite) TestListRepositoriesWithFilters() {
 		s.testCreds,
 		"super secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 
@@ -420,6 +429,7 @@ func (s *RepoTestSuite) TestListRepositoriesWithFilters() {
 		s.testCreds,
 		"super secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 
@@ -986,7 +996,9 @@ func (s *RepoTestSuite) TestAddRepoEntityEvent() {
 		s.Fixtures.CreateRepoParams.Name,
 		s.testCreds,
 		s.Fixtures.CreateRepoParams.WebhookSecret,
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 
 	s.Require().Nil(err)
 	entity, err := repo.GetEntity()

database/sql/scalesets.go

@@ -84,6 +84,7 @@ func (s *sqlDatabase) CreateEntityScaleSet(ctx context.Context, entity params.Fo
 		GitHubRunnerGroup:      param.GitHubRunnerGroup,
 		State:                  params.ScaleSetPendingCreate,
 		TemplateID:             param.TemplateID,
+		EnableShell:            param.EnableShell,
 	}
 
 	if len(param.ExtraSpecs) > 0 {
@@ -303,6 +304,10 @@ func (s *sqlDatabase) updateScaleSet(tx *gorm.DB, scaleSet ScaleSet, param param
 		scaleSet.TemplateID = param.TemplateID
 	}
 
+	if param.EnableShell != nil {
+		scaleSet.EnableShell = *param.EnableShell
+	}
+
 	if param.Name != "" {
 		scaleSet.Name = param.Name
 	}

database/sql/scalesets_test.go

@@ -62,17 +62,17 @@ func (s *ScaleSetsTestSuite) SetupTest() {
 	s.creds = garmTesting.CreateTestGithubCredentials(adminCtx, "new-creds", db, s.T(), githubEndpoint)
 
 	// create an organization for testing purposes
-	s.org, err = s.Store.CreateOrganization(s.adminCtx, "test-org", s.creds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin)
+	s.org, err = s.Store.CreateOrganization(s.adminCtx, "test-org", s.creds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin, false)
 	if err != nil {
 		s.FailNow(fmt.Sprintf("failed to create org: %s", err))
 	}
 
-	s.repo, err = s.Store.CreateRepository(s.adminCtx, "test-org", "test-repo", s.creds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin)
+	s.repo, err = s.Store.CreateRepository(s.adminCtx, "test-org", "test-repo", s.creds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin, false)
 	if err != nil {
 		s.FailNow(fmt.Sprintf("failed to create repo: %s", err))
 	}
 
-	s.enterprise, err = s.Store.CreateEnterprise(s.adminCtx, "test-enterprise", s.creds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin)
+	s.enterprise, err = s.Store.CreateEnterprise(s.adminCtx, "test-enterprise", s.creds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin, false)
 	if err != nil {
 		s.FailNow(fmt.Sprintf("failed to create enterprise: %s", err))
 	}
@@ -131,8 +131,8 @@ func (s *ScaleSetsTestSuite) callback(old, newSet params.ScaleSet) error {
 	s.Require().Equal(newSet.Flavor, "new-test-flavor")
 	s.Require().Equal(old.GitHubRunnerGroup, "test-group")
 	s.Require().Equal(newSet.GitHubRunnerGroup, "new-test-group")
-	s.Require().Equal(old.RunnerPrefix.Prefix, "garm")
-	s.Require().Equal(newSet.RunnerPrefix.Prefix, "test-prefix2")
+	s.Require().Equal(old.Prefix, "garm")
+	s.Require().Equal(newSet.Prefix, "test-prefix2")
 	s.Require().Equal(old.Enabled, false)
 	s.Require().Equal(newSet.Enabled, true)
 	return nil

database/sql/sql.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"log/slog"
 	"net/url"
+	"regexp"
 	"strings"
 
 	"gorm.io/driver/mysql"
@@ -470,6 +471,11 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
 		return fmt.Errorf("failed to get linux template for gitea: %w", err)
 	}
 
+	giteaWindowsData, err := templates.GetTemplateContent(commonParams.Windows, params.GiteaEndpointType)
+	if err != nil {
+		return fmt.Errorf("failed to get windows template for gitea: %w", err)
+	}
+
 	adminCtx := auth.GetAdminContext(s.ctx)
 
 	githubWindowsParams := params.CreateTemplateParams{
@@ -478,8 +484,9 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
 		OSType:      commonParams.Windows,
 		ForgeType:   params.GithubEndpointType,
 		Data:        githubWindowsData,
+		IsSystem:    true,
 	}
-	githubWindowsSystemTemplate, err := s.createSystemTemplate(adminCtx, githubWindowsParams)
+	githubWindowsSystemTemplate, err := s.CreateTemplate(adminCtx, githubWindowsParams)
 	if err != nil {
 		return fmt.Errorf("failed to create github windows template: %w", err)
 	}
@@ -490,8 +497,9 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
 		OSType:      commonParams.Linux,
 		ForgeType:   params.GithubEndpointType,
 		Data:        githubLinuxData,
+		IsSystem:    true,
 	}
-	githubLinuxSystemTemplate, err := s.createSystemTemplate(adminCtx, githubLinuxParams)
+	githubLinuxSystemTemplate, err := s.CreateTemplate(adminCtx, githubLinuxParams)
 	if err != nil {
 		return fmt.Errorf("failed to create github linux template: %w", err)
 	}
@@ -502,12 +510,26 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
 		OSType:      commonParams.Linux,
 		ForgeType:   params.GiteaEndpointType,
 		Data:        giteaLinuxData,
+		IsSystem:    true,
 	}
-	giteaLinuxSystemTemplate, err := s.createSystemTemplate(adminCtx, giteaLinuxParams)
+	giteaLinuxSystemTemplate, err := s.CreateTemplate(adminCtx, giteaLinuxParams)
 	if err != nil {
 		return fmt.Errorf("failed to create gitea linux template: %w", err)
 	}
 
+	giteaWindowsParams := params.CreateTemplateParams{
+		Name:        "gitea_windows",
+		Description: "Default Windows runner install template for Gitea",
+		OSType:      commonParams.Windows,
+		ForgeType:   params.GiteaEndpointType,
+		Data:        giteaWindowsData,
+		IsSystem:    true,
+	}
+	giteaWindowsSystemTemplate, err := s.CreateTemplate(adminCtx, giteaWindowsParams)
+	if err != nil {
+		return fmt.Errorf("failed to create gitea windows template: %w", err)
+	}
+
 	getTplID := func(forgeType params.EndpointType, osType commonParams.OSType) uint {
 		var templateID uint
 		switch forgeType {
@@ -515,6 +537,8 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
 			switch osType {
 			case commonParams.Linux:
 				templateID = giteaLinuxSystemTemplate.ID
+			case commonParams.Windows:
+				templateID = giteaWindowsSystemTemplate.ID
 			default:
 				return 0
 			}
@@ -582,64 +606,142 @@ func (s *sqlDatabase) ensureTemplates(migrateTemplates bool) error {
 	return nil
 }
 
+// dropIndexIfExists drops an index if it exists
+func (s *sqlDatabase) dropIndexIfExists(model interface{}, indexName string) {
+	if s.conn.Migrator().HasIndex(model, indexName) {
+		if err := s.conn.Migrator().DropIndex(model, indexName); err != nil {
+			slog.With(slog.Any("error", err)).
+				Error(fmt.Sprintf("failed to drop index %s", indexName))
+		}
+	}
+}
+
+// migratePoolNullIDs updates pools to set null IDs instead of zero UUIDs
+func (s *sqlDatabase) migratePoolNullIDs() error {
+	if !s.conn.Migrator().HasTable(&Pool{}) {
+		return nil
+	}
+
+	zeroUUID := "00000000-0000-0000-0000-000000000000"
+	updates := []struct {
+		column string
+		query  string
+	}{
+		{"repo_id", fmt.Sprintf("update pools set repo_id=NULL where repo_id='%s'", zeroUUID)},
+		{"org_id", fmt.Sprintf("update pools set org_id=NULL where org_id='%s'", zeroUUID)},
+		{"enterprise_id", fmt.Sprintf("update pools set enterprise_id=NULL where enterprise_id='%s'", zeroUUID)},
+	}
+
+	for _, update := range updates {
+		if err := s.conn.Exec(update.query).Error; err != nil {
+			return fmt.Errorf("error updating pools %s: %w", update.column, err)
+		}
+	}
+	return nil
+}
+
+// migrateGithubEndpointType adds and initializes endpoint_type column
+func (s *sqlDatabase) migrateGithubEndpointType() error {
+	if !s.conn.Migrator().HasTable(&GithubEndpoint{}) {
+		return nil
+	}
+
+	if s.conn.Migrator().HasColumn(&GithubEndpoint{}, "endpoint_type") {
+		return nil
+	}
+
+	if err := s.conn.Migrator().AutoMigrate(&GithubEndpoint{}); err != nil {
+		return fmt.Errorf("error migrating github endpoints: %w", err)
+	}
+
+	if err := s.conn.Exec("update github_endpoints set endpoint_type = 'github' where endpoint_type is null").Error; err != nil {
+		return fmt.Errorf("error updating github endpoints: %w", err)
+	}
+
+	return nil
+}
+
+// migrateControllerInfo updates controller info with new fields
+func (s *sqlDatabase) migrateControllerInfo(hasMinAgeField, hasAgentURL bool) error {
+	if hasMinAgeField && hasAgentURL {
+		return nil
+	}
+
+	var controller ControllerInfo
+	if err := s.conn.First(&controller).Error; err != nil {
+		if errors.Is(err, gorm.ErrRecordNotFound) {
+			return nil
+		}
+		return fmt.Errorf("error fetching controller info: %w", err)
+	}
+
+	if !hasMinAgeField {
+		controller.MinimumJobAgeBackoff = 30
+	}
+
+	if controller.GARMAgentReleasesURL == "" {
+		controller.GARMAgentReleasesURL = appdefaults.GARMAgentDefaultReleasesURL
+	}
+
+	if !hasAgentURL && controller.WebhookBaseURL != "" {
+		matchWebhooksPath := regexp.MustCompile(`/webhooks(/)?$`)
+		controller.AgentURL = matchWebhooksPath.ReplaceAllLiteralString(controller.WebhookBaseURL, `/agent`)
+	}
+
+	if err := s.conn.Save(&controller).Error; err != nil {
+		return fmt.Errorf("error updating controller info: %w", err)
+	}
+
+	return nil
+}
+
+// preMigrationChecks performs checks before running migrations
+func (s *sqlDatabase) preMigrationChecks() (needsCredentialMigration, migrateTemplates, hasMinAgeField, hasAgentURL bool) {
+	// Check if credentials need migration
+	needsCredentialMigration = !s.conn.Migrator().HasTable(&GithubCredentials{}) ||
+		!s.conn.Migrator().HasTable(&GithubEndpoint{})
+
+	// Check if templates need migration
+	migrateTemplates = !s.conn.Migrator().HasTable(&Template{})
+
+	// Check for controller info fields
+	if s.conn.Migrator().HasTable(&ControllerInfo{}) {
+		hasMinAgeField = s.conn.Migrator().HasColumn(&ControllerInfo{}, "minimum_job_age_backoff")
+		hasAgentURL = s.conn.Migrator().HasColumn(&ControllerInfo{}, "agent_url")
+	}
+
+	return
+}
+
 func (s *sqlDatabase) migrateDB() error {
-	if s.conn.Migrator().HasIndex(&Organization{}, "idx_organizations_name") {
-		if err := s.conn.Migrator().DropIndex(&Organization{}, "idx_organizations_name"); err != nil {
-			slog.With(slog.Any("error", err)).Error("failed to drop index idx_organizations_name")
-		}
-	}
-
-	if s.conn.Migrator().HasIndex(&Repository{}, "idx_owner") {
-		if err := s.conn.Migrator().DropIndex(&Repository{}, "idx_owner"); err != nil {
-			slog.With(slog.Any("error", err)).Error("failed to drop index idx_owner")
-		}
-	}
+	// Drop obsolete indexes
+	s.dropIndexIfExists(&Organization{}, "idx_organizations_name")
+	s.dropIndexIfExists(&Repository{}, "idx_owner")
 
+	// Run cascade migration
 	if err := s.cascadeMigration(); err != nil {
 		return fmt.Errorf("error running cascade migration: %w", err)
 	}
 
-	if s.conn.Migrator().HasTable(&Pool{}) {
-		if err := s.conn.Exec("update pools set repo_id=NULL where repo_id='00000000-0000-0000-0000-000000000000'").Error; err != nil {
-			return fmt.Errorf("error updating pools %w", err)
-		}
-
-		if err := s.conn.Exec("update pools set org_id=NULL where org_id='00000000-0000-0000-0000-000000000000'").Error; err != nil {
-			return fmt.Errorf("error updating pools: %w", err)
-		}
-
-		if err := s.conn.Exec("update pools set enterprise_id=NULL where enterprise_id='00000000-0000-0000-0000-000000000000'").Error; err != nil {
-			return fmt.Errorf("error updating pools: %w", err)
-		}
+	// Migrate pool null IDs
+	if err := s.migratePoolNullIDs(); err != nil {
+		return err
 	}
 
+	// Migrate workflows
 	if err := s.migrateWorkflow(); err != nil {
 		return fmt.Errorf("error migrating workflows: %w", err)
 	}
 
-	if s.conn.Migrator().HasTable(&GithubEndpoint{}) {
-		if !s.conn.Migrator().HasColumn(&GithubEndpoint{}, "endpoint_type") {
-			if err := s.conn.Migrator().AutoMigrate(&GithubEndpoint{}); err != nil {
-				return fmt.Errorf("error migrating github endpoints: %w", err)
-			}
-			if err := s.conn.Exec("update github_endpoints set endpoint_type = 'github' where endpoint_type is null").Error; err != nil {
-				return fmt.Errorf("error updating github endpoints: %w", err)
-			}
-		}
+	// Migrate GitHub endpoint type
+	if err := s.migrateGithubEndpointType(); err != nil {
+		return err
 	}
 
-	var needsCredentialMigration bool
-	if !s.conn.Migrator().HasTable(&GithubCredentials{}) || !s.conn.Migrator().HasTable(&GithubEndpoint{}) {
-		needsCredentialMigration = true
-	}
-
-	var hasMinAgeField bool
-	if s.conn.Migrator().HasTable(&ControllerInfo{}) && s.conn.Migrator().HasColumn(&ControllerInfo{}, "minimum_job_age_backoff") {
-		hasMinAgeField = true
-	}
-
-	migrateTemplates := !s.conn.Migrator().HasTable(&Template{})
+	// Check if we need to migrate credentials and templates
+	needsCredentialMigration, migrateTemplates, hasMinAgeField, hasAgentURL := s.preMigrationChecks()
 
+	// Run main schema migration
 	s.conn.Exec("PRAGMA foreign_keys = OFF")
 	if err := s.conn.AutoMigrate(
 		&User{},
@@ -672,30 +774,24 @@ func (s *sqlDatabase) migrateDB() error {
 
 	s.conn.Exec("PRAGMA foreign_keys = ON")
 
-	if !hasMinAgeField {
-		var controller ControllerInfo
-		if err := s.conn.First(&controller).Error; err != nil {
-			if !errors.Is(err, gorm.ErrRecordNotFound) {
-				return fmt.Errorf("error updating controller info: %w", err)
-			}
-		} else {
-			controller.MinimumJobAgeBackoff = 30
-			if err := s.conn.Save(&controller).Error; err != nil {
-				return fmt.Errorf("error updating controller info: %w", err)
-			}
-		}
+	// Migrate controller info if needed
+	if err := s.migrateControllerInfo(hasMinAgeField, hasAgentURL); err != nil {
+		return err
 	}
 
+	// Ensure github endpoint exists
 	if err := s.ensureGithubEndpoint(); err != nil {
 		return fmt.Errorf("error ensuring github endpoint: %w", err)
 	}
 
+	// Migrate credentials if needed
 	if needsCredentialMigration {
 		if err := s.migrateCredentialsToDB(); err != nil {
 			return fmt.Errorf("error migrating credentials: %w", err)
 		}
 	}
 
+	// Ensure templates exist
 	if err := s.ensureTemplates(migrateTemplates); err != nil {
 		return fmt.Errorf("failed to create default templates: %w", err)
 	}

database/sql/templates.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/google/uuid"
 	"gorm.io/gorm"
 
 	runnerErrors "github.com/cloudbase/garm-provider-common/errors"
@@ -142,70 +143,40 @@ func (s *sqlDatabase) GetTemplateByName(ctx context.Context, name string) (param
 	return ret, nil
 }
 
-func (s *sqlDatabase) createSystemTemplate(ctx context.Context, param params.CreateTemplateParams) (template params.Template, err error) {
-	if !auth.IsAdmin(ctx) {
+func (s *sqlDatabase) CreateTemplate(ctx context.Context, param params.CreateTemplateParams) (template params.Template, err error) {
+	if param.IsSystem && !auth.IsAdmin(ctx) {
 		return params.Template{}, runnerErrors.ErrUnauthorized
 	}
-	defer func() {
-		if err == nil {
-			s.sendNotify(common.TemplateEntityType, common.CreateOperation, template)
+	var userID *uuid.UUID
+	if !param.IsSystem {
+		parsedID, err := getUIDFromContext(ctx)
+		if err != nil {
+			return params.Template{}, fmt.Errorf("error creating template: %w", err)
 		}
-	}()
-	sealed, err := s.marshalAndSeal(param.Data)
-	if err != nil {
-		return params.Template{}, fmt.Errorf("failed to seal data: %w", err)
-	}
-	tpl := Template{
-		UserID:      nil,
-		Name:        param.Name,
-		Description: param.Description,
-		OSType:      param.OSType,
-		Data:        sealed,
-		ForgeType:   param.ForgeType,
-	}
-
-	if err := s.conn.Create(&tpl).Error; err != nil {
-		if errors.Is(err, gorm.ErrDuplicatedKey) {
-			return params.Template{}, runnerErrors.NewConflictError("a template name already exists with the specified name")
-		}
-		return params.Template{}, fmt.Errorf("error creating template: %w", err)
-	}
-
-	template, err = s.sqlToParamTemplate(tpl)
-	if err != nil {
-		return params.Template{}, fmt.Errorf("failed to convert template: %w", err)
-	}
-
-	return template, nil
-}
-
-func (s *sqlDatabase) CreateTemplate(ctx context.Context, param params.CreateTemplateParams) (template params.Template, err error) {
-	userID, err := getUIDFromContext(ctx)
-	if err != nil {
-		return params.Template{}, fmt.Errorf("error creating template: %w", err)
+		userID = &parsedID
 	}
 	defer func() {
 		if err == nil {
 			s.sendNotify(common.TemplateEntityType, common.CreateOperation, template)
 		}
 	}()
-
-	sealed, err := s.marshalAndSeal(param.Data)
-	if err != nil {
-		return params.Template{}, fmt.Errorf("failed to seal data: %w", err)
-	}
-	tpl := Template{
-		UserID:      &userID,
-		Name:        param.Name,
-		Description: param.Description,
-		OSType:      param.OSType,
-		Data:        sealed,
-		ForgeType:   param.ForgeType,
-	}
 	if err := param.Validate(); err != nil {
 		return params.Template{}, fmt.Errorf("failed to validate create params: %w", err)
 	}
 
+	sealed, err := s.marshalAndSeal(param.Data)
+	if err != nil {
+		return params.Template{}, fmt.Errorf("failed to seal data: %w", err)
+	}
+	tpl := Template{
+		UserID:      userID,
+		Name:        param.Name,
+		Description: param.Description,
+		OSType:      param.OSType,
+		Data:        sealed,
+		ForgeType:   param.ForgeType,
+	}
+
 	if err := s.conn.Create(&tpl).Error; err != nil {
 		if errors.Is(err, gorm.ErrDuplicatedKey) {
 			return params.Template{}, runnerErrors.NewConflictError("a template name already exists with the specified name")

database/sql/templates_test.go

@@ -192,7 +192,7 @@ func (s *TemplatesTestSuite) TestListTemplatesWithForgeTypeFilter() {
 }
 
 func (s *TemplatesTestSuite) TestListTemplatesWithNameFilter() {
-	partialName := "system"
+	partialName := params.SystemUser
 	templates, err := s.Store.ListTemplates(s.adminCtx, nil, nil, &partialName)
 	s.Require().Nil(err)
 	s.Require().Len(templates, 1)
@@ -201,7 +201,7 @@ func (s *TemplatesTestSuite) TestListTemplatesWithNameFilter() {
 
 func (s *TemplatesTestSuite) TestListTemplatesDBFetchErr() {
 	s.Fixtures.SQLMock.
-		ExpectQuery(regexp.QuoteMeta("SELECT `templates`.`id`,`templates`.`created_at`,`templates`.`updated_at`,`templates`.`deleted_at`,`templates`.`name`,`templates`.`user_id`,`templates`.`description`,`templates`.`os_type`,`templates`.`forge_type` FROM `templates` WHERE `templates`.`deleted_at` IS NULL")).
+		ExpectQuery(regexp.QuoteMeta("SELECT `templates`.`id`,`templates`.`created_at`,`templates`.`updated_at`,`templates`.`deleted_at`,`templates`.`name`,`templates`.`user_id`,`templates`.`description`,`templates`.`os_type`,`templates`.`forge_type`,`templates`.`agent_mode` FROM `templates` WHERE `templates`.`deleted_at` IS NULL")).
 		WillReturnError(fmt.Errorf("mocked fetching templates error"))
 
 	_, err := s.StoreSQLMocked.ListTemplates(s.adminCtx, nil, nil, nil)
@@ -320,12 +320,13 @@ func (s *TemplatesTestSuite) TestCreateTemplateSystemAndUserConflict() {
 	// Now try to create a system template with the same name using direct access to createSystemTemplate
 	// This should succeed since the unique constraint is on (name, user_id) and system templates have user_id = NULL
 	sqlDB := s.Store.(*sqlDatabase)
-	_, err = sqlDB.createSystemTemplate(s.adminCtx, params.CreateTemplateParams{
+	_, err = sqlDB.CreateTemplate(s.adminCtx, params.CreateTemplateParams{
 		Name:        templateName,
 		Description: "System template with same name",
 		OSType:      commonParams.Windows,
 		ForgeType:   params.GithubEndpointType,
 		Data:        []byte(`{"provider": "azure", "image": "windows-2022"}`),
+		IsSystem:    true,
 	})
 
 	// This should succeed because system templates (user_id = NULL) and user templates

database/sql/util.go

@@ -19,6 +19,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"log/slog"
 
 	"github.com/google/uuid"
 	"gorm.io/datatypes"
@@ -73,6 +74,16 @@ func (s *sqlDatabase) sqlToParamsInstance(instance Instance) (params.Instance, e
 		JitConfiguration:  jitConfig,
 		GitHubRunnerGroup: instance.GitHubRunnerGroup,
 		AditionalLabels:   labels,
+		Heartbeat:         instance.Heartbeat,
+	}
+
+	if len(instance.Capabilities) > 0 {
+		var caps params.AgentCapabilities
+		if err := json.Unmarshal(instance.Capabilities, &caps); err == nil {
+			ret.Capabilities = caps
+		} else {
+			slog.ErrorContext(s.ctx, "failed to unmarshal capabilities", "instance_name", instance.Name, "error", err)
+		}
 	}
 
 	if instance.ScaleSetFkID != nil {
@@ -150,6 +161,7 @@ func (s *sqlDatabase) sqlToCommonOrganization(org Organization, detailed bool) (
 		Endpoint:         endpoint,
 		CreatedAt:        org.CreatedAt,
 		UpdatedAt:        org.UpdatedAt,
+		AgentMode:        org.AgentMode,
 	}
 
 	var forgeCreds params.ForgeCredentials
@@ -222,6 +234,7 @@ func (s *sqlDatabase) sqlToCommonEnterprise(enterprise Enterprise, detailed bool
 		CreatedAt:        enterprise.CreatedAt,
 		UpdatedAt:        enterprise.UpdatedAt,
 		Endpoint:         endpoint,
+		AgentMode:        enterprise.AgentMode,
 	}
 
 	if enterprise.CredentialsID != nil {
@@ -285,6 +298,7 @@ func (s *sqlDatabase) sqlToCommonPool(pool Pool) (params.Pool, error) {
 		Priority:               pool.Priority,
 		CreatedAt:              pool.CreatedAt,
 		UpdatedAt:              pool.UpdatedAt,
+		EnableShell:            pool.EnableShell,
 	}
 
 	if pool.TemplateID != nil && *pool.TemplateID != 0 {
@@ -361,6 +375,7 @@ func (s *sqlDatabase) sqlToCommonScaleSet(scaleSet ScaleSet) (params.ScaleSet, e
 		ExtendedState:          scaleSet.ExtendedState,
 		LastMessageID:          scaleSet.LastMessageID,
 		DesiredRunnerCount:     scaleSet.DesiredRunnerCount,
+		EnableShell:            scaleSet.EnableShell,
 	}
 
 	if scaleSet.TemplateID != nil && *scaleSet.TemplateID != 0 {
@@ -435,6 +450,7 @@ func (s *sqlDatabase) sqlToCommonRepository(repo Repository, detailed bool) (par
 		CreatedAt:        repo.CreatedAt,
 		UpdatedAt:        repo.UpdatedAt,
 		Endpoint:         endpoint,
+		AgentMode:        repo.AgentMode,
 	}
 
 	if repo.CredentialsID != nil && repo.GiteaCredentialsID != nil {
@@ -531,6 +547,10 @@ func (s *sqlDatabase) updatePool(tx *gorm.DB, pool Pool, param params.UpdatePool
 		pool.Flavor = param.Flavor
 	}
 
+	if param.EnableShell != nil {
+		pool.EnableShell = *param.EnableShell
+	}
+
 	if param.Image != "" {
 		pool.Image = param.Image
 	}
@@ -737,23 +757,35 @@ func (s *sqlDatabase) addRepositoryEvent(ctx context.Context, repoID string, eve
 		Message:    statusMessage,
 		EventType:  event,
 		EventLevel: eventLevel,
+		RepoID:     repo.ID,
 	}
 
-	if err := s.conn.Model(&repo).Association("Events").Append(&msg); err != nil {
+	// Use Create instead of Association.Append to avoid loading all existing events
+	if err := s.conn.Create(&msg).Error; err != nil {
 		return fmt.Errorf("error adding status message: %w", err)
 	}
 
 	if maxEvents > 0 {
-		var latestEvents []RepositoryEvent
-		q := s.conn.Model(&RepositoryEvent{}).
-			Limit(maxEvents).Order("id desc").
-			Where("repo_id = ?", repo.ID).Find(&latestEvents)
-		if q.Error != nil {
-			return fmt.Errorf("error fetching latest events: %w", q.Error)
+		var count int64
+		if err := s.conn.Model(&RepositoryEvent{}).Where("repo_id = ?", repo.ID).Count(&count).Error; err != nil {
+			return fmt.Errorf("error counting events: %w", err)
 		}
-		if len(latestEvents) == maxEvents {
-			lastInList := latestEvents[len(latestEvents)-1]
-			if err := s.conn.Where("repo_id = ? and id < ?", repo.ID, lastInList.ID).Unscoped().Delete(&RepositoryEvent{}).Error; err != nil {
+
+		if count > int64(maxEvents) {
+			// Get the ID of the Nth most recent event
+			var cutoffEvent RepositoryEvent
+			if err := s.conn.Model(&RepositoryEvent{}).
+				Select("id").
+				Where("repo_id = ?", repo.ID).
+				Order("id desc").
+				Offset(maxEvents - 1).
+				Limit(1).
+				First(&cutoffEvent).Error; err != nil {
+				return fmt.Errorf("error finding cutoff event: %w", err)
+			}
+
+			// Delete all events older than the cutoff
+			if err := s.conn.Where("repo_id = ? and id < ?", repo.ID, cutoffEvent.ID).Unscoped().Delete(&RepositoryEvent{}).Error; err != nil {
 				return fmt.Errorf("error deleting old events: %w", err)
 			}
 		}
@@ -771,23 +803,35 @@ func (s *sqlDatabase) addOrgEvent(ctx context.Context, orgID string, event param
 		Message:    statusMessage,
 		EventType:  event,
 		EventLevel: eventLevel,
+		OrgID:      org.ID,
 	}
 
-	if err := s.conn.Model(&org).Association("Events").Append(&msg); err != nil {
+	// Use Create instead of Association.Append to avoid loading all existing events
+	if err := s.conn.Create(&msg).Error; err != nil {
 		return fmt.Errorf("error adding status message: %w", err)
 	}
 
 	if maxEvents > 0 {
-		var latestEvents []OrganizationEvent
-		q := s.conn.Model(&OrganizationEvent{}).
-			Limit(maxEvents).Order("id desc").
-			Where("org_id = ?", org.ID).Find(&latestEvents)
-		if q.Error != nil {
-			return fmt.Errorf("error fetching latest events: %w", q.Error)
+		var count int64
+		if err := s.conn.Model(&OrganizationEvent{}).Where("org_id = ?", org.ID).Count(&count).Error; err != nil {
+			return fmt.Errorf("error counting events: %w", err)
 		}
-		if len(latestEvents) == maxEvents {
-			lastInList := latestEvents[len(latestEvents)-1]
-			if err := s.conn.Where("org_id = ? and id < ?", org.ID, lastInList.ID).Unscoped().Delete(&OrganizationEvent{}).Error; err != nil {
+
+		if count > int64(maxEvents) {
+			// Get the ID of the Nth most recent event
+			var cutoffEvent OrganizationEvent
+			if err := s.conn.Model(&OrganizationEvent{}).
+				Select("id").
+				Where("org_id = ?", org.ID).
+				Order("id desc").
+				Offset(maxEvents - 1).
+				Limit(1).
+				First(&cutoffEvent).Error; err != nil {
+				return fmt.Errorf("error finding cutoff event: %w", err)
+			}
+
+			// Delete all events older than the cutoff
+			if err := s.conn.Where("org_id = ? and id < ?", org.ID, cutoffEvent.ID).Unscoped().Delete(&OrganizationEvent{}).Error; err != nil {
 				return fmt.Errorf("error deleting old events: %w", err)
 			}
 		}
@@ -802,26 +846,38 @@ func (s *sqlDatabase) addEnterpriseEvent(ctx context.Context, entID string, even
 	}
 
 	msg := EnterpriseEvent{
-		Message:    statusMessage,
-		EventType:  event,
-		EventLevel: eventLevel,
+		Message:      statusMessage,
+		EventType:    event,
+		EventLevel:   eventLevel,
+		EnterpriseID: ent.ID,
 	}
 
-	if err := s.conn.Model(&ent).Association("Events").Append(&msg); err != nil {
+	// Use Create instead of Association.Append to avoid loading all existing events
+	if err := s.conn.Create(&msg).Error; err != nil {
 		return fmt.Errorf("error adding status message: %w", err)
 	}
 
 	if maxEvents > 0 {
-		var latestEvents []EnterpriseEvent
-		q := s.conn.Model(&EnterpriseEvent{}).
-			Limit(maxEvents).Order("id desc").
-			Where("enterprise_id = ?", ent.ID).Find(&latestEvents)
-		if q.Error != nil {
-			return fmt.Errorf("error fetching latest events: %w", q.Error)
+		var count int64
+		if err := s.conn.Model(&EnterpriseEvent{}).Where("enterprise_id = ?", ent.ID).Count(&count).Error; err != nil {
+			return fmt.Errorf("error counting events: %w", err)
 		}
-		if len(latestEvents) == maxEvents {
-			lastInList := latestEvents[len(latestEvents)-1]
-			if err := s.conn.Where("enterprise_id = ? and id < ?", ent.ID, lastInList.ID).Unscoped().Delete(&EnterpriseEvent{}).Error; err != nil {
+
+		if count > int64(maxEvents) {
+			// Get the ID of the Nth most recent event
+			var cutoffEvent EnterpriseEvent
+			if err := s.conn.Model(&EnterpriseEvent{}).
+				Select("id").
+				Where("enterprise_id = ?", ent.ID).
+				Order("id desc").
+				Offset(maxEvents - 1).
+				Limit(1).
+				First(&cutoffEvent).Error; err != nil {
+				return fmt.Errorf("error finding cutoff event: %w", err)
+			}
+
+			// Delete all events older than the cutoff
+			if err := s.conn.Where("enterprise_id = ? and id < ?", ent.ID, cutoffEvent.ID).Unscoped().Delete(&EnterpriseEvent{}).Error; err != nil {
 				return fmt.Errorf("error deleting old events: %w", err)
 			}
 		}
@@ -996,7 +1052,7 @@ func (s *sqlDatabase) sqlToParamTemplate(template Template) (params.Template, er
 		}
 	}
 
-	owner := "system"
+	owner := params.SystemUser
 	if template.UserID != nil {
 		owner = template.User.Username
 	}

database/watcher/filters.go

@@ -333,3 +333,17 @@ func WithInstanceStatusFilter(statuses ...commonParams.InstanceStatus) dbCommon.
 		return false
 	}
 }
+
+func WithInstanceFilter(instance params.Instance) dbCommon.PayloadFilterFunc {
+	return func(payload dbCommon.ChangePayload) bool {
+		if payload.EntityType != dbCommon.InstanceEntityType {
+			return false
+		}
+		payloadInstance, ok := payload.Payload.(params.Instance)
+		if !ok {
+			return false
+		}
+
+		return payloadInstance.Name == instance.Name
+	}
+}

database/watcher/watcher_store_test.go

@@ -169,7 +169,7 @@ func (s *WatcherStoreTestSuite) TestInstanceWatcher() {
 	creds := garmTesting.CreateTestGithubCredentials(s.ctx, "test-creds", s.store, s.T(), ep)
 	s.T().Cleanup(func() { s.store.DeleteGithubCredentials(s.ctx, creds.ID) })
 
-	repo, err := s.store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotEmpty(repo.ID)
 	s.T().Cleanup(func() { s.store.DeleteRepository(s.ctx, repo.ID) })
@@ -193,10 +193,11 @@ func (s *WatcherStoreTestSuite) TestInstanceWatcher() {
 	s.T().Cleanup(func() { s.store.DeleteEntityPool(s.ctx, entity, pool.ID) })
 
 	createInstanceParams := params.CreateInstanceParams{
-		Name:   "test-instance",
-		OSType: commonParams.Linux,
-		OSArch: commonParams.Amd64,
-		Status: commonParams.InstanceCreating,
+		Name:         "test-instance",
+		OSType:       commonParams.Linux,
+		OSArch:       commonParams.Amd64,
+		Status:       commonParams.InstanceCreating,
+		RunnerStatus: params.RunnerIdle,
 	}
 	instance, err := s.store.CreateInstance(s.ctx, pool.ID, createInstanceParams)
 	s.Require().NoError(err)
@@ -274,7 +275,7 @@ func (s *WatcherStoreTestSuite) TestScaleSetInstanceWatcher() {
 	creds := garmTesting.CreateTestGithubCredentials(s.ctx, "test-creds", s.store, s.T(), ep)
 	s.T().Cleanup(func() { s.store.DeleteGithubCredentials(s.ctx, creds.ID) })
 
-	repo, err := s.store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotEmpty(repo.ID)
 	s.T().Cleanup(func() { s.store.DeleteRepository(s.ctx, repo.ID) })
@@ -299,10 +300,11 @@ func (s *WatcherStoreTestSuite) TestScaleSetInstanceWatcher() {
 	s.T().Cleanup(func() { s.store.DeleteScaleSetByID(s.ctx, scaleSet.ID) })
 
 	createInstanceParams := params.CreateInstanceParams{
-		Name:   "test-instance",
-		OSType: commonParams.Linux,
-		OSArch: commonParams.Amd64,
-		Status: commonParams.InstanceCreating,
+		Name:         "test-instance",
+		OSType:       commonParams.Linux,
+		OSArch:       commonParams.Amd64,
+		Status:       commonParams.InstanceCreating,
+		RunnerStatus: params.RunnerIdle,
 	}
 	instance, err := s.store.CreateScaleSetInstance(s.ctx, scaleSet.ID, createInstanceParams)
 	s.Require().NoError(err)
@@ -384,7 +386,7 @@ func (s *WatcherStoreTestSuite) TestPoolWatcher() {
 		}
 	})
 
-	repo, err := s.store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotEmpty(repo.ID)
 	s.T().Cleanup(func() { s.store.DeleteRepository(s.ctx, repo.ID) })
@@ -506,7 +508,7 @@ func (s *WatcherStoreTestSuite) TestScaleSetWatcher() {
 		}
 	})
 
-	repo, err := s.store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotEmpty(repo.ID)
 	s.T().Cleanup(func() { s.store.DeleteRepository(s.ctx, repo.ID) })
@@ -667,7 +669,7 @@ func (s *WatcherStoreTestSuite) TestEnterpriseWatcher() {
 	creds := garmTesting.CreateTestGithubCredentials(s.ctx, "test-creds", s.store, s.T(), ep)
 	s.T().Cleanup(func() { s.store.DeleteGithubCredentials(s.ctx, creds.ID) })
 
-	ent, err := s.store.CreateEnterprise(s.ctx, "test-enterprise", creds, "test-secret", params.PoolBalancerTypeRoundRobin)
+	ent, err := s.store.CreateEnterprise(s.ctx, "test-enterprise", creds, "test-secret", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotEmpty(ent.ID)
 
@@ -734,7 +736,7 @@ func (s *WatcherStoreTestSuite) TestOrgWatcher() {
 	creds := garmTesting.CreateTestGithubCredentials(s.ctx, "test-creds", s.store, s.T(), ep)
 	s.T().Cleanup(func() { s.store.DeleteGithubCredentials(s.ctx, creds.ID) })
 
-	org, err := s.store.CreateOrganization(s.ctx, "test-org", creds, "test-secret", params.PoolBalancerTypeRoundRobin)
+	org, err := s.store.CreateOrganization(s.ctx, "test-org", creds, "test-secret", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotEmpty(org.ID)
 
@@ -801,7 +803,7 @@ func (s *WatcherStoreTestSuite) TestRepoWatcher() {
 	creds := garmTesting.CreateTestGithubCredentials(s.ctx, "test-creds", s.store, s.T(), ep)
 	s.T().Cleanup(func() { s.store.DeleteGithubCredentials(s.ctx, creds.ID) })
 
-	repo, err := s.store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin)
+	repo, err := s.store.CreateRepository(s.ctx, "test-owner", "test-repo", creds, "test-secret", params.PoolBalancerTypeRoundRobin, false)
 	s.Require().NoError(err)
 	s.Require().NotEmpty(repo.ID)
 

go.mod

@@ -31,6 +31,7 @@ require (
 	golang.org/x/mod v0.31.0
 	golang.org/x/oauth2 v0.34.0
 	golang.org/x/sync v0.19.0
+	golang.org/x/term v0.38.0
 	gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gorm.io/datatypes v1.2.7
@@ -101,7 +102,6 @@ require (
 	go.yaml.in/yaml/v3 v3.0.4 // indirect
 	golang.org/x/net v0.48.0 // indirect
 	golang.org/x/sys v0.39.0 // indirect
-	golang.org/x/term v0.38.0 // indirect
 	golang.org/x/text v0.32.0 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

internal/templates/templates.go

@@ -26,15 +26,11 @@ type WrapperContext struct {
 
 func GetTemplateContent(osType commonParams.OSType, forge params.EndpointType) ([]byte, error) {
 	switch forge {
-	case params.GithubEndpointType:
+	case params.GithubEndpointType, params.GiteaEndpointType:
 		switch osType {
 		case commonParams.Linux, commonParams.Windows:
 		default:
-			return nil, runnerErrors.NewNotFoundError("could not find template for forge github and OS type: %q", osType)
-		}
-	case params.GiteaEndpointType:
-		if osType != commonParams.Linux {
-			return nil, runnerErrors.NewNotFoundError("could not find template for gitea with OS type: %q", osType)
+			return nil, runnerErrors.NewNotFoundError("could not find template for forge %s and OS type: %q", forge, osType)
 		}
 	default:
 		return nil, runnerErrors.NewNotFoundError("could not find template for forge type: %q", forge)

internal/templates/userdata/gitea_linux_userdata.tmpl

@@ -55,6 +55,80 @@ function fail() {
 	exit 1
 }
 
+INSTANCE_METADATA=$(curl \
+    --retry 5 \
+    --retry-delay 5 \
+    --retry-connrefused \
+    --fail -s \
+    -H 'Accept: application/json' \
+    -H "Authorization: Bearer ${BEARER_TOKEN}" \
+    "$METADATA_URL/runner-metadata" 2>&1) || fail "failed to get instance metadata: $INSTANCE_METADATA"
+
+AGENT_MODE=$(echo "$INSTANCE_METADATA" | jq -r '.agent_mode // empty')
+if [ "$AGENT_MODE" == "true" ]; then
+    sendStatus "Agent mode is enabled; setting up agent"
+    DOWNLOAD_URL=$(echo "$INSTANCE_METADATA" | jq -r '.agent_tools.download_url // empty')
+    if [ -z "$DOWNLOAD_URL" ]; then
+        fail "agent mode is enabled, but no agent tools found in metadata"
+    fi
+    AGENT_URL=$(echo "$INSTANCE_METADATA" | jq -r '.metadata_access.agent_url // empty')
+    if [ -z "$AGENT_URL" ]; then
+        fail "agent mode is enabled, but agent_url was not found in metadata"
+    fi
+    AGENT_TOKEN=$(echo "$INSTANCE_METADATA" | jq -r '.agent_token // empty')
+    if [ -z "$AGENT_TOKEN" ]; then
+        fail "agent mode is enabled, but agent_token was not found in metadata"
+    fi
+    AGENT_SHELL=$(echo "$INSTANCE_METADATA" | jq -r '.agent_shell_enabled // false')
+    sendStatus "Downloading agent from $DOWNLOAD_URL"
+    sudo curl --retry 5 \
+        --retry-delay 5 \
+        --retry-connrefused \
+        --fail -L \
+        -H "Authorization: Bearer ${BEARER_TOKEN}" \
+        -o /usr/local/bin/garm-agent "$DOWNLOAD_URL" || fail "failed to download garm-agent"
+    sudo chmod +x /usr/local/bin/garm-agent || fail "failed to make garm-agent executable"
+    sudo mkdir -p /var/log/garm-agent || fail "failed to create /var/log/garm-agent"
+    sudo chown {{ .RunnerUsername }}:{{ .RunnerUsername }} /var/log/garm-agent || fail "failed to chown /var/log/garm-agent"
+    sudo mkdir -p /etc/garm-agent || fail "failed to create /etc/garm"
+    sudo chown {{ .RunnerUsername }}:{{ .RunnerUsername }} /etc/garm-agent || fail "failed to change owner on /etc/garm-agent"
+
+    sendStatus "Creating config and systemd unit"
+    cat > /etc/garm-agent/garm-agent.toml << EOF
+server_url = "$AGENT_URL"
+log_file = "/var/log/garm-agent/garm-agent.log"
+work_dir = "$RUN_HOME"
+enable_shell = $AGENT_SHELL
+token = "$AGENT_TOKEN"
+runner_cmdline = ["$RUN_HOME/act_runner", "daemon", "--once"]
+state_db_path = "/etc/garm-agent/agent-state.db"
+EOF
+
+    cat > /tmp/garm-agent.service << EOF
+[Unit]
+Description=GARM agent
+After=multi-user.target
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/garm-agent daemon --config /etc/garm-agent/garm-agent.toml
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+RestartSec=5s
+User={{ .RunnerUsername }}
+Environment=TERM=xterm-256color
+Environment=LANG=en_US.UTF-8
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+    sudo mv /tmp/garm-agent.service /etc/systemd/system/garm-agent.service || fail "failed to create /etc/systemd/system/garm-agent.service"
+    sudo chown root:root /etc/systemd/system/garm-agent.service || fail "failed to change owner on /etc/systemd/system/garm-agent.service"
+    sendStatus "Reloading systemd unit"
+    sudo systemctl daemon-reload || fail "failed to reload systemd"
+fi
+
 function downloadAndExtractRunner() {
 	sendStatus "downloading tools from {{ .DownloadURL }}"
     mkdir -p "$RUN_HOME" || fail "failed to create actions-runner folder"
@@ -124,7 +198,12 @@ sudo systemctl daemon-reload || fail "failed to reload systemd"
 
 AGENT_ID=""
 sendStatus "starting service"
-sudo systemctl enable --now $SVC_NAME
+if [ "$AGENT_MODE" == "true" ]; then
+    sendStatus "Enabling garm-agent service"
+    sudo systemctl enable --now garm-agent
+else
+    sudo systemctl enable --now $SVC_NAME
+fi
 
 set +e
 AGENT_ID=$(grep '"id"' "$RUN_HOME"/.runner |  tr -d -c 0-9)

internal/templates/userdata/gitea_windows_userdata.tmpl

@@ -0,0 +1,596 @@
+#ps1_sysnative
+Param(
+	[Parameter(Mandatory=$false)]
+	[string]$Token="{{.CallbackToken}}"
+)
+
+$ErrorActionPreference="Stop"
+
+function Start-ExecuteWithRetry {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory=$true)]
+        [ScriptBlock]$ScriptBlock,
+        [int]$MaxRetryCount=10,
+        [int]$RetryInterval=3,
+        [string]$RetryMessage,
+        [array]$ArgumentList=@()
+    )
+    PROCESS {
+        $currentErrorActionPreference = $ErrorActionPreference
+        $ErrorActionPreference = "Continue"
+        $retryCount = 0
+        while ($true) {
+            try {
+                $res = Invoke-Command -ScriptBlock $ScriptBlock -ArgumentList $ArgumentList
+                $ErrorActionPreference = $currentErrorActionPreference
+                return $res
+            } catch [System.Exception] {
+                $retryCount++
+
+                if ($_.Exception -is [System.Net.WebException]) {
+                    $webResponse = $_.Exception.Response
+					# Skip retry on Error: 4XX (e.g. 401 Unauthorized, 404 Not Found etc.)
+                    if ($webResponse -and $webResponse.StatusCode -ge 400 -and $webResponse.StatusCode -lt 500) {
+                        # Skip retry on 4xx errors
+                        Write-Output "Encountered non-retryable error (4xx): $($_.Exception.Message)"
+                        $ErrorActionPreference = $currentErrorActionPreference
+                        throw
+                    }
+                }
+
+                if ($retryCount -gt $MaxRetryCount) {
+                    $ErrorActionPreference = $currentErrorActionPreference
+                    throw
+                } else {
+                    if ($RetryMessage) {
+                        Write-Output $RetryMessage
+                    } elseif ($_) {
+                        Write-Output $_
+                    }
+                    Start-Sleep -Seconds $RetryInterval
+                }
+            }
+        }
+    }
+}
+
+function Get-RandomString {
+    [CmdletBinding()]
+    Param(
+        [int]$Length=13
+    )
+    PROCESS {
+        if($Length -lt 6) {
+            $Length = 6
+        }
+        $special = @(44, 45, 46, 64)
+        $numeric = 48..57
+        $upper = 65..90
+        $lower = 97..122
+
+        $passwd = [System.Collections.Generic.List[object]](New-object "System.Collections.Generic.List[object]")
+        for($i=0; $i -lt $Length-4; $i++){
+            $c = get-random -input ($special + $numeric + $upper + $lower)
+            $passwd.Add([char]$c)
+        }
+
+        $passwd.Add([char](get-random -input $numeric))
+        $passwd.Add([char](get-random -input $special))
+        $passwd.Add([char](get-random -input $upper))
+        $passwd.Add([char](get-random -input $lower))
+
+        $Random = New-Object Random
+        return [string]::join("",($passwd|Sort-Object {$Random.Next()}))
+    }
+}
+
+Add-Type -TypeDefinition @"
+using System;
+using System.Runtime.InteropServices;
+using System.Text;
+
+public class GrantSysPrivileges
+{
+    [StructLayout(LayoutKind.Sequential)]
+    public struct LSA_UNICODE_STRING
+    {
+        public ushort Length;
+        public ushort MaximumLength;
+        public IntPtr Buffer;
+    }
+
+    [StructLayout(LayoutKind.Sequential)]
+    public struct LSA_OBJECT_ATTRIBUTES
+    {
+        public int Length;
+        public IntPtr RootDirectory;
+        public IntPtr ObjectName;
+        public uint Attributes;
+        public IntPtr SecurityDescriptor;
+        public IntPtr SecurityQualityOfService;
+    }
+
+    [DllImport("advapi32.dll", SetLastError=true)]
+    public static extern uint LsaOpenPolicy(
+        ref LSA_UNICODE_STRING SystemName,
+        ref LSA_OBJECT_ATTRIBUTES ObjectAttributes,
+        uint DesiredAccess,
+        out IntPtr PolicyHandle
+    );
+
+    [DllImport("advapi32.dll", SetLastError=true)]
+    public static extern uint LsaAddAccountRights(
+        IntPtr PolicyHandle,
+        IntPtr AccountSid,
+        LSA_UNICODE_STRING[] UserRights,
+        uint CountOfRights
+    );
+
+    [DllImport("advapi32.dll")]
+    public static extern uint LsaClose(IntPtr PolicyHandle);
+
+    [DllImport("advapi32.dll")]
+    public static extern uint LsaNtStatusToWinError(uint status);
+
+    public const uint POLICY_ALL_ACCESS = 0x00F0FFF;
+
+    public static uint GrantPrivilege(byte[] sid, string[] rights)
+    {
+        LSA_OBJECT_ATTRIBUTES loa = new LSA_OBJECT_ATTRIBUTES();
+        LSA_UNICODE_STRING systemName = new LSA_UNICODE_STRING();
+
+        IntPtr policyHandle;
+        uint result = LsaOpenPolicy(ref systemName, ref loa, POLICY_ALL_ACCESS, out policyHandle);
+        if (result != 0)
+        {
+            return LsaNtStatusToWinError(result);
+        }
+
+        LSA_UNICODE_STRING[] userRights = new LSA_UNICODE_STRING[rights.Length];
+        for (int i = 0; i < rights.Length; i++)
+        {
+            byte[] bytes = Encoding.Unicode.GetBytes(rights[i]);
+            IntPtr ptr = Marshal.AllocHGlobal(bytes.Length);
+            Marshal.Copy(bytes, 0, ptr, bytes.Length);
+
+            userRights[i].Buffer = ptr;
+            userRights[i].Length = (ushort)bytes.Length;
+            userRights[i].MaximumLength = (ushort)(bytes.Length);
+        }
+
+        IntPtr sidPtr = Marshal.AllocHGlobal(sid.Length);
+        Marshal.Copy(sid, 0, sidPtr, sid.Length);
+
+        result = LsaAddAccountRights(policyHandle, sidPtr, userRights, (uint)rights.Length);
+        LsaClose(policyHandle);
+
+        foreach (var right in userRights)
+        {
+            Marshal.FreeHGlobal(right.Buffer);
+        }
+        Marshal.FreeHGlobal(sidPtr);
+
+        return LsaNtStatusToWinError(result);
+    }
+}
+"@ -Language CSharp
+
+function Invoke-FastWebRequest {
+	[CmdletBinding()]
+	Param(
+		[Parameter(Mandatory=$True,ValueFromPipeline=$true,Position=0)]
+		[System.Uri]$Uri,
+		[Parameter(Position=1)]
+		[string]$OutFile,
+		[Hashtable]$Headers=@{},
+		[switch]$SkipIntegrityCheck=$false
+	)
+	PROCESS
+	{
+		if(!([System.Management.Automation.PSTypeName]'System.Net.Http.HttpClient').Type)
+		{
+			$assembly = [System.Reflection.Assembly]::LoadWithPartialName("System.Net.Http")
+		}
+
+		if(!$OutFile) {
+			$OutFile = $Uri.PathAndQuery.Substring($Uri.PathAndQuery.LastIndexOf("/") + 1)
+			if(!$OutFile) {
+				throw "The ""OutFile"" parameter needs to be specified"
+			}
+		}
+
+		$fragment = $Uri.Fragment.Trim('#')
+		if ($fragment) {
+			$details = $fragment.Split("=")
+			$algorithm = $details[0]
+			$hash = $details[1]
+		}
+
+		if (!$SkipIntegrityCheck -and $fragment -and (Test-Path $OutFile)) {
+			try {
+				return (Test-FileIntegrity -File $OutFile -Algorithm $algorithm -ExpectedHash $hash)
+			} catch {
+				Remove-Item $OutFile
+			}
+		}
+
+		$client = new-object System.Net.Http.HttpClient
+		foreach ($k in $Headers.Keys){
+			$client.DefaultRequestHeaders.Add($k, $Headers[$k])
+		}
+		$task = $client.GetStreamAsync($Uri)
+		$response = $task.Result
+		if($task.IsFaulted) {
+			$msg = "Request for URL '{0}' is faulted. Task status: {1}." -f @($Uri, $task.Status)
+			if($task.Exception) {
+				$msg += "Exception details: {0}" -f @($task.Exception)
+			}
+			Throw $msg
+		}
+		$outStream = New-Object IO.FileStream $OutFile, Create, Write, None
+
+		try {
+			$totRead = 0
+			$buffer = New-Object Byte[] 1MB
+			while (($read = $response.Read($buffer, 0, $buffer.Length)) -gt 0) {
+				$totRead += $read
+				$outStream.Write($buffer, 0, $read);
+			}
+		}
+		finally {
+			$outStream.Close()
+		}
+		if(!$SkipIntegrityCheck -and $fragment) {
+			Test-FileIntegrity -File $OutFile -Algorithm $algorithm -ExpectedHash $hash
+		}
+	}
+}
+
+function Import-Certificate() {
+	[CmdletBinding()]
+	param (
+		[parameter(Mandatory=$true)]
+		$CertificateData,
+		[parameter(Mandatory=$false)]
+		[System.Security.Cryptography.X509Certificates.StoreLocation]$StoreLocation="LocalMachine",
+		[parameter(Mandatory=$false)]
+		[System.Security.Cryptography.X509Certificates.StoreName]$StoreName="TrustedPublisher"
+	)
+	PROCESS
+	{
+		$store = New-Object System.Security.Cryptography.X509Certificates.X509Store(
+			$StoreName, $StoreLocation)
+		$store.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite)
+		$cert = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new($CertificateData)
+		$store.Add($cert)
+	}
+}
+
+function Invoke-APICall() {
+	[CmdletBinding()]
+	param (
+		[parameter(Mandatory=$true)]
+		[object]$Payload,
+		[parameter(Mandatory=$true)]
+		[string]$CallbackURL
+	)
+	PROCESS{
+		Invoke-WebRequest -UseBasicParsing -Method Post -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $CallbackURL -Body (ConvertTo-Json $Payload) | Out-Null
+	}
+}
+
+function Update-GarmStatus() {
+	[CmdletBinding()]
+	param (
+		[parameter(Mandatory=$true)]
+		[string]$Message,
+		[parameter(Mandatory=$false)]
+		[int64]$AgentID=0,
+		[parameter(Mandatory=$false)]
+		[string]$Status="installing",
+		[parameter(Mandatory=$true)]
+		[string]$CallbackURL
+	)
+	PROCESS{
+		$body = @{
+			"status"=$Status
+			"message"=$Message
+		}
+
+		if ($AgentID -ne 0) {
+			$body["agent_id"] = $AgentID
+		}
+		Invoke-APICall -Payload $body -CallbackURL $CallbackURL | Out-Null
+	}
+}
+
+function Invoke-GarmSuccess() {
+	[CmdletBinding()]
+	param (
+		[parameter(Mandatory=$true)]
+		[string]$Message,
+		[parameter(Mandatory=$true)]
+		[int64]$AgentID,
+		[parameter(Mandatory=$true)]
+		[string]$CallbackURL
+	)
+	PROCESS{
+		Update-GarmStatus -Message $Message -AgentID $AgentID -CallbackURL $CallbackURL -Status "idle" | Out-Null
+	}
+}
+
+function Invoke-GarmFailure() {
+	[CmdletBinding()]
+	param (
+		[parameter(Mandatory=$true)]
+		[string]$Message,
+		[parameter(Mandatory=$true)]
+		[string]$CallbackURL
+	)
+	PROCESS{
+		Update-GarmStatus -Message $Message -CallbackURL $CallbackURL -Status "failed" | Out-Null
+		Throw $Message
+	}
+}
+
+function Set-SystemInfo {
+    [CmdletBinding()]
+    param (
+        [parameter(Mandatory=$true)]
+        [string]$CallbackURL,
+        [parameter(Mandatory=$true)]
+        [string]$RunnerDir,
+		[parameter(Mandatory=$true)]
+        [string]$BearerToken
+    )
+
+    # Construct the path to the .runner file
+    $agentInfoFile = Join-Path $RunnerDir ".runner"
+
+    # Read and parse the JSON content from the .runner file
+    $agentInfo = ConvertFrom-Json (Get-Content -Raw -Path $agentInfoFile)
+    $AgentId = $agentInfo.agent_id
+
+    # Retrieve OS information
+    $osInfo = Get-WmiObject -Class Win32_OperatingSystem
+    $osName = $osInfo.Caption
+    $osVersion = $osInfo.Version
+
+    # Strip status from the callback URL
+    if ($CallbackUrl -match '^(.*)/status(/)?$') {
+        $CallbackUrl = $matches[1]
+    }
+
+    $SysInfoUrl = "$CallbackUrl/system-info/"
+    $Payload = @{
+        os_name    = $OSName
+        os_version = $OSVersion
+        agent_id   = $AgentId
+    } | ConvertTo-Json
+
+    # Send the POST request
+    try {
+        Invoke-RestMethod -Uri $SysInfoUrl -Method Post -Body $Payload -ContentType 'application/json' -Headers @{ 'Authorization' = "Bearer $BearerToken" } -ErrorAction Stop
+    } catch {
+        Write-Output "Failed to send the system information."
+    }
+}
+
+$CallbackURL="{{.CallbackURL}}"
+if (!($CallbackURL -match "^(.*)/status(/)?$")) {
+	$CallbackURL = "$CallbackURL/status"
+}
+$GHRunnerGroup = "{{.GitHubRunnerGroup}}"
+try {
+    $instanceMetadata = (wget -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri {{.MetadataURL}}/runner-metadata)
+    $metadata = ConvertFrom-Json $instanceMetadata.Content
+} catch {
+	Invoke-GarmFailure -Message "failed to get runner metadata: $_" -CallbackURL "{{.CallbackURL}}" | Out-Null
+}
+
+function Get-IsAgentMode {
+    return ($metadata.agent_mode -eq $true)
+}
+
+function Get-AgentURL {
+    $url = $metadata.metadata_access.agent_url
+    if (!$url) {
+        Throw("missing agent URL")
+    }
+    return $url
+}
+
+function Get-AgentToken {
+    $token = $metadata.agent_token
+    if (!$token) {
+        Throw("missing agent Token")
+    }
+    return $token
+}
+
+function Get-AgentDownloadURL {
+    $url = $metadata.agent_tools.download_url
+    if (!$url) {
+        Throw("missing agent download URL")
+    }
+    return $url
+}
+
+function Get-AgentShellEnabled {
+    $shellEnabled = $metadata.agent_shell_enabled
+    if ($shellEnabled) {return "true"}
+    return "false"
+}
+
+function Install-GarmAgent {
+    [CmdletBinding()]
+    param (
+        [parameter(Mandatory=$true)]
+        [System.Management.Automation.PSCredential]$pscreds,
+        [parameter(Mandatory=$true)]
+        [string]$runnerExecutable
+    )
+    Update-GarmStatus -Message "Agent mode is enabled" -CallbackURL $CallbackURL | Out-Null
+    $agentDir = "C:\garm-agent"
+    mkdir -Force $agentDir
+    $agentURL = Get-AgentURL
+    $agentDownloadURL = Get-AgentDownloadURL
+    $agentToken = Get-AgentToken
+    $agentDownloadHeaders=@{
+        "Authorization"="Bearer $Token"
+    }
+    $shellEnabled = Get-AgentShellEnabled
+    $runnerExecutable = $runnerExecutable.Replace('\', '/')
+    # '
+    Set-Content "$agentDir\garm-agent.toml" @"
+server_url = "$agentURL"
+log_file = "C:/garm-agent/garm-agent.log"
+shell = "cmd.exe"
+enable_shell = $shellEnabled
+work_dir = "C:/actions-runner/"
+token = "$agentToken"
+runner_cmdline = ["$runnerExecutable", "daemon", "--once"]
+state_db_path = "C:/garm-agent/agent-state.db"
+"@
+
+    Update-GarmStatus -Message "Downloading agent from: $agentDownloadURL" -CallbackURL $CallbackURL | Out-Null
+    Start-ExecuteWithRetry -ScriptBlock {
+        Invoke-FastWebRequest -Headers $agentDownloadHeaders -Uri "$agentDownloadURL" -OutFile $agentDir\garm-agent.exe
+    } -MaxRetryCount 5 -RetryInterval 5 -RetryMessage "Retrying download of runner..."
+
+    try {
+        New-Service -Name garm-agent -BinaryPathName "$agentDir\garm-agent.exe daemon --config $agentDir\garm-agent.toml" -DisplayName "garm-agent" -Description "GARM agent" -StartupType Automatic -Credential $pscreds
+    } catch {
+        Invoke-GarmFailure -CallbackURL $CallbackURL -Message "failed to set up garm agent $_"
+    }
+    Start-Service garm-agent
+}
+
+function Install-NSSM {
+    [CmdletBinding()]
+    param (
+        [parameter(Mandatory=$true)]
+        [string]$username,
+        [parameter(Mandatory=$true)]
+        [string]$password,
+        [parameter(Mandatory=$true)]
+        [string]$runnerDir
+    )
+
+	Start-ExecuteWithRetry -ScriptBlock {
+		Invoke-FastWebRequest -Uri "https://nssm.cc/ci/nssm-2.24-103-gdee49fc.zip" -OutFile "$runnerDir/nssm.zip"
+	} -MaxRetryCount 5 -RetryInterval 5 -RetryMessage "Retrying download of nssm..."
+	Expand-Archive -Path "$runnerDir/nssm.zip" -DestinationPath "$runnerDir/nssm" -Force
+    mv "$runnerDir\nssm\nssm-2.24-103-gdee49fc\win64\nssm.exe" "$runnerDir/nssm.exe"
+    rm -Recurse -Force "$runnerDir\nssm"
+    rm -Force "$runnerDir\nssm.zip"
+    $nssm="$runnerDir/nssm.exe"
+	& $nssm install GiteaActRunner "$runnerDir/act_runner.exe"
+	& $nssm set GiteaActRunner AppParameters daemon
+	& $nssm set GiteaActRunner AppStdout $runnerDir\stdout.log
+	& $nssm set GiteaActRunner AppStderr $runnerDir\stderr.log
+	& $nssm set GiteaActRunner AppStopMethodSkip 6
+	& $nssm set GiteaActRunner AppStopMethodConsole 1000
+	& $nssm set GiteaActRunner AppThrottle 5000
+    & $nssm set GiteaActRunner ObjectName $username $password
+	& $nssm start GiteaActRunner
+}
+
+
+
+function Install-Runner() {
+	$CallbackURL="{{.CallbackURL}}"
+	if (!($CallbackURL -match "^(.*)/status(/)?$")) {
+		$CallbackURL = "$CallbackURL/status"
+	}
+
+	if ($Token.Length -eq 0) {
+		Throw "missing callback authentication token"
+	}
+	try {
+		$MetadataURL="{{.MetadataURL}}"
+		$DownloadURL="{{.DownloadURL}}"
+		if($MetadataURL -eq ""){
+			Throw "missing metadata URL"
+		}
+        $runnerDir = "C:\actions-runner"
+        $runnerExecutable = Join-Path $runnerDir "act_runner.exe"
+
+		# Create user with administrator rights to run service as
+		$userPasswd = Get-RandomString -Length 10
+		$secPasswd = ConvertTo-SecureString "$userPasswd" -AsPlainText -Force
+		$userName = "runner"
+		$user = Get-LocalUser -Name $userName -ErrorAction SilentlyContinue
+		if (-not $user) {
+			New-LocalUser -Name $userName -Password $secPasswd -PasswordNeverExpires -UserMayNotChangePassword
+		} else {
+			Set-LocalUser -PasswordNeverExpires $true -Name $userName -Password $secPasswd
+		}
+		$pscreds = New-Object System.Management.Automation.PSCredential (".\$userName", $secPasswd)
+		$hasUser = Get-LocalGroupMember -SID S-1-5-32-544 -Member $userName -ErrorAction SilentlyContinue
+		if (-not $hasUser){
+			Add-LocalGroupMember -SID S-1-5-32-544 -Member $userName
+		}
+		$ntAcct = New-Object System.Security.Principal.NTAccount($userName)
+		$sid = $ntAcct.Translate([System.Security.Principal.SecurityIdentifier])
+		$sidBytes = New-Object byte[] ($sid.BinaryLength)
+		$sid.GetBinaryForm($sidBytes, 0)
+
+		$result = [GrantSysPrivileges]::GrantPrivilege($sidBytes, ("SeBatchLogonRight", "SeServiceLogonRight"))
+		if ($result -ne 0) {
+		    Throw "Failed to grant privileges"
+		}
+
+		$bundle = wget -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $MetadataURL/system/cert-bundle
+		$converted = ConvertFrom-Json $bundle
+		foreach ($i in $converted.root_certificates.psobject.Properties){
+			$data = [System.Convert]::FromBase64String($i.Value)
+			Import-Certificate -CertificateData $data -StoreName Root -StoreLocation LocalMachine
+		}
+
+
+		# Check if a cached runner is available
+		if (-not (Test-Path $runnerDir)) {
+			# No cached runner found, proceed to download and extract
+			Update-GarmStatus -CallbackURL $CallbackURL -Message "downloading tools from {{ .DownloadURL }}"
+            mkdir $runnerDir
+
+			Start-ExecuteWithRetry -ScriptBlock {
+				Invoke-FastWebRequest -Uri "{{ .DownloadURL }}" -OutFile $runnerExecutable
+			} -MaxRetryCount 5 -RetryInterval 5 -RetryMessage "Retrying download of runner..."
+		} else {
+			Update-GarmStatus -CallbackURL $CallbackURL -Message "using cached runner found at $runnerDir"
+		}
+
+		# Ensure runner has full access to actions-runner folder
+		$runnerACL = Get-Acl $runnerDir
+		$runnerACL.SetAccessRule((New-Object System.Security.AccessControl.FileSystemAccessRule(
+		    $userName, "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow"
+		)))
+		Set-Acl -Path $runnerDir -AclObject $runnerAcl
+
+		Update-GarmStatus -CallbackURL $CallbackURL -Message "configuring runner"
+		cd $runnerDir
+
+		$GithubRegistrationToken = Start-ExecuteWithRetry -ScriptBlock {
+			Invoke-WebRequest -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $MetadataURL/runner-registration-token/
+		} -MaxRetryCount 5 -RetryInterval 5 -RetryMessage "Retrying download of GitHub registration token..."
+        & $runnerExecutable register --ephemeral --no-interactive --instance "{{ .RepoURL }}" --token $GithubRegistrationToken --name "{{ .RunnerName }}" --labels "{{ .RunnerLabels }}"
+		if ($LASTEXITCODE) {
+			Throw "Failed to configure runner. Err code $LASTEXITCODE"
+		}
+		$agentInfoFile = Join-Path $runnerDir ".runner"
+		$agentInfo = ConvertFrom-Json (gc -raw $agentInfoFile)
+		Set-SystemInfo -CallbackURL $CallbackURL -RunnerDir $runnerDir -BearerToken $Token
+        if ((Get-IsAgentMode)) {
+            Install-GarmAgent -pscreds $pscreds -runnerExecutable $runnerExecutable
+        } else {
+            Install-NSSM -username $userName -password $userPasswd -runnerDir $runnerDir
+        }
+		Invoke-GarmSuccess -CallbackURL $CallbackURL -Message "runner successfully installed" -AgentID $agentInfo.id
+	} catch {
+		Invoke-GarmFailure -CallbackURL $CallbackURL -Message $_
+	}
+}
+Install-Runner

internal/templates/userdata/github_linux_userdata.tmpl

@@ -2,15 +2,24 @@
 
 set -e
 set -o pipefail
+set -x
 
 {{- if .EnableBootDebug }}
 set -x
 {{- end }}
 
+# Edit the templates directly in the browser. Intellisense is available for golang templates
+# {{ .DownloadURL }}
+
 CALLBACK_URL="{{ .CallbackURL }}"
 METADATA_URL="{{ .MetadataURL }}"
 BEARER_TOKEN="{{ .CallbackToken }}"
 
+# touch /tmp/hold
+# while true;do
+#    [ -e /tmp/hold ] && sleep 2 || break
+# done
+
 RUN_HOME="/home/{{ .RunnerUsername }}/actions-runner"
 
 if [ -z "$METADATA_URL" ];then
@@ -55,6 +64,80 @@ function fail() {
 	exit 1
 }
 
+INSTANCE_METADATA=$(curl \
+    --retry 5 \
+    --retry-delay 5 \
+    --retry-connrefused \
+    --fail -s \
+    -H 'Accept: application/json' \
+    -H "Authorization: Bearer ${BEARER_TOKEN}" \
+    "$METADATA_URL/runner-metadata" 2>&1) || fail "failed to get instance metadata: $INSTANCE_METADATA"
+
+AGENT_MODE=$(echo "$INSTANCE_METADATA" | jq -r '.agent_mode // empty')
+if [ "$AGENT_MODE" == "true" ]; then
+    sendStatus "Agent mode is enabled; setting up agent"
+    DOWNLOAD_URL=$(echo "$INSTANCE_METADATA" | jq -r '.agent_tools.download_url // empty')
+    if [ -z "$DOWNLOAD_URL" ]; then
+        fail "agent mode is enabled, but no agent tools found in metadata"
+    fi
+    AGENT_URL=$(echo "$INSTANCE_METADATA" | jq -r '.metadata_access.agent_url // empty')
+    if [ -z "$AGENT_URL" ]; then
+        fail "agent mode is enabled, but agent_url was not found in metadata"
+    fi
+    AGENT_TOKEN=$(echo "$INSTANCE_METADATA" | jq -r '.agent_token // empty')
+    if [ -z "$AGENT_TOKEN" ]; then
+        fail "agent mode is enabled, but agent_token was not found in metadata"
+    fi
+    AGENT_SHELL=$(echo "$INSTANCE_METADATA" | jq -r '.agent_shell_enabled // false')
+    sendStatus "Downloading agent from $DOWNLOAD_URL"
+    sudo curl --retry 5 \
+        --retry-delay 5 \
+        --retry-connrefused \
+        --fail -L \
+        -H "Authorization: Bearer ${BEARER_TOKEN}" \
+        -o /usr/local/bin/garm-agent "$DOWNLOAD_URL" || fail "failed to download garm-agent"
+    sudo chmod +x /usr/local/bin/garm-agent || fail "failed to make garm-agent executable"
+    sudo mkdir -p /var/log/garm-agent || fail "failed to create /var/log/garm-agent"
+    sudo chown {{ .RunnerUsername }}:{{ .RunnerUsername }} /var/log/garm-agent || fail "failed to chown /var/log/garm-agent"
+    sudo mkdir -p /etc/garm-agent || fail "failed to create /etc/garm"
+    sudo chown {{ .RunnerUsername }}:{{ .RunnerUsername }} /etc/garm-agent || fail "failed to change owner on /etc/garm-agent"
+
+    sendStatus "Creating config and systemd unit"
+    cat > /etc/garm-agent/garm-agent.toml << EOF
+server_url = "$AGENT_URL"
+log_file = "/var/log/garm-agent/garm-agent.log"
+work_dir = "$RUN_HOME"
+enable_shell = $AGENT_SHELL
+token = "$AGENT_TOKEN"
+runner_cmdline = ["/bin/bash", "-C", "/home/runner/actions-runner/run.sh"]
+state_db_path = "/etc/garm-agent/agent-state.db"
+EOF
+
+    cat > /tmp/garm-agent.service << EOF
+[Unit]
+Description=GARM agent
+After=multi-user.target
+
+[Service]
+Type=simple
+ExecStart=/usr/local/bin/garm-agent daemon --config /etc/garm-agent/garm-agent.toml
+ExecReload=/bin/kill -HUP $MAINPID
+Restart=always
+RestartSec=5s
+User={{ .RunnerUsername }}
+Environment=TERM=xterm-256color
+Environment=LANG=en_US.UTF-8
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+    sudo mv /tmp/garm-agent.service /etc/systemd/system/garm-agent.service || fail "failed to create /etc/systemd/system/garm-agent.service"
+    sudo chown root:root /etc/systemd/system/garm-agent.service || fail "failed to change owner on /etc/systemd/system/garm-agent.service"
+    sendStatus "Reloading systemd unit"
+    sudo systemctl daemon-reload || fail "failed to reload systemd"
+fi
+
 function downloadAndExtractRunner() {
 	sendStatus "downloading tools from {{ .DownloadURL }}"
 	if [ ! -z "{{ .TempDownloadToken }}" ]; then
@@ -116,14 +199,13 @@ fi
 
 sendStatus "enabling runner service"
 cp "$RUN_HOME"/bin/runsvc.sh "$RUN_HOME"/ || fail "failed to copy runsvc.sh"
-# Chown is not needed for the cached runner
-# sudo chown {{ .RunnerUsername }}:{{ .RunnerGroup }} -R /home/{{ .RunnerUsername }} || fail "failed to change owner"
 sudo systemctl daemon-reload || fail "failed to reload systemd"
 sudo systemctl enable $SVC_NAME
 {{- else}}
 
 GITHUB_TOKEN=$(curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/")
 
+
 set +e
 attempt=1
 while true; do
@@ -172,10 +254,14 @@ if [ -f "$RUN_HOME/env.sh" ];then
 	source env.sh
 	popd
 fi
-sudo systemctl start $SVC_NAME || fail "failed to start service"
+if [ "$AGENT_MODE" != "true" ]; then
+    sudo systemctl start $SVC_NAME || fail "failed to start service"
+fi
 {{- else}}
-sendStatus "starting service"
-sudo ./svc.sh start || fail "failed to start service"
+if [ "$AGENT_MODE" != "true" ]; then
+    sendStatus "starting service"
+    sudo ./svc.sh start || fail "failed to start service"
+fi
 
 set +e
 AGENT_ID=$(grep "agentId" "$RUN_HOME"/.runner |  tr -d -c 0-9)
@@ -185,4 +271,8 @@ fi
 set -e
 {{- end}}
 systemInfo $AGENT_ID
-success "runner successfully installed" $AGENT_ID
+if [ "$AGENT_MODE" == "true" ]; then
+  sendStatus "Starting garm-agent service"
+  sudo systemctl enable --now garm-agent || sendStatus "failed to start garm-agent"
+fi
+success "runner successfully installed" $AGENT_ID

internal/templates/userdata/github_windows_userdata.tmpl

@@ -377,14 +377,93 @@ function Set-SystemInfo {
     }
 }
 
+$CallbackURL="{{.CallbackURL}}"
+if (!($CallbackURL -match "^(.*)/status(/)?$")) {
+	$CallbackURL = "$CallbackURL/status"
+}
 $GHRunnerGroup = "{{.GitHubRunnerGroup}}"
+try {
+    $instanceMetadata = (wget -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri {{.MetadataURL}}/runner-metadata)
+    $metadata = ConvertFrom-Json $instanceMetadata.Content
+} catch {
+	Invoke-GarmFailure -Message "failed to get runner metadata: $_" -CallbackURL "{{.CallbackURL}}" | Out-Null
+}
+
+function Get-IsAgentMode {
+    return ($metadata.agent_mode -eq $true)
+}
+
+function Get-AgentURL {
+    $url = $metadata.metadata_access.agent_url
+    if (!$url) {
+        Throw("missing agent URL")
+    }
+    return $url
+}
+
+function Get-AgentToken {
+    $token = $metadata.agent_token
+    if (!$token) {
+        Throw("missing agent Token")
+    }
+    return $token
+}
+
+function Get-AgentDownloadURL {
+    $url = $metadata.agent_tools.download_url
+    if (!$url) {
+        Throw("missing agent download URL")
+    }
+    return $url
+}
+
+function Get-AgentShellEnabled {
+    $shellEnabled = $metadata.agent_shell_enabled
+    if ($shellEnabled) {return "true"}
+    return "false"
+}
+
+function Install-GarmAgent {
+    [CmdletBinding()]
+    param (
+        [parameter(Mandatory=$true)]
+        [System.Management.Automation.PSCredential]$pscreds
+    )
+    Update-GarmStatus -Message "Agent mode is enabled" -CallbackURL $CallbackURL | Out-Null
+    $agentDir = "C:\garm-agent"
+    mkdir -Force $agentDir
+    $agentURL = Get-AgentURL
+    $agentDownloadURL = Get-AgentDownloadURL
+    $agentToken = Get-AgentToken
+    $agentDownloadHeaders=@{
+        "Authorization"="Bearer $Token"
+    }
+    $shellEnabled = Get-AgentShellEnabled
+    Set-Content "$agentDir\garm-agent.toml" @"
+server_url = "$agentURL"
+log_file = "C:/garm-agent/garm-agent.log"
+shell = "cmd.exe"
+enable_shell = $shellEnabled
+work_dir = "C:/actions-runner/"
+token = "$agentToken"
+runner_cmdline = ["C:\\Windows\\system32\\cmd.exe", "/C", "C:\\actions-runner\\run.cmd"]
+state_db_path = "C:/garm-agent/agent-state.db"
+"@
+
+    Update-GarmStatus -Message "Downloading agent from: $agentDownloadURL" -CallbackURL $CallbackURL | Out-Null
+    Start-ExecuteWithRetry -ScriptBlock {
+        Invoke-FastWebRequest -Headers $agentDownloadHeaders -Uri "$agentDownloadURL" -OutFile $agentDir\garm-agent.exe
+    } -MaxRetryCount 5 -RetryInterval 5 -RetryMessage "Retrying download of runner..."
+
+    try {
+        New-Service -Name garm-agent -BinaryPathName "$agentDir\garm-agent.exe daemon --config $agentDir\garm-agent.toml" -DisplayName "garm-agent" -Description "GARM agent" -StartupType Automatic -Credential $pscreds
+    } catch {
+        Invoke-GarmFailure -CallbackURL $CallbackURL -Message "failed to set up garm agent $_"
+    }
+    Start-Service garm-agent
+}
 
 function Install-Runner() {
-	$CallbackURL="{{.CallbackURL}}"
-	if (!($CallbackURL -match "^(.*)/status(/)?$")) {
-		$CallbackURL = "$CallbackURL/status"
-	}
-
 	if ($Token.Length -eq 0) {
 		Throw "missing callback authentication token"
 	}
@@ -480,8 +559,12 @@ function Install-Runner() {
 
 		Update-GarmStatus -CallbackURL $CallbackURL -Message "Creating system service"
 		$SVC_NAME=(gc -raw $serviceNameFile)
-		New-Service -Name "$SVC_NAME" -BinaryPathName "C:\actions-runner\bin\RunnerService.exe" -DisplayName "$SVC_NAME" -Description "GitHub Actions Runner ($SVC_NAME)" -StartupType Automatic -Credential $pscreds
-		Start-Service "$SVC_NAME"
+        if (!(Get-IsAgentMode)) {
+		    New-Service -Name "$SVC_NAME" -BinaryPathName "C:\actions-runner\bin\RunnerService.exe" -DisplayName "$SVC_NAME" -Description "GitHub Actions Runner ($SVC_NAME)" -StartupType Automatic -Credential $pscreds
+		    Start-Service "$SVC_NAME"
+        } else {
+            Install-GarmAgent $pscreds
+        }
 		Set-SystemInfo -CallbackURL $CallbackURL -RunnerDir $runnerDir -BearerToken $Token
 		Update-GarmStatus -Message "runner successfully installed" -CallbackURL $CallbackURL -Status "idle" | Out-Null
 
@@ -489,17 +572,34 @@ function Install-Runner() {
 		$GithubRegistrationToken = Start-ExecuteWithRetry -ScriptBlock {
 			Invoke-WebRequest -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $MetadataURL/runner-registration-token/
 		} -MaxRetryCount 5 -RetryInterval 5 -RetryMessage "Retrying download of GitHub registration token..."
-		{{- if .GitHubRunnerGroup }}
-		./config.cmd --unattended --url "{{ .RepoURL }}" --token $GithubRegistrationToken --runnergroup {{.GitHubRunnerGroup}} --name "{{ .RunnerName }}" --labels "{{ .RunnerLabels }}" --no-default-labels --ephemeral --runasservice --windowslogonaccount "$userName" --windowslogonpassword "$userPasswd"
-		{{- else}}
-		./config.cmd --unattended --url "{{ .RepoURL }}" --token $GithubRegistrationToken --name "{{ .RunnerName }}" --labels "{{ .RunnerLabels }}" --no-default-labels --ephemeral --runasservice --windowslogonaccount "$userName" --windowslogonpassword "$userPasswd"
-		{{- end}}
+        $argList = @{
+            "--unattended" = $null;
+            "--url" =  "{{ .RepoURL }}";
+            "--token" = $GithubRegistrationToken;
+            "--name" = "{{ .RunnerName }}";
+            "--labels" = "{{ .RunnerLabels }}"
+            "--no-default-labels" = $null;
+            "--ephemeral" = $null;
+        }
+        {{- if .GitHubRunnerGroup }}
+            $argList["--runnergroup"] = {{.GitHubRunnerGroup}}
+        {{- end }}
+        if (!(Get-IsAgentMode)) {
+            $argList["--runasservice"] = $null
+            $argList["--windowslogonaccount"] = "$userName"
+            $argList["--windowslogonpassword"] = $userPasswd
+        }
+
+		./config.cmd @argList
 		if ($LASTEXITCODE) {
 			Throw "Failed to configure runner. Err code $LASTEXITCODE"
 		}
 		$agentInfoFile = Join-Path $runnerDir ".runner"
 		$agentInfo = ConvertFrom-Json (gc -raw $agentInfoFile)
 		Set-SystemInfo -CallbackURL $CallbackURL -RunnerDir $runnerDir -BearerToken $Token
+        if ((Get-IsAgentMode)) {
+            Install-GarmAgent $pscreds
+        }
 		Invoke-GarmSuccess -CallbackURL $CallbackURL -Message "runner successfully installed" -AgentID $agentInfo.agentId
 		{{- end }}
 	} catch {

params/params.go

@@ -108,6 +108,11 @@ const (
 	JobStatusCompleted  JobStatus = "completed"
 )
 
+const (
+	// SystemUser is a virtual user that identifies the system itself.
+	SystemUser = "system"
+)
+
 const (
 	ForgeEntityTypeRepository   ForgeEntityType = "repository"
 	ForgeEntityTypeOrganization ForgeEntityType = "organization"
@@ -173,6 +178,97 @@ const (
 	MessageTypeJobAvailable = "JobAvailable"
 )
 
+var InstanceStatusTransitions = map[commonParams.InstanceStatus][]commonParams.InstanceStatus{
+	commonParams.InstanceRunning: {
+		commonParams.InstancePendingDelete,
+		commonParams.InstancePendingForceDelete,
+		commonParams.InstanceStopped,
+		commonParams.InstanceStatusUnknown,
+	},
+	commonParams.InstanceStopped: {
+		commonParams.InstancePendingDelete,
+		commonParams.InstancePendingForceDelete,
+		commonParams.InstanceRunning,
+		commonParams.InstanceStatusUnknown,
+	},
+	commonParams.InstanceError: {
+		commonParams.InstancePendingDelete,
+		commonParams.InstancePendingForceDelete,
+		commonParams.InstanceStatusUnknown,
+		commonParams.InstanceDeleting,
+	},
+	commonParams.InstancePendingDelete: {
+		commonParams.InstanceDeleting,
+		commonParams.InstancePendingForceDelete,
+	},
+	commonParams.InstancePendingForceDelete: {
+		commonParams.InstanceDeleting,
+	},
+	commonParams.InstanceDeleting: {
+		commonParams.InstanceError,
+		commonParams.InstanceDeleted,
+	},
+	commonParams.InstanceDeleted: {}, // no further transitions possible
+	commonParams.InstancePendingCreate: {
+		commonParams.InstancePendingDelete,
+		commonParams.InstanceCreating,
+		commonParams.InstancePendingForceDelete,
+	},
+	commonParams.InstanceCreating: {
+		commonParams.InstanceError,
+		commonParams.InstanceRunning,
+	},
+	commonParams.InstanceStatusUnknown: {
+		commonParams.InstanceRunning,
+		commonParams.InstanceStopped,
+		commonParams.InstanceError,
+		commonParams.InstancePendingDelete,
+		commonParams.InstancePendingForceDelete,
+		commonParams.InstanceDeleting,
+		commonParams.InstanceDeleted,
+		commonParams.InstancePendingCreate,
+		commonParams.InstanceCreating,
+	},
+}
+
+var RunnerStatusTransitions = map[RunnerStatus][]RunnerStatus{
+	RunnerPending: {
+		RunnerFailed,
+		RunnerInstalling,
+		RunnerTerminated,
+		RunnerPending,
+	},
+	RunnerInstalling: {
+		RunnerFailed,
+		RunnerIdle,
+		RunnerTerminated,
+		RunnerInstalling,
+	},
+	RunnerIdle: {
+		RunnerOffline,
+		RunnerActive,
+		RunnerTerminated,
+		RunnerIdle,
+	},
+	RunnerActive: {
+		RunnerTerminated,
+		RunnerActive,
+	},
+	RunnerFailed: {
+		RunnerTerminated,
+		RunnerFailed,
+	},
+	RunnerOffline: {
+		RunnerIdle,
+		RunnerActive,
+		RunnerTerminated,
+		RunnerOffline,
+	},
+	RunnerTerminated: {
+		RunnerTerminated,
+	},
+}
+
 // swagger:model StatusMessage
 type StatusMessage struct {
 	CreatedAt  time.Time  `json:"created_at,omitempty"`
@@ -264,6 +360,10 @@ type Instance struct {
 	// Job is the current job that is being serviced by this runner.
 	Job *Job `json:"job,omitempty"`
 
+	// Heartbeat is the last recorded heartbeat from the runner
+	Heartbeat    time.Time         `json:"heartbeat"`
+	Capabilities AgentCapabilities `json:"capabilities"`
+
 	// Do not serialize sensitive info.
 	CallbackURL      string            `json:"-"`
 	MetadataURL      string            `json:"-"`
@@ -372,6 +472,7 @@ type Pool struct {
 	Tags           []Tag               `json:"tags,omitempty"`
 	Enabled        bool                `json:"enabled,omitempty"`
 	Instances      []Instance          `json:"instances,omitempty"`
+	EnableShell    bool                `json:"enable_shell"`
 
 	RepoID   string `json:"repo_id,omitempty"`
 	RepoName string `json:"repo_name,omitempty"`
@@ -527,6 +628,7 @@ type ScaleSet struct {
 	Enabled            bool                `json:"enabled,omitempty"`
 	Instances          []Instance          `json:"instances,omitempty"`
 	DesiredRunnerCount int                 `json:"desired_runner_count,omitempty"`
+	EnableShell        bool                `json:"enable_shell"`
 
 	Endpoint ForgeEndpoint `json:"endpoint,omitempty"`
 
@@ -630,6 +732,7 @@ type Repository struct {
 
 	CredentialsID uint             `json:"credentials_id,omitempty"`
 	Credentials   ForgeCredentials `json:"credentials,omitempty"`
+	AgentMode     bool             `json:"agent_mode"`
 
 	PoolManagerStatus PoolManagerStatus `json:"pool_manager_status,omitempty"`
 	PoolBalancerType  PoolBalancerType  `json:"pool_balancing_type,omitempty"`
@@ -666,6 +769,7 @@ func (r Repository) GetEntity() (ForgeEntity, error) {
 		WebhookSecret:    r.WebhookSecret,
 		CreatedAt:        r.CreatedAt,
 		UpdatedAt:        r.UpdatedAt,
+		AgentMode:        r.AgentMode,
 	}, nil
 }
 
@@ -709,6 +813,7 @@ type Organization struct {
 	CreatedAt         time.Time         `json:"created_at,omitempty"`
 	UpdatedAt         time.Time         `json:"updated_at,omitempty"`
 	Events            []EntityEvent     `json:"events,omitempty"`
+	AgentMode         bool              `json:"agent_mode"`
 	// Do not serialize sensitive info.
 	WebhookSecret string `json:"-"`
 }
@@ -730,6 +835,7 @@ func (o Organization) GetEntity() (ForgeEntity, error) {
 		Credentials:      o.Credentials,
 		CreatedAt:        o.CreatedAt,
 		UpdatedAt:        o.UpdatedAt,
+		AgentMode:        o.AgentMode,
 	}, nil
 }
 
@@ -769,6 +875,7 @@ type Enterprise struct {
 	CreatedAt         time.Time         `json:"created_at,omitempty"`
 	UpdatedAt         time.Time         `json:"updated_at,omitempty"`
 	Events            []EntityEvent     `json:"events,omitempty"`
+	AgentMode         bool              `json:"agent_mode"`
 	// Do not serialize sensitive info.
 	WebhookSecret string `json:"-"`
 }
@@ -790,6 +897,7 @@ func (e Enterprise) GetEntity() (ForgeEntity, error) {
 		Credentials:      e.Credentials,
 		CreatedAt:        e.CreatedAt,
 		UpdatedAt:        e.UpdatedAt,
+		AgentMode:        e.AgentMode,
 	}, nil
 }
 
@@ -869,6 +977,15 @@ type ControllerInfo struct {
 	// Functionally it is the same as WebhookURL, but it allows us to safely manage webhooks
 	// from GARM without accidentally removing webhooks from other services or GARM controllers.
 	ControllerWebhookURL string `json:"controller_webhook_url,omitempty"`
+	// AgentURL is the URL where the GARM agent will connect. If set behind a reverse proxy, this
+	// URL must be configured to allow websocket connections.
+	AgentURL string `json:"agent_url,omitempty"`
+	// GARMAgentReleasesURL is the URL from where GARM can fetch garm-agent binaries. This URL must
+	// have an API response compatible with the github releases API.
+	// The default value for this field is: https://api.github.com/repos/cloudbase/garm-agent/releases
+	GARMAgentReleasesURL string `json:"garm_agent_releases_url"`
+	// SyncGARMAgentTools enables or disables automatic sync of garm-agent tools.
+	SyncGARMAgentTools bool `json:"enable_agent_tools_sync"`
 	// MinimumJobAgeBackoff is the minimum time in seconds that a job must be in queued state
 	// before GARM will attempt to allocate a runner for it. When set to a non zero value,
 	// GARM will ignore the job until the job's age is greater than this value. When using
@@ -1183,6 +1300,7 @@ type ForgeEntity struct {
 	PoolBalancerType PoolBalancerType `json:"pool_balancing_type,omitempty"`
 	CreatedAt        time.Time        `json:"created_at,omitempty"`
 	UpdatedAt        time.Time        `json:"updated_at,omitempty"`
+	AgentMode        bool             `json:"agent_mode"`
 
 	WebhookSecret string `json:"-"`
 }
@@ -1349,6 +1467,7 @@ type GARMAgentTool struct {
 	Version     string              `json:"version"`
 	OSType      commonParams.OSType `json:"os_type"`
 	OSArch      commonParams.OSArch `json:"os_arch"`
+	DownloadURL string              `json:"download_url"`
 }
 
 // swagger:model GARMAgentToolsPaginatedResponse
@@ -1358,6 +1477,7 @@ type GARMAgentToolsPaginatedResponse = PaginatedResponse[GARMAgentTool]
 type MetadataServiceAccessDetails struct {
 	CallbackURL string `json:"callback_url"`
 	MetadataURL string `json:"metadata_url"`
+	AgentURL    string `json:"agent_url"`
 }
 
 // swagger:model InstanceMetadata
@@ -1376,7 +1496,18 @@ type InstanceMetadata struct {
 	// Also, the instance metadata should never be saved to disk, and the metadata URL is only
 	// accessible during setup of the runner. The API returns unauthorized once the runner
 	// transitions to failed/idle.
-	ExtraSpecs  map[string]any                         `json:"extra_specs,omitempty"`
-	JITEnabled  bool                                   `json:"jit_enabled"`
-	RunnerTools commonParams.RunnerApplicationDownload `json:"runner_tools"`
+	ExtraSpecs map[string]any `json:"extra_specs,omitempty"`
+	// Agent mode indicates whether or not we need to install the GARM agent on the runner.
+	AgentMode bool `json:"agent_mode"`
+	// AgentTools represents the garm agent download details.
+	AgentTools        *GARMAgentTool                         `json:"agent_tools,omitempty"`
+	AgentToken        string                                 `json:"agent_token,omitempty"`
+	AgentShellEnabled bool                                   `json:"agent_shell_enabled,omitempty"`
+	JITEnabled        bool                                   `json:"jit_enabled"`
+	RunnerTools       commonParams.RunnerApplicationDownload `json:"runner_tools"`
+}
+
+// swagger:model AgentCapabilities
+type AgentCapabilities struct {
+	Shell bool `json:"has_shell"`
 }

params/requests.go

@@ -20,6 +20,7 @@ import (
 	"encoding/pem"
 	"fmt"
 	"net/url"
+	"time"
 
 	runnerErrors "github.com/cloudbase/garm-provider-common/errors"
 	commonParams "github.com/cloudbase/garm-provider-common/params"
@@ -45,6 +46,7 @@ type CreateRepoParams struct {
 	WebhookSecret    string           `json:"webhook_secret,omitempty"`
 	PoolBalancerType PoolBalancerType `json:"pool_balancer_type,omitempty"`
 	ForgeType        EndpointType     `json:"forge_type,omitempty"`
+	AgentMode        bool             `json:"agent_mode,omitempty"`
 }
 
 func (c *CreateRepoParams) Validate() error {
@@ -86,6 +88,7 @@ type CreateOrgParams struct {
 	WebhookSecret    string           `json:"webhook_secret,omitempty"`
 	PoolBalancerType PoolBalancerType `json:"pool_balancer_type,omitempty"`
 	ForgeType        EndpointType     `json:"forge_type,omitempty"`
+	AgentMode        bool             `json:"agent_mode,omitempty"`
 }
 
 func (c *CreateOrgParams) Validate() error {
@@ -121,6 +124,7 @@ type CreateEnterpriseParams struct {
 	CredentialsName  string           `json:"credentials_name,omitempty"`
 	WebhookSecret    string           `json:"webhook_secret,omitempty"`
 	PoolBalancerType PoolBalancerType `json:"pool_balancer_type,omitempty"`
+	AgentMode        bool             `json:"agent_mode,omitempty"`
 }
 
 func (c *CreateEnterpriseParams) Validate() error {
@@ -168,6 +172,7 @@ type UpdatePoolParams struct {
 	OSType                 commonParams.OSType `json:"os_type,omitempty"`
 	OSArch                 commonParams.OSArch `json:"os_arch,omitempty"`
 	ExtraSpecs             json.RawMessage     `json:"extra_specs,omitempty"`
+	EnableShell            *bool               `json:"enable_shell"`
 	// GithubRunnerGroup is the github runner group in which the runners of this
 	// pool will be added to.
 	// The runner group must be created by someone with access to the enterprise.
@@ -208,6 +213,7 @@ type CreatePoolParams struct {
 	Enabled                bool                `json:"enabled,omitempty"`
 	RunnerBootstrapTimeout uint                `json:"runner_bootstrap_timeout,omitempty"`
 	ExtraSpecs             json.RawMessage     `json:"extra_specs,omitempty"`
+	EnableShell            bool                `json:"enable_shell"`
 	// GithubRunnerGroup is the github runner group in which the runners of this
 	// pool will be added to.
 	// The runner group must be created by someone with access to the enterprise.
@@ -257,10 +263,12 @@ type UpdateInstanceParams struct {
 	Status           commonParams.InstanceStatus `json:"status,omitempty"`
 	RunnerStatus     RunnerStatus                `json:"runner_status,omitempty"`
 	ProviderFault    []byte                      `json:"provider_fault,omitempty"`
+	Heartbeat        *time.Time                  `json:"heartbeat,omitempty"`
 	AgentID          int64                       `json:"-"`
 	CreateAttempt    int                         `json:"-"`
 	TokenFetched     *bool                       `json:"-"`
 	JitConfiguration map[string]string           `json:"-"`
+	Capabilities     *AgentCapabilities          `json:"-"`
 }
 
 type UpdateUserParams struct {
@@ -291,6 +299,7 @@ type UpdateEntityParams struct {
 	CredentialsName  string           `json:"credentials_name,omitempty"`
 	WebhookSecret    string           `json:"webhook_secret,omitempty"`
 	PoolBalancerType PoolBalancerType `json:"pool_balancer_type,omitempty"`
+	AgentMode        *bool            `json:"agent_mode,omitempty"`
 }
 
 type InstanceUpdateMessage struct {
@@ -537,6 +546,9 @@ type UpdateControllerParams struct {
 	MetadataURL          *string `json:"metadata_url,omitempty"`
 	CallbackURL          *string `json:"callback_url,omitempty"`
 	WebhookURL           *string `json:"webhook_url,omitempty"`
+	AgentURL             *string `json:"agent_url,omitempty"`
+	GARMAgentReleasesURL *string `json:"garm_agent_releases_url,omitempty"`
+	SyncGARMAgentTools   *bool   `json:"enable_agent_tools_sync,omitempty"`
 	MinimumJobAgeBackoff *uint   `json:"minimum_job_age_backoff,omitempty"`
 }
 
@@ -562,6 +574,13 @@ func (u UpdateControllerParams) Validate() error {
 		}
 	}
 
+	if u.AgentURL != nil {
+		u, err := url.Parse(*u.AgentURL)
+		if err != nil || u.Scheme == "" || u.Host == "" {
+			return runnerErrors.NewBadRequestError("invalid agent_url")
+		}
+	}
+
 	return nil
 }
 
@@ -584,6 +603,7 @@ type CreateScaleSetParams struct {
 	Enabled                bool                `json:"enabled,omitempty"`
 	RunnerBootstrapTimeout uint                `json:"runner_bootstrap_timeout,omitempty"`
 	ExtraSpecs             json.RawMessage     `json:"extra_specs,omitempty"`
+	EnableShell            bool                `json:"enable_shell"`
 	// GithubRunnerGroup is the github runner group in which the runners of this
 	// pool will be added to.
 	// The runner group must be created by someone with access to the enterprise.
@@ -633,6 +653,7 @@ type UpdateScaleSetParams struct {
 	OSType                 commonParams.OSType `json:"os_type,omitempty"`
 	OSArch                 commonParams.OSArch `json:"os_arch,omitempty"`
 	ExtraSpecs             json.RawMessage     `json:"extra_specs,omitempty"`
+	EnableShell            *bool               `json:"enable_shell"`
 	// GithubRunnerGroup is the github runner group in which the runners of this
 	// pool will be added to.
 	// The runner group must be created by someone with access to the enterprise.
@@ -827,6 +848,7 @@ type CreateTemplateParams struct {
 	Data        []byte              `json:"data"`
 	OSType      commonParams.OSType `json:"os_type"`
 	ForgeType   EndpointType        `json:"forge_type,omitempty"`
+	IsSystem    bool                `json:"-"`
 }
 
 func (c *CreateTemplateParams) Validate() error {
@@ -889,3 +911,23 @@ type CreateFileObjectParams struct {
 	Size        int64    `json:"size"`
 	Tags        []string `json:"tags"`
 }
+
+// swagger:model CreateGARMToolParams
+type CreateGARMToolParams struct {
+	Name        string              `json:"name"`
+	Description string              `json:"description"`
+	Size        int64               `json:"size"`
+	OSType      commonParams.OSType `json:"os_type"`
+	OSArch      commonParams.OSArch `json:"os_arch"`
+	Version     string              `json:"version"`
+}
+
+// swagger:model RestoreTemplateRequest
+type RestoreTemplateRequest struct {
+	Forge  EndpointType        `json:"forge"`
+	OSType commonParams.OSType `json:"os_type"`
+	// RestoreAll indicates whether or not to restore all known
+	// system owned templates. If set, the Forge and OSType params
+	// are ignored.
+	RestoreAll bool `json:"restore_all"`
+}

runner/agent.go

@@ -0,0 +1,115 @@
+package runner
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"time"
+
+	runnerErrors "github.com/cloudbase/garm-provider-common/errors"
+	commonParams "github.com/cloudbase/garm-provider-common/params"
+	"github.com/cloudbase/garm/auth"
+	"github.com/cloudbase/garm/params"
+)
+
+func (r *Runner) RecordAgentHeartbeat(ctx context.Context) error {
+	instance, err := auth.InstanceParams(ctx)
+	if err != nil {
+		slog.With(slog.Any("error", err)).ErrorContext(
+			ctx, "failed to get instance params")
+		return runnerErrors.ErrUnauthorized
+	}
+	now := time.Now().UTC()
+	updateParams := params.UpdateInstanceParams{
+		Heartbeat: &now,
+	}
+
+	if _, err := r.store.UpdateInstance(ctx, instance.Name, updateParams); err != nil {
+		return fmt.Errorf("failed to record heartbeat: %w", err)
+	}
+	return nil
+}
+
+func (r *Runner) SetInstanceCapabilities(ctx context.Context, caps params.AgentCapabilities) error {
+	instance, err := auth.InstanceParams(ctx)
+	if err != nil {
+		slog.With(slog.Any("error", err)).ErrorContext(
+			ctx, "failed to get instance params")
+		return runnerErrors.ErrUnauthorized
+	}
+
+	updateParams := params.UpdateInstanceParams{
+		Capabilities: &caps,
+	}
+
+	if _, err := r.store.UpdateInstance(ctx, instance.ID, updateParams); err != nil {
+		return fmt.Errorf("failed to update capabilities: %w", err)
+	}
+	return nil
+}
+
+func (r *Runner) SetInstanceToPendingDelete(ctx context.Context) error {
+	instance, err := auth.InstanceParams(ctx)
+	if err != nil {
+		slog.With(slog.Any("error", err)).ErrorContext(
+			ctx, "failed to get instance params")
+		return runnerErrors.ErrUnauthorized
+	}
+
+	updateParams := params.UpdateInstanceParams{
+		Status: commonParams.InstancePendingDelete,
+	}
+
+	if _, err := r.store.UpdateInstance(r.ctx, instance.ID, updateParams); err != nil {
+		return fmt.Errorf("failed to set instance to pending_delete: %w", err)
+	}
+	return nil
+}
+
+func (r *Runner) GetAgentJWTToken(ctx context.Context, runnerName string) (string, error) {
+	var instance params.Instance
+	var err error
+	if !auth.IsAdmin(ctx) {
+		instance, err = auth.InstanceParams(ctx)
+		if err != nil {
+			return "", runnerErrors.ErrUnauthorized
+		}
+
+		// A runner bootstrap token can get an agent token for itself.
+		if instance.Name != runnerName || auth.InstanceIsAgent(ctx) {
+			return "", runnerErrors.ErrUnauthorized
+		}
+	} else {
+		instance, err = r.GetInstance(ctx, runnerName)
+		if err != nil {
+			return "", fmt.Errorf("failed to get runner: %w", err)
+		}
+	}
+
+	var entityGetter params.EntityGetter
+	switch {
+	case instance.PoolID != "":
+		entityGetter, err = r.GetPoolByID(ctx, instance.PoolID)
+	case instance.ScaleSetID != 0:
+		entityGetter, err = r.GetScaleSetByID(ctx, instance.ScaleSetID)
+	}
+	if err != nil {
+		return "", fmt.Errorf("failed to get entity: %w", err)
+	}
+
+	entity, err := entityGetter.GetEntity()
+	if err != nil {
+		return "", fmt.Errorf("failed to get entity: %w", err)
+	}
+
+	dbEntity, err := r.store.GetForgeEntity(ctx, entity.EntityType, entity.ID)
+	if err != nil {
+		return "", fmt.Errorf("failed to get entity from DB: %w", err)
+	}
+
+	agentToken, err := r.tokenGetter.NewAgentJWTToken(instance, dbEntity)
+	if err != nil {
+		return "", fmt.Errorf("failed to get agent token: %w", err)
+	}
+	return agentToken, nil
+}

runner/enterprises.go

@@ -52,7 +52,7 @@ func (r *Runner) CreateEnterprise(ctx context.Context, param params.CreateEnterp
 		return params.Enterprise{}, runnerErrors.NewConflictError("enterprise %s already exists", param.Name)
 	}
 
-	enterprise, err = r.store.CreateEnterprise(ctx, param.Name, creds, param.WebhookSecret, param.PoolBalancerType)
+	enterprise, err = r.store.CreateEnterprise(ctx, param.Name, creds, param.WebhookSecret, param.PoolBalancerType, param.AgentMode)
 	if err != nil {
 		return params.Enterprise{}, fmt.Errorf("error creating enterprise: %w", err)
 	}

runner/enterprises_test.go

@@ -88,6 +88,7 @@ func (s *EnterpriseTestSuite) SetupTest() {
 			s.testCreds,
 			fmt.Sprintf("test-webhook-secret-%v", i),
 			params.PoolBalancerTypeRoundRobin,
+			false,
 		)
 		if err != nil {
 			s.FailNow(fmt.Sprintf("failed to create database object (test-enterprise-%v): %+v", i, err))
@@ -244,6 +245,7 @@ func (s *EnterpriseTestSuite) TestListEnterprisesWithFilters() {
 		s.testCreds,
 		"super secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 	enterprise2, err := s.Fixtures.Store.CreateEnterprise(
@@ -252,6 +254,7 @@ func (s *EnterpriseTestSuite) TestListEnterprisesWithFilters() {
 		s.testCreds,
 		"super secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 	enterprise3, err := s.Fixtures.Store.CreateEnterprise(
@@ -260,6 +263,7 @@ func (s *EnterpriseTestSuite) TestListEnterprisesWithFilters() {
 		s.ghesCreds,
 		"super secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	s.Require().NoError(err)
 	orgs, err := s.Runner.ListEnterprises(

runner/garm_tools.go

@@ -0,0 +1,223 @@
+// Copyright 2025 Cloudbase Solutions SRL
+//
+//    Licensed under the Apache License, Version 2.0 (the "License"); you may
+//    not use this file except in compliance with the License. You may obtain
+//    a copy of the License at
+//
+//         http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+//    License for the specific language governing permissions and limitations
+//    under the License.
+
+package runner
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log/slog"
+
+	runnerErrors "github.com/cloudbase/garm-provider-common/errors"
+	"github.com/cloudbase/garm/auth"
+	"github.com/cloudbase/garm/params"
+)
+
+var (
+	garmAgentFileTag          = "category=garm-agent"
+	garmAgentOSTypeWindowsTag = "os_type=windows"
+	garmAgentOSTypeLinuxTag   = "os_type=linux"
+	garmAgentOSArchAMD64Tag   = "os_arch=amd64"
+	garmAgentOSArchARM64Tag   = "os_arch=arm64"
+)
+
+func (r *Runner) ListGARMTools(ctx context.Context) ([]params.GARMAgentTool, error) {
+	if !auth.IsAdmin(ctx) {
+		return nil, runnerErrors.ErrUnauthorized
+	}
+
+	ret := []params.GARMAgentTool{}
+	var next uint64 = 1
+	for {
+		allAgentTools, err := r.store.SearchFileObjectByTags(r.ctx, []string{garmAgentFileTag}, next, 100)
+		if err != nil {
+			return nil, fmt.Errorf("failed to list files: %w", err)
+		}
+		if allAgentTools.TotalCount == 0 {
+			return nil, nil
+		}
+		for _, tool := range allAgentTools.Results {
+			parsed, err := fileObjectToGARMTool(tool, "")
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse object with ID %d", tool.ID)
+			}
+			ret = append(ret, parsed)
+		}
+		if allAgentTools.NextPage == nil {
+			break
+		}
+		next = *allAgentTools.NextPage
+	}
+	return ret, nil
+}
+
+func (r *Runner) CreateGARMTool(ctx context.Context, param params.CreateGARMToolParams, reader io.Reader) (params.FileObject, error) {
+	if !auth.IsAdmin(ctx) {
+		return params.FileObject{}, runnerErrors.ErrUnauthorized
+	}
+
+	// Validate version is provided
+	if param.Version == "" {
+		return params.FileObject{}, runnerErrors.NewBadRequestError("version is required")
+	}
+
+	// Build tags based on OS type and arch
+	var osTypeTag, osArchTag string
+	switch param.OSType {
+	case "windows":
+		osTypeTag = garmAgentOSTypeWindowsTag
+	case "linux":
+		osTypeTag = garmAgentOSTypeLinuxTag
+	default:
+		return params.FileObject{}, runnerErrors.NewBadRequestError("invalid os_type: must be 'windows' or 'linux'")
+	}
+
+	switch param.OSArch {
+	case "amd64":
+		osArchTag = garmAgentOSArchAMD64Tag
+	case "arm64":
+		osArchTag = garmAgentOSArchARM64Tag
+	default:
+		return params.FileObject{}, runnerErrors.NewBadRequestError("invalid os_arch: must be 'amd64' or 'arm64'")
+	}
+
+	// Build tags: category, os_type, os_arch, version
+	tags := []string{
+		garmAgentFileTag,
+		osTypeTag,
+		osArchTag,
+		fmt.Sprintf("version=%s", param.Version),
+	}
+
+	// Create the file object params
+	createParams := params.CreateFileObjectParams{
+		Name:        param.Name,
+		Description: param.Description,
+		Size:        param.Size,
+		Tags:        tags,
+	}
+
+	// Upload the new binary
+	newTool, err := r.store.CreateFileObject(ctx, createParams, reader)
+	if err != nil {
+		return params.FileObject{}, fmt.Errorf("failed to upload garm-agent tool: %w", err)
+	}
+	slog.DebugContext(ctx, "uploaded new garm-agent tool",
+		"tool_id", newTool.ID,
+		"name", newTool.Name,
+		"os_type", param.OSType,
+		"os_arch", param.OSArch,
+		"version", param.Version,
+		"size", newTool.Size)
+
+	// Clean up old versions (keep only the newly uploaded one)
+	// Build tags to find all binaries with same OS/ARCH (excluding version)
+	cleanupTags := []string{garmAgentFileTag, osTypeTag, osArchTag}
+
+	// Delete all except the one we just uploaded
+	// Paginate through all results to ensure we delete everything
+	deletedCount := 0
+	page := uint64(1)
+	pageSize := uint64(100)
+
+	for {
+		allTools, err := r.store.SearchFileObjectByTags(ctx, cleanupTags, page, pageSize)
+		if err != nil {
+			slog.ErrorContext(ctx, "failed to search for old garm-agent versions during cleanup",
+				"error", err,
+				"os_type", param.OSType,
+				"os_arch", param.OSArch,
+				"new_tool_id", newTool.ID,
+				"page", page)
+			// Don't fail - upload succeeded
+			break
+		}
+
+		for _, tool := range allTools.Results {
+			if tool.ID != newTool.ID {
+				// Delete old version directly via store (bypass API check since this is internal)
+				if err := r.store.DeleteFileObject(ctx, tool.ID); err != nil {
+					slog.WarnContext(ctx, "failed to delete old garm-agent version during cleanup",
+						"error", err,
+						"tool_id", tool.ID,
+						"tool_name", tool.Name,
+						"os_type", param.OSType,
+						"os_arch", param.OSArch)
+					continue
+				}
+				deletedCount++
+				slog.DebugContext(ctx, "deleted old garm-agent version",
+					"tool_id", tool.ID,
+					"tool_name", tool.Name,
+					"tags", tool.Tags)
+			}
+		}
+
+		// Check if there's a next page
+		if allTools.NextPage == nil {
+			break
+		}
+		page = *allTools.NextPage
+	}
+
+	if deletedCount > 0 {
+		slog.InfoContext(ctx, "cleaned up old garm-agent versions",
+			"deleted_count", deletedCount,
+			"os_type", param.OSType,
+			"os_arch", param.OSArch,
+			"kept_version", param.Version)
+	}
+
+	return newTool, nil
+}
+
+func (r *Runner) DeleteGarmTool(ctx context.Context, osType, osArch string) error {
+	if !auth.IsAdmin(ctx) {
+		return runnerErrors.ErrUnauthorized
+	}
+
+	// Build tags based on OS type and arch
+	tags := []string{garmAgentFileTag}
+
+	switch osType {
+	case "windows":
+		tags = append(tags, garmAgentOSTypeWindowsTag)
+	case "linux":
+		tags = append(tags, garmAgentOSTypeLinuxTag)
+	default:
+		return runnerErrors.NewBadRequestError("invalid os_type: must be 'windows' or 'linux'")
+	}
+
+	switch osArch {
+	case "amd64":
+		tags = append(tags, garmAgentOSArchAMD64Tag)
+	case "arm64":
+		tags = append(tags, garmAgentOSArchARM64Tag)
+	default:
+		return runnerErrors.NewBadRequestError("invalid os_arch: must be 'amd64' or 'arm64'")
+	}
+
+	// Delete all tools matching these tags
+	deletedCount, err := r.store.DeleteFileObjectsByTags(ctx, tags)
+	if err != nil {
+		return fmt.Errorf("failed to delete garm-agent tools: %w", err)
+	}
+
+	if deletedCount == 0 {
+		return runnerErrors.NewNotFoundError("no garm-agent tools found for %s/%s", osType, osArch)
+	}
+
+	return nil
+}

runner/garm_tools_test.go

@@ -0,0 +1,465 @@
+// Copyright 2025 Cloudbase Solutions SRL
+//
+//    Licensed under the Apache License, Version 2.0 (the "License"); you may
+//    not use this file except in compliance with the License. You may obtain
+//    a copy of the License at
+//
+//         http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+//    License for the specific language governing permissions and limitations
+//    under the License.
+
+//go:build testing
+// +build testing
+
+package runner
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/suite"
+
+	runnerErrors "github.com/cloudbase/garm-provider-common/errors"
+	commonParams "github.com/cloudbase/garm-provider-common/params"
+	"github.com/cloudbase/garm/database"
+	dbCommon "github.com/cloudbase/garm/database/common"
+	garmTesting "github.com/cloudbase/garm/internal/testing"
+	"github.com/cloudbase/garm/params"
+)
+
+var (
+	windowsAMD64ToolsTags = []string{
+		garmAgentFileTag,
+		garmAgentOSTypeWindowsTag,
+		garmAgentOSArchAMD64Tag,
+	}
+	windowsARM64ToolsTags = []string{
+		garmAgentFileTag,
+		garmAgentOSTypeWindowsTag,
+		garmAgentOSArchARM64Tag,
+	}
+	linuxARM64ToolsTags = []string{
+		garmAgentFileTag,
+		garmAgentOSTypeLinuxTag,
+		garmAgentOSArchARM64Tag,
+	}
+)
+
+type GARMToolsTestSuite struct {
+	suite.Suite
+	AdminContext        context.Context
+	UnauthorizedContext context.Context
+	Store               dbCommon.Store
+	Runner              *Runner
+}
+
+func (s *GARMToolsTestSuite) SetupTest() {
+	dbCfg := garmTesting.GetTestSqliteDBConfig(s.T())
+	db, err := database.NewDatabase(context.Background(), dbCfg)
+	if err != nil {
+		s.FailNow(fmt.Sprintf("failed to create db connection: %s", err))
+	}
+
+	adminCtx := garmTesting.ImpersonateAdminContext(context.Background(), db, s.T())
+
+	s.AdminContext = adminCtx
+	s.UnauthorizedContext = context.Background()
+	s.Store = db
+
+	runner := &Runner{
+		ctx:   adminCtx,
+		store: db,
+	}
+	s.Runner = runner
+}
+
+func (s *GARMToolsTestSuite) TestCreateGARMToolUnauthorized() {
+	param := params.CreateGARMToolParams{
+		Name:        "garm-agent-linux-amd64",
+		Description: "GARM agent for Linux AMD64",
+		Size:        1024,
+		OSType:      "linux",
+		OSArch:      "amd64",
+		Version:     "v1.0.0",
+	}
+
+	reader := bytes.NewReader([]byte("test binary content"))
+	_, err := s.Runner.CreateGARMTool(s.UnauthorizedContext, param, reader)
+
+	s.Require().Error(err)
+	s.Equal(runnerErrors.ErrUnauthorized, err)
+}
+
+func (s *GARMToolsTestSuite) TestCreateGARMToolMissingVersion() {
+	param := params.CreateGARMToolParams{
+		Name:        "garm-agent-linux-amd64",
+		Description: "GARM agent for Linux AMD64",
+		Size:        1024,
+		OSType:      "linux",
+		OSArch:      "amd64",
+		Version:     "",
+	}
+
+	reader := bytes.NewReader([]byte("test binary content"))
+	_, err := s.Runner.CreateGARMTool(s.AdminContext, param, reader)
+
+	s.Require().Error(err)
+	s.Contains(err.Error(), "version is required")
+}
+
+func (s *GARMToolsTestSuite) TestCreateGARMToolInvalidOSType() {
+	param := params.CreateGARMToolParams{
+		Name:        "garm-agent-invalid-amd64",
+		Description: "Invalid OS type",
+		Size:        1024,
+		OSType:      "invalid",
+		OSArch:      "amd64",
+		Version:     "v1.0.0",
+	}
+
+	reader := bytes.NewReader([]byte("test binary content"))
+	_, err := s.Runner.CreateGARMTool(s.AdminContext, param, reader)
+
+	s.Require().Error(err)
+	s.Contains(err.Error(), "invalid os_type")
+}
+
+func (s *GARMToolsTestSuite) TestCreateGARMToolInvalidOSArch() {
+	param := params.CreateGARMToolParams{
+		Name:        "garm-agent-linux-invalid",
+		Description: "Invalid arch",
+		Size:        1024,
+		OSType:      "linux",
+		OSArch:      "invalid",
+		Version:     "v1.0.0",
+	}
+
+	reader := bytes.NewReader([]byte("test binary content"))
+	_, err := s.Runner.CreateGARMTool(s.AdminContext, param, reader)
+
+	s.Require().Error(err)
+	s.Contains(err.Error(), "invalid os_arch")
+}
+
+func (s *GARMToolsTestSuite) TestCreateGARMToolSuccess() {
+	content := []byte("test binary content for linux amd64")
+	param := params.CreateGARMToolParams{
+		Name:        "garm-agent-linux-amd64",
+		Description: "GARM agent for Linux AMD64",
+		Size:        int64(len(content)),
+		OSType:      commonParams.Linux,
+		OSArch:      commonParams.Amd64,
+		Version:     "v1.0.0",
+	}
+
+	reader := bytes.NewReader(content)
+	tool, err := s.Runner.CreateGARMTool(s.AdminContext, param, reader)
+
+	s.Require().NoError(err)
+	s.Equal(param.Name, tool.Name)
+	s.Equal(param.Description, tool.Description)
+	s.Equal(int64(len(content)), tool.Size)
+
+	// Verify tags
+	expectedTags := []string{
+		"category=garm-agent",
+		"os_type=linux",
+		"os_arch=amd64",
+		"version=v1.0.0",
+	}
+	s.ElementsMatch(expectedTags, tool.Tags)
+}
+
+func (s *GARMToolsTestSuite) TestCreateGARMToolCleansUpOldVersions() {
+	// Upload version 1.0.0
+	param1 := params.CreateGARMToolParams{
+		Name:        "garm-agent-linux-arm64",
+		Description: "GARM agent for Linux ARM64 v1.0.0",
+		Size:        1024,
+		OSType:      commonParams.Linux,
+		OSArch:      commonParams.Arm64,
+		Version:     "v1.0.0",
+	}
+	reader1 := bytes.NewReader([]byte("version 1.0.0 binary"))
+	tool1, err := s.Runner.CreateGARMTool(s.AdminContext, param1, reader1)
+	s.Require().NoError(err)
+
+	// Upload version 1.1.0
+	param2 := params.CreateGARMToolParams{
+		Name:        "garm-agent-linux-arm64",
+		Description: "GARM agent for Linux ARM64 v1.1.0",
+		Size:        2048,
+		OSType:      commonParams.Linux,
+		OSArch:      commonParams.Arm64,
+		Version:     "v1.1.0",
+	}
+	reader2 := bytes.NewReader([]byte("version 1.1.0 binary"))
+	tool2, err := s.Runner.CreateGARMTool(s.AdminContext, param2, reader2)
+	s.Require().NoError(err)
+
+	// Verify v1.0.0 was deleted
+	_, err = s.Store.GetFileObject(s.AdminContext, tool1.ID)
+	s.Require().Error(err)
+	s.Contains(err.Error(), "could not find file object")
+
+	// Verify v1.1.0 still exists
+	existing, err := s.Store.GetFileObject(s.AdminContext, tool2.ID)
+	s.Require().NoError(err)
+	s.Equal(tool2.ID, existing.ID)
+
+	// Search for all linux/arm64 tools - should only find v1.1.0
+	results, err := s.Store.SearchFileObjectByTags(s.AdminContext, linuxARM64ToolsTags, 1, 10)
+	s.Require().NoError(err)
+	s.Equal(uint64(1), results.TotalCount)
+	s.Len(results.Results, 1)
+	s.Equal(tool2.ID, results.Results[0].ID)
+}
+
+func (s *GARMToolsTestSuite) TestCreateGARMToolPaginationCleanup() {
+	// Upload 150 old versions for windows/amd64
+	for i := 1; i <= 150; i++ {
+		param := params.CreateGARMToolParams{
+			Name:        fmt.Sprintf("garm-agent-windows-amd64-v%d", i),
+			Description: fmt.Sprintf("Version %d", i),
+			Size:        int64(i * 100),
+			OSType:      commonParams.Windows,
+			OSArch:      commonParams.Amd64,
+			Version:     fmt.Sprintf("v0.0.%d", i),
+		}
+		reader := bytes.NewReader([]byte(fmt.Sprintf("version %d binary", i)))
+		_, err := s.Runner.CreateGARMTool(s.AdminContext, param, reader)
+		s.Require().NoError(err)
+	}
+
+	// Upload the latest version
+	latestParam := params.CreateGARMToolParams{
+		Name:        "garm-agent-windows-amd64",
+		Description: "Latest version",
+		Size:        99999,
+		OSType:      commonParams.Windows,
+		OSArch:      commonParams.Amd64,
+		Version:     "v2.0.0",
+	}
+	reader := bytes.NewReader([]byte("latest version binary"))
+	latestTool, err := s.Runner.CreateGARMTool(s.AdminContext, latestParam, reader)
+	s.Require().NoError(err)
+
+	// Verify only the latest version exists
+	results, err := s.Store.SearchFileObjectByTags(s.AdminContext, windowsAMD64ToolsTags, 1, 200)
+	s.Require().NoError(err)
+	s.Equal(uint64(1), results.TotalCount)
+	s.Len(results.Results, 1)
+	s.Equal(latestTool.ID, results.Results[0].ID)
+	s.Equal("v2.0.0", getVersionFromTags(results.Results[0].Tags))
+}
+
+func (s *GARMToolsTestSuite) TestCreateGARMToolDoesNotAffectOtherPlatforms() {
+	// Upload linux/amd64 v1.0.0
+	param1 := params.CreateGARMToolParams{
+		Name:        "garm-agent-linux-amd64",
+		Description: "Linux AMD64",
+		Size:        1024,
+		OSType:      commonParams.Linux,
+		OSArch:      commonParams.Amd64,
+		Version:     "v1.0.0",
+	}
+	reader1 := bytes.NewReader([]byte("linux amd64 binary"))
+	tool1, err := s.Runner.CreateGARMTool(s.AdminContext, param1, reader1)
+	s.Require().NoError(err)
+
+	// Upload windows/amd64 v1.0.0
+	param2 := params.CreateGARMToolParams{
+		Name:        "garm-agent-windows-amd64",
+		Description: "Windows AMD64",
+		Size:        2048,
+		OSType:      commonParams.Windows,
+		OSArch:      commonParams.Amd64,
+		Version:     "v1.0.0",
+	}
+	reader2 := bytes.NewReader([]byte("windows amd64 binary"))
+	tool2, err := s.Runner.CreateGARMTool(s.AdminContext, param2, reader2)
+	s.Require().NoError(err)
+
+	// Verify both still exist
+	_, err = s.Store.GetFileObject(s.AdminContext, tool1.ID)
+	s.Require().NoError(err)
+
+	_, err = s.Store.GetFileObject(s.AdminContext, tool2.ID)
+	s.Require().NoError(err)
+}
+
+func (s *GARMToolsTestSuite) TestDeleteGarmToolUnauthorized() {
+	err := s.Runner.DeleteGarmTool(s.UnauthorizedContext, "linux", "amd64")
+	s.Require().Error(err)
+	s.Equal(runnerErrors.ErrUnauthorized, err)
+}
+
+func (s *GARMToolsTestSuite) TestDeleteGarmToolInvalidOSType() {
+	err := s.Runner.DeleteGarmTool(s.AdminContext, "invalid", "amd64")
+	s.Require().Error(err)
+	s.Contains(err.Error(), "invalid os_type")
+}
+
+func (s *GARMToolsTestSuite) TestDeleteGarmToolInvalidOSArch() {
+	err := s.Runner.DeleteGarmTool(s.AdminContext, "linux", "invalid")
+	s.Require().Error(err)
+	s.Contains(err.Error(), "invalid os_arch")
+}
+
+func (s *GARMToolsTestSuite) TestDeleteGarmToolNotFound() {
+	err := s.Runner.DeleteGarmTool(s.AdminContext, "linux", "amd64")
+	s.Require().Error(err)
+	s.Contains(err.Error(), "no garm-agent tools found")
+}
+
+func (s *GARMToolsTestSuite) TestDeleteGarmToolSuccess() {
+	// Create a tool
+	param := params.CreateGARMToolParams{
+		Name:        "garm-agent-linux-amd64",
+		Description: "GARM agent for Linux AMD64",
+		Size:        1024,
+		OSType:      commonParams.Linux,
+		OSArch:      commonParams.Amd64,
+		Version:     "v1.0.0",
+	}
+	reader := bytes.NewReader([]byte("test binary"))
+	tool, err := s.Runner.CreateGARMTool(s.AdminContext, param, reader)
+	s.Require().NoError(err)
+
+	// Delete it
+	err = s.Runner.DeleteGarmTool(s.AdminContext, "linux", "amd64")
+	s.Require().NoError(err)
+
+	// Verify it's gone
+	_, err = s.Store.GetFileObject(s.AdminContext, tool.ID)
+	s.Require().Error(err)
+	s.Contains(err.Error(), "could not find file object")
+}
+
+func (s *GARMToolsTestSuite) TestDeleteGarmToolDeletesAllVersions() {
+	// Create multiple versions using windows/arm64
+	for i := 1; i <= 5; i++ {
+		reader := bytes.NewReader([]byte(fmt.Sprintf("version %d", i)))
+		// CreateGARMTool only keeps the latest, so we need to use the store directly
+		// to create multiple versions
+		tags := windowsARM64ToolsTags
+		tags = append(tags, fmt.Sprintf("version=v1.%d.0", i))
+		createParam := params.CreateFileObjectParams{
+			Name:        fmt.Sprintf("garm-agent-windows-arm64-v%d", i),
+			Description: fmt.Sprintf("Version %d", i),
+			Size:        int64(i * 100),
+			Tags:        tags,
+		}
+		_, err := s.Store.CreateFileObject(s.AdminContext, createParam, reader)
+		s.Require().NoError(err)
+	}
+
+	// Verify we have 5 versions
+	results, err := s.Store.SearchFileObjectByTags(s.AdminContext, windowsARM64ToolsTags, 1, 10)
+	s.Require().NoError(err)
+	s.Equal(uint64(5), results.TotalCount)
+
+	// Delete all
+	err = s.Runner.DeleteGarmTool(s.AdminContext, "windows", "arm64")
+	s.Require().NoError(err)
+
+	// Verify all are gone
+	results, err = s.Store.SearchFileObjectByTags(s.AdminContext, windowsARM64ToolsTags, 1, 10)
+	s.Require().NoError(err)
+	s.Equal(uint64(0), results.TotalCount)
+}
+
+func (s *GARMToolsTestSuite) TestListGARMToolsUnauthorized() {
+	_, err := s.Runner.ListGARMTools(s.UnauthorizedContext)
+	s.Require().Error(err)
+	s.Equal(runnerErrors.ErrUnauthorized, err)
+}
+
+func (s *GARMToolsTestSuite) TestListGARMToolsEmpty() {
+	tools, err := s.Runner.ListGARMTools(s.AdminContext)
+	s.Require().NoError(err)
+	s.Empty(tools)
+}
+
+func (s *GARMToolsTestSuite) TestListGARMToolsSinglePlatform() {
+	// Create one tool
+	param := params.CreateGARMToolParams{
+		Name:        "garm-agent-linux-amd64",
+		Description: "GARM agent for Linux AMD64",
+		Size:        1024,
+		OSType:      commonParams.Linux,
+		OSArch:      commonParams.Amd64,
+		Version:     "v1.0.0",
+	}
+	reader := bytes.NewReader([]byte("test binary"))
+	_, err := s.Runner.CreateGARMTool(s.AdminContext, param, reader)
+	s.Require().NoError(err)
+
+	tools, err := s.Runner.ListGARMTools(s.AdminContext)
+	s.Require().NoError(err)
+	s.Len(tools, 1)
+	s.Equal("linux", string(tools[0].OSType))
+	s.Equal("amd64", string(tools[0].OSArch))
+}
+
+func (s *GARMToolsTestSuite) TestListGARMToolsMultiplePlatforms() {
+	// Create tools for all supported platforms
+	platforms := []struct {
+		osType commonParams.OSType
+		osArch commonParams.OSArch
+	}{
+		{commonParams.Linux, commonParams.Amd64},
+		{commonParams.Linux, commonParams.Arm64},
+		{commonParams.Windows, commonParams.Amd64},
+		{commonParams.Windows, commonParams.Arm64},
+	}
+
+	for _, p := range platforms {
+		param := params.CreateGARMToolParams{
+			Name:        fmt.Sprintf("garm-agent-%s-%s", p.osType, p.osArch),
+			Description: fmt.Sprintf("GARM agent for %s %s", p.osType, p.osArch),
+			Size:        1024,
+			OSType:      p.osType,
+			OSArch:      p.osArch,
+			Version:     "v1.0.0",
+		}
+		reader := bytes.NewReader([]byte(fmt.Sprintf("%s %s binary", p.osType, p.osArch)))
+		_, err := s.Runner.CreateGARMTool(s.AdminContext, param, reader)
+		s.Require().NoError(err)
+	}
+
+	tools, err := s.Runner.ListGARMTools(s.AdminContext)
+	s.Require().NoError(err)
+	s.Len(tools, 4)
+
+	// Verify we have all platforms
+	foundPlatforms := make(map[string]bool)
+	for _, tool := range tools {
+		key := fmt.Sprintf("%s-%s", tool.OSType, tool.OSArch)
+		foundPlatforms[key] = true
+	}
+	s.True(foundPlatforms["linux-amd64"])
+	s.True(foundPlatforms["linux-arm64"])
+	s.True(foundPlatforms["windows-amd64"])
+	s.True(foundPlatforms["windows-arm64"])
+}
+
+// Helper function to extract version from tags
+func getVersionFromTags(tags []string) string {
+	for _, tag := range tags {
+		if len(tag) > 8 && tag[:8] == "version=" {
+			return tag[8:]
+		}
+	}
+	return ""
+}
+
+func TestGARMToolsTestSuite(t *testing.T) {
+	suite.Run(t, new(GARMToolsTestSuite))
+}

runner/interfaces.go

@@ -44,9 +44,15 @@ type EnterprisePoolManager interface {
 }
 
 //go:generate go run github.com/vektra/mockery/v2@latest
-
 type PoolManagerController interface {
 	RepoPoolManager
 	OrgPoolManager
 	EnterprisePoolManager
 }
+
+type AgentStoreOps interface {
+	RecordAgentHeartbeat(ctx context.Context) error
+	AddInstanceStatusMessage(ctx context.Context, param params.InstanceUpdateMessage) error
+	SetInstanceToPendingDelete(ctx context.Context) error
+	SetInstanceCapabilities(ctx context.Context, capabilities params.AgentCapabilities) error
+}

runner/metadata.go

@@ -22,7 +22,9 @@ import (
 	"errors"
 	"fmt"
 	"html/template"
+	"io"
 	"log/slog"
+	"net/url"
 	"strings"
 
 	"github.com/cloudbase/garm-provider-common/cloudconfig"
@@ -179,6 +181,7 @@ func (r *Runner) GetInstanceMetadata(ctx context.Context) (params.InstanceMetada
 
 	var entityGetter params.EntityGetter
 	var extraSpecs json.RawMessage
+	var enableShell bool
 	switch {
 	case instance.PoolID != "":
 		pool, err := r.store.GetPoolByID(r.ctx, instance.PoolID)
@@ -187,6 +190,7 @@ func (r *Runner) GetInstanceMetadata(ctx context.Context) (params.InstanceMetada
 		}
 		entityGetter = pool
 		extraSpecs = pool.ExtraSpecs
+		enableShell = pool.EnableShell
 	case instance.ScaleSetID != 0:
 		scaleSet, err := r.store.GetScaleSetByID(r.ctx, instance.ScaleSetID)
 		if err != nil {
@@ -194,6 +198,7 @@ func (r *Runner) GetInstanceMetadata(ctx context.Context) (params.InstanceMetada
 		}
 		entityGetter = scaleSet
 		extraSpecs = scaleSet.ExtraSpecs
+		enableShell = scaleSet.EnableShell
 	default:
 		// This is not actually an unauthorized scenario. This case means that an
 		// instance was created but it does not belong to any pool or scale set.
@@ -220,9 +225,28 @@ func (r *Runner) GetInstanceMetadata(ctx context.Context) (params.InstanceMetada
 		MetadataAccess: params.MetadataServiceAccessDetails{
 			CallbackURL: instance.CallbackURL,
 			MetadataURL: instance.MetadataURL,
+			AgentURL:    cache.ControllerInfo().AgentURL,
 		},
-		ForgeType:  dbEntity.Credentials.ForgeType,
-		JITEnabled: len(instance.JitConfiguration) > 0,
+		ForgeType:         dbEntity.Credentials.ForgeType,
+		JITEnabled:        len(instance.JitConfiguration) > 0,
+		AgentMode:         dbEntity.AgentMode,
+		AgentShellEnabled: enableShell,
+	}
+
+	if dbEntity.AgentMode {
+		agentTools, err := r.GetGARMTools(ctx, 0, 25)
+		if err != nil {
+			return params.InstanceMetadata{}, fmt.Errorf("failed to find garm agent tools: %w", err)
+		}
+		if agentTools.TotalCount == 0 {
+			return params.InstanceMetadata{}, runnerErrors.NewConflictError("agent mode is enabled, but agent tools not available")
+		}
+		ret.AgentTools = &agentTools.Results[0]
+		agentToken, err := r.GetAgentJWTToken(r.ctx, instance.Name)
+		if err != nil {
+			return params.InstanceMetadata{}, fmt.Errorf("failed to get agent token: %w", err)
+		}
+		ret.AgentToken = agentToken
 	}
 
 	if len(dbEntity.Credentials.Endpoint.CACertBundle) > 0 {
@@ -310,6 +334,9 @@ func (r *Runner) GetRunnerInstallScript(ctx context.Context) ([]byte, error) {
 		return nil, runnerErrors.NewConflictError("pool or scale set has no template associated and no template is defined in extra_specs")
 	}
 
+	if specs.ExtraContext == nil {
+		specs.ExtraContext = map[string]string{}
+	}
 	installCtx, err := r.getRunnerInstallTemplateContext(instance, entity, token, specs.ExtraContext)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get runner install context: %w", err)
@@ -490,9 +517,49 @@ func (r *Runner) GetRootCertificateBundle(ctx context.Context) (params.Certifica
 	return bundle, nil
 }
 
+func fileObjectToGARMTool(obj params.FileObject, downloadURL string) (params.GARMAgentTool, error) {
+	var version string
+	var osType string
+	var osArch string
+	for _, val := range obj.Tags {
+		if strings.HasPrefix(val, "version=") {
+			version = val[8:]
+		}
+		if strings.HasPrefix(val, "os_arch=") {
+			osArch = val[8:]
+		}
+		if strings.HasPrefix(val, "os_type=") {
+			osType = val[8:]
+		}
+	}
+	switch {
+	case version == "":
+		return params.GARMAgentTool{}, runnerErrors.NewConflictError("missing version for tools %d", obj.ID)
+	case osType == "":
+		return params.GARMAgentTool{}, runnerErrors.NewConflictError("missing os_type for tools %d", obj.ID)
+	case osArch == "":
+		return params.GARMAgentTool{}, runnerErrors.NewConflictError("missing os_arch for tools %d", obj.ID)
+	}
+	res := params.GARMAgentTool{
+		ID:          obj.ID,
+		Name:        obj.Name,
+		Size:        obj.Size,
+		SHA256SUM:   obj.SHA256,
+		Description: obj.Description,
+		CreatedAt:   obj.CreatedAt,
+		UpdatedAt:   obj.UpdatedAt,
+		FileType:    obj.FileType,
+		OSType:      commonParams.OSType(osType),
+		OSArch:      commonParams.OSArch(osArch),
+		DownloadURL: downloadURL,
+		Version:     version,
+	}
+	return res, nil
+}
+
 func (r *Runner) GetGARMTools(ctx context.Context, page, pageSize uint64) (params.GARMAgentToolsPaginatedResponse, error) {
 	tags := []string{
-		"category=garm-agent",
+		garmAgentFileTag,
 	}
 	instance, err := validateInstanceState(ctx)
 	if err != nil {
@@ -511,35 +578,14 @@ func (r *Runner) GetGARMTools(ctx context.Context, page, pageSize uint64) (param
 
 	var tools []params.GARMAgentTool
 	for _, val := range files.Results {
-		tags := val.Tags
-		var version string
-		var osType string
-		var osArch string
-		for _, val := range tags {
-			if strings.HasPrefix(val, "version=") {
-				version = val[8:]
-			}
-			if strings.HasPrefix(val, "os_arch=") {
-				osArch = val[8:]
-			}
-			if strings.HasPrefix(val, "os_type=") {
-				osType = val[8:]
-			}
+		objectIDAsString := fmt.Sprintf("%d", val.ID)
+		downloadURL, err := url.JoinPath(instance.MetadataURL, "tools/garm-agent", objectIDAsString, "download")
+		if err != nil {
+			return params.GARMAgentToolsPaginatedResponse{}, fmt.Errorf("failed to construct agent tools download URL: %w", err)
 		}
-		res := params.GARMAgentTool{
-			ID:          val.ID,
-			Name:        val.Name,
-			Size:        val.Size,
-			SHA256SUM:   val.SHA256,
-			Description: val.Description,
-			CreatedAt:   val.CreatedAt,
-			UpdatedAt:   val.UpdatedAt,
-			FileType:    val.FileType,
-			OSType:      commonParams.OSType(osType),
-			OSArch:      commonParams.OSArch(osArch),
-		}
-		if version != "" {
-			res.Version = version
+		res, err := fileObjectToGARMTool(val, downloadURL)
+		if err != nil {
+			return params.GARMAgentToolsPaginatedResponse{}, fmt.Errorf("failed parse tools object: %w", err)
 		}
 		tools = append(tools, res)
 	}
@@ -552,3 +598,81 @@ func (r *Runner) GetGARMTools(ctx context.Context, page, pageSize uint64) (param
 		Results:      tools,
 	}, nil
 }
+
+func (r *Runner) ShowGARMTools(ctx context.Context, toolsID uint) (params.GARMAgentTool, error) {
+	instance, err := validateInstanceState(ctx)
+	if err != nil {
+		if !auth.IsAdmin(ctx) {
+			return params.GARMAgentTool{}, runnerErrors.ErrUnauthorized
+		}
+	}
+
+	tools, err := r.store.GetFileObject(r.ctx, toolsID)
+	if err != nil {
+		return params.GARMAgentTool{}, fmt.Errorf("failed to list files: %w", err)
+	}
+
+	var version string
+	var osType string
+	var osArch string
+	var category string
+	for _, val := range tools.Tags {
+		if strings.HasPrefix(val, "version=") {
+			version = val[8:]
+		}
+		if strings.HasPrefix(val, "os_arch=") {
+			osArch = val[8:]
+		}
+		if strings.HasPrefix(val, "os_type=") {
+			osType = val[8:]
+		}
+		if strings.HasPrefix(val, "category=") {
+			category = val[9:]
+		}
+	}
+	if category != "garm-agent" {
+		slog.InfoContext(ctx, "selected object is not marked as garm-agent", "object_id", toolsID, "instance", instance.Name)
+		return params.GARMAgentTool{}, runnerErrors.ErrUnauthorized
+	}
+	if osType != string(instance.OSType) {
+		return params.GARMAgentTool{}, runnerErrors.NewBadRequestError("requested tools OS type (%s) does not match instance OS type (%s)", osType, instance.OSType)
+	}
+	if osArch != string(instance.OSArch) {
+		return params.GARMAgentTool{}, runnerErrors.NewBadRequestError("requested tools OS arch (%s) does not match instance OS arch (%s)", osArch, instance.OSArch)
+	}
+	agentIDAsString := fmt.Sprintf("%d", tools.ID)
+	downloadURL, err := url.JoinPath(instance.MetadataURL, "tools/garm-agent", agentIDAsString, "download")
+	if err != nil {
+		return params.GARMAgentTool{}, fmt.Errorf("failed to construct agent tools download URL: %w", err)
+	}
+	res := params.GARMAgentTool{
+		ID:          tools.ID,
+		Name:        tools.Name,
+		Size:        tools.Size,
+		SHA256SUM:   tools.SHA256,
+		Description: tools.Description,
+		CreatedAt:   tools.CreatedAt,
+		UpdatedAt:   tools.UpdatedAt,
+		FileType:    tools.FileType,
+		OSType:      commonParams.OSType(osType),
+		OSArch:      commonParams.OSArch(osArch),
+		DownloadURL: downloadURL,
+	}
+	if version != "" {
+		res.Version = version
+	}
+	return res, nil
+}
+
+func (r *Runner) GetGARMToolsReadHandler(ctx context.Context, toolsID uint) (io.ReadCloser, error) {
+	toolsDetails, err := r.ShowGARMTools(ctx, toolsID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to validate tools request: %w", err)
+	}
+
+	readCloser, err := r.store.OpenFileObjectContent(ctx, toolsDetails.ID)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file object: %w", err)
+	}
+	return readCloser, nil
+}

runner/metadata_test.go

@@ -77,7 +77,7 @@ func (s *MetadataTestSuite) SetupTest() {
 	testCreds := garmTesting.CreateTestGithubCredentials(s.adminCtx, "test-creds", db, s.T(), s.githubEndpoint)
 
 	// Create test organization
-	org, err := db.CreateOrganization(s.adminCtx, "test-org", testCreds, "test-webhook-secret", params.PoolBalancerTypeRoundRobin)
+	org, err := db.CreateOrganization(s.adminCtx, "test-org", testCreds, "test-webhook-secret", params.PoolBalancerTypeRoundRobin, false)
 	if err != nil {
 		s.FailNow(fmt.Sprintf("failed to create test org: %s", err))
 	}

runner/object_store.go

@@ -15,6 +15,7 @@ package runner
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 
@@ -27,7 +28,11 @@ func (r *Runner) CreateFileObject(ctx context.Context, param params.CreateFileOb
 	if !auth.IsAdmin(ctx) {
 		return params.FileObject{}, runnerErrors.ErrUnauthorized
 	}
-
+	for _, val := range param.Tags {
+		if val == garmAgentFileTag {
+			return params.FileObject{}, runnerErrors.NewBadRequestError("cannot create garm-agent tools via object storage API")
+		}
+	}
 	fileObj, err := r.store.CreateFileObject(ctx, param, reader)
 	if err != nil {
 		return params.FileObject{}, fmt.Errorf("failed to create file object: %w", err)
@@ -53,12 +58,43 @@ func (r *Runner) DeleteFileObject(ctx context.Context, objID uint) error {
 		return runnerErrors.ErrUnauthorized
 	}
 
+	object, err := r.store.GetFileObject(ctx, objID)
+	if err != nil {
+		if errors.Is(err, runnerErrors.ErrNotFound) {
+			return nil
+		}
+		return fmt.Errorf("failed to query object in DB: %w", err)
+	}
+	for _, val := range object.Tags {
+		if val == garmAgentFileTag {
+			return runnerErrors.NewBadRequestError("cannot delete garm-agent tools via object storage API")
+		}
+	}
 	if err := r.store.DeleteFileObject(ctx, objID); err != nil {
 		return fmt.Errorf("failed to delete file object: %w", err)
 	}
 	return nil
 }
 
+func (r *Runner) DeleteFileObjectsByTags(ctx context.Context, tags []string) (int64, error) {
+	if !auth.IsAdmin(ctx) {
+		return 0, runnerErrors.ErrUnauthorized
+	}
+
+	// Check if any of the tags include garm-agent tag
+	for _, tag := range tags {
+		if tag == garmAgentFileTag {
+			return 0, runnerErrors.NewBadRequestError("cannot delete garm-agent tools via object storage API")
+		}
+	}
+
+	deletedCount, err := r.store.DeleteFileObjectsByTags(ctx, tags)
+	if err != nil {
+		return 0, fmt.Errorf("failed to delete file objects by tags: %w", err)
+	}
+	return deletedCount, nil
+}
+
 func (r *Runner) ListFileObjects(ctx context.Context, page, pageSize uint64, tags []string) (params.FileObjectPaginatedResponse, error) {
 	if !auth.IsAdmin(ctx) {
 		return params.FileObjectPaginatedResponse{}, runnerErrors.ErrUnauthorized
@@ -82,6 +118,21 @@ func (r *Runner) UpdateFileObject(ctx context.Context, objID uint, param params.
 		return params.FileObject{}, runnerErrors.ErrUnauthorized
 	}
 
+	object, err := r.store.GetFileObject(ctx, objID)
+	if err != nil {
+		return params.FileObject{}, fmt.Errorf("failed to query object in DB: %w", err)
+	}
+	for _, val := range object.Tags {
+		if val == garmAgentFileTag {
+			return params.FileObject{}, runnerErrors.NewBadRequestError("cannot update garm-agent tools via object storage API")
+		}
+	}
+
+	for _, val := range param.Tags {
+		if val == garmAgentFileTag {
+			return params.FileObject{}, runnerErrors.NewBadRequestError("cannot update garm-agent tools via object storage API")
+		}
+	}
 	resp, err := r.store.UpdateFileObject(ctx, objID, param)
 	if err != nil {
 		return params.FileObject{}, fmt.Errorf("failed to update object: %w", err)

runner/object_store_test.go

@@ -129,6 +129,22 @@ func (s *ObjectStoreTestSuite) TestCreateFileObjectUnauthorized() {
 	s.Require().ErrorIs(err, runnerErrors.ErrUnauthorized)
 }
 
+func (s *ObjectStoreTestSuite) TestCreateFileObjectWithGarmAgentTag() {
+	content := []byte("garm-agent tool content")
+	reader := bytes.NewReader(content)
+
+	createParams := params.CreateFileObjectParams{
+		Name: "garm-agent-tool.bin",
+		Size: int64(len(content)),
+		Tags: []string{garmAgentFileTag, "test"},
+	}
+
+	_, err := s.Runner.CreateFileObject(s.Fixtures.AdminContext, createParams, reader)
+
+	s.Require().NotNil(err)
+	s.Require().Contains(err.Error(), "cannot create garm-agent tools via object storage API")
+}
+
 func (s *ObjectStoreTestSuite) TestGetFileObject() {
 	fileObj, err := s.Runner.GetFileObject(s.Fixtures.AdminContext, s.Fixtures.TestFileObject.ID)
 
@@ -187,6 +203,29 @@ func (s *ObjectStoreTestSuite) TestDeleteFileObjectNotFound() {
 	s.Require().Nil(err)
 }
 
+func (s *ObjectStoreTestSuite) TestDeleteFileObjectWithGarmAgentTag() {
+	// Create a file with garm-agent tag
+	content := []byte("garm-agent tool")
+	param := params.CreateFileObjectParams{
+		Name: "garm-agent-delete-test.bin",
+		Size: int64(len(content)),
+		Tags: []string{garmAgentFileTag},
+	}
+	// Create directly via store to bypass the API restriction
+	fileObj, err := s.Fixtures.Store.CreateFileObject(s.Fixtures.AdminContext, param, bytes.NewReader(content))
+	s.Require().Nil(err)
+
+	// Try to delete via API
+	err = s.Runner.DeleteFileObject(s.Fixtures.AdminContext, fileObj.ID)
+
+	s.Require().NotNil(err)
+	s.Require().Contains(err.Error(), "cannot delete garm-agent tools via object storage API")
+
+	// Verify file still exists
+	_, err = s.Fixtures.Store.GetFileObject(s.Fixtures.AdminContext, fileObj.ID)
+	s.Require().Nil(err)
+}
+
 func (s *ObjectStoreTestSuite) TestListFileObjects() {
 	// Create additional test files
 	for i := 1; i <= 3; i++ {
@@ -318,7 +357,62 @@ func (s *ObjectStoreTestSuite) TestUpdateFileObjectNotFound() {
 	_, err := s.Runner.UpdateFileObject(s.Fixtures.AdminContext, 99999, s.Fixtures.UpdateObjectParams)
 
 	s.Require().NotNil(err)
-	s.Require().Contains(err.Error(), "failed to update object")
+	s.Require().Contains(err.Error(), "failed to query object in DB")
+}
+
+func (s *ObjectStoreTestSuite) TestUpdateFileObjectWithGarmAgentTag() {
+	// Create a file with garm-agent tag
+	content := []byte("garm-agent tool")
+	param := params.CreateFileObjectParams{
+		Name: "garm-agent-update-test.bin",
+		Size: int64(len(content)),
+		Tags: []string{garmAgentFileTag},
+	}
+	// Create directly via store to bypass the API restriction
+	fileObj, err := s.Fixtures.Store.CreateFileObject(s.Fixtures.AdminContext, param, bytes.NewReader(content))
+	s.Require().Nil(err)
+
+	// Try to update via API
+	newName := "updated-agent-tool.bin"
+	updateParams := params.UpdateFileObjectParams{
+		Name: &newName,
+		Tags: []string{"updated"},
+	}
+	_, err = s.Runner.UpdateFileObject(s.Fixtures.AdminContext, fileObj.ID, updateParams)
+
+	s.Require().NotNil(err)
+	s.Require().Contains(err.Error(), "cannot update garm-agent tools via object storage API")
+
+	// Verify file is unchanged
+	unchanged, err := s.Fixtures.Store.GetFileObject(s.Fixtures.AdminContext, fileObj.ID)
+	s.Require().Nil(err)
+	s.Require().Equal(param.Name, unchanged.Name)
+}
+
+func (s *ObjectStoreTestSuite) TestUpdateFileObjectAddingGarmAgentTag() {
+	// Create a regular file
+	content := []byte("regular file")
+	param := params.CreateFileObjectParams{
+		Name: "regular-file.txt",
+		Size: int64(len(content)),
+		Tags: []string{"regular"},
+	}
+	fileObj, err := s.Fixtures.Store.CreateFileObject(s.Fixtures.AdminContext, param, bytes.NewReader(content))
+	s.Require().Nil(err)
+
+	// Try to add garm-agent tag via update
+	updateParams := params.UpdateFileObjectParams{
+		Tags: []string{garmAgentFileTag, "updated"},
+	}
+	_, err = s.Runner.UpdateFileObject(s.Fixtures.AdminContext, fileObj.ID, updateParams)
+
+	s.Require().NotNil(err)
+	s.Require().Contains(err.Error(), "cannot update garm-agent tools via object storage API")
+
+	// Verify file tags are unchanged
+	unchanged, err := s.Fixtures.Store.GetFileObject(s.Fixtures.AdminContext, fileObj.ID)
+	s.Require().Nil(err)
+	s.Require().ElementsMatch(param.Tags, unchanged.Tags)
 }
 
 func (s *ObjectStoreTestSuite) TestGetFileObjectReader() {
@@ -348,6 +442,98 @@ func (s *ObjectStoreTestSuite) TestGetFileObjectReaderNotFound() {
 	s.Require().Contains(err.Error(), "failed to open file object")
 }
 
+func (s *ObjectStoreTestSuite) TestDeleteFileObjectsByTags() {
+	// Create multiple test files with specific tags
+	for i := 1; i <= 5; i++ {
+		content := []byte(fmt.Sprintf("test file %d", i))
+		var tags []string
+		if i <= 3 {
+			// First 3 files have matching tags
+			tags = []string{"tag1=value1", "tag2=value2", "test"}
+		} else {
+			// Last 2 files have different tags
+			tags = []string{"tag1=value1", "other"}
+		}
+		param := params.CreateFileObjectParams{
+			Name: fmt.Sprintf("bulk-delete-test-%d.txt", i),
+			Size: int64(len(content)),
+			Tags: tags,
+		}
+		_, err := s.Fixtures.Store.CreateFileObject(s.Fixtures.AdminContext, param, bytes.NewReader(content))
+		s.Require().Nil(err)
+	}
+
+	// Delete files matching BOTH tags
+	deletedCount, err := s.Runner.DeleteFileObjectsByTags(
+		s.Fixtures.AdminContext,
+		[]string{"tag1=value1", "tag2=value2"},
+	)
+
+	s.Require().Nil(err)
+	s.Require().Equal(int64(3), deletedCount)
+
+	// Verify the right files were deleted
+	allObjects, err := s.Fixtures.Store.ListFileObjects(s.Fixtures.AdminContext, 0, 100)
+	s.Require().Nil(err)
+
+	// Count how many bulk-delete-test files remain
+	remainingCount := 0
+	for _, obj := range allObjects.Results {
+		if bytes.Contains([]byte(obj.Name), []byte("bulk-delete-test")) {
+			remainingCount++
+			// Should only be the last 2 files
+			s.Require().Contains(obj.Tags, "other")
+		}
+	}
+	s.Require().Equal(2, remainingCount)
+}
+
+func (s *ObjectStoreTestSuite) TestDeleteFileObjectsByTagsUnauthorized() {
+	deletedCount, err := s.Runner.DeleteFileObjectsByTags(
+		s.Fixtures.UnauthorizedContext,
+		[]string{"tag1", "tag2"},
+	)
+
+	s.Require().NotNil(err)
+	s.Require().ErrorIs(err, runnerErrors.ErrUnauthorized)
+	s.Require().Equal(int64(0), deletedCount)
+}
+
+func (s *ObjectStoreTestSuite) TestDeleteFileObjectsByTagsWithGarmAgentTag() {
+	// Try to delete with garm-agent tag
+	deletedCount, err := s.Runner.DeleteFileObjectsByTags(
+		s.Fixtures.AdminContext,
+		[]string{"category=garm-agent", "test"},
+	)
+
+	s.Require().NotNil(err)
+	s.Require().Contains(err.Error(), "cannot delete garm-agent tools via object storage API")
+	s.Require().Equal(int64(0), deletedCount)
+}
+
+func (s *ObjectStoreTestSuite) TestDeleteFileObjectsByTagsNoMatches() {
+	// Try to delete with tags that don't match anything
+	deletedCount, err := s.Runner.DeleteFileObjectsByTags(
+		s.Fixtures.AdminContext,
+		[]string{"nonexistent-tag1", "nonexistent-tag2"},
+	)
+
+	s.Require().Nil(err)
+	s.Require().Equal(int64(0), deletedCount)
+}
+
+func (s *ObjectStoreTestSuite) TestDeleteFileObjectsByTagsEmptyTags() {
+	// Try to delete with empty tags list
+	deletedCount, err := s.Runner.DeleteFileObjectsByTags(
+		s.Fixtures.AdminContext,
+		[]string{},
+	)
+
+	s.Require().NotNil(err)
+	s.Require().Contains(err.Error(), "no tags provided")
+	s.Require().Equal(int64(0), deletedCount)
+}
+
 func TestObjectStoreTestSuite(t *testing.T) {
 	suite.Run(t, new(ObjectStoreTestSuite))
 }

runner/organizations.go

@@ -62,7 +62,7 @@ func (r *Runner) CreateOrganization(ctx context.Context, param params.CreateOrgP
 		return params.Organization{}, runnerErrors.NewConflictError("organization %s already exists", param.Name)
 	}
 
-	org, err = r.store.CreateOrganization(ctx, param.Name, creds, param.WebhookSecret, param.PoolBalancerType)
+	org, err = r.store.CreateOrganization(ctx, param.Name, creds, param.WebhookSecret, param.PoolBalancerType, param.AgentMode)
 	if err != nil {
 		return params.Organization{}, fmt.Errorf("error creating organization: %w", err)
 	}

runner/organizations_test.go

@@ -89,6 +89,7 @@ func (s *OrgTestSuite) SetupTest() {
 			s.testCreds,
 			fmt.Sprintf("test-webhook-secret-%v", i),
 			params.PoolBalancerTypeRoundRobin,
+			false,
 		)
 		if err != nil {
 			s.FailNow(fmt.Sprintf("failed to create database object (test-org-%v)", i))
@@ -257,7 +258,9 @@ func (s *OrgTestSuite) TestListOrganizationsWithFilter() {
 		"test-org",
 		s.testCreds,
 		"super-secret",
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 	s.Require().NoError(err)
 
 	org2, err := s.Fixtures.Store.CreateOrganization(
@@ -265,7 +268,9 @@ func (s *OrgTestSuite) TestListOrganizationsWithFilter() {
 		"test-org",
 		s.giteaTestCreds,
 		"super-secret",
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 	s.Require().NoError(err)
 
 	org3, err := s.Fixtures.Store.CreateOrganization(
@@ -273,7 +278,9 @@ func (s *OrgTestSuite) TestListOrganizationsWithFilter() {
 		"test-org2",
 		s.giteaTestCreds,
 		"super-secret",
-		params.PoolBalancerTypeRoundRobin)
+		params.PoolBalancerTypeRoundRobin,
+		false,
+	)
 	s.Require().NoError(err)
 
 	orgs, err := s.Runner.ListOrganizations(

runner/pool/pool.go

@@ -1269,7 +1269,7 @@ func (r *basePoolManager) retryFailedInstancesForOnePool(ctx context.Context, po
 			// TODO(gabriel-samfira): Incrementing CreateAttempt should be done within a transaction.
 			// It's fairly safe to do here (for now), as there should be no other code path that updates
 			// an instance in this state.
-			var tokenFetched bool = len(instance.JitConfiguration) > 0
+			tokenFetched := len(instance.JitConfiguration) > 0
 			updateParams := params.UpdateInstanceParams{
 				CreateAttempt: instance.CreateAttempt + 1,
 				TokenFetched:  &tokenFetched,

runner/pools_test.go

@@ -69,7 +69,7 @@ func (s *PoolTestSuite) SetupTest() {
 	s.secondaryTestCreds = garmTesting.CreateTestGithubCredentials(s.adminCtx, "secondary-creds", db, s.T(), s.githubEndpoint)
 
 	// create an organization for testing purposes
-	org, err := db.CreateOrganization(s.adminCtx, "test-org", s.testCreds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin)
+	org, err := db.CreateOrganization(s.adminCtx, "test-org", s.testCreds, "test-webhookSecret", params.PoolBalancerTypeRoundRobin, false)
 	if err != nil {
 		s.FailNow(fmt.Sprintf("failed to create org: %s", err))
 	}

runner/repositories.go

@@ -61,7 +61,7 @@ func (r *Runner) CreateRepository(ctx context.Context, param params.CreateRepoPa
 		return params.Repository{}, runnerErrors.NewConflictError("repository %s/%s already exists", param.Owner, param.Name)
 	}
 
-	repo, err = r.store.CreateRepository(ctx, param.Owner, param.Name, creds, param.WebhookSecret, param.PoolBalancerType)
+	repo, err = r.store.CreateRepository(ctx, param.Owner, param.Name, creds, param.WebhookSecret, param.PoolBalancerType, param.AgentMode)
 	if err != nil {
 		return params.Repository{}, fmt.Errorf("error creating repository: %w", err)
 	}
@@ -248,7 +248,7 @@ func (r *Runner) findTemplate(ctx context.Context, entity params.ForgeEntity, os
 		}
 		for _, val := range tpls {
 			slog.InfoContext(ctx, "considering template", "name", val.Name, "os_type", val.OSType, "pool_os_type", osType, "forge_type", val.ForgeType, "pool_forge_type", entity.Credentials.ForgeType, "owner", val.Owner)
-			if val.OSType == osType && val.ForgeType == entity.Credentials.ForgeType && val.Owner == "system" {
+			if val.OSType == osType && val.ForgeType == entity.Credentials.ForgeType && val.Owner == params.SystemUser {
 				template = val
 				break
 			}

runner/repositories_test.go

@@ -93,6 +93,7 @@ func (s *RepoTestSuite) SetupTest() {
 			s.testCreds,
 			fmt.Sprintf("test-webhook-secret-%v", i),
 			params.PoolBalancerTypeRoundRobin,
+			false,
 		)
 		if err != nil {
 			s.FailNow(fmt.Sprintf("failed to create database object (test-repo-%v): %q", i, err))
@@ -275,6 +276,7 @@ func (s *RepoTestSuite) TestListRepositoriesWithFilters() {
 		s.testCreds,
 		"test-webhook-secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	if err != nil {
 		s.FailNow(fmt.Sprintf("failed to create database object (example-repo): %q", err))
@@ -287,6 +289,7 @@ func (s *RepoTestSuite) TestListRepositoriesWithFilters() {
 		s.testCreds,
 		"test-webhook-secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	if err != nil {
 		s.FailNow(fmt.Sprintf("failed to create database object (example-repo): %q", err))
@@ -299,6 +302,7 @@ func (s *RepoTestSuite) TestListRepositoriesWithFilters() {
 		s.giteaTestCreds,
 		"test-webhook-secret",
 		params.PoolBalancerTypeRoundRobin,
+		false,
 	)
 	if err != nil {
 		s.FailNow(fmt.Sprintf("failed to create database object (example-repo): %q", err))

runner/runner.go

@@ -76,6 +76,7 @@ func NewRunner(ctx context.Context, cfg config.Config, db dbCommon.Store, token
 		ctx:             ctx,
 		config:          cfg,
 		store:           db,
+		tokenGetter:     token,
 		poolManagerCtrl: poolManagerCtrl,
 		providers:       providers,
 	}
@@ -228,9 +229,10 @@ func (p *poolManagerCtrl) GetEnterprisePoolManagers() (map[string]common.PoolMan
 type Runner struct {
 	mux sync.Mutex
 
-	config config.Config
-	ctx    context.Context
-	store  dbCommon.Store
+	config      config.Config
+	ctx         context.Context
+	store       dbCommon.Store
+	tokenGetter auth.InstanceTokenGetter
 
 	poolManagerCtrl PoolManagerController
 

runner/templates.go

@@ -21,6 +21,7 @@ import (
 	runnerErrors "github.com/cloudbase/garm-provider-common/errors"
 	commonParams "github.com/cloudbase/garm-provider-common/params"
 	"github.com/cloudbase/garm/auth"
+	"github.com/cloudbase/garm/internal/templates"
 	"github.com/cloudbase/garm/params"
 )
 
@@ -62,6 +63,94 @@ func (r *Runner) GetTemplateByName(ctx context.Context, templateName string) (pa
 	return template, nil
 }
 
+func (r *Runner) RestoreTemplate(ctx context.Context, param params.RestoreTemplateRequest) error {
+	if !auth.IsAdmin(ctx) {
+		return runnerErrors.ErrUnauthorized
+	}
+
+	// Determine which templates to restore
+	var templatesConfig []struct {
+		OS    commonParams.OSType
+		Forge params.EndpointType
+	}
+
+	if param.RestoreAll {
+		// Restore all system templates
+		for _, os := range []commonParams.OSType{commonParams.Linux, commonParams.Windows} {
+			for _, forge := range []params.EndpointType{params.GiteaEndpointType, params.GithubEndpointType} {
+				templatesConfig = append(templatesConfig, struct {
+					OS    commonParams.OSType
+					Forge params.EndpointType
+				}{OS: os, Forge: forge})
+			}
+		}
+	} else {
+		// Restore specific template
+		templatesConfig = append(templatesConfig, struct {
+			OS    commonParams.OSType
+			Forge params.EndpointType
+		}{OS: param.OSType, Forge: param.Forge})
+	}
+
+	// Process each template
+	for _, cfg := range templatesConfig {
+		// Get the template content from internal/templates
+		templateContent, err := templates.GetTemplateContent(cfg.OS, cfg.Forge)
+		if err != nil {
+			return fmt.Errorf("failed to get template content for %s/%s: %w", cfg.Forge, cfg.OS, err)
+		}
+
+		// Find existing system template for this OS/Forge combination
+		existingTemplates, err := r.ListTemplates(ctx, &cfg.OS, &cfg.Forge, nil)
+		if err != nil {
+			return fmt.Errorf("failed to list templates for %s/%s: %w", cfg.Forge, cfg.OS, err)
+		}
+
+		var systemTemplate *params.Template
+		for _, tpl := range existingTemplates {
+			if tpl.Owner == params.SystemUser || tpl.Owner == "" {
+				systemTemplate = &tpl
+				break
+			}
+		}
+
+		// Generate template name
+		templateName := fmt.Sprintf("%s_%s", cfg.Forge, cfg.OS)
+		description := fmt.Sprintf("Default template for %s runners on %s", cfg.Forge, cfg.OS)
+
+		if systemTemplate != nil {
+			// Update existing system template
+			updateParams := params.UpdateTemplateParams{
+				Data: templateContent,
+			}
+			// Only update name if it was changed by user (different from expected system name)
+			if systemTemplate.Name != templateName {
+				updateParams.Name = &templateName
+			}
+			_, err := r.UpdateTemplate(ctx, systemTemplate.ID, updateParams)
+			if err != nil {
+				return fmt.Errorf("failed to update system template %d for %s/%s: %w", systemTemplate.ID, cfg.Forge, cfg.OS, err)
+			}
+		} else {
+			// Create new system template
+			createParams := params.CreateTemplateParams{
+				Name:        templateName,
+				Description: description,
+				Data:        templateContent,
+				OSType:      cfg.OS,
+				ForgeType:   cfg.Forge,
+				IsSystem:    true,
+			}
+			_, err := r.CreateTemplate(ctx, createParams)
+			if err != nil {
+				return fmt.Errorf("failed to create system template for %s/%s: %w", cfg.Forge, cfg.OS, err)
+			}
+		}
+	}
+
+	return nil
+}
+
 func (r *Runner) ListTemplates(ctx context.Context, osType *commonParams.OSType, forgeType *params.EndpointType, partialName *string) ([]params.Template, error) {
 	if !auth.IsAdmin(ctx) {
 		return nil, runnerErrors.ErrUnauthorized

runner/templates_test.go

@@ -321,6 +321,258 @@ func (s *TemplateTestSuite) TestDeleteTemplateNotFound() {
 	s.Require().Nil(err) // Should not error for not found templates
 }
 
+func (s *TemplateTestSuite) TestRestoreTemplateSpecific() {
+	osType := commonParams.Linux
+	forgeType := params.GithubEndpointType
+	templates, err := s.Runner.ListTemplates(s.adminCtx, &osType, &forgeType, nil)
+	s.Require().Nil(err)
+	s.Require().GreaterOrEqual(len(templates), 1, "Expected at least one github_linux template from migration")
+
+	var systemTemplate *params.Template
+	for _, tpl := range templates {
+		if tpl.Owner == params.SystemUser {
+			systemTemplate = &tpl
+			break
+		}
+	}
+	s.Require().NotNil(systemTemplate, "Expected system template for github_linux")
+
+	modifiedName := "modified_template_name"
+	modifiedData := []byte("modified template content for testing")
+	updateParams := params.UpdateTemplateParams{
+		Name: &modifiedName,
+		Data: modifiedData,
+	}
+	_, err = s.Runner.UpdateTemplate(s.adminCtx, systemTemplate.ID, updateParams)
+	s.Require().Nil(err)
+
+	updatedTemplate, err := s.Runner.GetTemplate(s.adminCtx, systemTemplate.ID)
+	s.Require().Nil(err)
+	s.Require().Equal(modifiedName, updatedTemplate.Name)
+	s.Require().Equal(modifiedData, updatedTemplate.Data)
+
+	restoreParams := params.RestoreTemplateRequest{
+		Forge:      params.GithubEndpointType,
+		OSType:     commonParams.Linux,
+		RestoreAll: false,
+	}
+	err = s.Runner.RestoreTemplate(s.adminCtx, restoreParams)
+	s.Require().Nil(err)
+
+	// Verify the template was restored
+	restoredTemplate, err := s.Runner.GetTemplate(s.adminCtx, systemTemplate.ID)
+	s.Require().Nil(err)
+	// Name should be restored to the system default
+	s.Require().Equal("github_linux", restoredTemplate.Name)
+	// Data should be restored to original template content (not the modified content)
+	s.Require().NotEqual(modifiedData, restoredTemplate.Data)
+	// Should match the original template data or be close to it (content from internal/templates)
+	s.Require().NotEmpty(restoredTemplate.Data)
+	// Verify it's still a system template
+	s.Require().Equal(params.SystemUser, restoredTemplate.Owner)
+
+	// Verify the data is different from what we modified (restored back to system template)
+	s.Require().NotEqual(string(modifiedData), string(restoredTemplate.Data))
+}
+
+func (s *TemplateTestSuite) TestRestoreTemplateAll() {
+	// Get all system templates
+	allTemplates, err := s.Runner.ListTemplates(s.adminCtx, nil, nil, nil)
+	s.Require().Nil(err)
+
+	// Find all system templates
+	systemTemplates := []params.Template{}
+	for _, tpl := range allTemplates {
+		if tpl.Owner == params.SystemUser {
+			systemTemplates = append(systemTemplates, tpl)
+		}
+	}
+	// We should have at least 4 system templates (github/gitea x linux/windows)
+	s.Require().GreaterOrEqual(len(systemTemplates), 4, "Expected at least 4 system templates")
+
+	// Modify all system templates
+	modifiedTemplateIDs := make(map[uint]struct {
+		originalName string
+		originalData []byte
+	})
+
+	for _, tpl := range systemTemplates {
+		modifiedName := "modified_" + tpl.Name
+		modifiedData := []byte("modified content for " + tpl.Name)
+
+		updateParams := params.UpdateTemplateParams{
+			Name: &modifiedName,
+			Data: modifiedData,
+		}
+		_, err := s.Runner.UpdateTemplate(s.adminCtx, tpl.ID, updateParams)
+		s.Require().Nil(err)
+
+		modifiedTemplateIDs[tpl.ID] = struct {
+			originalName string
+			originalData []byte
+		}{
+			originalName: tpl.Name,
+			originalData: tpl.Data,
+		}
+	}
+
+	// Verify all templates were modified
+	for templateID := range modifiedTemplateIDs {
+		template, err := s.Runner.GetTemplate(s.adminCtx, templateID)
+		s.Require().Nil(err)
+		s.Require().Contains(template.Name, "modified_", "Template name should contain 'modified_'")
+	}
+
+	// Restore all templates
+	restoreParams := params.RestoreTemplateRequest{
+		RestoreAll: true,
+	}
+	err = s.Runner.RestoreTemplate(s.adminCtx, restoreParams)
+	s.Require().Nil(err)
+
+	// Verify all templates were restored
+	for templateID := range modifiedTemplateIDs {
+		template, err := s.Runner.GetTemplate(s.adminCtx, templateID)
+		s.Require().Nil(err)
+
+		// Name should not contain "modified_" anymore
+		s.Require().NotContains(template.Name, "modified_", "Template name should be restored")
+		// Should still be a system template
+		s.Require().Equal(params.SystemUser, template.Owner)
+		// Data should be restored from internal/templates
+		s.Require().NotEmpty(template.Data)
+		s.Require().NotContains(string(template.Data), "modified content", "Template data should be restored")
+	}
+
+	// Verify we can still find templates for each OS/Forge combination
+	combinations := []struct {
+		os    commonParams.OSType
+		forge params.EndpointType
+	}{
+		{commonParams.Linux, params.GithubEndpointType},
+		{commonParams.Windows, params.GithubEndpointType},
+		{commonParams.Linux, params.GiteaEndpointType},
+		{commonParams.Windows, params.GiteaEndpointType},
+	}
+
+	for _, combo := range combinations {
+		templates, err := s.Runner.ListTemplates(s.adminCtx, &combo.os, &combo.forge, nil)
+		s.Require().Nil(err)
+
+		foundSystem := false
+		for _, tpl := range templates {
+			if tpl.Owner == params.SystemUser {
+				foundSystem = true
+				break
+			}
+		}
+		s.Require().True(foundSystem, "Should have system template for %s/%s", combo.forge, combo.os)
+	}
+}
+
+func (s *TemplateTestSuite) TestRestoreTemplateMissingTemplate() {
+	// Delete a system template
+	osType := commonParams.Windows
+	forgeType := params.GiteaEndpointType
+	templates, err := s.Runner.ListTemplates(s.adminCtx, &osType, &forgeType, nil)
+	s.Require().Nil(err)
+
+	var systemTemplate *params.Template
+	for _, tpl := range templates {
+		if tpl.Owner == params.SystemUser {
+			systemTemplate = &tpl
+			break
+		}
+	}
+	s.Require().NotNil(systemTemplate, "Expected system template for gitea_windows")
+
+	// Delete the template
+	err = s.Runner.DeleteTemplate(s.adminCtx, systemTemplate.ID)
+	s.Require().Nil(err)
+
+	// Verify it's deleted
+	templates, err = s.Runner.ListTemplates(s.adminCtx, &osType, &forgeType, nil)
+	s.Require().Nil(err)
+
+	foundSystem := false
+	for _, tpl := range templates {
+		if tpl.Owner == params.SystemUser {
+			foundSystem = true
+			break
+		}
+	}
+	s.Require().False(foundSystem, "System template should be deleted")
+
+	restoreParams := params.RestoreTemplateRequest{
+		Forge:      params.GiteaEndpointType,
+		OSType:     commonParams.Windows,
+		RestoreAll: false,
+	}
+	err = s.Runner.RestoreTemplate(s.adminCtx, restoreParams)
+	s.Require().Nil(err)
+
+	templates, err = s.Runner.ListTemplates(s.adminCtx, &osType, &forgeType, nil)
+	s.Require().Nil(err)
+
+	foundSystem = false
+	var recreatedTemplateID uint
+	for _, tpl := range templates {
+		if tpl.Owner == params.SystemUser {
+			foundSystem = true
+			recreatedTemplateID = tpl.ID
+			break
+		}
+	}
+	s.Require().True(foundSystem, "System template should be recreated")
+	s.Require().NotZero(recreatedTemplateID)
+
+	// Get the full template with data
+	recreatedTemplate, err := s.Runner.GetTemplate(s.adminCtx, recreatedTemplateID)
+	s.Require().Nil(err)
+	s.Require().Equal("gitea_windows", recreatedTemplate.Name)
+	s.Require().NotEmpty(recreatedTemplate.Data)
+}
+
+func (s *TemplateTestSuite) TestRestoreTemplateUnauthorized() {
+	restoreParams := params.RestoreTemplateRequest{
+		Forge:      params.GithubEndpointType,
+		OSType:     commonParams.Linux,
+		RestoreAll: false,
+	}
+
+	err := s.Runner.RestoreTemplate(s.nonAdminCtx, restoreParams)
+
+	s.Require().NotNil(err)
+	s.Require().ErrorIs(err, runnerErrors.ErrUnauthorized)
+}
+
+func (s *TemplateTestSuite) TestRestoreTemplatePreservesUserTemplates() {
+	// Create a user template with the same OS/Forge as a system template
+	userTemplate, err := s.Runner.CreateTemplate(s.adminCtx, params.CreateTemplateParams{
+		Name:        "user-github-linux-template",
+		Description: "User's custom template",
+		OSType:      commonParams.Linux,
+		ForgeType:   params.GithubEndpointType,
+		Data:        []byte("user custom template data"),
+	})
+	s.Require().Nil(err)
+	s.Require().NotEqual(params.SystemUser, userTemplate.Owner, "User template should not be system owned")
+
+	// Restore all templates
+	restoreParams := params.RestoreTemplateRequest{
+		RestoreAll: true,
+	}
+	err = s.Runner.RestoreTemplate(s.adminCtx, restoreParams)
+	s.Require().Nil(err)
+
+	// Verify user template still exists and wasn't modified
+	userTemplateAfter, err := s.Runner.GetTemplate(s.adminCtx, userTemplate.ID)
+	s.Require().Nil(err)
+	s.Require().Equal(userTemplate.Name, userTemplateAfter.Name)
+	s.Require().Equal(userTemplate.Data, userTemplateAfter.Data)
+	s.Require().NotEqual(params.SystemUser, userTemplateAfter.Owner)
+}
+
 func TestTemplateTestSuite(t *testing.T) {
 	suite.Run(t, new(TemplateTestSuite))
 }

util/appdefaults/appdefaults.go

@@ -49,6 +49,9 @@ const (
 	GiteaRunnerReleasesURL = "https://gitea.com/api/v1/repos/gitea/act_runner/releases"
 	// GiteaRunnerMinimumVersion is the minimum version we need in order to support ephemeral runners.
 	GiteaRunnerMinimumVersion = "v0.2.12"
+
+	// GARM agent releases URL
+	GARMAgentDefaultReleasesURL = "https://api.github.com/repos/cloudbase/garm-agent/releases"
 )
 
 var Version string

webapp/assets/_app/immutable/chunks/B-zJt1QP.js

@@ -1 +1 @@
-import{s as t,p as r}from"./DeKGyprh.js";const e={get data(){return r.data},get error(){return r.error},get form(){return r.form},get params(){return r.params},get route(){return r.route},get state(){return r.state},get status(){return r.status},get url(){return r.url}};t.updated.check;const s=e;export{s as p};
+import{s as t,p as r}from"./BSYpqPvJ.js";const e={get data(){return r.data},get error(){return r.error},get form(){return r.form},get params(){return r.params},get route(){return r.route},get state(){return r.state},get status(){return r.status},get url(){return r.url}};t.updated.check;const s=e;export{s as p};

webapp/assets/_app/immutable/chunks/B5vVe1Fo.js

@@ -1 +0,0 @@
-import"./DsnmJJEf.js";import{i as D}from"./TJn6xDN9.js";import{p as E,E as B,l as t,q as s,g as e,m as o,h as P,f as T,t as q,i as S,b as F,c as G,u as I,d as _,k as a,r as z}from"./DniTuB_0.js";import{i as J,h as K,s as N,j as O}from"./DD3srElq.js";import{l as j,p as l}from"./DbNhg6mG.js";var Q=T('<button><svg fill="none" stroke="currentColor" viewBox="0 0 24 24"><!></svg></button>');function Z(C,i){const M=j(i,["children","$$slots","$$events","$$legacy"]),L=j(M,["action","disabled","title","ariaLabel","size"]);E(i,!1);const u=o(),h=o(),k=o(),f=o(),g=o(),v=o(),n=o(),m=o(),b=o(),A=B();let r=l(i,"action",8,"edit"),x=l(i,"disabled",8,!1),w=l(i,"title",8,""),y=l(i,"ariaLabel",8,""),c=l(i,"size",8,"md");function H(){x()||A("click")}t(()=>{},()=>{a(u,"transition-colors focus:outline-none focus:ring-2 focus:ring-offset-2 dark:focus:ring-offset-gray-900 cursor-pointer disabled:cursor-not-allowed disabled:opacity-50")}),t(()=>s(c()),()=>{a(h,{sm:"p-1",md:"p-2"}[c()])}),t(()=>s(r()),()=>{a(k,{edit:"text-indigo-600 dark:text-indigo-400 hover:text-indigo-900 dark:hover:text-indigo-300 focus:ring-indigo-500",delete:"text-red-600 dark:text-red-400 hover:text-red-900 dark:hover:text-red-300 focus:ring-red-500",view:"text-gray-600 dark:text-gray-400 hover:text-gray-900 dark:hover:text-gray-300 focus:ring-gray-500",add:"text-green-600 dark:text-green-400 hover:text-green-900 dark:hover:text-green-300 focus:ring-green-500",copy:"text-blue-600 dark:text-blue-400 hover:text-blue-900 dark:hover:text-blue-300 focus:ring-blue-500",download:"text-blue-600 dark:text-blue-400 hover:text-blue-900 dark:hover:text-blue-300 focus:ring-blue-500"}[r()])}),t(()=>s(c()),()=>{a(f,c()==="sm"?"h-4 w-4":"h-5 w-5")}),t(()=>(e(u),e(h),e(k)),()=>{a(g,[e(u),e(h),e(k)].join(" "))}),t(()=>{},()=>{a(v,{edit:'<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z" />',delete:'<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 7l-.867 12.142A2 2 0 0116.138 21H7.862a2 2 0 01-1.995-1.858L5 7m5 4v6m4-6v6m1-10V4a1 1 0 00-1-1h-4a1 1 0 00-1 1v3M4 7h16" />',view:'<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 12a3 3 0 11-6 0 3 3 0 016 0z" /><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M2.458 12C3.732 7.943 7.523 5 12 5c4.478 0 8.268 2.943 9.542 7-1.274 4.057-5.064 7-9.542 7-4.477 0-8.268-2.943-9.542-7z" />',add:'<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 6v6m0 0v6m0-6h6m-6 0H6" />',copy:'<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M8 16H6a2 2 0 01-2-2V6a2 2 0 012-2h8a2 2 0 012 2v2m-6 12h8a2 2 0 002-2v-8a2 2 0 00-2-2h-8a2 2 0 00-2 2v8a2 2 0 002 2z" />',download:'<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-4l-4 4m0 0l-4-4m4 4V4" />'})}),t(()=>{},()=>{a(n,{edit:"Edit",delete:"Delete",view:"View",add:"Add",copy:"Clone",download:"Download"})}),t(()=>(s(w()),e(n),s(r())),()=>{a(m,w()||e(n)[r()])}),t(()=>(s(y()),e(n),s(r())),()=>{a(b,y()||`${e(n)[r()]} item`)}),P(),D();var d=Q();J(d,()=>({type:"button",class:e(g),disabled:x(),title:e(m),"aria-label":e(b),...L}));var p=_(d),V=_(p);K(V,()=>(e(v),s(r()),I(()=>e(v)[r()])),!0),z(p),z(d),q(()=>N(p,0,O(e(f)))),S("click",d,H),F(C,d),G()}export{Z as A};

webapp/assets/_app/immutable/chunks/B6XSaEl0.js

@@ -1 +0,0 @@
-import"./DsnmJJEf.js";import{i as q}from"./TJn6xDN9.js";import{p as A,E as F,f as y,s as l,d as t,r as a,z as $,D as b,b as o,t as p,e as n,c as G}from"./DniTuB_0.js";import{p as v,i as H}from"./DbNhg6mG.js";import{M as I}from"./PC4wJWhj.js";import{B as w}from"./DD3srElq.js";var J=y('<p class="mt-1 font-medium text-gray-900 dark:text-white"> </p>'),K=y('<div class="max-w-xl w-full p-6"><div class="mx-auto flex items-center justify-center h-12 w-12 rounded-full bg-red-100 dark:bg-red-900 mb-4"><svg class="h-6 w-6 text-red-600 dark:text-red-400" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 9v2m0 4h.01m-6.938 4h13.856c1.54 0 2.502-1.667 1.732-2.5L13.732 4c-.77-.833-1.964-.833-2.732 0L3.732 16.5c-.77.833.192 2.5 1.732 2.5z"></path></svg></div> <div class="text-center"><h3 class="text-lg leading-6 font-medium text-gray-900 dark:text-white mb-2"> </h3> <div class="text-sm text-gray-500 dark:text-gray-400"><p> </p> <!></div></div> <div class="mt-6 flex justify-end space-x-3"><!> <!></div></div>');function W(D,s){A(s,!1);let M=v(s,"title",8),j=v(s,"message",8),g=v(s,"itemName",8,""),d=v(s,"loading",8,!1);const c=F();function B(){c("confirm")}q(),I(D,{$$events:{close:()=>c("close")},children:(C,O)=>{var m=K(),f=l(t(m),2),u=t(f),P=t(u,!0);a(u);var h=l(u,2),x=t(h),z=t(x,!0);a(x);var E=l(x,2);{var L=e=>{var i=J(),r=t(i,!0);a(i),p(()=>n(r,g())),o(e,i)};H(E,e=>{g()&&e(L)})}a(h),a(f);var _=l(f,2),k=t(_);w(k,{variant:"secondary",get disabled(){return d()},$$events:{click:()=>c("close")},children:(e,i)=>{$();var r=b("Cancel");o(e,r)},$$slots:{default:!0}});var N=l(k,2);w(N,{variant:"danger",get disabled(){return d()},get loading(){return d()},$$events:{click:B},children:(e,i)=>{$();var r=b();p(()=>n(r,d()?"Deleting...":"Delete")),o(e,r)},$$slots:{default:!0}}),a(_),a(m),p(()=>{n(P,M()),n(z,j())}),o(C,m)},$$slots:{default:!0}}),G()}export{W as D};

webapp/assets/_app/immutable/chunks/B7If18kh.js

@@ -1,4 +1,4 @@
-import{d as o}from"./DD3srElq.js";function l(r){if(!r)return"N/A";try{return(typeof r=="string"?new Date(r):r).toLocaleString()}catch{return"Invalid Date"}}function f(r,e="w-4 h-4"){return r==="gitea"?`<svg class="${e}" xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 640 640"><path d="m395.9 484.2-126.9-61c-12.5-6-17.9-21.2-11.8-33.8l61-126.9c6-12.5 21.2-17.9 33.8-11.8 17.2 8.3 27.1 13 27.1 13l-.1-109.2 16.7-.1.1 117.1s57.4 24.2 83.1 40.1c3.7 2.3 10.2 6.8 12.9 14.4 2.1 6.1 2 13.1-1 19.3l-61 126.9c-6.2 12.7-21.4 18.1-33.9 12" style="fill:#fff"/><path d="M622.7 149.8c-4.1-4.1-9.6-4-9.6-4s-117.2 6.6-177.9 8c-13.3.3-26.5.6-39.6.7v117.2c-5.5-2.6-11.1-5.3-16.6-7.9 0-36.4-.1-109.2-.1-109.2-29 .4-89.2-2.2-89.2-2.2s-141.4-7.1-156.8-8.5c-9.8-.6-22.5-2.1-39 1.5-8.7 1.8-33.5 7.4-53.8 26.9C-4.9 212.4 6.6 276.2 8 285.8c1.7 11.7 6.9 44.2 31.7 72.5 45.8 56.1 144.4 54.8 144.4 54.8s12.1 28.9 30.6 55.5c25 33.1 50.7 58.9 75.7 62 63 0 188.9-.1 188.9-.1s12 .1 28.3-10.3c14-8.5 26.5-23.4 26.5-23.4S547 483 565 451.5c5.5-9.7 10.1-19.1 14.1-28 0 0 55.2-117.1 55.2-231.1-1.1-34.5-9.6-40.6-11.6-42.6M125.6 353.9c-25.9-8.5-36.9-18.7-36.9-18.7S69.6 321.8 60 295.4c-16.5-44.2-1.4-71.2-1.4-71.2s8.4-22.5 38.5-30c13.8-3.7 31-3.1 31-3.1s7.1 59.4 15.7 94.2c7.2 29.2 24.8 77.7 24.8 77.7s-26.1-3.1-43-9.1m300.3 107.6s-6.1 14.5-19.6 15.4c-5.8.4-10.3-1.2-10.3-1.2s-.3-.1-5.3-2.1l-112.9-55s-10.9-5.7-12.8-15.6c-2.2-8.1 2.7-18.1 2.7-18.1L322 273s4.8-9.7 12.2-13c.6-.3 2.3-1 4.5-1.5 8.1-2.1 18 2.8 18 2.8L467.4 315s12.6 5.7 15.3 16.2c1.9 7.4-.5 14-1.8 17.2-6.3 15.4-55 113.1-55 113.1" style="fill:#609926"/><path d="M326.8 380.1c-8.2.1-15.4 5.8-17.3 13.8s2 16.3 9.1 20c7.7 4 17.5 1.8 22.7-5.4 5.1-7.1 4.3-16.9-1.8-23.1l24-49.1c1.5.1 3.7.2 6.2-.5 4.1-.9 7.1-3.6 7.1-3.6 4.2 1.8 8.6 3.8 13.2 6.1 4.8 2.4 9.3 4.9 13.4 7.3.9.5 1.8 1.1 2.8 1.9 1.6 1.3 3.4 3.1 4.7 5.5 1.9 5.5-1.9 14.9-1.9 14.9-2.3 7.6-18.4 40.6-18.4 40.6-8.1-.2-15.3 5-17.7 12.5-2.6 8.1 1.1 17.3 8.9 21.3s17.4 1.7 22.5-5.3c5-6.8 4.6-16.3-1.1-22.6 1.9-3.7 3.7-7.4 5.6-11.3 5-10.4 13.5-30.4 13.5-30.4.9-1.7 5.7-10.3 2.7-21.3-2.5-11.4-12.6-16.7-12.6-16.7-12.2-7.9-29.2-15.2-29.2-15.2s0-4.1-1.1-7.1c-1.1-3.1-2.8-5.1-3.9-6.3 4.7-9.7 9.4-19.3 14.1-29-4.1-2-8.1-4-12.2-6.1-4.8 9.8-9.7 19.7-14.5 29.5-6.7-.1-12.9 3.5-16.1 9.4-3.4 6.3-2.7 14.1 1.9 19.8z" style="fill:#609926"/></svg>`:r==="github"?`<div class="inline-flex ${e}"><svg class="${e} dark:hidden" width="98" height="96" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 98 96"><path fill-rule="evenodd" clip-rule="evenodd" d="M48.854 0C21.839 0 0 22 0 49.217c0 21.756 13.993 40.172 33.405 46.69 2.427.49 3.316-1.059 3.316-2.362 0-1.141-.08-5.052-.08-9.127-13.59 2.934-16.42-5.867-16.42-5.867-2.184-5.704-5.42-7.17-5.42-7.17-4.448-3.015.324-3.015.324-3.015 4.934.326 7.523 5.052 7.523 5.052 4.367 7.496 11.404 5.378 14.235 4.074.404-3.178 1.699-5.378 3.074-6.6-10.839-1.141-22.243-5.378-22.243-24.283 0-5.378 1.94-9.778 5.014-13.2-.485-1.222-2.184-6.275.486-13.038 0 0 4.125-1.304 13.426 5.052a46.97 46.97 0 0 1 12.214-1.63c4.125 0 8.33.571 12.213 1.63 9.302-6.356 13.427-5.052 13.427-5.052 2.67 6.763.97 11.816.485 13.038 3.155 3.422 5.015 7.822 5.015 13.2 0 18.905-11.404 23.06-22.324 24.283 1.78 1.548 3.316 4.481 3.316 9.126 0 6.6-.08 11.897-.08 13.526 0 1.304.89 2.853 3.316 2.364 19.412-6.52 33.405-24.935 33.405-46.691C97.707 22 75.788 0 48.854 0z" fill="#24292f"/></svg><svg class="${e} hidden dark:block" width="98" height="96" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 98 96"><path fill-rule="evenodd" clip-rule="evenodd" d="M48.854 0C21.839 0 0 22 0 49.217c0 21.756 13.993 40.172 33.405 46.69 2.427.49 3.316-1.059 3.316-2.362 0-1.141-.08-5.052-.08-9.127-13.59 2.934-16.42-5.867-16.42-5.867-2.184-5.704-5.42-7.17-5.42-7.17-4.448-3.015.324-3.015.324-3.015 4.934.326 7.523 5.052 7.523 5.052 4.367 7.496 11.404 5.378 14.235 4.074.404-3.178 1.699-5.378 3.074-6.6-10.839-1.141-22.243-5.378-22.243-24.283 0-5.378 1.94-9.778 5.014-13.2-.485-1.222-2.184-6.275.486-13.038 0 0 4.125-1.304 13.426 5.052a46.97 46.97 0 0 1 12.214-1.63c4.125 0 8.33.571 12.213 1.63 9.302-6.356 13.427-5.052 13.427-5.052 2.67 6.763.97 11.816.485 13.038 3.155 3.422 5.015 7.822 5.015 13.2 0 18.905-11.404 23.06-22.324 24.283 1.78 1.548 3.316 4.481 3.316 9.126 0 6.6-.08 11.897-.08 13.526 0 1.304.89 2.853 3.316 2.364 19.412-6.52 33.405-24.935 33.405-46.691C97.707 22 75.788 0 48.854 0z" fill="#fff"/></svg></div>`:`<svg class="${e} text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+import{d as o}from"./CQh-7xkh.js";function l(r){if(!r)return"N/A";try{return(typeof r=="string"?new Date(r):r).toLocaleString()}catch{return"Invalid Date"}}function f(r,e="w-4 h-4"){return r==="gitea"?`<svg class="${e}" xmlns="http://www.w3.org/2000/svg" xml:space="preserve" viewBox="0 0 640 640"><path d="m395.9 484.2-126.9-61c-12.5-6-17.9-21.2-11.8-33.8l61-126.9c6-12.5 21.2-17.9 33.8-11.8 17.2 8.3 27.1 13 27.1 13l-.1-109.2 16.7-.1.1 117.1s57.4 24.2 83.1 40.1c3.7 2.3 10.2 6.8 12.9 14.4 2.1 6.1 2 13.1-1 19.3l-61 126.9c-6.2 12.7-21.4 18.1-33.9 12" style="fill:#fff"/><path d="M622.7 149.8c-4.1-4.1-9.6-4-9.6-4s-117.2 6.6-177.9 8c-13.3.3-26.5.6-39.6.7v117.2c-5.5-2.6-11.1-5.3-16.6-7.9 0-36.4-.1-109.2-.1-109.2-29 .4-89.2-2.2-89.2-2.2s-141.4-7.1-156.8-8.5c-9.8-.6-22.5-2.1-39 1.5-8.7 1.8-33.5 7.4-53.8 26.9C-4.9 212.4 6.6 276.2 8 285.8c1.7 11.7 6.9 44.2 31.7 72.5 45.8 56.1 144.4 54.8 144.4 54.8s12.1 28.9 30.6 55.5c25 33.1 50.7 58.9 75.7 62 63 0 188.9-.1 188.9-.1s12 .1 28.3-10.3c14-8.5 26.5-23.4 26.5-23.4S547 483 565 451.5c5.5-9.7 10.1-19.1 14.1-28 0 0 55.2-117.1 55.2-231.1-1.1-34.5-9.6-40.6-11.6-42.6M125.6 353.9c-25.9-8.5-36.9-18.7-36.9-18.7S69.6 321.8 60 295.4c-16.5-44.2-1.4-71.2-1.4-71.2s8.4-22.5 38.5-30c13.8-3.7 31-3.1 31-3.1s7.1 59.4 15.7 94.2c7.2 29.2 24.8 77.7 24.8 77.7s-26.1-3.1-43-9.1m300.3 107.6s-6.1 14.5-19.6 15.4c-5.8.4-10.3-1.2-10.3-1.2s-.3-.1-5.3-2.1l-112.9-55s-10.9-5.7-12.8-15.6c-2.2-8.1 2.7-18.1 2.7-18.1L322 273s4.8-9.7 12.2-13c.6-.3 2.3-1 4.5-1.5 8.1-2.1 18 2.8 18 2.8L467.4 315s12.6 5.7 15.3 16.2c1.9 7.4-.5 14-1.8 17.2-6.3 15.4-55 113.1-55 113.1" style="fill:#609926"/><path d="M326.8 380.1c-8.2.1-15.4 5.8-17.3 13.8s2 16.3 9.1 20c7.7 4 17.5 1.8 22.7-5.4 5.1-7.1 4.3-16.9-1.8-23.1l24-49.1c1.5.1 3.7.2 6.2-.5 4.1-.9 7.1-3.6 7.1-3.6 4.2 1.8 8.6 3.8 13.2 6.1 4.8 2.4 9.3 4.9 13.4 7.3.9.5 1.8 1.1 2.8 1.9 1.6 1.3 3.4 3.1 4.7 5.5 1.9 5.5-1.9 14.9-1.9 14.9-2.3 7.6-18.4 40.6-18.4 40.6-8.1-.2-15.3 5-17.7 12.5-2.6 8.1 1.1 17.3 8.9 21.3s17.4 1.7 22.5-5.3c5-6.8 4.6-16.3-1.1-22.6 1.9-3.7 3.7-7.4 5.6-11.3 5-10.4 13.5-30.4 13.5-30.4.9-1.7 5.7-10.3 2.7-21.3-2.5-11.4-12.6-16.7-12.6-16.7-12.2-7.9-29.2-15.2-29.2-15.2s0-4.1-1.1-7.1c-1.1-3.1-2.8-5.1-3.9-6.3 4.7-9.7 9.4-19.3 14.1-29-4.1-2-8.1-4-12.2-6.1-4.8 9.8-9.7 19.7-14.5 29.5-6.7-.1-12.9 3.5-16.1 9.4-3.4 6.3-2.7 14.1 1.9 19.8z" style="fill:#609926"/></svg>`:r==="github"?`<div class="inline-flex ${e}"><svg class="${e} dark:hidden" width="98" height="96" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 98 96"><path fill-rule="evenodd" clip-rule="evenodd" d="M48.854 0C21.839 0 0 22 0 49.217c0 21.756 13.993 40.172 33.405 46.69 2.427.49 3.316-1.059 3.316-2.362 0-1.141-.08-5.052-.08-9.127-13.59 2.934-16.42-5.867-16.42-5.867-2.184-5.704-5.42-7.17-5.42-7.17-4.448-3.015.324-3.015.324-3.015 4.934.326 7.523 5.052 7.523 5.052 4.367 7.496 11.404 5.378 14.235 4.074.404-3.178 1.699-5.378 3.074-6.6-10.839-1.141-22.243-5.378-22.243-24.283 0-5.378 1.94-9.778 5.014-13.2-.485-1.222-2.184-6.275.486-13.038 0 0 4.125-1.304 13.426 5.052a46.97 46.97 0 0 1 12.214-1.63c4.125 0 8.33.571 12.213 1.63 9.302-6.356 13.427-5.052 13.427-5.052 2.67 6.763.97 11.816.485 13.038 3.155 3.422 5.015 7.822 5.015 13.2 0 18.905-11.404 23.06-22.324 24.283 1.78 1.548 3.316 4.481 3.316 9.126 0 6.6-.08 11.897-.08 13.526 0 1.304.89 2.853 3.316 2.364 19.412-6.52 33.405-24.935 33.405-46.691C97.707 22 75.788 0 48.854 0z" fill="#24292f"/></svg><svg class="${e} hidden dark:block" width="98" height="96" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 98 96"><path fill-rule="evenodd" clip-rule="evenodd" d="M48.854 0C21.839 0 0 22 0 49.217c0 21.756 13.993 40.172 33.405 46.69 2.427.49 3.316-1.059 3.316-2.362 0-1.141-.08-5.052-.08-9.127-13.59 2.934-16.42-5.867-16.42-5.867-2.184-5.704-5.42-7.17-5.42-7.17-4.448-3.015.324-3.015.324-3.015 4.934.326 7.523 5.052 7.523 5.052 4.367 7.496 11.404 5.378 14.235 4.074.404-3.178 1.699-5.378 3.074-6.6-10.839-1.141-22.243-5.378-22.243-24.283 0-5.378 1.94-9.778 5.014-13.2-.485-1.222-2.184-6.275.486-13.038 0 0 4.125-1.304 13.426 5.052a46.97 46.97 0 0 1 12.214-1.63c4.125 0 8.33.571 12.213 1.63 9.302-6.356 13.427-5.052 13.427-5.052 2.67 6.763.97 11.816.485 13.038 3.155 3.422 5.015 7.822 5.015 13.2 0 18.905-11.404 23.06-22.324 24.283 1.78 1.548 3.316 4.481 3.316 9.126 0 6.6-.08 11.897-.08 13.526 0 1.304.89 2.853 3.316 2.364 19.412-6.52 33.405-24.935 33.405-46.691C97.707 22 75.788 0 48.854 0z" fill="#fff"/></svg></div>`:`<svg class="${e} text-gray-400" fill="none" stroke="currentColor" viewBox="0 0 24 24">
 			<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M10.325 4.317c.426-1.756 2.924-1.756 3.35 0a1.724 1.724 0 002.573 1.066c1.543-.94 3.31.826 2.37 2.37a1.724 1.724 0 001.065 2.572c1.756.426 1.756 2.924 0 3.35a1.724 1.724 0 00-1.066 2.573c.94 1.543-.826 3.31-2.37 2.37a1.724 1.724 0 00-2.572 1.065c-.426 1.756-2.924 1.756-3.35 0a1.724 1.724 0 00-2.573-1.066c-1.543.94-3.31-.826-2.37-2.37a1.724 1.724 0 00-1.065-2.572c-1.756-.426-1.756-2.924 0-3.35a1.724 1.724 0 001.066-2.573c-.94-1.543.826-3.31 2.37-2.37.996.608 2.296.07 2.572-1.065z" />
 			<path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 12a3 3 0 11-6 0 3 3 0 016 0z" />
 		</svg>`}function d(r,e){if(r.repo_name)return r.repo_name;if(r.org_name)return r.org_name;if(r.enterprise_name)return r.enterprise_name;if(r.repo_id&&!r.repo_name&&e?.repositories){const n=e.repositories.find(t=>t.id===r.repo_id);return n?`${n.owner}/${n.name}`:"Unknown Entity"}if(r.org_id&&!r.org_name&&e?.organizations){const n=e.organizations.find(t=>t.id===r.org_id);return n&&n.name?n.name:"Unknown Entity"}if(r.enterprise_id&&!r.enterprise_name&&e?.enterprises){const n=e.enterprises.find(t=>t.id===r.enterprise_id);return n&&n.name?n.name:"Unknown Entity"}return"Unknown Entity"}function p(r){return r.repo_id?"repository":r.org_id?"organization":r.enterprise_id?"enterprise":"unknown"}function g(r){return r.repo_id?o(`/repositories/${r.repo_id}`):r.org_id?o(`/organizations/${r.org_id}`):r.enterprise_id?o(`/enterprises/${r.enterprise_id}`):"#"}function w(r){r&&(r.scrollTop=r.scrollHeight)}function m(r){return{newPerPage:r,newCurrentPage:1}}function v(r){return r.pool_manager_status?.running?{text:"Running",variant:"success"}:{text:"Stopped",variant:"error"}}function _(r){switch(r.toLowerCase()){case"error":return{text:"Error",variant:"error"};case"warning":return{text:"Warning",variant:"warning"};case"info":return{text:"Info",variant:"info"};default:return{text:r,variant:"info"}}}function i(r,e,n){if(!e.trim())return r;const t=e.toLowerCase();return r.filter(s=>typeof n=="function"?n(s).toLowerCase().includes(t):n.some(a=>s[a]?.toString().toLowerCase().includes(t)))}function h(r,e){return i(r,e,["name","owner"])}function x(r,e){return i(r,e,["name"])}function k(r,e){return i(r,e,n=>[n.name||"",n.description||"",n.endpoint?.name||""].join(" "))}function E(r,e){return i(r,e,["name","description","base_url","api_base_url"])}function L(r,e,n){return r.slice((e-1)*n,e*n)}export{E as a,l as b,m as c,_ as d,d as e,k as f,f as g,i as h,p as i,g as j,v as k,x as l,h as m,L as p,w as s};

webapp/assets/_app/immutable/chunks/BFX8vI-4.js

@@ -1 +1 @@
-import"./DsnmJJEf.js";import{i as u}from"./TJn6xDN9.js";import{p as v,E as m,f as h,d as r,r as d,i as t,b as k,c as g}from"./DniTuB_0.js";import{f as b}from"./DD3srElq.js";var w=h('<div class="fixed inset-0 bg-black/30 dark:bg-black/50 overflow-y-auto h-full w-full z-50 flex items-center justify-center p-4" role="dialog" aria-modal="true" tabindex="-1"><div class="relative mx-auto bg-white dark:bg-gray-800 rounded-lg shadow-lg" role="document"><!></div></div>');function M(s,i){v(i,!1);const l=m();function n(){l("close")}function c(o){o.stopPropagation()}function f(o){o.key==="Escape"&&l("close")}u();var a=w(),e=r(a),p=r(e);b(p,i,"default",{}),d(e),d(a),t("click",e,c),t("click",a,n),t("keydown",a,f),k(s,a),g()}export{M};
+import"./DsnmJJEf.js";import{i as u}from"./De-I_rkH.js";import{p as v,E as m,f as h,d as r,r as d,i as t,b as k,c as g}from"./BIqNNOMq.js";import{f as b}from"./CQh-7xkh.js";var w=h('<div class="fixed inset-0 bg-black/30 dark:bg-black/50 overflow-y-auto h-full w-full z-50 flex items-center justify-center p-4" role="dialog" aria-modal="true" tabindex="-1"><div class="relative mx-auto bg-white dark:bg-gray-800 rounded-lg shadow-lg" role="document"><!></div></div>');function M(s,i){v(i,!1);const l=m();function n(){l("close")}function c(o){o.stopPropagation()}function f(o){o.key==="Escape"&&l("close")}u();var a=w(),e=r(a),p=r(e);b(p,i,"default",{}),d(e),d(a),t("click",e,c),t("click",a,n),t("keydown",a,f),k(s,a),g()}export{M};

webapp/assets/_app/immutable/chunks/BIfA0Pjo.js

@@ -1 +1 @@
-import{s as e}from"./DeKGyprh.js";const r=()=>{const s=e;return{page:{subscribe:s.page.subscribe},navigating:{subscribe:s.navigating.subscribe},updated:s.updated}},b={subscribe(s){return r().page.subscribe(s)}};export{b as p};
+import{s as e}from"./BSYpqPvJ.js";const r=()=>{const s=e;return{page:{subscribe:s.page.subscribe},navigating:{subscribe:s.navigating.subscribe},updated:s.updated}},b={subscribe(s){return r().page.subscribe(s)}};export{b as p};