DevFW-CICD/garm
15
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

Merge pull request #143 from gabriel-samfira/move-code-to-external-package

Browse source 
Move code to external package
This commit is contained in:
Gabriel 2023-07-24 15:44:49 +03:00 • committed by GitHub
commit 851a9bd0ae
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
81 changed files with 1194 additions and 909 deletions
Download patch file Download diff file Expand all files Collapse all files

4
apiserver/controllers/controllers.go
View file

@ -21,13 +21,13 @@ import (
"net/http"
"strings"
gErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/apiserver/params"
"github.com/cloudbase/garm/auth"
gErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/metrics"
runnerParams "github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner"
"github.com/cloudbase/garm/util"
wsWriter "github.com/cloudbase/garm/websocket"
"github.com/gorilla/websocket"

2
apiserver/controllers/enterprises.go
View file

@ -19,8 +19,8 @@ import (
"log"
"net/http"
gErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/apiserver/params"
gErrors "github.com/cloudbase/garm/errors"
runnerParams "github.com/cloudbase/garm/params"
"github.com/gorilla/mux"

2
apiserver/controllers/instances.go
View file

@ -19,8 +19,8 @@ import (
"log"
"net/http"
gErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/apiserver/params"
gErrors "github.com/cloudbase/garm/errors"
runnerParams "github.com/cloudbase/garm/params"
"github.com/gorilla/mux"

2
apiserver/controllers/organizations.go
View file

@ -19,8 +19,8 @@ import (
"log"
"net/http"
gErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/apiserver/params"
gErrors "github.com/cloudbase/garm/errors"
runnerParams "github.com/cloudbase/garm/params"
"github.com/gorilla/mux"

2
apiserver/controllers/pools.go
View file

@ -19,8 +19,8 @@ import (
"log"
"net/http"
gErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/apiserver/params"
gErrors "github.com/cloudbase/garm/errors"
runnerParams "github.com/cloudbase/garm/params"
"github.com/gorilla/mux"

2
apiserver/controllers/repositories.go
View file

@ -19,8 +19,8 @@ import (
"log"
"net/http"
gErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/apiserver/params"
gErrors "github.com/cloudbase/garm/errors"
runnerParams "github.com/cloudbase/garm/params"
"github.com/gorilla/mux"

2
apiserver/routers/routers.go
View file

@ -54,9 +54,9 @@ import (
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/apiserver/controllers"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/util"
)
func WithMetricsRouter(parentRouter *mux.Router, disableAuth bool, metricsMiddlerware auth.Middleware) *mux.Router {

6
apiserver/swagger.yaml
View file

@ -337,13 +337,13 @@ paths:
name: enterpriseID
required: true
type: string
- description: Parameters used to update the enterprise.
- description: Parameters used when updating the enterprise.
in: body
name: Body
required: true
schema:
$ref: '#/definitions/UpdateEntityParams'
description: Parameters used to update the enterprise.
description: Parameters used when updating the enterprise.
type: object
responses:
"200":
@ -354,7 +354,7 @@ paths:
description: APIErrorResponse
schema:
$ref: '#/definitions/APIErrorResponse'
summary: Update an enterprise with the given parameters.
summary: Update enterprise with the given parameters.
tags:
- enterprises
/enterprises/{enterpriseID}/instances:

4
auth/auth.go
View file

@ -18,11 +18,11 @@ import (
"context"
"time"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/golang-jwt/jwt"
"github.com/nbutton23/zxcvbn-go"

9
auth/context.go
View file

@ -18,7 +18,6 @@ import (
"context"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/providers/common"
)
type contextFlags string
@ -65,16 +64,16 @@ func InstanceTokenFetched(ctx context.Context) bool {
return elem.(bool)
}
func SetInstanceRunnerStatus(ctx context.Context, val common.RunnerStatus) context.Context {
func SetInstanceRunnerStatus(ctx context.Context, val params.RunnerStatus) context.Context {
return context.WithValue(ctx, instanceRunnerStatus, val)
}
func InstanceRunnerStatus(ctx context.Context) common.RunnerStatus {
func InstanceRunnerStatus(ctx context.Context) params.RunnerStatus {
elem := ctx.Value(instanceRunnerStatus)
if elem == nil {
return common.RunnerPending
return params.RunnerPending
}
return elem.(common.RunnerStatus)
return elem.(params.RunnerStatus)
}
func SetInstanceName(ctx context.Context, val string) context.Context {

5
auth/instance_middleware.go
View file

@ -21,12 +21,11 @@ import (
"strings"
"time"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/config"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
providerCommon "github.com/cloudbase/garm/runner/providers/common"
"github.com/golang-jwt/jwt"
"github.com/pkg/errors"
@ -149,7 +148,7 @@ func (amw *instanceMiddleware) Middleware(next http.Handler) http.Handler {
}
runnerStatus := InstanceRunnerStatus(ctx)
if runnerStatus != providerCommon.RunnerInstalling && runnerStatus != providerCommon.RunnerPending {
if runnerStatus != params.RunnerInstalling && runnerStatus != params.RunnerPending {
// Instances that have finished installing can no longer authenticate to the API
invalidAuthResponse(w)
return

2
auth/jwt.go
View file

@ -22,10 +22,10 @@ import (
"net/http"
"strings"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
apiParams "github.com/cloudbase/garm/apiserver/params"
"github.com/cloudbase/garm/config"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/golang-jwt/jwt"
)

2
client/enterprises/enterprises_client.go
View file

@ -384,7 +384,7 @@ func (a *Client) ListEnterprises(params *ListEnterprisesParams, authInfo runtime
}
/*
UpdateEnterprise updates an enterprise with the given parameters
UpdateEnterprise updates enterprise with the given parameters
*/
func (a *Client) UpdateEnterprise(params *UpdateEnterpriseParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*UpdateEnterpriseOK, error) {
// TODO: Validate the params before sending

2
client/enterprises/update_enterprise_parameters.go
View file

@ -65,7 +65,7 @@ type UpdateEnterpriseParams struct {
/* Body.
Parameters used to update the enterprise.
Parameters used when updating the enterprise.
*/
Body garm_params.UpdateEntityParams

2
cmd/garm-cli/cmd/log.go
View file

@ -10,8 +10,8 @@ import (
"os/signal"
"time"
"github.com/cloudbase/garm-provider-common/util"
apiParams "github.com/cloudbase/garm/apiserver/params"
"github.com/cloudbase/garm/util"
"github.com/gorilla/websocket"
"github.com/spf13/cobra"

10
cmd/garm-cli/cmd/pool.go
View file

@ -25,6 +25,8 @@ import (
"github.com/jedib0t/go-pretty/v6/table"
"github.com/pkg/errors"
"github.com/spf13/cobra"
commonParams "github.com/cloudbase/garm-provider-common/params"
)
var (
@ -192,8 +194,8 @@ var poolAddCmd = &cobra.Command{
MinIdleRunners: poolMinIdleRunners,
Image: poolImage,
Flavor: poolFlavor,
OSType: params.OSType(poolOSType),
OSArch: params.OSArch(poolOSArch),
OSType: commonParams.OSType(poolOSType),
OSArch: commonParams.OSArch(poolOSArch),
Tags: tags,
Enabled: poolEnabled,
RunnerBootstrapTimeout: poolRunnerBootstrapTimeout,
@ -280,11 +282,11 @@ explicitly remove them using the runner delete command.
}
if cmd.Flags().Changed("os-type") {
poolUpdateParams.OSType = params.OSType(poolOSType)
poolUpdateParams.OSType = commonParams.OSType(poolOSType)
}
if cmd.Flags().Changed("os-arch") {
poolUpdateParams.OSArch = params.OSArch(poolOSArch)
poolUpdateParams.OSArch = commonParams.OSArch(poolOSArch)
}
if cmd.Flags().Changed("max-runners") {

2
cmd/garm-cli/config/config.go
View file

@ -23,7 +23,7 @@ import (
"github.com/BurntSushi/toml"
"github.com/pkg/errors"
runnerErrors "github.com/cloudbase/garm/errors"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
)
const (

9
cmd/garm/main.go
View file

@ -27,6 +27,7 @@ import (
"syscall"
"time"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/apiserver/controllers"
"github.com/cloudbase/garm/apiserver/routers"
"github.com/cloudbase/garm/auth"
@ -35,7 +36,6 @@ import (
"github.com/cloudbase/garm/database/common"
"github.com/cloudbase/garm/metrics"
"github.com/cloudbase/garm/runner"
"github.com/cloudbase/garm/util"
"github.com/cloudbase/garm/util/appdefaults"
"github.com/cloudbase/garm/websocket"
lumberjack "gopkg.in/natefinch/lumberjack.v2"
@ -52,6 +52,11 @@ var (
var Version string
var signals = []os.Signal{
os.Interrupt,
syscall.SIGTERM,
}
func maybeInitController(db common.Store) error {
if _, err := db.ControllerInfo(); err == nil {
return nil
@ -79,7 +84,7 @@ func main() {
log.Fatalf("Fetching config: %+v", err)
}
logWriter, err := util.GetLoggingWriter(cfg)
logWriter, err := util.GetLoggingWriter(cfg.Default.LogFile)
if err != nil {
log.Fatalf("fetching log writer: %+v", err)
}

14
cmd/garm/signal_nix.go
View file

@ -1,14 +0,0 @@
//go:build !windows
// +build !windows
package main
import (
"os"
"syscall"
)
var signals = []os.Signal{
os.Interrupt,
syscall.SIGTERM,
}

10
cmd/garm/signal_windows.go
View file

@ -1,10 +0,0 @@
//go:build windows && !linux
// +build windows,!linux
package main
import "os"
var signals = []os.Signal{
os.Interrupt,
}

2
config/external.go
View file

@ -19,7 +19,7 @@ import (
"os"
"path/filepath"
"github.com/cloudbase/garm/util/exec"
"github.com/cloudbase/garm-provider-common/util/exec"
"github.com/pkg/errors"
)

2
database/sql/controller.go
View file

@ -15,7 +15,7 @@
package sql
import (
runnerErrors "github.com/cloudbase/garm/errors"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/params"
"github.com/google/uuid"

2
database/sql/controller_test.go
View file

@ -19,8 +19,8 @@ import (
"fmt"
"testing"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
garmTesting "github.com/cloudbase/garm/internal/testing"
"github.com/stretchr/testify/suite"

4
database/sql/enterprise.go
View file

@ -3,9 +3,9 @@ package sql
import (
"context"
runnerErrors "github.com/cloudbase/garm/errors"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/google/uuid"
"github.com/pkg/errors"

2
database/sql/enterprise_test.go
View file

@ -24,8 +24,8 @@ import (
"github.com/cloudbase/garm/params"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
garmTesting "github.com/cloudbase/garm/internal/testing"
"github.com/stretchr/testify/suite"

2
database/sql/instances.go
View file

@ -18,7 +18,7 @@ import (
"context"
"encoding/json"
runnerErrors "github.com/cloudbase/garm/errors"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/params"
"github.com/google/uuid"

17
database/sql/instances_test.go
View file

@ -22,10 +22,11 @@ import (
"sort"
"testing"
commonParams "github.com/cloudbase/garm-provider-common/params"
dbCommon "github.com/cloudbase/garm/database/common"
garmTesting "github.com/cloudbase/garm/internal/testing"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/providers/common"
"gopkg.in/DATA-DOG/go-sqlmock.v1"
@ -109,8 +110,8 @@ func (s *InstancesTestSuite) SetupTest() {
OSType: "linux",
OSArch: "amd64",
CallbackURL: "https://garm.example.com/",
Status: common.InstanceRunning,
RunnerStatus: common.RunnerIdle,
Status: commonParams.InstanceRunning,
RunnerStatus: params.RunnerIdle,
},
)
if err != nil {
@ -156,18 +157,18 @@ func (s *InstancesTestSuite) SetupTest() {
ProviderID: "update-provider-test",
OSName: "ubuntu",
OSVersion: "focal",
Status: common.InstancePendingDelete,
RunnerStatus: common.RunnerActive,
Status: commonParams.InstancePendingDelete,
RunnerStatus: params.RunnerActive,
AgentID: 4,
CreateAttempt: 3,
Addresses: []params.Address{
Addresses: []commonParams.Address{
{
Address: "12.10.12.10",
Type: params.PublicAddress,
Type: commonParams.PublicAddress,
},
{
Address: "10.1.1.2",
Type: params.PrivateAddress,
Type: commonParams.PrivateAddress,
},
},
},

2
database/sql/jobs.go
View file

@ -4,8 +4,8 @@ import (
"context"
"encoding/json"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/google/uuid"
"github.com/pkg/errors"

14
database/sql/models.go
View file

@ -17,8 +17,8 @@ package sql
import (
"time"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/providers/common"
"github.com/google/uuid"
"github.com/pkg/errors"
@ -63,8 +63,8 @@ type Pool struct {
RunnerBootstrapTimeout uint
Image string `gorm:"index:idx_pool_type"`
Flavor string `gorm:"index:idx_pool_type"`
OSType params.OSType
OSArch params.OSArch
OSType commonParams.OSType
OSArch commonParams.OSArch
Tags []*Tag `gorm:"many2many:pool_tags;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"`
Enabled bool
// ExtraSpecs is an opaque json that gets sent to the provider
@ -143,13 +143,13 @@ type Instance struct {
ProviderID *string `gorm:"uniqueIndex"`
Name string `gorm:"uniqueIndex"`
AgentID int64
OSType params.OSType
OSArch params.OSArch
OSType commonParams.OSType
OSArch commonParams.OSArch
OSName string
OSVersion string
Addresses []Address `gorm:"foreignKey:InstanceID;constraint:OnDelete:CASCADE,OnUpdate:CASCADE;"`
Status common.InstanceStatus
RunnerStatus common.RunnerStatus
Status commonParams.InstanceStatus
RunnerStatus params.RunnerStatus
CallbackURL string
MetadataURL string
ProviderFault []byte `gorm:"type:longblob"`

4
database/sql/organizations.go
View file

@ -18,9 +18,9 @@ import (
"context"
"fmt"
runnerErrors "github.com/cloudbase/garm/errors"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/google/uuid"
"github.com/pkg/errors"

2
database/sql/organizations_test.go
View file

@ -22,8 +22,8 @@ import (
"sort"
"testing"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
garmTesting "github.com/cloudbase/garm/internal/testing"
"github.com/cloudbase/garm/params"

2
database/sql/pools.go
View file

@ -18,7 +18,7 @@ import (
"context"
"fmt"
runnerErrors "github.com/cloudbase/garm/errors"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/params"
"github.com/google/uuid"

4
database/sql/repositories.go
View file

@ -18,9 +18,9 @@ import (
"context"
"fmt"
runnerErrors "github.com/cloudbase/garm/errors"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/google/uuid"
"github.com/pkg/errors"

4
database/sql/users.go
View file

@ -18,9 +18,9 @@ import (
"context"
"fmt"
runnerErrors "github.com/cloudbase/garm/errors"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/pkg/errors"
"gorm.io/gorm"

10
database/sql/util.go
View file

@ -18,12 +18,14 @@ import (
"encoding/json"
"fmt"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/pkg/errors"
"gorm.io/datatypes"
"gorm.io/gorm"
commonParams "github.com/cloudbase/garm-provider-common/params"
)
func (s *sqlDatabase) sqlToParamsInstance(instance Instance) params.Instance {
@ -75,10 +77,10 @@ func (s *sqlDatabase) sqlToParamsInstance(instance Instance) params.Instance {
return ret
}
func (s *sqlDatabase) sqlAddressToParamsAddress(addr Address) params.Address {
return params.Address{
func (s *sqlDatabase) sqlAddressToParamsAddress(addr Address) commonParams.Address {
return commonParams.Address{
Address: addr.Address,
Type: params.AddressType(addr.Type),
Type: commonParams.AddressType(addr.Type),
}
}

8
doc/extra_specs.md Normal file
View file

@ -0,0 +1,8 @@
# ExtraSpecs
ExtraSpecs is an opaque raw json that gets sent to the provider as part of the bootstrap params for instances. It can contain any kind of data needed by providers. The contents of this field means nothing to garm itself. We don't act on the information in this field at all. We only validate that it's a proper json.
However, during the installation phase of the runners, GARM providers can leverage the information set in this field to augment the process in many ways. This can be used for anything rangin from overriding provider config values, to supplying a different runner install template, to passing in information that is relevant only to specific providers.
For example, the [external OpenStack provider](https://github.com/cloudbase/garm-provider-openstack) uses this to [override](https://github.com/cloudbase/garm-provider-openstack#tweaking-the-provider) things like `security groups`, `storage backends`, `network ids`, etc.

9
go.mod
View file

@ -4,6 +4,7 @@ go 1.20
require (
github.com/BurntSushi/toml v1.2.1
github.com/cloudbase/garm-provider-common v0.0.0-20230724114054-7aa0a3dfbce0
github.com/go-openapi/errors v0.20.4
github.com/go-openapi/runtime v0.26.0
github.com/go-openapi/strfmt v0.21.7
@ -19,20 +20,16 @@ require (
github.com/juju/retry v1.0.0
github.com/lxc/lxd v0.0.0-20230325180147-8d608287b0ce
github.com/manifoldco/promptui v0.9.0
github.com/mattn/go-isatty v0.0.18
github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.14.0
github.com/spf13/cobra v1.6.1
github.com/stretchr/testify v1.8.2
github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569
golang.org/x/crypto v0.7.0
golang.org/x/oauth2 v0.8.0
golang.org/x/sync v0.1.0
golang.org/x/sys v0.8.0
gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/yaml.v3 v3.0.1
gorm.io/datatypes v1.1.1
gorm.io/driver/mysql v1.4.7
gorm.io/driver/sqlite v1.4.4
@ -76,6 +73,7 @@ require (
github.com/kr/fs v0.1.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-sqlite3 v1.14.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
@ -95,10 +93,12 @@ require (
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 // indirect
go.mongodb.org/mongo-driver v1.11.3 // indirect
go.opentelemetry.io/otel v1.14.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect
golang.org/x/net v0.10.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/term v0.8.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.30.0 // indirect
@ -106,4 +106,5 @@ require (
gopkg.in/httprequest.v1 v1.2.1 // indirect
gopkg.in/macaroon.v2 v2.1.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

6
go.sum
View file

@ -25,6 +25,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04=
github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cloudbase/garm-provider-common v0.0.0-20230724114054-7aa0a3dfbce0 h1:5ScMXea/ZIcUbw1aXAgN8xTqSG84AOf5Maf5hBC82wQ=
github.com/cloudbase/garm-provider-common v0.0.0-20230724114054-7aa0a3dfbce0/go.mod h1:RKzgL0MXkNeGfloQpE2swz/y4LWJr5+2Wd45bSXPB0k=
github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I=
github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs=
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
@ -231,8 +233,8 @@ github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYt
github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE=
github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=

83
params/params.go
View file

@ -18,7 +18,8 @@ import (
"encoding/json"
"time"
"github.com/cloudbase/garm/runner/providers/common"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/util/appdefaults"
"github.com/google/go-github/v53/github"
@ -27,13 +28,11 @@ import (
type (
PoolType string
AddressType string
EventType string
EventLevel string
OSType string
OSArch string
ProviderType string
JobStatus string
RunnerStatus string
)
const (
@ -55,11 +54,6 @@ const (
EnterprisePool PoolType = "enterprise"
)
const (
PublicAddress AddressType = "public"
PrivateAddress AddressType = "private"
)
const (
StatusEvent EventType = "status"
FetchTokenEvent EventType = "fetchToken"
@ -72,23 +66,14 @@ const (
)
const (
Windows OSType = "windows"
Linux OSType = "linux"
Unknown OSType = "unknown"
RunnerIdle RunnerStatus = "idle"
RunnerPending RunnerStatus = "pending"
RunnerTerminated RunnerStatus = "terminated"
RunnerInstalling RunnerStatus = "installing"
RunnerFailed RunnerStatus = "failed"
RunnerActive RunnerStatus = "active"
)
const (
Amd64 OSArch = "amd64"
I386 OSArch = "i386"
Arm64 OSArch = "arm64"
Arm OSArch = "arm"
)
type Address struct {
Address string `json:"address"`
Type AddressType `json:"type"`
}
type StatusMessage struct {
CreatedAt time.Time `json:"created_at"`
Message string `json:"message"`
@ -116,7 +101,7 @@ type Instance struct {
// OSType is the operating system type. For now, only Linux and
// Windows are supported.
OSType OSType `json:"os_type,omitempty"`
OSType commonParams.OSType `json:"os_type,omitempty"`
// OSName is the name of the OS. Eg: ubuntu, centos, etc.
OSName string `json:"os_name,omitempty"`
@ -125,17 +110,17 @@ type Instance struct {
OSVersion string `json:"os_version,omitempty"`
// OSArch is the operating system architecture.
OSArch OSArch `json:"os_arch,omitempty"`
OSArch commonParams.OSArch `json:"os_arch,omitempty"`
// Addresses is a list of IP addresses the provider reports
// for this instance.
Addresses []Address `json:"addresses,omitempty"`
Addresses []commonParams.Address `json:"addresses,omitempty"`
// Status is the status of the instance inside the provider (eg: running, stopped, etc)
Status common.InstanceStatus `json:"status,omitempty"`
Status commonParams.InstanceStatus `json:"status,omitempty"`
// RunnerStatus is the github runner status as it appears on GitHub.
RunnerStatus common.RunnerStatus `json:"runner_status,omitempty"`
RunnerStatus RunnerStatus `json:"runner_status,omitempty"`
// PoolID is the ID of the garm pool to which a runner belongs.
PoolID string `json:"pool_id,omitempty"`
@ -208,10 +193,10 @@ type BootstrapInstance struct {
CACertBundle []byte `json:"ca-cert-bundle"`
// OSArch is the target OS CPU architecture of the runner.
OSArch OSArch `json:"arch"`
OSArch commonParams.OSArch `json:"arch"`
// OSType is the target OS platform of the runner (windows, linux).
OSType OSType `json:"os_type"`
OSType commonParams.OSType `json:"os_type"`
// Flavor is the platform specific abstraction that defines what resources will be allocated
// to the runner (CPU, RAM, disk space, etc). This field is meaningful to the provider which
@ -245,24 +230,24 @@ type Tag struct {
type Pool struct {
RunnerPrefix
ID string `json:"id"`
ProviderName string `json:"provider_name"`
MaxRunners uint `json:"max_runners"`
MinIdleRunners uint `json:"min_idle_runners"`
Image string `json:"image"`
Flavor string `json:"flavor"`
OSType OSType `json:"os_type"`
OSArch OSArch `json:"os_arch"`
Tags []Tag `json:"tags"`
Enabled bool `json:"enabled"`
Instances []Instance `json:"instances"`
RepoID string `json:"repo_id,omitempty"`
RepoName string `json:"repo_name,omitempty"`
OrgID string `json:"org_id,omitempty"`
OrgName string `json:"org_name,omitempty"`
EnterpriseID string `json:"enterprise_id,omitempty"`
EnterpriseName string `json:"enterprise_name,omitempty"`
RunnerBootstrapTimeout uint `json:"runner_bootstrap_timeout"`
ID string `json:"id"`
ProviderName string `json:"provider_name"`
MaxRunners uint `json:"max_runners"`
MinIdleRunners uint `json:"min_idle_runners"`
Image string `json:"image"`
Flavor string `json:"flavor"`
OSType commonParams.OSType `json:"os_type"`
OSArch commonParams.OSArch `json:"os_arch"`
Tags []Tag `json:"tags"`
Enabled bool `json:"enabled"`
Instances []Instance `json:"instances"`
RepoID string `json:"repo_id,omitempty"`
RepoName string `json:"repo_name,omitempty"`
OrgID string `json:"org_id,omitempty"`
OrgName string `json:"org_name,omitempty"`
EnterpriseID string `json:"enterprise_id,omitempty"`
EnterpriseName string `json:"enterprise_name,omitempty"`
RunnerBootstrapTimeout uint `json:"runner_bootstrap_timeout"`
// ExtraSpecs is an opaque raw json that gets sent to the provider
// as part of the bootstrap params for instances. It can contain
// any kind of data needed by providers. The contents of this field means

81
params/requests.go
View file

@ -18,16 +18,17 @@ import (
"encoding/json"
"fmt"
"github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/runner/providers/common"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm-provider-common/errors"
)
const DefaultRunnerPrefix = "garm"
type InstanceRequest struct {
Name string `json:"name"`
OSType OSType `json:"os_type"`
OSVersion string `json:"os_version"`
Name string `json:"name"`
OSType commonParams.OSType `json:"os_type"`
OSVersion string `json:"os_version"`
}
type CreateRepoParams struct {
@ -108,16 +109,16 @@ type NewUserParams struct {
type UpdatePoolParams struct {
RunnerPrefix
Tags []string `json:"tags,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
MaxRunners *uint `json:"max_runners,omitempty"`
MinIdleRunners *uint `json:"min_idle_runners,omitempty"`
RunnerBootstrapTimeout *uint `json:"runner_bootstrap_timeout,omitempty"`
Image string `json:"image"`
Flavor string `json:"flavor"`
OSType OSType `json:"os_type"`
OSArch OSArch `json:"os_arch"`
ExtraSpecs json.RawMessage `json:"extra_specs,omitempty"`
Tags []string `json:"tags,omitempty"`
Enabled *bool `json:"enabled,omitempty"`
MaxRunners *uint `json:"max_runners,omitempty"`
MinIdleRunners *uint `json:"min_idle_runners,omitempty"`
RunnerBootstrapTimeout *uint `json:"runner_bootstrap_timeout,omitempty"`
Image string `json:"image"`
Flavor string `json:"flavor"`
OSType commonParams.OSType `json:"os_type"`
OSArch commonParams.OSArch `json:"os_arch"`
ExtraSpecs json.RawMessage `json:"extra_specs,omitempty"`
// GithubRunnerGroup is the github runner group in which the runners of this
// pool will be added to.
// The runner group must be created by someone with access to the enterprise.
@ -126,10 +127,10 @@ type UpdatePoolParams struct {
type CreateInstanceParams struct {
Name string
OSType OSType
OSArch OSArch
Status common.InstanceStatus
RunnerStatus common.RunnerStatus
OSType commonParams.OSType
OSArch commonParams.OSArch
Status commonParams.InstanceStatus
RunnerStatus RunnerStatus
CallbackURL string
MetadataURL string
// GithubRunnerGroup is the github runner group to which the runner belongs.
@ -142,17 +143,17 @@ type CreateInstanceParams struct {
type CreatePoolParams struct {
RunnerPrefix
ProviderName string `json:"provider_name"`
MaxRunners uint `json:"max_runners"`
MinIdleRunners uint `json:"min_idle_runners"`
Image string `json:"image"`
Flavor string `json:"flavor"`
OSType OSType `json:"os_type"`
OSArch OSArch `json:"os_arch"`
Tags []string `json:"tags"`
Enabled bool `json:"enabled"`
RunnerBootstrapTimeout uint `json:"runner_bootstrap_timeout"`
ExtraSpecs json.RawMessage `json:"extra_specs,omitempty"`
ProviderName string `json:"provider_name"`
MaxRunners uint `json:"max_runners"`
MinIdleRunners uint `json:"min_idle_runners"`
Image string `json:"image"`
Flavor string `json:"flavor"`
OSType commonParams.OSType `json:"os_type"`
OSArch commonParams.OSArch `json:"os_arch"`
Tags []string `json:"tags"`
Enabled bool `json:"enabled"`
RunnerBootstrapTimeout uint `json:"runner_bootstrap_timeout"`
ExtraSpecs json.RawMessage `json:"extra_specs,omitempty"`
// GithubRunnerGroup is the github runner group in which the runners of this
// pool will be added to.
// The runner group must be created by someone with access to the enterprise.
@ -195,14 +196,14 @@ type UpdateInstanceParams struct {
OSVersion string `json:"os_version,omitempty"`
// Addresses is a list of IP addresses the provider reports
// for this instance.
Addresses []Address `json:"addresses,omitempty"`
Addresses []commonParams.Address `json:"addresses,omitempty"`
// Status is the status of the instance inside the provider (eg: running, stopped, etc)
Status common.InstanceStatus `json:"status,omitempty"`
RunnerStatus common.RunnerStatus `json:"runner_status,omitempty"`
ProviderFault []byte `json:"provider_fault,omitempty"`
AgentID int64 `json:"-"`
CreateAttempt int `json:"-"`
TokenFetched *bool `json:"-"`
Status commonParams.InstanceStatus `json:"status,omitempty"`
RunnerStatus RunnerStatus `json:"runner_status,omitempty"`
ProviderFault []byte `json:"provider_fault,omitempty"`
AgentID int64 `json:"-"`
CreateAttempt int `json:"-"`
TokenFetched *bool `json:"-"`
}
type UpdateUserParams struct {
@ -233,7 +234,7 @@ type UpdateEntityParams struct {
}
type InstanceUpdateMessage struct {
Status common.RunnerStatus `json:"status"`
Message string `json:"message"`
AgentID *int64 `json:"agent_id"`
Status RunnerStatus `json:"status"`
Message string `json:"message"`
AgentID *int64 `json:"agent_id"`
}

36
runner/common/mocks/Provider.go
View file

@ -5,8 +5,10 @@ package mocks
import (
context "context"
params "github.com/cloudbase/garm/params"
garm_provider_commonparams "github.com/cloudbase/garm-provider-common/params"
mock "github.com/stretchr/testify/mock"
params "github.com/cloudbase/garm/params"
)
// Provider is an autogenerated mock type for the Provider type
@ -29,21 +31,21 @@ func (_m *Provider) AsParams() params.Provider {
}
// CreateInstance provides a mock function with given fields: ctx, bootstrapParams
func (_m *Provider) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) {
func (_m *Provider) CreateInstance(ctx context.Context, bootstrapParams garm_provider_commonparams.BootstrapInstance) (garm_provider_commonparams.ProviderInstance, error) {
ret := _m.Called(ctx, bootstrapParams)
var r0 params.Instance
var r0 garm_provider_commonparams.ProviderInstance
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, params.BootstrapInstance) (params.Instance, error)); ok {
if rf, ok := ret.Get(0).(func(context.Context, garm_provider_commonparams.BootstrapInstance) (garm_provider_commonparams.ProviderInstance, error)); ok {
return rf(ctx, bootstrapParams)
}
if rf, ok := ret.Get(0).(func(context.Context, params.BootstrapInstance) params.Instance); ok {
if rf, ok := ret.Get(0).(func(context.Context, garm_provider_commonparams.BootstrapInstance) garm_provider_commonparams.ProviderInstance); ok {
r0 = rf(ctx, bootstrapParams)
} else {
r0 = ret.Get(0).(params.Instance)
r0 = ret.Get(0).(garm_provider_commonparams.ProviderInstance)
}
if rf, ok := ret.Get(1).(func(context.Context, params.BootstrapInstance) error); ok {
if rf, ok := ret.Get(1).(func(context.Context, garm_provider_commonparams.BootstrapInstance) error); ok {
r1 = rf(ctx, bootstrapParams)
} else {
r1 = ret.Error(1)
@ -67,18 +69,18 @@ func (_m *Provider) DeleteInstance(ctx context.Context, instance string) error {
}
// GetInstance provides a mock function with given fields: ctx, instance
func (_m *Provider) GetInstance(ctx context.Context, instance string) (params.Instance, error) {
func (_m *Provider) GetInstance(ctx context.Context, instance string) (garm_provider_commonparams.ProviderInstance, error) {
ret := _m.Called(ctx, instance)
var r0 params.Instance
var r0 garm_provider_commonparams.ProviderInstance
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) (params.Instance, error)); ok {
if rf, ok := ret.Get(0).(func(context.Context, string) (garm_provider_commonparams.ProviderInstance, error)); ok {
return rf(ctx, instance)
}
if rf, ok := ret.Get(0).(func(context.Context, string) params.Instance); ok {
if rf, ok := ret.Get(0).(func(context.Context, string) garm_provider_commonparams.ProviderInstance); ok {
r0 = rf(ctx, instance)
} else {
r0 = ret.Get(0).(params.Instance)
r0 = ret.Get(0).(garm_provider_commonparams.ProviderInstance)
}
if rf, ok := ret.Get(1).(func(context.Context, string) error); ok {
@ -91,19 +93,19 @@ func (_m *Provider) GetInstance(ctx context.Context, instance string) (params.In
}
// ListInstances provides a mock function with given fields: ctx, poolID
func (_m *Provider) ListInstances(ctx context.Context, poolID string) ([]params.Instance, error) {
func (_m *Provider) ListInstances(ctx context.Context, poolID string) ([]garm_provider_commonparams.ProviderInstance, error) {
ret := _m.Called(ctx, poolID)
var r0 []params.Instance
var r0 []garm_provider_commonparams.ProviderInstance
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, string) ([]params.Instance, error)); ok {
if rf, ok := ret.Get(0).(func(context.Context, string) ([]garm_provider_commonparams.ProviderInstance, error)); ok {
return rf(ctx, poolID)
}
if rf, ok := ret.Get(0).(func(context.Context, string) []params.Instance); ok {
if rf, ok := ret.Get(0).(func(context.Context, string) []garm_provider_commonparams.ProviderInstance); ok {
r0 = rf(ctx, poolID)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]params.Instance)
r0 = ret.Get(0).([]garm_provider_commonparams.ProviderInstance)
}
}

7
runner/common/provider.go
View file

@ -17,19 +17,20 @@ package common
import (
"context"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
)
//go:generate mockery --all
type Provider interface {
// CreateInstance creates a new compute instance in the provider.
CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error)
CreateInstance(ctx context.Context, bootstrapParams commonParams.BootstrapInstance) (commonParams.ProviderInstance, error)
// Delete instance will delete the instance in a provider.
DeleteInstance(ctx context.Context, instance string) error
// GetInstance will return details about one instance.
GetInstance(ctx context.Context, instance string) (params.Instance, error)
GetInstance(ctx context.Context, instance string) (commonParams.ProviderInstance, error)
// ListInstances will list all instances for a provider.
ListInstances(ctx context.Context, poolID string) ([]params.Instance, error)
ListInstances(ctx context.Context, poolID string) ([]commonParams.ProviderInstance, error)
// RemoveAllInstances will remove all instances created by this provider.
RemoveAllInstances(ctx context.Context) error
// Stop shuts down the instance.

2
runner/enterprises.go
View file

@ -6,8 +6,8 @@ import (
"log"
"strings"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/auth"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/util/appdefaults"

2
runner/enterprises_test.go
View file

@ -19,11 +19,11 @@ import (
"fmt"
"testing"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/database"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
garmTesting "github.com/cloudbase/garm/internal/testing"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"

2
runner/organizations.go
View file

@ -20,8 +20,8 @@ import (
"log"
"strings"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/auth"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/util/appdefaults"

2
runner/organizations_test.go
View file

@ -19,11 +19,11 @@ import (
"fmt"
"testing"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/database"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
garmTesting "github.com/cloudbase/garm/internal/testing"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"

2
runner/pool/enterprise.go
View file

@ -7,8 +7,8 @@ import (
"strings"
"sync"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/util"

2
runner/pool/organization.go
View file

@ -21,8 +21,8 @@ import (
"strings"
"sync"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/util"

77
runner/pool/pool.go
View file

@ -25,13 +25,14 @@ import (
"sync"
"time"
commonParams "github.com/cloudbase/garm-provider-common/params"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/auth"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
providerCommon "github.com/cloudbase/garm/runner/providers/common"
"github.com/cloudbase/garm/util"
"github.com/google/go-github/v53/github"
"github.com/google/uuid"
@ -176,7 +177,7 @@ func (r *basePoolManager) HandleWorkflowJob(job params.WorkflowJob) error {
}
// update instance workload state.
if _, err := r.setInstanceRunnerStatus(jobParams.RunnerName, providerCommon.RunnerTerminated); err != nil {
if _, err := r.setInstanceRunnerStatus(jobParams.RunnerName, params.RunnerTerminated); err != nil {
if errors.Is(err, runnerErrors.ErrNotFound) {
return nil
}
@ -184,7 +185,7 @@ func (r *basePoolManager) HandleWorkflowJob(job params.WorkflowJob) error {
return errors.Wrap(err, "updating runner")
}
r.log("marking instance %s as pending_delete", util.SanitizeLogEntry(jobParams.RunnerName))
if _, err := r.setInstanceStatus(jobParams.RunnerName, providerCommon.InstancePendingDelete, nil); err != nil {
if _, err := r.setInstanceStatus(jobParams.RunnerName, commonParams.InstancePendingDelete, nil); err != nil {
if errors.Is(err, runnerErrors.ErrNotFound) {
return nil
}
@ -206,7 +207,7 @@ func (r *basePoolManager) HandleWorkflowJob(job params.WorkflowJob) error {
}
// update instance workload state.
instance, err := r.setInstanceRunnerStatus(jobParams.RunnerName, providerCommon.RunnerActive)
instance, err := r.setInstanceRunnerStatus(jobParams.RunnerName, params.RunnerActive)
if err != nil {
if errors.Is(err, runnerErrors.ErrNotFound) {
return nil
@ -370,9 +371,9 @@ func (r *basePoolManager) cleanupOrphanedProviderRunners(runners []*github.Runne
}
defer r.keyMux.Unlock(instance.Name, false)
switch providerCommon.InstanceStatus(instance.Status) {
case providerCommon.InstancePendingCreate,
providerCommon.InstancePendingDelete:
switch commonParams.InstanceStatus(instance.Status) {
case commonParams.InstancePendingCreate,
commonParams.InstancePendingDelete:
// this instance is in the process of being created or is awaiting deletion.
// Instances in pending_create did not get a chance to register themselves in,
// github so we let them be for now.
@ -380,7 +381,7 @@ func (r *basePoolManager) cleanupOrphanedProviderRunners(runners []*github.Runne
}
switch instance.RunnerStatus {
case providerCommon.RunnerPending, providerCommon.RunnerInstalling:
case params.RunnerPending, params.RunnerInstalling:
// runner is still installing. We give it a chance to finish.
r.log("runner %s is still installing, give it a chance to finish", instance.Name)
continue
@ -394,7 +395,7 @@ func (r *basePoolManager) cleanupOrphanedProviderRunners(runners []*github.Runne
if ok := runnerNames[instance.Name]; !ok {
// Set pending_delete on DB field. Allow consolidate() to remove it.
if _, err := r.setInstanceStatus(instance.Name, providerCommon.InstancePendingDelete, nil); err != nil {
if _, err := r.setInstanceStatus(instance.Name, commonParams.InstancePendingDelete, nil); err != nil {
r.log("failed to update runner %s status: %s", instance.Name, err)
return errors.Wrap(err, "updating runner")
}
@ -455,7 +456,7 @@ func (r *basePoolManager) reapTimedOutRunners(runners []*github.Runner) error {
// even though, technically the runner is online and fully functional. This is why we check here for
// both the runner status as reported by GitHub and the runner status as reported by the provider.
// If the runner is "offline" and marked as "failed", it should be safe to reap it.
if runner, ok := runnersByName[instance.Name]; !ok || (runner.GetStatus() == "offline" && instance.RunnerStatus == providerCommon.RunnerFailed) {
if runner, ok := runnersByName[instance.Name]; !ok || (runner.GetStatus() == "offline" && instance.RunnerStatus == params.RunnerFailed) {
r.log("reaping timed-out/failed runner %s", instance.Name)
if err := r.ForceDeleteRunner(instance); err != nil {
r.log("failed to update runner %s status: %s", instance.Name, err)
@ -466,13 +467,13 @@ func (r *basePoolManager) reapTimedOutRunners(runners []*github.Runner) error {
return nil
}
func instanceInList(instanceName string, instances []params.Instance) (params.Instance, bool) {
func instanceInList(instanceName string, instances []commonParams.ProviderInstance) (commonParams.ProviderInstance, bool) {
for _, val := range instances {
if val.Name == instanceName {
return val, true
}
}
return params.Instance{}, false
return commonParams.ProviderInstance{}, false
}
// cleanupOrphanedGithubRunners will forcefully remove any github runners that appear
@ -480,7 +481,7 @@ func instanceInList(instanceName string, instances []params.Instance) (params.In
// This may happen if someone manually deletes the instance in the provider. We need to
// first remove the instance from github, and then from our database.
func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner) error {
poolInstanceCache := map[string][]params.Instance{}
poolInstanceCache := map[string][]commonParams.ProviderInstance{}
g, ctx := errgroup.WithContext(r.ctx)
for _, runner := range runners {
if !r.isManagedRunner(labelsFromRunner(runner)) {
@ -513,8 +514,8 @@ func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner)
continue
}
switch providerCommon.InstanceStatus(dbInstance.Status) {
case providerCommon.InstancePendingDelete, providerCommon.InstanceDeleting:
switch commonParams.InstanceStatus(dbInstance.Status) {
case commonParams.InstancePendingDelete, commonParams.InstanceDeleting:
// already marked for deletion or is in the process of being deleted.
// Let consolidate take care of it.
continue
@ -531,7 +532,7 @@ func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner)
return fmt.Errorf("unknown provider %s for pool %s", pool.ProviderName, pool.ID)
}
var poolInstances []params.Instance
var poolInstances []commonParams.ProviderInstance
poolInstances, ok = poolInstanceCache[pool.ID]
if !ok {
r.log("updating instances cache for pool %s", pool.ID)
@ -578,7 +579,7 @@ func (r *basePoolManager) cleanupOrphanedGithubRunners(runners []*github.Runner)
return nil
}
if providerInstance.Status == providerCommon.InstanceRunning {
if providerInstance.Status == commonParams.InstanceRunning {
// instance is running, but github reports runner as offline. Log the event.
// This scenario may require manual intervention.
// Perhaps it just came online and github did not yet change it's status?
@ -633,7 +634,7 @@ func (r *basePoolManager) fetchInstance(runnerName string) (params.Instance, err
return runner, nil
}
func (r *basePoolManager) setInstanceRunnerStatus(runnerName string, status providerCommon.RunnerStatus) (params.Instance, error) {
func (r *basePoolManager) setInstanceRunnerStatus(runnerName string, status params.RunnerStatus) (params.Instance, error) {
updateParams := params.UpdateInstanceParams{
RunnerStatus: status,
}
@ -658,7 +659,7 @@ func (r *basePoolManager) updateInstance(runnerName string, update params.Update
return instance, nil
}
func (r *basePoolManager) setInstanceStatus(runnerName string, status providerCommon.InstanceStatus, providerFault []byte) (params.Instance, error) {
func (r *basePoolManager) setInstanceStatus(runnerName string, status commonParams.InstanceStatus, providerFault []byte) (params.Instance, error) {
updateParams := params.UpdateInstanceParams{
Status: status,
ProviderFault: providerFault,
@ -681,8 +682,8 @@ func (r *basePoolManager) AddRunner(ctx context.Context, poolID string, aditiona
createParams := params.CreateInstanceParams{
Name: name,
Status: providerCommon.InstancePendingCreate,
RunnerStatus: providerCommon.RunnerPending,
Status: commonParams.InstancePendingCreate,
RunnerStatus: params.RunnerPending,
OSArch: pool.OSArch,
OSType: pool.OSType,
CallbackURL: r.helper.GetCallbackURL(),
@ -751,7 +752,7 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error
return errors.Wrap(err, "fetching instance jwt token")
}
bootstrapArgs := params.BootstrapInstance{
bootstrapArgs := commonParams.BootstrapInstance{
Name: instance.Name,
Tools: r.tools,
RepoURL: r.helper.GithubURL(),
@ -787,7 +788,7 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error
return errors.Wrap(err, "creating instance")
}
if providerInstance.Status == providerCommon.InstanceError {
if providerInstance.Status == commonParams.InstanceError {
instanceIDToDelete = instance.ProviderID
if instanceIDToDelete == "" {
instanceIDToDelete = instance.Name
@ -910,17 +911,17 @@ func (r *basePoolManager) controllerLabel() string {
return fmt.Sprintf("%s%s", controllerLabelPrefix, r.controllerID)
}
func (r *basePoolManager) updateArgsFromProviderInstance(providerInstance params.Instance) params.UpdateInstanceParams {
func (r *basePoolManager) updateArgsFromProviderInstance(providerInstance commonParams.ProviderInstance) params.UpdateInstanceParams {
return params.UpdateInstanceParams{
ProviderID: providerInstance.ProviderID,
OSName: providerInstance.OSName,
OSVersion: providerInstance.OSVersion,
Addresses: providerInstance.Addresses,
Status: providerInstance.Status,
RunnerStatus: providerInstance.RunnerStatus,
ProviderFault: providerInstance.ProviderFault,
}
}
func (r *basePoolManager) scaleDownOnePool(ctx context.Context, pool params.Pool) error {
r.log("scaling down pool %s", pool.ID)
if !pool.Enabled {
@ -939,7 +940,7 @@ func (r *basePoolManager) scaleDownOnePool(ctx context.Context, pool params.Pool
// consideration for scale-down. The 5 minute grace period prevents a situation where a
// "queued" workflow triggers the creation of a new idle runner, and this routine reaps
// an idle runner before they have a chance to pick up a job.
if inst.RunnerStatus == providerCommon.RunnerIdle && inst.Status == providerCommon.InstanceRunning && time.Since(inst.UpdatedAt).Minutes() > 2 {
if inst.RunnerStatus == params.RunnerIdle && inst.Status == commonParams.InstanceRunning && time.Since(inst.UpdatedAt).Minutes() > 2 {
idleWorkers = append(idleWorkers, inst)
}
}
@ -1026,7 +1027,7 @@ func (r *basePoolManager) ensureIdleRunnersForOnePool(pool params.Pool) error {
idleOrPendingWorkers := []params.Instance{}
for _, inst := range existingInstances {
if inst.RunnerStatus != providerCommon.RunnerActive && inst.RunnerStatus != providerCommon.RunnerTerminated {
if inst.RunnerStatus != params.RunnerActive && inst.RunnerStatus != params.RunnerTerminated {
idleOrPendingWorkers = append(idleOrPendingWorkers, inst)
}
}
@ -1066,7 +1067,7 @@ func (r *basePoolManager) retryFailedInstancesForOnePool(ctx context.Context, po
g, errCtx := errgroup.WithContext(ctx)
for _, instance := range existingInstances {
if instance.Status != providerCommon.InstanceError {
if instance.Status != commonParams.InstanceError {
continue
}
if instance.CreateAttempt >= maxCreateAttempts {
@ -1105,7 +1106,7 @@ func (r *basePoolManager) retryFailedInstancesForOnePool(ctx context.Context, po
updateParams := params.UpdateInstanceParams{
CreateAttempt: instance.CreateAttempt + 1,
TokenFetched: &tokenFetched,
Status: providerCommon.InstancePendingCreate,
Status: commonParams.InstancePendingCreate,
}
r.log("queueing previously failed instance %s for retry", instance.Name)
// Set instance to pending create and wait for retry.
@ -1216,7 +1217,7 @@ func (r *basePoolManager) deletePendingInstances() error {
r.log("removing instances in pending_delete")
for _, instance := range instances {
if instance.Status != providerCommon.InstancePendingDelete {
if instance.Status != commonParams.InstancePendingDelete {
// not in pending_delete status. Skip.
continue
}
@ -1230,7 +1231,7 @@ func (r *basePoolManager) deletePendingInstances() error {
// Set the status to deleting before launching the goroutine that removes
// the runner from the provider (which can take a long time).
if _, err := r.setInstanceStatus(instance.Name, providerCommon.InstanceDeleting, nil); err != nil {
if _, err := r.setInstanceStatus(instance.Name, commonParams.InstanceDeleting, nil); err != nil {
r.log("failed to update runner %s status: %q", instance.Name, err)
r.keyMux.Unlock(instance.Name, false)
continue
@ -1246,7 +1247,7 @@ func (r *basePoolManager) deletePendingInstances() error {
r.log("failed to remove instance %s: %s", instance.Name, err)
// failed to remove from provider. Set the status back to pending_delete, which
// will retry the operation.
if _, err := r.setInstanceStatus(instance.Name, providerCommon.InstancePendingDelete, nil); err != nil {
if _, err := r.setInstanceStatus(instance.Name, commonParams.InstancePendingDelete, nil); err != nil {
r.log("failed to update runner %s status: %s", instance.Name, err)
}
}
@ -1277,7 +1278,7 @@ func (r *basePoolManager) addPendingInstances() error {
return fmt.Errorf("failed to fetch instances from store: %w", err)
}
for _, instance := range instances {
if instance.Status != providerCommon.InstancePendingCreate {
if instance.Status != commonParams.InstancePendingCreate {
// not in pending_create status. Skip.
continue
}
@ -1291,7 +1292,7 @@ func (r *basePoolManager) addPendingInstances() error {
// Set the instance to "creating" before launching the goroutine. This will ensure that addPendingInstances()
// won't attempt to create the runner a second time.
if _, err := r.setInstanceStatus(instance.Name, providerCommon.InstanceCreating, nil); err != nil {
if _, err := r.setInstanceStatus(instance.Name, commonParams.InstanceCreating, nil); err != nil {
r.log("failed to update runner %s status: %s", instance.Name, err)
r.keyMux.Unlock(instance.Name, false)
// We failed to transition the instance to Creating. This means that garm will retry to create this instance
@ -1305,7 +1306,7 @@ func (r *basePoolManager) addPendingInstances() error {
if err := r.addInstanceToProvider(instance); err != nil {
r.log("failed to add instance to provider: %s", err)
errAsBytes := []byte(err.Error())
if _, err := r.setInstanceStatus(instance.Name, providerCommon.InstanceError, errAsBytes); err != nil {
if _, err := r.setInstanceStatus(instance.Name, commonParams.InstanceError, errAsBytes); err != nil {
r.log("failed to update runner %s status: %s", instance.Name, err)
}
r.log("failed to create instance in provider: %s", err)
@ -1431,7 +1432,7 @@ func (r *basePoolManager) ForceDeleteRunner(runner params.Instance) error {
}
r.log("setting instance status for: %v", runner.Name)
if _, err := r.setInstanceStatus(runner.Name, providerCommon.InstancePendingDelete, nil); err != nil {
if _, err := r.setInstanceStatus(runner.Name, commonParams.InstancePendingDelete, nil); err != nil {
r.log("failed to update runner %s status: %s", runner.Name, err)
return errors.Wrap(err, "updating runner")
}

2
runner/pool/repository.go
View file

@ -21,8 +21,8 @@ import (
"strings"
"sync"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/util"

2
runner/pool/util.go
View file

@ -7,7 +7,7 @@ import (
"sync"
"sync/atomic"
runnerErrors "github.com/cloudbase/garm/errors"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/params"
)

2
runner/pools.go
View file

@ -18,8 +18,8 @@ import (
"context"
"fmt"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/auth"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/pkg/errors"

2
runner/pools_test.go
View file

@ -19,11 +19,11 @@ import (
"fmt"
"testing"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/database"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
garmTesting "github.com/cloudbase/garm/internal/testing"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"

63
runner/providers/common/common.go
View file

@ -1,63 +0,0 @@
// Copyright 2022 Cloudbase Solutions SRL
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package common
type InstanceStatus string
type RunnerStatus string
const (
InstanceRunning InstanceStatus = "running"
InstanceStopped InstanceStatus = "stopped"
InstanceError InstanceStatus = "error"
InstancePendingDelete InstanceStatus = "pending_delete"
InstanceDeleting InstanceStatus = "deleting"
InstancePendingCreate InstanceStatus = "pending_create"
InstanceCreating InstanceStatus = "creating"
InstanceStatusUnknown InstanceStatus = "unknown"
RunnerIdle RunnerStatus = "idle"
RunnerPending RunnerStatus = "pending"
RunnerTerminated RunnerStatus = "terminated"
RunnerInstalling RunnerStatus = "installing"
RunnerFailed RunnerStatus = "failed"
RunnerActive RunnerStatus = "active"
)
// IsValidStatus checks if the given status is valid.
func IsValidStatus(status InstanceStatus) bool {
switch status {
case InstanceRunning, InstanceError, InstancePendingCreate,
InstancePendingDelete, InstanceStatusUnknown, InstanceStopped,
InstanceCreating, InstanceDeleting:
return true
default:
return false
}
}
// IsProviderValidStatus checks if the given status is valid for the provider.
// A provider should only return a status indicating that the instance is in a
// lifecycle state that it can influence. The sole purpose of a provider is to
// manage the lifecycle of an instance. Statuses that indicate an instance should
// be created or removed, will be set by the controller.
func IsValidProviderStatus(status InstanceStatus) bool {
switch status {
case InstanceRunning, InstanceError, InstanceStopped:
return true
default:
return false
}
}

26
runner/providers/external/execution/exit_codes.go vendored
View file

@ -1,26 +0,0 @@
package execution
import (
"errors"
gErrors "github.com/cloudbase/garm/errors"
)
const (
// ExitCodeNotFound is an exit code that indicates a Not Found error
ExitCodeNotFound int = 30
// ExitCodeDuplicate is an exit code that indicates a duplicate error
ExitCodeDuplicate int = 31
)
func ResolveErrorToExitCode(err error) int {
if err != nil {
if errors.Is(err, gErrors.ErrNotFound) {
return ExitCodeNotFound
} else if errors.Is(err, gErrors.ErrDuplicateEntity) {
return ExitCodeDuplicate
}
return 1
}
return 0
}

52
runner/providers/external/external.go vendored
View file

@ -7,13 +7,15 @@ import (
"log"
"os/exec"
"github.com/cloudbase/garm-provider-common/execution"
commonParams "github.com/cloudbase/garm-provider-common/params"
garmErrors "github.com/cloudbase/garm-provider-common/errors"
garmExec "github.com/cloudbase/garm-provider-common/util/exec"
"github.com/cloudbase/garm/config"
garmErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
providerCommon "github.com/cloudbase/garm/runner/providers/common"
"github.com/cloudbase/garm/runner/providers/external/execution"
garmExec "github.com/cloudbase/garm/util/exec"
"github.com/pkg/errors"
)
@ -44,7 +46,7 @@ type external struct {
execPath string
}
func (e *external) validateResult(inst params.Instance) error {
func (e *external) validateResult(inst commonParams.ProviderInstance) error {
if inst.ProviderID == "" {
return garmErrors.NewProviderError("missing provider ID")
}
@ -57,7 +59,7 @@ func (e *external) validateResult(inst params.Instance) error {
// we can still function without this info (I think)
log.Printf("WARNING: missing OS information")
}
if !providerCommon.IsValidProviderStatus(inst.Status) {
if !IsValidProviderStatus(inst.Status) {
return garmErrors.NewProviderError("invalid status returned (%s)", inst.Status)
}
@ -65,7 +67,7 @@ func (e *external) validateResult(inst params.Instance) error {
}
// CreateInstance creates a new compute instance in the provider.
func (e *external) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) {
func (e *external) CreateInstance(ctx context.Context, bootstrapParams commonParams.BootstrapInstance) (commonParams.ProviderInstance, error) {
asEnv := []string{
fmt.Sprintf("GARM_COMMAND=%s", execution.CreateInstanceCommand),
fmt.Sprintf("GARM_CONTROLLER_ID=%s", e.controllerID),
@ -75,21 +77,21 @@ func (e *external) CreateInstance(ctx context.Context, bootstrapParams params.Bo
asJs, err := json.Marshal(bootstrapParams)
if err != nil {
return params.Instance{}, errors.Wrap(err, "serializing bootstrap params")
return commonParams.ProviderInstance{}, errors.Wrap(err, "serializing bootstrap params")
}
out, err := garmExec.Exec(ctx, e.execPath, asJs, asEnv)
if err != nil {
return params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err)
return commonParams.ProviderInstance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err)
}
var param params.Instance
var param commonParams.ProviderInstance
if err := json.Unmarshal(out, &param); err != nil {
return params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err)
return commonParams.ProviderInstance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err)
}
if err := e.validateResult(param); err != nil {
return params.Instance{}, garmErrors.NewProviderError("failed to validate result: %s", err)
return commonParams.ProviderInstance{}, garmErrors.NewProviderError("failed to validate result: %s", err)
}
retAsJs, _ := json.MarshalIndent(param, "", " ")
@ -118,7 +120,7 @@ func (e *external) DeleteInstance(ctx context.Context, instance string) error {
}
// GetInstance will return details about one instance.
func (e *external) GetInstance(ctx context.Context, instance string) (params.Instance, error) {
func (e *external) GetInstance(ctx context.Context, instance string) (commonParams.ProviderInstance, error) {
asEnv := []string{
fmt.Sprintf("GARM_COMMAND=%s", execution.GetInstanceCommand),
fmt.Sprintf("GARM_CONTROLLER_ID=%s", e.controllerID),
@ -130,23 +132,23 @@ func (e *external) GetInstance(ctx context.Context, instance string) (params.Ins
// know when the error is ErrNotFound.
out, err := garmExec.Exec(ctx, e.execPath, nil, asEnv)
if err != nil {
return params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err)
return commonParams.ProviderInstance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err)
}
var param params.Instance
var param commonParams.ProviderInstance
if err := json.Unmarshal(out, &param); err != nil {
return params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err)
return commonParams.ProviderInstance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err)
}
if err := e.validateResult(param); err != nil {
return params.Instance{}, garmErrors.NewProviderError("failed to validate result: %s", err)
return commonParams.ProviderInstance{}, garmErrors.NewProviderError("failed to validate result: %s", err)
}
return param, nil
}
// ListInstances will list all instances for a provider.
func (e *external) ListInstances(ctx context.Context, poolID string) ([]params.Instance, error) {
func (e *external) ListInstances(ctx context.Context, poolID string) ([]commonParams.ProviderInstance, error) {
asEnv := []string{
fmt.Sprintf("GARM_COMMAND=%s", execution.ListInstancesCommand),
fmt.Sprintf("GARM_CONTROLLER_ID=%s", e.controllerID),
@ -156,20 +158,22 @@ func (e *external) ListInstances(ctx context.Context, poolID string) ([]params.I
out, err := garmExec.Exec(ctx, e.execPath, nil, asEnv)
if err != nil {
return []params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err)
return []commonParams.ProviderInstance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err)
}
var param []params.Instance
var param []commonParams.ProviderInstance
if err := json.Unmarshal(out, &param); err != nil {
return []params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err)
return []commonParams.ProviderInstance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err)
}
for _, inst := range param {
ret := make([]commonParams.ProviderInstance, len(param))
for idx, inst := range param {
if err := e.validateResult(inst); err != nil {
return []params.Instance{}, garmErrors.NewProviderError("failed to validate result: %s", err)
return []commonParams.ProviderInstance{}, garmErrors.NewProviderError("failed to validate result: %s", err)
}
ret[idx] = inst
}
return param, nil
return ret, nil
}
// RemoveAllInstances will remove all instances created by this provider.

21
runner/providers/external/util.go vendored Normal file
View file

@ -0,0 +1,21 @@
package external
import (
commonParams "github.com/cloudbase/garm-provider-common/params"
)
// IsProviderValidStatus checks if the given status is valid for the provider.
// A provider should only return a status indicating that the instance is in a
// lifecycle state that it can influence. The sole purpose of a provider is to
// manage the lifecycle of an instance. Statuses that indicate an instance should
// be created or removed, will be set by the controller.
func IsValidProviderStatus(status commonParams.InstanceStatus) bool {
switch status {
case commonParams.InstanceRunning, commonParams.InstanceError,
commonParams.InstanceStopped, commonParams.InstanceStatusUnknown:
return true
default:
return false
}
}

2
runner/providers/lxd/images.go
View file

@ -18,8 +18,8 @@ import (
"fmt"
"strings"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/config"
runnerErrors "github.com/cloudbase/garm/errors"
lxd "github.com/lxc/lxd/client"
"github.com/lxc/lxd/shared/api"

58
runner/providers/lxd/lxd.go
View file

@ -21,16 +21,18 @@ import (
"sync"
"time"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/config"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/util"
"github.com/google/go-github/v53/github"
lxd "github.com/lxc/lxd/client"
"github.com/lxc/lxd/shared/api"
"github.com/pkg/errors"
"github.com/cloudbase/garm-provider-common/cloudconfig"
commonParams "github.com/cloudbase/garm-provider-common/params"
)
var _ common.Provider = &LXD{}
@ -66,16 +68,16 @@ var (
"arm64": "arm64",
}
configToLXDArchMap map[params.OSArch]string = map[params.OSArch]string{
params.Amd64: "x86_64",
params.Arm64: "aarch64",
params.Arm: "armv7l",
configToLXDArchMap map[commonParams.OSArch]string = map[commonParams.OSArch]string{
commonParams.Amd64: "x86_64",
commonParams.Arm64: "aarch64",
commonParams.Arm: "armv7l",
}
lxdToConfigArch map[string]params.OSArch = map[string]params.OSArch{
"x86_64": params.Amd64,
"aarch64": params.Arm64,
"armv7l": params.Arm,
lxdToConfigArch map[string]commonParams.OSArch = map[string]commonParams.OSArch{
"x86_64": commonParams.Amd64,
"aarch64": commonParams.Arm64,
"armv7l": commonParams.Arm,
}
)
@ -171,10 +173,10 @@ func (l *LXD) getProfiles(flavor string) ([]string, error) {
return ret, nil
}
func (l *LXD) getTools(tools []*github.RunnerApplicationDownload, osType params.OSType, architecture string) (github.RunnerApplicationDownload, error) {
func (l *LXD) getTools(tools []*github.RunnerApplicationDownload, osType commonParams.OSType, architecture string) (github.RunnerApplicationDownload, error) {
// Validate image OS. Linux only for now.
switch osType {
case params.Linux:
case commonParams.Linux:
default:
return github.RunnerApplicationDownload{}, fmt.Errorf("this provider does not support OS type: %s", osType)
}
@ -210,7 +212,7 @@ func (l *LXD) secureBootEnabled() string {
return "false"
}
func (l *LXD) getCreateInstanceArgs(bootstrapParams params.BootstrapInstance, specs extraSpecs) (api.InstancesPost, error) {
func (l *LXD) getCreateInstanceArgs(bootstrapParams commonParams.BootstrapInstance, specs extraSpecs) (api.InstancesPost, error) {
if bootstrapParams.Name == "" {
return api.InstancesPost{}, runnerErrors.NewBadRequestError("missing name")
}
@ -237,7 +239,7 @@ func (l *LXD) getCreateInstanceArgs(bootstrapParams params.BootstrapInstance, sp
bootstrapParams.UserDataOptions.DisableUpdatesOnBoot = specs.DisableUpdates
bootstrapParams.UserDataOptions.ExtraPackages = specs.ExtraPackages
cloudCfg, err := util.GetCloudConfig(bootstrapParams, tools, bootstrapParams.Name)
cloudCfg, err := cloudconfig.GetCloudConfig(bootstrapParams, tools, bootstrapParams.Name)
if err != nil {
return api.InstancesPost{}, errors.Wrap(err, "generating cloud-config")
}
@ -313,40 +315,40 @@ func (l *LXD) launchInstance(createArgs api.InstancesPost) error {
}
// CreateInstance creates a new compute instance in the provider.
func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) {
func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams commonParams.BootstrapInstance) (commonParams.ProviderInstance, error) {
extraSpecs, err := parseExtraSpecsFromBootstrapParams(bootstrapParams)
if err != nil {
return params.Instance{}, errors.Wrap(err, "parsing extra specs")
return commonParams.ProviderInstance{}, errors.Wrap(err, "parsing extra specs")
}
args, err := l.getCreateInstanceArgs(bootstrapParams, extraSpecs)
if err != nil {
return params.Instance{}, errors.Wrap(err, "fetching create args")
return commonParams.ProviderInstance{}, errors.Wrap(err, "fetching create args")
}
if err := l.launchInstance(args); err != nil {
return params.Instance{}, errors.Wrap(err, "creating instance")
return commonParams.ProviderInstance{}, errors.Wrap(err, "creating instance")
}
ret, err := l.waitInstanceHasIP(ctx, args.Name)
if err != nil {
return params.Instance{}, errors.Wrap(err, "fetching instance")
return commonParams.ProviderInstance{}, errors.Wrap(err, "fetching instance")
}
return ret, nil
}
// GetInstance will return details about one instance.
func (l *LXD) GetInstance(ctx context.Context, instanceName string) (params.Instance, error) {
func (l *LXD) GetInstance(ctx context.Context, instanceName string) (commonParams.ProviderInstance, error) {
cli, err := l.getCLI()
if err != nil {
return params.Instance{}, errors.Wrap(err, "fetching client")
return commonParams.ProviderInstance{}, errors.Wrap(err, "fetching client")
}
instance, _, err := cli.GetInstanceFull(instanceName)
if err != nil {
if isNotFoundError(err) {
return params.Instance{}, errors.Wrapf(runnerErrors.ErrNotFound, "fetching instance: %q", err)
return commonParams.ProviderInstance{}, errors.Wrapf(runnerErrors.ErrNotFound, "fetching instance: %q", err)
}
return params.Instance{}, errors.Wrap(err, "fetching instance")
return commonParams.ProviderInstance{}, errors.Wrap(err, "fetching instance")
}
return lxdInstanceToAPIInstance(instance), nil
@ -418,10 +420,10 @@ type listResponse struct {
}
// ListInstances will list all instances for a provider.
func (l *LXD) ListInstances(ctx context.Context, poolID string) ([]params.Instance, error) {
func (l *LXD) ListInstances(ctx context.Context, poolID string) ([]commonParams.ProviderInstance, error) {
cli, err := l.getCLI()
if err != nil {
return []params.Instance{}, errors.Wrap(err, "fetching client")
return []commonParams.ProviderInstance{}, errors.Wrap(err, "fetching client")
}
result := make(chan listResponse, 1)
@ -443,14 +445,14 @@ func (l *LXD) ListInstances(ctx context.Context, poolID string) ([]params.Instan
select {
case res := <-result:
if res.err != nil {
return []params.Instance{}, errors.Wrap(res.err, "fetching instances")
return []commonParams.ProviderInstance{}, errors.Wrap(res.err, "fetching instances")
}
instances = res.instances
case <-time.After(time.Second * 60):
return []params.Instance{}, errors.Wrap(runnerErrors.ErrTimeout, "fetching instances from provider")
return []commonParams.ProviderInstance{}, errors.Wrap(runnerErrors.ErrTimeout, "fetching instances from provider")
}
ret := []params.Instance{}
ret := []commonParams.ProviderInstance{}
for _, instance := range instances {
if id, ok := instance.ExpandedConfig[controllerIDKeyName]; ok && id == l.controllerID {

4
runner/providers/lxd/specs.go
View file

@ -17,7 +17,7 @@ package lxd
import (
"encoding/json"
"github.com/cloudbase/garm/params"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/pkg/errors"
)
@ -26,7 +26,7 @@ type extraSpecs struct {
ExtraPackages []string `json:"extra_packages"`
}
func parseExtraSpecsFromBootstrapParams(bootstrapParams params.BootstrapInstance) (extraSpecs, error) {
func parseExtraSpecsFromBootstrapParams(bootstrapParams commonParams.BootstrapInstance) (extraSpecs, error) {
specs := extraSpecs{}
if bootstrapParams.ExtraSpecs == nil {
return specs, nil

36
runner/providers/lxd/util.go
View file

@ -25,10 +25,10 @@ import (
"strings"
"time"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/providers/common"
"github.com/cloudbase/garm/util"
"github.com/juju/clock"
"github.com/juju/retry"
@ -61,7 +61,7 @@ func isNotFoundError(err error) bool {
return false
}
func lxdInstanceToAPIInstance(instance *api.InstanceFull) params.Instance {
func lxdInstanceToAPIInstance(instance *api.InstanceFull) commonParams.ProviderInstance {
lxdOS, ok := instance.ExpandedConfig["image.os"]
if !ok {
log.Printf("failed to find OS in instance config")
@ -77,7 +77,7 @@ func lxdInstanceToAPIInstance(instance *api.InstanceFull) params.Instance {
if !ok {
log.Printf("failed to find OS type in fallback location")
}
osType = params.OSType(osTypeFromTag)
osType = commonParams.OSType(osTypeFromTag)
}
osRelease, ok := instance.ExpandedConfig["image.release"]
@ -86,16 +86,16 @@ func lxdInstanceToAPIInstance(instance *api.InstanceFull) params.Instance {
}
state := instance.State
addresses := []params.Address{}
addresses := []commonParams.Address{}
if state.Network != nil {
for _, details := range state.Network {
for _, addr := range details.Addresses {
if addr.Scope != "global" {
continue
}
addresses = append(addresses, params.Address{
addresses = append(addresses, commonParams.Address{
Address: addr.Address,
Type: params.PublicAddress,
Type: commonParams.PublicAddress,
})
}
}
@ -106,7 +106,7 @@ func lxdInstanceToAPIInstance(instance *api.InstanceFull) params.Instance {
log.Printf("failed to find OS architecture")
}
return params.Instance{
return commonParams.ProviderInstance{
OSArch: instanceArch,
ProviderID: instance.Name,
Name: instance.Name,
@ -118,14 +118,14 @@ func lxdInstanceToAPIInstance(instance *api.InstanceFull) params.Instance {
}
}
func lxdStatusToProviderStatus(status string) common.InstanceStatus {
func lxdStatusToProviderStatus(status string) commonParams.InstanceStatus {
switch status {
case "Running":
return common.InstanceRunning
return commonParams.InstanceRunning
case "Stopped":
return common.InstanceStopped
return commonParams.InstanceStopped
default:
return common.InstanceStatusUnknown
return commonParams.InstanceStatusUnknown
}
}
@ -186,9 +186,9 @@ func projectName(cfg config.LXD) string {
return DefaultProjectName
}
func resolveArchitecture(osArch params.OSArch) (string, error) {
func resolveArchitecture(osArch commonParams.OSArch) (string, error) {
if string(osArch) == "" {
return configToLXDArchMap[params.Amd64], nil
return configToLXDArchMap[commonParams.Amd64], nil
}
arch, ok := configToLXDArchMap[osArch]
if !ok {
@ -199,8 +199,8 @@ func resolveArchitecture(osArch params.OSArch) (string, error) {
// waitDeviceActive is a function capable of figuring out when a Equinix Metal
// device is active
func (l *LXD) waitInstanceHasIP(ctx context.Context, instanceName string) (params.Instance, error) {
var p params.Instance
func (l *LXD) waitInstanceHasIP(ctx context.Context, instanceName string) (commonParams.ProviderInstance, error) {
var p commonParams.ProviderInstance
var errIPNotFound error = fmt.Errorf("ip not found")
err := retry.Call(retry.CallArgs{
Func: func() error {
@ -227,7 +227,7 @@ func (l *LXD) waitInstanceHasIP(ctx context.Context, instanceName string) (param
})
if err != nil && err != errIPNotFound {
return params.Instance{}, err
return commonParams.ProviderInstance{}, err
}
return p, nil

2
runner/repositories.go
View file

@ -20,8 +20,8 @@ import (
"log"
"strings"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/auth"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/util/appdefaults"

2
runner/repositories_test.go
View file

@ -19,11 +19,11 @@ import (
"fmt"
"testing"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/database"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
garmTesting "github.com/cloudbase/garm/internal/testing"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"

15
runner/runner.go
View file

@ -29,16 +29,17 @@ import (
"sync"
"time"
commonParams "github.com/cloudbase/garm-provider-common/params"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/config"
dbCommon "github.com/cloudbase/garm/database/common"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/runner/pool"
"github.com/cloudbase/garm/runner/providers"
providerCommon "github.com/cloudbase/garm/runner/providers/common"
"github.com/cloudbase/garm/util"
"golang.org/x/sync/errgroup"
"github.com/google/uuid"
@ -761,7 +762,7 @@ func (r *Runner) appendTagsToCreatePoolParams(param params.CreatePoolParams) (pa
return param, nil
}
func (r *Runner) processTags(osArch string, osType params.OSType, tags []string) ([]string, error) {
func (r *Runner) processTags(osArch string, osType commonParams.OSType, tags []string) ([]string, error) {
// github automatically adds the "self-hosted" tag as well as the OS type (linux, windows, etc)
// and architecture (arm, x64, etc) to all self hosted runners. When a workflow job comes in, we try
// to find a pool based on the labels that are set in the workflow. If we don't explicitly define these
@ -857,7 +858,7 @@ func (r *Runner) GetInstanceGithubRegistrationToken(ctx context.Context) (string
}
status := auth.InstanceRunnerStatus(ctx)
if status != providerCommon.RunnerPending && status != providerCommon.RunnerInstalling {
if status != params.RunnerPending && status != params.RunnerInstalling {
return "", runnerErrors.ErrUnauthorized
}
@ -943,9 +944,9 @@ func (r *Runner) ForceDeleteRunner(ctx context.Context, instanceName string) err
}
switch instance.Status {
case providerCommon.InstanceRunning, providerCommon.InstanceError:
case commonParams.InstanceRunning, commonParams.InstanceError:
default:
return runnerErrors.NewBadRequestError("runner must be in %q or %q state", providerCommon.InstanceRunning, providerCommon.InstanceError)
return runnerErrors.NewBadRequestError("runner must be in %q or %q state", commonParams.InstanceRunning, commonParams.InstanceError)
}
poolMgr, err := r.getPoolManagerFromInstance(ctx, instance)

2
runner/types.go
View file

@ -14,7 +14,7 @@
package runner
import "github.com/cloudbase/garm/params"
import "github.com/cloudbase/garm-provider-common/params"
type HookTargetType string

14
util/appdefaults/appdefaults.go
View file

@ -19,11 +19,6 @@ const (
// configuration file.
DefaultConfigFilePath = "/etc/garm/config.toml"
// DefaultUser is the default username that should exist on the instances.
DefaultUser = "runner"
// DefaultUserShell is the shell for the default user.
DefaultUserShell = "/bin/bash"
// DefaultPoolQueueSize is the default size for a pool queue.
DefaultPoolQueueSize = 10
@ -33,12 +28,3 @@ const (
// uploadBaseURL is the default URL for guthub uploads.
GithubDefaultUploadBaseURL = "https://uploads.github.com/"
)
var (
// DefaultUserGroups are the groups the default user will be part of.
DefaultUserGroups = []string{
"sudo", "adm", "cdrom", "dialout",
"dip", "video", "plugdev", "netdev",
"docker", "lxd",
}
)

437
util/util.go
View file

@ -15,186 +15,20 @@
package util
import (
"bytes"
"compress/gzip"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/binary"
"fmt"
"io"
"math/big"
"net/http"
"os"
"path"
"regexp"
"strings"
"unicode"
"unicode/utf16"
"github.com/cloudbase/garm/cloudconfig"
"github.com/cloudbase/garm/config"
runnerErrors "github.com/cloudbase/garm/errors"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/util/appdefaults"
"github.com/google/go-github/v53/github"
"github.com/google/uuid"
gorillaHandlers "github.com/gorilla/handlers"
"github.com/pkg/errors"
"github.com/teris-io/shortid"
"golang.org/x/crypto/bcrypt"
"golang.org/x/oauth2"
lumberjack "gopkg.in/natefinch/lumberjack.v2"
)
const alphanumeric = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
// From: https://www.alexedwards.net/blog/validation-snippets-for-go#email-validation
var rxEmail = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$")
var (
OSToOSTypeMap map[string]params.OSType = map[string]params.OSType{
"almalinux": params.Linux,
"alma": params.Linux,
"alpine": params.Linux,
"archlinux": params.Linux,
"arch": params.Linux,
"centos": params.Linux,
"ubuntu": params.Linux,
"rhel": params.Linux,
"suse": params.Linux,
"opensuse": params.Linux,
"fedora": params.Linux,
"debian": params.Linux,
"flatcar": params.Linux,
"gentoo": params.Linux,
"rockylinux": params.Linux,
"rocky": params.Linux,
"windows": params.Windows,
}
githubArchMapping map[string]string = map[string]string{
"x86_64": "x64",
"amd64": "x64",
"armv7l": "arm",
"aarch64": "arm64",
"x64": "x64",
"arm": "arm",
"arm64": "arm64",
}
githubOSTypeMap map[string]string = map[string]string{
"linux": "linux",
"windows": "win",
}
//
githubOSTag = map[params.OSType]string{
params.Linux: "Linux",
params.Windows: "Windows",
}
)
// ResolveToGithubArch returns the cpu architecture as it is defined in the GitHub
// tools download list. We use it to find the proper tools for the OS/Arch combo we're
// deploying.
func ResolveToGithubArch(arch string) (string, error) {
ghArch, ok := githubArchMapping[arch]
if !ok {
return "", runnerErrors.NewNotFoundError("arch %s is unknown", arch)
}
return ghArch, nil
}
// ResolveToGithubArch returns the OS type as it is defined in the GitHub
// tools download list. We use it to find the proper tools for the OS/Arch combo we're
// deploying.
func ResolveToGithubOSType(osType string) (string, error) {
ghOS, ok := githubOSTypeMap[osType]
if !ok {
return "", runnerErrors.NewNotFoundError("os %s is unknown", osType)
}
return ghOS, nil
}
// ResolveToGithubTag returns the default OS tag that self hosted runners automatically
// (and forcefully) adds to every runner that gets deployed. We need to keep track of those
// tags internally as well.
func ResolveToGithubTag(os params.OSType) (string, error) {
ghOS, ok := githubOSTag[os]
if !ok {
return "", runnerErrors.NewNotFoundError("os %s is unknown", os)
}
return ghOS, nil
}
// IsValidEmail returs a bool indicating if an email is valid
func IsValidEmail(email string) bool {
if len(email) > 254 || !rxEmail.MatchString(email) {
return false
}
return true
}
func IsAlphanumeric(s string) bool {
for _, r := range s {
if !unicode.IsLetter(r) && !unicode.IsNumber(r) {
return false
}
}
return true
}
// GetLoggingWriter returns a new io.Writer suitable for logging.
func GetLoggingWriter(cfg *config.Config) (io.Writer, error) {
var writer io.Writer = os.Stdout
if cfg.Default.LogFile != "" {
dirname := path.Dir(cfg.Default.LogFile)
if _, err := os.Stat(dirname); err != nil {
if !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to create log folder")
}
if err := os.MkdirAll(dirname, 0o711); err != nil {
return nil, fmt.Errorf("failed to create log folder")
}
}
writer = &lumberjack.Logger{
Filename: cfg.Default.LogFile,
MaxSize: 500, // megabytes
MaxBackups: 3,
MaxAge: 28, // days
Compress: true, // disabled by default
}
}
return writer, nil
}
func ConvertFileToBase64(file string) (string, error) {
bytes, err := os.ReadFile(file)
if err != nil {
return "", errors.Wrap(err, "reading file")
}
return base64.StdEncoding.EncodeToString(bytes), nil
}
func OSToOSType(os string) (params.OSType, error) {
osType, ok := OSToOSTypeMap[strings.ToLower(os)]
if !ok {
return params.Unknown, fmt.Errorf("no OS to OS type mapping for %s", os)
}
return osType, nil
}
func GithubClient(ctx context.Context, token string, credsDetails params.GithubCredentials) (common.GithubClient, common.GithubEnterpriseClient, error) {
var roots *x509.CertPool
if credsDetails.CABundle != nil && len(credsDetails.CABundle) > 0 {
@ -224,274 +58,3 @@ func GithubClient(ctx context.Context, token string, credsDetails params.GithubC
return ghClient.Actions, ghClient.Enterprise, nil
}
func GetCloudConfig(bootstrapParams params.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) (string, error) {
if tools.Filename == nil {
return "", fmt.Errorf("missing tools filename")
}
if tools.DownloadURL == nil {
return "", fmt.Errorf("missing tools download URL")
}
var tempToken string
if tools.TempDownloadToken != nil {
tempToken = *tools.TempDownloadToken
}
installRunnerParams := cloudconfig.InstallRunnerParams{
FileName: *tools.Filename,
DownloadURL: *tools.DownloadURL,
TempDownloadToken: tempToken,
MetadataURL: bootstrapParams.MetadataURL,
RunnerUsername: appdefaults.DefaultUser,
RunnerGroup: appdefaults.DefaultUser,
RepoURL: bootstrapParams.RepoURL,
RunnerName: runnerName,
RunnerLabels: strings.Join(bootstrapParams.Labels, ","),
CallbackURL: bootstrapParams.CallbackURL,
CallbackToken: bootstrapParams.InstanceToken,
GitHubRunnerGroup: bootstrapParams.GitHubRunnerGroup,
}
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
installRunnerParams.CABundle = string(bootstrapParams.CACertBundle)
}
installScript, err := cloudconfig.InstallRunnerScript(installRunnerParams, bootstrapParams.OSType)
if err != nil {
return "", errors.Wrap(err, "generating script")
}
var asStr string
switch bootstrapParams.OSType {
case params.Linux:
cloudCfg := cloudconfig.NewDefaultCloudInitConfig()
if bootstrapParams.UserDataOptions.DisableUpdatesOnBoot {
cloudCfg.PackageUpgrade = false
cloudCfg.Packages = []string{}
}
for _, pkg := range bootstrapParams.UserDataOptions.ExtraPackages {
cloudCfg.AddPackage(pkg)
}
cloudCfg.AddSSHKey(bootstrapParams.SSHKeys...)
cloudCfg.AddFile(installScript, "/install_runner.sh", "root:root", "755")
cloudCfg.AddRunCmd(fmt.Sprintf("su -l -c /install_runner.sh %s", appdefaults.DefaultUser))
cloudCfg.AddRunCmd("rm -f /install_runner.sh")
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
if err := cloudCfg.AddCACert(bootstrapParams.CACertBundle); err != nil {
return "", errors.Wrap(err, "adding CA cert bundle")
}
}
var err error
asStr, err = cloudCfg.Serialize()
if err != nil {
return "", errors.Wrap(err, "creating cloud config")
}
case params.Windows:
asStr = string(installScript)
default:
return "", fmt.Errorf("unknown os type: %s", bootstrapParams.OSType)
}
return asStr, nil
}
func GetTools(osType params.OSType, osArch params.OSArch, tools []*github.RunnerApplicationDownload) (github.RunnerApplicationDownload, error) {
// Validate image OS. Linux only for now.
switch osType {
case params.Linux:
case params.Windows:
default:
return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS type: %s", osType)
}
switch osArch {
case params.Amd64:
case params.Arm:
case params.Arm64:
default:
return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS arch: %s", osArch)
}
// Find tools for OS/Arch.
for _, tool := range tools {
if tool == nil {
continue
}
if tool.OS == nil || tool.Architecture == nil {
continue
}
ghArch, err := ResolveToGithubArch(string(osArch))
if err != nil {
continue
}
ghOS, err := ResolveToGithubOSType(string(osType))
if err != nil {
continue
}
if *tool.Architecture == ghArch && *tool.OS == ghOS {
return *tool, nil
}
}
return github.RunnerApplicationDownload{}, fmt.Errorf("failed to find tools for OS %s and arch %s", osType, osArch)
}
// GetRandomString returns a secure random string
func GetRandomString(n int) (string, error) {
data := make([]byte, n)
_, err := rand.Read(data)
if err != nil {
return "", errors.Wrap(err, "getting random data")
}
for i, b := range data {
data[i] = alphanumeric[b%byte(len(alphanumeric))]
}
return string(data), nil
}
func Aes256EncodeString(target string, passphrase string) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
toEncrypt := []byte(target)
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return nil, errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, errors.Wrap(err, "creating new aead")
}
nonce := make([]byte, aesgcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, errors.Wrap(err, "creating nonce")
}
ciphertext := aesgcm.Seal(nonce, nonce, toEncrypt, nil)
return ciphertext, nil
}
func Aes256DecodeString(target []byte, passphrase string) (string, error) {
if len(passphrase) != 32 {
return "", fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return "", errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return "", errors.Wrap(err, "creating new aead")
}
nonceSize := aesgcm.NonceSize()
if len(target) < nonceSize {
return "", fmt.Errorf("failed to decrypt text")
}
nonce, ciphertext := target[:nonceSize], target[nonceSize:]
plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", fmt.Errorf("failed to decrypt text")
}
return string(plaintext), nil
}
// PaswsordToBcrypt returns a bcrypt hash of the specified password using the default cost
func PaswsordToBcrypt(password string) (string, error) {
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return "", fmt.Errorf("failed to hash password")
}
return string(hashedPassword), nil
}
func NewLoggingMiddleware(writer io.Writer) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return gorillaHandlers.CombinedLoggingHandler(writer, next)
}
}
func SanitizeLogEntry(entry string) string {
return strings.Replace(strings.Replace(entry, "\n", "", -1), "\r", "", -1)
}
func toBase62(uuid []byte) string {
var i big.Int
i.SetBytes(uuid[:])
return i.Text(62)
}
func NewID() string {
short, err := shortid.Generate()
if err == nil {
return toBase62([]byte(short))
}
newUUID := uuid.New()
return toBase62(newUUID[:])
}
func UTF16FromString(s string) ([]uint16, error) {
buf := make([]uint16, 0, len(s)*2+1)
for _, r := range s {
buf = utf16.AppendRune(buf, r)
}
return utf16.AppendRune(buf, '\x00'), nil
}
func UTF16ToString(s []uint16) string {
for i, v := range s {
if v == 0 {
s = s[0:i]
break
}
}
return string(utf16.Decode(s))
}
func Uint16ToByteArray(u []uint16) []byte {
ret := make([]byte, (len(u)-1)*2)
for i := 0; i < len(u)-1; i++ {
binary.LittleEndian.PutUint16(ret[i*2:], uint16(u[i]))
}
return ret
}
func UTF16EncodedByteArrayFromString(s string) ([]byte, error) {
asUint16, err := UTF16FromString(s)
if err != nil {
return nil, fmt.Errorf("failed to encode to uint16: %w", err)
}
asBytes := Uint16ToByteArray(asUint16)
return asBytes, nil
}
func CompressData(data []byte) ([]byte, error) {
var b bytes.Buffer
gz := gzip.NewWriter(&b)
_, err := gz.Write(data)
if err != nil {
return nil, fmt.Errorf("failed to compress data: %w", err)
}
if err = gz.Flush(); err != nil {
return nil, fmt.Errorf("failed to flush buffer: %w", err)
}
if err = gz.Close(); err != nil {
return nil, fmt.Errorf("failed to close buffer: %w", err)
}
return b.Bytes(), nil
}

10
cloudconfig/cloudconfig.go → vendor/github.com/cloudbase/garm-provider-common/cloudconfig/cloudconfig.go generated vendored
View file

@ -21,7 +21,7 @@ import (
"strings"
"sync"
"github.com/cloudbase/garm/util/appdefaults"
"github.com/cloudbase/garm-provider-common/defaults"
"github.com/pkg/errors"
"gopkg.in/yaml.v3"
@ -36,10 +36,10 @@ func NewDefaultCloudInitConfig() *CloudInit {
},
SystemInfo: &SystemInfo{
DefaultUser: DefaultUser{
Name: appdefaults.DefaultUser,
Home: fmt.Sprintf("/home/%s", appdefaults.DefaultUser),
Shell: appdefaults.DefaultUserShell,
Groups: appdefaults.DefaultUserGroups,
Name: defaults.DefaultUser,
Home: fmt.Sprintf("/home/%s", defaults.DefaultUser),
Shell: defaults.DefaultUserShell,
Groups: defaults.DefaultUserGroups,
Sudo: "ALL=(ALL) NOPASSWD:ALL",
},
},

74
cloudconfig/templates.go → vendor/github.com/cloudbase/garm-provider-common/cloudconfig/templates.go generated vendored
View file

@ -19,7 +19,7 @@ import (
"fmt"
"text/template"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm-provider-common/params"
"github.com/pkg/errors"
)
@ -28,6 +28,10 @@ var CloudConfigTemplate = `#!/bin/bash
set -e
set -o pipefail
{{- if .EnableBootDebug }}
set -x
{{- end }}
CALLBACK_URL="{{ .CallbackURL }}"
METADATA_URL="{{ .MetadataURL }}"
BEARER_TOKEN="{{ .CallbackToken }}"
@ -410,31 +414,59 @@ function Install-Runner() {
Install-Runner
`
// InstallRunnerParams holds the parameters needed to render the runner install script.
type InstallRunnerParams struct {
FileName string
DownloadURL string
RunnerUsername string
RunnerGroup string
RepoURL string
MetadataURL string
RunnerName string
RunnerLabels string
CallbackURL string
CallbackToken string
// FileName is the name of the file that will be downloaded from the download URL.
// This will be the runner archive downloaded from GitHub.
FileName string
// DownloadURL is the URL from which the runner archive will be downloaded.
DownloadURL string
// RunnerUsername is the username of the user that will run the runner service.
RunnerUsername string
// RunnerGroup is the group of the user that will run the runner service.
RunnerGroup string
// RepoURL is the URL or the github repo the github runner agent needs to configure itself.
RepoURL string
// MetadataURL is the URL where instances can fetch information needed to set themselves up.
// This URL is set in the GARM config file.
MetadataURL string
// RunnerName is the name of the runner. GARM will use this to register the runner with GitHub.
RunnerName string
// RunnerLabels is a comma separated list of labels that will be added to the runner.
RunnerLabels string
// CallbackURL is the URL where the instance can send a post, signaling progress or status.
// This URL is set in the GARM config file.
CallbackURL string
// CallbackToken is the token that needs to be set by the instance in the headers in order to call
// the CallbackURL.
CallbackToken string
// TempDownloadToken is the token that needs to be set by the instance in the headers in order to download
// the githun runner. This is usually needed when using garm against a GHES instance.
TempDownloadToken string
CABundle string
// CABundle is a CA certificate bundle which will be sent to instances and which will tipically be installed
// as a system wide trusted root CA by either cloud-init or whatever mechanism the provider will use to set
// up the runner.
CABundle string
// GitHubRunnerGroup is the github runner group in which the newly installed runner should be added to.
GitHubRunnerGroup string
// EnableBootDebug will enable bash debug mode.
EnableBootDebug bool
// ExtraContext is a map of extra context that will be passed to the runner install template.
// This option is useful for situations in which you're supplying your own template and you need
// to pass in information that is not available in the default template.
ExtraContext map[string]string
}
func InstallRunnerScript(installParams InstallRunnerParams, osType params.OSType) ([]byte, error) {
var tpl string
switch osType {
case params.Linux:
tpl = CloudConfigTemplate
case params.Windows:
tpl = WindowsSetupScriptTemplate
default:
return nil, fmt.Errorf("unsupported os type: %s", osType)
func InstallRunnerScript(installParams InstallRunnerParams, osType params.OSType, tpl string) ([]byte, error) {
if tpl == "" {
switch osType {
case params.Linux:
tpl = CloudConfigTemplate
case params.Windows:
tpl = WindowsSetupScriptTemplate
default:
return nil, fmt.Errorf("unsupported os type: %s", osType)
}
}
t, err := template.New("").Parse(tpl)

197
vendor/github.com/cloudbase/garm-provider-common/cloudconfig/util.go generated vendored Normal file
View file

@ -0,0 +1,197 @@
package cloudconfig
import (
"encoding/json"
"fmt"
"sort"
"strings"
"github.com/cloudbase/garm-provider-common/defaults"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/google/go-github/v53/github"
"github.com/pkg/errors"
)
// CloudConfigSpec is a struct that holds extra specs that can be used to customize user data.
type CloudConfigSpec struct {
// RunnerInstallTemplate can be used to override the default runner install template.
// If used, the caller is responsible for the correctness of the template as well as the
// suitability of the template for the target OS.
RunnerInstallTemplate []byte `json:"runner_install_template"`
// PreInstallScripts is a map of pre-install scripts that will be run before the
// runner install script. These will run as root and can be used to prep a generic image
// before we attempt to install the runner. The key of the map is the name of the script
// as it will be written to disk. The value is a byte array with the contents of the script.
//
// These scripts will be added and run in alphabetical order.
//
// On Linux, we will set the executable flag. On Windows, the name matters as Windows looks for an
// extension to determine if the file is an executable or not. In theory this can hold binaries,
// but in most cases this will most likely hold scripts. We do not currenly validate the payload,
// so it's up to the user what they upload here.
// Caution needs to be exercised when using this feature, as the total size of userdata is limited
// on most providers.
PreInstallScripts map[string][]byte `json:"pre_install_scripts"`
// ExtraContext is a map of extra context that will be passed to the runner install template.
ExtraContext map[string]string `json:"extra_context"`
}
func sortMapKeys(m map[string][]byte) []string {
var keys []string
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return keys
}
// GetSpecs returns the cloud config specific extra specs from the bootstrap params.
func GetSpecs(bootstrapParams commonParams.BootstrapInstance) (CloudConfigSpec, error) {
var extraSpecs CloudConfigSpec
if len(bootstrapParams.ExtraSpecs) == 0 {
return extraSpecs, nil
}
if err := json.Unmarshal(bootstrapParams.ExtraSpecs, &extraSpecs); err != nil {
return CloudConfigSpec{}, errors.Wrap(err, "unmarshaling extra specs")
}
if extraSpecs.ExtraContext == nil {
extraSpecs.ExtraContext = map[string]string{}
}
if extraSpecs.PreInstallScripts == nil {
extraSpecs.PreInstallScripts = map[string][]byte{}
}
return extraSpecs, nil
}
// GetRunnerInstallScript returns the runner install script for the given bootstrap params.
// This function will return either the default script for the given OS type or will use the supplied template
// if one is provided.
func GetRunnerInstallScript(bootstrapParams commonParams.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) ([]byte, error) {
if tools.Filename == nil {
return nil, fmt.Errorf("missing tools filename")
}
if tools.DownloadURL == nil {
return nil, fmt.Errorf("missing tools download URL")
}
var tempToken string
if tools.TempDownloadToken != nil {
tempToken = *tools.TempDownloadToken
}
extraSpecs, err := GetSpecs(bootstrapParams)
if err != nil {
return nil, errors.Wrap(err, "getting specs")
}
installRunnerParams := InstallRunnerParams{
FileName: *tools.Filename,
DownloadURL: *tools.DownloadURL,
TempDownloadToken: tempToken,
MetadataURL: bootstrapParams.MetadataURL,
RunnerUsername: defaults.DefaultUser,
RunnerGroup: defaults.DefaultUser,
RepoURL: bootstrapParams.RepoURL,
RunnerName: runnerName,
RunnerLabels: strings.Join(bootstrapParams.Labels, ","),
CallbackURL: bootstrapParams.CallbackURL,
CallbackToken: bootstrapParams.InstanceToken,
GitHubRunnerGroup: bootstrapParams.GitHubRunnerGroup,
ExtraContext: extraSpecs.ExtraContext,
EnableBootDebug: bootstrapParams.UserDataOptions.EnableBootDebug,
}
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
installRunnerParams.CABundle = string(bootstrapParams.CACertBundle)
}
installScript, err := InstallRunnerScript(installRunnerParams, bootstrapParams.OSType, string(extraSpecs.RunnerInstallTemplate))
if err != nil {
return nil, errors.Wrap(err, "generating script")
}
return installScript, nil
}
// GetCloudInitConfig returns the cloud-init specific userdata config. This config can be used on most clouds
// for most Linux machines. The install runner script must be generated separately either by GetRunnerInstallScript()
// or some other means.
func GetCloudInitConfig(bootstrapParams commonParams.BootstrapInstance, installScript []byte) (string, error) {
extraSpecs, err := GetSpecs(bootstrapParams)
if err != nil {
return "", errors.Wrap(err, "getting specs")
}
cloudCfg := NewDefaultCloudInitConfig()
if bootstrapParams.UserDataOptions.DisableUpdatesOnBoot {
cloudCfg.PackageUpgrade = false
cloudCfg.Packages = []string{}
}
for _, pkg := range bootstrapParams.UserDataOptions.ExtraPackages {
cloudCfg.AddPackage(pkg)
}
if len(extraSpecs.PreInstallScripts) > 0 {
names := sortMapKeys(extraSpecs.PreInstallScripts)
for _, name := range names {
script := extraSpecs.PreInstallScripts[name]
cloudCfg.AddFile(script, fmt.Sprintf("/garm-pre-install/%s", name), "root:root", "755")
cloudCfg.AddRunCmd(fmt.Sprintf("/garm-pre-install/%s", name))
}
}
cloudCfg.AddRunCmd("rm -rf /garm-pre-install")
cloudCfg.AddSSHKey(bootstrapParams.SSHKeys...)
cloudCfg.AddFile(installScript, "/install_runner.sh", "root:root", "755")
cloudCfg.AddRunCmd(fmt.Sprintf("su -l -c /install_runner.sh %s", defaults.DefaultUser))
cloudCfg.AddRunCmd("rm -f /install_runner.sh")
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
if err := cloudCfg.AddCACert(bootstrapParams.CACertBundle); err != nil {
return "", errors.Wrap(err, "adding CA cert bundle")
}
}
asStr, err := cloudCfg.Serialize()
if err != nil {
return "", errors.Wrap(err, "creating cloud config")
}
return asStr, nil
}
// GetCloudConfig is a helper function that generates a cloud-init config for Linux and a powershell script for Windows.
// In most cases this function should do, but in situations where a more custom approach is needed, you may need to call
// GetCloudInitConfig() or GetRunnerInstallScript() directly and compose the final userdata in a different way.
// The extra specs PreInstallScripts is only supported on Linux via cloud-init by this function. On some providers, like Azure
// Windows initialization scripts are run by creating a separate CustomScriptExtension resource for each individual script.
// On other clouds it may be different. This function aims to be generic, which is why it only supports the PreInstallScripts
// via cloud-init.
func GetCloudConfig(bootstrapParams commonParams.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) (string, error) {
installScript, err := GetRunnerInstallScript(bootstrapParams, tools, runnerName)
if err != nil {
return "", errors.Wrap(err, "generating script")
}
var asStr string
switch bootstrapParams.OSType {
case commonParams.Linux:
cloudCfg, err := GetCloudInitConfig(bootstrapParams, installScript)
if err != nil {
return "", errors.Wrap(err, "getting cloud init config")
}
return cloudCfg, nil
case commonParams.Windows:
asStr = string(installScript)
default:
return "", fmt.Errorf("unknown os type: %s", bootstrapParams.OSType)
}
return asStr, nil
}

17
vendor/github.com/cloudbase/garm-provider-common/defaults/defaults.go generated vendored Normal file
View file

@ -0,0 +1,17 @@
package defaults
const (
// DefaultUser is the default username that should exist on the instances.
DefaultUser = "runner"
// DefaultUserShell is the shell for the default user.
DefaultUserShell = "/bin/bash"
)
var (
// DefaultUserGroups are the groups the default user will be part of.
DefaultUserGroups = []string{
"sudo", "adm", "cdrom", "dialout",
"dip", "video", "plugdev", "netdev",
"docker", "lxd",
}
)

0
errors/errors.go → vendor/github.com/cloudbase/garm-provider-common/errors/errors.go generated vendored
View file

0
runner/providers/external/execution/commands.go → vendor/github.com/cloudbase/garm-provider-common/execution/commands.go generated vendored
View file

23
runner/providers/external/execution/execution.go → vendor/github.com/cloudbase/garm-provider-common/execution/execution.go generated vendored
View file

@ -4,15 +4,36 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"github.com/cloudbase/garm/params"
gErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/params"
"github.com/mattn/go-isatty"
)
const (
// ExitCodeNotFound is an exit code that indicates a Not Found error
ExitCodeNotFound int = 30
// ExitCodeDuplicate is an exit code that indicates a duplicate error
ExitCodeDuplicate int = 31
)
func ResolveErrorToExitCode(err error) int {
if err != nil {
if errors.Is(err, gErrors.ErrNotFound) {
return ExitCodeNotFound
} else if errors.Is(err, gErrors.ErrDuplicateEntity) {
return ExitCodeDuplicate
}
return 1
}
return 0
}
func GetEnvironment() (Environment, error) {
env := Environment{
Command: ExecutionCommand(os.Getenv("GARM_COMMAND")),

8
runner/providers/external/execution/interface.go → vendor/github.com/cloudbase/garm-provider-common/execution/interface.go generated vendored
View file

@ -3,7 +3,7 @@ package execution
import (
"context"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm-provider-common/params"
)
// ExternalProvider defines an interface that external providers need to implement.
@ -11,13 +11,13 @@ import (
// decouple it, in case it may diverge from native providers.
type ExternalProvider interface {
// CreateInstance creates a new compute instance in the provider.
CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error)
CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.ProviderInstance, error)
// Delete instance will delete the instance in a provider.
DeleteInstance(ctx context.Context, instance string) error
// GetInstance will return details about one instance.
GetInstance(ctx context.Context, instance string) (params.Instance, error)
GetInstance(ctx context.Context, instance string) (params.ProviderInstance, error)
// ListInstances will list all instances for a provider.
ListInstances(ctx context.Context, poolID string) ([]params.Instance, error)
ListInstances(ctx context.Context, poolID string) ([]params.ProviderInstance, error)
// RemoveAllInstances will remove all instances created by this provider.
RemoveAllInstances(ctx context.Context) error
// Stop shuts down the instance.

149
vendor/github.com/cloudbase/garm-provider-common/params/params.go generated vendored Normal file
View file

@ -0,0 +1,149 @@
package params
import (
"encoding/json"
"github.com/google/go-github/v53/github"
)
type (
AddressType string
InstanceStatus string
OSType string
OSArch string
)
const (
Windows OSType = "windows"
Linux OSType = "linux"
Unknown OSType = "unknown"
)
const (
Amd64 OSArch = "amd64"
I386 OSArch = "i386"
Arm64 OSArch = "arm64"
Arm OSArch = "arm"
)
const (
InstanceRunning InstanceStatus = "running"
InstanceStopped InstanceStatus = "stopped"
InstanceError InstanceStatus = "error"
InstancePendingDelete InstanceStatus = "pending_delete"
InstanceDeleting InstanceStatus = "deleting"
InstancePendingCreate InstanceStatus = "pending_create"
InstanceCreating InstanceStatus = "creating"
InstanceStatusUnknown InstanceStatus = "unknown"
)
const (
PublicAddress AddressType = "public"
PrivateAddress AddressType = "private"
)
type UserDataOptions struct {
DisableUpdatesOnBoot bool `json:"disable_updates_on_boot"`
ExtraPackages []string `json:"extra_packages"`
EnableBootDebug bool `json:"enable_boot_debug"`
}
type BootstrapInstance struct {
Name string `json:"name"`
Tools []*github.RunnerApplicationDownload `json:"tools"`
// RepoURL is the URL the github runner agent needs to configure itself.
RepoURL string `json:"repo_url"`
// CallbackUrl is the URL where the instance can send a post, signaling
// progress or status.
CallbackURL string `json:"callback-url"`
// MetadataURL is the URL where instances can fetch information needed to set themselves up.
MetadataURL string `json:"metadata-url"`
// InstanceToken is the token that needs to be set by the instance in the headers
// in order to send updated back to the garm via CallbackURL.
InstanceToken string `json:"instance-token"`
// SSHKeys are the ssh public keys we may want to inject inside the runners, if the
// provider supports it.
SSHKeys []string `json:"ssh-keys"`
// ExtraSpecs is an opaque raw json that gets sent to the provider
// as part of the bootstrap params for instances. It can contain
// any kind of data needed by providers. The contents of this field means
// nothing to garm itself. We don't act on the information in this field at
// all. We only validate that it's a proper json.
ExtraSpecs json.RawMessage `json:"extra_specs,omitempty"`
// GitHubRunnerGroup is the github runner group in which the newly installed runner
// should be added to. The runner group must be created by someone with access to the
// enterprise.
GitHubRunnerGroup string `json:"github-runner-group"`
// CACertBundle is a CA certificate bundle which will be sent to instances and which
// will tipically be installed as a system wide trusted root CA. by either cloud-init
// or whatever mechanism the provider will use to set up the runner.
CACertBundle []byte `json:"ca-cert-bundle"`
// OSArch is the target OS CPU architecture of the runner.
OSArch OSArch `json:"arch"`
// OSType is the target OS platform of the runner (windows, linux).
OSType OSType `json:"os_type"`
// Flavor is the platform specific abstraction that defines what resources will be allocated
// to the runner (CPU, RAM, disk space, etc). This field is meaningful to the provider which
// handles the actual creation.
Flavor string `json:"flavor"`
// Image is the platform specific identifier of the operating system template that will be used
// to spin up a new machine.
Image string `json:"image"`
// Labels are a list of github runner labels that will be added to the runner.
Labels []string `json:"labels"`
// PoolID is the ID of the garm pool to which this runner belongs.
PoolID string `json:"pool_id"`
// UserDataOptions are the options for the user data generation.
UserDataOptions UserDataOptions `json:"user_data_options"`
}
type Address struct {
Address string `json:"address"`
Type AddressType `json:"type"`
}
type ProviderInstance struct {
// PeoviderID is the unique ID the provider associated
// with the compute instance. We use this to identify the
// instance in the provider.
ProviderID string `json:"provider_id,omitempty"`
// Name is the name associated with an instance. Depending on
// the provider, this may or may not be useful in the context of
// the provider, but we can use it internally to identify the
// instance.
Name string `json:"name,omitempty"`
// OSType is the operating system type. For now, only Linux and
// Windows are supported.
OSType OSType `json:"os_type,omitempty"`
// OSName is the name of the OS. Eg: ubuntu, centos, etc.
OSName string `json:"os_name,omitempty"`
// OSVersion is the version of the operating system.
OSVersion string `json:"os_version,omitempty"`
// OSArch is the operating system architecture.
OSArch OSArch `json:"os_arch,omitempty"`
// Addresses is a list of IP addresses the provider reports
// for this instance.
Addresses []Address `json:"addresses,omitempty"`
// Status is the status of the instance inside the provider (eg: running, stopped, etc)
Status InstanceStatus `json:"status,omitempty"`
// ProviderFault holds any error messages captured from the IaaS provider that is
// responsible for managing the lifecycle of the runner.
ProviderFault []byte `json:"provider_fault,omitempty"`
}

0
util/exec/exec.go → vendor/github.com/cloudbase/garm-provider-common/util/exec/exec.go generated vendored
View file

0
util/exec/exec_nix.go → vendor/github.com/cloudbase/garm-provider-common/util/exec/exec_nix.go generated vendored
View file

0
util/exec/exec_windows.go → vendor/github.com/cloudbase/garm-provider-common/util/exec/exec_windows.go generated vendored
View file

387
vendor/github.com/cloudbase/garm-provider-common/util/util.go generated vendored Normal file
View file

@ -0,0 +1,387 @@
// Copyright 2023 Cloudbase Solutions SRL
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package util
import (
"bytes"
"compress/gzip"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/base64"
"encoding/binary"
"fmt"
"io"
"math/big"
"net/http"
"os"
"path"
"regexp"
"strings"
"unicode"
"unicode/utf16"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/google/go-github/v53/github"
"github.com/google/uuid"
gorillaHandlers "github.com/gorilla/handlers"
"github.com/pkg/errors"
"github.com/teris-io/shortid"
"golang.org/x/crypto/bcrypt"
lumberjack "gopkg.in/natefinch/lumberjack.v2"
)
const alphanumeric = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
// From: https://www.alexedwards.net/blog/validation-snippets-for-go#email-validation
var rxEmail = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$")
var (
OSToOSTypeMap map[string]commonParams.OSType = map[string]commonParams.OSType{
"almalinux": commonParams.Linux,
"alma": commonParams.Linux,
"alpine": commonParams.Linux,
"archlinux": commonParams.Linux,
"arch": commonParams.Linux,
"centos": commonParams.Linux,
"ubuntu": commonParams.Linux,
"rhel": commonParams.Linux,
"suse": commonParams.Linux,
"opensuse": commonParams.Linux,
"fedora": commonParams.Linux,
"debian": commonParams.Linux,
"flatcar": commonParams.Linux,
"gentoo": commonParams.Linux,
"rockylinux": commonParams.Linux,
"rocky": commonParams.Linux,
"windows": commonParams.Windows,
}
githubArchMapping map[string]string = map[string]string{
"x86_64": "x64",
"amd64": "x64",
"armv7l": "arm",
"aarch64": "arm64",
"x64": "x64",
"arm": "arm",
"arm64": "arm64",
}
githubOSTypeMap map[string]string = map[string]string{
"linux": "linux",
"windows": "win",
}
//
githubOSTag = map[commonParams.OSType]string{
commonParams.Linux: "Linux",
commonParams.Windows: "Windows",
}
)
// ResolveToGithubArch returns the cpu architecture as it is defined in the GitHub
// tools download list. We use it to find the proper tools for the OS/Arch combo we're
// deploying.
func ResolveToGithubArch(arch string) (string, error) {
ghArch, ok := githubArchMapping[arch]
if !ok {
return "", runnerErrors.NewNotFoundError("arch %s is unknown", arch)
}
return ghArch, nil
}
// ResolveToGithubArch returns the OS type as it is defined in the GitHub
// tools download list. We use it to find the proper tools for the OS/Arch combo we're
// deploying.
func ResolveToGithubOSType(osType string) (string, error) {
ghOS, ok := githubOSTypeMap[osType]
if !ok {
return "", runnerErrors.NewNotFoundError("os %s is unknown", osType)
}
return ghOS, nil
}
// ResolveToGithubTag returns the default OS tag that self hosted runners automatically
// (and forcefully) adds to every runner that gets deployed. We need to keep track of those
// tags internally as well.
func ResolveToGithubTag(os commonParams.OSType) (string, error) {
ghOS, ok := githubOSTag[os]
if !ok {
return "", runnerErrors.NewNotFoundError("os %s is unknown", os)
}
return ghOS, nil
}
// IsValidEmail returs a bool indicating if an email is valid
func IsValidEmail(email string) bool {
if len(email) > 254 || !rxEmail.MatchString(email) {
return false
}
return true
}
func IsAlphanumeric(s string) bool {
for _, r := range s {
if !unicode.IsLetter(r) && !unicode.IsNumber(r) {
return false
}
}
return true
}
// GetLoggingWriter returns a new io.Writer suitable for logging.
func GetLoggingWriter(logFile string) (io.Writer, error) {
var writer io.Writer = os.Stdout
if logFile != "" {
dirname := path.Dir(logFile)
if _, err := os.Stat(dirname); err != nil {
if !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to create log folder")
}
if err := os.MkdirAll(dirname, 0o711); err != nil {
return nil, fmt.Errorf("failed to create log folder")
}
}
writer = &lumberjack.Logger{
Filename: logFile,
MaxSize: 500, // megabytes
MaxBackups: 3,
MaxAge: 28, // days
Compress: true, // disabled by default
}
}
return writer, nil
}
func ConvertFileToBase64(file string) (string, error) {
bytes, err := os.ReadFile(file)
if err != nil {
return "", errors.Wrap(err, "reading file")
}
return base64.StdEncoding.EncodeToString(bytes), nil
}
func OSToOSType(os string) (commonParams.OSType, error) {
osType, ok := OSToOSTypeMap[strings.ToLower(os)]
if !ok {
return commonParams.Unknown, fmt.Errorf("no OS to OS type mapping for %s", os)
}
return osType, nil
}
func GetTools(osType commonParams.OSType, osArch commonParams.OSArch, tools []*github.RunnerApplicationDownload) (github.RunnerApplicationDownload, error) {
// Validate image OS. Linux only for now.
switch osType {
case commonParams.Linux:
case commonParams.Windows:
default:
return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS type: %s", osType)
}
switch osArch {
case commonParams.Amd64:
case commonParams.Arm:
case commonParams.Arm64:
default:
return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS arch: %s", osArch)
}
// Find tools for OS/Arch.
for _, tool := range tools {
if tool == nil {
continue
}
if tool.OS == nil || tool.Architecture == nil {
continue
}
ghArch, err := ResolveToGithubArch(string(osArch))
if err != nil {
continue
}
ghOS, err := ResolveToGithubOSType(string(osType))
if err != nil {
continue
}
if *tool.Architecture == ghArch && *tool.OS == ghOS {
return *tool, nil
}
}
return github.RunnerApplicationDownload{}, fmt.Errorf("failed to find tools for OS %s and arch %s", osType, osArch)
}
// GetRandomString returns a secure random string
func GetRandomString(n int) (string, error) {
data := make([]byte, n)
_, err := rand.Read(data)
if err != nil {
return "", errors.Wrap(err, "getting random data")
}
for i, b := range data {
data[i] = alphanumeric[b%byte(len(alphanumeric))]
}
return string(data), nil
}
func Aes256EncodeString(target string, passphrase string) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
toEncrypt := []byte(target)
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return nil, errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, errors.Wrap(err, "creating new aead")
}
nonce := make([]byte, aesgcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, errors.Wrap(err, "creating nonce")
}
ciphertext := aesgcm.Seal(nonce, nonce, toEncrypt, nil)
return ciphertext, nil
}
func Aes256DecodeString(target []byte, passphrase string) (string, error) {
if len(passphrase) != 32 {
return "", fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return "", errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return "", errors.Wrap(err, "creating new aead")
}
nonceSize := aesgcm.NonceSize()
if len(target) < nonceSize {
return "", fmt.Errorf("failed to decrypt text")
}
nonce, ciphertext := target[:nonceSize], target[nonceSize:]
plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", fmt.Errorf("failed to decrypt text")
}
return string(plaintext), nil
}
// PaswsordToBcrypt returns a bcrypt hash of the specified password using the default cost
func PaswsordToBcrypt(password string) (string, error) {
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return "", fmt.Errorf("failed to hash password")
}
return string(hashedPassword), nil
}
func NewLoggingMiddleware(writer io.Writer) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return gorillaHandlers.CombinedLoggingHandler(writer, next)
}
}
func SanitizeLogEntry(entry string) string {
return strings.Replace(strings.Replace(entry, "\n", "", -1), "\r", "", -1)
}
func toBase62(uuid []byte) string {
var i big.Int
i.SetBytes(uuid[:])
return i.Text(62)
}
func NewID() string {
short, err := shortid.Generate()
if err == nil {
return toBase62([]byte(short))
}
newUUID := uuid.New()
return toBase62(newUUID[:])
}
func UTF16FromString(s string) ([]uint16, error) {
buf := make([]uint16, 0, len(s)*2+1)
for _, r := range s {
buf = utf16.AppendRune(buf, r)
}
return utf16.AppendRune(buf, '\x00'), nil
}
func UTF16ToString(s []uint16) string {
for i, v := range s {
if v == 0 {
s = s[0:i]
break
}
}
return string(utf16.Decode(s))
}
func Uint16ToByteArray(u []uint16) []byte {
ret := make([]byte, (len(u)-1)*2)
for i := 0; i < len(u)-1; i++ {
binary.LittleEndian.PutUint16(ret[i*2:], uint16(u[i]))
}
return ret
}
func UTF16EncodedByteArrayFromString(s string) ([]byte, error) {
asUint16, err := UTF16FromString(s)
if err != nil {
return nil, fmt.Errorf("failed to encode to uint16: %w", err)
}
asBytes := Uint16ToByteArray(asUint16)
return asBytes, nil
}
func CompressData(data []byte) ([]byte, error) {
var b bytes.Buffer
gz := gzip.NewWriter(&b)
_, err := gz.Write(data)
if err != nil {
return nil, fmt.Errorf("failed to compress data: %w", err)
}
if err = gz.Flush(); err != nil {
return nil, fmt.Errorf("failed to flush buffer: %w", err)
}
if err = gz.Close(); err != nil {
return nil, fmt.Errorf("failed to close buffer: %w", err)
}
return b.Bytes(), nil
}

11
vendor/modules.txt vendored
View file

@ -34,6 +34,15 @@ github.com/cespare/xxhash/v2
# github.com/chzyer/readline v1.5.1
## explicit; go 1.15
github.com/chzyer/readline
# github.com/cloudbase/garm-provider-common v0.0.0-20230724114054-7aa0a3dfbce0
## explicit; go 1.20
github.com/cloudbase/garm-provider-common/cloudconfig
github.com/cloudbase/garm-provider-common/defaults
github.com/cloudbase/garm-provider-common/errors
github.com/cloudbase/garm-provider-common/execution
github.com/cloudbase/garm-provider-common/params
github.com/cloudbase/garm-provider-common/util
github.com/cloudbase/garm-provider-common/util/exec
# github.com/cloudflare/circl v1.3.3
## explicit; go 1.19
github.com/cloudflare/circl/dh/x25519
@ -219,7 +228,7 @@ github.com/mailru/easyjson/jwriter
github.com/manifoldco/promptui
github.com/manifoldco/promptui/list
github.com/manifoldco/promptui/screenbuf
# github.com/mattn/go-isatty v0.0.18
# github.com/mattn/go-isatty v0.0.19
## explicit; go 1.15
github.com/mattn/go-isatty
# github.com/mattn/go-runewidth v0.0.14