diff --git a/apiserver/controllers/controllers.go b/apiserver/controllers/controllers.go index ae9d3b41..2c1fd42a 100644 --- a/apiserver/controllers/controllers.go +++ b/apiserver/controllers/controllers.go @@ -22,12 +22,12 @@ import ( "strings" gErrors "github.com/cloudbase/garm-provider-common/errors" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/apiserver/params" "github.com/cloudbase/garm/auth" "github.com/cloudbase/garm/metrics" runnerParams "github.com/cloudbase/garm/params" "github.com/cloudbase/garm/runner" - "github.com/cloudbase/garm/util" wsWriter "github.com/cloudbase/garm/websocket" "github.com/gorilla/websocket" diff --git a/apiserver/routers/routers.go b/apiserver/routers/routers.go index 77ff0cd4..55e42684 100644 --- a/apiserver/routers/routers.go +++ b/apiserver/routers/routers.go @@ -54,9 +54,9 @@ import ( "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/apiserver/controllers" "github.com/cloudbase/garm/auth" - "github.com/cloudbase/garm/util" ) func WithMetricsRouter(parentRouter *mux.Router, disableAuth bool, metricsMiddlerware auth.Middleware) *mux.Router { diff --git a/auth/auth.go b/auth/auth.go index 5097947e..d912bee6 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -19,10 +19,10 @@ import ( "time" runnerErrors "github.com/cloudbase/garm-provider-common/errors" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/config" "github.com/cloudbase/garm/database/common" "github.com/cloudbase/garm/params" - "github.com/cloudbase/garm/util" "github.com/golang-jwt/jwt" "github.com/nbutton23/zxcvbn-go" diff --git a/cmd/garm-cli/cmd/log.go b/cmd/garm-cli/cmd/log.go index 4b7e031c..9e6669ce 100644 --- a/cmd/garm-cli/cmd/log.go +++ b/cmd/garm-cli/cmd/log.go @@ -10,8 +10,8 @@ import ( "os/signal" "time" + "github.com/cloudbase/garm-provider-common/util" apiParams "github.com/cloudbase/garm/apiserver/params" - "github.com/cloudbase/garm/util" "github.com/gorilla/websocket" "github.com/spf13/cobra" diff --git a/cmd/garm/main.go b/cmd/garm/main.go index 1b34f5c0..04a45f0b 100644 --- a/cmd/garm/main.go +++ b/cmd/garm/main.go @@ -27,6 +27,7 @@ import ( "syscall" "time" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/apiserver/controllers" "github.com/cloudbase/garm/apiserver/routers" "github.com/cloudbase/garm/auth" @@ -35,7 +36,6 @@ import ( "github.com/cloudbase/garm/database/common" "github.com/cloudbase/garm/metrics" "github.com/cloudbase/garm/runner" - "github.com/cloudbase/garm/util" "github.com/cloudbase/garm/util/appdefaults" "github.com/cloudbase/garm/websocket" lumberjack "gopkg.in/natefinch/lumberjack.v2" @@ -84,7 +84,7 @@ func main() { log.Fatalf("Fetching config: %+v", err) } - logWriter, err := util.GetLoggingWriter(cfg) + logWriter, err := util.GetLoggingWriter(cfg.Default.LogFile) if err != nil { log.Fatalf("fetching log writer: %+v", err) } diff --git a/database/sql/enterprise.go b/database/sql/enterprise.go index 0b3da280..0a1ea81b 100644 --- a/database/sql/enterprise.go +++ b/database/sql/enterprise.go @@ -4,8 +4,8 @@ import ( "context" runnerErrors "github.com/cloudbase/garm-provider-common/errors" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/params" - "github.com/cloudbase/garm/util" "github.com/google/uuid" "github.com/pkg/errors" diff --git a/database/sql/organizations.go b/database/sql/organizations.go index efacf671..c0a48d4d 100644 --- a/database/sql/organizations.go +++ b/database/sql/organizations.go @@ -19,8 +19,8 @@ import ( "fmt" runnerErrors "github.com/cloudbase/garm-provider-common/errors" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/params" - "github.com/cloudbase/garm/util" "github.com/google/uuid" "github.com/pkg/errors" diff --git a/database/sql/repositories.go b/database/sql/repositories.go index 2d935e74..007a2f6e 100644 --- a/database/sql/repositories.go +++ b/database/sql/repositories.go @@ -19,8 +19,8 @@ import ( "fmt" runnerErrors "github.com/cloudbase/garm-provider-common/errors" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/params" - "github.com/cloudbase/garm/util" "github.com/google/uuid" "github.com/pkg/errors" diff --git a/database/sql/users.go b/database/sql/users.go index b3c11c02..78922b80 100644 --- a/database/sql/users.go +++ b/database/sql/users.go @@ -19,8 +19,8 @@ import ( "fmt" runnerErrors "github.com/cloudbase/garm-provider-common/errors" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/params" - "github.com/cloudbase/garm/util" "github.com/pkg/errors" "gorm.io/gorm" diff --git a/database/sql/util.go b/database/sql/util.go index 5c0552af..fa6706d2 100644 --- a/database/sql/util.go +++ b/database/sql/util.go @@ -18,8 +18,8 @@ import ( "encoding/json" "fmt" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/params" - "github.com/cloudbase/garm/util" "github.com/pkg/errors" "gorm.io/datatypes" diff --git a/go.mod b/go.mod index 3733583f..0be8ca73 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.2 - github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 golang.org/x/crypto v0.7.0 golang.org/x/oauth2 v0.8.0 golang.org/x/sync v0.1.0 @@ -95,6 +94,7 @@ require ( github.com/sirupsen/logrus v1.9.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.0 // indirect + github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 // indirect go.mongodb.org/mongo-driver v1.11.3 // indirect go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect diff --git a/runner/common/mocks/Provider.go b/runner/common/mocks/Provider.go index 7f9d801f..ca8bddcf 100644 --- a/runner/common/mocks/Provider.go +++ b/runner/common/mocks/Provider.go @@ -5,8 +5,10 @@ package mocks import ( context "context" - params "github.com/cloudbase/garm/params" + garm_provider_commonparams "github.com/cloudbase/garm-provider-common/params" mock "github.com/stretchr/testify/mock" + + params "github.com/cloudbase/garm/params" ) // Provider is an autogenerated mock type for the Provider type @@ -29,21 +31,21 @@ func (_m *Provider) AsParams() params.Provider { } // CreateInstance provides a mock function with given fields: ctx, bootstrapParams -func (_m *Provider) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) { +func (_m *Provider) CreateInstance(ctx context.Context, bootstrapParams garm_provider_commonparams.BootstrapInstance) (params.Instance, error) { ret := _m.Called(ctx, bootstrapParams) var r0 params.Instance var r1 error - if rf, ok := ret.Get(0).(func(context.Context, params.BootstrapInstance) (params.Instance, error)); ok { + if rf, ok := ret.Get(0).(func(context.Context, garm_provider_commonparams.BootstrapInstance) (params.Instance, error)); ok { return rf(ctx, bootstrapParams) } - if rf, ok := ret.Get(0).(func(context.Context, params.BootstrapInstance) params.Instance); ok { + if rf, ok := ret.Get(0).(func(context.Context, garm_provider_commonparams.BootstrapInstance) params.Instance); ok { r0 = rf(ctx, bootstrapParams) } else { r0 = ret.Get(0).(params.Instance) } - if rf, ok := ret.Get(1).(func(context.Context, params.BootstrapInstance) error); ok { + if rf, ok := ret.Get(1).(func(context.Context, garm_provider_commonparams.BootstrapInstance) error); ok { r1 = rf(ctx, bootstrapParams) } else { r1 = ret.Error(1) diff --git a/runner/common/provider.go b/runner/common/provider.go index 2def7385..f62dc77b 100644 --- a/runner/common/provider.go +++ b/runner/common/provider.go @@ -17,13 +17,14 @@ package common import ( "context" + commonParams "github.com/cloudbase/garm-provider-common/params" "github.com/cloudbase/garm/params" ) //go:generate mockery --all type Provider interface { // CreateInstance creates a new compute instance in the provider. - CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) + CreateInstance(ctx context.Context, bootstrapParams commonParams.BootstrapInstance) (params.Instance, error) // Delete instance will delete the instance in a provider. DeleteInstance(ctx context.Context, instance string) error // GetInstance will return details about one instance. diff --git a/runner/pool/pool.go b/runner/pool/pool.go index 7a3256cf..e0dd3a03 100644 --- a/runner/pool/pool.go +++ b/runner/pool/pool.go @@ -28,11 +28,11 @@ import ( commonParams "github.com/cloudbase/garm-provider-common/params" runnerErrors "github.com/cloudbase/garm-provider-common/errors" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/auth" dbCommon "github.com/cloudbase/garm/database/common" "github.com/cloudbase/garm/params" "github.com/cloudbase/garm/runner/common" - "github.com/cloudbase/garm/util" "github.com/google/go-github/v53/github" "github.com/google/uuid" @@ -752,7 +752,7 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error return errors.Wrap(err, "fetching instance jwt token") } - bootstrapArgs := params.BootstrapInstance{ + bootstrapArgs := commonParams.BootstrapInstance{ Name: instance.Name, Tools: r.tools, RepoURL: r.helper.GithubURL(), diff --git a/runner/providers/external/external.go b/runner/providers/external/external.go index 7c97a4a2..a1a9c3d8 100644 --- a/runner/providers/external/external.go +++ b/runner/providers/external/external.go @@ -9,7 +9,7 @@ import ( "github.com/cloudbase/garm-provider-common/execution" - providerParams "github.com/cloudbase/garm-provider-common/params" + commonParams "github.com/cloudbase/garm-provider-common/params" garmErrors "github.com/cloudbase/garm-provider-common/errors" "github.com/cloudbase/garm/config" @@ -46,7 +46,7 @@ type external struct { execPath string } -func (e *external) validateResult(inst providerParams.ProviderInstance) error { +func (e *external) validateResult(inst commonParams.ProviderInstance) error { if inst.ProviderID == "" { return garmErrors.NewProviderError("missing provider ID") } @@ -67,7 +67,7 @@ func (e *external) validateResult(inst providerParams.ProviderInstance) error { } // CreateInstance creates a new compute instance in the provider. -func (e *external) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) { +func (e *external) CreateInstance(ctx context.Context, bootstrapParams commonParams.BootstrapInstance) (params.Instance, error) { asEnv := []string{ fmt.Sprintf("GARM_COMMAND=%s", execution.CreateInstanceCommand), fmt.Sprintf("GARM_CONTROLLER_ID=%s", e.controllerID), @@ -85,7 +85,7 @@ func (e *external) CreateInstance(ctx context.Context, bootstrapParams params.Bo return params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err) } - var param providerParams.ProviderInstance + var param commonParams.ProviderInstance if err := json.Unmarshal(out, ¶m); err != nil { return params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err) } @@ -135,7 +135,7 @@ func (e *external) GetInstance(ctx context.Context, instance string) (params.Ins return params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err) } - var param providerParams.ProviderInstance + var param commonParams.ProviderInstance if err := json.Unmarshal(out, ¶m); err != nil { return params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err) } @@ -161,7 +161,7 @@ func (e *external) ListInstances(ctx context.Context, poolID string) ([]params.I return []params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err) } - var param []providerParams.ProviderInstance + var param []commonParams.ProviderInstance if err := json.Unmarshal(out, ¶m); err != nil { return []params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err) } diff --git a/runner/providers/lxd/lxd.go b/runner/providers/lxd/lxd.go index f64ee457..530ec74b 100644 --- a/runner/providers/lxd/lxd.go +++ b/runner/providers/lxd/lxd.go @@ -22,10 +22,10 @@ import ( "time" runnerErrors "github.com/cloudbase/garm-provider-common/errors" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/config" "github.com/cloudbase/garm/params" "github.com/cloudbase/garm/runner/common" - "github.com/cloudbase/garm/util" "github.com/google/go-github/v53/github" lxd "github.com/lxc/lxd/client" @@ -212,7 +212,7 @@ func (l *LXD) secureBootEnabled() string { return "false" } -func (l *LXD) getCreateInstanceArgs(bootstrapParams params.BootstrapInstance, specs extraSpecs) (api.InstancesPost, error) { +func (l *LXD) getCreateInstanceArgs(bootstrapParams commonParams.BootstrapInstance, specs extraSpecs) (api.InstancesPost, error) { if bootstrapParams.Name == "" { return api.InstancesPost{}, runnerErrors.NewBadRequestError("missing name") } @@ -315,7 +315,7 @@ func (l *LXD) launchInstance(createArgs api.InstancesPost) error { } // CreateInstance creates a new compute instance in the provider. -func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) { +func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams commonParams.BootstrapInstance) (params.Instance, error) { extraSpecs, err := parseExtraSpecsFromBootstrapParams(bootstrapParams) if err != nil { return params.Instance{}, errors.Wrap(err, "parsing extra specs") diff --git a/runner/providers/lxd/specs.go b/runner/providers/lxd/specs.go index 202473b7..0471a536 100644 --- a/runner/providers/lxd/specs.go +++ b/runner/providers/lxd/specs.go @@ -17,7 +17,7 @@ package lxd import ( "encoding/json" - "github.com/cloudbase/garm/params" + commonParams "github.com/cloudbase/garm-provider-common/params" "github.com/pkg/errors" ) @@ -26,7 +26,7 @@ type extraSpecs struct { ExtraPackages []string `json:"extra_packages"` } -func parseExtraSpecsFromBootstrapParams(bootstrapParams params.BootstrapInstance) (extraSpecs, error) { +func parseExtraSpecsFromBootstrapParams(bootstrapParams commonParams.BootstrapInstance) (extraSpecs, error) { specs := extraSpecs{} if bootstrapParams.ExtraSpecs == nil { return specs, nil diff --git a/runner/providers/lxd/util.go b/runner/providers/lxd/util.go index fe6c517c..aa70c8f2 100644 --- a/runner/providers/lxd/util.go +++ b/runner/providers/lxd/util.go @@ -27,9 +27,9 @@ import ( commonParams "github.com/cloudbase/garm-provider-common/params" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/config" "github.com/cloudbase/garm/params" - "github.com/cloudbase/garm/util" "github.com/juju/clock" "github.com/juju/retry" diff --git a/runner/runner.go b/runner/runner.go index 226aa9f0..b8120adc 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -32,6 +32,7 @@ import ( commonParams "github.com/cloudbase/garm-provider-common/params" runnerErrors "github.com/cloudbase/garm-provider-common/errors" + "github.com/cloudbase/garm-provider-common/util" "github.com/cloudbase/garm/auth" "github.com/cloudbase/garm/config" dbCommon "github.com/cloudbase/garm/database/common" @@ -39,7 +40,6 @@ import ( "github.com/cloudbase/garm/runner/common" "github.com/cloudbase/garm/runner/pool" "github.com/cloudbase/garm/runner/providers" - "github.com/cloudbase/garm/util" "golang.org/x/sync/errgroup" "github.com/google/uuid" diff --git a/util/util.go b/util/util.go index 4600fc08..db2b86f0 100644 --- a/util/util.go +++ b/util/util.go @@ -15,188 +15,20 @@ package util import ( - "bytes" - "compress/gzip" "context" - "crypto/aes" - "crypto/cipher" - "crypto/rand" "crypto/tls" "crypto/x509" - "encoding/base64" - "encoding/binary" "fmt" - "io" - "math/big" "net/http" - "os" - "path" - "regexp" - "strings" - "unicode" - "unicode/utf16" - "github.com/cloudbase/garm-provider-common/cloudconfig" - runnerErrors "github.com/cloudbase/garm-provider-common/errors" - "github.com/cloudbase/garm/config" "github.com/cloudbase/garm/params" "github.com/cloudbase/garm/runner/common" - "github.com/cloudbase/garm-provider-common/defaults" - commonParams "github.com/cloudbase/garm-provider-common/params" - "github.com/google/go-github/v53/github" - "github.com/google/uuid" - gorillaHandlers "github.com/gorilla/handlers" "github.com/pkg/errors" - "github.com/teris-io/shortid" - "golang.org/x/crypto/bcrypt" "golang.org/x/oauth2" - lumberjack "gopkg.in/natefinch/lumberjack.v2" ) -const alphanumeric = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" - -// From: https://www.alexedwards.net/blog/validation-snippets-for-go#email-validation -var rxEmail = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$") - -var ( - OSToOSTypeMap map[string]commonParams.OSType = map[string]commonParams.OSType{ - "almalinux": commonParams.Linux, - "alma": commonParams.Linux, - "alpine": commonParams.Linux, - "archlinux": commonParams.Linux, - "arch": commonParams.Linux, - "centos": commonParams.Linux, - "ubuntu": commonParams.Linux, - "rhel": commonParams.Linux, - "suse": commonParams.Linux, - "opensuse": commonParams.Linux, - "fedora": commonParams.Linux, - "debian": commonParams.Linux, - "flatcar": commonParams.Linux, - "gentoo": commonParams.Linux, - "rockylinux": commonParams.Linux, - "rocky": commonParams.Linux, - "windows": commonParams.Windows, - } - - githubArchMapping map[string]string = map[string]string{ - "x86_64": "x64", - "amd64": "x64", - "armv7l": "arm", - "aarch64": "arm64", - "x64": "x64", - "arm": "arm", - "arm64": "arm64", - } - - githubOSTypeMap map[string]string = map[string]string{ - "linux": "linux", - "windows": "win", - } - - // - githubOSTag = map[commonParams.OSType]string{ - commonParams.Linux: "Linux", - commonParams.Windows: "Windows", - } -) - -// ResolveToGithubArch returns the cpu architecture as it is defined in the GitHub -// tools download list. We use it to find the proper tools for the OS/Arch combo we're -// deploying. -func ResolveToGithubArch(arch string) (string, error) { - ghArch, ok := githubArchMapping[arch] - if !ok { - return "", runnerErrors.NewNotFoundError("arch %s is unknown", arch) - } - - return ghArch, nil -} - -// ResolveToGithubArch returns the OS type as it is defined in the GitHub -// tools download list. We use it to find the proper tools for the OS/Arch combo we're -// deploying. -func ResolveToGithubOSType(osType string) (string, error) { - ghOS, ok := githubOSTypeMap[osType] - if !ok { - return "", runnerErrors.NewNotFoundError("os %s is unknown", osType) - } - - return ghOS, nil -} - -// ResolveToGithubTag returns the default OS tag that self hosted runners automatically -// (and forcefully) adds to every runner that gets deployed. We need to keep track of those -// tags internally as well. -func ResolveToGithubTag(os commonParams.OSType) (string, error) { - ghOS, ok := githubOSTag[os] - if !ok { - return "", runnerErrors.NewNotFoundError("os %s is unknown", os) - } - - return ghOS, nil -} - -// IsValidEmail returs a bool indicating if an email is valid -func IsValidEmail(email string) bool { - if len(email) > 254 || !rxEmail.MatchString(email) { - return false - } - return true -} - -func IsAlphanumeric(s string) bool { - for _, r := range s { - if !unicode.IsLetter(r) && !unicode.IsNumber(r) { - return false - } - } - return true -} - -// GetLoggingWriter returns a new io.Writer suitable for logging. -func GetLoggingWriter(cfg *config.Config) (io.Writer, error) { - var writer io.Writer = os.Stdout - if cfg.Default.LogFile != "" { - dirname := path.Dir(cfg.Default.LogFile) - if _, err := os.Stat(dirname); err != nil { - if !os.IsNotExist(err) { - return nil, fmt.Errorf("failed to create log folder") - } - if err := os.MkdirAll(dirname, 0o711); err != nil { - return nil, fmt.Errorf("failed to create log folder") - } - } - writer = &lumberjack.Logger{ - Filename: cfg.Default.LogFile, - MaxSize: 500, // megabytes - MaxBackups: 3, - MaxAge: 28, // days - Compress: true, // disabled by default - } - } - return writer, nil -} - -func ConvertFileToBase64(file string) (string, error) { - bytes, err := os.ReadFile(file) - if err != nil { - return "", errors.Wrap(err, "reading file") - } - - return base64.StdEncoding.EncodeToString(bytes), nil -} - -func OSToOSType(os string) (commonParams.OSType, error) { - osType, ok := OSToOSTypeMap[strings.ToLower(os)] - if !ok { - return commonParams.Unknown, fmt.Errorf("no OS to OS type mapping for %s", os) - } - return osType, nil -} - func GithubClient(ctx context.Context, token string, credsDetails params.GithubCredentials) (common.GithubClient, common.GithubEnterpriseClient, error) { var roots *x509.CertPool if credsDetails.CABundle != nil && len(credsDetails.CABundle) > 0 { @@ -226,274 +58,3 @@ func GithubClient(ctx context.Context, token string, credsDetails params.GithubC return ghClient.Actions, ghClient.Enterprise, nil } - -func GetCloudConfig(bootstrapParams params.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) (string, error) { - if tools.Filename == nil { - return "", fmt.Errorf("missing tools filename") - } - - if tools.DownloadURL == nil { - return "", fmt.Errorf("missing tools download URL") - } - - var tempToken string - if tools.TempDownloadToken != nil { - tempToken = *tools.TempDownloadToken - } - - installRunnerParams := cloudconfig.InstallRunnerParams{ - FileName: *tools.Filename, - DownloadURL: *tools.DownloadURL, - TempDownloadToken: tempToken, - MetadataURL: bootstrapParams.MetadataURL, - RunnerUsername: defaults.DefaultUser, - RunnerGroup: defaults.DefaultUser, - RepoURL: bootstrapParams.RepoURL, - RunnerName: runnerName, - RunnerLabels: strings.Join(bootstrapParams.Labels, ","), - CallbackURL: bootstrapParams.CallbackURL, - CallbackToken: bootstrapParams.InstanceToken, - GitHubRunnerGroup: bootstrapParams.GitHubRunnerGroup, - } - if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 { - installRunnerParams.CABundle = string(bootstrapParams.CACertBundle) - } - - installScript, err := cloudconfig.InstallRunnerScript(installRunnerParams, bootstrapParams.OSType) - if err != nil { - return "", errors.Wrap(err, "generating script") - } - - var asStr string - switch bootstrapParams.OSType { - case commonParams.Linux: - cloudCfg := cloudconfig.NewDefaultCloudInitConfig() - - if bootstrapParams.UserDataOptions.DisableUpdatesOnBoot { - cloudCfg.PackageUpgrade = false - cloudCfg.Packages = []string{} - } - for _, pkg := range bootstrapParams.UserDataOptions.ExtraPackages { - cloudCfg.AddPackage(pkg) - } - - cloudCfg.AddSSHKey(bootstrapParams.SSHKeys...) - cloudCfg.AddFile(installScript, "/install_runner.sh", "root:root", "755") - cloudCfg.AddRunCmd(fmt.Sprintf("su -l -c /install_runner.sh %s", defaults.DefaultUser)) - cloudCfg.AddRunCmd("rm -f /install_runner.sh") - if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 { - if err := cloudCfg.AddCACert(bootstrapParams.CACertBundle); err != nil { - return "", errors.Wrap(err, "adding CA cert bundle") - } - } - var err error - asStr, err = cloudCfg.Serialize() - if err != nil { - return "", errors.Wrap(err, "creating cloud config") - } - case commonParams.Windows: - asStr = string(installScript) - default: - return "", fmt.Errorf("unknown os type: %s", bootstrapParams.OSType) - } - - return asStr, nil -} - -func GetTools(osType commonParams.OSType, osArch commonParams.OSArch, tools []*github.RunnerApplicationDownload) (github.RunnerApplicationDownload, error) { - // Validate image OS. Linux only for now. - switch osType { - case commonParams.Linux: - case commonParams.Windows: - default: - return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS type: %s", osType) - } - - switch osArch { - case commonParams.Amd64: - case commonParams.Arm: - case commonParams.Arm64: - default: - return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS arch: %s", osArch) - } - - // Find tools for OS/Arch. - for _, tool := range tools { - if tool == nil { - continue - } - if tool.OS == nil || tool.Architecture == nil { - continue - } - - ghArch, err := ResolveToGithubArch(string(osArch)) - if err != nil { - continue - } - - ghOS, err := ResolveToGithubOSType(string(osType)) - if err != nil { - continue - } - if *tool.Architecture == ghArch && *tool.OS == ghOS { - return *tool, nil - } - } - return github.RunnerApplicationDownload{}, fmt.Errorf("failed to find tools for OS %s and arch %s", osType, osArch) -} - -// GetRandomString returns a secure random string -func GetRandomString(n int) (string, error) { - data := make([]byte, n) - _, err := rand.Read(data) - if err != nil { - return "", errors.Wrap(err, "getting random data") - } - for i, b := range data { - data[i] = alphanumeric[b%byte(len(alphanumeric))] - } - - return string(data), nil -} - -func Aes256EncodeString(target string, passphrase string) ([]byte, error) { - if len(passphrase) != 32 { - return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)") - } - - toEncrypt := []byte(target) - block, err := aes.NewCipher([]byte(passphrase)) - if err != nil { - return nil, errors.Wrap(err, "creating cipher") - } - - aesgcm, err := cipher.NewGCM(block) - if err != nil { - return nil, errors.Wrap(err, "creating new aead") - } - - nonce := make([]byte, aesgcm.NonceSize()) - if _, err := io.ReadFull(rand.Reader, nonce); err != nil { - return nil, errors.Wrap(err, "creating nonce") - } - - ciphertext := aesgcm.Seal(nonce, nonce, toEncrypt, nil) - return ciphertext, nil -} - -func Aes256DecodeString(target []byte, passphrase string) (string, error) { - if len(passphrase) != 32 { - return "", fmt.Errorf("invalid passphrase length (expected length 32 characters)") - } - - block, err := aes.NewCipher([]byte(passphrase)) - if err != nil { - return "", errors.Wrap(err, "creating cipher") - } - - aesgcm, err := cipher.NewGCM(block) - if err != nil { - return "", errors.Wrap(err, "creating new aead") - } - - nonceSize := aesgcm.NonceSize() - if len(target) < nonceSize { - return "", fmt.Errorf("failed to decrypt text") - } - - nonce, ciphertext := target[:nonceSize], target[nonceSize:] - plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil) - if err != nil { - return "", fmt.Errorf("failed to decrypt text") - } - return string(plaintext), nil -} - -// PaswsordToBcrypt returns a bcrypt hash of the specified password using the default cost -func PaswsordToBcrypt(password string) (string, error) { - hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) - if err != nil { - return "", fmt.Errorf("failed to hash password") - } - return string(hashedPassword), nil -} - -func NewLoggingMiddleware(writer io.Writer) func(http.Handler) http.Handler { - return func(next http.Handler) http.Handler { - return gorillaHandlers.CombinedLoggingHandler(writer, next) - } -} - -func SanitizeLogEntry(entry string) string { - return strings.Replace(strings.Replace(entry, "\n", "", -1), "\r", "", -1) -} - -func toBase62(uuid []byte) string { - var i big.Int - i.SetBytes(uuid[:]) - return i.Text(62) -} - -func NewID() string { - short, err := shortid.Generate() - if err == nil { - return toBase62([]byte(short)) - } - newUUID := uuid.New() - return toBase62(newUUID[:]) -} - -func UTF16FromString(s string) ([]uint16, error) { - buf := make([]uint16, 0, len(s)*2+1) - for _, r := range s { - buf = utf16.AppendRune(buf, r) - } - return utf16.AppendRune(buf, '\x00'), nil -} - -func UTF16ToString(s []uint16) string { - for i, v := range s { - if v == 0 { - s = s[0:i] - break - } - } - return string(utf16.Decode(s)) -} - -func Uint16ToByteArray(u []uint16) []byte { - ret := make([]byte, (len(u)-1)*2) - for i := 0; i < len(u)-1; i++ { - binary.LittleEndian.PutUint16(ret[i*2:], uint16(u[i])) - } - return ret -} - -func UTF16EncodedByteArrayFromString(s string) ([]byte, error) { - asUint16, err := UTF16FromString(s) - if err != nil { - return nil, fmt.Errorf("failed to encode to uint16: %w", err) - } - asBytes := Uint16ToByteArray(asUint16) - return asBytes, nil -} - -func CompressData(data []byte) ([]byte, error) { - var b bytes.Buffer - gz := gzip.NewWriter(&b) - - _, err := gz.Write(data) - if err != nil { - return nil, fmt.Errorf("failed to compress data: %w", err) - } - - if err = gz.Flush(); err != nil { - return nil, fmt.Errorf("failed to flush buffer: %w", err) - } - - if err = gz.Close(); err != nil { - return nil, fmt.Errorf("failed to close buffer: %w", err) - } - - return b.Bytes(), nil -} diff --git a/vendor/github.com/cloudbase/garm-provider-common/util/util.go b/vendor/github.com/cloudbase/garm-provider-common/util/util.go new file mode 100644 index 00000000..ce5d4e74 --- /dev/null +++ b/vendor/github.com/cloudbase/garm-provider-common/util/util.go @@ -0,0 +1,462 @@ +// Copyright 2022 Cloudbase Solutions SRL +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package util + +import ( + "bytes" + "compress/gzip" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "encoding/binary" + "fmt" + "io" + "math/big" + "net/http" + "os" + "path" + "regexp" + "strings" + "unicode" + "unicode/utf16" + + "github.com/cloudbase/garm-provider-common/cloudconfig" + runnerErrors "github.com/cloudbase/garm-provider-common/errors" + + "github.com/cloudbase/garm-provider-common/defaults" + commonParams "github.com/cloudbase/garm-provider-common/params" + + "github.com/google/go-github/v53/github" + "github.com/google/uuid" + gorillaHandlers "github.com/gorilla/handlers" + "github.com/pkg/errors" + "github.com/teris-io/shortid" + "golang.org/x/crypto/bcrypt" + lumberjack "gopkg.in/natefinch/lumberjack.v2" +) + +const alphanumeric = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + +// From: https://www.alexedwards.net/blog/validation-snippets-for-go#email-validation +var rxEmail = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$") + +var ( + OSToOSTypeMap map[string]commonParams.OSType = map[string]commonParams.OSType{ + "almalinux": commonParams.Linux, + "alma": commonParams.Linux, + "alpine": commonParams.Linux, + "archlinux": commonParams.Linux, + "arch": commonParams.Linux, + "centos": commonParams.Linux, + "ubuntu": commonParams.Linux, + "rhel": commonParams.Linux, + "suse": commonParams.Linux, + "opensuse": commonParams.Linux, + "fedora": commonParams.Linux, + "debian": commonParams.Linux, + "flatcar": commonParams.Linux, + "gentoo": commonParams.Linux, + "rockylinux": commonParams.Linux, + "rocky": commonParams.Linux, + "windows": commonParams.Windows, + } + + githubArchMapping map[string]string = map[string]string{ + "x86_64": "x64", + "amd64": "x64", + "armv7l": "arm", + "aarch64": "arm64", + "x64": "x64", + "arm": "arm", + "arm64": "arm64", + } + + githubOSTypeMap map[string]string = map[string]string{ + "linux": "linux", + "windows": "win", + } + + // + githubOSTag = map[commonParams.OSType]string{ + commonParams.Linux: "Linux", + commonParams.Windows: "Windows", + } +) + +// ResolveToGithubArch returns the cpu architecture as it is defined in the GitHub +// tools download list. We use it to find the proper tools for the OS/Arch combo we're +// deploying. +func ResolveToGithubArch(arch string) (string, error) { + ghArch, ok := githubArchMapping[arch] + if !ok { + return "", runnerErrors.NewNotFoundError("arch %s is unknown", arch) + } + + return ghArch, nil +} + +// ResolveToGithubArch returns the OS type as it is defined in the GitHub +// tools download list. We use it to find the proper tools for the OS/Arch combo we're +// deploying. +func ResolveToGithubOSType(osType string) (string, error) { + ghOS, ok := githubOSTypeMap[osType] + if !ok { + return "", runnerErrors.NewNotFoundError("os %s is unknown", osType) + } + + return ghOS, nil +} + +// ResolveToGithubTag returns the default OS tag that self hosted runners automatically +// (and forcefully) adds to every runner that gets deployed. We need to keep track of those +// tags internally as well. +func ResolveToGithubTag(os commonParams.OSType) (string, error) { + ghOS, ok := githubOSTag[os] + if !ok { + return "", runnerErrors.NewNotFoundError("os %s is unknown", os) + } + + return ghOS, nil +} + +// IsValidEmail returs a bool indicating if an email is valid +func IsValidEmail(email string) bool { + if len(email) > 254 || !rxEmail.MatchString(email) { + return false + } + return true +} + +func IsAlphanumeric(s string) bool { + for _, r := range s { + if !unicode.IsLetter(r) && !unicode.IsNumber(r) { + return false + } + } + return true +} + +// GetLoggingWriter returns a new io.Writer suitable for logging. +func GetLoggingWriter(logFile string) (io.Writer, error) { + var writer io.Writer = os.Stdout + if logFile != "" { + dirname := path.Dir(logFile) + if _, err := os.Stat(dirname); err != nil { + if !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to create log folder") + } + if err := os.MkdirAll(dirname, 0o711); err != nil { + return nil, fmt.Errorf("failed to create log folder") + } + } + writer = &lumberjack.Logger{ + Filename: logFile, + MaxSize: 500, // megabytes + MaxBackups: 3, + MaxAge: 28, // days + Compress: true, // disabled by default + } + } + return writer, nil +} + +func ConvertFileToBase64(file string) (string, error) { + bytes, err := os.ReadFile(file) + if err != nil { + return "", errors.Wrap(err, "reading file") + } + + return base64.StdEncoding.EncodeToString(bytes), nil +} + +func OSToOSType(os string) (commonParams.OSType, error) { + osType, ok := OSToOSTypeMap[strings.ToLower(os)] + if !ok { + return commonParams.Unknown, fmt.Errorf("no OS to OS type mapping for %s", os) + } + return osType, nil +} + +func GetCloudConfig(bootstrapParams commonParams.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) (string, error) { + if tools.Filename == nil { + return "", fmt.Errorf("missing tools filename") + } + + if tools.DownloadURL == nil { + return "", fmt.Errorf("missing tools download URL") + } + + var tempToken string + if tools.TempDownloadToken != nil { + tempToken = *tools.TempDownloadToken + } + + installRunnerParams := cloudconfig.InstallRunnerParams{ + FileName: *tools.Filename, + DownloadURL: *tools.DownloadURL, + TempDownloadToken: tempToken, + MetadataURL: bootstrapParams.MetadataURL, + RunnerUsername: defaults.DefaultUser, + RunnerGroup: defaults.DefaultUser, + RepoURL: bootstrapParams.RepoURL, + RunnerName: runnerName, + RunnerLabels: strings.Join(bootstrapParams.Labels, ","), + CallbackURL: bootstrapParams.CallbackURL, + CallbackToken: bootstrapParams.InstanceToken, + GitHubRunnerGroup: bootstrapParams.GitHubRunnerGroup, + } + if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 { + installRunnerParams.CABundle = string(bootstrapParams.CACertBundle) + } + + installScript, err := cloudconfig.InstallRunnerScript(installRunnerParams, bootstrapParams.OSType) + if err != nil { + return "", errors.Wrap(err, "generating script") + } + + var asStr string + switch bootstrapParams.OSType { + case commonParams.Linux: + cloudCfg := cloudconfig.NewDefaultCloudInitConfig() + + if bootstrapParams.UserDataOptions.DisableUpdatesOnBoot { + cloudCfg.PackageUpgrade = false + cloudCfg.Packages = []string{} + } + for _, pkg := range bootstrapParams.UserDataOptions.ExtraPackages { + cloudCfg.AddPackage(pkg) + } + + cloudCfg.AddSSHKey(bootstrapParams.SSHKeys...) + cloudCfg.AddFile(installScript, "/install_runner.sh", "root:root", "755") + cloudCfg.AddRunCmd(fmt.Sprintf("su -l -c /install_runner.sh %s", defaults.DefaultUser)) + cloudCfg.AddRunCmd("rm -f /install_runner.sh") + if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 { + if err := cloudCfg.AddCACert(bootstrapParams.CACertBundle); err != nil { + return "", errors.Wrap(err, "adding CA cert bundle") + } + } + var err error + asStr, err = cloudCfg.Serialize() + if err != nil { + return "", errors.Wrap(err, "creating cloud config") + } + case commonParams.Windows: + asStr = string(installScript) + default: + return "", fmt.Errorf("unknown os type: %s", bootstrapParams.OSType) + } + + return asStr, nil +} + +func GetTools(osType commonParams.OSType, osArch commonParams.OSArch, tools []*github.RunnerApplicationDownload) (github.RunnerApplicationDownload, error) { + // Validate image OS. Linux only for now. + switch osType { + case commonParams.Linux: + case commonParams.Windows: + default: + return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS type: %s", osType) + } + + switch osArch { + case commonParams.Amd64: + case commonParams.Arm: + case commonParams.Arm64: + default: + return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS arch: %s", osArch) + } + + // Find tools for OS/Arch. + for _, tool := range tools { + if tool == nil { + continue + } + if tool.OS == nil || tool.Architecture == nil { + continue + } + + ghArch, err := ResolveToGithubArch(string(osArch)) + if err != nil { + continue + } + + ghOS, err := ResolveToGithubOSType(string(osType)) + if err != nil { + continue + } + if *tool.Architecture == ghArch && *tool.OS == ghOS { + return *tool, nil + } + } + return github.RunnerApplicationDownload{}, fmt.Errorf("failed to find tools for OS %s and arch %s", osType, osArch) +} + +// GetRandomString returns a secure random string +func GetRandomString(n int) (string, error) { + data := make([]byte, n) + _, err := rand.Read(data) + if err != nil { + return "", errors.Wrap(err, "getting random data") + } + for i, b := range data { + data[i] = alphanumeric[b%byte(len(alphanumeric))] + } + + return string(data), nil +} + +func Aes256EncodeString(target string, passphrase string) ([]byte, error) { + if len(passphrase) != 32 { + return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)") + } + + toEncrypt := []byte(target) + block, err := aes.NewCipher([]byte(passphrase)) + if err != nil { + return nil, errors.Wrap(err, "creating cipher") + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return nil, errors.Wrap(err, "creating new aead") + } + + nonce := make([]byte, aesgcm.NonceSize()) + if _, err := io.ReadFull(rand.Reader, nonce); err != nil { + return nil, errors.Wrap(err, "creating nonce") + } + + ciphertext := aesgcm.Seal(nonce, nonce, toEncrypt, nil) + return ciphertext, nil +} + +func Aes256DecodeString(target []byte, passphrase string) (string, error) { + if len(passphrase) != 32 { + return "", fmt.Errorf("invalid passphrase length (expected length 32 characters)") + } + + block, err := aes.NewCipher([]byte(passphrase)) + if err != nil { + return "", errors.Wrap(err, "creating cipher") + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return "", errors.Wrap(err, "creating new aead") + } + + nonceSize := aesgcm.NonceSize() + if len(target) < nonceSize { + return "", fmt.Errorf("failed to decrypt text") + } + + nonce, ciphertext := target[:nonceSize], target[nonceSize:] + plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil) + if err != nil { + return "", fmt.Errorf("failed to decrypt text") + } + return string(plaintext), nil +} + +// PaswsordToBcrypt returns a bcrypt hash of the specified password using the default cost +func PaswsordToBcrypt(password string) (string, error) { + hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + return "", fmt.Errorf("failed to hash password") + } + return string(hashedPassword), nil +} + +func NewLoggingMiddleware(writer io.Writer) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return gorillaHandlers.CombinedLoggingHandler(writer, next) + } +} + +func SanitizeLogEntry(entry string) string { + return strings.Replace(strings.Replace(entry, "\n", "", -1), "\r", "", -1) +} + +func toBase62(uuid []byte) string { + var i big.Int + i.SetBytes(uuid[:]) + return i.Text(62) +} + +func NewID() string { + short, err := shortid.Generate() + if err == nil { + return toBase62([]byte(short)) + } + newUUID := uuid.New() + return toBase62(newUUID[:]) +} + +func UTF16FromString(s string) ([]uint16, error) { + buf := make([]uint16, 0, len(s)*2+1) + for _, r := range s { + buf = utf16.AppendRune(buf, r) + } + return utf16.AppendRune(buf, '\x00'), nil +} + +func UTF16ToString(s []uint16) string { + for i, v := range s { + if v == 0 { + s = s[0:i] + break + } + } + return string(utf16.Decode(s)) +} + +func Uint16ToByteArray(u []uint16) []byte { + ret := make([]byte, (len(u)-1)*2) + for i := 0; i < len(u)-1; i++ { + binary.LittleEndian.PutUint16(ret[i*2:], uint16(u[i])) + } + return ret +} + +func UTF16EncodedByteArrayFromString(s string) ([]byte, error) { + asUint16, err := UTF16FromString(s) + if err != nil { + return nil, fmt.Errorf("failed to encode to uint16: %w", err) + } + asBytes := Uint16ToByteArray(asUint16) + return asBytes, nil +} + +func CompressData(data []byte) ([]byte, error) { + var b bytes.Buffer + gz := gzip.NewWriter(&b) + + _, err := gz.Write(data) + if err != nil { + return nil, fmt.Errorf("failed to compress data: %w", err) + } + + if err = gz.Flush(); err != nil { + return nil, fmt.Errorf("failed to flush buffer: %w", err) + } + + if err = gz.Close(); err != nil { + return nil, fmt.Errorf("failed to close buffer: %w", err) + } + + return b.Bytes(), nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 16cabe51..3528211d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -41,6 +41,7 @@ github.com/cloudbase/garm-provider-common/defaults github.com/cloudbase/garm-provider-common/errors github.com/cloudbase/garm-provider-common/execution github.com/cloudbase/garm-provider-common/params +github.com/cloudbase/garm-provider-common/util # github.com/cloudflare/circl v1.3.3 ## explicit; go 1.19 github.com/cloudflare/circl/dh/x25519