DevFW-CICD/garm
16
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Move most of util package

Browse source 
Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2023-07-22 22:39:17 +00:00
parent ed651bb7d0
commit e775c9c11d
22 changed files with 499 additions and 472 deletions
Download patch file Download diff file Expand all files Collapse all files

2
apiserver/controllers/controllers.go
View file

@ -22,12 +22,12 @@ import (
"strings"
gErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/apiserver/params"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/metrics"
runnerParams "github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner"
"github.com/cloudbase/garm/util"
wsWriter "github.com/cloudbase/garm/websocket"
"github.com/gorilla/websocket"

2
apiserver/routers/routers.go
View file

@ -54,9 +54,9 @@ import (
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/apiserver/controllers"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/util"
)
func WithMetricsRouter(parentRouter *mux.Router, disableAuth bool, metricsMiddlerware auth.Middleware) *mux.Router {

2
auth/auth.go
View file

@ -19,10 +19,10 @@ import (
"time"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/database/common"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/golang-jwt/jwt"
"github.com/nbutton23/zxcvbn-go"

2
cmd/garm-cli/cmd/log.go
View file

@ -10,8 +10,8 @@ import (
"os/signal"
"time"
"github.com/cloudbase/garm-provider-common/util"
apiParams "github.com/cloudbase/garm/apiserver/params"
"github.com/cloudbase/garm/util"
"github.com/gorilla/websocket"
"github.com/spf13/cobra"

4
cmd/garm/main.go
View file

@ -27,6 +27,7 @@ import (
"syscall"
"time"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/apiserver/controllers"
"github.com/cloudbase/garm/apiserver/routers"
"github.com/cloudbase/garm/auth"
@ -35,7 +36,6 @@ import (
"github.com/cloudbase/garm/database/common"
"github.com/cloudbase/garm/metrics"
"github.com/cloudbase/garm/runner"
"github.com/cloudbase/garm/util"
"github.com/cloudbase/garm/util/appdefaults"
"github.com/cloudbase/garm/websocket"
lumberjack "gopkg.in/natefinch/lumberjack.v2"
@ -84,7 +84,7 @@ func main() {
log.Fatalf("Fetching config: %+v", err)
}
logWriter, err := util.GetLoggingWriter(cfg)
logWriter, err := util.GetLoggingWriter(cfg.Default.LogFile)
if err != nil {
log.Fatalf("fetching log writer: %+v", err)
}

2
database/sql/enterprise.go
View file

@ -4,8 +4,8 @@ import (
"context"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/google/uuid"
"github.com/pkg/errors"

2
database/sql/organizations.go
View file

@ -19,8 +19,8 @@ import (
"fmt"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/google/uuid"
"github.com/pkg/errors"

2
database/sql/repositories.go
View file

@ -19,8 +19,8 @@ import (
"fmt"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/google/uuid"
"github.com/pkg/errors"

2
database/sql/users.go
View file

@ -19,8 +19,8 @@ import (
"fmt"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/pkg/errors"
"gorm.io/gorm"

2
database/sql/util.go
View file

@ -18,8 +18,8 @@ import (
"encoding/json"
"fmt"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/pkg/errors"
"gorm.io/datatypes"

2
go.mod
View file

@ -25,7 +25,6 @@ require (
github.com/prometheus/client_golang v1.14.0
github.com/spf13/cobra v1.6.1
github.com/stretchr/testify v1.8.2
github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569
golang.org/x/crypto v0.7.0
golang.org/x/oauth2 v0.8.0
golang.org/x/sync v0.1.0
@ -95,6 +94,7 @@ require (
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 // indirect
go.mongodb.org/mongo-driver v1.11.3 // indirect
go.opentelemetry.io/otel v1.14.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect

12
runner/common/mocks/Provider.go
View file

@ -5,8 +5,10 @@ package mocks
import (
context "context"
params "github.com/cloudbase/garm/params"
garm_provider_commonparams "github.com/cloudbase/garm-provider-common/params"
mock "github.com/stretchr/testify/mock"
params "github.com/cloudbase/garm/params"
)
// Provider is an autogenerated mock type for the Provider type
@ -29,21 +31,21 @@ func (_m *Provider) AsParams() params.Provider {
}
// CreateInstance provides a mock function with given fields: ctx, bootstrapParams
func (_m *Provider) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) {
func (_m *Provider) CreateInstance(ctx context.Context, bootstrapParams garm_provider_commonparams.BootstrapInstance) (params.Instance, error) {
ret := _m.Called(ctx, bootstrapParams)
var r0 params.Instance
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, params.BootstrapInstance) (params.Instance, error)); ok {
if rf, ok := ret.Get(0).(func(context.Context, garm_provider_commonparams.BootstrapInstance) (params.Instance, error)); ok {
return rf(ctx, bootstrapParams)
}
if rf, ok := ret.Get(0).(func(context.Context, params.BootstrapInstance) params.Instance); ok {
if rf, ok := ret.Get(0).(func(context.Context, garm_provider_commonparams.BootstrapInstance) params.Instance); ok {
r0 = rf(ctx, bootstrapParams)
} else {
r0 = ret.Get(0).(params.Instance)
}
if rf, ok := ret.Get(1).(func(context.Context, params.BootstrapInstance) error); ok {
if rf, ok := ret.Get(1).(func(context.Context, garm_provider_commonparams.BootstrapInstance) error); ok {
r1 = rf(ctx, bootstrapParams)
} else {
r1 = ret.Error(1)

3
runner/common/provider.go
View file

@ -17,13 +17,14 @@ package common
import (
"context"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm/params"
)
//go:generate mockery --all
type Provider interface {
// CreateInstance creates a new compute instance in the provider.
CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error)
CreateInstance(ctx context.Context, bootstrapParams commonParams.BootstrapInstance) (params.Instance, error)
// Delete instance will delete the instance in a provider.
DeleteInstance(ctx context.Context, instance string) error
// GetInstance will return details about one instance.

4
runner/pool/pool.go
View file

@ -28,11 +28,11 @@ import (
commonParams "github.com/cloudbase/garm-provider-common/params"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/auth"
dbCommon "github.com/cloudbase/garm/database/common"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/util"
"github.com/google/go-github/v53/github"
"github.com/google/uuid"
@ -752,7 +752,7 @@ func (r *basePoolManager) addInstanceToProvider(instance params.Instance) error
return errors.Wrap(err, "fetching instance jwt token")
}
bootstrapArgs := params.BootstrapInstance{
bootstrapArgs := commonParams.BootstrapInstance{
Name: instance.Name,
Tools: r.tools,
RepoURL: r.helper.GithubURL(),

12
runner/providers/external/external.go vendored
View file

@ -9,7 +9,7 @@ import (
"github.com/cloudbase/garm-provider-common/execution"
providerParams "github.com/cloudbase/garm-provider-common/params"
commonParams "github.com/cloudbase/garm-provider-common/params"
garmErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/config"
@ -46,7 +46,7 @@ type external struct {
execPath string
}
func (e *external) validateResult(inst providerParams.ProviderInstance) error {
func (e *external) validateResult(inst commonParams.ProviderInstance) error {
if inst.ProviderID == "" {
return garmErrors.NewProviderError("missing provider ID")
}
@ -67,7 +67,7 @@ func (e *external) validateResult(inst providerParams.ProviderInstance) error {
}
// CreateInstance creates a new compute instance in the provider.
func (e *external) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) {
func (e *external) CreateInstance(ctx context.Context, bootstrapParams commonParams.BootstrapInstance) (params.Instance, error) {
asEnv := []string{
fmt.Sprintf("GARM_COMMAND=%s", execution.CreateInstanceCommand),
fmt.Sprintf("GARM_CONTROLLER_ID=%s", e.controllerID),
@ -85,7 +85,7 @@ func (e *external) CreateInstance(ctx context.Context, bootstrapParams params.Bo
return params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err)
}
var param providerParams.ProviderInstance
var param commonParams.ProviderInstance
if err := json.Unmarshal(out, &param); err != nil {
return params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err)
}
@ -135,7 +135,7 @@ func (e *external) GetInstance(ctx context.Context, instance string) (params.Ins
return params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err)
}
var param providerParams.ProviderInstance
var param commonParams.ProviderInstance
if err := json.Unmarshal(out, &param); err != nil {
return params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err)
}
@ -161,7 +161,7 @@ func (e *external) ListInstances(ctx context.Context, poolID string) ([]params.I
return []params.Instance{}, garmErrors.NewProviderError("provider binary %s returned error: %s", e.execPath, err)
}
var param []providerParams.ProviderInstance
var param []commonParams.ProviderInstance
if err := json.Unmarshal(out, &param); err != nil {
return []params.Instance{}, garmErrors.NewProviderError("failed to decode response from binary: %s", err)
}

6
runner/providers/lxd/lxd.go
View file

@ -22,10 +22,10 @@ import (
"time"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/util"
"github.com/google/go-github/v53/github"
lxd "github.com/lxc/lxd/client"
@ -212,7 +212,7 @@ func (l *LXD) secureBootEnabled() string {
return "false"
}
func (l *LXD) getCreateInstanceArgs(bootstrapParams params.BootstrapInstance, specs extraSpecs) (api.InstancesPost, error) {
func (l *LXD) getCreateInstanceArgs(bootstrapParams commonParams.BootstrapInstance, specs extraSpecs) (api.InstancesPost, error) {
if bootstrapParams.Name == "" {
return api.InstancesPost{}, runnerErrors.NewBadRequestError("missing name")
}
@ -315,7 +315,7 @@ func (l *LXD) launchInstance(createArgs api.InstancesPost) error {
}
// CreateInstance creates a new compute instance in the provider.
func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams params.BootstrapInstance) (params.Instance, error) {
func (l *LXD) CreateInstance(ctx context.Context, bootstrapParams commonParams.BootstrapInstance) (params.Instance, error) {
extraSpecs, err := parseExtraSpecsFromBootstrapParams(bootstrapParams)
if err != nil {
return params.Instance{}, errors.Wrap(err, "parsing extra specs")

4
runner/providers/lxd/specs.go
View file

@ -17,7 +17,7 @@ package lxd
import (
"encoding/json"
"github.com/cloudbase/garm/params"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/pkg/errors"
)
@ -26,7 +26,7 @@ type extraSpecs struct {
ExtraPackages []string `json:"extra_packages"`
}
func parseExtraSpecsFromBootstrapParams(bootstrapParams params.BootstrapInstance) (extraSpecs, error) {
func parseExtraSpecsFromBootstrapParams(bootstrapParams commonParams.BootstrapInstance) (extraSpecs, error) {
specs := extraSpecs{}
if bootstrapParams.ExtraSpecs == nil {
return specs, nil

2
runner/providers/lxd/util.go
View file

@ -27,9 +27,9 @@ import (
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/util"
"github.com/juju/clock"
"github.com/juju/retry"

2
runner/runner.go
View file

@ -32,6 +32,7 @@ import (
commonParams "github.com/cloudbase/garm-provider-common/params"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/util"
"github.com/cloudbase/garm/auth"
"github.com/cloudbase/garm/config"
dbCommon "github.com/cloudbase/garm/database/common"
@ -39,7 +40,6 @@ import (
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm/runner/pool"
"github.com/cloudbase/garm/runner/providers"
"github.com/cloudbase/garm/util"
"golang.org/x/sync/errgroup"
"github.com/google/uuid"

439
util/util.go
View file

@ -15,188 +15,20 @@
package util
import (
"bytes"
"compress/gzip"
"context"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/base64"
"encoding/binary"
"fmt"
"io"
"math/big"
"net/http"
"os"
"path"
"regexp"
"strings"
"unicode"
"unicode/utf16"
"github.com/cloudbase/garm-provider-common/cloudconfig"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm/config"
"github.com/cloudbase/garm/params"
"github.com/cloudbase/garm/runner/common"
"github.com/cloudbase/garm-provider-common/defaults"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/google/go-github/v53/github"
"github.com/google/uuid"
gorillaHandlers "github.com/gorilla/handlers"
"github.com/pkg/errors"
"github.com/teris-io/shortid"
"golang.org/x/crypto/bcrypt"
"golang.org/x/oauth2"
lumberjack "gopkg.in/natefinch/lumberjack.v2"
)
const alphanumeric = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
// From: https://www.alexedwards.net/blog/validation-snippets-for-go#email-validation
var rxEmail = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$")
var (
OSToOSTypeMap map[string]commonParams.OSType = map[string]commonParams.OSType{
"almalinux": commonParams.Linux,
"alma": commonParams.Linux,
"alpine": commonParams.Linux,
"archlinux": commonParams.Linux,
"arch": commonParams.Linux,
"centos": commonParams.Linux,
"ubuntu": commonParams.Linux,
"rhel": commonParams.Linux,
"suse": commonParams.Linux,
"opensuse": commonParams.Linux,
"fedora": commonParams.Linux,
"debian": commonParams.Linux,
"flatcar": commonParams.Linux,
"gentoo": commonParams.Linux,
"rockylinux": commonParams.Linux,
"rocky": commonParams.Linux,
"windows": commonParams.Windows,
}
githubArchMapping map[string]string = map[string]string{
"x86_64": "x64",
"amd64": "x64",
"armv7l": "arm",
"aarch64": "arm64",
"x64": "x64",
"arm": "arm",
"arm64": "arm64",
}
githubOSTypeMap map[string]string = map[string]string{
"linux": "linux",
"windows": "win",
}
//
githubOSTag = map[commonParams.OSType]string{
commonParams.Linux: "Linux",
commonParams.Windows: "Windows",
}
)
// ResolveToGithubArch returns the cpu architecture as it is defined in the GitHub
// tools download list. We use it to find the proper tools for the OS/Arch combo we're
// deploying.
func ResolveToGithubArch(arch string) (string, error) {
ghArch, ok := githubArchMapping[arch]
if !ok {
return "", runnerErrors.NewNotFoundError("arch %s is unknown", arch)
}
return ghArch, nil
}
// ResolveToGithubArch returns the OS type as it is defined in the GitHub
// tools download list. We use it to find the proper tools for the OS/Arch combo we're
// deploying.
func ResolveToGithubOSType(osType string) (string, error) {
ghOS, ok := githubOSTypeMap[osType]
if !ok {
return "", runnerErrors.NewNotFoundError("os %s is unknown", osType)
}
return ghOS, nil
}
// ResolveToGithubTag returns the default OS tag that self hosted runners automatically
// (and forcefully) adds to every runner that gets deployed. We need to keep track of those
// tags internally as well.
func ResolveToGithubTag(os commonParams.OSType) (string, error) {
ghOS, ok := githubOSTag[os]
if !ok {
return "", runnerErrors.NewNotFoundError("os %s is unknown", os)
}
return ghOS, nil
}
// IsValidEmail returs a bool indicating if an email is valid
func IsValidEmail(email string) bool {
if len(email) > 254 || !rxEmail.MatchString(email) {
return false
}
return true
}
func IsAlphanumeric(s string) bool {
for _, r := range s {
if !unicode.IsLetter(r) && !unicode.IsNumber(r) {
return false
}
}
return true
}
// GetLoggingWriter returns a new io.Writer suitable for logging.
func GetLoggingWriter(cfg *config.Config) (io.Writer, error) {
var writer io.Writer = os.Stdout
if cfg.Default.LogFile != "" {
dirname := path.Dir(cfg.Default.LogFile)
if _, err := os.Stat(dirname); err != nil {
if !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to create log folder")
}
if err := os.MkdirAll(dirname, 0o711); err != nil {
return nil, fmt.Errorf("failed to create log folder")
}
}
writer = &lumberjack.Logger{
Filename: cfg.Default.LogFile,
MaxSize: 500, // megabytes
MaxBackups: 3,
MaxAge: 28, // days
Compress: true, // disabled by default
}
}
return writer, nil
}
func ConvertFileToBase64(file string) (string, error) {
bytes, err := os.ReadFile(file)
if err != nil {
return "", errors.Wrap(err, "reading file")
}
return base64.StdEncoding.EncodeToString(bytes), nil
}
func OSToOSType(os string) (commonParams.OSType, error) {
osType, ok := OSToOSTypeMap[strings.ToLower(os)]
if !ok {
return commonParams.Unknown, fmt.Errorf("no OS to OS type mapping for %s", os)
}
return osType, nil
}
func GithubClient(ctx context.Context, token string, credsDetails params.GithubCredentials) (common.GithubClient, common.GithubEnterpriseClient, error) {
var roots *x509.CertPool
if credsDetails.CABundle != nil && len(credsDetails.CABundle) > 0 {
@ -226,274 +58,3 @@ func GithubClient(ctx context.Context, token string, credsDetails params.GithubC
return ghClient.Actions, ghClient.Enterprise, nil
}
func GetCloudConfig(bootstrapParams params.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) (string, error) {
if tools.Filename == nil {
return "", fmt.Errorf("missing tools filename")
}
if tools.DownloadURL == nil {
return "", fmt.Errorf("missing tools download URL")
}
var tempToken string
if tools.TempDownloadToken != nil {
tempToken = *tools.TempDownloadToken
}
installRunnerParams := cloudconfig.InstallRunnerParams{
FileName: *tools.Filename,
DownloadURL: *tools.DownloadURL,
TempDownloadToken: tempToken,
MetadataURL: bootstrapParams.MetadataURL,
RunnerUsername: defaults.DefaultUser,
RunnerGroup: defaults.DefaultUser,
RepoURL: bootstrapParams.RepoURL,
RunnerName: runnerName,
RunnerLabels: strings.Join(bootstrapParams.Labels, ","),
CallbackURL: bootstrapParams.CallbackURL,
CallbackToken: bootstrapParams.InstanceToken,
GitHubRunnerGroup: bootstrapParams.GitHubRunnerGroup,
}
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
installRunnerParams.CABundle = string(bootstrapParams.CACertBundle)
}
installScript, err := cloudconfig.InstallRunnerScript(installRunnerParams, bootstrapParams.OSType)
if err != nil {
return "", errors.Wrap(err, "generating script")
}
var asStr string
switch bootstrapParams.OSType {
case commonParams.Linux:
cloudCfg := cloudconfig.NewDefaultCloudInitConfig()
if bootstrapParams.UserDataOptions.DisableUpdatesOnBoot {
cloudCfg.PackageUpgrade = false
cloudCfg.Packages = []string{}
}
for _, pkg := range bootstrapParams.UserDataOptions.ExtraPackages {
cloudCfg.AddPackage(pkg)
}
cloudCfg.AddSSHKey(bootstrapParams.SSHKeys...)
cloudCfg.AddFile(installScript, "/install_runner.sh", "root:root", "755")
cloudCfg.AddRunCmd(fmt.Sprintf("su -l -c /install_runner.sh %s", defaults.DefaultUser))
cloudCfg.AddRunCmd("rm -f /install_runner.sh")
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
if err := cloudCfg.AddCACert(bootstrapParams.CACertBundle); err != nil {
return "", errors.Wrap(err, "adding CA cert bundle")
}
}
var err error
asStr, err = cloudCfg.Serialize()
if err != nil {
return "", errors.Wrap(err, "creating cloud config")
}
case commonParams.Windows:
asStr = string(installScript)
default:
return "", fmt.Errorf("unknown os type: %s", bootstrapParams.OSType)
}
return asStr, nil
}
func GetTools(osType commonParams.OSType, osArch commonParams.OSArch, tools []*github.RunnerApplicationDownload) (github.RunnerApplicationDownload, error) {
// Validate image OS. Linux only for now.
switch osType {
case commonParams.Linux:
case commonParams.Windows:
default:
return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS type: %s", osType)
}
switch osArch {
case commonParams.Amd64:
case commonParams.Arm:
case commonParams.Arm64:
default:
return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS arch: %s", osArch)
}
// Find tools for OS/Arch.
for _, tool := range tools {
if tool == nil {
continue
}
if tool.OS == nil || tool.Architecture == nil {
continue
}
ghArch, err := ResolveToGithubArch(string(osArch))
if err != nil {
continue
}
ghOS, err := ResolveToGithubOSType(string(osType))
if err != nil {
continue
}
if *tool.Architecture == ghArch && *tool.OS == ghOS {
return *tool, nil
}
}
return github.RunnerApplicationDownload{}, fmt.Errorf("failed to find tools for OS %s and arch %s", osType, osArch)
}
// GetRandomString returns a secure random string
func GetRandomString(n int) (string, error) {
data := make([]byte, n)
_, err := rand.Read(data)
if err != nil {
return "", errors.Wrap(err, "getting random data")
}
for i, b := range data {
data[i] = alphanumeric[b%byte(len(alphanumeric))]
}
return string(data), nil
}
func Aes256EncodeString(target string, passphrase string) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
toEncrypt := []byte(target)
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return nil, errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, errors.Wrap(err, "creating new aead")
}
nonce := make([]byte, aesgcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, errors.Wrap(err, "creating nonce")
}
ciphertext := aesgcm.Seal(nonce, nonce, toEncrypt, nil)
return ciphertext, nil
}
func Aes256DecodeString(target []byte, passphrase string) (string, error) {
if len(passphrase) != 32 {
return "", fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return "", errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return "", errors.Wrap(err, "creating new aead")
}
nonceSize := aesgcm.NonceSize()
if len(target) < nonceSize {
return "", fmt.Errorf("failed to decrypt text")
}
nonce, ciphertext := target[:nonceSize], target[nonceSize:]
plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", fmt.Errorf("failed to decrypt text")
}
return string(plaintext), nil
}
// PaswsordToBcrypt returns a bcrypt hash of the specified password using the default cost
func PaswsordToBcrypt(password string) (string, error) {
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return "", fmt.Errorf("failed to hash password")
}
return string(hashedPassword), nil
}
func NewLoggingMiddleware(writer io.Writer) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return gorillaHandlers.CombinedLoggingHandler(writer, next)
}
}
func SanitizeLogEntry(entry string) string {
return strings.Replace(strings.Replace(entry, "\n", "", -1), "\r", "", -1)
}
func toBase62(uuid []byte) string {
var i big.Int
i.SetBytes(uuid[:])
return i.Text(62)
}
func NewID() string {
short, err := shortid.Generate()
if err == nil {
return toBase62([]byte(short))
}
newUUID := uuid.New()
return toBase62(newUUID[:])
}
func UTF16FromString(s string) ([]uint16, error) {
buf := make([]uint16, 0, len(s)*2+1)
for _, r := range s {
buf = utf16.AppendRune(buf, r)
}
return utf16.AppendRune(buf, '\x00'), nil
}
func UTF16ToString(s []uint16) string {
for i, v := range s {
if v == 0 {
s = s[0:i]
break
}
}
return string(utf16.Decode(s))
}
func Uint16ToByteArray(u []uint16) []byte {
ret := make([]byte, (len(u)-1)*2)
for i := 0; i < len(u)-1; i++ {
binary.LittleEndian.PutUint16(ret[i*2:], uint16(u[i]))
}
return ret
}
func UTF16EncodedByteArrayFromString(s string) ([]byte, error) {
asUint16, err := UTF16FromString(s)
if err != nil {
return nil, fmt.Errorf("failed to encode to uint16: %w", err)
}
asBytes := Uint16ToByteArray(asUint16)
return asBytes, nil
}
func CompressData(data []byte) ([]byte, error) {
var b bytes.Buffer
gz := gzip.NewWriter(&b)
_, err := gz.Write(data)
if err != nil {
return nil, fmt.Errorf("failed to compress data: %w", err)
}
if err = gz.Flush(); err != nil {
return nil, fmt.Errorf("failed to flush buffer: %w", err)
}
if err = gz.Close(); err != nil {
return nil, fmt.Errorf("failed to close buffer: %w", err)
}
return b.Bytes(), nil
}

462
vendor/github.com/cloudbase/garm-provider-common/util/util.go generated vendored Normal file
View file

@ -0,0 +1,462 @@
// Copyright 2022 Cloudbase Solutions SRL
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package util
import (
"bytes"
"compress/gzip"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/base64"
"encoding/binary"
"fmt"
"io"
"math/big"
"net/http"
"os"
"path"
"regexp"
"strings"
"unicode"
"unicode/utf16"
"github.com/cloudbase/garm-provider-common/cloudconfig"
runnerErrors "github.com/cloudbase/garm-provider-common/errors"
"github.com/cloudbase/garm-provider-common/defaults"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/google/go-github/v53/github"
"github.com/google/uuid"
gorillaHandlers "github.com/gorilla/handlers"
"github.com/pkg/errors"
"github.com/teris-io/shortid"
"golang.org/x/crypto/bcrypt"
lumberjack "gopkg.in/natefinch/lumberjack.v2"
)
const alphanumeric = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
// From: https://www.alexedwards.net/blog/validation-snippets-for-go#email-validation
var rxEmail = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+\\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$")
var (
OSToOSTypeMap map[string]commonParams.OSType = map[string]commonParams.OSType{
"almalinux": commonParams.Linux,
"alma": commonParams.Linux,
"alpine": commonParams.Linux,
"archlinux": commonParams.Linux,
"arch": commonParams.Linux,
"centos": commonParams.Linux,
"ubuntu": commonParams.Linux,
"rhel": commonParams.Linux,
"suse": commonParams.Linux,
"opensuse": commonParams.Linux,
"fedora": commonParams.Linux,
"debian": commonParams.Linux,
"flatcar": commonParams.Linux,
"gentoo": commonParams.Linux,
"rockylinux": commonParams.Linux,
"rocky": commonParams.Linux,
"windows": commonParams.Windows,
}
githubArchMapping map[string]string = map[string]string{
"x86_64": "x64",
"amd64": "x64",
"armv7l": "arm",
"aarch64": "arm64",
"x64": "x64",
"arm": "arm",
"arm64": "arm64",
}
githubOSTypeMap map[string]string = map[string]string{
"linux": "linux",
"windows": "win",
}
//
githubOSTag = map[commonParams.OSType]string{
commonParams.Linux: "Linux",
commonParams.Windows: "Windows",
}
)
// ResolveToGithubArch returns the cpu architecture as it is defined in the GitHub
// tools download list. We use it to find the proper tools for the OS/Arch combo we're
// deploying.
func ResolveToGithubArch(arch string) (string, error) {
ghArch, ok := githubArchMapping[arch]
if !ok {
return "", runnerErrors.NewNotFoundError("arch %s is unknown", arch)
}
return ghArch, nil
}
// ResolveToGithubArch returns the OS type as it is defined in the GitHub
// tools download list. We use it to find the proper tools for the OS/Arch combo we're
// deploying.
func ResolveToGithubOSType(osType string) (string, error) {
ghOS, ok := githubOSTypeMap[osType]
if !ok {
return "", runnerErrors.NewNotFoundError("os %s is unknown", osType)
}
return ghOS, nil
}
// ResolveToGithubTag returns the default OS tag that self hosted runners automatically
// (and forcefully) adds to every runner that gets deployed. We need to keep track of those
// tags internally as well.
func ResolveToGithubTag(os commonParams.OSType) (string, error) {
ghOS, ok := githubOSTag[os]
if !ok {
return "", runnerErrors.NewNotFoundError("os %s is unknown", os)
}
return ghOS, nil
}
// IsValidEmail returs a bool indicating if an email is valid
func IsValidEmail(email string) bool {
if len(email) > 254 || !rxEmail.MatchString(email) {
return false
}
return true
}
func IsAlphanumeric(s string) bool {
for _, r := range s {
if !unicode.IsLetter(r) && !unicode.IsNumber(r) {
return false
}
}
return true
}
// GetLoggingWriter returns a new io.Writer suitable for logging.
func GetLoggingWriter(logFile string) (io.Writer, error) {
var writer io.Writer = os.Stdout
if logFile != "" {
dirname := path.Dir(logFile)
if _, err := os.Stat(dirname); err != nil {
if !os.IsNotExist(err) {
return nil, fmt.Errorf("failed to create log folder")
}
if err := os.MkdirAll(dirname, 0o711); err != nil {
return nil, fmt.Errorf("failed to create log folder")
}
}
writer = &lumberjack.Logger{
Filename: logFile,
MaxSize: 500, // megabytes
MaxBackups: 3,
MaxAge: 28, // days
Compress: true, // disabled by default
}
}
return writer, nil
}
func ConvertFileToBase64(file string) (string, error) {
bytes, err := os.ReadFile(file)
if err != nil {
return "", errors.Wrap(err, "reading file")
}
return base64.StdEncoding.EncodeToString(bytes), nil
}
func OSToOSType(os string) (commonParams.OSType, error) {
osType, ok := OSToOSTypeMap[strings.ToLower(os)]
if !ok {
return commonParams.Unknown, fmt.Errorf("no OS to OS type mapping for %s", os)
}
return osType, nil
}
func GetCloudConfig(bootstrapParams commonParams.BootstrapInstance, tools github.RunnerApplicationDownload, runnerName string) (string, error) {
if tools.Filename == nil {
return "", fmt.Errorf("missing tools filename")
}
if tools.DownloadURL == nil {
return "", fmt.Errorf("missing tools download URL")
}
var tempToken string
if tools.TempDownloadToken != nil {
tempToken = *tools.TempDownloadToken
}
installRunnerParams := cloudconfig.InstallRunnerParams{
FileName: *tools.Filename,
DownloadURL: *tools.DownloadURL,
TempDownloadToken: tempToken,
MetadataURL: bootstrapParams.MetadataURL,
RunnerUsername: defaults.DefaultUser,
RunnerGroup: defaults.DefaultUser,
RepoURL: bootstrapParams.RepoURL,
RunnerName: runnerName,
RunnerLabels: strings.Join(bootstrapParams.Labels, ","),
CallbackURL: bootstrapParams.CallbackURL,
CallbackToken: bootstrapParams.InstanceToken,
GitHubRunnerGroup: bootstrapParams.GitHubRunnerGroup,
}
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
installRunnerParams.CABundle = string(bootstrapParams.CACertBundle)
}
installScript, err := cloudconfig.InstallRunnerScript(installRunnerParams, bootstrapParams.OSType)
if err != nil {
return "", errors.Wrap(err, "generating script")
}
var asStr string
switch bootstrapParams.OSType {
case commonParams.Linux:
cloudCfg := cloudconfig.NewDefaultCloudInitConfig()
if bootstrapParams.UserDataOptions.DisableUpdatesOnBoot {
cloudCfg.PackageUpgrade = false
cloudCfg.Packages = []string{}
}
for _, pkg := range bootstrapParams.UserDataOptions.ExtraPackages {
cloudCfg.AddPackage(pkg)
}
cloudCfg.AddSSHKey(bootstrapParams.SSHKeys...)
cloudCfg.AddFile(installScript, "/install_runner.sh", "root:root", "755")
cloudCfg.AddRunCmd(fmt.Sprintf("su -l -c /install_runner.sh %s", defaults.DefaultUser))
cloudCfg.AddRunCmd("rm -f /install_runner.sh")
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {
if err := cloudCfg.AddCACert(bootstrapParams.CACertBundle); err != nil {
return "", errors.Wrap(err, "adding CA cert bundle")
}
}
var err error
asStr, err = cloudCfg.Serialize()
if err != nil {
return "", errors.Wrap(err, "creating cloud config")
}
case commonParams.Windows:
asStr = string(installScript)
default:
return "", fmt.Errorf("unknown os type: %s", bootstrapParams.OSType)
}
return asStr, nil
}
func GetTools(osType commonParams.OSType, osArch commonParams.OSArch, tools []*github.RunnerApplicationDownload) (github.RunnerApplicationDownload, error) {
// Validate image OS. Linux only for now.
switch osType {
case commonParams.Linux:
case commonParams.Windows:
default:
return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS type: %s", osType)
}
switch osArch {
case commonParams.Amd64:
case commonParams.Arm:
case commonParams.Arm64:
default:
return github.RunnerApplicationDownload{}, fmt.Errorf("unsupported OS arch: %s", osArch)
}
// Find tools for OS/Arch.
for _, tool := range tools {
if tool == nil {
continue
}
if tool.OS == nil || tool.Architecture == nil {
continue
}
ghArch, err := ResolveToGithubArch(string(osArch))
if err != nil {
continue
}
ghOS, err := ResolveToGithubOSType(string(osType))
if err != nil {
continue
}
if *tool.Architecture == ghArch && *tool.OS == ghOS {
return *tool, nil
}
}
return github.RunnerApplicationDownload{}, fmt.Errorf("failed to find tools for OS %s and arch %s", osType, osArch)
}
// GetRandomString returns a secure random string
func GetRandomString(n int) (string, error) {
data := make([]byte, n)
_, err := rand.Read(data)
if err != nil {
return "", errors.Wrap(err, "getting random data")
}
for i, b := range data {
data[i] = alphanumeric[b%byte(len(alphanumeric))]
}
return string(data), nil
}
func Aes256EncodeString(target string, passphrase string) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
toEncrypt := []byte(target)
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return nil, errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, errors.Wrap(err, "creating new aead")
}
nonce := make([]byte, aesgcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, errors.Wrap(err, "creating nonce")
}
ciphertext := aesgcm.Seal(nonce, nonce, toEncrypt, nil)
return ciphertext, nil
}
func Aes256DecodeString(target []byte, passphrase string) (string, error) {
if len(passphrase) != 32 {
return "", fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return "", errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return "", errors.Wrap(err, "creating new aead")
}
nonceSize := aesgcm.NonceSize()
if len(target) < nonceSize {
return "", fmt.Errorf("failed to decrypt text")
}
nonce, ciphertext := target[:nonceSize], target[nonceSize:]
plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", fmt.Errorf("failed to decrypt text")
}
return string(plaintext), nil
}
// PaswsordToBcrypt returns a bcrypt hash of the specified password using the default cost
func PaswsordToBcrypt(password string) (string, error) {
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
if err != nil {
return "", fmt.Errorf("failed to hash password")
}
return string(hashedPassword), nil
}
func NewLoggingMiddleware(writer io.Writer) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return gorillaHandlers.CombinedLoggingHandler(writer, next)
}
}
func SanitizeLogEntry(entry string) string {
return strings.Replace(strings.Replace(entry, "\n", "", -1), "\r", "", -1)
}
func toBase62(uuid []byte) string {
var i big.Int
i.SetBytes(uuid[:])
return i.Text(62)
}
func NewID() string {
short, err := shortid.Generate()
if err == nil {
return toBase62([]byte(short))
}
newUUID := uuid.New()
return toBase62(newUUID[:])
}
func UTF16FromString(s string) ([]uint16, error) {
buf := make([]uint16, 0, len(s)*2+1)
for _, r := range s {
buf = utf16.AppendRune(buf, r)
}
return utf16.AppendRune(buf, '\x00'), nil
}
func UTF16ToString(s []uint16) string {
for i, v := range s {
if v == 0 {
s = s[0:i]
break
}
}
return string(utf16.Decode(s))
}
func Uint16ToByteArray(u []uint16) []byte {
ret := make([]byte, (len(u)-1)*2)
for i := 0; i < len(u)-1; i++ {
binary.LittleEndian.PutUint16(ret[i*2:], uint16(u[i]))
}
return ret
}
func UTF16EncodedByteArrayFromString(s string) ([]byte, error) {
asUint16, err := UTF16FromString(s)
if err != nil {
return nil, fmt.Errorf("failed to encode to uint16: %w", err)
}
asBytes := Uint16ToByteArray(asUint16)
return asBytes, nil
}
func CompressData(data []byte) ([]byte, error) {
var b bytes.Buffer
gz := gzip.NewWriter(&b)
_, err := gz.Write(data)
if err != nil {
return nil, fmt.Errorf("failed to compress data: %w", err)
}
if err = gz.Flush(); err != nil {
return nil, fmt.Errorf("failed to flush buffer: %w", err)
}
if err = gz.Close(); err != nil {
return nil, fmt.Errorf("failed to close buffer: %w", err)
}
return b.Bytes(), nil
}

1
vendor/modules.txt vendored
View file

@ -41,6 +41,7 @@ github.com/cloudbase/garm-provider-common/defaults
github.com/cloudbase/garm-provider-common/errors
github.com/cloudbase/garm-provider-common/execution
github.com/cloudbase/garm-provider-common/params
github.com/cloudbase/garm-provider-common/util
# github.com/cloudflare/circl v1.3.3
## explicit; go 1.19
github.com/cloudflare/circl/dh/x25519