Update garm-provider-common and go-github

* Updates the garm-provider-common and go-github packages.
* Update sqlToParamsInstance to return an error when unmarshaling

This change is needed to pull in the new Seal/Unseal functions in common.

Signed-off-by: Gabriel Adrian Samfira <gsamfira@cloudbasesolutions.com>
This commit is contained in:
Gabriel Adrian Samfira 2023-08-28 08:11:44 +00:00
parent 4348999cb1
commit d700b790ac
310 changed files with 10994 additions and 1955 deletions

View file

@ -40,7 +40,6 @@ if [ -z "$METADATA_URL" ];then
echo "no token is available and METADATA_URL is not set"
exit 1
fi
GITHUB_TOKEN=$(curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/")
function call() {
PAYLOAD="$1"
@ -53,11 +52,18 @@ function sendStatus() {
call "{\"status\": \"installing\", \"message\": \"$MSG\"}"
}
{{- if .UseJITConfig }}
function success() {
MSG="$1"
call "{\"status\": \"idle\", \"message\": \"$MSG\"}"
}
{{- else}}
function success() {
MSG="$1"
ID=$2
call "{\"status\": \"idle\", \"message\": \"$MSG\", \"agent_id\": $ID}"
}
{{- end}}
function fail() {
MSG="$1"
@ -105,16 +111,6 @@ function downloadAndExtractRunner() {
# chown {{ .RunnerUsername }}:{{ .RunnerGroup }} -R /home/{{ .RunnerUsername }}/actions-runner/ || fail "failed to change owner"
}
TEMP_TOKEN=""
GH_RUNNER_GROUP="{{.GitHubRunnerGroup}}"
# $RUNNER_GROUP_OPT will be added to the config.sh line. If it's empty, nothing happens
# if it holds a value, it will be part of the command.
RUNNER_GROUP_OPT=""
if [ ! -z $GH_RUNNER_GROUP ];then
RUNNER_GROUP_OPT="--runnergroup=$GH_RUNNER_GROUP"
fi
CACHED_RUNNER=$(getCachedToolsPath)
if [ -z "$CACHED_RUNNER" ];then
downloadAndExtractRunner
@ -130,11 +126,46 @@ fi
sendStatus "configuring runner"
{{- if .UseJITConfig }}
function getRunnerFile() {
curl --retry 5 --retry-delay 5 \
--retry-connrefused --fail -s \
-X GET -H 'Accept: application/json' \
-H "Authorization: Bearer ${BEARER_TOKEN}" \
"${METADATA_URL}/$1" -o "$2"
}
sendStatus "downloading JIT credentials"
getRunnerFile "credentials/runner" "/home/{{ .RunnerUsername }}/actions-runner/.runner" || fail "failed to get runner file"
getRunnerFile "credentials/credentials" "/home/{{ .RunnerUsername }}/actions-runner/.credentials" || fail "failed to get credentials file"
getRunnerFile "credentials/credentials_rsaparams" "/home/{{ .RunnerUsername }}/actions-runner/.credentials_rsaparams" || fail "failed to get credentials_rsaparams file"
getRunnerFile "system/service-name" "/home/{{ .RunnerUsername }}/actions-runner/.service" || fail "failed to get service name file"
sed -i 's/$/\.service/' /home/{{ .RunnerUsername }}/actions-runner/.service
SVC_NAME=$(cat /home/{{ .RunnerUsername }}/actions-runner/.service)
sendStatus "generating systemd unit file"
getRunnerFile "systemd/unit-file?runAsUser={{ .RunnerUsername }}" "$SVC_NAME" || fail "failed to get service file"
sudo mv $SVC_NAME /etc/systemd/system/ || fail "failed to move service file"
sendStatus "enabling runner service"
cp /home/{{ .RunnerUsername }}/actions-runner/bin/runsvc.sh /home/{{ .RunnerUsername }}/actions-runner/ || fail "failed to copy runsvc.sh"
sudo chown {{ .RunnerUsername }}:{{ .RunnerGroup }} -R /home/{{ .RunnerUsername }} || fail "failed to change owner"
sudo systemctl daemon-reload || fail "failed to reload systemd"
sudo systemctl enable $SVC_NAME
{{- else}}
GITHUB_TOKEN=$(curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/")
set +e
attempt=1
while true; do
ERROUT=$(mktemp)
./config.sh --unattended --url "{{ .RepoURL }}" --token "$GITHUB_TOKEN" $RUNNER_GROUP_OPT --name "{{ .RunnerName }}" --labels "{{ .RunnerLabels }}" --ephemeral 2>$ERROUT
{{- if .GitHubRunnerGroup }}
./config.sh --unattended --url "{{ .RepoURL }}" --token "$GITHUB_TOKEN" --runnergroup {{.GitHubRunnerGroup}} --name "{{ .RunnerName }}" --labels "{{ .RunnerLabels }}" --ephemeral 2>$ERROUT
{{- else}}
./config.sh --unattended --url "{{ .RepoURL }}" --token "$GITHUB_TOKEN" --name "{{ .RunnerName }}" --labels "{{ .RunnerLabels }}" --ephemeral 2>$ERROUT
{{- end}}
if [ $? -eq 0 ]; then
rm $ERROUT || true
sendStatus "runner successfully configured after $attempt attempt(s)"
@ -161,12 +192,17 @@ set -e
sendStatus "installing runner service"
sudo ./svc.sh install {{ .RunnerUsername }} || fail "failed to install service"
{{- end}}
if [ -e "/sys/fs/selinux" ];then
sudo chcon -h user_u:object_r:bin_t /home/runner/ || fail "failed to change selinux context"
sudo chcon -R -h {{ .RunnerUsername }}:object_r:bin_t /home/runner/* || fail "failed to change selinux context"
fi
{{- if .UseJITConfig }}
sudo systemctl start $SVC_NAME || fail "failed to start service"
success "runner successfully installed"
{{- else}}
sendStatus "starting service"
sudo ./svc.sh start || fail "failed to start service"
@ -176,8 +212,8 @@ if [ $? -ne 0 ];then
fail "failed to get agent ID"
fi
set -e
success "runner successfully installed" $AGENT_ID
{{- end}}
`
var WindowsSetupScriptTemplate = `#ps1_sysnative
@ -263,10 +299,10 @@ function Import-Certificate() {
[CmdletBinding()]
param (
[parameter(Mandatory=$true)]
[string]$CertificatePath,
[parameter(Mandatory=$true)]
$CertificateData,
[parameter(Mandatory=$false)]
[System.Security.Cryptography.X509Certificates.StoreLocation]$StoreLocation="LocalMachine",
[parameter(Mandatory=$true)]
[parameter(Mandatory=$false)]
[System.Security.Cryptography.X509Certificates.StoreName]$StoreName="TrustedPublisher"
)
PROCESS
@ -274,8 +310,7 @@ function Import-Certificate() {
$store = New-Object System.Security.Cryptography.X509Certificates.X509Store(
$StoreName, $StoreLocation)
$store.Open([System.Security.Cryptography.X509Certificates.OpenFlags]::ReadWrite)
$cert = New-Object System.Security.Cryptography.X509Certificates.X509Certificate2(
$CertificatePath)
$cert = [System.Security.Cryptography.X509Certificates.X509Certificate2]::new($CertificateData)
$store.Add($cert)
}
}
@ -298,14 +333,22 @@ function Update-GarmStatus() {
param (
[parameter(Mandatory=$true)]
[string]$Message,
[parameter(Mandatory=$false)]
[int64]$AgentID=0,
[parameter(Mandatory=$false)]
[string]$Status="installing",
[parameter(Mandatory=$true)]
[string]$CallbackURL
)
PROCESS{
$body = @{
"status"="installing"
"status"=$Status
"message"=$Message
}
if ($AgentID -ne 0) {
$body["AgentID"] = $AgentID
}
Invoke-APICall -Payload $body -CallbackURL $CallbackURL | Out-Null
}
}
@ -321,12 +364,7 @@ function Invoke-GarmSuccess() {
[string]$CallbackURL
)
PROCESS{
$body = @{
"status"="idle"
"message"=$Message
"agent_id"=$AgentID
}
Invoke-APICall -Payload $body -CallbackURL $CallbackURL | Out-Null
Update-GarmStatus -Message $Message -AgentID $AgentID -CallbackURL $CallbackURL -Status "idle" | Out-Null
}
}
@ -339,18 +377,11 @@ function Invoke-GarmFailure() {
[string]$CallbackURL
)
PROCESS{
$body = @{
"status"="failed"
"message"=$Message
}
Invoke-APICall -Payload $body -CallbackURL $CallbackURL | Out-Null
Update-GarmStatus -Message $Message -CallbackURL $CallbackURL -Status "failed" | Out-Null
Throw $Message
}
}
$PEMData = @"
{{.CABundle}}
"@
$GHRunnerGroup = "{{.GitHubRunnerGroup}}"
function Install-Runner() {
@ -369,12 +400,13 @@ function Install-Runner() {
Throw "missing metadata URL"
}
if($PEMData.Trim().Length -gt 0){
Set-Content $env:TMP\garm-ca.pem $PEMData
Import-Certificate -CertificatePath $env:TMP\garm-ca.pem
$bundle = wget -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $MetadataURL/system/cert-bundle
$converted = ConvertFrom-Json $bundle
foreach ($i in $converted.root_certificates.psobject.Properties){
$data = [System.Convert]::FromBase64String($i.Value)
Import-Certificate -CertificateData $data -StoreName Root -StoreLocation LocalMachine
}
$GithubRegistrationToken = Invoke-WebRequest -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $MetadataURL/runner-registration-token/
Update-GarmStatus -CallbackURL $CallbackURL -Message "downloading tools from $DownloadURL"
$downloadToken="{{.TempDownloadToken}}"
@ -393,17 +425,42 @@ function Install-Runner() {
Update-GarmStatus -CallbackURL $CallbackURL -Message "extracting runner"
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory($downloadPath, "$runnerDir")
$runnerGroupOpt = ""
if ($GHRunnerGroup.Length -gt 0){
$runnerGroupOpt = "--runnergroup $GHRunnerGroup"
}
Update-GarmStatus -CallbackURL $CallbackURL -Message "configuring and starting runner"
cd $runnerDir
./config.cmd --unattended --url "{{ .RepoURL }}" --token $GithubRegistrationToken $runnerGroupOpt --name "{{ .RunnerName }}" --labels "{{ .RunnerLabels }}" --ephemeral --runasservice
{{- if .UseJITConfig }}
Update-GarmStatus -CallbackURL $CallbackURL -Message "downloading JIT credentials"
wget -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $MetadataURL/credentials/runner -OutFile (Join-Path $runnerDir ".runner")
wget -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $MetadataURL/credentials/credentials -OutFile (Join-Path $runnerDir ".credentials")
Add-Type -AssemblyName System.Security
$rsaData = (wget -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $MetadataURL/credentials/credentials_rsaparams)
$encodedBytes = [System.Text.Encoding]::UTF8.GetBytes($rsaData)
$protectedBytes = [Security.Cryptography.ProtectedData]::Protect( $encodedBytes, $null, [Security.Cryptography.DataProtectionScope]::LocalMachine )
[System.IO.File]::WriteAllBytes((Join-Path $runnerDir ".credentials_rsaparams"), $protectedBytes)
$serviceNameFile = (Join-Path $runnerDir ".service")
wget -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $MetadataURL/system/service-name -OutFile $serviceNameFile
Update-GarmStatus -CallbackURL $CallbackURL -Message "Creating system service"
$SVC_NAME=(gc -raw $serviceNameFile)
New-Service -Name "$SVC_NAME" -BinaryPathName "C:\runner\bin\RunnerService.exe" -DisplayName "$SVC_NAME" -Description "GitHub Actions Runner ($SVC_NAME)" -StartupType Automatic
Start-Service "$SVC_NAME"
Update-GarmStatus -Message "runner successfully installed" -CallbackURL $CallbackURL -Status "idle" | Out-Null
{{- else }}
$GithubRegistrationToken = Invoke-WebRequest -UseBasicParsing -Headers @{"Accept"="application/json"; "Authorization"="Bearer $Token"} -Uri $MetadataURL/runner-registration-token/
{{- if .GitHubRunnerGroup }}
./config.cmd --unattended --url "{{ .RepoURL }}" --token $GithubRegistrationToken --runnergroup {{.GitHubRunnerGroup}} --name "{{ .RunnerName }}" --labels "{{ .RunnerLabels }}" --ephemeral --runasservice
{{- else}}
./config.cmd --unattended --url "{{ .RepoURL }}" --token $GithubRegistrationToken --name "{{ .RunnerName }}" --labels "{{ .RunnerLabels }}" --ephemeral --runasservice
{{- end}}
$agentInfoFile = Join-Path $runnerDir ".runner"
$agentInfo = ConvertFrom-Json (gc -raw $agentInfoFile)
Invoke-GarmSuccess -CallbackURL $CallbackURL -Message "runner successfully installed" -AgentID $agentInfo.agentId
{{- end }}
} catch {
Invoke-GarmFailure -CallbackURL $CallbackURL -Message $_
}
@ -452,6 +509,8 @@ type InstallRunnerParams struct {
// This option is useful for situations in which you're supplying your own template and you need
// to pass in information that is not available in the default template.
ExtraContext map[string]string
// UseJITConfig indicates whether to attempt to configure the runner using JIT or a registration token.
UseJITConfig bool
}
func InstallRunnerScript(installParams InstallRunnerParams, osType params.OSType, tpl string) ([]byte, error) {

View file

@ -8,7 +8,7 @@ import (
"github.com/cloudbase/garm-provider-common/defaults"
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/google/go-github/v53/github"
"github.com/google/go-github/v54/github"
"github.com/pkg/errors"
)
@ -105,6 +105,7 @@ func GetRunnerInstallScript(bootstrapParams commonParams.BootstrapInstance, tool
GitHubRunnerGroup: bootstrapParams.GitHubRunnerGroup,
ExtraContext: extraSpecs.ExtraContext,
EnableBootDebug: bootstrapParams.UserDataOptions.EnableBootDebug,
UseJITConfig: bootstrapParams.JitConfigEnabled,
}
if bootstrapParams.CACertBundle != nil && len(bootstrapParams.CACertBundle) > 0 {

View file

@ -3,7 +3,7 @@ package params
import (
"encoding/json"
"github.com/google/go-github/v53/github"
"github.com/google/go-github/v54/github"
)
type (
@ -104,6 +104,12 @@ type BootstrapInstance struct {
// UserDataOptions are the options for the user data generation.
UserDataOptions UserDataOptions `json:"user_data_options"`
// JitConfigEnabled is a flag that indicates if the runner should be configured to use
// just-in-time configuration. If set to true, providers must attempt to fetch the JIT configuration
// from the metadata service instead of the runner registration token. The runner registration token
// is not available if the runner is configured to use JIT.
JitConfigEnabled bool `json:"jit_config_enabled"`
}
type Address struct {

View file

@ -0,0 +1,156 @@
package util
import (
"bytes"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"github.com/minio/sio"
"github.com/pkg/errors"
"golang.org/x/crypto/hkdf"
)
type Envelope struct {
Nonce [32]byte `json:"nonce"`
Data []byte `json:"data"`
}
// Seal will encrypt the given data using a derived key from the given passphrase.
// This function is meant to be used with small datasets like passwords, keys and
// secrets of any type, before they are saved to disk.
func Seal(data []byte, passphrase []byte) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
var nonce [32]byte
if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil {
return nil, fmt.Errorf("failed to read random data: %w", err)
}
// derive an encryption key from the master key and the nonce
var key [32]byte
kdf := hkdf.New(sha256.New, passphrase, nonce[:], nil)
if _, err := io.ReadFull(kdf, key[:]); err != nil {
return nil, fmt.Errorf("failed to derive encryption key: %w", err)
}
input := bytes.NewReader(data)
output := bytes.NewBuffer(nil)
if _, err := sio.Encrypt(output, input, sio.Config{Key: key[:]}); err != nil {
return nil, fmt.Errorf("failed to encrypt data: %w", err)
}
envelope := Envelope{
Data: output.Bytes(),
Nonce: nonce,
}
asJs, err := json.Marshal(envelope)
if err != nil {
return nil, fmt.Errorf("failed to marshal envelope: %w", err)
}
return asJs, nil
}
// Unseal will decrypt the given data using a derived key from the given passphrase.
// This function is meant to be used with small datasets like passwords, keys and
// secrets of any type, after they are read from disk.
func Unseal(data []byte, passphrase []byte) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
var envelope Envelope
if err := json.Unmarshal(data, &envelope); err != nil {
return Aes256Decode(data, string(passphrase))
}
// derive an encryption key from the master key and the nonce
var key [32]byte
kdf := hkdf.New(sha256.New, passphrase, envelope.Nonce[:], nil)
if _, err := io.ReadFull(kdf, key[:]); err != nil {
return nil, fmt.Errorf("failed to derive encryption key: %w", err)
}
input := bytes.NewReader(envelope.Data)
output := bytes.NewBuffer(nil)
if _, err := sio.Decrypt(output, input, sio.Config{Key: key[:]}); err != nil {
return nil, fmt.Errorf("failed to decrypt data: %w", err)
}
return output.Bytes(), nil
}
func Aes256Encode(target []byte, passphrase string) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return nil, errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, errors.Wrap(err, "creating new aead")
}
nonce := make([]byte, aesgcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, errors.Wrap(err, "creating nonce")
}
ciphertext := aesgcm.Seal(nonce, nonce, target, nil)
return ciphertext, nil
}
func Aes256EncodeString(target string, passphrase string) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
return Aes256Encode([]byte(target), passphrase)
}
func Aes256Decode(target []byte, passphrase string) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return nil, errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, errors.Wrap(err, "creating new aead")
}
nonceSize := aesgcm.NonceSize()
if len(target) < nonceSize {
return nil, fmt.Errorf("failed to decrypt text")
}
nonce, ciphertext := target[:nonceSize], target[nonceSize:]
plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return nil, fmt.Errorf("failed to decrypt text")
}
return plaintext, nil
}
func Aes256DecodeString(target []byte, passphrase string) (string, error) {
data, err := Aes256Decode(target, passphrase)
if err != nil {
return "", err
}
return string(data), nil
}

View file

@ -17,8 +17,6 @@ package util
import (
"bytes"
"compress/gzip"
"crypto/aes"
"crypto/cipher"
"crypto/rand"
"encoding/base64"
"encoding/binary"
@ -37,7 +35,7 @@ import (
commonParams "github.com/cloudbase/garm-provider-common/params"
"github.com/google/go-github/v53/github"
"github.com/google/go-github/v54/github"
"github.com/google/uuid"
gorillaHandlers "github.com/gorilla/handlers"
"github.com/pkg/errors"
@ -244,59 +242,6 @@ func GetRandomString(n int) (string, error) {
return string(data), nil
}
func Aes256EncodeString(target string, passphrase string) ([]byte, error) {
if len(passphrase) != 32 {
return nil, fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
toEncrypt := []byte(target)
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return nil, errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return nil, errors.Wrap(err, "creating new aead")
}
nonce := make([]byte, aesgcm.NonceSize())
if _, err := io.ReadFull(rand.Reader, nonce); err != nil {
return nil, errors.Wrap(err, "creating nonce")
}
ciphertext := aesgcm.Seal(nonce, nonce, toEncrypt, nil)
return ciphertext, nil
}
func Aes256DecodeString(target []byte, passphrase string) (string, error) {
if len(passphrase) != 32 {
return "", fmt.Errorf("invalid passphrase length (expected length 32 characters)")
}
block, err := aes.NewCipher([]byte(passphrase))
if err != nil {
return "", errors.Wrap(err, "creating cipher")
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return "", errors.Wrap(err, "creating new aead")
}
nonceSize := aesgcm.NonceSize()
if len(target) < nonceSize {
return "", fmt.Errorf("failed to decrypt text")
}
nonce, ciphertext := target[:nonceSize], target[nonceSize:]
plaintext, err := aesgcm.Open(nil, nonce, ciphertext, nil)
if err != nil {
return "", fmt.Errorf("failed to decrypt text")
}
return string(plaintext), nil
}
// PaswsordToBcrypt returns a bcrypt hash of the specified password using the default cost
func PaswsordToBcrypt(password string) (string, error) {
hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)