Merge pull request #57 from mercedes-benz/runner-prefix

feat: allow to configure the runner name
This commit is contained in:
Gabriel 2023-01-19 13:33:59 +02:00 committed by GitHub
commit c92dfb5d20
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 571 additions and 5 deletions

View file

@ -63,6 +63,7 @@ var orgPoolAddCmd = &cobra.Command{
newPoolParams := params.CreatePoolParams{
ProviderName: poolProvider,
MaxRunners: poolMaxRunners,
RunnerPrefix: poolRunnerPrefix,
MinIdleRunners: poolMinIdleRunners,
Image: poolImage,
Flavor: poolFlavor,
@ -196,6 +197,10 @@ explicitly remove them using the runner delete command.
poolUpdateParams.OSArch = config.OSArch(poolOSArch)
}
if cmd.Flags().Changed("runner-prefix") {
poolUpdateParams.RunnerPrefix = poolRunnerPrefix
}
if cmd.Flags().Changed("max-runners") {
poolUpdateParams.MaxRunners = &poolMaxRunners
}
@ -225,6 +230,7 @@ func init() {
orgPoolAddCmd.Flags().StringVar(&poolTags, "tags", "", "A comma separated list of tags to assign to this runner.")
orgPoolAddCmd.Flags().StringVar(&poolOSType, "os-type", "linux", "Operating system type (windows, linux, etc).")
orgPoolAddCmd.Flags().StringVar(&poolOSArch, "os-arch", "amd64", "Operating system architecture (amd64, arm, etc).")
orgPoolAddCmd.Flags().StringVar(&poolRunnerPrefix, "runner-prefix", "", "The name prefix to use for runners in this pool.")
orgPoolAddCmd.Flags().UintVar(&poolMaxRunners, "max-runners", 5, "The maximum number of runner this pool will create.")
orgPoolAddCmd.Flags().UintVar(&poolMinIdleRunners, "min-idle-runners", 1, "Attempt to maintain a minimum of idle self-hosted runners of this type.")
orgPoolAddCmd.Flags().BoolVar(&poolEnabled, "enabled", false, "Enable this pool.")
@ -238,6 +244,7 @@ func init() {
orgPoolUpdateCmd.Flags().StringVar(&poolTags, "tags", "", "A comma separated list of tags to assign to this runner.")
orgPoolUpdateCmd.Flags().StringVar(&poolOSType, "os-type", "linux", "Operating system type (windows, linux, etc).")
orgPoolUpdateCmd.Flags().StringVar(&poolOSArch, "os-arch", "amd64", "Operating system architecture (amd64, arm, etc).")
orgPoolUpdateCmd.Flags().StringVar(&poolRunnerPrefix, "runner-prefix", "", "The name prefix to use for runners in this pool.")
orgPoolUpdateCmd.Flags().UintVar(&poolMaxRunners, "max-runners", 5, "The maximum number of runner this pool will create.")
orgPoolUpdateCmd.Flags().UintVar(&poolMinIdleRunners, "min-idle-runners", 1, "Attempt to maintain a minimum of idle self-hosted runners of this type.")
orgPoolUpdateCmd.Flags().BoolVar(&poolEnabled, "enabled", false, "Enable this pool.")

View file

@ -45,7 +45,7 @@ var poolListCmd = &cobra.Command{
Aliases: []string{"ls"},
Short: "List pools",
Long: `List pools of repositories, orgs or all of the above.
This command will list pools from one repo, one org or all pools
on the system. The list flags are mutually exclusive. You must however
specify one of them.
@ -167,6 +167,7 @@ var poolAddCmd = &cobra.Command{
tags := strings.Split(poolTags, ",")
newPoolParams := params.CreatePoolParams{
RunnerPrefix: poolRunnerPrefix,
ProviderName: poolProvider,
MaxRunners: poolMaxRunners,
MinIdleRunners: poolMinIdleRunners,
@ -257,6 +258,10 @@ explicitly remove them using the runner delete command.
poolUpdateParams.MinIdleRunners = &poolMinIdleRunners
}
if cmd.Flags().Changed("runner-prefix") {
poolUpdateParams.RunnerPrefix = poolRunnerPrefix
}
if cmd.Flags().Changed("enabled") {
poolUpdateParams.Enabled = &poolEnabled
}
@ -287,6 +292,7 @@ func init() {
poolUpdateCmd.Flags().StringVar(&poolTags, "tags", "", "A comma separated list of tags to assign to this runner.")
poolUpdateCmd.Flags().StringVar(&poolOSType, "os-type", "linux", "Operating system type (windows, linux, etc).")
poolUpdateCmd.Flags().StringVar(&poolOSArch, "os-arch", "amd64", "Operating system architecture (amd64, arm, etc).")
poolUpdateCmd.Flags().StringVar(&poolRunnerPrefix, "runner-prefix", "", "The name prefix to use for runners in this pool.")
poolUpdateCmd.Flags().UintVar(&poolMaxRunners, "max-runners", 5, "The maximum number of runner this pool will create.")
poolUpdateCmd.Flags().UintVar(&poolMinIdleRunners, "min-idle-runners", 1, "Attempt to maintain a minimum of idle self-hosted runners of this type.")
poolUpdateCmd.Flags().BoolVar(&poolEnabled, "enabled", false, "Enable this pool.")
@ -295,6 +301,7 @@ func init() {
poolAddCmd.Flags().StringVar(&poolProvider, "provider-name", "", "The name of the provider where runners will be created.")
poolAddCmd.Flags().StringVar(&poolImage, "image", "", "The provider-specific image name to use for runners in this pool.")
poolAddCmd.Flags().StringVar(&poolFlavor, "flavor", "", "The flavor to use for this runner.")
poolAddCmd.Flags().StringVar(&poolRunnerPrefix, "runner-prefix", "", "The name prefix to use for runners in this pool.")
poolAddCmd.Flags().StringVar(&poolTags, "tags", "", "A comma separated list of tags to assign to this runner.")
poolAddCmd.Flags().StringVar(&poolOSType, "os-type", "linux", "Operating system type (windows, linux, etc).")
poolAddCmd.Flags().StringVar(&poolOSArch, "os-arch", "amd64", "Operating system architecture (amd64, arm, etc).")

View file

@ -28,6 +28,7 @@ var (
poolProvider string
poolMaxRunners uint
poolMinIdleRunners uint
poolRunnerPrefix string
poolImage string
poolFlavor string
poolOSType string
@ -77,6 +78,7 @@ var repoPoolAddCmd = &cobra.Command{
newPoolParams := params.CreatePoolParams{
ProviderName: poolProvider,
MaxRunners: poolMaxRunners,
RunnerPrefix: poolRunnerPrefix,
MinIdleRunners: poolMinIdleRunners,
Image: poolImage,
Flavor: poolFlavor,
@ -182,6 +184,10 @@ explicitly remove them using the runner delete command.
poolUpdateParams.OSArch = config.OSArch(poolOSArch)
}
if cmd.Flags().Changed("runner-prefix") {
poolUpdateParams.RunnerPrefix = poolRunnerPrefix
}
if cmd.Flags().Changed("max-runners") {
poolUpdateParams.MaxRunners = &poolMaxRunners
}
@ -211,6 +217,7 @@ func init() {
repoPoolAddCmd.Flags().StringVar(&poolTags, "tags", "", "A comma separated list of tags to assign to this runner.")
repoPoolAddCmd.Flags().StringVar(&poolOSType, "os-type", "linux", "Operating system type (windows, linux, etc).")
repoPoolAddCmd.Flags().StringVar(&poolOSArch, "os-arch", "amd64", "Operating system architecture (amd64, arm, etc).")
repoPoolAddCmd.Flags().StringVar(&poolRunnerPrefix, "runner-prefix", "", "The name prefix to use for runners in this pool.")
repoPoolAddCmd.Flags().UintVar(&poolMaxRunners, "max-runners", 5, "The maximum number of runner this pool will create.")
repoPoolAddCmd.Flags().UintVar(&poolMinIdleRunners, "min-idle-runners", 1, "Attempt to maintain a minimum of idle self-hosted runners of this type.")
repoPoolAddCmd.Flags().BoolVar(&poolEnabled, "enabled", false, "Enable this pool.")
@ -224,6 +231,7 @@ func init() {
repoPoolUpdateCmd.Flags().StringVar(&poolTags, "tags", "", "A comma separated list of tags to assign to this runner.")
repoPoolUpdateCmd.Flags().StringVar(&poolOSType, "os-type", "linux", "Operating system type (windows, linux, etc).")
repoPoolUpdateCmd.Flags().StringVar(&poolOSArch, "os-arch", "amd64", "Operating system architecture (amd64, arm, etc).")
repoPoolUpdateCmd.Flags().StringVar(&poolRunnerPrefix, "runner-prefix", "", "The name prefix to use for runners in this pool.")
repoPoolUpdateCmd.Flags().UintVar(&poolMaxRunners, "max-runners", 5, "The maximum number of runner this pool will create.")
repoPoolUpdateCmd.Flags().UintVar(&poolMinIdleRunners, "min-idle-runners", 1, "Attempt to maintain a minimum of idle self-hosted runners of this type.")
repoPoolUpdateCmd.Flags().BoolVar(&poolEnabled, "enabled", false, "Enable this pool.")
@ -241,7 +249,7 @@ func init() {
func formatPools(pools []params.Pool) {
t := table.NewWriter()
header := table.Row{"ID", "Image", "Flavor", "Tags", "Belongs to", "Level", "Enabled"}
header := table.Row{"ID", "Image", "Flavor", "Tags", "Belongs to", "Level", "Enabled", "Runner Prefix"}
t.AppendHeader(header)
for _, pool := range pools {
@ -262,7 +270,7 @@ func formatPools(pools []params.Pool) {
belongsTo = pool.EnterpriseName
level = "enterprise"
}
t.AppendRow(table.Row{pool.ID, pool.Image, pool.Flavor, strings.Join(tags, " "), belongsTo, level, pool.Enabled})
t.AppendRow(table.Row{pool.ID, pool.Image, pool.Flavor, strings.Join(tags, " "), belongsTo, level, pool.Enabled, pool.RunnerPrefix})
t.AppendSeparator()
}
fmt.Println(t.Render())
@ -307,6 +315,7 @@ func formatOnePool(pool params.Pool) {
t.AppendRow(table.Row{"Belongs to", belongsTo})
t.AppendRow(table.Row{"Level", level})
t.AppendRow(table.Row{"Enabled", pool.Enabled})
t.AppendRow(table.Row{"Runner Prefix", pool.RunnerPrefix})
if len(pool.Instances) > 0 {
for _, instance := range pool.Instances {

View file

@ -145,6 +145,7 @@ func (s *sqlDatabase) CreateEnterprisePool(ctx context.Context, enterpriseID str
ProviderName: param.ProviderName,
MaxRunners: param.MaxRunners,
MinIdleRunners: param.MinIdleRunners,
RunnerPrefix: param.RunnerPrefix,
Image: param.Image,
Flavor: param.Flavor,
OSType: param.OSType,

View file

@ -56,6 +56,7 @@ type Pool struct {
Base
ProviderName string `gorm:"index:idx_pool_type"`
RunnerPrefix string
MaxRunners uint
MinIdleRunners uint
RunnerBootstrapTimeout uint

View file

@ -159,6 +159,7 @@ func (s *sqlDatabase) CreateOrganizationPool(ctx context.Context, orgId string,
ProviderName: param.ProviderName,
MaxRunners: param.MaxRunners,
MinIdleRunners: param.MinIdleRunners,
RunnerPrefix: param.RunnerPrefix,
Image: param.Image,
Flavor: param.Flavor,
OSType: param.OSType,

View file

@ -167,6 +167,7 @@ func (s *sqlDatabase) CreateRepositoryPool(ctx context.Context, repoId string, p
ProviderName: param.ProviderName,
MaxRunners: param.MaxRunners,
MinIdleRunners: param.MinIdleRunners,
RunnerPrefix: param.RunnerPrefix,
Image: param.Image,
Flavor: param.Flavor,
OSType: param.OSType,

View file

@ -110,6 +110,7 @@ func (s *sqlDatabase) sqlToCommonPool(pool Pool) params.Pool {
ProviderName: pool.ProviderName,
MaxRunners: pool.MaxRunners,
MinIdleRunners: pool.MinIdleRunners,
RunnerPrefix: pool.RunnerPrefix,
Image: pool.Image,
Flavor: pool.Flavor,
OSArch: pool.OSArch,
@ -218,6 +219,8 @@ func (s *sqlDatabase) updatePool(pool Pool, param params.UpdatePoolParams) (para
pool.Image = param.Image
}
pool.RunnerPrefix = param.GetRunnerPrefix()
if param.MaxRunners != nil {
pool.MaxRunners = *param.MaxRunners
}

1
go.mod
View file

@ -61,6 +61,7 @@ require (
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.4.0 // indirect
github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 // indirect
github.com/xdg-go/stringprep v1.0.3 // indirect
golang.org/x/net v0.0.0-20220325170049-de3da57026de // indirect
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect

2
go.sum
View file

@ -248,6 +248,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569 h1:xzABM9let0HLLqFypcxvLmlvEciCHL7+Lv+4vwZqecI=
github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569/go.mod h1:2Ly+NIftZN4de9zRmENdYbvPQeaVIYKWpLFStLFEBgI=
github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCOIs=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

View file

@ -138,6 +138,7 @@ type Tag struct {
type Pool struct {
ID string `json:"id"`
RunnerPrefix string `json:"runner_prefix"`
ProviderName string `json:"provider_name"`
MaxRunners uint `json:"max_runners"`
MinIdleRunners uint `json:"min_idle_runners"`

View file

@ -21,6 +21,8 @@ import (
"garm/runner/providers/common"
)
const DefaultRunnerPrefix = "garm"
type InstanceRequest struct {
Name string `json:"name"`
OSType config.OSType `json:"os_type"`
@ -102,10 +104,18 @@ type UpdatePoolParams struct {
RunnerBootstrapTimeout *uint `json:"runner_bootstrap_timeout,omitempty"`
Image string `json:"image"`
Flavor string `json:"flavor"`
RunnerPrefix string `json:"runner_prefix"`
OSType config.OSType `json:"os_type"`
OSArch config.OSArch `json:"os_arch"`
}
func (p *UpdatePoolParams) GetRunnerPrefix() string {
if p.RunnerPrefix == "" {
p.RunnerPrefix = DefaultRunnerPrefix
}
return p.RunnerPrefix
}
type CreateInstanceParams struct {
Name string
OSType config.OSType
@ -119,6 +129,7 @@ type CreateInstanceParams struct {
type CreatePoolParams struct {
ProviderName string `json:"provider_name"`
RunnerPrefix string `json:"runner_prefix"`
MaxRunners uint `json:"max_runners"`
MinIdleRunners uint `json:"min_idle_runners"`
Image string `json:"image"`

View file

@ -32,8 +32,8 @@ import (
providerCommon "garm/runner/providers/common"
"github.com/google/go-github/v48/github"
"github.com/google/uuid"
"github.com/pkg/errors"
"github.com/teris-io/shortid"
)
var (
@ -388,13 +388,20 @@ func (r *basePoolManager) acquireNewInstance(job params.WorkflowJob) error {
return nil
}
// use own alphabet to avoid '-' and '_' in the shortid
const shortidABC = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
func (r *basePoolManager) AddRunner(ctx context.Context, poolID string) error {
pool, err := r.helper.GetPoolByID(poolID)
if err != nil {
return errors.Wrap(err, "fetching pool")
}
name := fmt.Sprintf("garm-%s", uuid.New())
prefix := pool.RunnerPrefix
if prefix == "" {
prefix = params.DefaultRunnerPrefix
}
name := fmt.Sprintf("%s-%s", prefix, shortid.MustNew(0, shortidABC, 42).String())
createParams := params.CreateInstanceParams{
Name: name,

3
vendor/github.com/teris-io/shortid/.gitignore generated vendored Normal file
View file

@ -0,0 +1,3 @@
.idea/
vendor/
Gopkg.lock

19
vendor/github.com/teris-io/shortid/.travis.yml generated vendored Normal file
View file

@ -0,0 +1,19 @@
language: go
arch:
- amd64
- ppc64le
go:
- 1.8
before_install:
- go get
- touch coverage.txt
- pip install --user codecov
script:
- go test -coverprofile=coverage.txt -covermode=atomic ./...
after_success:
- codecov

18
vendor/github.com/teris-io/shortid/LICENSE generated vendored Normal file
View file

@ -0,0 +1,18 @@
MIT License
Permission is hereby granted, free of charge, to any person obtaining a copy of this
software and associated documentation files (the "Software"), to deal in the Software
without restriction, including without limitation the rights to use, copy, modify,
merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be included in all copies
or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

109
vendor/github.com/teris-io/shortid/README.md generated vendored Normal file
View file

@ -0,0 +1,109 @@
[![Build status][buildimage]][build] [![Coverage][codecovimage]][codecov] [![GoReportCard][cardimage]][card] [![API documentation][docsimage]][docs]
# Generator of unique non-sequential short Ids
The package `shortid`enables the generation of short, fully unique,
non-sequential and by default URL friendly Ids at a rate of hundreds of thousand per second. It
guarantees uniqueness during the time period until 2050!
The package is heavily inspired by the node.js [shortid][nodeshortid] library (see more detail below).
The easiest way to start generating Ids is:
fmt.Printf(shortid.Generate())
fmt.Printf(shortid.Generate())
The recommended one is to initialise and reuse a generator specific to a given worker:
sid, err := shortid.New(1, shortid.DefaultABC, 2342)
// then either:
fmt.Printf(sid.Generate())
fmt.Printf(sid.Generate())
// or:
shortid.SetDefault(sid)
// followed by:
fmt.Printf(shortid.Generate())
fmt.Printf(shortid.Generate())
### Id Length
The standard Id length is 9 symbols when generated at a rate of 1 Id per millisecond,
occasionally it reaches 11 (at the rate of a few thousand Ids per millisecond) and very-very
rarely it can go beyond that during continuous generation at full throttle on high-performant
hardware. A test generating 500k Ids at full throttle on conventional hardware generated the
following Ids at the head and the tail (length > 9 is expected for this test):
-NDveu-9Q
iNove6iQ9J
NVDve6-9Q
VVDvc6i99J
NVovc6-QQy
VVoveui9QC
...
tFmGc6iQQs
KpTvcui99k
KFTGcuiQ9p
KFmGeu-Q9O
tFTvcu-QQt
tpTveu-99u
### Life span
The package guarantees the generation of unique Ids with no collisions for 34 years
(1/1/2016-1/1/2050) using the same worker Id within a single (although can be concurrent)
application provided application restarts take longer than 1 millisecond. The package supports
up to 32 workers all providing unique sequences from each other.
### Implementation details
Although heavily inspired by the node.js [shortid][nodeshortid] library this is
not just a Go port. This implementation
* is safe to concurrency (test included);
* does not require any yearly version/epoch resets (test included);
* provides stable Id size over a the whole range of operation at the rate of 1ms (test included);
* guarantees no collisions: due to guaranteed fixed size of Ids between milliseconds and because
multiple requests within the same ms lead to longer Ids with the prefix unique to the ms (tests
included);
* supports 32 instead of 16 workers (test included)
The algorithm uses less randomness than the original node.js implementation, which permits to extend
the life span as well as reduce and guarantee the length. In general terms, each Id has the
following 3 pieces of information encoded: the millisecond since epoch (first 8 symbols, epoch:
1/1/2016), the worker Id (9th symbol), the running concurrent counter within the millisecond (only
if required, spanning over all remaining symbols).
The element of randomness per symbol is 1/2 for the worker and the millisecond data and 0 for the
counter. The original algorithm of the node.js library uses 1/4 throughout. Here 0 means no
randomness, i.e. every value is encoded using a 64-base alphabet directly; 1/2 means one of two
matching symbols of the supplied alphabet is used randomly, 1/4 one of four matching symbols. All
methods accepting the parameters that govern the randomness are exported and can be used to directly
implement an algorithm with e.g. more randomness, but with longer Ids and shorter life spans.
### License and copyright
Copyright (c) 2016. Oleg Sklyar and teris.io. MIT license applies. All rights reserved.
**[Original algorithm][nodeshortid]:** Copyright (c) 2015 Dylan Greene, contributors. The same MIT
license applies. Many thanks to Dylan for putting together the original node.js library, which
inspired this "port":
**Seed computation:** based on The Central Randomizer 1.3. Copyright (c) 1997 Paul Houle (houle@msc.cornell.edu)
[go]: https://golang.org
[nodeshortid]: https://github.com/dylang/shortid
[build]: https://travis-ci.org/teris-io/shortid
[buildimage]: https://travis-ci.org/teris-io/shortid.svg?branch=master
[codecov]: https://codecov.io/github/teris-io/shortid?branch=master
[codecovimage]: https://codecov.io/github/teris-io/shortid/coverage.svg?branch=master
[card]: http://goreportcard.com/report/teris-io/shortid
[cardimage]: https://goreportcard.com/badge/github.com/teris-io/shortid
[docs]: https://godoc.org/github.com/teris-io/shortid
[docsimage]: http://img.shields.io/badge/godoc-reference-blue.svg?style=flat

362
vendor/github.com/teris-io/shortid/shortid.go generated vendored Normal file
View file

@ -0,0 +1,362 @@
// Copyright (c) 2016-2017. Oleg Sklyar & teris.io. All rights reserved.
// See the LICENSE file in the project root for licensing information.
// Original algorithm:
// Copyright (c) 2015 Dylan Greene, contributors: https://github.com/dylang/shortid.
// MIT-license as found in the LICENSE file.
// Seed computation: based on The Central Randomizer 1.3
// Copyright (c) 1997 Paul Houle (houle@msc.cornell.edu)
// Package shortid enables the generation of short, unique, non-sequential and by default URL friendly
// Ids. The package is heavily inspired by the node.js https://github.com/dylang/shortid library.
//
// Id Length
//
// The standard Id length is 9 symbols when generated at a rate of 1 Id per millisecond,
// occasionally it reaches 11 (at the rate of a few thousand Ids per millisecond) and very-very
// rarely it can go beyond that during continuous generation at full throttle on high-performant
// hardware. A test generating 500k Ids at full throttle on conventional hardware generated the
// following Ids at the head and the tail (length > 9 is expected for this test):
//
// -NDveu-9Q
// iNove6iQ9J
// NVDve6-9Q
// VVDvc6i99J
// NVovc6-QQy
// VVoveui9QC
// ...
// tFmGc6iQQs
// KpTvcui99k
// KFTGcuiQ9p
// KFmGeu-Q9O
// tFTvcu-QQt
// tpTveu-99u
//
// Life span
//
// The package guarantees the generation of unique Ids with zero collisions for 34 years
// (1/1/2016-1/1/2050) using the same worker Id within a single (although concurrent) application if
// application restarts take longer than 1 millisecond. The package supports up to 32 works, all
// providing unique sequences.
//
// Implementation details
//
// Although heavily inspired by the node.js shortid library this is
// not a simple Go port. In addition it
//
// - is safe to concurrency;
// - does not require any yearly version/epoch resets;
// - provides stable Id size over a long period at the rate of 1ms;
// - guarantees no collisions (due to guaranteed fixed size of Ids between milliseconds and because
// multiple requests within the same ms lead to longer Ids with the prefix unique to the ms);
// - supports 32 over 16 workers.
//
// The algorithm uses less randomness than the original node.js implementation, which permits to
// extend the life span as well as reduce and guarantee the length. In general terms, each Id
// has the following 3 pieces of information encoded: the millisecond (first 8 symbols), the worker
// Id (9th symbol), running concurrent counter within the same millisecond, only if required, over
// all remaining symbols. The element of randomness per symbol is 1/2 for the worker and the
// millisecond and 0 for the counter. Here 0 means no randomness, i.e. every value is encoded using
// a 64-base alphabet; 1/2 means one of two matching symbols of the supplied alphabet, 1/4 one of
// four matching symbols. The original algorithm of the node.js module uses 1/4 throughout.
//
// All methods accepting the parameters that govern the randomness are exported and can be used
// to directly implement an algorithm with e.g. more randomness, but with longer Ids and shorter
// life spans.
package shortid
import (
randc "crypto/rand"
"errors"
"fmt"
"math"
randm "math/rand"
"sync"
"sync/atomic"
"time"
"unsafe"
)
// Version defined the library version.
const Version = 1.1
// DefaultABC is the default URL-friendly alphabet.
const DefaultABC = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_-"
// Abc represents a shuffled alphabet used to generate the Ids and provides methods to
// encode data.
type Abc struct {
alphabet []rune
}
// Shortid type represents a short Id generator working with a given alphabet.
type Shortid struct {
abc Abc
worker uint
epoch time.Time // ids can be generated for 34 years since this date
ms uint // ms since epoch for the last id
count uint // request count within the same ms
mx sync.Mutex // locks access to ms and count
}
var shortid *Shortid
func init() {
shortid = MustNew(0, DefaultABC, 1)
}
// GetDefault retrieves the default short Id generator initialised with the default alphabet,
// worker=0 and seed=1. The default can be overwritten using SetDefault.
func GetDefault() *Shortid {
return (*Shortid)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&shortid))))
}
// SetDefault overwrites the default generator.
func SetDefault(sid *Shortid) {
target := (*unsafe.Pointer)(unsafe.Pointer(&shortid))
source := unsafe.Pointer(sid)
atomic.SwapPointer(target, source)
}
// Generate generates an Id using the default generator.
func Generate() (string, error) {
return shortid.Generate()
}
// MustGenerate acts just like Generate, but panics instead of returning errors.
func MustGenerate() string {
id, err := Generate()
if err == nil {
return id
}
panic(err)
}
// New constructs an instance of the short Id generator for the given worker number [0,31], alphabet
// (64 unique symbols) and seed value (to shuffle the alphabet). The worker number should be
// different for multiple or distributed processes generating Ids into the same data space. The
// seed, on contrary, should be identical.
func New(worker uint8, alphabet string, seed uint64) (*Shortid, error) {
if worker > 31 {
return nil, errors.New("expected worker in the range [0,31]")
}
abc, err := NewAbc(alphabet, seed)
if err == nil {
sid := &Shortid{
abc: abc,
worker: uint(worker),
epoch: time.Date(2016, time.January, 1, 0, 0, 0, 0, time.UTC),
ms: 0,
count: 0,
}
return sid, nil
}
return nil, err
}
// MustNew acts just like New, but panics instead of returning errors.
func MustNew(worker uint8, alphabet string, seed uint64) *Shortid {
sid, err := New(worker, alphabet, seed)
if err == nil {
return sid
}
panic(err)
}
// Generate generates a new short Id.
func (sid *Shortid) Generate() (string, error) {
return sid.GenerateInternal(nil, sid.epoch)
}
// MustGenerate acts just like Generate, but panics instead of returning errors.
func (sid *Shortid) MustGenerate() string {
id, err := sid.Generate()
if err == nil {
return id
}
panic(err)
}
// GenerateInternal should only be used for testing purposes.
func (sid *Shortid) GenerateInternal(tm *time.Time, epoch time.Time) (string, error) {
ms, count := sid.getMsAndCounter(tm, epoch)
idrunes := make([]rune, 9)
if tmp, err := sid.abc.Encode(ms, 8, 5); err == nil {
copy(idrunes, tmp) // first 8 symbols
} else {
return "", err
}
if tmp, err := sid.abc.Encode(sid.worker, 1, 5); err == nil {
idrunes[8] = tmp[0]
} else {
return "", err
}
if count > 0 {
if countrunes, err := sid.abc.Encode(count, 0, 6); err == nil {
// only extend if really need it
idrunes = append(idrunes, countrunes...)
} else {
return "", err
}
}
return string(idrunes), nil
}
func (sid *Shortid) getMsAndCounter(tm *time.Time, epoch time.Time) (uint, uint) {
sid.mx.Lock()
defer sid.mx.Unlock()
var ms uint
if tm != nil {
ms = uint(tm.Sub(epoch).Nanoseconds() / 1000000)
} else {
ms = uint(time.Now().Sub(epoch).Nanoseconds() / 1000000)
}
if ms == sid.ms {
sid.count++
} else {
sid.count = 0
sid.ms = ms
}
return sid.ms, sid.count
}
// String returns a string representation of the short Id generator.
func (sid *Shortid) String() string {
return fmt.Sprintf("Shortid(worker=%v, epoch=%v, abc=%v)", sid.worker, sid.epoch, sid.abc)
}
// Abc returns the instance of alphabet used for representing the Ids.
func (sid *Shortid) Abc() Abc {
return sid.abc
}
// Epoch returns the value of epoch used as the beginning of millisecond counting (normally
// 2016-01-01 00:00:00 local time)
func (sid *Shortid) Epoch() time.Time {
return sid.epoch
}
// Worker returns the value of worker for this short Id generator.
func (sid *Shortid) Worker() uint {
return sid.worker
}
// NewAbc constructs a new instance of shuffled alphabet to be used for Id representation.
func NewAbc(alphabet string, seed uint64) (Abc, error) {
runes := []rune(alphabet)
if len(runes) != len(DefaultABC) {
return Abc{}, fmt.Errorf("alphabet must contain %v unique characters", len(DefaultABC))
}
if nonUnique(runes) {
return Abc{}, errors.New("alphabet must contain unique characters only")
}
abc := Abc{alphabet: nil}
abc.shuffle(alphabet, seed)
return abc, nil
}
// MustNewAbc acts just like NewAbc, but panics instead of returning errors.
func MustNewAbc(alphabet string, seed uint64) Abc {
res, err := NewAbc(alphabet, seed)
if err == nil {
return res
}
panic(err)
}
func nonUnique(runes []rune) bool {
found := make(map[rune]struct{})
for _, r := range runes {
if _, seen := found[r]; !seen {
found[r] = struct{}{}
}
}
return len(found) < len(runes)
}
func (abc *Abc) shuffle(alphabet string, seed uint64) {
source := []rune(alphabet)
for len(source) > 1 {
seed = (seed*9301 + 49297) % 233280
i := int(seed * uint64(len(source)) / 233280)
abc.alphabet = append(abc.alphabet, source[i])
source = append(source[:i], source[i+1:]...)
}
abc.alphabet = append(abc.alphabet, source[0])
}
// Encode encodes a given value into a slice of runes of length nsymbols. In case nsymbols==0, the
// length of the result is automatically computed from data. Even if fewer symbols is required to
// encode the data than nsymbols, all positions are used encoding 0 where required to guarantee
// uniqueness in case further data is added to the sequence. The value of digits [4,6] represents
// represents n in 2^n, which defines how much randomness flows into the algorithm: 4 -- every value
// can be represented by 4 symbols in the alphabet (permitting at most 16 values), 5 -- every value
// can be represented by 2 symbols in the alphabet (permitting at most 32 values), 6 -- every value
// is represented by exactly 1 symbol with no randomness (permitting 64 values).
func (abc *Abc) Encode(val, nsymbols, digits uint) ([]rune, error) {
if digits < 4 || 6 < digits {
return nil, fmt.Errorf("allowed digits range [4,6], found %v", digits)
}
var computedSize uint = 1
if val >= 1 {
computedSize = uint(math.Log2(float64(val)))/digits + 1
}
if nsymbols == 0 {
nsymbols = computedSize
} else if nsymbols < computedSize {
return nil, fmt.Errorf("cannot accommodate data, need %v digits, got %v", computedSize, nsymbols)
}
mask := 1<<digits - 1
random := make([]int, int(nsymbols))
// no random component if digits == 6
if digits < 6 {
copy(random, maskedRandomInts(len(random), 0x3f-mask))
}
res := make([]rune, int(nsymbols))
for i := range res {
shift := digits * uint(i)
index := (int(val>>shift) & mask) | random[i]
res[i] = abc.alphabet[index]
}
return res, nil
}
// MustEncode acts just like Encode, but panics instead of returning errors.
func (abc *Abc) MustEncode(val, size, digits uint) []rune {
res, err := abc.Encode(val, size, digits)
if err == nil {
return res
}
panic(err)
}
func maskedRandomInts(size, mask int) []int {
ints := make([]int, size)
bytes := make([]byte, size)
if _, err := randc.Read(bytes); err == nil {
for i, b := range bytes {
ints[i] = int(b) & mask
}
} else {
for i := range ints {
ints[i] = randm.Intn(0xff) & mask
}
}
return ints
}
// String returns a string representation of the Abc instance.
func (abc Abc) String() string {
return fmt.Sprintf("Abc{alphabet='%v')", abc.Alphabet())
}
// Alphabet returns the alphabet used as an immutable string.
func (abc Abc) Alphabet() string {
return string(abc.alphabet)
}

3
vendor/modules.txt vendored
View file

@ -168,6 +168,9 @@ github.com/stretchr/testify/assert
github.com/stretchr/testify/mock
github.com/stretchr/testify/require
github.com/stretchr/testify/suite
# github.com/teris-io/shortid v0.0.0-20220617161101-71ec9f2aa569
## explicit; go 1.18
github.com/teris-io/shortid
# github.com/xdg-go/stringprep v1.0.3
## explicit; go 1.11
# golang.org/x/crypto v0.0.0-20220321153916-2c7772ba3064