diff --git a/.buildkite/pipeline.yaml b/.buildkite/pipeline.yaml index 8d2ca3568..c4cac17ac 100644 --- a/.buildkite/pipeline.yaml +++ b/.buildkite/pipeline.yaml @@ -12,10 +12,6 @@ steps: command: .buildkite/prettier-check.sh agents: { queue: "vagrant" } - - label: ":git: :sleuth_or_spy:" - command: .buildkite/verify-release/verify-release.sh - agents: { queue: standard } - - label: ":rice: docker-compose-test" command: .buildkite/vagrant-run.sh docker-test artifact_paths: ./*.log diff --git a/.buildkite/verify-release/go.mod b/.buildkite/verify-release/go.mod deleted file mode 100644 index 1cfee15ff..000000000 --- a/.buildkite/verify-release/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module verify-release - -go 1.16 - -require golang.org/x/mod v0.4.2 diff --git a/.buildkite/verify-release/go.sum b/.buildkite/verify-release/go.sum deleted file mode 100644 index b52a2e0a7..000000000 --- a/.buildkite/verify-release/go.sum +++ /dev/null @@ -1,13 +0,0 @@ -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= -golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/.buildkite/verify-release/verify-release.go b/.buildkite/verify-release/verify-release.go deleted file mode 100755 index 01818c3f4..000000000 --- a/.buildkite/verify-release/verify-release.go +++ /dev/null @@ -1,152 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "io/fs" - "log" - "os" - "os/exec" - "path/filepath" - "strings" - - "golang.org/x/mod/semver" -) - -var verbose bool - -func main() { - flag.BoolVar(&verbose, "verbose", false, "print all paths visited") - - flag.Parse() - - branch, err := getBranch() - if err != nil { - log.Fatalf("when getting branch %s", err) - } - - if !isReleaseBranch(branch) { - fmt.Fprintf(os.Stderr, "branch %q is not a release branch\n", branch) - os.Exit(0) - } - - paths := flag.Args() - err = validate(paths) - if err != nil { - log.Fatal(err) - } -} - -func validate(paths []string) error { - - var validationErrors []validationError - for _, p := range paths { - err := filepath.WalkDir(p, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return fmt.Errorf("when walking over %q: %s", path, err) - } - - if d.IsDir() { - return nil - } - - ext := filepath.Ext(path) - if !(ext == ".yaml" || ext == ".yml" || ext == ".sh") { - return nil - } - if verbose { - log.Println(path) - } - - b, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("when reading %q: %w", path, err) - } - - contents := string(b) - - for _, tag := range []string{"insiders", "latest"} { - if strings.Contains(contents, fmt.Sprintf(":%s", tag)) { - validationErrors = append(validationErrors, validationError{path, tag}) - } - } - return nil - }) - - if err != nil { - return err - } - - } - - if len(validationErrors) > 0 { - message := "FILES WITH INVALID TAGS:" - - for _, e := range validationErrors { - message = fmt.Sprintf("%s\n%s", message, e.Error()) - } - - return fmt.Errorf(message) - } - - return nil -} - -type validationError struct { - path string - tag string -} - -func (e *validationError) Error() string { - return fmt.Sprintf("%s: %q", e.path, e.tag) -} - -func isReleaseBranch(branch string) bool { - if verbose { - log.Printf("branch = %q", branch) - } - - version := strings.TrimPrefix(branch, "publish-") - - if !strings.HasPrefix(version, "v") { - version = fmt.Sprintf("v%s", version) - } - - return semver.IsValid(version) -} - -func getBranch() (string, error) { - // tag, branch, git else fail - tag := os.Getenv("BUILDKITE_TAG") - if tag != "" { - return tag, nil - } - - branch := os.Getenv("BUILDKITE_BRANCH") - if branch != "" { - return branch, nil - } - - branch, err := branchFromGit() - if err != nil { - return "", fmt.Errorf("when running git: %w", err) - } - - if branch != "" { - return branch, nil - } - - return "", fmt.Errorf("unable to determine branch") - -} - -func branchFromGit() (string, error) { - cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD") - out, err := cmd.Output() - if err != nil { - return "", err - } - - branch := string(out) - return strings.TrimSpace(branch), nil -} diff --git a/.buildkite/verify-release/verify-release.sh b/.buildkite/verify-release/verify-release.sh deleted file mode 100755 index 2e48672c0..000000000 --- a/.buildkite/verify-release/verify-release.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -cd "$(dirname "${BASH_SOURCE[0]}")/../.." -set -euxo pipefail - -ROOT="$(pwd)" -cd .buildkite/verify-release - -echo "--- Check to see if semver tag are set in release branch" -go run verify-release.go -verbose=true "${ROOT}" diff --git a/.tool-versions b/.tool-versions index 4df9eb887..da16608db 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,6 +1,5 @@ nodejs 16.7.0 yarn 1.22.4 shellcheck 0.7.1 -golang 1.19.8 github-cli 2.46.0 asdf:bosmak/asdf-checkov latest diff --git a/go.mod b/go.mod deleted file mode 100644 index f1e6ab114..000000000 --- a/go.mod +++ /dev/null @@ -1,30 +0,0 @@ -module github.com/sourcegraph/deploy-sourcegraph-docker - -go 1.19 - -require ( - github.com/Masterminds/semver v1.5.0 - github.com/sourcegraph/run v0.12.0 - github.com/sourcegraph/sourcegraph/dev/ci/images v0.0.0-20240222005846-e3158c0c2717 - github.com/sourcegraph/update-docker-tags v0.10.0 - github.com/urfave/cli/v2 v2.25.7 -) - -require ( - bitbucket.org/creachadair/shell v0.0.7 // indirect - github.com/Masterminds/semver/v3 v3.1.0 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/djherbis/buffer v1.2.0 // indirect - github.com/djherbis/nio/v3 v3.0.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/itchyny/gojq v0.12.11 // indirect - github.com/itchyny/timefmt-go v0.1.5 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/rogpeppe/go-internal v1.8.1 // indirect - github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - go.bobheadxi.dev/streamline v1.2.1 // indirect - go.opentelemetry.io/otel v1.11.0 // indirect - go.opentelemetry.io/otel/trace v1.11.0 // indirect -) diff --git a/go.sum b/go.sum deleted file mode 100644 index 616ac9a8e..000000000 --- a/go.sum +++ /dev/null @@ -1,74 +0,0 @@ -bitbucket.org/creachadair/shell v0.0.7 h1:Z96pB6DkSb7F3Y3BBnJeOZH2gazyMTWlvecSD4vDqfk= -bitbucket.org/creachadair/shell v0.0.7/go.mod h1:oqtXSSvSYr4624lnnabXHaBsYW6RD80caLi2b3hJk0U= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/semver/v3 v3.1.0 h1:Y2lUDsFKVRSYGojLJ1yLxSXdMmMYTYls0rCvoqmMUQk= -github.com/Masterminds/semver/v3 v3.1.0/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/djherbis/buffer v1.1.0/go.mod h1:VwN8VdFkMY0DCALdY8o00d3IZ6Amz/UNVMWcSaJT44o= -github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ= -github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= -github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= -github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/hexops/autogold v1.3.1 h1:YgxF9OHWbEIUjhDbpnLhgVsjUDsiHDTyDfy2lrfdlzo= -github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= -github.com/hexops/valast v1.4.3 h1:oBoGERMJh6UZdRc6cduE1CTPK+VAdXA59Y1HFgu3sm0= -github.com/itchyny/gojq v0.12.11 h1:YhLueoHhHiN4mkfM+3AyJV6EPcCxKZsOnYf+aVSwaQw= -github.com/itchyny/gojq v0.12.11/go.mod h1:o3FT8Gkbg/geT4pLI0tF3hvip5F3Y/uskjRz9OYa38g= -github.com/itchyny/timefmt-go v0.1.5 h1:G0INE2la8S6ru/ZI5JecgyzbbJNs5lG1RcBqa7Jm6GE= -github.com/itchyny/timefmt-go v0.1.5/go.mod h1:nEP7L+2YmAbT2kZ2HfSs1d8Xtw9LY8D2stDBckWakZ8= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= -github.com/nightlyone/lockfile v1.0.0 h1:RHep2cFKK4PonZJDdEl4GmkabuhbsRMgk/k3uAmxBiA= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= -github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sourcegraph/run v0.12.0 h1:3A8w5e8HIYPfafHekvmdmmh42RHKGVhmiTZAPJclg7I= -github.com/sourcegraph/run v0.12.0/go.mod h1:PwaP936BTnAJC1cqR5rSbG5kOs/EWStTK3lqvMX5GUA= -github.com/sourcegraph/sourcegraph/dev/ci/images v0.0.0-20231101211343-d046aeda10e4 h1:Mi4SkXsHkJBGrzbrarAU0gCT/GpwP4OnAVrqynAUjoQ= -github.com/sourcegraph/sourcegraph/dev/ci/images v0.0.0-20231101211343-d046aeda10e4/go.mod h1:HpOGaRM0iHPNrEQvpB4/3E9dAUjutyhgjg9vrOOeXPY= -github.com/sourcegraph/sourcegraph/dev/ci/images v0.0.0-20240222005846-e3158c0c2717 h1:xzO3kAjylr2urK0w5Yu95n98jSwpTQ4w/rvf8QA5pwY= -github.com/sourcegraph/sourcegraph/dev/ci/images v0.0.0-20240222005846-e3158c0c2717/go.mod h1:HpOGaRM0iHPNrEQvpB4/3E9dAUjutyhgjg9vrOOeXPY= -github.com/sourcegraph/update-docker-tags v0.10.0 h1:KpY1HtvSt0FzDM/sR0jQF5StNbiPKpqmHhE0jpBEa0A= -github.com/sourcegraph/update-docker-tags v0.10.0/go.mod h1:lCDsoUnk/ASDbm7UxE7tU3qAUhQxA7xykELHXcLeKFM= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= -github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= -github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= -go.bobheadxi.dev/streamline v1.2.1 h1:IqKSA1TbeuDqCzYNAwtlh8sqf3tsQus8XgJdkCWFT8c= -go.bobheadxi.dev/streamline v1.2.1/go.mod h1:yJsVXOSBFLgAKvsnf6WmIzmB2A65nWqkR/sRNxJPa74= -go.opentelemetry.io/otel v1.11.0 h1:kfToEGMDq6TrVrJ9Vht84Y8y9enykSZzDDZglV0kIEk= -go.opentelemetry.io/otel v1.11.0/go.mod h1:H2KtuEphyMvlhZ+F7tg9GRhAOe60moNx61Ex+WmiKkk= -go.opentelemetry.io/otel/sdk v1.11.0 h1:ZnKIL9V9Ztaq+ME43IUi/eo22mNsb6a7tGfzaOWB5fo= -go.opentelemetry.io/otel/trace v1.11.0 h1:20U/Vj42SX+mASlXLmSGBg6jpI1jQtv682lZtTAOVFI= -go.opentelemetry.io/otel/trace v1.11.0/go.mod h1:nyYjis9jy0gytE9LXGU+/m1sHTKbRY0fX0hulNNDP1U= -golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA= -golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= -golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM= diff --git a/renovate.json b/renovate.json index a58a74a9c..bbd0f43d0 100644 --- a/renovate.json +++ b/renovate.json @@ -21,12 +21,6 @@ "ignoreUnstable": false, "semanticCommits": false, "automerge": true - }, - { - "groupName": "Sourcegraph Docker images list", - "packagePatterns": ["github.com/sourcegraph/sourcegraph/enterprise/dev/ci/images"], - "allowedVersions": "~=0.0.0", - "automerge": true } ], "masterIssue": true, diff --git a/test/upgrade-test.go b/test/upgrade-test.go deleted file mode 100644 index e544b9b0f..000000000 --- a/test/upgrade-test.go +++ /dev/null @@ -1,574 +0,0 @@ -package main - -import ( - "bytes" - "context" - "errors" - "fmt" - "log" - "os" - "path/filepath" - "sort" - "strings" - "time" - - "github.com/Masterminds/semver" - "github.com/urfave/cli/v2" - - "github.com/sourcegraph/run" -) - -// This is the main entry point for a docker-compose upgrade test tool intended for use within CI/CD pipelines, but also usable by developers for quick smoke tests -func main() { - app := &cli.App{ - Name: "upgrade-test", - Usage: "Upgrade test is an upgrade smoke test for Sourcegraph's docker-compose deployment type.", - Flags: []cli.Flag{ - &cli.BoolFlag{ - Name: "verbose", - Aliases: []string{"v"}, - Usage: "Stream verbose output to stdout.", - }, - }, - Commands: []*cli.Command{ - { - Name: "standard", - Aliases: []string{"std"}, - Usage: "Runs a standard upgrade test. between specified versions \n\nExample:\n\nupgrade-test standard -vs 5.0.0,5.1.0", - Flags: []cli.Flag{ - &cli.StringSliceFlag{ - Name: "versions", - Aliases: []string{"vs"}, - Usage: "A sequence of versions to do standard upgrades through.", - Required: true, - }, - &cli.BoolFlag{ - Name: "order-versions", - Aliases: []string{"ovs"}, - Usage: "Order versions in the sequence provided by the user, before commencing upgrades.", - }, - }, - Action: func(cCtx *cli.Context) error { - ctx := cCtx.Context - // get flags - verbose := cCtx.Bool("verbose") - order := cCtx.Bool("order-versions") - // Process user provided versions - userVersions := cCtx.StringSlice("versions") - var versions []*semver.Version - for _, v := range userVersions { - ver, err := semver.NewVersion(v) - if err != nil { - return fmt.Errorf("Invalid version: %s", v) - } - - versions = append(versions, ver) - } - - env, err := initTestEnv(ctx, verbose) - if err != nil { - return fmt.Errorf("Failed to initialize env: %w", err) - } - defer env.cleanup() - - if err := testStandardUpgrade(ctx, env, verbose, order, versions); err != nil { - return fmt.Errorf("Standard upgrade failed: %w", err) - } - return nil - }, - }, - { - Name: "multiversion", - Aliases: []string{"mvu"}, - Usage: "Runs a multiversion upgrade between two specified versions \n\nExample:\n\nupgrade-test multiversion -f 5.0.0 -t 5.1.0", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "from", - Aliases: []string{"f"}, - Usage: "Sourcegraph version to upgrade from", - Required: true, - }, - &cli.StringFlag{ - Name: "to", - Aliases: []string{"t"}, - Usage: "Sourcegraph version to upgrade to", - Required: true, - }, - }, - Action: func(cCtx *cli.Context) error { - ctx := cCtx.Context - // get flags - verbose := cCtx.Bool("verbose") - from, err := semver.NewVersion(cCtx.String("from")) - if err != nil { - return fmt.Errorf("Invalid 'from' version: %s", cCtx.String("from")) - } - to, err := semver.NewVersion(cCtx.String("to")) - if err != nil { - return fmt.Errorf("Invalid 'to' version: %s", cCtx.String("to")) - } - env, err := initTestEnv(ctx, verbose) - if err != nil { - return fmt.Errorf("Failed to initialize env: %w", err) - } - defer env.cleanup() - - // Test multiversion upgrade between specified versions - if err := testMultiversionUpgrade(ctx, env, verbose, from, to); err != nil { - return fmt.Errorf("Multiversion upgrade failed: %w", err) - } - return nil - }, - }, - }, - } - - if err := app.Run(os.Args); err != nil { - log.Fatal(err) - } - -} - -// Test env is used to track the state of the test environment, run docker-compose commands, and clean up after the test. -type testEnv struct { - ctx context.Context - tmpDir string - dcDir string - verbose bool - cleanup func() -} - -// initTestEnv -// - clones the deploy-sourcegraph-docker repo into a temp directory, -// - prunes ALL docker volumes, -// - kills all running containers -// -// Warning this clears all containers/volumes on the host! -func initTestEnv(ctx context.Context, verbose bool) (*testEnv, error) { - fmt.Println("Initializing test environment ...") - // Clear current docker environment - if err := dockerClean(ctx, verbose); err != nil { - return nil, fmt.Errorf("failed to clean docker environment during initialization: %w", err) - } - if err := dockerPrune(ctx, verbose); err != nil { - return nil, fmt.Errorf("failed to run docker prune during initialization: %w", err) - } - - // Create a temp directory - tmpDir, err := os.MkdirTemp("", "upgrade-test") - if err != nil { - return nil, fmt.Errorf("Failed to create temp directory: %w", err) - } - if verbose { - fmt.Printf("Created temp directory: %s\n", tmpDir) - } - // Clone the deploy-sourcegraph-docker repo into the tmp directory - if verbose { - err := run.Cmd(ctx, "git", "clone", "https://github.com/sourcegraph/deploy-sourcegraph-docker.git"). - Dir(tmpDir). - Run().Stream(os.Stdout) - if err != nil { - return nil, fmt.Errorf("failed to clone deploy-sourcegraph-docker repo: %w", err) - } - fmt.Println("Cloned deploy-sourcegraph-docker repo to: ", filepath.Join(tmpDir, "deploy-sourcegraph-docker")) - } else { - if err := run.Cmd(ctx, "git", "clone", "https://github.com/sourcegraph/deploy-sourcegraph-docker.git"). - Dir(tmpDir).Run().Wait(); err != nil { - return nil, fmt.Errorf("failed to clone deploy-sourcegraph-docker repo: %w", err) - } - } - // Create cleanup function to remove the temp directory - cleanup := func() { - if err := os.RemoveAll(tmpDir); err != nil { - fmt.Printf("Failed to remove temp directory(%s): %v", tmpDir, err) - } - if err := dockerClean(ctx, verbose); err != nil { - fmt.Printf("failed to clean docker environment after testing: %v", err) - } - return - } - // do init things - return &testEnv{ - ctx: ctx, - tmpDir: tmpDir, - dcDir: filepath.Join(tmpDir, "deploy-sourcegraph-docker", "docker-compose"), - verbose: verbose, - cleanup: cleanup, - }, nil -} - -// checkout a version tag -func (env *testEnv) gitCheckoutVersion(version *semver.Version) error { - fmt.Println("Checking out version " + fmt.Sprintf("v%s", version.String())) - if env.verbose { - err := run.Cmd(env.ctx, "git", "checkout", fmt.Sprintf("v%s", version.String())). - Dir(env.dcDir). - Run().Stream(os.Stdout) - if err != nil { - return fmt.Errorf("failed to checkout version %s: %s", fmt.Sprintf("v%s", version.String()), err) - } - } else { - err := run.Cmd(env.ctx, "git", "checkout", fmt.Sprintf("v%s", version.String())). - Dir(env.dcDir). - Run().Wait() - if err != nil { - return fmt.Errorf("failed to checkout version %s: %s", fmt.Sprintf("v%s", version.String()), err) - } - } - return nil -} - -// Get latest migrator version -func (env *testEnv) getLatestMigrator() (*semver.Version, error) { - tags, err := run.Cmd(env.ctx, "git", "for-each-ref", "--format", "'%(refname:short)'", "refs/tags"). - Dir(env.dcDir).Run().Lines() - if err != nil { - return nil, err - } - // Loop through tags and find the latest semver tag - var latest *semver.Version - for _, tag := range tags { - v, err := semver.NewVersion(tag) - if err != nil { - continue // skip non-matching tags - } - - if latest == nil || v.GreaterThan(latest) { - latest = v - } - } - - if latest == nil { - return nil, errors.New("No valid semver tags found") - } - - if env.verbose { - fmt.Println("Latest version found: " + latest.String()) - } - return latest, nil -} - -// composeUpTimeout creates a timed context to handle for the failure of migrator during `docker-compose up` -// Migrator "up" in docker-compose can crash loop if it fails, with the migrator continually trying to run the "up" command -// and the frontend waiting to initialize after migrator's successful completion. -// -// This command has a 80 sec timeout and will check the migrator logs to display the failure if the timeout is reached. -func (env *testEnv) composeUpTimeout(images ...string) error { - timeout := 80 * time.Second - tCtx, cancel := context.WithTimeout(env.ctx, timeout) - defer cancel() - - done := make(chan error) - - go func() { - err := env.composeUp(images...) - done <- err - }() - - select { - case err := <-done: - return err - case <-tCtx.Done(): - fmt.Println("\nDocker-compose up timed out afer 60 seconds, checking migrator logs for failure...") - if err := run.Cmd(env.ctx, "docker", "logs", "migrator").Run().Stream(os.Stdout); err != nil { - return fmt.Errorf("Error checking migrator logs: %s\n", err) - } - return fmt.Errorf("migrator failed docker-compose up after timeout, check logs for details.)") - } -} - -// Docker compose up -func (env *testEnv) composeUp(images ...string) error { - fmt.Println("Starting docker-compose up...") - if env.verbose { - if err := run.Cmd(env.ctx, append([]string{"docker-compose", "up", "-d"}, images...)...). - Dir(env.dcDir). - Run().Stream(os.Stdout); err != nil { - return fmt.Errorf("failed to run docker-compose up: %w", err) - } - } else { - if err := run.Cmd(env.ctx, append([]string{"docker-compose", "up", "-d"}, images...)...). - Dir(env.dcDir). - Run().Wait(); err != nil { - return fmt.Errorf("failed to run docker-compose up: %w", err) - } - } - return nil -} - -// Docker compose up -func (env *testEnv) composeDown(images ...string) error { - fmt.Println("Starting docker-compose down...") - if env.verbose { - if err := run.Cmd(env.ctx, append([]string{"docker-compose", "down", "--remove-orphans"}, images...)...). - Dir(env.dcDir). - Run().Stream(os.Stdout); err != nil { - return fmt.Errorf("failed to run docker-compose down: %w", err) - } - } else { - if err := run.Cmd(env.ctx, append([]string{"docker-compose", "down", "--remove-orphans"}, images...)...). - Dir(env.dcDir). - Run().Wait(); err != nil { - return fmt.Errorf("failed to run docker-compose down: %w", err) - } - } - return nil -} - -// Standard upgrade test tests the migrator `up` command, iterating over a slice of versions in the order provided. -func testStandardUpgrade(ctx context.Context, env *testEnv, verbose, ordered bool, versions []*semver.Version) error { - if ordered { - sort.Sort(semver.Collection(versions)) - } - for _, version := range versions { - // git checkout version tag - if err := env.gitCheckoutVersion(version); err != nil { - return fmt.Errorf("failed to checkout version %s: %s", version, err) - } - // Bring up instance with migrator "up" default - if err := env.composeUpTimeout(); err != nil { - return fmt.Errorf("failed to run docker-compose up at version %s: %s", version, err) - } - // Check for drift and other "up" operations - if err := validateUpgrade(ctx, env, verbose, version); err != nil { - return fmt.Errorf("Error validating upgrade: %s", err) - } - // Bring down deployment - if err := env.composeDown(); err != nil { - return fmt.Errorf("failed to run docker-compose down at version %s: %s", version, err) - } - } - return nil -} - -// Multiversion upgrade test tests the migrator `upgrade` command. -func testMultiversionUpgrade(ctx context.Context, env *testEnv, verbose bool, from, to *semver.Version) error { - // git checkout and initialize deployment - if err := env.gitCheckoutVersion(from); err != nil { - return fmt.Errorf("failed to checkout version %s: %s", from, err) - } - if err := env.composeUpTimeout(); err != nil { - return fmt.Errorf("failed to run docker-compose up %s init: %s", fmt.Sprintf("v%s", from.String()), err) - } - // Check for schema drift init of "from" version - if err := migratorDrift(ctx, env, verbose, from, "frontend"); err != nil { - return fmt.Errorf("schema drift detected on initial deploy at %s: %s", fmt.Sprintf("v%s", from.String()), err) - } - if err := migratorDrift(ctx, env, verbose, from, "codeintel"); err != nil { - return fmt.Errorf("schema drift detected on initial deploy at %s: %s", fmt.Sprintf("v%s", from.String()), err) - } - if err := migratorDrift(ctx, env, verbose, from, "codeinsights"); err != nil { - return fmt.Errorf("schema drift detected on initial deploy at %s: %s", fmt.Sprintf("v%s", from.String()), err) - } - // Bring down init deployment - if err := env.composeDown(); err != nil { - return fmt.Errorf("failed to run docker-compose down at %s: %s", from, err) - } - - // Start just the dbs to be targeted by migrator - if err := env.composeUp("pgsql", "codeintel-db", "codeinsights-db"); err != nil { - return fmt.Errorf("failed to run docker-compose up for databases at %s: %s", fmt.Sprintf("v%s", from.String()), err) - } - - // Here we run the migrator with the "upgrade" command, first we check the version of "to", - // this is because in v5.1.0 a check is added to ensure migrator is always run at the latest possible image version. - // For our testing migrator will run the upgrade using the specified "to" version. - c, err := semver.NewConstraint(">= 5.1.0") - if err != nil { - return fmt.Errorf("failed to create semver constraint: %s", err) - } - if c.Check(to) { - if err := migratorUpgrade(ctx, verbose, from, to, "-ignore-migrator-update"); err != nil { - return fmt.Errorf("multiversion upgrade from %s to %s failed: %s", from, to, err) - } - } else { - if err := migratorUpgrade(ctx, verbose, from, to); err != nil { - return fmt.Errorf("multiversion upgrade from %s to %s failed: %s", from, to, err) - } - } - // git checkout "to" version and start the deployment - if err := env.gitCheckoutVersion(to); err != nil { - return fmt.Errorf("failed to checkout version %s: %s", to, err) - } - if err := env.composeUpTimeout(); err != nil { - return fmt.Errorf("failed to run final docker-compose up at %s: %s", to, err) - } - // Validate the migrator "upgrade" command accomplished required tasks - if err := validateUpgrade(ctx, env, verbose, to); err != nil { - return fmt.Errorf("Error validating upgrade: %s", err) - } - if err := env.composeDown(); err != nil { - return fmt.Errorf("failed to run docker-compose down after upgrade: %s", err) - } - return nil -} - -// Ensure that the expected steps have correctly executed durring the upgradenv. -// - Check versions.version has been updated -// - Check migration_logs for failures -// - Check for schema drift -func validateUpgrade(ctx context.Context, env *testEnv, verbose bool, version *semver.Version) error { - fmt.Printf("\nChecking version %s for version update, failed migrations, and drift.", fmt.Sprintf("v%s", version.String())) - - // Validate pgsql database versions.version row was set correctly. - fmt.Printf("\nChecking that versions.version has been updated... ") - var versionFromDB bytes.Buffer - err := run.Cmd(ctx, "docker", "exec", "pgsql", - "psql", "-U", "sg", - "-c", - "'SELECT", "version", "FROM", "versions;'", "-t").Run().Stream(&versionFromDB) - // Validate upgrade is run after an upgrade is performed and takes the upgrade "to" version. - // If the db doesn't return the "to" version, error. - if versionFromDB.String() == version.String() { - return fmt.Errorf("versions.version not updated to %s", fmt.Sprintf("v%s", version.String())) - } else if err != nil { - return fmt.Errorf("failed to validate upgraded version %s: %s", fmt.Sprintf("v%s", version.String()), err) - } - // Check for failed migrations in migration_logs table. Valid after v3.36.0 when table was introduced - // If the count of rows in the "migration_logs" table with column "success"=false is greater than 0, error - c, err := semver.NewConstraint(">= 3.36.0") - if err != nil { - return fmt.Errorf("failed to create semver constraint: %s", err) - } - if c.Check(version) { - fmt.Println("Checking for failed migrations... ") - var numFailedMigrations bytes.Buffer - err := run.Cmd(ctx, "docker", "exec", "pgsql", - "psql", "-U", "sg", - "-c", - "'SELECT", "COUNT(*)", "FROM", "migration_logs", "WHERE", "success=false;'", "-t"). - Run().Stream(&numFailedMigrations) - if err != nil { - return fmt.Errorf("failed to query pgsql for failed migrations count: %s", err) - } - // Check for failed migrations - if strings.TrimSpace(numFailedMigrations.String()) != "0" { - return fmt.Errorf("found failed migrations in migration_logs table for version %s", fmt.Sprintf("v%s", version.String())) - } - } - - // Check for schema drift with docker run migrator ... command drift - if err := migratorDrift(ctx, env, verbose, version, "frontend"); err != nil { - return fmt.Errorf("schema drift detected after multiversion upgrade at %s: %s", fmt.Sprintf("v%s", version.String()), err) - } - if err := migratorDrift(ctx, env, verbose, version, "codeintel"); err != nil { - return fmt.Errorf("schema drift detected after multiversion upgrade at %s: %s", fmt.Sprintf("v%s", version.String()), err) - } - if err := migratorDrift(ctx, env, verbose, version, "codeinsights"); err != nil { - return fmt.Errorf("schema drift detected after multiversion upgrade at %s: %s", fmt.Sprintf("v%s", version.String()), err) - } - - return nil -} - -// Construct the base string for the docker run migrator commands -func migratorBaseString(ctx context.Context, migratorVersion *semver.Version) []string { - migratorBase := []string{"docker", "run", "--rm", - "--name", "migrator_" + migratorVersion.String(), - "-e", "PGHOST=pgsql", - "-e", "PGPORT=5432", - "-e", "PGUSER=sg", - "-e", "PGPASSWORD=sg", - "-e", "PGDATABASE=sg", - "-e", "PGSSLMODE=disable", - "-e", "CODEINTEL_PGHOST=codeintel-db", - "-e", "CODEINTEL_PGPORT=5432", - "-e", "CODEINTEL_PGUSER=sg", - "-e", "CODEINTEL_PGPASSWORD=sg", - "-e", "CODEINTEL_PGDATABASE=sg", - "-e", "CODEINTEL_PGSSLMODE=disable", - "-e", "CODEINSIGHTS_PGHOST=codeinsights-db", - "-e", "CODEINSIGHTS_PGPORT=5432", - "-e", "CODEINSIGHTS_PGUSER=postgres", - "-e", "CODEINSIGHTS_PGPASSWORD=password", - "-e", "CODEINSIGHTS_PGDATABASE=postgres", - "-e", "CODEINSIGHTS_PGSSLMODE=disable", - "--network=docker-compose_sourcegraph", - "sourcegraph/migrator:" + migratorVersion.String(), - } - return migratorBase -} - -// Check drift with docker run migrator, use latest migrator version -func migratorDrift(ctx context.Context, env *testEnv, verbose bool, version *semver.Version, db string, migratorArgs ...string) error { - fmt.Println("Checking for drift with docker run migrator... ") - // Get latest migrator version - tag, err := env.getLatestMigrator() - if err != nil { - return fmt.Errorf("failed to select latest migrator version: %s", err) - } - // Construct the docker run command for migrator - migratorBase := migratorBaseString(ctx, tag) - migratorCmd := append(migratorBase, "drift", "--db="+db, "--version="+fmt.Sprintf("v%s", version.String())) - migratorCmd = append(migratorCmd, migratorArgs...) - if verbose { - fmt.Println("Running... ", migratorCmd) - } - // Run the constructed docker run migrator command - if err = run.Cmd(ctx, migratorCmd...).Run().Stream(os.Stdout); err != nil { - return fmt.Errorf("drift check failed for %s version %s: %s", db, fmt.Sprintf("v%s", version.String()), err) - } else { - fmt.Println("No schema drift found in", db) - return nil - } -} - -// Check drift with docker run migrator -func migratorUpgrade(ctx context.Context, verbose bool, vFrom, vTo *semver.Version, migratorArgs ...string) error { - fmt.Printf("\nPerforming MVU from %s to %s... ", fmt.Sprintf("v%s", vFrom.String()), fmt.Sprintf("v%s", vTo.String())) - // Construct the docker run command for migrator - migratorBase := migratorBaseString(ctx, vTo) - migratorCmd := append(migratorBase, "upgrade", "--from="+fmt.Sprintf("v%s", vFrom.String()), "--to="+fmt.Sprintf("v%s", vTo.String())) - migratorCmd = append(migratorCmd, migratorArgs...) - if verbose { - fmt.Println("Running...\n", migratorCmd) - } - // Run the constructed docker run migrator command - err := run.Cmd(ctx, migratorCmd...).Run().Stream(os.Stdout) - if err != nil { - return fmt.Errorf("multiversion upgrade failed from %s to %s: %s", fmt.Sprintf("v%s", vFrom.String()), fmt.Sprintf("v%s", vTo.String()), err) - } - return nil -} - -// Docker commands - -// Prune docker volumes -func dockerPrune(ctx context.Context, verbose bool) error { - fmt.Println("Pruning docker volumes...") - if verbose { - if err := run.Cmd(ctx, "docker", "volume", "prune", "-a", "-f").Run().Stream(os.Stdout); err != nil { - return fmt.Errorf("failed to prune docker volumes: %s", err) - } - } else { - if err := run.Cmd(ctx, "docker", "volume", "prune", "-a", "-f").Run().Wait(); err != nil { - return fmt.Errorf("failed to prune docker volumes: %s", err) - } - } - return nil -} - -// Stop and clean all docker containers on init -func dockerClean(ctx context.Context, verbose bool) error { - fmt.Println("Stopping and cleaning all docker containers...") - // Get all docker containers to stop and clean - containers, err := run.Cmd(ctx, "docker", "ps", "-aq").Run().Lines() - if err != nil { - return fmt.Errorf("failed to stop and clean all docker containers: %s", err) - } - // Don't clean if there are no containers - if len(containers) != 0 { - // construct the docker rm command for all containers - rmCmd := append([]string{"docker", "rm", "-f"}, containers...) - if verbose { - if err = run.Cmd(ctx, rmCmd...).Run().Stream(os.Stdout); err != nil { - return fmt.Errorf("failed to stop and clean all docker containers: %s", err) - } - } else { - if err := run.Cmd(ctx, rmCmd...).Run().Wait(); err != nil { - return fmt.Errorf("failed to stop and clean all docker containers: %s", err) - } - } - } - return nil -} diff --git a/tools/tools.go b/tools/tools.go deleted file mode 100644 index c2dae5113..000000000 --- a/tools/tools.go +++ /dev/null @@ -1,8 +0,0 @@ -//+build tools - -package tools - -import ( - // pins update-docker-tags CLI - _ "github.com/sourcegraph/update-docker-tags" -)