v1.1.5

.github/workflows/codeql.yml

@@ -0,0 +1,25 @@
+name: "CodeQL analysis"
+
+on: 
+ push:
+ pull_request:
+ schedule:
+ - cron: '0 0 * * 0'
+
+jobs:
+ codeql:
+
+ strategy:
+ fail-fast: false
+
+ runs-on: ubuntu-latest # windows-latest and ubuntu-latest are supported. macos-latest is not supported at this time. 
+
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+
+ - name: Initialize CodeQL 
+ uses: github/codeql-action/init@v1
+
+ - name: Perform CodeQL Analysis
+ uses: github/codeql-action/analyze@v1

.github/workflows/golangci-lint.yml

@@ -0,0 +1,21 @@
+name: golangci-lint
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+permissions:
+ contents: read
+ # Optional: allow read access to pull request. Use with `only-new-issues` option.
+ # pull-requests: read
+jobs:
+ golangci:
+ name: lint
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/setup-go@v3
+ with:
+ go-version: 1.16
+ - uses: actions/checkout@v3
+ - name: golangci-lint
+ uses: golangci/golangci-lint-action@v3

.golangci.yml

@@ -0,0 +1,16 @@
+run:
+ timeout: 5m
+ modules-download-mode: readonly
+
+linters:
+ disable:
+ - errcheck
+ - staticcheck
+ enable:
+ - gosimple
+ - govet
+ - noctx
+ - rowserrcheck
+ - sqlclosecheck
+ - unused
+

RELEASE_VERSION

@@ -1 +1 @@
-1.1.2
+1.1.5

build.sh

@@ -18,31 +18,32 @@ function build {
  GOOS=$3
  GOARCH=$4
 
- if ! go version | egrep -q 'go(1\.1[56])' ; then
+ if ! go version | egrep -q 'go1\.(1[5-9]|[2-9][0-9]{1})' ; then
  echo "go version must be 1.15 or above"
  exit 1
  fi
 
- # TODO: remove GO111MODULE once gh-ost uses Go modules
- echo "Building ${osname} binary"
+ echo "Building ${osname}-${GOARCH} binary"
  export GOOS
  export GOARCH
  GO111MODULE=off go build -ldflags "$ldflags" -o $buildpath/$target go/cmd/gh-ost/main.go
 
  if [ $? -ne 0 ]; then
- echo "Build failed for ${osname}"
+ echo "Build failed for ${osname} ${GOARCH}."
  exit 1
  fi
 
- (cd $buildpath && tar cfz ./gh-ost-binary-${osshort}-${timestamp}.tar.gz $target)
+ (cd $buildpath && tar cfz ./gh-ost-binary-${osshort}-${GOARCH}-${timestamp}.tar.gz $target)
 
- if [ "$GOOS" == "linux" ] ; then
+ # build RPM and deb for Linux, x86-64 only
+ if [ "$GOOS" == "linux" ] && [ "$GOARCH" == "amd64" ] ; then
  echo "Creating Distro full packages"
  builddir=$(setuptree)
  cp $buildpath/$target $builddir/gh-ost/usr/bin
  cd $buildpath
- fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach <[email protected]>' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" .
- fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'shlomi-noach <[email protected]>' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files .
+ fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'GitHub' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t rpm --rpm-rpmbuild-define "_build_id_links none" --rpm-os linux .
+ fpm -v "${RELEASE_VERSION}" --epoch 1 -f -s dir -n gh-ost -m 'GitHub' --description "GitHub's Online Schema Migrations for MySQL " --url "https://github.com/github/gh-ost" --vendor "GitHub" --license "Apache 2.0" -C $builddir/gh-ost --prefix=/ -t deb --deb-no-default-config-files .
+ cd -
  fi
 }
 
@@ -63,10 +64,18 @@ main() {
  mkdir -p ${buildpath}
  rm -rf ${buildpath:?}/*
  build GNU/Linux linux linux amd64
- # build macOS osx darwin amd64
+ build GNU/Linux linux linux arm64
+ build macOS osx darwin amd64
+ build macOS osx darwin arm64
 
  echo "Binaries found in:"
  find $buildpath/gh-ost* -type f -maxdepth 1
+
+ echo "Checksums:"
+ (cd $buildpath && shasum -a256 gh-ost* 2>/dev/null)
 }
 
+. script/bootstrap
+cd .gopath/src/github.com/github/gh-ost
+
 main "$@"

doc/command-line-flags.md

@@ -22,7 +22,7 @@ If, for some reason, you do not wish `gh-ost` to connect to a replica, you may c
 
 ### approve-renamed-columns
 
-When your migration issues a column rename (`change column old_name new_name ...`) `gh-ost` analyzes the statement to try and associate the old column name with new column name. Otherwise the new structure may also look like some column was dropped and another was added.
+When your migration issues a column rename (`change column old_name new_name ...`) `gh-ost` analyzes the statement to try and associate the old column name with new column name. Otherwise, the new structure may also look like some column was dropped and another was added.
 
 `gh-ost` will print out what it thinks the _rename_ implied, but will not issue the migration unless you provide with `--approve-renamed-columns`.
 
@@ -32,7 +32,7 @@ If you think `gh-ost` is mistaken and that there's actually no _rename_ involved
 
 `gh-ost` infers the identity of the master server by crawling up the replication topology. You may explicitly tell `gh-ost` the identity of the master host via `--assume-master-host=the.master.com`. This is useful in:
 
-- _master-master_ topologies (together with [`--allow-master-master`](#allow-master-master)), where `gh-ost` can arbitrarily pick one of the co-masters and you prefer that it picks a specific one
+- _master-master_ topologies (together with [`--allow-master-master`](#allow-master-master)), where `gh-ost` can arbitrarily pick one of the co-masters, and you prefer that it picks a specific one
 - _tungsten replicator_ topologies (together with [`--tungsten`](#tungsten)), where `gh-ost` is unable to crawl and detect the master
 
 ### assume-rbr
@@ -61,7 +61,13 @@ Comma delimited status-name=threshold, same format as [`--max-load`](#max-load).
 
 `--critical-load` defines a threshold that, when met, `gh-ost` panics and bails out. The default behavior is to bail out immediately when meeting this threshold.
 
-This may sometimes lead to migrations bailing out on a very short spike, that, while in itself is impacting production and is worth investigating, isn't reason enough to kill a 10 hour migration.
+This may sometimes lead to migrations bailing out on a very short spike, that, while in itself is impacting production and is worth investigating, isn't reason enough to kill a 10-hour migration.
+
+### critical-load-hibernate-seconds
+
+When `--critical-load-hibernate-seconds` is non-zero (e.g. `--critical-load-hibernate-seconds=300`), `critical-load` does not panic and bail out; instead, `gh-ost` goes into hibernation for the specified duration. It will not read/write anything from/to any server during this time. Execution then continues upon waking from hibernation.
+
+If `critical-load` is met again, `gh-ost` will repeat this cycle, and never panic and bail out.
 
 ### critical-load-interval-millis
 
@@ -98,7 +104,7 @@ Noteworthy is that setting `--dml-batch-size` to higher value _does not_ mean `g
 
 ### exact-rowcount
 
-A `gh-ost` execution need to copy whatever rows you have in your existing table onto the ghost table. This can, and often be, a large number. Exactly what that number is?
+A `gh-ost` execution need to copy whatever rows you have in your existing table onto the ghost table. This can and often will be, a large number. Exactly what that number is?
 `gh-ost` initially estimates the number of rows in your table by issuing an `explain select * from your_table`. This will use statistics on your table and return with a rough estimate. How rough? It might go as low as half or as high as double the actual number of rows in your table. This is the same method as used in [`pt-online-schema-change`](https://www.percona.com/doc/percona-toolkit/2.2/pt-online-schema-change.html).
 
 `gh-ost` also supports the `--exact-rowcount` flag. When this flag is given, two things happen:
@@ -135,6 +141,10 @@ Add this flag when executing on a 1st generation Google Cloud Platform (GCP).
 
 Default 100. See [`subsecond-lag`](subsecond-lag.md) for details.
 
+### hooks-status-interval
+
+Defaults to 60 seconds. Configures how often the `gh-ost-on-status` hook is called, see [`hooks`](hooks.md) for full details on how to use hooks.
+
 ### initially-drop-ghost-table
 
 `gh-ost` maintains two tables while migrating: the _ghost_ table (which is synced from your original table and finally replaces it) and a changelog table, which is used internally for bookkeeping. By default, it panics and aborts if it sees those tables upon startup. Provide `--initially-drop-ghost-table` and `--initially-drop-old-table` to let `gh-ost` know it's OK to drop them beforehand.
@@ -230,7 +240,7 @@ Provide a command delimited list of replicas; `gh-ost` will throttle when any of
 
 ### throttle-http
 
-Provide a HTTP endpoint; `gh-ost` will issue `HEAD` requests on given URL and throttle whenever response status code is not `200`. The URL can be queried and updated dynamically via [interactive commands](interactive-commands.md). Empty URL disables the HTTP check.
+Provide an HTTP endpoint; `gh-ost` will issue `HEAD` requests on given URL and throttle whenever response status code is not `200`. The URL can be queried and updated dynamically via [interactive commands](interactive-commands.md). Empty URL disables the HTTP check.
 
 ### timestamp-old-table
 

doc/hooks.md

@@ -68,6 +68,7 @@ The following variables are available on all hooks:
 - `GH_OST_INSPECTED_LAG` - lag in seconds (floating point) of inspected server
 - `GH_OST_HEARTBEAT_LAG` - lag in seconds (floating point) of heartbeat
 - `GH_OST_PROGRESS` - progress pct ([0..100], floating point) of migration
+- `GH_OST_ETA_SECONDS` - estimated duration until migration finishes in seconds
 - `GH_OST_MIGRATED_HOST`
 - `GH_OST_INSPECTED_HOST`
 - `GH_OST_EXECUTING_HOST`

doc/triggerless-design.md

@@ -112,7 +112,7 @@ It is also interesting to observe that `gh-ost` is the only application writing
 
 When `gh-ost` pauses (throttles), it issues no writes on the ghost table. Because there are no triggers, write workload is decoupled from the `gh-ost` write workload. And because we're using an asynchronous approach, the algorithm already handles a time difference between a master write time and the ghost apply time. A difference of a few microseconds is no different from a difference of minutes or hours.
 
-When `gh-ost` [throttles](throttle.md), either by replication lag, `max-load` setting or and explicit [interactive user command](interactive-commands.md), the master is back to normal. It sees no more writes on the ghost table.
+When `gh-ost` [throttles](throttle.md), either by replication lag, `max-load` setting or an explicit [interactive user command](interactive-commands.md), the master is back to normal. It sees no more writes on the ghost table.
 An exception is the ongoing heartbeat writes onto the changelog table, which we consider to be negligible.
 
 #### Testability

go/base/context.go

@@ -1,5 +1,5 @@
 /*
- Copyright 2016 GitHub Inc.
+ Copyright 2022 GitHub Inc.
  See https://github.com/github/gh-ost/blob/master/LICENSE
 */
 
@@ -15,7 +15,7 @@ import (
  "sync/atomic"
  "time"
 
- "github.com/satori/go.uuid"
+ uuid "github.com/satori/go.uuid"
 
  "github.com/github/gh-ost/go/mysql"
  "github.com/github/gh-ost/go/sql"
@@ -83,6 +83,8 @@ type MigrationContext struct {
  AlterStatement string
  AlterStatementOptions string // anything following the 'ALTER TABLE [schema.]table' from AlterStatement
 
+ countMutex sync.Mutex
+ countTableRowsCancelFunc func()
  CountTableRows bool
  ConcurrentCountTableRows bool
  AllowedRunningOnMaster bool
@@ -141,6 +143,7 @@ type MigrationContext struct {
  HooksHintMessage string
  HooksHintOwner string
  HooksHintToken string
+ HooksStatusIntervalSec int64
 
  DropServeSocket bool
  ServeSocketFile string
@@ -184,7 +187,9 @@ type MigrationContext struct {
  CurrentLag int64
  currentProgress uint64
  etaNanoseonds int64
+ ThrottleHTTPIntervalMillis int64
  ThrottleHTTPStatusCode int64
+ ThrottleHTTPTimeoutMillis int64
  controlReplicasLagResult mysql.ReplicationLagResult
  TotalRowsCopied int64
  TotalDMLEventsApplied int64
@@ -426,6 +431,36 @@ func (this *MigrationContext) IsTransactionalTable() bool {
  return false
 }
 
+// SetCountTableRowsCancelFunc sets the cancel function for the CountTableRows query context
+func (this *MigrationContext) SetCountTableRowsCancelFunc(f func()) {
+ this.countMutex.Lock()
+ defer this.countMutex.Unlock()
+
+ this.countTableRowsCancelFunc = f
+}
+
+// IsCountingTableRows returns true if the migration has a table count query running
+func (this *MigrationContext) IsCountingTableRows() bool {
+ this.countMutex.Lock()
+ defer this.countMutex.Unlock()
+
+ return this.countTableRowsCancelFunc != nil
+}
+
+// CancelTableRowsCount cancels the CountTableRows query context. It is safe to
+// call function even when IsCountingTableRows is false.
+func (this *MigrationContext) CancelTableRowsCount() {
+ this.countMutex.Lock()
+ defer this.countMutex.Unlock()
+
+ if this.countTableRowsCancelFunc == nil {
+ return
+ }
+
+ this.countTableRowsCancelFunc()
+ this.countTableRowsCancelFunc = nil
+}
+
 // ElapsedTime returns time since very beginning of the process
 func (this *MigrationContext) ElapsedTime() time.Duration {
  return time.Since(this.StartTime)
@@ -552,8 +587,8 @@ func (this *MigrationContext) SetMaxLagMillisecondsThrottleThreshold(maxLagMilli
 }
 
 func (this *MigrationContext) SetChunkSize(chunkSize int64) {
- if chunkSize < 100 {
- chunkSize = 100
+ if chunkSize < 10 {
+ chunkSize = 10
  }
  if chunkSize > 100000 {
  chunkSize = 100000

go/base/context_test.go

@@ -1,11 +1,13 @@
 /*
- Copyright 2016 GitHub Inc.
+ Copyright 2022 GitHub Inc.
  See https://github.com/github/gh-ost/blob/master/LICENSE
 */
 
 package base
 
 import (
+ "io/ioutil"
+ "os"
  "testing"
  "time"
 
@@ -56,3 +58,65 @@ func TestGetTableNames(t *testing.T) {
  test.S(t).ExpectEquals(context.GetChangelogTableName(), "_tmp_ghc")
  }
 }
+
+func TestReadConfigFile(t *testing.T) {
+ {
+ context := NewMigrationContext()
+ context.ConfigFile = "/does/not/exist"
+ if err := context.ReadConfigFile(); err == nil {
+ t.Fatal("Expected .ReadConfigFile() to return an error, got nil")
+ }
+ }
+ {
+ f, err := ioutil.TempFile("", t.Name())
+ if err != nil {
+ t.Fatalf("Failed to create tmp file: %v", err)
+ }
+ defer os.Remove(f.Name())
+
+ f.Write([]byte("[client]"))
+ context := NewMigrationContext()
+ context.ConfigFile = f.Name()
+ if err := context.ReadConfigFile(); err != nil {
+ t.Fatalf(".ReadConfigFile() failed: %v", err)
+ }
+ }
+ {
+ f, err := ioutil.TempFile("", t.Name())
+ if err != nil {
+ t.Fatalf("Failed to create tmp file: %v", err)
+ }
+ defer os.Remove(f.Name())
+
+ f.Write([]byte("[client]\nuser=test\npassword=123456"))
+ context := NewMigrationContext()
+ context.ConfigFile = f.Name()
+ if err := context.ReadConfigFile(); err != nil {
+ t.Fatalf(".ReadConfigFile() failed: %v", err)
+ }
+
+ if context.config.Client.User != "test" {
+ t.Fatalf("Expected client user %q, got %q", "test", context.config.Client.User)
+ } else if context.config.Client.Password != "123456" {
+ t.Fatalf("Expected client password %q, got %q", "123456", context.config.Client.Password)
+ }
+ }
+ {
+ f, err := ioutil.TempFile("", t.Name())
+ if err != nil {
+ t.Fatalf("Failed to create tmp file: %v", err)
+ }
+ defer os.Remove(f.Name())
+
+ f.Write([]byte("[osc]\nmax_load=10"))
+ context := NewMigrationContext()
+ context.ConfigFile = f.Name()
+ if err := context.ReadConfigFile(); err != nil {
+ t.Fatalf(".ReadConfigFile() failed: %v", err)
+ }
+
+ if context.config.Osc.Max_Load != "10" {
+ t.Fatalf("Expected osc 'max_load' %q, got %q", "10", context.config.Osc.Max_Load)
+ }
+ }
+}