Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: shmel1k/qumomf
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: v1.2.1
Choose a base ref
...
head repository: shmel1k/qumomf
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: master
Choose a head ref
  • 3 commits
  • 4 files changed
  • 2 contributors

Commits on Aug 4, 2021

  1. [analysis] add dead followers to logs

    Aleksandr Petrukhin committed Aug 4, 2021
    Copy the full SHA
    1f09fa4 View commit details

Commits on Aug 6, 2021

  1. [failover] fix missing logs

    Aleksandr Petrukhin committed Aug 6, 2021
    Copy the full SHA
    20beb45 View commit details
  2. Merge pull request #68 from shmel1k/feature/dead_followers_to_logs

    [analysis] add dead followers to logs
    shmel1k authored Aug 6, 2021
    Copy the full SHA
    c8bdc13 View commit details
Showing with 16 additions and 6 deletions.
  1. +1 −1 go.mod
  2. +2 −0 internal/vshard/orchestrator/analysis.go
  3. +6 −2 internal/vshard/orchestrator/failover.go
  4. +7 −3 internal/vshard/orchestrator/monitor.go
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -11,7 +11,7 @@ require (
github.com/rs/zerolog v1.18.0
github.com/satori/go.uuid v1.2.0 // indirect
github.com/stretchr/testify v1.4.0
github.com/tarantool/go-tarantool v0.0.0-20191229181800-f4ece3508d87
github.com/tarantool/go-tarantool v0.0.0-20191229181800-f4ece3508d87 // indirect
github.com/tinylib/msgp v1.1.1 // indirect
github.com/viciious/go-tarantool v0.0.0-20190828171136-ede812c03707
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82
2 changes: 2 additions & 0 deletions internal/vshard/orchestrator/analysis.go
Original file line number Diff line number Diff line change
@@ -53,6 +53,8 @@ type ReplicationAnalysis struct {
CountReplicatingReplicas int // Total number of replicas confirmed replication
CountInconsistentVShardConf int // Total number of replicas with other than master vshard configuration
State ReplicaSetState
// DeadFollowers is a list with followers that are not currently connected to leader.
DeadFollowers []string
}

func (a ReplicationAnalysis) String() string {
8 changes: 6 additions & 2 deletions internal/vshard/orchestrator/failover.go
Original file line number Diff line number Diff line change
@@ -154,13 +154,17 @@ func (f *failover) checkAndRecover(ctx context.Context, analysis *ReplicationAna
recvFunc, desc := f.getCheckAndRecoveryFunc(analysis.State)
if recvFunc == nil {
if desc != "" {
logger.Warn().Msg(desc)
logger.Warn().
Strs("dead_followers", analysis.DeadFollowers).
Msg(desc)
}
return
}

f.cluster.StartRecovery()
logger.Warn().Msg(desc)
logger.Warn().
Strs("dead_followers", analysis.DeadFollowers).
Msg(desc)
logger.Info().Msgf("Cluster snapshot before recovery: %s", f.cluster.Dump())
recoveries := recvFunc(ctx, analysis)
for _, recv := range recoveries {
10 changes: 7 additions & 3 deletions internal/vshard/orchestrator/monitor.go
Original file line number Diff line number Diff line change
@@ -110,20 +110,23 @@ func analyze(set vshard.ReplicaSet, logger zerolog.Logger) *ReplicationAnalysis
countInconsistentVShardConf := 0
masterMasterReplication := false
followers := set.Followers()
var deadFollowers []string
for i := range followers {
r := &followers[i]
countReplicas++
if r.LastCheckValid {
countWorkingReplicas++

status := r.StorageInfo.Replication.Status
if status == vshard.StatusFollow {
switch status {
case vshard.StatusFollow:
countReplicatingReplicas++
} else if status == vshard.StatusMaster {
case vshard.StatusMaster:
countReplicatingReplicas++
masterMasterReplication = true

logger.Warn().Msgf("Found M-M replication ('%s'-'%s'), ('%s'-'%s')", set.MasterUUID, r.UUID, set.MasterURI, r.URI)
default:
deadFollowers = append(deadFollowers, string(r.UUID))
}

if r.VShardFingerprint != master.VShardFingerprint {
@@ -168,6 +171,7 @@ func analyze(set vshard.ReplicaSet, logger zerolog.Logger) *ReplicationAnalysis
CountReplicatingReplicas: countReplicatingReplicas,
CountInconsistentVShardConf: countInconsistentVShardConf,
State: state,
DeadFollowers: deadFollowers,
}
}