Added option to write logs to a file

Integrated to lib provides logs rotation.

Author: Pavel Parshin

cmd/qumomf/main.go

@@ -3,16 +3,21 @@ package main
 import (
 	"context"
 	"flag"
+	"fmt"
+	"io"
 	"log/syslog"
 	"net/http"
 	"os"
 	"os/signal"
+	"path"
 	"syscall"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
+	"golang.org/x/sys/unix"
+	"gopkg.in/natefinch/lumberjack.v2"
 
 	"github.com/shmel1k/qumomf/internal/config"
 	"github.com/shmel1k/qumomf/internal/coordinator"
@@ -80,26 +85,57 @@ func main() {
 func initLogger(cfg *config.Config) zerolog.Logger {
 	zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
 
-	logLevel, err := zerolog.ParseLevel(cfg.Qumomf.LogLevel)
+	loggingCfg := cfg.Qumomf.Logging
+
+	logLevel, err := zerolog.ParseLevel(loggingCfg.Level)
 	if err != nil {
-		log.Warn().Msgf("Unknown Level String: '%s', defaulting to DebugLevel", cfg.Qumomf.LogLevel)
+		log.Warn().Msgf("Unknown Level String: '%s', defaulting to DebugLevel", loggingCfg.Level)
 		logLevel = zerolog.DebugLevel
 	}
 
-	base := zerolog.New(os.Stdout).Level(logLevel).With().Timestamp().Logger()
+	writers := make([]io.Writer, 0, 1)
+	writers = append(writers, os.Stdout)
 
-	if cfg.Qumomf.EnableSysLog {
+	if loggingCfg.SysLogEnabled {
 		w, err := syslog.New(syslog.LOG_INFO, "qumomf")
 		if err != nil {
 			log.Warn().Err(err).Msg("Unable to connect to the system log daemon")
-			return base
+		} else {
+			writers = append(writers, zerolog.SyslogLevelWriter(w))
+		}
+	}
+
+	if loggingCfg.FileLoggingEnabled {
+		w, err := newRollingLogFile(&loggingCfg)
+		if err != nil {
+			log.Warn().Err(err).Msg("Unable to init file logger")
+		} else {
+			writers = append(writers, w)
 		}
-		syslogWriter := zerolog.SyslogLevelWriter(w)
+	}
+
+	var baseLogger zerolog.Logger
+	if len(writers) == 1 {
+		baseLogger = zerolog.New(writers[0])
+	} else {
+		return zerolog.New(zerolog.MultiLevelWriter(writers...))
+	}
+
+	return baseLogger.Level(logLevel).With().Timestamp().Logger()
+}
 
-		return zerolog.New(zerolog.MultiLevelWriter(os.Stdout, syslogWriter)).Level(logLevel).With().Timestamp().Logger()
+func newRollingLogFile(cfg *config.Logging) (io.Writer, error) {
+	dir := path.Dir(cfg.Filename)
+	if unix.Access(dir, unix.W_OK) != nil {
+		return nil, fmt.Errorf("no permissions to write logs to dir: %s", dir)
 	}
 
-	return base
+	return &lumberjack.Logger{
+		Filename:   cfg.Filename,
+		MaxBackups: cfg.MaxBackups,
+		MaxSize:    cfg.MaxSize,
+		MaxAge:     cfg.MaxAge,
+	}, nil
 }
 
 func initHTTPServer(port string) *http.Server {

config/qumomf.conf.yml

@@ -1,11 +1,22 @@
 qumomf:
   # TCP port to listen.
   port: ':8080'
-  # Verbose level of logging: trace, debug, info, warn, error, fatal, panic.
-  # To disable logging, pass an empty string.
-  log_level: "debug"
-  # Write to the local syslog daemon as well as to stdout.
-  enable_syslog: false
+  logging:
+    # Verbose level of logging: trace, debug, info, warn, error, fatal, panic.
+    # To disable logging, pass an empty string.
+    level: 'debug'
+    # Write logs to the local syslog daemon.
+    syslog_enabled: false
+    # Write logs to the file.
+    file_enabled: true
+    # Absolute path to the log output file.
+    file_name: '/var/log/qumomf.log'
+    # The max size in MB of the logfile before it's rolled.
+    file_max_size: 256
+    # The max number of rolled files to keep.
+    file_max_backups: 3
+    # The max age in days to keep a logfile.
+    file_max_age: 5
   # Indicates whether qumomf should run in the readonly mode:
   # no auto failover will be executed.
   # Can be overwritten by cluster-specific options.

config/qumomf.daemon.min.conf.yml

@@ -1,7 +1,13 @@
 qumomf:
   port: ':8080'
-  log_level: "debug"
-  enable_syslog: false
+  logging:
+    level: 'debug'
+    syslog_enabled: false
+    file_enabled: true
+    file_name: '/var/log/qumomf.log'
+    file_max_size: 256
+    file_max_backups: 3
+    file_max_age: 5
   readonly: true
   cluster_discovery_time: '5s'
   cluster_recovery_time: '1s'

example/qumomf.yml

@@ -1,6 +1,13 @@
 qumomf:
   port: ':8080'
-  log_level: "debug"
+  logging:
+    level: 'debug'
+    syslog_enabled: false
+    file_enabled: true
+    file_name: '/home/pavel/work/qumomf/src/github.com/shmel1k/qumomf/bin/qumomf.log'
+    file_max_size: 256
+    file_max_backups: 3
+    file_max_age: 5
   readonly: true
   cluster_discovery_time: '5s'
   cluster_recovery_time: '1s'

go.mod

@@ -11,7 +11,9 @@ require (
 	github.com/tarantool/go-tarantool v0.0.0-20191229181800-f4ece3508d87
 	github.com/tinylib/msgp v1.1.1 // indirect
 	github.com/viciious/go-tarantool v0.0.0-20190828171136-ede812c03707
+	golang.org/x/sys v0.0.0-20200122134326-e047566fdf82
 	google.golang.org/appengine v1.6.5 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/vmihailenco/msgpack.v2 v2.9.1 // indirect
 	gopkg.in/yaml.v2 v2.2.8
 )

go.sum

@@ -98,6 +98,7 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -111,6 +112,8 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/vmihailenco/msgpack.v2 v2.9.1 h1:kb0VV7NuIojvRfzwslQeP3yArBqJHW9tOl4t38VS1jM=
 gopkg.in/vmihailenco/msgpack.v2 v2.9.1/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/config/config.go

@@ -10,6 +10,12 @@ import (
 
 const (
 	defaultLogLevel                  = "debug"
+	defaultSysLogEnabled             = false
+	defaultFileLoggingEnabled        = false
+	defaultLogFilename               = "/var/log/qumomf.log"
+	defaultLogFileMaxSize            = 256
+	defaultLogFileMaxBackups         = 3
+	defaultLogFileMaxAge             = 5
 	defaultReadOnly                  = true
 	defaultUser                      = "guest"
 	defaultPassword                  = "guest"
@@ -31,8 +37,7 @@ type Config struct {
 	// Qumomf is a set of global options determines qumomf's behavior.
 	Qumomf struct {
 		Port                      string        `yaml:"port"`
-		LogLevel                  string        `yaml:"log_level"`
-		EnableSysLog              bool          `yaml:"enable_syslog"`
+		Logging                   Logging       `yaml:"logging"`
 		ReadOnly                  bool          `yaml:"readonly"`
 		ClusterDiscoveryTime      time.Duration `yaml:"cluster_discovery_time"`
 		ClusterRecoveryTime       time.Duration `yaml:"cluster_recovery_time"`
@@ -57,6 +62,16 @@ type Config struct {
 	Clusters   map[string]ClusterConfig `yaml:"clusters"`
 }
 
+type Logging struct {
+	Level              string `yaml:"level"`
+	SysLogEnabled      bool   `yaml:"syslog_enabled"`
+	FileLoggingEnabled bool   `yaml:"file_enabled"`
+	Filename           string `yaml:"file_name"`
+	MaxSize            int    `yaml:"file_max_size"`    // megabytes
+	MaxBackups         int    `yaml:"file_max_backups"` // files
+	MaxAge             int    `yaml:"file_max_age"`     // days
+}
+
 type ConnectConfig struct {
 	User           *string        `yaml:"user"`
 	Password       *string        `yaml:"password"`
@@ -140,7 +155,15 @@ func (c *Config) withDefaults() {
 
 	base := &c.Qumomf
 	base.ReadOnly = defaultReadOnly
-	base.LogLevel = defaultLogLevel
+
+	base.Logging.Level = defaultLogLevel
+	base.Logging.SysLogEnabled = defaultSysLogEnabled
+	base.Logging.FileLoggingEnabled = defaultFileLoggingEnabled
+	base.Logging.Filename = defaultLogFilename
+	base.Logging.MaxSize = defaultLogFileMaxSize
+	base.Logging.MaxBackups = defaultLogFileMaxBackups
+	base.Logging.MaxAge = defaultLogFileMaxAge
+
 	base.ClusterDiscoveryTime = defaultClusterDiscoveryTime
 	base.ClusterRecoveryTime = defaultClusterRecoveryTime
 	base.ShardRecoveryBlockTime = defaultShardRecoveryBlockTime

internal/config/config_test.go

@@ -24,8 +24,16 @@ func TestSetup_ValidPath(t *testing.T) {
 	require.NotNil(t, cfg)
 
 	assert.Equal(t, ":8080", cfg.Qumomf.Port)
-	assert.Equal(t, "debug", cfg.Qumomf.LogLevel)
-	assert.True(t, cfg.Qumomf.EnableSysLog)
+
+	loggingCfg := cfg.Qumomf.Logging
+	assert.Equal(t, "debug", loggingCfg.Level)
+	assert.True(t, loggingCfg.SysLogEnabled)
+	assert.True(t, loggingCfg.FileLoggingEnabled)
+	assert.Equal(t, "/var/log/qumomf.log", loggingCfg.Filename)
+	assert.Equal(t, 256, loggingCfg.MaxSize)
+	assert.Equal(t, 3, loggingCfg.MaxBackups)
+	assert.Equal(t, 5, loggingCfg.MaxAge)
+
 	assert.True(t, cfg.Qumomf.ReadOnly)
 	assert.Equal(t, 60*time.Second, cfg.Qumomf.ClusterDiscoveryTime)
 	assert.Equal(t, 5*time.Second, cfg.Qumomf.ClusterRecoveryTime)

internal/config/testdata/qumomf-full.conf.yml

@@ -1,7 +1,13 @@
 qumomf:
   port: ':8080'
-  log_level: "debug"
-  enable_syslog: true
+  logging:
+    level: 'debug'
+    syslog_enabled: true
+    file_enabled: true
+    file_name: '/var/log/qumomf.log'
+    file_max_size: 256
+    file_max_backups: 3
+    file_max_age: 5
   readonly: true
   cluster_discovery_time: '60s'
   cluster_recovery_time: '5s'