From 8ebf041b9d72d8c349fc5d5c15377372f1bc37ea Mon Sep 17 00:00:00 2001 From: whiteCcinn <471113744@qq.com> Date: Wed, 19 May 2021 15:59:57 +0800 Subject: [PATCH] support recover && signal && IPC --- .gitignore | 2 + README.md | 30 ++++++++++- daemon.go | 105 ++++++++++++++++++++++++++++++++++++-- example/daemon_recover.go | 45 ++++++++++++++++ example/daemon_signal.go | 57 +++++++++++++++++++++ pipe.go | 41 +++++++++++++++ signal.go | 60 ++++++++++++++++++++++ 7 files changed, 334 insertions(+), 6 deletions(-) create mode 100644 example/daemon_recover.go create mode 100644 example/daemon_signal.go create mode 100644 pipe.go create mode 100644 signal.go diff --git a/.gitignore b/.gitignore index f9a0367..ecd08b1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,8 @@ # vendor/ example/backgroud example/daemon +example/daemon_recover +example/daemon_signal example/*.log go.sum .idea \ No newline at end of file diff --git a/README.md b/README.md index 1111580..4ede253 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,19 @@ Go daemon mode for the process - restart callback - custom logger - worked time +- panic recover -[example](https://github.com/whiteCcinn/daemon/tree/main/example) +## Installation +```shell +go get github.com/whiteCcinn/daemon +``` + +## Examples： - [backgroud](https://github.com/whiteCcinn/daemon/blob/main/example/backgroud.go) - [daemon](https://github.com/whiteCcinn/daemon/blob/main/example/daemon.go) +- [daemon-recover](https://github.com/whiteCcinn/daemon/blob/main/example/daemon_recover.go) +- [daemon-signal](https://github.com/whiteCcinn/daemon/blob/main/example/daemon_signal.go) ## Log @@ -49,6 +57,26 @@ main.main() /www/example/daemon.go:38 +0x2c8 [supervisor(2924)] [count:2/2; errNum:0/0] [heart -pid=2936 exit] [2936-worked 10.0428623s] [err: exit status 2] [supervisor(2924)] [reboot too many times quit] + + +[process(1648)] [started] +[supervisor(1642)] [heart --pid=1648] +2021/05/19 07:46:12 1648 start... +2021/05/19 07:46:22 1648 end +[supervisor(1642)] [count:1/2; errNum:0/0] [heart -pid=1648 exit] [1648-worked 10.0249616s] +[process(1661)] [started] +[supervisor(1642)] [heart --pid=1661] +2021/05/19 07:46:22 restart... +2021/05/19 07:46:22 1661 start... +2021/05/19 07:46:32 1661 end +[supervisor(1642)] [count:2/2; errNum:0/0] [heart -pid=1661 exit] [1661-worked 10.0243316s] +[supervisor(1642)] [reboot too many times quit] +[process(1782)] [started] +[supervisor(1775)] [heart --pid=1782] +2021/05/19 07:50:59 1782 start... +2021/05/19 07:51:05 sigterm +[supervisor(1775)] [stop heart -pid 1782] [safety exit] +2021/05/19 07:51:09 1782 end... ``` ## Terminal diff --git a/daemon.go b/daemon.go index a2b3301..0b14db8 100644 --- a/daemon.go +++ b/daemon.go @@ -2,6 +2,7 @@ package daemon import ( "context" + "encoding/json" "fmt" "github.com/erikdubbelboer/gspt" "io" @@ -47,7 +48,8 @@ type Context struct { ProcAttr syscall.SysProcAttr - Logger io.Writer + Logger io.Writer + PanicLogger io.Writer Env []string Args []string @@ -62,8 +64,9 @@ type Context struct { RestoreTime time.Duration *exec.Cmd - Pid int // supervisor pid - CPid int // main pid + ExtraFiles []*os.File + Pid int // supervisor pid + CPid int // main pid RestartCallback } @@ -111,7 +114,7 @@ func Background(ctx context.Context, dctx *Context, opts ...Option) (*exec.Cmd, dctx.CPid = cmd.Process.Pid if !defaultOption.exit { dctx.log("[process(%d)] [started]\n", dctx.CPid) - dctx.log("[supervisor(%d)] [watch --pid=%d]\n", dctx.Pid, dctx.CPid) + dctx.log("[supervisor(%d)] [heart --pid=%d]\n", dctx.Pid, dctx.CPid) } } @@ -136,6 +139,14 @@ func startProc(ctx context.Context, dctx *Context) (*exec.Cmd, error) { cmd.Stdout = dctx.Logger } + if dctx.PanicLogger == nil { + dctx.PanicLogger = dctx.Logger + } + + if dctx.ExtraFiles != nil { + cmd.ExtraFiles = dctx.ExtraFiles + } + if err := ctx.Err(); err != nil { return nil, err } @@ -169,6 +180,28 @@ func (dctx *Context) Run(ctx context.Context) error { os.Exit(0) } count++ + + r, w, err := os.Pipe() + if err != nil { + dctx.log("[supervisor(%d)] [create pipe failed] [err: %v]\n", dctx.Pid, err) + os.Exit(2) + } + + // 因为不需要从父进程发送到子进程，所以不需要w2 + r2, _, err := os.Pipe() + if err != nil { + dctx.log("[supervisor(%d)] [create pipe failed] [err: %v]\n", dctx.Pid, err) + os.Exit(2) + } + + extraFile := make([]*os.File, 0, 2) + // so fd(3) = w + extraFile = append(extraFile, w, r2) + if dctx.ExtraFiles != nil { + extraFile = append(extraFile, dctx.ExtraFiles...) + } + dctx.ExtraFiles = extraFile + begin := time.Now() cmd, err := Background(ctx, dctx, WithNoExit()) if err != nil { @@ -179,8 +212,26 @@ func (dctx *Context) Run(ctx context.Context) error { // child process if cmd == nil { + exitFunc := func(sig os.Signal) (err error) { + // this is fd(3) + pipe := os.NewFile(uintptr(3), "pipe") + message := PipeMessage{ + Type: ProcessToSupervisor, + Behavior: WantSafetyClose, + } + err = json.NewEncoder(pipe).Encode(message) + if err != nil { + panic(err) + } + return + } + SetSigHandler(exitFunc, syscall.SIGINT) + SetSigHandler(exitFunc, syscall.SIGTERM) + break } + + // supervisor process gspt.SetProcTitle(fmt.Sprintf("heart -pid %d", dctx.CPid)) if count > 2 || isReset { if dctx.RestartCallback != nil { @@ -188,6 +239,26 @@ func (dctx *Context) Run(ctx context.Context) error { } } + // 从子进程获取数据 + go func() { + for { + var data PipeMessage + decoder := json.NewDecoder(r) + if err := decoder.Decode(&data); err != nil { + log.Printf("decode r, err: %v", err) + break + } + if data.Type != ProcessToSupervisor { + continue + } + + if data.Behavior == WantSafetyClose { + dctx.log("[supervisor(%d)] [stop heart -pid %d] [safety exit]\n", dctx.Pid, dctx.CPid) + os.Exit(0) + } + } + }() + // parent process wait child process exit err = cmd.Wait() end := time.Now() @@ -209,16 +280,40 @@ func (dctx *Context) Run(ctx context.Context) error { dctx.log("[supervisor(%d)] [%s] [heart -pid=%d exit] [%d-worked %v] [err: %v]\n", dctx.Pid, dInfo, dctx.CPid, dctx.CPid, cost, err) } else { dctx.log("[supervisor(%d)] [%s] [heart -pid=%d exit] [%d-worked %v]\n", dctx.Pid, dInfo, dctx.CPid, dctx.CPid, cost) - } } return nil } +// output log-message to Context.Logger func (dctx *Context) log(format string, args ...interface{}) { _, fe := fmt.Fprintf(dctx.Logger, format, args...) if fe != nil { log.Fatal(fe) } } + +func (dctx *Context) logPanic(format string, args ...interface{}) { + _, fe := fmt.Fprintf(dctx.PanicLogger, format, args...) + if fe != nil { + log.Fatal(fe) + } +} + +// WithRecovery wraps goroutine startup call with force recovery. +// it will dump current goroutine stack into log if catch any recover result. +// exec: execute logic function. +// recoverFn: handler will be called after recover and before dump stack, passing `nil` means noop. +func (dctx *Context) WithRecovery(exec func(), recoverFn func(r interface{})) { + defer func() { + r := recover() + if recoverFn != nil { + recoverFn(r) + } + if r != nil { + dctx.logPanic("panic in the recoverable goroutine, error: %v\n", r) + } + }() + exec() +} diff --git a/example/daemon_recover.go b/example/daemon_recover.go new file mode 100644 index 0000000..f1bbc51 --- /dev/null +++ b/example/daemon_recover.go @@ -0,0 +1,45 @@ +package main + +import ( + "context" + "github.com/whiteCcinn/daemon" + "log" + "os" + "syscall" + "time" +) + +func main() { + logName := "daemon.log" + panicLogName := "panic-daemon.log" + stdout, err := os.OpenFile(logName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) + panicStdout, err := os.OpenFile(panicLogName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) + if err != nil { + log.Fatal(err) + } + + ctx := context.Background() + dctx := daemon.Context{ + ProcAttr: syscall.SysProcAttr{}, + //Logger: os.Stdout, + Logger: stdout, + PanicLogger: panicStdout, + MaxCount: 2, + RestartCallback: func(ctx context.Context) { + log.Println("restart...") + }, + } + + err = dctx.Run(ctx) + if err != nil { + log.Fatal(err) + } + + // belong func main() + dctx.WithRecovery(func() { + log.Println(os.Getpid(), "start...") + time.Sleep(time.Second * 3) + panic("This trigger panic") + log.Println(os.Getpid(), "end") + }, nil) +} diff --git a/example/daemon_signal.go b/example/daemon_signal.go new file mode 100644 index 0000000..171e3eb --- /dev/null +++ b/example/daemon_signal.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "github.com/whiteCcinn/daemon" + "log" + "os" + "syscall" + "time" +) + +func main() { + logName := "daemon.log" + stdout, err := os.OpenFile(logName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666) + if err != nil { + log.Fatal(err) + } + + ctx := context.Background() + dctx := daemon.Context{ + ProcAttr: syscall.SysProcAttr{}, + //Logger: os.Stdout, + Logger: stdout, + MaxCount: 2, + RestartCallback: func(ctx context.Context) { + log.Println("restart...") + }, + } + + err = dctx.Run(ctx) + if err != nil { + log.Fatal(err) + } + + // belong func main() + dctx.WithRecovery(func() { + daemon.SetSigHandler(func(sig os.Signal) (err error) { + log.Println("sigint") + return + }, syscall.SIGINT) + + daemon.SetSigHandler(func(sig os.Signal) (err error) { + log.Println("sigterm") + return nil + }, syscall.SIGTERM) + + go func() { + err := daemon.ServeSignals() + if err != nil { + log.Println(err) + } + }() + log.Println(os.Getpid(), "start...") + time.Sleep(time.Second * 10) + log.Println(os.Getpid(), "end...") + }, nil) +} diff --git a/pipe.go b/pipe.go new file mode 100644 index 0000000..deb6434 --- /dev/null +++ b/pipe.go @@ -0,0 +1,41 @@ +package daemon + +type PipeMessageType int + +const ( + SupervisorToProcess PipeMessageType = iota + 1 + ProcessToSupervisor +) + +type ProcessBehavior int + +const ( + WantSafetyClose ProcessBehavior = iota + 1 +) + +func (pmt PipeMessageType) String() (s string) { + switch pmt { + case SupervisorToProcess: + s = "The Process sends messages to the Supervisor" + case ProcessToSupervisor: + s = "The Supervisor sends messages to the Process" + default: + s = "Unknown PipeMessageType" + } + return +} + +func (pb ProcessBehavior) String() (s string) { + switch pb { + case WantSafetyClose: + s = "Expect a safe exit" + default: + s = "Unknown ProcessBehavior" + } + return +} + +type PipeMessage struct { + Type PipeMessageType + Behavior ProcessBehavior +} diff --git a/signal.go b/signal.go new file mode 100644 index 0000000..0391ddc --- /dev/null +++ b/signal.go @@ -0,0 +1,60 @@ +package daemon + +import ( + "errors" + "os" + "os/signal" + "syscall" +) + +// ErrStop should be returned signal handler function +// for termination of handling signals. +var ErrStop = errors.New("stop serve signals") + +// SignalHandlerFunc is the interface for signal handler functions. +type SignalHandlerFunc func(sig os.Signal) (err error) + +// SetSigHandler sets handler for the given signals. +// SIGTERM has the default handler, he returns ErrStop. +func SetSigHandler(handler SignalHandlerFunc, signals ...os.Signal) { + for _, sig := range signals { + handlers[sig] = append(handlers[sig], handler) + } +} + +// ServeSignals calls handlers for system signals. +func ServeSignals() (err error) { + signals := make([]os.Signal, 0, len(handlers)) + for sig := range handlers { + signals = append(signals, sig) + } + + ch := make(chan os.Signal, 8) + signal.Notify(ch, signals...) + + for sig := range ch { + for _, f := range handlers[sig] { + err = f(sig) + if err == ErrStop { + err = nil + } + if err != nil { + break + } + } + } + + signal.Stop(ch) + + return +} + +var handlers = make(map[os.Signal][]SignalHandlerFunc) + +func init() { + handlers[syscall.SIGTERM] = []SignalHandlerFunc{sigtermDefaultHandler} +} + +func sigtermDefaultHandler(sig os.Signal) error { + return ErrStop +}