MB-44216: Race in closing moss store and obtaining stats

+ There appears to be racey code path in obtaining stats for a
  moss store that has since been closed.
+ Fetch histograms from moss within lock to avoid this panic ..

    http: panic serving 127.0.0.1:36252: runtime error: invalid memory address or nil pointer dereference
    goroutine 1069 [running]:
    net/http.(*conn).serve.func1(0xc0004301e0)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/net/http/server.go:1767 +0x139
    panic(0xffeba0, 0x1cb22b0)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/runtime/panic.go:679 +0x1b2
    github.com/couchbase/moss.(*Store).Histograms(...)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/couchbase/[email protected]/store_stats.go:89
    github.com/blevesearch/bleve/v2/index/upsidedown/store/moss.(*stats).statsMap(0xc0000110c0, 0x7faa69bc5a70)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/[email protected]/index/upsidedown/store/moss/stats.go:48 +0x27b
    github.com/blevesearch/bleve/v2/index/upsidedown/store/moss.(*Store).StatsMap(0xc0020f2820, 0x13431c0)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/[email protected]/index/upsidedown/store/moss/store.go:214 +0x2f
    github.com/blevesearch/bleve/v2/index/upsidedown.(*indexStat).statsMap(0xc0020f2780, 0xc0040ba780)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/[email protected]/index/upsidedown/stats.go:46 +0x478
    github.com/blevesearch/bleve/v2/index/upsidedown.(*UpsideDownCouch).StatsMap(0xc0020f0480, 0x6)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/[email protected]/index/upsidedown/upsidedown.go:1022 +0x2f
    github.com/blevesearch/bleve/v2.(*IndexStat).statsMap(0xc0020efb40, 0xc0040ba750)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/[email protected]/index_stats.go:31 +0x4a
    github.com/blevesearch/bleve/v2.(*indexImpl).StatsMap(0xc0002c8690, 0x0)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/blevesearch/bleve/[email protected]/index_impl.go:791 +0x2f
    github.com/couchbase/cbft.(*BleveDest).StatsMap(0xc0020ddad0, 0xf930e0, 0xc003eba100, 0xc00225e270)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/cbft/pindex_bleve.go:1911 +0xb2
    github.com/couchbase/cbft.addPIndexStats(0xc001cd1300, 0xc003ed9b00, 0x11a15dd, 0x13)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/cbft/ns_server.go:596 +0x73
    github.com/couchbase/cbft.gatherIndexStats(0xc000422e00, 0xc001fdf000, 0x1, 0xc0003d5a40, 0x1c, 0xc0044c5460)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/cbft/ns_server.go:350 +0x165a
    github.com/couchbase/cbft.(*PrometheusHighMetricsHandler).ServeHTTP(0xc0003e7fe0, 0x1343c40, 0xc003aa0760, 0xc0039dad00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/cbft/prometheus.go:113 +0x144
    github.com/couchbase/cbgt/rest.(*HandlerWithRESTMeta).ServeHTTP(0xc00050e480, 0x13455c0, 0xc000518ee0, 0xc0039dad00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/cbgt/rest/rest.go:254 +0x10c
    github.com/couchbase/cbft.(*AuthVersionHandler).ServeHTTP(0xc0004cd700, 0x13455c0, 0xc000518ee0, 0xc0039dad00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/cbft/rest.go:174 +0x13d
    main.exportMuxRoutesToHttprouter.func1.1(0x13455c0, 0xc000518ee0, 0xc0039dac00, 0x0, 0x0, 0x0)
        cbft/cmd/cbft/main.go:729 +0xa1
    github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc0002a3240, 0x13455c0, 0xc000518ee0, 0xc0039dac00)
        /home/couchbase/jenkins/workspace/couchbase-server-unix/server_build/gopkg/go-1.13.7/pkg/mod/github.com/julienschmidt/[email protected]/router.go:344 +0x932
    net/http.serverHandler.ServeHTTP(0xc000214460, 0x13455c0, 0xc000518ee0, 0xc0039dac00)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/net/http/server.go:2802 +0xa4
    net/http.(*conn).serve(0xc0004301e0, 0x1349ec0, 0xc001a2e200)
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/net/http/server.go:1890 +0x875
    created by net/http.(*Server).Serve
        /home/couchbase/.cbdepscache/exploded/x86_64/go-1.13.7/go/src/net/http/server.go:2928 +0x384

Author: abhinavdangeti

index/upsidedown/store/moss/lower.go

@@ -553,3 +553,13 @@ func (w *mossStoreWrapper) Actual() *moss.Store {
 	w.m.Unlock()
 	return rv
 }
+
+func (w *mossStoreWrapper) histograms() string {
+	var rv string
+	w.m.Lock()
+	if w.s != nil {
+		rv = w.s.Histograms().String()
+	}
+	w.m.Unlock()
+	return rv
+}

index/upsidedown/store/moss/stats.go

@@ -45,7 +45,7 @@ func (s *stats) statsMap() map[string]interface{} {
 	}
 
 	if msw, ok := s.s.llstore.(*mossStoreWrapper); ok {
-		ms["store_histograms"] = msw.Actual().Histograms().String()
+		ms["store_histograms"] = msw.histograms()
 	}
 
 	ms["coll_histograms"] = s.s.ms.Histograms().String()