Merge pull request #1059 from wakatime/feature/legacy-offline-db

Use new offline db filename and location

USAGE.md

@@ -102,7 +102,7 @@ some/submodule/name = new project name
 | status_bar_enabled             | Turns on wakatime status bar for certain editors. | _bool_ | `true` |
 | status_bar_coding_activity     | Enables displaying Today's code stats in the status bar of some editors. When false, only the WakaTime icon is displayed in the status bar. | _bool_ | `true` |
 | status_bar_hide_categories     | When `true`, --today only displays the total code stats, never displaying Categories in the output. | _bool_ | `false` |
-| offline                        | Enables saving code stats locally to ~/.wakatime.bdb when offline, and syncing to the dashboard later when back online. | _bool_ | `true` |
+| offline                        | Enables saving code stats locally to ~/.wakatime/offline_heartbeats.bdb when offline, and syncing to the dashboard later when back online. | _bool_ | `true` |
 | proxy                          | Optional proxy configuration. Supports HTTPS, SOCKS and NTLM proxies. For ex: `https://user:pass@host:port`, `socks5://user:pass@host:port`, `domain\\user:pass` | _string_ | |
 | no_ssl_verify                  | Disables SSL certificate verification for HTTPS requests. By default, SSL certificates are verified. | _bool_ | `false` |
 | ssl_certs_file                 | Path to a CA certs file. By default, uses bundled Letsencrypt CA cert along with system ca certs. | _filepath_ | |

cmd/offlinesync/offlinesync.go

@@ -2,6 +2,7 @@ package offlinesync
 
 import (
 	"fmt"
+	"os"
 
 	cmdapi "github.com/wakatime/wakatime-cli/cmd/api"
 	"github.com/wakatime/wakatime-cli/cmd/params"
@@ -25,8 +26,16 @@ func Run(v *viper.Viper) (int, error) {
 		)
 	}
 
-	err = SyncOfflineActivity(v, queueFilepath)
+	queueFilepathLegacy, err := offline.QueueFilepathLegacy()
 	if err != nil {
+		log.Warnf("legacy offline sync failed: failed to load offline queue filepath: %s", err)
+	}
+
+	if err = syncOfflineActivityLegacy(v, queueFilepathLegacy); err != nil {
+		log.Warnf("legacy offline sync failed: %s", err)
+	}
+
+	if err = SyncOfflineActivity(v, queueFilepath); err != nil {
 		if errwaka, ok := err.(wakaerror.Error); ok {
 			return errwaka.ExitCode(), fmt.Errorf("offline sync failed: %s", errwaka.Message())
 		}
@@ -42,6 +51,49 @@ func Run(v *viper.Viper) (int, error) {
 	return exitcode.Success, nil
 }
 
+// syncOfflineActivityLegacy syncs the old offline activity by sending heartbeats
+// from the legacy offline queue to the WakaTime API.
+func syncOfflineActivityLegacy(v *viper.Viper, queueFilepath string) error {
+	if queueFilepath == "" {
+		return nil
+	}
+
+	paramOffline := params.LoadOfflineParams(v)
+
+	paramAPI, err := params.LoadAPIParams(v)
+	if err != nil {
+		return fmt.Errorf("failed to load API parameters: %w", err)
+	}
+
+	apiClient, err := cmdapi.NewClientWithoutAuth(paramAPI)
+	if err != nil {
+		return fmt.Errorf("failed to initialize api client: %w", err)
+	}
+
+	if paramOffline.QueueFileLegacy != "" {
+		queueFilepath = paramOffline.QueueFileLegacy
+	}
+
+	handle := heartbeat.NewHandle(apiClient,
+		offline.WithSync(queueFilepath, paramOffline.SyncMax),
+		apikey.WithReplacing(apikey.Config{
+			DefaultAPIKey: paramAPI.Key,
+			MapPatterns:   paramAPI.KeyPatterns,
+		}),
+	)
+
+	_, err = handle(nil)
+	if err != nil {
+		return err
+	}
+
+	if err := os.Remove(queueFilepath); err != nil {
+		log.Warnf("failed to delete legacy offline file: %s", err)
+	}
+
+	return nil
+}
+
 // SyncOfflineActivity syncs offline activity by sending heartbeats
 // from the offline queue to the WakaTime API.
 func SyncOfflineActivity(v *viper.Viper, queueFilepath string) error {

cmd/offlinesync/offlinesync_internal_test.go

@@ -0,0 +1,147 @@
+package offlinesync
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
+)
+
+func TestSyncOfflineActivityLegacy(t *testing.T) {
+	testServerURL, router, tearDown := setupTestServer()
+	defer tearDown()
+
+	var (
+		plugin   = "plugin/0.0.1"
+		numCalls int
+	)
+
+	router.HandleFunc("/users/current/heartbeats.bulk", func(w http.ResponseWriter, req *http.Request) {
+		numCalls++
+
+		// check request
+		assert.Equal(t, http.MethodPost, req.Method)
+		assert.Equal(t, []string{"application/json"}, req.Header["Accept"])
+		assert.Equal(t, []string{"application/json"}, req.Header["Content-Type"])
+		assert.Equal(t, []string{"Basic MDAwMDAwMDAtMDAwMC00MDAwLTgwMDAtMDAwMDAwMDAwMDAw"}, req.Header["Authorization"])
+		assert.True(t, strings.HasSuffix(req.Header["User-Agent"][0], plugin), fmt.Sprintf(
+			"%q should have suffix %q",
+			req.Header["User-Agent"][0],
+			plugin,
+		))
+
+		expectedBody, err := os.ReadFile("testdata/api_heartbeats_request_template.json")
+		require.NoError(t, err)
+
+		body, err := io.ReadAll(req.Body)
+		require.NoError(t, err)
+
+		assert.JSONEq(t, string(expectedBody), string(body))
+
+		// send response
+		w.WriteHeader(http.StatusCreated)
+
+		f, err := os.Open("testdata/api_heartbeats_response.json")
+		require.NoError(t, err)
+		defer f.Close()
+
+		_, err = io.Copy(w, f)
+		require.NoError(t, err)
+	})
+
+	// setup offline queue
+	f, err := os.CreateTemp(t.TempDir(), "")
+	require.NoError(t, err)
+
+	// early close to avoid file locking in Windows
+	f.Close()
+
+	db, err := bolt.Open(f.Name(), 0600, nil)
+	require.NoError(t, err)
+
+	dataGo, err := os.ReadFile("testdata/heartbeat_go.json")
+	require.NoError(t, err)
+
+	dataPy, err := os.ReadFile("testdata/heartbeat_py.json")
+	require.NoError(t, err)
+
+	dataJs, err := os.ReadFile("testdata/heartbeat_js.json")
+	require.NoError(t, err)
+
+	insertHeartbeatRecords(t, db, "heartbeats", []heartbeatRecord{
+		{
+			ID:        "1592868367.219124-file-coding-wakatime-cli-heartbeat-/tmp/main.go-true",
+			Heartbeat: string(dataGo),
+		},
+		{
+			ID:        "1592868386.079084-file-debugging-wakatime-summary-/tmp/main.py-false",
+			Heartbeat: string(dataPy),
+		},
+		{
+			ID:        "1592868394.084354-file-building-wakatime-todaygoal-/tmp/main.js-false",
+			Heartbeat: string(dataJs),
+		},
+	})
+
+	err = db.Close()
+	require.NoError(t, err)
+
+	v := viper.New()
+	v.Set("api-url", testServerURL)
+	v.Set("key", "00000000-0000-4000-8000-000000000000")
+	v.Set("sync-offline-activity", 100)
+	v.Set("plugin", plugin)
+
+	err = syncOfflineActivityLegacy(v, f.Name())
+	require.NoError(t, err)
+
+	assert.NoFileExists(t, f.Name())
+
+	assert.Eventually(t, func() bool { return numCalls == 1 }, time.Second, 50*time.Millisecond)
+}
+
+func setupTestServer() (string, *http.ServeMux, func()) {
+	router := http.NewServeMux()
+	srv := httptest.NewServer(router)
+
+	return srv.URL, router, func() { srv.Close() }
+}
+
+type heartbeatRecord struct {
+	ID        string
+	Heartbeat string
+}
+
+func insertHeartbeatRecords(t *testing.T, db *bolt.DB, bucket string, hh []heartbeatRecord) {
+	for _, h := range hh {
+		insertHeartbeatRecord(t, db, bucket, h)
+	}
+}
+
+func insertHeartbeatRecord(t *testing.T, db *bolt.DB, bucket string, h heartbeatRecord) {
+	t.Helper()
+
+	err := db.Update(func(tx *bolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte(bucket))
+		if err != nil {
+			return fmt.Errorf("failed to create bucket: %s", err)
+		}
+
+		err = b.Put([]byte(h.ID), []byte(h.Heartbeat))
+		if err != nil {
+			return fmt.Errorf("failed put heartbeat: %s", err)
+		}
+
+		return nil
+	})
+	require.NoError(t, err)
+}

cmd/params/params.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/url"
 	"os"
 	"os/exec"
@@ -129,10 +130,11 @@ type (
 
 	// Offline contains offline related parameters.
 	Offline struct {
-		Disabled  bool
-		QueueFile string
-		PrintMax  int
-		SyncMax   int
+		Disabled        bool
+		QueueFile       string
+		QueueFileLegacy string
+		PrintMax        int
+		SyncMax         int
 	}
 
 	// ProjectParams params for project name sanitization.
@@ -653,10 +655,11 @@ func LoadOfflineParams(v *viper.Viper) Offline {
 	}
 
 	return Offline{
-		Disabled:  disabled,
-		QueueFile: vipertools.GetString(v, "offline-queue-file"),
-		PrintMax:  v.GetInt("print-offline-heartbeats"),
-		SyncMax:   syncMax,
+		Disabled:        disabled,
+		QueueFile:       vipertools.GetString(v, "offline-queue-file"),
+		QueueFileLegacy: vipertools.GetString(v, "offline-queue-file-legacy"),
+		PrintMax:        v.GetInt("print-offline-heartbeats"),
+		SyncMax:         syncMax,
 	}
 }
 
@@ -730,7 +733,7 @@ func readExtraHeartbeats() ([]heartbeat.Heartbeat, error) {
 	in := bufio.NewReader(os.Stdin)
 
 	input, err := in.ReadString('\n')
-	if err != nil {
+	if err != nil && err != io.EOF {
 		log.Debugf("failed to read data from stdin: %s", err)
 	}
 
@@ -1036,10 +1039,11 @@ func (p Heartbeat) String() string {
 // String implements fmt.Stringer interface.
 func (p Offline) String() string {
 	return fmt.Sprintf(
-		"disabled: %t, print max: %d, queue file: '%s', num sync max: %d",
+		"disabled: %t, print max: %d, queue file: '%s', queue file legacy: '%s', num sync max: %d",
 		p.Disabled,
 		p.PrintMax,
 		p.QueueFile,
+		p.QueueFileLegacy,
 		p.SyncMax,
 	)
 }

cmd/params/params_test.go

@@ -1687,6 +1687,15 @@ func TestLoad_OfflineQueueFile(t *testing.T) {
 	assert.Equal(t, "/path/to/file", params.QueueFile)
 }
 
+func TestLoad_OfflineQueueFileLegacy(t *testing.T) {
+	v := viper.New()
+	v.Set("offline-queue-file-legacy", "/path/to/file")
+
+	params := paramscmd.LoadOfflineParams(v)
+
+	assert.Equal(t, "/path/to/file", params.QueueFileLegacy)
+}
+
 func TestLoad_OfflineSyncMax(t *testing.T) {
 	v := viper.New()
 	v.Set("sync-offline-activity", 42)
@@ -2487,15 +2496,17 @@ func TestHeartbeat_String(t *testing.T) {
 
 func TestOffline_String(t *testing.T) {
 	offline := paramscmd.Offline{
-		Disabled:  true,
-		PrintMax:  6,
-		QueueFile: "/path/to/queue.file",
-		SyncMax:   12,
+		Disabled:        true,
+		PrintMax:        6,
+		QueueFile:       "/path/to/queue.file",
+		QueueFileLegacy: "/path/to/legacy.file",
+		SyncMax:         12,
 	}
 
 	assert.Equal(
 		t,
-		"disabled: true, print max: 6, queue file: '/path/to/queue.file', num sync max: 12",
+		"disabled: true, print max: 6, queue file: '/path/to/queue.file',"+
+			" queue file legacy: '/path/to/legacy.file', num sync max: 12",
 		offline.String(),
 	)
 }

cmd/root.go

@@ -181,6 +181,11 @@ func setFlags(cmd *cobra.Command, v *viper.Viper) {
 		"",
 		"(internal) Specify an offline queue file, which will be used instead of the default one.",
 	)
+	flags.String(
+		"offline-queue-file-legacy",
+		"",
+		"(internal) Specify the legacy offline queue file, which will be used instead of the default one.",
+	)
 	flags.String(
 		"output",
 		"",
@@ -216,7 +221,7 @@ func setFlags(cmd *cobra.Command, v *viper.Viper) {
 	flags.Int(
 		"sync-offline-activity",
 		offline.SyncMaxDefault,
-		fmt.Sprintf("Amount of offline activity to sync from your local ~/.wakatime.bdb bolt"+
+		fmt.Sprintf("Amount of offline activity to sync from your local ~/.wakatime/offline_heartbeats.bdb bolt"+
 			" file to your WakaTime Dashboard before exiting. Can be zero or"+
 			" a positive integer. Defaults to %d, meaning after sending a heartbeat"+
 			" while online, all queued offline heartbeats are sent to WakaTime API, up"+
@@ -259,6 +264,7 @@ func setFlags(cmd *cobra.Command, v *viper.Viper) {
 
 	// hide internal flags
 	_ = flags.MarkHidden("offline-queue-file")
+	_ = flags.MarkHidden("offline-queue-file-legacy")
 	_ = flags.MarkHidden("user-agent")
 
 	err := v.BindPFlags(flags)

go.mod

@@ -1,6 +1,6 @@
 module github.com/wakatime/wakatime-cli
 
-go 1.22.4
+go 1.22.5
 
 require (
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358