From ebe8d43aa3053a8468ec3555c6cf0a39f659914a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Henrique=20Guard=C3=A3o=20Gandarez?= Date: Tue, 19 Nov 2024 09:52:53 -0300 Subject: [PATCH 1/2] Bump changelog action to v1.5.0 --- .github/workflows/on_push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/on_push.yml b/.github/workflows/on_push.yml index b755dc4e..cf2e3820 100644 --- a/.github/workflows/on_push.yml +++ b/.github/workflows/on_push.yml @@ -644,7 +644,7 @@ jobs: # Run only for develop branch if: ${{ github.ref == 'refs/heads/develop' }} name: Changelog for develop - uses: gandarez/changelog-action@v1.4.0 + uses: gandarez/changelog-action@v1.5.0 id: changelog-develop with: current_tag: ${{ github.sha }} From 6e993372ff2b41af25a456be1e8feddb11a544c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Henrique=20Guard=C3=A3o=20Gandarez?= Date: Thu, 21 Nov 2024 11:53:45 -0300 Subject: [PATCH 2/2] Accept boolean value for exclude and include params --- cmd/heartbeat/heartbeat_test.go | 38 ++++++++ cmd/params/params.go | 69 ++++++++------- cmd/params/params_internal_test.go | 17 ++-- cmd/params/params_test.go | 136 +++++++++++++++++++++-------- 4 files changed, 185 insertions(+), 75 deletions(-) diff --git a/cmd/heartbeat/heartbeat_test.go b/cmd/heartbeat/heartbeat_test.go index 2b40ff92..959c944f 100644 --- a/cmd/heartbeat/heartbeat_test.go +++ b/cmd/heartbeat/heartbeat_test.go @@ -234,6 +234,44 @@ func TestSendHeartbeats_WithFiltering_Exclude(t *testing.T) { assert.Equal(t, 0, numCalls) } +func TestSendHeartbeats_WithFiltering_Exclude_All(t *testing.T) { + resetSingleton(t) + + testServerURL, router, tearDown := setupTestServer() + defer tearDown() + + var numCalls int + + router.HandleFunc("/users/current/heartbeats.bulk", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + + numCalls++ + }) + + v := viper.New() + v.SetDefault("sync-offline-activity", 1000) + v.Set("api-url", testServerURL) + v.Set("category", "debugging") + v.Set("entity", `\tmp\main.go`) + v.Set("exclude", `true`) + v.Set("entity-type", "file") + v.Set("key", "00000000-0000-4000-8000-000000000000") + v.Set("plugin", "plugin") + v.Set("time", 1585598059.1) + v.Set("timeout", 5) + v.Set("write", true) + + offlineQueueFile, err := os.CreateTemp(t.TempDir(), "") + require.NoError(t, err) + + defer offlineQueueFile.Close() + + err = cmdheartbeat.SendHeartbeats(context.Background(), v, offlineQueueFile.Name()) + require.NoError(t, err) + + assert.Equal(t, 0, numCalls) +} + func TestSendHeartbeats_ExtraHeartbeats(t *testing.T) { resetSingleton(t) diff --git a/cmd/params/params.go b/cmd/params/params.go index 457dd01f..bdda273c 100644 --- a/cmd/params/params.go +++ b/cmd/params/params.go @@ -437,12 +437,17 @@ func LoadHeartbeatParams(ctx context.Context, v *viper.Viper) (Heartbeat, error) timeSecs = float64(time.Now().UnixNano()) / 1000000000 } + filterParams, err := loadFilterParams(ctx, v) + if err != nil { + return Heartbeat{}, fmt.Errorf("failed to load filter params: %s", err) + } + projectParams, err := loadProjectParams(ctx, v) if err != nil { return Heartbeat{}, fmt.Errorf("failed to parse project params: %s", err) } - sanitizeParams, err := loadSanitizeParams(v) + sanitizeParams, err := loadSanitizeParams(ctx, v) if err != nil { return Heartbeat{}, fmt.Errorf("failed to load sanitize params: %s", err) } @@ -469,34 +474,30 @@ func LoadHeartbeatParams(ctx context.Context, v *viper.Viper) (Heartbeat, error) LinesInFile: linesInFile, LocalFile: vipertools.GetString(v, "local-file"), Time: timeSecs, - Filter: loadFilterParams(ctx, v), + Filter: filterParams, Project: projectParams, Sanitize: sanitizeParams, }, nil } -func loadFilterParams(ctx context.Context, v *viper.Viper) FilterParams { +func loadFilterParams(ctx context.Context, v *viper.Viper) (FilterParams, error) { exclude := v.GetStringSlice("exclude") exclude = append(exclude, v.GetStringSlice("settings.exclude")...) exclude = append(exclude, v.GetStringSlice("settings.ignore")...) - logger := log.Extract(ctx) - var excludePatterns []regex.Regex for _, s := range exclude { - // make all regex case insensitive - if !strings.HasPrefix(s, "(?i)") { - s = "(?i)" + s - } - - compiled, err := regex.Compile(s) + patterns, err := parseBoolOrRegexList(ctx, s) if err != nil { - logger.Warnf("failed to compile exclude regex pattern %q", s) - continue + return FilterParams{}, fmt.Errorf( + "failed to parse regex exclude param %q: %s", + s, + err, + ) } - excludePatterns = append(excludePatterns, compiled) + excludePatterns = append(excludePatterns, patterns...) } include := v.GetStringSlice("include") @@ -505,18 +506,16 @@ func loadFilterParams(ctx context.Context, v *viper.Viper) FilterParams { var includePatterns []regex.Regex for _, s := range include { - // make all regex case insensitive - if !strings.HasPrefix(s, "(?i)") { - s = "(?i)" + s - } - - compiled, err := regex.Compile(s) + patterns, err := parseBoolOrRegexList(ctx, s) if err != nil { - logger.Warnf("failed to compile include regex pattern %q", s) - continue + return FilterParams{}, fmt.Errorf( + "failed to parse regex include param %q: %s", + s, + err, + ) } - includePatterns = append(includePatterns, compiled) + includePatterns = append(includePatterns, patterns...) } return FilterParams{ @@ -532,10 +531,10 @@ func loadFilterParams(ctx context.Context, v *viper.Viper) FilterParams { "include-only-with-project-file", "settings.include_only_with_project_file", ), - } + }, nil } -func loadSanitizeParams(v *viper.Viper) (SanitizeParams, error) { +func loadSanitizeParams(ctx context.Context, v *viper.Viper) (SanitizeParams, error) { // hide branch names hideBranchNamesStr := vipertools.FirstNonEmptyString( v, @@ -545,7 +544,7 @@ func loadSanitizeParams(v *viper.Viper) (SanitizeParams, error) { "settings.hidebranchnames", ) - hideBranchNamesPatterns, err := parseBoolOrRegexList(hideBranchNamesStr) + hideBranchNamesPatterns, err := parseBoolOrRegexList(ctx, hideBranchNamesStr) if err != nil { return SanitizeParams{}, fmt.Errorf( "failed to parse regex hide branch names param %q: %s", @@ -563,7 +562,7 @@ func loadSanitizeParams(v *viper.Viper) (SanitizeParams, error) { "settings.hideprojectnames", ) - hideProjectNamesPatterns, err := parseBoolOrRegexList(hideProjectNamesStr) + hideProjectNamesPatterns, err := parseBoolOrRegexList(ctx, hideProjectNamesStr) if err != nil { return SanitizeParams{}, fmt.Errorf( "failed to parse regex hide project names param %q: %s", @@ -583,7 +582,7 @@ func loadSanitizeParams(v *viper.Viper) (SanitizeParams, error) { "settings.hidefilenames", ) - hideFileNamesPatterns, err := parseBoolOrRegexList(hideFileNamesStr) + hideFileNamesPatterns, err := parseBoolOrRegexList(ctx, hideFileNamesStr) if err != nil { return SanitizeParams{}, fmt.Errorf( "failed to parse regex hide file names param %q: %s", @@ -602,7 +601,7 @@ func loadSanitizeParams(v *viper.Viper) (SanitizeParams, error) { } func loadProjectParams(ctx context.Context, v *viper.Viper) (ProjectParams, error) { - submodulesDisabled, err := parseBoolOrRegexList(vipertools.GetString(v, "git.submodules_disabled")) + submodulesDisabled, err := parseBoolOrRegexList(ctx, vipertools.GetString(v, "git.submodules_disabled")) if err != nil { return ProjectParams{}, fmt.Errorf( "failed to parse regex submodules disabled param: %s", @@ -1154,9 +1153,11 @@ func (p StatusBar) String() string { ) } -func parseBoolOrRegexList(s string) ([]regex.Regex, error) { +func parseBoolOrRegexList(ctx context.Context, s string) ([]regex.Regex, error) { var patterns []regex.Regex + logger := log.Extract(ctx) + s = strings.ReplaceAll(s, "\r", "\n") s = strings.Trim(s, "\n\t ") @@ -1174,9 +1175,15 @@ func parseBoolOrRegexList(s string) ([]regex.Regex, error) { continue } + // make all regex case insensitive + if !strings.HasPrefix(s, "(?i)") { + s = "(?i)" + s + } + compiled, err := regex.Compile(s) if err != nil { - return nil, fmt.Errorf("failed to compile regex %q: %s", s, err) + logger.Warnf("failed to compile regex pattern %q, it will be ignored", s) + continue } patterns = append(patterns, compiled) diff --git a/cmd/params/params_internal_test.go b/cmd/params/params_internal_test.go index 2d72e323..f7a7655d 100644 --- a/cmd/params/params_internal_test.go +++ b/cmd/params/params_internal_test.go @@ -1,6 +1,7 @@ package params import ( + "context" "regexp" "testing" "time" @@ -13,6 +14,8 @@ import ( ) func TestParseBoolOrRegexList(t *testing.T) { + ctx := context.Background() + tests := map[string]struct { Input string Expected []regex.Regex @@ -32,29 +35,29 @@ func TestParseBoolOrRegexList(t *testing.T) { "valid regex": { Input: "\t.?\n\t\n \n\t\twakatime.? \t\n", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile(".?")), - regex.NewRegexpWrap(regexp.MustCompile("wakatime.?")), + regex.NewRegexpWrap(regexp.MustCompile("(?i).?")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)wakatime.?")), }, }, "valid regex with windows style": { Input: "\t.?\r\n\t\t\twakatime.? \t\r\n", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile(".?")), - regex.NewRegexpWrap(regexp.MustCompile("wakatime.?")), + regex.NewRegexpWrap(regexp.MustCompile("(?i).?")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)wakatime.?")), }, }, "valid regex with old mac style": { Input: "\t.?\r\t\t\twakatime.? \t\r", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile(".?")), - regex.NewRegexpWrap(regexp.MustCompile("wakatime.?")), + regex.NewRegexpWrap(regexp.MustCompile("(?i).?")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)wakatime.?")), }, }, } for name, test := range tests { t.Run(name, func(t *testing.T) { - regex, err := parseBoolOrRegexList(test.Input) + regex, err := parseBoolOrRegexList(ctx, test.Input) require.NoError(t, err) assert.Equal(t, test.Expected, regex) diff --git a/cmd/params/params_test.go b/cmd/params/params_test.go index 484b690b..37d05ee1 100644 --- a/cmd/params/params_test.go +++ b/cmd/params/params_test.go @@ -9,7 +9,6 @@ import ( "path/filepath" "regexp" "runtime" - "strings" "sync" "testing" "time" @@ -844,6 +843,18 @@ func TestLoadHeartbeatParams_Filter_Exclude(t *testing.T) { assert.Equal(t, "(?i)wakatime.?", params.Filter.Exclude[5].String()) } +func TestLoadHeartbeatParams_Filter_Exclude_All(t *testing.T) { + v := viper.New() + v.Set("entity", "/path/to/file") + v.Set("exclude", []string{"true"}) + + params, err := cmdparams.LoadHeartbeatParams(context.Background(), v) + require.NoError(t, err) + + require.Len(t, params.Filter.Exclude, 1) + assert.Equal(t, ".*", params.Filter.Exclude[0].String()) +} + func TestLoadHeartbeatParams_Filter_Exclude_Multiline(t *testing.T) { v := viper.New() v.Set("entity", "/path/to/file") @@ -929,6 +940,18 @@ func TestLoadHeartbeatParams_Filter_Include(t *testing.T) { assert.Equal(t, "(?i)wakatime.+", params.Filter.Include[3].String()) } +func TestLoadHeartbeatParams_Filter_Include_All(t *testing.T) { + v := viper.New() + v.Set("entity", "/path/to/file") + v.Set("include", []string{"true"}) + + params, err := cmdparams.LoadHeartbeatParams(context.Background(), v) + require.NoError(t, err) + + require.Len(t, params.Filter.Include, 1) + assert.Equal(t, ".*", params.Filter.Include[0].String()) +} + func TestLoadHeartbeatParams_Filter_Include_IgnoresInvalidRegex(t *testing.T) { v := viper.New() v.Set("entity", "/path/to/file") @@ -1047,14 +1070,14 @@ func TestLoadHeartbeatParams_SanitizeParams_HideBranchNames_List(t *testing.T) { "regex": { ViperValue: "fix.*", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile("fix.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)fix.*")), }, }, "regex list": { ViperValue: ".*secret.*\nfix.*", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile(".*secret.*")), - regex.NewRegexpWrap(regexp.MustCompile("fix.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i).*secret.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)fix.*")), }, }, } @@ -1134,19 +1157,32 @@ func TestLoadHeartbeatParams_SanitizeParams_HideBranchNames_ConfigDeprecatedTwo( } func TestLoadHeartbeatParams_SanitizeParams_HideBranchNames_InvalidRegex(t *testing.T) { + logFile, err := os.CreateTemp(t.TempDir(), "") + require.NoError(t, err) + + defer logFile.Close() + + ctx := context.Background() + v := viper.New() v.Set("entity", "/path/to/file") v.Set("hide-branch-names", ".*secret.*\n[0-9+") + v.Set("log-file", logFile.Name()) - _, err := cmdparams.LoadHeartbeatParams(context.Background(), v) - require.Error(t, err) + logger, err := cmd.SetupLogging(ctx, v) + require.NoError(t, err) + + defer logger.Flush() + + ctx = log.ToContext(ctx, logger) + + _, err = cmdparams.LoadHeartbeatParams(ctx, v) + require.NoError(t, err) - assert.True(t, strings.HasPrefix( - err.Error(), - "failed to load sanitize params:"+ - " failed to parse regex hide branch names param \".*secret.*\\n[0-9+\":"+ - " failed to compile regex \"[0-9+\":", - )) + output, err := io.ReadAll(logFile) + require.NoError(t, err) + + assert.Contains(t, string(output), "failed to compile regex pattern \\\"(?i)[0-9+\\\", it will be ignored") } func TestLoadHeartbeatParams_SanitizeParams_HideProjectNames_True(t *testing.T) { @@ -1209,14 +1245,14 @@ func TestLoadHeartbeatParams_SanitizeParams_HideProjecthNames_List(t *testing.T) "regex": { ViperValue: "fix.*", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile("fix.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)fix.*")), }, }, "regex list": { ViperValue: ".*secret.*\nfix.*", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile(".*secret.*")), - regex.NewRegexpWrap(regexp.MustCompile("fix.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i).*secret.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)fix.*")), }, }, } @@ -1296,19 +1332,32 @@ func TestLoadHeartbeatParams_SanitizeParams_HideProjectNames_ConfigDeprecatedTwo } func TestLoadHeartbeatParams_SanitizeParams_HideProjectNames_InvalidRegex(t *testing.T) { + logFile, err := os.CreateTemp(t.TempDir(), "") + require.NoError(t, err) + + defer logFile.Close() + + ctx := context.Background() + v := viper.New() v.Set("entity", "/path/to/file") v.Set("hide-project-names", ".*secret.*\n[0-9+") + v.Set("log-file", logFile.Name()) - _, err := cmdparams.LoadHeartbeatParams(context.Background(), v) - require.Error(t, err) + logger, err := cmd.SetupLogging(ctx, v) + require.NoError(t, err) - assert.True(t, strings.HasPrefix( - err.Error(), - "failed to load sanitize params:"+ - " failed to parse regex hide project names param \".*secret.*\\n[0-9+\":"+ - " failed to compile regex \"[0-9+\":", - )) + defer logger.Flush() + + ctx = log.ToContext(ctx, logger) + + _, err = cmdparams.LoadHeartbeatParams(ctx, v) + require.NoError(t, err) + + output, err := io.ReadAll(logFile) + require.NoError(t, err) + + assert.Contains(t, string(output), "failed to compile regex pattern \\\"(?i)[0-9+\\\", it will be ignored") } func TestLoadHeartbeatParams_SanitizeParams_HideFileNames_True(t *testing.T) { @@ -1371,14 +1420,14 @@ func TestLoadHeartbeatParams_SanitizeParams_HideFileNames_List(t *testing.T) { "regex": { ViperValue: "fix.*", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile("fix.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)fix.*")), }, }, "regex list": { ViperValue: ".*secret.*\nfix.*", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile(".*secret.*")), - regex.NewRegexpWrap(regexp.MustCompile("fix.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i).*secret.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)fix.*")), }, }, } @@ -1493,19 +1542,32 @@ func TestLoadHeartbeatParams_SanitizeParams_HideFileNames_ConfigDeprecatedTwo(t } func TestLoadHeartbeatParams_SanitizeParams_HideFileNames_InvalidRegex(t *testing.T) { + logFile, err := os.CreateTemp(t.TempDir(), "") + require.NoError(t, err) + + defer logFile.Close() + + ctx := context.Background() + v := viper.New() v.Set("entity", "/path/to/file") v.Set("hide-file-names", ".*secret.*\n[0-9+") + v.Set("log-file", logFile.Name()) - _, err := cmdparams.LoadHeartbeatParams(context.Background(), v) - require.Error(t, err) + logger, err := cmd.SetupLogging(ctx, v) + require.NoError(t, err) + + defer logger.Flush() + + ctx = log.ToContext(ctx, logger) + + _, err = cmdparams.LoadHeartbeatParams(ctx, v) + require.NoError(t, err) + + output, err := io.ReadAll(logFile) + require.NoError(t, err) - assert.True(t, strings.HasPrefix( - err.Error(), - "failed to load sanitize params:"+ - " failed to parse regex hide file names param \".*secret.*\\n[0-9+\":"+ - " failed to compile regex \"[0-9+\":", - )) + assert.Contains(t, string(output), "failed to compile regex pattern \\\"(?i)[0-9+\\\", it will be ignored") } func TestLoadHeartbeatParams_SanitizeParams_HideProjectFolder(t *testing.T) { @@ -1601,14 +1663,14 @@ func TestLoadHeartbeatsParams_SubmodulesDisabled_List(t *testing.T) { "regex": { ViperValue: "fix.*", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile("fix.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)fix.*")), }, }, "regex_list": { ViperValue: "\n.*secret.*\nfix.*", Expected: []regex.Regex{ - regex.NewRegexpWrap(regexp.MustCompile(".*secret.*")), - regex.NewRegexpWrap(regexp.MustCompile("fix.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i).*secret.*")), + regex.NewRegexpWrap(regexp.MustCompile("(?i)fix.*")), }, }, }