diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 0e1478a..0b2bbc4 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,9 +4,7 @@ _Before opening an issue, search for similar bug reports or feature requests on **System info:** -* **Version used (`uc -v`):** -* **OS (e.g. from `/etc/*-release`):** -* **Kernel (`uname -a`):** +* **Verbose version info (`uc -vv`):** * **Install tools:** **Steps to reproduce:** diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a792aa..b92ce12 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,22 +7,38 @@ on: branches: [master] schedule: - cron: '0 16 */15 * *' + workflow_dispatch: + inputs: + force_run: + description: 'Force workflow run' + required: true + type: choice + options: [yes, no] + +permissions: + actions: read + contents: read + statuses: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + SRC_DIR: src/github.com/${{ github.repository }} jobs: Go: name: Go runs-on: ubuntu-latest - env: - SRC_DIR: src/github.com/${{ github.repository }} - strategy: matrix: - go: [ '1.17.x', '1.18.x' ] + go: [ '1.19.x', '1.20.x' ] steps: - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v4 with: go-version: ${{ matrix.go }} diff --git a/Makefile b/Makefile index adf6605..8b219e3 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ ################################################################################ -# This Makefile generated by GoMakeGen 2.1.0 using next command: +# This Makefile generated by GoMakeGen 2.2.0 using next command: # gomakegen --mod . # # More info: https://kaos.sh/gomakegen @@ -94,6 +94,6 @@ help: ## Show this info | sed 's/ifdef //' \ | awk 'BEGIN {FS = " .*?## "}; {printf " \033[32m%-14s\033[0m %s\n", $$1, $$2}' @echo -e '' - @echo -e '\033[90mGenerated by GoMakeGen 2.1.0\033[0m\n' + @echo -e '\033[90mGenerated by GoMakeGen 2.2.0\033[0m\n' ################################################################################ diff --git a/README.md b/README.md index 78fedc5..3e27843 100644 --- a/README.md +++ b/README.md @@ -22,22 +22,16 @@ #### From sources -To build the `uc` from scratch, make sure you have a working Go 1.16+ workspace (_[instructions](https://golang.org/doc/install)_), then: +To build the `uc` from scratch, make sure you have a working Go 1.19+ workspace (_[instructions](https://golang.org/doc/install)_), then: ``` -go get github.com/essentialkaos/uc -``` - -If you want to update `uc` to latest stable release, do: - -``` -go get -u github.com/essentialkaos/uc +go install github.com/essentialkaos/uc@latest ``` #### From [ESSENTIAL KAOS Public Repository](https://yum.kaos.st) ```bash -sudo yum install -y https://yum.kaos.st/get/$(uname -r).rpm +sudo yum install -y https://yum.kaos.st/kaos-repo-latest.el$(grep 'CPE_NAME' /etc/os-release | tr -d '"' | cut -d':' -f5).noarch.rpm sudo yum install uc ``` diff --git a/cli/cli.go b/cli/cli.go new file mode 100644 index 0000000..20e137e --- /dev/null +++ b/cli/cli.go @@ -0,0 +1,445 @@ +package cli + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2023 ESSENTIAL KAOS // +// Apache License, Version 2.0 // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "bufio" + "fmt" + "hash/crc64" + "os" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/essentialkaos/ek/v12/fmtc" + "github.com/essentialkaos/ek/v12/fmtutil" + "github.com/essentialkaos/ek/v12/fsutil" + "github.com/essentialkaos/ek/v12/options" + "github.com/essentialkaos/ek/v12/signal" + "github.com/essentialkaos/ek/v12/strutil" + "github.com/essentialkaos/ek/v12/usage" + "github.com/essentialkaos/ek/v12/usage/completion/bash" + "github.com/essentialkaos/ek/v12/usage/completion/fish" + "github.com/essentialkaos/ek/v12/usage/completion/zsh" + "github.com/essentialkaos/ek/v12/usage/man" + "github.com/essentialkaos/ek/v12/usage/update" + + "github.com/essentialkaos/uc/cli/support" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Application basic info +const ( + APP = "uc" + VER = "1.1.1" + DESC = "Tool for counting unique lines" +) + +// Constants with options names +const ( + OPT_MAX_LINES = "m:max" + OPT_DISTRIBUTION = "d:dist" + OPT_NO_PROGRESS = "np:no-progress" + OPT_NO_COLOR = "nc:no-color" + OPT_HELP = "h:help" + OPT_VER = "v:version" + + OPT_VERB_VER = "vv:verbose-version" + OPT_COMPLETION = "completion" + OPT_GENERATE_MAN = "generate-man" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +// MAX_SAMPLE_SIZE is maximum sample size +const MAX_SAMPLE_SIZE = 512 + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Stats contains data info +type Stats struct { + Counters map[uint64]uint32 // crc64 → num + Samples map[uint64]string // crc64 → sample (512 symbols) + LastReadLines uint64 + LastReadBytes float64 + TotalReadLines uint64 + TotalReadBytes float64 + LastReadDate time.Time + Finished bool + + mx *sync.Mutex +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// LineInfo is struct with line info +type LineInfo struct { + CRC uint64 + Num uint32 +} + +type linesSlice []LineInfo + +func (s linesSlice) Len() int { return len(s) } +func (s linesSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s linesSlice) Less(i, j int) bool { + return s[i].Num < s[j].Num +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// optMap is map with options +var optMap = options.Map{ + OPT_MAX_LINES: {Type: options.INT}, + OPT_DISTRIBUTION: {Type: options.BOOL}, + OPT_NO_PROGRESS: {Type: options.BOOL}, + OPT_NO_COLOR: {Type: options.BOOL}, + OPT_HELP: {Type: options.BOOL}, + OPT_VER: {Type: options.BOOL}, + + OPT_VERB_VER: {Type: options.BOOL}, + OPT_COMPLETION: {}, + OPT_GENERATE_MAN: {Type: options.BOOL}, +} + +// stats contains info about data +var stats *Stats + +// rawMode is raw mode flag +var rawMode bool + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Run is main application function +func Run(gitRev string, gomod []byte) { + runtime.GOMAXPROCS(1) + + preConfigureUI() + + args, errs := options.Parse(optMap) + + if len(errs) != 0 { + printError(errs[0].Error()) + os.Exit(1) + } + + configureUI() + + switch { + case options.Has(OPT_COMPLETION): + os.Exit(printCompletion()) + case options.Has(OPT_GENERATE_MAN): + printMan() + os.Exit(0) + case options.GetB(OPT_VER): + genAbout(gitRev).Print() + os.Exit(0) + case options.GetB(OPT_VERB_VER): + support.Print(APP, VER, gitRev, gomod) + os.Exit(0) + case options.GetB(OPT_HELP) || len(args) == 0: + genUsage().Print() + os.Exit(0) + } + + signal.Handlers{ + signal.INT: signalHandler, + signal.TERM: signalHandler, + signal.QUIT: signalHandler, + }.TrackAsync() + + processData(args.Get(0).String()) +} + +// preConfigureUI preconfigures UI based on information about user terminal +func preConfigureUI() { + term := os.Getenv("TERM") + + fmtc.DisableColors = true + + if term != "" { + switch { + case strings.Contains(term, "xterm"), + strings.Contains(term, "color"), + term == "screen": + fmtc.DisableColors = false + } + } + + if !fsutil.IsCharacterDevice("/dev/stdout") && os.Getenv("FAKETTY") == "" { + fmtc.DisableColors = true + rawMode = true + } + + if os.Getenv("NO_COLOR") != "" { + fmtc.DisableColors = true + } +} + +// configureUI configures user interface +func configureUI() { + if options.GetB(OPT_NO_COLOR) { + fmtc.DisableColors = true + } + + if options.GetB(OPT_NO_PROGRESS) { + rawMode = true + } +} + +// processData starts data processing +func processData(input string) { + var r *bufio.Reader + + stats = &Stats{ + Counters: make(map[uint64]uint32), + mx: &sync.Mutex{}, + } + + if input == "-" { + r = bufio.NewReader(os.Stdin) + } else { + fd, err := os.OpenFile(input, os.O_RDONLY, 0) + + if err != nil { + printError(err.Error()) + os.Exit(1) + } + + r = bufio.NewReader(fd) + } + + readData(bufio.NewScanner(r)) +} + +// readData reads data +func readData(s *bufio.Scanner) { + ct := crc64.MakeTable(crc64.ECMA) + dist := options.GetB(OPT_DISTRIBUTION) + maxLines, err := parseMaxLines(options.GetS(OPT_MAX_LINES)) + + if err != nil { + printError(err.Error()) + os.Exit(1) + } + + if dist { + stats.Samples = make(map[uint64]string) + } + + stats.LastReadDate = time.Now() + + if !rawMode { + go printProgress() + } + + for s.Scan() { + data := s.Bytes() + dataLen := float64(len(data)) + dataCrc := crc64.Checksum(data, ct) + + stats.mx.Lock() + + stats.Counters[dataCrc]++ + stats.LastReadBytes += dataLen + stats.LastReadLines++ + + stats.TotalReadLines++ + stats.TotalReadBytes += dataLen + + if dist { + _, exist := stats.Samples[dataCrc] + + if !exist { + stats.Samples[dataCrc] = strutil.Substr(string(data), 0, MAX_SAMPLE_SIZE) + } + } + + if maxLines > 0 && len(stats.Counters) == maxLines { + stats.mx.Unlock() + break + } + + stats.mx.Unlock() + } + + printResults() +} + +// printProgress shows data processing progress +func printProgress() { + for range time.NewTicker(time.Second / 4).C { + stats.mx.Lock() + + if stats.Finished { + break + } + + now := time.Now() + dur := now.Sub(stats.LastReadDate) + readSpeed := stats.LastReadBytes / dur.Seconds() + + fmtc.TPrintf( + "{s}%12s/s {s-}|{s} %-12s {s-}|{s} %12s/s {s-}|{s} %-12s{!}", + fmtutil.PrettyNum(stats.LastReadLines), + fmtutil.PrettyNum(stats.TotalReadLines), + fmtutil.PrettySize(readSpeed), + fmtutil.PrettySize(stats.TotalReadBytes), + ) + + stats.LastReadLines = 0 + stats.LastReadBytes = 0 + stats.LastReadDate = now + + stats.mx.Unlock() + } +} + +// printResults shows results +func printResults() { + stats.mx.Lock() + + stats.Finished = true + + if options.GetB(OPT_DISTRIBUTION) { + printDistribution() + } else { + fmtc.TPrintln(len(stats.Counters)) + } + + stats.mx.Unlock() +} + +// printDistribution prints distrubution info +func printDistribution() { + var distData linesSlice + + for crc, num := range stats.Counters { + distData = append(distData, LineInfo{crc, num}) + } + + sort.Sort(sort.Reverse(distData)) + + for _, info := range distData { + fmtc.TPrintf(" %7d %s\n", info.Num, stats.Samples[info.CRC]) + } +} + +// parseMaxLines parses max line option +func parseMaxLines(maxLines string) (int, error) { + if maxLines == "" { + return 0, nil + } + + maxLines = strings.ToUpper(maxLines) + + mp := 1 + + switch { + case strings.HasSuffix(maxLines, "K"): + maxLines = strutil.Exclude(maxLines, "K") + mp = 1000 + case strings.HasSuffix(maxLines, "M"): + mp = 1000 * 1000 + maxLines = strutil.Exclude(maxLines, "M") + } + + num, err := strconv.Atoi(maxLines) + + if err != nil { + return 0, err + } + + return num * mp, nil +} + +// signalHandler is signal handler +func signalHandler() { + printResults() + os.Exit(0) +} + +// printError prints error message to console +func printError(f string, a ...interface{}) { + fmtc.Fprintf(os.Stderr, "{r}"+f+"{!}\n", a...) +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// printCompletion prints completion for given shell +func printCompletion() int { + switch options.GetS(OPT_COMPLETION) { + case "bash": + fmt.Printf(bash.Generate(genUsage(), APP)) + case "fish": + fmt.Printf(fish.Generate(genUsage(), APP)) + case "zsh": + fmt.Printf(zsh.Generate(genUsage(), optMap, APP)) + default: + return 1 + } + + return 0 +} + +// printMan prints man page +func printMan() { + fmt.Println( + man.Generate( + genUsage(), + genAbout(""), + ), + ) +} + +// genUsage generates usage info +func genUsage() *usage.Info { + info := usage.NewInfo(APP, "file") + + info.AddOption(OPT_DISTRIBUTION, "Show number of occurrences for every line") + info.AddOption(OPT_MAX_LINES, "Max number of unique lines", "num") + info.AddOption(OPT_NO_PROGRESS, "Disable progress output") + info.AddOption(OPT_NO_PROGRESS, "Disable progress output") + info.AddOption(OPT_NO_COLOR, "Disable colors in output") + info.AddOption(OPT_HELP, "Show this help message") + info.AddOption(OPT_VER, "Show version") + + info.AddExample("file.txt", "Count unique lines in file.txt") + info.AddExample("-d file.txt", "Show distribution for file.txt") + info.AddExample("-d -m 5k file.txt", "Show distribution for file.txt with 5,000 uniq lines max") + info.AddRawExample( + "cat file.txt | "+APP+" -", + "Count unique lines in stdin data", + ) + + return info +} + +// genAbout generates info about version +func genAbout(gitRev string) *usage.About { + about := &usage.About{ + App: APP, + Version: VER, + Desc: DESC, + Year: 2009, + Owner: "ESSENTIAL KAOS", + License: "Apache License, Version 2.0 ", + BugTracker: "https://github.com/essentialkaos/uc", + UpdateChecker: usage.UpdateChecker{"essentialkaos/uc", update.GitHubChecker}, + } + + if gitRev != "" { + about.Build = "git:" + gitRev + } + + return about +} diff --git a/cli/support/support.go b/cli/support/support.go new file mode 100644 index 0000000..c5d80fd --- /dev/null +++ b/cli/support/support.go @@ -0,0 +1,166 @@ +package support + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2023 ESSENTIAL KAOS // +// Apache License, Version 2.0 // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "fmt" + "os" + "runtime" + "strings" + + "github.com/essentialkaos/ek/v12/fmtc" + "github.com/essentialkaos/ek/v12/fmtutil" + "github.com/essentialkaos/ek/v12/fsutil" + "github.com/essentialkaos/ek/v12/hash" + "github.com/essentialkaos/ek/v12/strutil" + "github.com/essentialkaos/ek/v12/system" + + "github.com/essentialkaos/depsy" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Print prints verbose info about application, system, dependencies and +// important environment +func Print(app, ver, gitRev string, gomod []byte) { + fmtutil.SeparatorTitleColorTag = "{s-}" + fmtutil.SeparatorFullscreen = false + fmtutil.SeparatorColorTag = "{s-}" + fmtutil.SeparatorSize = 80 + + showApplicationInfo(app, ver, gitRev) + showOSInfo() + showDepsInfo(gomod) + + fmtutil.Separator(false) +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// showApplicationInfo shows verbose information about application +func showApplicationInfo(app, ver, gitRev string) { + fmtutil.Separator(false, "APPLICATION INFO") + + printInfo(7, "Name", app) + printInfo(7, "Version", ver) + + printInfo(7, "Go", fmtc.Sprintf( + "%s {s}(%s/%s){!}", + strings.TrimLeft(runtime.Version(), "go"), + runtime.GOOS, runtime.GOARCH, + )) + + if gitRev != "" { + if !fmtc.DisableColors && fmtc.IsTrueColorSupported() { + printInfo(7, "Git SHA", gitRev+getHashColorBullet(gitRev)) + } else { + printInfo(7, "Git SHA", gitRev) + } + } + + bin, _ := os.Executable() + binSHA := hash.FileHash(bin) + + if binSHA != "" { + binSHA = strutil.Head(binSHA, 7) + if !fmtc.DisableColors && fmtc.IsTrueColorSupported() { + printInfo(7, "Bin SHA", binSHA+getHashColorBullet(binSHA)) + } else { + printInfo(7, "Bin SHA", binSHA) + } + } +} + +// showOSInfo shows verbose information about system +func showOSInfo() { + osInfo, err := system.GetOSInfo() + + if err == nil { + fmtutil.Separator(false, "OS INFO") + + printInfo(12, "Name", osInfo.Name) + printInfo(12, "Pretty Name", osInfo.PrettyName) + printInfo(12, "Version", osInfo.VersionID) + printInfo(12, "ID", osInfo.ID) + printInfo(12, "ID Like", osInfo.IDLike) + printInfo(12, "Version ID", osInfo.VersionID) + printInfo(12, "Version Code", osInfo.VersionCodename) + printInfo(12, "CPE", osInfo.CPEName) + } + + systemInfo, err := system.GetSystemInfo() + + if err != nil { + return + } else { + if osInfo == nil { + fmtutil.Separator(false, "SYSTEM INFO") + printInfo(12, "Name", systemInfo.OS) + } + } + + printInfo(12, "Arch", systemInfo.Arch) + printInfo(12, "Kernel", systemInfo.Kernel) + + containerEngine := "No" + + switch { + case fsutil.IsExist("/.dockerenv"): + containerEngine = "Yes (Docker)" + case fsutil.IsExist("/run/.containerenv"): + containerEngine = "Yes (Podman)" + } + + fmtc.NewLine() + + printInfo(12, "Container", containerEngine) +} + +// showDepsInfo shows information about all dependencies +func showDepsInfo(gomod []byte) { + deps := depsy.Extract(gomod, false) + + if len(deps) == 0 { + return + } + + fmtutil.Separator(false, "DEPENDENCIES") + + for _, dep := range deps { + if dep.Extra == "" { + fmtc.Printf(" {s}%8s{!} %s\n", dep.Version, dep.Path) + } else { + fmtc.Printf(" {s}%8s{!} %s {s-}(%s){!}\n", dep.Version, dep.Path, dep.Extra) + } + } +} + +// getHashColorBullet return bullet with color from hash +func getHashColorBullet(v string) string { + if len(v) > 6 { + v = strutil.Head(v, 6) + } + + return fmtc.Sprintf(" {#" + strutil.Head(v, 6) + "}● {!}") +} + +// printInfo formats and prints info record +func printInfo(size int, name, value string) { + name = name + ":" + size++ + + if value == "" { + fm := fmt.Sprintf(" {*}%%-%ds{!} {s-}—{!}\n", size) + fmtc.Printf(fm, name) + } else { + fm := fmt.Sprintf(" {*}%%-%ds{!} %%s\n", size) + fmtc.Printf(fm, name, value) + } +} + +// ////////////////////////////////////////////////////////////////////////////////// // diff --git a/common/uc.spec b/common/uc.spec index c216101..36e55cf 100644 --- a/common/uc.spec +++ b/common/uc.spec @@ -1,28 +1,24 @@ ################################################################################ -# rpmbuilder:relative-pack true +%define debug_package %{nil} ################################################################################ -%define debug_package %{nil} +Summary: Simple utility for counting unique lines +Name: uc +Version: 1.1.1 +Release: 0%{?dist} +Group: Applications/System +License: Apache License, Version 2.0 +URL: https://kaos.sh/uc -################################################################################ - -Summary: Simple utility for counting unique lines -Name: uc -Version: 1.1.0 -Release: 0%{?dist} -Group: Applications/System -License: Apache License, Version 2.0 -URL: https://kaos.sh/uc - -Source0: https://source.kaos.st/%{name}/%{name}-%{version}.tar.bz2 +Source0: https://source.kaos.st/%{name}/%{name}-%{version}.tar.bz2 -BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) +BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -BuildRequires: golang >= 1.17 +BuildRequires: golang >= 1.19 -Provides: %{name} = %{version}-%{release} +Provides: %{name} = %{version}-%{release} ################################################################################ @@ -35,9 +31,14 @@ Simple utility for counting unique lines. %setup -q %build -export GOPATH=$(pwd) -pushd src/github.com/essentialkaos/%{name} - go build -mod vendor -o $GOPATH/%{name} %{name}.go +if [[ ! -d "%{name}/vendor" ]] ; then + echo "This package requires vendored dependencies" + exit 1 +fi + +pushd %{name} + go build %{name}.go + cp LICENSE .. popd %install @@ -46,9 +47,9 @@ rm -rf %{buildroot} install -dm 755 %{buildroot}%{_bindir} install -dm 755 %{buildroot}%{_mandir}/man1 -install -pm 755 %{name} %{buildroot}%{_bindir}/ +install -pm 755 %{name}/%{name} %{buildroot}%{_bindir}/ -./%{name} --generate-man > %{buildroot}%{_mandir}/man1/%{name}.1 +./%{name}/%{name} --generate-man > %{buildroot}%{_mandir}/man1/%{name}.1 %clean rm -rf %{buildroot} @@ -64,6 +65,14 @@ rm -rf %{buildroot} ################################################################################ %changelog +* Mon Mar 06 2023 Anton Novojilov - 1.1.1-0 +- Added verbose info output +- Dependencies update +- Code refactoring + +* Thu Dec 01 2022 Anton Novojilov - 1.1.0-1 +- Fixed build using sources from source.kaos.st + * Wed Aug 10 2022 Anton Novojilov - 1.1.0-0 - Minor UI improvements - Fixed bug with parsing max number of unique lines diff --git a/go.mod b/go.mod index 410473f..8cc5b90 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,10 @@ module github.com/essentialkaos/uc -go 1.17 +go 1.18 -require github.com/essentialkaos/ek/v12 v12.53.0 +require ( + github.com/essentialkaos/depsy v1.0.0 + github.com/essentialkaos/ek/v12 v12.63.0 +) -require golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b // indirect +require golang.org/x/sys v0.6.0 // indirect diff --git a/go.sum b/go.sum index 425e2c8..a0cb15a 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,10 @@ -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/essentialkaos/check v1.3.0 h1:ria+8o22RCLdt2D/1SHQsEH5Mmy5S+iWHaGHrrbPUc0= -github.com/essentialkaos/check v1.3.0/go.mod h1:PhxzfJWlf5L/skuyhzBLIvjMB5Xu9TIyDIsqpY5MvB8= -github.com/essentialkaos/ek/v12 v12.53.0 h1:sBSzM4ZQ487wRqAIB7kfftqMSi8/HXIr5exJlBbdljA= -github.com/essentialkaos/ek/v12 v12.53.0/go.mod h1:Y8ln7hqABw8GT1vWuU7cCJfZAdE1uxmOYZvOVv8HRzo= -github.com/essentialkaos/go-linenoise/v3 v3.4.0/go.mod h1:t1kNLY2bSMQCy1JXOefD2BDLs/TTPMtTv3DFNV5uDSI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/essentialkaos/check v1.4.0 h1:kWdFxu9odCxUqo1NNFNJmguGrDHgwi3A8daXX1nkuKk= +github.com/essentialkaos/depsy v1.0.0 h1:FikBtTnNhk+xFO/hFr+CfiKs6QnA3wMD6tGL0XTEUkc= +github.com/essentialkaos/depsy v1.0.0/go.mod h1:XVsB2eVUonEzmLKQP3ig2P6v2+WcHVgJ10zm0JLqFMM= +github.com/essentialkaos/ek/v12 v12.63.0 h1:9yaEu5W3bx//9y52ShqYCoFDKOcwEdrnvgSkUYyatgI= +github.com/essentialkaos/ek/v12 v12.63.0/go.mod h1:9MlSuHpewu7OZ9tM9dLFHvoA8dflBIUPCA0Ctt97wRs= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= -github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b h1:2n253B2r0pYSmEV+UNCQoPfU/FiaizQEK5Gu4Bq4JE8= -golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/uc.go b/uc.go index 4393f12..d08f46b 100644 --- a/uc.go +++ b/uc.go @@ -2,428 +2,27 @@ package main // ////////////////////////////////////////////////////////////////////////////////// // // // -// Copyright (c) 2022 ESSENTIAL KAOS // +// Copyright (c) 2023 ESSENTIAL KAOS // // Apache License, Version 2.0 // // // // ////////////////////////////////////////////////////////////////////////////////// // import ( - "bufio" - "fmt" - "hash/crc64" - "os" - "runtime" - "sort" - "strconv" - "strings" - "sync" - "time" + _ "embed" - "github.com/essentialkaos/ek/v12/fmtc" - "github.com/essentialkaos/ek/v12/fmtutil" - "github.com/essentialkaos/ek/v12/fsutil" - "github.com/essentialkaos/ek/v12/options" - "github.com/essentialkaos/ek/v12/signal" - "github.com/essentialkaos/ek/v12/strutil" - "github.com/essentialkaos/ek/v12/usage" - "github.com/essentialkaos/ek/v12/usage/completion/bash" - "github.com/essentialkaos/ek/v12/usage/completion/fish" - "github.com/essentialkaos/ek/v12/usage/completion/zsh" - "github.com/essentialkaos/ek/v12/usage/man" - "github.com/essentialkaos/ek/v12/usage/update" + CLI "github.com/essentialkaos/uc/cli" ) // ////////////////////////////////////////////////////////////////////////////////// // -// Application basic info -const ( - APP = "uc" - VER = "1.1.0" - DESC = "Tool for counting unique lines" -) - -// Constants with options names -const ( - OPT_MAX_LINES = "m:max" - OPT_DISTRIBUTION = "d:dist" - OPT_NO_PROGRESS = "np:no-progress" - OPT_NO_COLOR = "nc:no-color" - OPT_HELP = "h:help" - OPT_VER = "v:version" - - OPT_COMPLETION = "completion" - OPT_GENERATE_MAN = "generate-man" -) - -// ////////////////////////////////////////////////////////////////////////////////// // - -// MAX_SAMPLE_SIZE is maximum sample size -const MAX_SAMPLE_SIZE = 512 - -// ////////////////////////////////////////////////////////////////////////////////// // - -// Stats contains data info -type Stats struct { - Counters map[uint64]uint32 // crc64 → num - Samples map[uint64]string // crc64 → sample (512 symbols) - LastReadLines uint64 - LastReadBytes float64 - TotalReadLines uint64 - TotalReadBytes float64 - LastReadDate time.Time - Finished bool - - mx *sync.Mutex -} - -// ////////////////////////////////////////////////////////////////////////////////// // +//go:embed go.mod +var gomod []byte -// LineInfo is struct with line info -type LineInfo struct { - CRC uint64 - Num uint32 -} - -type linesSlice []LineInfo - -func (s linesSlice) Len() int { return len(s) } -func (s linesSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s linesSlice) Less(i, j int) bool { - return s[i].Num < s[j].Num -} - -// ////////////////////////////////////////////////////////////////////////////////// // - -// optMap is map with options -var optMap = options.Map{ - OPT_MAX_LINES: {Type: options.INT}, - OPT_DISTRIBUTION: {Type: options.BOOL}, - OPT_NO_PROGRESS: {Type: options.BOOL}, - OPT_NO_COLOR: {Type: options.BOOL}, - OPT_HELP: {Type: options.BOOL, Alias: "u:usage"}, - OPT_VER: {Type: options.BOOL, Alias: "ver"}, - - OPT_COMPLETION: {}, - OPT_GENERATE_MAN: {Type: options.BOOL}, -} - -// stats contains info about data -var stats *Stats - -// rawMode is raw mode flag -var rawMode bool +// gitrev is short hash of the latest git commit +var gitrev string // ////////////////////////////////////////////////////////////////////////////////// // -// main is main func func main() { - runtime.GOMAXPROCS(1) - - args, errs := options.Parse(optMap) - - if len(errs) != 0 { - printError("Options parsing errors:") - - for _, err := range errs { - printError(" %v", err) - } - - os.Exit(1) - } - - if options.Has(OPT_COMPLETION) { - os.Exit(genCompletion()) - } - - if options.Has(OPT_GENERATE_MAN) { - genMan() - os.Exit(0) - } - - configureUI() - - if options.GetB(OPT_VER) { - showAbout() - os.Exit(0) - } - - if options.GetB(OPT_HELP) || len(args) == 0 { - showUsage() - os.Exit(0) - } - - signal.Handlers{ - signal.INT: signalHandler, - signal.TERM: signalHandler, - signal.QUIT: signalHandler, - }.TrackAsync() - - processData(args.Get(0).String()) -} - -// configureUI configures user interface -func configureUI() { - if options.GetB(OPT_NO_COLOR) { - fmtc.DisableColors = true - } - - if !fsutil.IsCharacterDevice("/dev/stdout") && os.Getenv("FAKETTY") == "" { - rawMode = true - } - - if options.GetB(OPT_NO_PROGRESS) { - rawMode = true - } -} - -// processData starts data processing -func processData(input string) { - var r *bufio.Reader - - stats = &Stats{ - Counters: make(map[uint64]uint32), - mx: &sync.Mutex{}, - } - - if input == "-" { - r = bufio.NewReader(os.Stdin) - } else { - fd, err := os.OpenFile(input, os.O_RDONLY, 0) - - if err != nil { - printError(err.Error()) - os.Exit(1) - } - - r = bufio.NewReader(fd) - } - - readData(bufio.NewScanner(r)) -} - -// readData reads data -func readData(s *bufio.Scanner) { - ct := crc64.MakeTable(crc64.ECMA) - dist := options.GetB(OPT_DISTRIBUTION) - maxLines, err := parseMaxLines(options.GetS(OPT_MAX_LINES)) - - if err != nil { - printError(err.Error()) - os.Exit(1) - } - - if dist { - stats.Samples = make(map[uint64]string) - } - - stats.LastReadDate = time.Now() - - if !rawMode { - go printProgress() - } - - for s.Scan() { - data := s.Bytes() - dataLen := float64(len(data)) - dataCrc := crc64.Checksum(data, ct) - - stats.mx.Lock() - - stats.Counters[dataCrc]++ - stats.LastReadBytes += dataLen - stats.LastReadLines++ - - stats.TotalReadLines++ - stats.TotalReadBytes += dataLen - - if dist { - _, exist := stats.Samples[dataCrc] - - if !exist { - stats.Samples[dataCrc] = strutil.Substr(string(data), 0, MAX_SAMPLE_SIZE) - } - } - - if maxLines > 0 && len(stats.Counters) == maxLines { - stats.mx.Unlock() - break - } - - stats.mx.Unlock() - } - - printResults() -} - -// printProgress shows data processing progress -func printProgress() { - for range time.NewTicker(time.Second / 4).C { - stats.mx.Lock() - - if stats.Finished { - break - } - - now := time.Now() - dur := now.Sub(stats.LastReadDate) - readSpeed := stats.LastReadBytes / dur.Seconds() - - fmtc.TPrintf( - "{s}%12s/s {s-}|{s} %-12s {s-}|{s} %12s/s {s-}|{s} %-12s{!}", - fmtutil.PrettyNum(stats.LastReadLines), - fmtutil.PrettyNum(stats.TotalReadLines), - fmtutil.PrettySize(readSpeed), - fmtutil.PrettySize(stats.TotalReadBytes), - ) - - stats.LastReadLines = 0 - stats.LastReadBytes = 0 - stats.LastReadDate = now - - stats.mx.Unlock() - } -} - -// printResults shows results -func printResults() { - stats.mx.Lock() - - stats.Finished = true - - if options.GetB(OPT_DISTRIBUTION) { - printDistribution() - } else { - fmtc.TPrintln(len(stats.Counters)) - } - - stats.mx.Unlock() -} - -// printDistribution prints distrubution info -func printDistribution() { - var distData linesSlice - - for crc, num := range stats.Counters { - distData = append(distData, LineInfo{crc, num}) - } - - sort.Sort(sort.Reverse(distData)) - - for _, info := range distData { - fmtc.TPrintf(" %7d %s\n", info.Num, stats.Samples[info.CRC]) - } -} - -// parseMaxLines parses max line option -func parseMaxLines(maxLines string) (int, error) { - if maxLines == "" { - return 0, nil - } - - maxLines = strings.ToUpper(maxLines) - - mp := 1 - - switch { - case strings.HasSuffix(maxLines, "K"): - maxLines = strutil.Exclude(maxLines, "K") - mp = 1000 - case strings.HasSuffix(maxLines, "M"): - mp = 1000 * 1000 - maxLines = strutil.Exclude(maxLines, "M") - } - - num, err := strconv.Atoi(maxLines) - - if err != nil { - return 0, err - } - - return num * mp, nil -} - -// signalHandler is signal handler -func signalHandler() { - printResults() - os.Exit(0) -} - -// printError prints error message to console -func printError(f string, a ...interface{}) { - fmtc.Fprintf(os.Stderr, "{r}"+f+"{!}\n", a...) -} - -// ////////////////////////////////////////////////////////////////////////////////// // - -// showUsage prints usage info -func showUsage() { - genUsage().Render() -} - -// showAbout prints info about version -func showAbout() { - genAbout().Render() -} - -// genCompletion generates completion for different shells -func genCompletion() int { - switch options.GetS(OPT_COMPLETION) { - case "bash": - fmt.Printf(bash.Generate(genUsage(), APP)) - case "fish": - fmt.Printf(fish.Generate(genUsage(), APP)) - case "zsh": - fmt.Printf(zsh.Generate(genUsage(), optMap, APP)) - default: - return 1 - } - - return 0 -} - -// genMan generates man page -func genMan() { - fmt.Println( - man.Generate( - genUsage(), - genAbout(), - ), - ) -} - -// genUsage generates usage info -func genUsage() *usage.Info { - info := usage.NewInfo(APP, "file") - - info.AddOption(OPT_DISTRIBUTION, "Show number of occurrences for every line") - info.AddOption(OPT_MAX_LINES, "Max number of unique lines", "num") - info.AddOption(OPT_NO_PROGRESS, "Disable progress output") - info.AddOption(OPT_NO_PROGRESS, "Disable progress output") - info.AddOption(OPT_NO_COLOR, "Disable colors in output") - info.AddOption(OPT_HELP, "Show this help message") - info.AddOption(OPT_VER, "Show version") - - info.AddExample("file.txt", "Count unique lines in file.txt") - info.AddExample("-d file.txt", "Show distribution for file.txt") - info.AddExample("-d -m 5k file.txt", "Show distribution for file.txt with 5,000 uniq lines max") - info.AddRawExample( - "cat file.txt | "+APP+" -", - "Count unique lines in stdin data", - ) - - return info -} - -// genAbout generates info about version -func genAbout() *usage.About { - return &usage.About{ - App: APP, - Version: VER, - Desc: DESC, - Year: 2009, - Owner: "ESSENTIAL KAOS", - License: "Apache License, Version 2.0 ", - BugTracker: "https://github.com/essentialkaos/uc", - UpdateChecker: usage.UpdateChecker{"essentialkaos/uc", update.GitHubChecker}, - } + CLI.Run(gitrev, gomod) }