Skip to content

Commit

Permalink
✨ Upload files instead of multi-part form. (#127)
Browse files Browse the repository at this point in the history
Upload issues and deps files instead of using multi-part form.
Much simpler and more easily supports the addon staging the issues and
deps files on disk rather than streaming. The more atomic approach will
prevent transaction deadlock which can more easily occur when the
addon-analyzer builder reported an error (which it should never do).

Requires: konveyor/tackle2-hub#743

---------

Signed-off-by: Jeff Ortel <[email protected]>
  • Loading branch information
jortel authored Sep 27, 2024
1 parent 953224b commit 63cc40a
Show file tree
Hide file tree
Showing 7 changed files with 83 additions and 52 deletions.
23 changes: 5 additions & 18 deletions builder/deps.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,30 +14,15 @@ type Deps struct {
Path string
}

// Reader returns a reader.
func (b *Deps) Reader() (r io.Reader) {
r, w := io.Pipe()
go func() {
var err error
defer func() {
if err != nil {
_ = w.CloseWithError(err)
} else {
_ = w.Close()
}
}()
err = b.Write(w)
}()
return
}

// Write deps to the writer.
// Write deps section.
func (b *Deps) Write(writer io.Writer) (err error) {
input, err := b.read()
if err != nil {
return
}
encoder := yaml.NewEncoder(writer)
_, _ = writer.Write([]byte(api.BeginDepsMarker))
_, _ = writer.Write([]byte{'\n'})
for _, p := range input {
for _, d := range p.Dependencies {
err = encoder.Encode(
Expand All @@ -54,6 +39,8 @@ func (b *Deps) Write(writer io.Writer) (err error) {
}
}
}
_, _ = writer.Write([]byte(api.EndDepsMarker))
_, _ = writer.Write([]byte{'\n'})
return
}

Expand Down
28 changes: 9 additions & 19 deletions builder/issue.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,30 +31,18 @@ func (b *Issues) RuleError() (r *RuleError) {
return &b.ruleErr
}

// Reader returns a reader.
func (b *Issues) Reader() (r io.Reader) {
r, w := io.Pipe()
go func() {
var err error
defer func() {
if err != nil {
_ = w.CloseWithError(err)
} else {
_ = w.Close()
}
}()
err = b.Write(w)
}()
return
}

// Write issues to the writer.
// Write issues section.
func (b *Issues) Write(writer io.Writer) (err error) {
input, err := b.read()
if err != nil {
return
}
encoder := yaml.NewEncoder(writer)
_, _ = writer.Write([]byte(api.BeginIssuesMarker))
_, _ = writer.Write([]byte{'\n'})
if err != nil {
return
}
for _, ruleset := range input {
b.ruleErr.Append(ruleset)
for ruleid, v := range ruleset.Violations {
Expand Down Expand Up @@ -98,6 +86,8 @@ func (b *Issues) Write(writer io.Writer) (err error) {
}
}
}
_, _ = writer.Write([]byte(api.EndIssuesMarker))
_, _ = writer.Write([]byte{'\n'})
return
}

Expand Down Expand Up @@ -139,7 +129,7 @@ func (b *Issues) Tags() (tags []string) {
}

// Facts builds facts.
func (b *Issues) Facts() (facts api.FactMap) {
func (b *Issues) Facts() (facts api.Map) {
return
}

Expand Down
46 changes: 46 additions & 0 deletions builder/manifest.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package builder

import (
"os"

"github.com/konveyor/tackle2-hub/api"
"gopkg.in/yaml.v2"
)

// Manifest file.
type Manifest struct {
Analysis api.Analysis
Issues *Issues
Deps *Deps
Path string
}

// Write manifest file.
func (m *Manifest) Write() (err error) {
m.Path = "manifest.yaml"
file, err := os.Create(m.Path)
if err != nil {
return
}
defer func() {
_ = file.Close()
}()
_, _ = file.Write([]byte(api.BeginMainMarker))
_, _ = file.Write([]byte{'\n'})
encoder := yaml.NewEncoder(file)
err = encoder.Encode(m.Analysis)
if err != nil {
return
}
_, _ = file.Write([]byte(api.EndMainMarker))
_, _ = file.Write([]byte{'\n'})
err = m.Issues.Write(file)
if err != nil {
return
}
err = m.Deps.Write(file)
if err != nil {
return
}
return
}
2 changes: 0 additions & 2 deletions cmd/analyzer.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (
"github.com/konveyor/tackle2-addon/command"
)

type RuleError = builder.RuleError

// Analyzer application analyzer.
type Analyzer struct {
*Data
Expand Down
24 changes: 14 additions & 10 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -148,24 +148,28 @@ func updateApplication(d *Data, appId uint, issues *builder.Issues, deps *builde
}
//
// Analysis.
appAnalysis := addon.Application.Analysis(appId)
mark := time.Now()
analysis := &api.Analysis{}
manifest := builder.Manifest{
Analysis: api.Analysis{},
Issues: issues,
Deps: deps,
}
if d.Mode.Repository != nil {
analysis.Commit, err = d.Mode.Repository.Head()
manifest.Analysis.Commit, err = d.Mode.Repository.Head()
if err != nil {
return
}
}
err = appAnalysis.Create(
analysis,
binding.MIMEYAML,
issues.Reader(),
deps.Reader())
err = manifest.Write()
if err != nil {
return
}
mark := time.Now()
analysis := addon.Application.Analysis(appId)
reported, err := analysis.Create(manifest.Path, binding.MIMEYAML)
if err != nil {
return
}
addon.Activity("Analysis reported. duration: %s", time.Since(mark))
addon.Activity("Analysis %d reported. duration: %s", reported.ID, time.Since(mark))
// Facts.
facts := addon.Application.Facts(appId)
facts.Source(Source)
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/gin-gonic/gin v1.9.1
github.com/konveyor/analyzer-lsp v0.4.0-alpha.1.0.20240603131628-bc4ff29956a2
github.com/konveyor/tackle2-addon v0.5.0-rc.2.0.20240813152219-3153a61e7c46
github.com/konveyor/tackle2-hub v0.5.0-rc.1.0.20240726125502-8bb3c0911660
github.com/konveyor/tackle2-hub v0.5.1-0.20240926152344-e15a8a4fbf23
github.com/onsi/gomega v1.27.6
github.com/rogpeppe/go-internal v1.10.0
go.lsp.dev/uri v0.3.0
Expand Down Expand Up @@ -63,6 +63,7 @@ require (
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-sqlite3 v1.14.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
Expand Down
9 changes: 7 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi
github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8=
github.com/andygrunwald/go-jira v1.16.0 h1:PU7C7Fkk5L96JvPc6vDVIrd99vdPnYudHu4ju2c2ikQ=
github.com/andygrunwald/go-jira v1.16.0/go.mod h1:UQH4IBVxIYWbgagc0LF/k9FRs9xjIiQ8hIcC6HfLwFU=
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bombsimon/logrusr/v3 v3.0.0 h1:tcAoLfuAhKP9npBxWzSdpsvKPQt1XV02nSf2lZA82TQ=
Expand All @@ -32,6 +33,7 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE=
github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
Expand Down Expand Up @@ -116,6 +118,7 @@ github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc=
Expand All @@ -141,8 +144,8 @@ github.com/konveyor/analyzer-lsp v0.4.0-alpha.1.0.20240603131628-bc4ff29956a2 h1
github.com/konveyor/analyzer-lsp v0.4.0-alpha.1.0.20240603131628-bc4ff29956a2/go.mod h1:GXkSykQ84oE1SyMvFko9s9wRn/FMdl4efLLWSjMX2nU=
github.com/konveyor/tackle2-addon v0.5.0-rc.2.0.20240813152219-3153a61e7c46 h1:cHLMtHTCiFxYNMejsZvhFTuByjipjlm/5Zt5+hdYRVY=
github.com/konveyor/tackle2-addon v0.5.0-rc.2.0.20240813152219-3153a61e7c46/go.mod h1:1cHTnmMGtYzv0GIMiyEMPkJe+hOlzbhxwRal5e7Mog0=
github.com/konveyor/tackle2-hub v0.5.0-rc.1.0.20240726125502-8bb3c0911660 h1:joaeY9ndWjNd6rgwyhoa+lgcIRiNwvAcR9LWQvAyN10=
github.com/konveyor/tackle2-hub v0.5.0-rc.1.0.20240726125502-8bb3c0911660/go.mod h1:5c5A3i/oARdUp1yo+iYJFFvsIlsbsVGBZT7/CQji9YY=
github.com/konveyor/tackle2-hub v0.5.1-0.20240926152344-e15a8a4fbf23 h1:EnSIrmEte86RvQx8/QsCnm/Wo/F+z0NMowTdoZJpUhc=
github.com/konveyor/tackle2-hub v0.5.1-0.20240926152344-e15a8a4fbf23/go.mod h1:PeqkGgjIbCjaK/zudHGcBG4jB3Wbyi+lAIDLUjgjbMI=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
Expand All @@ -160,6 +163,8 @@ github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6
github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
Expand Down

0 comments on commit 63cc40a

Please sign in to comment.