From 32f4c5fcc777e6ee7b8692c0538dff5ece0bfc2c Mon Sep 17 00:00:00 2001 From: Jeff Ortel Date: Sun, 11 Aug 2024 17:51:09 -0700 Subject: [PATCH] manifest file with markers. Signed-off-by: Jeff Ortel --- api/analysis.go | 197 ++++++++++++++++++++++++++++++++--------- binding/application.go | 17 +--- hack/add/analysis.sh | 63 +++++-------- 3 files changed, 182 insertions(+), 95 deletions(-) diff --git a/api/analysis.go b/api/analysis.go index 733e1f18..d7cac7e3 100644 --- a/api/analysis.go +++ b/api/analysis.go @@ -1,12 +1,15 @@ package api import ( + "bufio" "bytes" "encoding/json" "errors" + "fmt" "io" "net/http" "os" + "regexp" "sort" "strconv" "strings" @@ -315,7 +318,7 @@ func (h AnalysisHandler) AppList(ctx *gin.Context) { // @description - dependencies: file that multiple api.TechDependency resources. // @tags analyses // @produce json -// @success 201 {object} api.AnalysisManifest +// @success 201 {object} api.Ref // @router /application/{id}/analyses [post] // @param id path int true "Application ID" func (h AnalysisHandler) AppCreate(ctx *gin.Context) { @@ -332,43 +335,34 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { return } } - // - // Analysis - manifest := &AnalysisManifest{} - err := h.Bind(ctx, manifest) - if err != nil { - _ = ctx.Error(err) - return - } - r := Analysis{} - r.Commit = manifest.Commit - analysis := r.Model() - analysis.ApplicationID = id - analysis.CreateUser = h.BaseHandler.CurrentUser(ctx) db := h.DB(ctx) - db.Logger = db.Logger.LogMode(logger.Error) - err = db.Create(analysis).Error + // + // Manifest + ref := &Ref{} + err := h.Bind(ctx, ref) if err != nil { _ = ctx.Error(err) return } - // - // Issues file := &model.File{} - err = db.First(file, manifest.Issues.ID).Error + err = db.First(file, ref.ID).Error if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } - reader, err := os.Open(file.Path) + reader := &AnalysisReader{} + f, err := reader.open( + file.Path, + "___BEGIN-ANALYSIS___", + "___END-ANALYSIS___") if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() d, err := h.Decoder(ctx, file.Encoding, reader) if err != nil { @@ -376,6 +370,37 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { _ = ctx.Error(err) return } + r := &Analysis{} + err = d.Decode(r) + if err != nil { + err = &BadRequestError{err.Error()} + _ = ctx.Error(err) + return + } + analysis := r.Model() + analysis.ApplicationID = id + analysis.CreateUser = h.BaseHandler.CurrentUser(ctx) + db.Logger = db.Logger.LogMode(logger.Error) + err = db.Create(analysis).Error + if err != nil { + _ = ctx.Error(err) + return + } + // + // Issues + reader = &AnalysisReader{} + f, err = reader.open( + file.Path, + "___BEGIN-ISSUES___", + "___END-ISSUES___") + if err != nil { + err = &BadRequestError{err.Error()} + _ = ctx.Error(err) + return + } + defer func() { + _ = f.Close() + }() for { r := &Issue{} err = d.Decode(r) @@ -399,28 +424,19 @@ func (h AnalysisHandler) AppCreate(ctx *gin.Context) { } // // Dependencies - file = &model.File{} - err = db.First(file, manifest.Dependencies.ID).Error - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } - reader, err = os.Open(file.Path) + reader = &AnalysisReader{} + f, err = reader.open( + file.Path, + "___BEGIN-DEPENDENCIES___", + "___END-DEPENDENCIES___") if err != nil { err = &BadRequestError{err.Error()} _ = ctx.Error(err) return } defer func() { - _ = reader.Close() + _ = f.Close() }() - d, err = h.Decoder(ctx, file.Encoding, reader) - if err != nil { - err = &BadRequestError{err.Error()} - _ = ctx.Error(err) - return - } var deps []*TechDependency for { r := &TechDependency{} @@ -2345,13 +2361,6 @@ type DepAppReport struct { } `json:"dependency"` } -// AnalysisManifest resource. -type AnalysisManifest struct { - Commit string `json:"commit,omitempty" yaml:",omitempty"` - Issues Ref `json:"issues"` - Dependencies Ref `json:"dependencies"` -} - // FactMap map. type FactMap map[string]any @@ -2858,3 +2867,105 @@ func (r *yamlEncoder) embed(object any) encoder { r.write(s) return r } + +// AnalysisReader manifest reader. +type AnalysisReader struct { + file *os.File + marker map[string]int64 + begin int64 + end int64 + read int64 +} + +// scan manifest and catalog position of markers. +func (r *AnalysisReader) scan(path string) (err error) { + if r.marker != nil { + return + } + r.file, err = os.Open(path) + if err != nil { + return + } + defer func() { + _ = r.file.Close() + }() + pattern, err := regexp.Compile(`^___[A-Z-]+___$`) + if err != nil { + return + } + p := int64(0) + r.marker = make(map[string]int64) + scanner := bufio.NewScanner(r.file) + for scanner.Scan() { + content := scanner.Text() + content = strings.TrimSpace(content) + if pattern.Match([]byte(content)) { + r.marker[content] = p + } + p += int64(len(content)) + p++ + } + + return +} + +// open returns a read delimited by the specified markers. +func (r *AnalysisReader) open(path, begin, end string) (reader io.ReadCloser, err error) { + found := false + err = r.scan(path) + if err != nil { + return + } + r.begin, found = r.marker[begin] + if !found { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s not found.", begin), + } + return + } + r.end, found = r.marker[end] + if !found { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s not found.", end), + } + return + } + if r.begin >= r.end { + err = &BadRequestError{ + Reason: fmt.Sprintf("marker: %s must preceed %s.", begin, end), + } + return + } + r.begin = r.begin + int64(len(begin)) + r.begin += 1 + r.read = r.end - r.begin + r.file, err = os.Open(path) + if err != nil { + return + } + _, err = r.file.Seek(r.begin, io.SeekStart) + reader = r + return +} + +// Read bytes. +func (r *AnalysisReader) Read(b []byte) (n int, err error) { + n, err = r.file.Read(b) + if n == 0 || err != nil { + return + } + if int64(n) > r.read { + n = int(r.read) + } + r.read -= int64(n) + if n < 1 { + err = io.EOF + } + return +} + +// Close the reader. +func (r *AnalysisReader) Close() (err error) { + err = r.file.Close() + return +} diff --git a/binding/application.go b/binding/application.go index c65ae29a..be8a5abb 100644 --- a/binding/application.go +++ b/binding/application.go @@ -320,7 +320,7 @@ type Analysis struct { // as individual documents. // The `deps` is the path to a file containing api.TechDependency(s) // as individual documents. -func (h *Analysis) Create(commit, encoding, issues, deps string) (err error) { +func (h *Analysis) Create(manifest, encoding string) (err error) { switch encoding { case "": encoding = binding.MIMEJSON @@ -331,26 +331,17 @@ func (h *Analysis) Create(commit, encoding, issues, deps string) (err error) { "Encoding: %s not supported", encoding) } - r := &api.AnalysisManifest{Commit: commit} file := File{client: h.client} - f, err := file.PostEncoded(issues, encoding) + f, err := file.PostEncoded(manifest, encoding) if err != nil { return } - r.Issues = api.Ref{ID: f.ID} - defer func() { - _ = file.Delete(f.ID) - }() - f, err = file.PostEncoded(deps, encoding) - if err != nil { - return - } - r.Dependencies = api.Ref{ID: f.ID} + ref := &api.Ref{ID: f.ID} defer func() { _ = file.Delete(f.ID) }() path := Path(api.AppAnalysesRoot).Inject(Params{api.ID: h.appId}) - err = h.client.Post(path, r) + err = h.client.Post(path, ref) if err != nil { return } diff --git a/hack/add/analysis.sh b/hack/add/analysis.sh index a842e9ff..cd48448b 100755 --- a/hack/add/analysis.sh +++ b/hack/add/analysis.sh @@ -8,15 +8,13 @@ nRuleSet="${2:-10}" nIssue="${3:-10}" nIncident="${4:-25}" tmp=/tmp/${self}-${pid} -iPath="/tmp/issues.yaml" -dPath="/tmp/deps.yaml" +file="/tmp/manifest.yaml" echo " Application: ${appId}" echo " RuleSets: ${nRuleSet}" echo " Issues: ${nIssue}" echo " Incidents: ${nIncident}" -echo " Issues path: ${iPath}" -echo " Deps path: ${dPath}" +echo " Manifest path: ${file}" sources=( konveyor.io/source=oraclejdk @@ -62,11 +60,18 @@ konveyor.io/target=hibernate konveyor.io/target=jbpm ) +# +# Analysis +# +echo "___BEGIN-ANALYSIS___" > ${file} +echo "--- +commit: "1234" +" >> ${file} +echo "___END-ANALYSIS___" >> ${file} # # Issues # -file=${iPath} -echo "" > ${file} +echo "___BEGIN-ISSUES___" >> ${file} for r in $(seq 1 ${nRuleSet}) do for i in $(seq 1 ${nIssue}) @@ -150,17 +155,18 @@ fi done done done +echo "___END-ISSUES___ +___BEGIN-DEPENDENCIES___" >> ${file} # # Deps # -file=${dPath} echo -n "--- name: github.com/jboss version: 4.0 labels: - konveyor.io/language=java - konveyor.io/otherA=dog -" > ${file} +" >> ${file} echo -n "--- name: github.com/jboss version: 5.0 @@ -192,51 +198,31 @@ echo -n "--- name: github.com/java version: 8 " >> ${file} +echo "___END-DEPENDENCIES___" >> ${file} -echo "Report (files) GENERATED" +echo "Manifest (file) GENERATED: ${file}" # -# Post issues. -code=$(curl -kSs -o ${tmp} -w "%{http_code}" -F "file=@${iPath};type=application/x-yaml" http://${host}/files/issues) -if [ ! $? -eq 0 ] -then - exit $? -fi -case ${code} in - 201) - issueId=$(cat ${tmp}|jq .id) - echo "issues file: ${name} created. id=${issueId}" - ;; - *) - echo "create issues file - FAILED: ${code}." - cat ${tmp} - exit 1 -esac -# -# Post deps. -code=$(curl -kSs -o ${tmp} -w "%{http_code}" -F "file=@${dPath};application/x-yaml" http://${host}/files/deps) +# Post manifest. +code=$(curl -kSs -o ${tmp} -w "%{http_code}" -F "file=@${file};type=application/x-yaml" http://${host}/files/manifest) if [ ! $? -eq 0 ] then exit $? fi case ${code} in 201) - depId=$(cat ${tmp}|jq .id) - echo "deps file: ${name} created. id=${depId}" + manifestId=$(cat ${tmp}|jq .id) + echo "manifest (file): ${name} posted. id=${manifestId}" ;; *) - echo "create deps file - FAILED: ${code}." + echo "manifest (file) post - FAILED: ${code}." cat ${tmp} exit 1 esac # # Post analysis. d=" -commit: "42b22a90" -issues: - id: ${issueId} -dependencies: - id: ${depId} +id: ${manifestId} " code=$(curl -kSs -o ${tmp} -w "%{http_code}" ${host}/applications/${appId}/analyses -H "Content-Type:application/x-yaml" -d "${d}") if [ ! $? -eq 0 ] @@ -246,12 +232,11 @@ fi case ${code} in 201) id=$(cat ${tmp}|jq .id) - echo "analysis: ${name} created. id=${id}" + echo "analysis: ${name} posted. id=${id}" ;; *) - echo "create analysis - FAILED: ${code}." + echo "analysis post - FAILED: ${code}." cat ${tmp} exit 1 esac -echo "Report POSTED"