Skip to content

Commit

Permalink
Merge pull request #233 from anchore/javascript_parser_fix_author
Browse files Browse the repository at this point in the history 
Javascript parser fix author
  • Loading branch information
Toure Dunnon authored Oct 26, 2020
2 parents 15e2e32 + 7c42a74 commit 076454d
Show file tree
Hide file tree
Showing 8 changed files with 303 additions and 90 deletions.
15 changes: 15 additions & 0 deletions internal/parse.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package internal

import "regexp"

// MatchCaptureGroups takes a regular expression and string and returns all of the named capture group results in a map.
func MatchCaptureGroups(regEx *regexp.Regexp, str string) map[string]string {
match := regEx.FindStringSubmatch(str)
results := make(map[string]string)
for i, name := range regEx.SubexpNames() {
if i > 0 && i <= len(match) {
results[name] = match[i]
}
}
return results
}
81 changes: 37 additions & 44 deletions schema/json/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,62 +53,55 @@
"type": "integer"
},
"files": {
"anyOf": [
{
"type": "null"
},
{
"items": {
"anyOf": [
{
"items": {
"anyOf": [
{
"type": "string"
},
{
"properties": {
"checksum": {
"type": "string"
},
{
"digest": {
"properties": {
"checksum": {
"type": "string"
},
"digest": {
"properties": {
"algorithm": {
"type": "string"
},
"value": {
"type": "string"
}
},
"required": [
"algorithm",
"value"
],
"type": "object"
},
"ownerGid": {
"type": "string"
},
"ownerUid": {
"type": "string"
},
"path": {
"type": "string"
},
"permissions": {
"algorithm": {
"type": "string"
},
"size": {
"value": {
"type": "string"
}
},
"required": [
"path"
"algorithm",
"value"
],
"type": "object"
},
"ownerGid": {
"type": "string"
},
"ownerUid": {
"type": "string"
},
"path": {
"type": "string"
},
"permissions": {
"type": "string"
},
"size": {
"type": "string"
}
]
},
"type": "array"
}
]
},
"required": [
"path"
],
"type": "object"
}
]
},
"type": "array"
},
"gitCommitOfApkPort": {
"type": "string"
Expand Down
89 changes: 87 additions & 2 deletions syft/cataloger/javascript/parse_package_json.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ import (
"encoding/json"
"fmt"
"io"
"regexp"

"github.com/anchore/syft/internal"

"github.com/mitchellh/mapstructure"

"github.com/anchore/syft/syft/cataloger/common"
"github.com/anchore/syft/syft/pkg"
Expand All @@ -16,12 +21,90 @@ var _ common.ParserFn = parsePackageLock
type PackageJSON struct {
Version string `json:"version"`
Latest []string `json:"latest"`
Author string `json:"author"`
Author Author `json:"author"`
License string `json:"license"`
Name string `json:"name"`
Homepage string `json:"homepage"`
Description string `json:"description"`
Dependencies map[string]string `json:"dependencies"`
Repository Repository `json:"repository"`
}

type Author struct {
Name string `json:"name" mapstruct:"name"`
Email string `json:"email" mapstruct:"email"`
URL string `json:"url" mapstruct:"url"`
}

type Repository struct {
Type string `json:"type" mapstructure:"type"`
URL string `json:"url" mapstructure:"url"`
}

// match example: "author": "Isaac Z. Schlueter <[email protected]> (http://blog.izs.me)"
// ---> name: "Isaac Z. Schlueter" email: "[email protected]" url: "http://blog.izs.me"
var authorPattern = regexp.MustCompile(`^\s*(?P<name>[^<(]*)(\s+<(?P<email>.*)>)?(\s\((?P<url>.*)\))?\s*$`)

// Exports Author.UnmarshalJSON interface to help normalize the json structure.
func (a *Author) UnmarshalJSON(b []byte) error {
var authorStr string
var fields map[string]string
var author Author

if err := json.Unmarshal(b, &authorStr); err != nil {
// string parsing did not work, assume a map was given
// for more information: https://docs.npmjs.com/files/package.json#people-fields-author-contributors
if err := json.Unmarshal(b, &fields); err != nil {
return fmt.Errorf("unable to parse package.json author: %w", err)
}
} else {
// parse out "name <email> (url)" into an Author struct
fields = internal.MatchCaptureGroups(authorPattern, authorStr)
}

// translate the map into a structure
if err := mapstructure.Decode(fields, &author); err != nil {
return fmt.Errorf("unable to decode package.json author: %w", err)
}

*a = author

return nil
}

func (a *Author) AuthorString() string {
result := a.Name
if a.Email != "" {
result += fmt.Sprintf(" <%s>", a.Email)
}
if a.URL != "" {
result += fmt.Sprintf(" (%s)", a.URL)
}
return result
}

func (r *Repository) UnmarshalJSON(b []byte) error {
var repositoryStr string
var fields map[string]string
var repository Repository

if err := json.Unmarshal(b, &repositoryStr); err != nil {
// string parsing did not work, assume a map was given
// for more information: https://docs.npmjs.com/files/package.json#people-fields-author-contributors
if err := json.Unmarshal(b, &fields); err != nil {
return fmt.Errorf("unable to parse package.json author: %w", err)
}
// translate the map into a structure
if err := mapstructure.Decode(fields, &repository); err != nil {
return fmt.Errorf("unable to decode package.json author: %w", err)
}

*r = repository
} else {
r.URL = repositoryStr
}

return nil
}

// parsePackageJson parses a package.json and returns the discovered JavaScript packages.
Expand All @@ -45,8 +128,10 @@ func parsePackageJSON(_ string, reader io.Reader) ([]pkg.Package, error) {
Type: pkg.NpmPkg,
MetadataType: pkg.NpmPackageJSONMetadataType,
Metadata: pkg.NpmPackageJSONMetadata{
Author: p.Author,
Author: p.Author.AuthorString(),
Homepage: p.Homepage,
URL: p.Repository.URL,
Licenses: []string{p.License},
},
})
}
Expand Down
103 changes: 76 additions & 27 deletions syft/cataloger/javascript/parse_package_json_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,36 +9,85 @@ import (
)

func TestParsePackageJSON(t *testing.T) {
expected := pkg.Package{
Name: "npm",
Version: "6.14.6",
Type: pkg.NpmPkg,
Licenses: []string{"Artistic-2.0"},
Language: pkg.JavaScript,
MetadataType: pkg.NpmPackageJSONMetadataType,
Metadata: pkg.NpmPackageJSONMetadata{
Author: "Isaac Z. Schlueter <[email protected]> (http://blog.izs.me)",
Homepage: "https://docs.npmjs.com/",
tests := []struct {
Fixture string
ExpectedPkg pkg.Package
}{
{
Fixture: "test-fixtures/pkg-json/package.json",
ExpectedPkg: pkg.Package{
Name: "npm",
Version: "6.14.6",
Type: pkg.NpmPkg,
Licenses: []string{"Artistic-2.0"},
Language: pkg.JavaScript,
MetadataType: pkg.NpmPackageJSONMetadataType,
Metadata: pkg.NpmPackageJSONMetadata{
Author: "Isaac Z. Schlueter <[email protected]> (http://blog.izs.me)",
Homepage: "https://docs.npmjs.com/",
URL: "https://github.com/npm/cli",
Licenses: []string{"Artistic-2.0"},
},
},
},
{
Fixture: "test-fixtures/pkg-json/package-nested-author.json",
ExpectedPkg: pkg.Package{
Name: "npm",
Version: "6.14.6",
Type: pkg.NpmPkg,
Licenses: []string{"Artistic-2.0"},
Language: pkg.JavaScript,
MetadataType: pkg.NpmPackageJSONMetadataType,
Metadata: pkg.NpmPackageJSONMetadata{
Author: "Isaac Z. Schlueter <[email protected]> (http://blog.izs.me)",
Homepage: "https://docs.npmjs.com/",
URL: "https://github.com/npm/cli",
Licenses: []string{"Artistic-2.0"},
},
},
},
{
Fixture: "test-fixtures/pkg-json/package-repo-string.json",
ExpectedPkg: pkg.Package{
Name: "function-bind",
Version: "1.1.1",
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
Language: pkg.JavaScript,
MetadataType: pkg.NpmPackageJSONMetadataType,
Metadata: pkg.NpmPackageJSONMetadata{
Author: "Raynos <[email protected]>",
Homepage: "https://github.com/Raynos/function-bind",
URL: "git://github.com/Raynos/function-bind.git",
Licenses: []string{"MIT"},
},
},
},
}
fixture, err := os.Open("test-fixtures/pkg-json/package.json")
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}

actual, err := parsePackageJSON(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
if len(actual) != 1 {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=1", len(actual))
}
for _, test := range tests {
t.Run(test.Fixture, func(t *testing.T) {
fixture, err := os.Open(test.Fixture)
if err != nil {
t.Fatalf("failed to open fixture: %+v", err)
}

for _, d := range deep.Equal(actual[0], expected) {
t.Errorf("diff: %+v", d)
}
actual, err := parsePackageJSON(fixture.Name(), fixture)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}
if len(actual) != 1 {
for _, a := range actual {
t.Log(" ", a)
}
t.Fatalf("unexpected package count: %d!=1", len(actual))
}

for _, d := range deep.Equal(actual[0], test.ExpectedPkg) {

t.Errorf("diff: %+v", d)
}
})
}
}
16 changes: 16 additions & 0 deletions syft/cataloger/javascript/test-fixtures/pkg-json/package-nested-author.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"version": "6.14.6",
"name": "npm",
"description": "a package manager for JavaScript",
"homepage": "https://docs.npmjs.com/",
"author": {
"name": "Isaac Z. Schlueter",
"email": "[email protected]",
"url": "http://blog.izs.me"
},
"repository": {
"type": "git",
"url": "https://github.com/npm/cli"
},
"license": "Artistic-2.0"
}
Loading

0 comments on commit 076454d

Please sign in to comment.