refactor: unify Library and Package structs (#6633)

Signed-off-by: knqyf263 <knqyf263@gmail.com>
Co-authored-by: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com>
Co-authored-by: DmitriyLewen <dmitriy.lewen@smartforce.io>
This commit is contained in:
Teppei Fukuda
2024-05-07 16:25:52 +04:00
committed by GitHub
parent 39ebed45f8
commit 3eecfc6b6e
156 changed files with 3901 additions and 3675 deletions

View File

@@ -7,9 +7,10 @@ import (
"net/textproto"
"strings"
"github.com/samber/lo"
"golang.org/x/xerrors"
"github.com/aquasecurity/trivy/pkg/dependency/types"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/log"
xio "github.com/aquasecurity/trivy/pkg/x/io"
)
@@ -18,7 +19,7 @@ type Parser struct {
logger *log.Logger
}
func NewParser() types.Parser {
func NewParser() *Parser {
return &Parser{
logger: log.WithPrefix("python"),
}
@@ -26,7 +27,7 @@ func NewParser() types.Parser {
// Parse parses egg and wheel metadata.
// e.g. .egg-info/PKG-INFO and dist-info/METADATA
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) {
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) {
rd := textproto.NewReader(bufio.NewReader(r))
h, err := rd.ReadMIMEHeader()
if e := textproto.ProtocolError(""); errors.As(err, &e) {
@@ -82,11 +83,11 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency,
license = "file://" + h.Get("License-File")
}
return []types.Library{
return []ftypes.Package{
{
Name: name,
Version: version,
License: license,
Name: name,
Version: version,
Licenses: lo.Ternary(license != "", []string{license}, nil),
},
}, nil, nil
}

View File

@@ -8,14 +8,14 @@ import (
"github.com/stretchr/testify/require"
"github.com/aquasecurity/trivy/pkg/dependency/parser/python/packaging"
"github.com/aquasecurity/trivy/pkg/dependency/types"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
)
func TestParse(t *testing.T) {
tests := []struct {
name string
input string
want []types.Library
want []ftypes.Package
wantErr bool
}{
// listing dependencies based on METADATA/PKG-INFO files
@@ -33,16 +33,22 @@ func TestParse(t *testing.T) {
// cd /usr/lib/python3.9/site-packages/setuptools-52.0.0-py3.9.egg-info/
// cat PKG-INFO | grep -e "^Name:" -e "^Version:" -e "^License:" | cut -d" " -f2- | \
// tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\", \""$2"\", \""$3"\"\}\n")}'
want: []types.Library{{Name: "setuptools", Version: "51.3.3", License: "UNKNOWN"}},
want: []ftypes.Package{
{
Name: "setuptools",
Version: "51.3.3",
Licenses: []string{"UNKNOWN"},
},
},
},
{
name: "egg PKG-INFO with description containing non-RFC 7230 bytes",
input: "testdata/unidecode-egg-info.PKG-INFO",
want: []types.Library{
want: []ftypes.Package{
{
Name: "Unidecode",
Version: "0.4.1",
License: "UNKNOWN",
Name: "Unidecode",
Version: "0.4.1",
Licenses: []string{"UNKNOWN"},
},
},
},
@@ -55,7 +61,13 @@ func TestParse(t *testing.T) {
// cd /usr/lib/python3.9/site-packages/
// cat distlib-0.3.1-py3.9.egg-info | grep -e "^Name:" -e "^Version:" -e "^License:" | cut -d" " -f2- | \
// tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\", \""$2"\", \""$3"\"\}\n")}'
want: []types.Library{{Name: "distlib", Version: "0.3.1", License: "Python license"}},
want: []ftypes.Package{
{
Name: "distlib",
Version: "0.3.1",
Licenses: []string{"Python license"},
},
},
},
{
name: "wheel METADATA",
@@ -67,31 +79,53 @@ func TestParse(t *testing.T) {
// find dist-infos/ | grep -v METADATA | xargs rm -R
// for single METADATA file with known name
// cat "{{ libname }}.METADATA | grep -e "^Name:" -e "^Version:" -e "^License:" | cut -d" " -f2- | tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\", \""$2"\", \""$3"\"\}\n")}'
want: []types.Library{{Name: "simple", Version: "0.1.0", License: ""}},
// cat "{{ libname }}.METADATA | grep -e "^Name:" -e "^Version:" -e "^Licenses: []string{" | cut -d" " -f2- | tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\"}, \""$2"\", \""$3"\"\}\n")}'
want: []ftypes.Package{
{
Name: "simple",
Version: "0.1.0",
Licenses: nil,
},
},
},
{
name: "wheel METADATA",
// for single METADATA file with known name
// cat "{{ libname }}.METADATA | grep -e "^Name:" -e "^Version:" -e "^License:" | cut -d" " -f2- | tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\", \""$2"\", \""$3"\"\}\n")}'
// cat "{{ libname }}.METADATA | grep -e "^Name:" -e "^Version:" -e "^Licenses: []string{" | cut -d" " -f2- | tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\"}, \""$2"\", \""$3"\"\}\n")}'
input: "testdata/distlib-0.3.1.METADATA",
want: []types.Library{{Name: "distlib", Version: "0.3.1", License: "Python Software Foundation License"}},
want: []ftypes.Package{
{
Name: "distlib",
Version: "0.3.1",
Licenses: []string{"Python Software Foundation License"},
},
},
},
{
name: "wheel METADATA",
// Input defines "Classifier: License" but it ends at "OSI Approved" which doesn't define any specific license, thus "License" field is added to results
input: "testdata/asyncssh-2.14.2.METADATA",
want: []types.Library{{Name: "asyncssh", Version: "2.14.2", License: "Eclipse Public License v2.0"}},
want: []ftypes.Package{
{
Name: "asyncssh",
Version: "2.14.2",
Licenses: []string{"Eclipse Public License v2.0"},
},
},
},
{
name: "wheel METADATA",
// Input defines multiple "Classifier: License"
input: "testdata/pyphen-0.14.0.METADATA",
want: []types.Library{
{Name: "pyphen", Version: "0.14.0", License: "GNU General Public License v2 or later (GPLv2+), GNU Lesser General Public License v2 or later (LGPLv2+), Mozilla Public License 1.1 (MPL 1.1)"},
want: []ftypes.Package{
{
Name: "pyphen",
Version: "0.14.0",
Licenses: []string{"GNU General Public License v2 or later (GPLv2+), GNU Lesser General Public License v2 or later (LGPLv2+), Mozilla Public License 1.1 (MPL 1.1)"},
},
},
},
{
@@ -102,33 +136,33 @@ func TestParse(t *testing.T) {
{
name: "with License-Expression field",
input: "testdata/iniconfig-2.0.0.METADATA",
want: []types.Library{
want: []ftypes.Package{
{
Name: "iniconfig",
Version: "2.0.0",
License: "MIT",
Name: "iniconfig",
Version: "2.0.0",
Licenses: []string{"MIT"},
},
},
},
{
name: "with an empty license field but with license in Classifier",
input: "testdata/zipp-3.12.1.METADATA",
want: []types.Library{
want: []ftypes.Package{
{
Name: "zipp",
Version: "3.12.1",
License: "MIT License",
Name: "zipp",
Version: "3.12.1",
Licenses: []string{"MIT License"},
},
},
},
{
name: "without licenses, but with a license file (a license in Classifier was removed)",
input: "testdata/networkx-3.0.METADATA",
want: []types.Library{
want: []ftypes.Package{
{
Name: "networkx",
Version: "3.0",
License: "file://LICENSE.txt",
Name: "networkx",
Version: "3.0",
Licenses: []string{"file://LICENSE.txt"},
},
},
},

View File

@@ -10,7 +10,7 @@ import (
"golang.org/x/text/transform"
"golang.org/x/xerrors"
"github.com/aquasecurity/trivy/pkg/dependency/types"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
xio "github.com/aquasecurity/trivy/pkg/x/io"
)
@@ -24,11 +24,11 @@ const (
type Parser struct{}
func NewParser() types.Parser {
func NewParser() *Parser {
return &Parser{}
}
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) {
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) {
// `requirements.txt` can use byte order marks (BOM)
// e.g. on Windows `requirements.txt` can use UTF-16LE with BOM
// We need to override them to avoid the file being read incorrectly
@@ -36,7 +36,7 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency,
decodedReader := transform.NewReader(r, transformer)
scanner := bufio.NewScanner(decodedReader)
var libs []types.Library
var pkgs []ftypes.Package
for scanner.Scan() {
line := scanner.Text()
line = strings.ReplaceAll(line, " ", "")
@@ -49,7 +49,7 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency,
if len(s) != 2 {
continue
}
libs = append(libs, types.Library{
pkgs = append(pkgs, ftypes.Package{
Name: s[0],
Version: s[1],
})
@@ -57,7 +57,7 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency,
if err := scanner.Err(); err != nil {
return nil, nil, xerrors.Errorf("scan error: %w", err)
}
return libs, nil, nil
return pkgs, nil, nil
}
func rStripByKey(line, key string) string {

View File

@@ -8,13 +8,13 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/aquasecurity/trivy/pkg/dependency/types"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
)
func TestParse(t *testing.T) {
vectors := []struct {
file string
want []types.Library
want []ftypes.Package
}{
{
file: "testdata/requirements_flask.txt",

View File

@@ -1,9 +1,9 @@
package pip
import "github.com/aquasecurity/trivy/pkg/dependency/types"
import ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
var (
requirementsFlask = []types.Library{
requirementsFlask = []ftypes.Package{
{Name: "click", Version: "8.0.0"},
{Name: "Flask", Version: "2.0.0"},
{Name: "itsdangerous", Version: "2.0.0"},
@@ -12,45 +12,45 @@ var (
{Name: "Werkzeug", Version: "2.0.0"},
}
requirementsComments = []types.Library{
requirementsComments = []ftypes.Package{
{Name: "click", Version: "8.0.0"},
{Name: "Flask", Version: "2.0.0"},
{Name: "Jinja2", Version: "3.0.0"},
{Name: "MarkupSafe", Version: "2.0.0"},
}
requirementsSpaces = []types.Library{
requirementsSpaces = []ftypes.Package{
{Name: "click", Version: "8.0.0"},
{Name: "Flask", Version: "2.0.0"},
{Name: "itsdangerous", Version: "2.0.0"},
{Name: "Jinja2", Version: "3.0.0"},
}
requirementsNoVersion = []types.Library{
requirementsNoVersion = []ftypes.Package{
{Name: "Flask", Version: "2.0.0"},
}
requirementsOperator = []types.Library{
requirementsOperator = []ftypes.Package{
{Name: "Django", Version: "2.3.4"},
{Name: "SomeProject", Version: "5.4"},
}
requirementsHash = []types.Library{
requirementsHash = []ftypes.Package{
{Name: "FooProject", Version: "1.2"},
{Name: "Jinja2", Version: "3.0.0"},
}
requirementsHyphens = []types.Library{
requirementsHyphens = []ftypes.Package{
{Name: "oauth2-client", Version: "4.0.0"},
{Name: "python-gitlab", Version: "2.0.0"},
}
requirementsExtras = []types.Library{
requirementsExtras = []ftypes.Package{
{Name: "pyjwt", Version: "2.1.0"},
{Name: "celery", Version: "4.4.7"},
}
requirementsUtf16le = []types.Library{
requirementsUtf16le = []ftypes.Package{
{Name: "attrs", Version: "20.3.0"},
}
)

View File

@@ -7,7 +7,7 @@ import (
"github.com/liamg/jfather"
"golang.org/x/xerrors"
"github.com/aquasecurity/trivy/pkg/dependency/types"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
xio "github.com/aquasecurity/trivy/pkg/x/io"
)
@@ -22,11 +22,11 @@ type dependency struct {
type Parser struct{}
func NewParser() types.Parser {
func NewParser() *Parser {
return &Parser{}
}
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) {
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) {
var lockFile lockFile
input, err := io.ReadAll(r)
if err != nil {
@@ -36,20 +36,20 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency,
return nil, nil, xerrors.Errorf("failed to decode Pipenv.lock: %w", err)
}
var libs []types.Library
for pkgName, dependency := range lockFile.Default {
libs = append(libs, types.Library{
var pkgs []ftypes.Package
for pkgName, dep := range lockFile.Default {
pkgs = append(pkgs, ftypes.Package{
Name: pkgName,
Version: strings.TrimLeft(dependency.Version, "="),
Locations: []types.Location{
Version: strings.TrimLeft(dep.Version, "="),
Locations: []ftypes.Location{
{
StartLine: dependency.StartLine,
EndLine: dependency.EndLine,
StartLine: dep.StartLine,
EndLine: dep.EndLine,
},
},
})
}
return libs, nil, nil
return pkgs, nil, nil
}
// UnmarshalJSONWithMetadata needed to detect start and end lines of deps

View File

@@ -4,19 +4,18 @@ import (
"os"
"path"
"sort"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/aquasecurity/trivy/pkg/dependency/types"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
)
func TestParse(t *testing.T) {
vectors := []struct {
file string // Test input file
want []types.Library
want []ftypes.Package
}{
{
file: "testdata/Pipfile_normal.lock",
@@ -40,21 +39,8 @@ func TestParse(t *testing.T) {
got, _, err := NewParser().Parse(f)
require.NoError(t, err)
sort.Slice(got, func(i, j int) bool {
ret := strings.Compare(got[i].Name, got[j].Name)
if ret == 0 {
return got[i].Version < got[j].Version
}
return ret < 0
})
sort.Slice(v.want, func(i, j int) bool {
ret := strings.Compare(v.want[i].Name, v.want[j].Name)
if ret == 0 {
return v.want[i].Version < v.want[j].Version
}
return ret < 0
})
sort.Sort(ftypes.Packages(got))
sort.Sort(ftypes.Packages(v.want))
assert.Equal(t, v.want, got)
})

View File

@@ -1,6 +1,6 @@
package pipenv
import "github.com/aquasecurity/trivy/pkg/dependency/types"
import ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
var (
// docker run --name pipenv --rm -it python:3.9-alpine sh
@@ -11,13 +11,13 @@ var (
// pipenv graph --json | jq -rc '.[] | "{\"\(.package.package_name | ascii_downcase)\", \"\(.package.installed_version)\", \"\"},"'
// graph doesn't contain information about location of dependency in lock file.
// add locations manually
pipenvNormal = []types.Library{
{Name: "urllib3", Version: "1.24.2", Locations: []types.Location{{StartLine: 65, EndLine: 71}}},
{Name: "requests", Version: "2.21.0", Locations: []types.Location{{StartLine: 57, EndLine: 64}}},
{Name: "pyyaml", Version: "5.1", Locations: []types.Location{{StartLine: 40, EndLine: 56}}},
{Name: "idna", Version: "2.8", Locations: []types.Location{{StartLine: 33, EndLine: 39}}},
{Name: "chardet", Version: "3.0.4", Locations: []types.Location{{StartLine: 26, EndLine: 32}}},
{Name: "certifi", Version: "2019.3.9", Locations: []types.Location{{StartLine: 19, EndLine: 25}}},
pipenvNormal = []ftypes.Package{
{Name: "urllib3", Version: "1.24.2", Locations: []ftypes.Location{{StartLine: 65, EndLine: 71}}},
{Name: "requests", Version: "2.21.0", Locations: []ftypes.Location{{StartLine: 57, EndLine: 64}}},
{Name: "pyyaml", Version: "5.1", Locations: []ftypes.Location{{StartLine: 40, EndLine: 56}}},
{Name: "idna", Version: "2.8", Locations: []ftypes.Location{{StartLine: 33, EndLine: 39}}},
{Name: "chardet", Version: "3.0.4", Locations: []ftypes.Location{{StartLine: 26, EndLine: 32}}},
{Name: "certifi", Version: "2019.3.9", Locations: []ftypes.Location{{StartLine: 19, EndLine: 25}}},
}
// docker run --name pipenv --rm -it python:3.9-alpine bash
@@ -28,17 +28,17 @@ var (
// pipenv graph --json | jq -rc '.[] | "{\"\(.package.package_name | ascii_downcase)\", \"\(.package.installed_version)\", \"\"},"'
// graph doesn't contain information about location of dependency in lock file.
// add locations manually
pipenvDjango = []types.Library{
{Name: "urllib3", Version: "1.24.2", Locations: []types.Location{{StartLine: 95, EndLine: 101}}},
{Name: "sqlparse", Version: "0.3.0", Locations: []types.Location{{StartLine: 88, EndLine: 94}}},
{Name: "requests", Version: "2.21.0", Locations: []types.Location{{StartLine: 80, EndLine: 87}}},
{Name: "pyyaml", Version: "5.1", Locations: []types.Location{{StartLine: 63, EndLine: 79}}},
{Name: "pytz", Version: "2019.1", Locations: []types.Location{{StartLine: 56, EndLine: 62}}},
{Name: "idna", Version: "2.8", Locations: []types.Location{{StartLine: 49, EndLine: 55}}},
{Name: "djangorestframework", Version: "3.9.3", Locations: []types.Location{{StartLine: 41, EndLine: 48}}},
{Name: "django", Version: "2.2", Locations: []types.Location{{StartLine: 33, EndLine: 40}}},
{Name: "chardet", Version: "3.0.4", Locations: []types.Location{{StartLine: 26, EndLine: 32}}},
{Name: "certifi", Version: "2019.3.9", Locations: []types.Location{{StartLine: 19, EndLine: 25}}},
pipenvDjango = []ftypes.Package{
{Name: "urllib3", Version: "1.24.2", Locations: []ftypes.Location{{StartLine: 95, EndLine: 101}}},
{Name: "sqlparse", Version: "0.3.0", Locations: []ftypes.Location{{StartLine: 88, EndLine: 94}}},
{Name: "requests", Version: "2.21.0", Locations: []ftypes.Location{{StartLine: 80, EndLine: 87}}},
{Name: "pyyaml", Version: "5.1", Locations: []ftypes.Location{{StartLine: 63, EndLine: 79}}},
{Name: "pytz", Version: "2019.1", Locations: []ftypes.Location{{StartLine: 56, EndLine: 62}}},
{Name: "idna", Version: "2.8", Locations: []ftypes.Location{{StartLine: 49, EndLine: 55}}},
{Name: "djangorestframework", Version: "3.9.3", Locations: []ftypes.Location{{StartLine: 41, EndLine: 48}}},
{Name: "django", Version: "2.2", Locations: []ftypes.Location{{StartLine: 33, EndLine: 40}}},
{Name: "chardet", Version: "3.0.4", Locations: []ftypes.Location{{StartLine: 26, EndLine: 32}}},
{Name: "certifi", Version: "2019.3.9", Locations: []ftypes.Location{{StartLine: 19, EndLine: 25}}},
}
// docker run --name pipenv --rm -it python:3.9-alpine bash
@@ -49,30 +49,30 @@ var (
// pipenv graph --json | jq -rc '.[] | "{\"\(.package.package_name | ascii_downcase)\", \"\(.package.installed_version)\", \"\"},"'
// graph doesn't contain information about location of dependency in lock file.
// add locations manually
pipenvMany = []types.Library{
{Name: "urllib3", Version: "1.24.2", Locations: []types.Location{{StartLine: 237, EndLine: 244}}},
{Name: "sqlparse", Version: "0.3.0", Locations: []types.Location{{StartLine: 230, EndLine: 236}}},
{Name: "six", Version: "1.12.0", Locations: []types.Location{{StartLine: 222, EndLine: 229}}},
{Name: "simplejson", Version: "3.16.0", Locations: []types.Location{{StartLine: 204, EndLine: 221}}},
{Name: "s3transfer", Version: "0.2.0", Locations: []types.Location{{StartLine: 197, EndLine: 203}}},
{Name: "rsa", Version: "3.4.2", Locations: []types.Location{{StartLine: 190, EndLine: 196}}},
{Name: "requests", Version: "2.21.0", Locations: []types.Location{{StartLine: 182, EndLine: 189}}},
{Name: "pyyaml", Version: "3.13", Locations: []types.Location{{StartLine: 165, EndLine: 181}}},
{Name: "pytz", Version: "2019.1", Locations: []types.Location{{StartLine: 158, EndLine: 164}}},
{Name: "python-dateutil", Version: "2.8.0", Locations: []types.Location{{StartLine: 150, EndLine: 157}}},
{Name: "pyasn1", Version: "0.4.5", Locations: []types.Location{{StartLine: 142, EndLine: 149}}},
{Name: "markupsafe", Version: "1.1.1", Locations: []types.Location{{StartLine: 109, EndLine: 141}}},
{Name: "jmespath", Version: "0.9.4", Locations: []types.Location{{StartLine: 102, EndLine: 108}}},
{Name: "jinja2", Version: "2.10.1", Locations: []types.Location{{StartLine: 94, EndLine: 101}}},
{Name: "idna", Version: "2.8", Locations: []types.Location{{StartLine: 87, EndLine: 93}}},
{Name: "framework", Version: "0.1.0", Locations: []types.Location{{StartLine: 80, EndLine: 86}}},
{Name: "docutils", Version: "0.14", Locations: []types.Location{{StartLine: 72, EndLine: 79}}},
{Name: "djangorestframework", Version: "3.9.3", Locations: []types.Location{{StartLine: 64, EndLine: 71}}},
{Name: "django", Version: "2.2", Locations: []types.Location{{StartLine: 56, EndLine: 63}}},
{Name: "colorama", Version: "0.3.9", Locations: []types.Location{{StartLine: 49, EndLine: 55}}},
{Name: "chardet", Version: "3.0.4", Locations: []types.Location{{StartLine: 42, EndLine: 48}}},
{Name: "certifi", Version: "2019.3.9", Locations: []types.Location{{StartLine: 35, EndLine: 41}}},
{Name: "botocore", Version: "1.12.137", Locations: []types.Location{{StartLine: 27, EndLine: 34}}},
{Name: "awscli", Version: "1.16.147", Locations: []types.Location{{StartLine: 19, EndLine: 26}}},
pipenvMany = []ftypes.Package{
{Name: "urllib3", Version: "1.24.2", Locations: []ftypes.Location{{StartLine: 237, EndLine: 244}}},
{Name: "sqlparse", Version: "0.3.0", Locations: []ftypes.Location{{StartLine: 230, EndLine: 236}}},
{Name: "six", Version: "1.12.0", Locations: []ftypes.Location{{StartLine: 222, EndLine: 229}}},
{Name: "simplejson", Version: "3.16.0", Locations: []ftypes.Location{{StartLine: 204, EndLine: 221}}},
{Name: "s3transfer", Version: "0.2.0", Locations: []ftypes.Location{{StartLine: 197, EndLine: 203}}},
{Name: "rsa", Version: "3.4.2", Locations: []ftypes.Location{{StartLine: 190, EndLine: 196}}},
{Name: "requests", Version: "2.21.0", Locations: []ftypes.Location{{StartLine: 182, EndLine: 189}}},
{Name: "pyyaml", Version: "3.13", Locations: []ftypes.Location{{StartLine: 165, EndLine: 181}}},
{Name: "pytz", Version: "2019.1", Locations: []ftypes.Location{{StartLine: 158, EndLine: 164}}},
{Name: "python-dateutil", Version: "2.8.0", Locations: []ftypes.Location{{StartLine: 150, EndLine: 157}}},
{Name: "pyasn1", Version: "0.4.5", Locations: []ftypes.Location{{StartLine: 142, EndLine: 149}}},
{Name: "markupsafe", Version: "1.1.1", Locations: []ftypes.Location{{StartLine: 109, EndLine: 141}}},
{Name: "jmespath", Version: "0.9.4", Locations: []ftypes.Location{{StartLine: 102, EndLine: 108}}},
{Name: "jinja2", Version: "2.10.1", Locations: []ftypes.Location{{StartLine: 94, EndLine: 101}}},
{Name: "idna", Version: "2.8", Locations: []ftypes.Location{{StartLine: 87, EndLine: 93}}},
{Name: "framework", Version: "0.1.0", Locations: []ftypes.Location{{StartLine: 80, EndLine: 86}}},
{Name: "docutils", Version: "0.14", Locations: []ftypes.Location{{StartLine: 72, EndLine: 79}}},
{Name: "djangorestframework", Version: "3.9.3", Locations: []ftypes.Location{{StartLine: 64, EndLine: 71}}},
{Name: "django", Version: "2.2", Locations: []ftypes.Location{{StartLine: 56, EndLine: 63}}},
{Name: "colorama", Version: "0.3.9", Locations: []ftypes.Location{{StartLine: 49, EndLine: 55}}},
{Name: "chardet", Version: "3.0.4", Locations: []ftypes.Location{{StartLine: 42, EndLine: 48}}},
{Name: "certifi", Version: "2019.3.9", Locations: []ftypes.Location{{StartLine: 35, EndLine: 41}}},
{Name: "botocore", Version: "1.12.137", Locations: []ftypes.Location{{StartLine: 27, EndLine: 34}}},
{Name: "awscli", Version: "1.16.147", Locations: []ftypes.Location{{StartLine: 19, EndLine: 26}}},
}
)

View File

@@ -9,7 +9,6 @@ import (
version "github.com/aquasecurity/go-pep440-version"
"github.com/aquasecurity/trivy/pkg/dependency"
"github.com/aquasecurity/trivy/pkg/dependency/types"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/aquasecurity/trivy/pkg/log"
xio "github.com/aquasecurity/trivy/pkg/x/io"
@@ -39,61 +38,61 @@ func NewParser() *Parser {
}
}
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) {
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) {
var lockfile Lockfile
if _, err := toml.NewDecoder(r).Decode(&lockfile); err != nil {
return nil, nil, xerrors.Errorf("failed to decode poetry.lock: %w", err)
}
// Keep all installed versions
libVersions := p.parseVersions(lockfile)
pkgVersions := p.parseVersions(lockfile)
var libs []types.Library
var deps []types.Dependency
var pkgs []ftypes.Package
var deps []ftypes.Dependency
for _, pkg := range lockfile.Packages {
if pkg.Category == "dev" {
continue
}
pkgID := packageID(pkg.Name, pkg.Version)
libs = append(libs, types.Library{
pkgs = append(pkgs, ftypes.Package{
ID: pkgID,
Name: pkg.Name,
Version: pkg.Version,
})
dependsOn := p.parseDependencies(pkg.Dependencies, libVersions)
dependsOn := p.parseDependencies(pkg.Dependencies, pkgVersions)
if len(dependsOn) != 0 {
deps = append(deps, types.Dependency{
deps = append(deps, ftypes.Dependency{
ID: pkgID,
DependsOn: dependsOn,
})
}
}
return libs, deps, nil
return pkgs, deps, nil
}
// parseVersions stores all installed versions of libraries for use in dependsOn
// as the dependencies of libraries use version range.
// parseVersions stores all installed versions of packages for use in dependsOn
// as the dependencies of packages use version range.
func (p *Parser) parseVersions(lockfile Lockfile) map[string][]string {
libVersions := make(map[string][]string)
pkgVersions := make(map[string][]string)
for _, pkg := range lockfile.Packages {
if pkg.Category == "dev" {
continue
}
if vers, ok := libVersions[pkg.Name]; ok {
libVersions[pkg.Name] = append(vers, pkg.Version)
if vers, ok := pkgVersions[pkg.Name]; ok {
pkgVersions[pkg.Name] = append(vers, pkg.Version)
} else {
libVersions[pkg.Name] = []string{pkg.Version}
pkgVersions[pkg.Name] = []string{pkg.Version}
}
}
return libVersions
return pkgVersions
}
func (p *Parser) parseDependencies(deps map[string]any, libVersions map[string][]string) []string {
func (p *Parser) parseDependencies(deps map[string]any, pkgVersions map[string][]string) []string {
var dependsOn []string
for name, versRange := range deps {
if dep, err := p.parseDependency(name, versRange, libVersions); err != nil {
if dep, err := p.parseDependency(name, versRange, pkgVersions); err != nil {
p.logger.Debug("Failed to parse poetry dependency", log.Err(err))
} else if dep != "" {
dependsOn = append(dependsOn, dep)
@@ -105,9 +104,9 @@ func (p *Parser) parseDependencies(deps map[string]any, libVersions map[string][
return dependsOn
}
func (p *Parser) parseDependency(name string, versRange any, libVersions map[string][]string) (string, error) {
func (p *Parser) parseDependency(name string, versRange any, pkgVersions map[string][]string) (string, error) {
name = normalizePkgName(name)
vers, ok := libVersions[name]
vers, ok := pkgVersions[name]
if !ok {
return "", xerrors.Errorf("no version found for %q", name)
}

View File

@@ -8,34 +8,34 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/aquasecurity/trivy/pkg/dependency/types"
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
)
func TestParser_Parse(t *testing.T) {
tests := []struct {
name string
file string
wantLibs []types.Library
wantDeps []types.Dependency
wantPkgs []ftypes.Package
wantDeps []ftypes.Dependency
wantErr assert.ErrorAssertionFunc
}{
{
name: "normal",
file: "testdata/poetry_normal.lock",
wantLibs: poetryNormal,
wantPkgs: poetryNormal,
wantErr: assert.NoError,
},
{
name: "many",
file: "testdata/poetry_many.lock",
wantLibs: poetryMany,
wantPkgs: poetryMany,
wantDeps: poetryManyDeps,
wantErr: assert.NoError,
},
{
name: "flask",
file: "testdata/poetry_flask.lock",
wantLibs: poetryFlask,
wantPkgs: poetryFlask,
wantDeps: poetryFlaskDeps,
wantErr: assert.NoError,
},
@@ -47,11 +47,11 @@ func TestParser_Parse(t *testing.T) {
defer f.Close()
p := NewParser()
gotLibs, gotDeps, err := p.Parse(f)
gotPkgs, gotDeps, err := p.Parse(f)
if !tt.wantErr(t, err, fmt.Sprintf("Parse(%v)", tt.file)) {
return
}
assert.Equalf(t, tt.wantLibs, gotLibs, "Parse(%v)", tt.file)
assert.Equalf(t, tt.wantPkgs, gotPkgs, "Parse(%v)", tt.file)
assert.Equalf(t, tt.wantDeps, gotDeps, "Parse(%v)", tt.file)
})
}
@@ -62,7 +62,7 @@ func TestParseDependency(t *testing.T) {
name string
packageName string
versionRange interface{}
libsVersions map[string][]string
pkgsVersions map[string][]string
want string
wantErr string
}{
@@ -70,7 +70,7 @@ func TestParseDependency(t *testing.T) {
name: "handle package name",
packageName: "Test_project.Name",
versionRange: "*",
libsVersions: map[string][]string{
pkgsVersions: map[string][]string{
"test-project-name": {"1.0.0"},
},
want: "test-project-name@1.0.0",
@@ -79,7 +79,7 @@ func TestParseDependency(t *testing.T) {
name: "version range as string",
packageName: "test",
versionRange: ">=1.0.0",
libsVersions: map[string][]string{
pkgsVersions: map[string][]string{
"test": {"2.0.0"},
},
want: "test@2.0.0",
@@ -88,7 +88,7 @@ func TestParseDependency(t *testing.T) {
name: "version range == *",
packageName: "test",
versionRange: "*",
libsVersions: map[string][]string{
pkgsVersions: map[string][]string{
"test": {"3.0.0"},
},
want: "test@3.0.0",
@@ -100,23 +100,23 @@ func TestParseDependency(t *testing.T) {
"version": ">=4.8.3",
"markers": "python_version < \"3.8\"",
},
libsVersions: map[string][]string{
pkgsVersions: map[string][]string{
"test": {"5.0.0"},
},
want: "test@5.0.0",
},
{
name: "libsVersions doesn't contain required version",
name: "pkgsVersions doesn't contain required version",
packageName: "test",
versionRange: ">=1.0.0",
libsVersions: map[string][]string{},
pkgsVersions: map[string][]string{},
wantErr: "no version found",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NewParser().parseDependency(tt.packageName, tt.versionRange, tt.libsVersions)
got, err := NewParser().parseDependency(tt.packageName, tt.versionRange, tt.pkgsVersions)
if tt.wantErr != "" {
assert.ErrorContains(t, err, tt.wantErr)
return

View File

@@ -1,6 +1,6 @@
package poetry
import "github.com/aquasecurity/trivy/pkg/dependency/types"
import ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
var (
// docker run --name pipenv --rm -it python@sha256:e1141f10176d74d1a0e87a7c0a0a5a98dd98ec5ac12ce867768f40c6feae2fd9 sh
@@ -10,7 +10,7 @@ var (
// poetry new normal && cd normal
// poetry add pypi@2.1
// poetry show -a | awk '{gsub(/\(!\)/, ""); printf("{ID: \""$1"@"$2"\", Name: \""$1"\", Version: \""$2"\"},\n") }'
poetryNormal = []types.Library{
poetryNormal = []ftypes.Package{
{ID: "pypi@2.1", Name: "pypi", Version: "2.1"},
}
@@ -24,7 +24,7 @@ var (
// poetry show -a | awk '{gsub(/\(!\)/, ""); printf("{ID: \""$1"@"$2"\", Name: \""$1"\", Version: \""$2"\"},\n") }'
// `--no-dev` flag uncorrected returns deps. Then need to remove `dev` deps manually
// list of dev deps - cat poetry.lock | grep 'category = "dev"' -B 3
poetryMany = []types.Library{
poetryMany = []ftypes.Package{
{ID: "attrs@22.2.0", Name: "attrs", Version: "22.2.0"},
{ID: "backports-cached-property@1.0.2", Name: "backports-cached-property", Version: "1.0.2"},
{ID: "build@0.10.0", Name: "build", Version: "0.10.0"},
@@ -82,7 +82,7 @@ var (
}
// cat poetry.lock | grep "\[package.dependencies\]" -B 3 -A 8 - it might help to complete this slice
poetryManyDeps = []types.Dependency{
poetryManyDeps = []ftypes.Dependency{
{ID: "build@0.10.0", DependsOn: []string{"colorama@0.4.6", "importlib-metadata@6.0.0", "packaging@23.0", "pyproject-hooks@1.0.0", "tomli@2.0.1"}},
{ID: "cachecontrol@0.12.11", DependsOn: []string{"lockfile@0.12.2", "msgpack@1.0.4", "requests@2.28.2"}},
{ID: "cffi@1.15.1", DependsOn: []string{"pycparser@2.21"}},
@@ -115,7 +115,7 @@ var (
// poetry new web && cd web
// poetry add flask@1.0.3
// poetry show -a | awk '{gsub(/\(!\)/, ""); printf("{ID: \""$1"@"$2"\", Name: \""$1"\", Version: \""$2"\"},\n") }'
poetryFlask = []types.Library{
poetryFlask = []ftypes.Package{
{ID: "click@8.1.3", Name: "click", Version: "8.1.3"},
{ID: "colorama@0.4.6", Name: "colorama", Version: "0.4.6"},
{ID: "flask@1.0.3", Name: "flask", Version: "1.0.3"},
@@ -126,7 +126,7 @@ var (
}
// cat poetry.lock | grep "\[package.dependencies\]" -B 3 -A 8 - it might help to complete this slice
poetryFlaskDeps = []types.Dependency{
poetryFlaskDeps = []ftypes.Dependency{
{ID: "click@8.1.3", DependsOn: []string{"colorama@0.4.6"}},
{ID: "flask@1.0.3", DependsOn: []string{"click@8.1.3", "itsdangerous@2.1.2", "jinja2@3.1.2", "werkzeug@2.2.3"}},
{ID: "jinja2@3.1.2", DependsOn: []string{"markupsafe@2.1.2"}},