mirror of
https://github.com/aquasecurity/trivy.git
synced 2026-02-09 02:03:13 +08:00
refactor: unify Library and Package structs (#6633)
Signed-off-by: knqyf263 <knqyf263@gmail.com> Co-authored-by: DmitriyLewen <91113035+DmitriyLewen@users.noreply.github.com> Co-authored-by: DmitriyLewen <dmitriy.lewen@smartforce.io>
This commit is contained in:
@@ -7,9 +7,10 @@ import (
|
||||
"net/textproto"
|
||||
"strings"
|
||||
|
||||
"github.com/samber/lo"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
"github.com/aquasecurity/trivy/pkg/log"
|
||||
xio "github.com/aquasecurity/trivy/pkg/x/io"
|
||||
)
|
||||
@@ -18,7 +19,7 @@ type Parser struct {
|
||||
logger *log.Logger
|
||||
}
|
||||
|
||||
func NewParser() types.Parser {
|
||||
func NewParser() *Parser {
|
||||
return &Parser{
|
||||
logger: log.WithPrefix("python"),
|
||||
}
|
||||
@@ -26,7 +27,7 @@ func NewParser() types.Parser {
|
||||
|
||||
// Parse parses egg and wheel metadata.
|
||||
// e.g. .egg-info/PKG-INFO and dist-info/METADATA
|
||||
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) {
|
||||
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) {
|
||||
rd := textproto.NewReader(bufio.NewReader(r))
|
||||
h, err := rd.ReadMIMEHeader()
|
||||
if e := textproto.ProtocolError(""); errors.As(err, &e) {
|
||||
@@ -82,11 +83,11 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency,
|
||||
license = "file://" + h.Get("License-File")
|
||||
}
|
||||
|
||||
return []types.Library{
|
||||
return []ftypes.Package{
|
||||
{
|
||||
Name: name,
|
||||
Version: version,
|
||||
License: license,
|
||||
Name: name,
|
||||
Version: version,
|
||||
Licenses: lo.Ternary(license != "", []string{license}, nil),
|
||||
},
|
||||
}, nil, nil
|
||||
}
|
||||
|
||||
@@ -8,14 +8,14 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/aquasecurity/trivy/pkg/dependency/parser/python/packaging"
|
||||
"github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
)
|
||||
|
||||
func TestParse(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
want []types.Library
|
||||
want []ftypes.Package
|
||||
wantErr bool
|
||||
}{
|
||||
// listing dependencies based on METADATA/PKG-INFO files
|
||||
@@ -33,16 +33,22 @@ func TestParse(t *testing.T) {
|
||||
// cd /usr/lib/python3.9/site-packages/setuptools-52.0.0-py3.9.egg-info/
|
||||
// cat PKG-INFO | grep -e "^Name:" -e "^Version:" -e "^License:" | cut -d" " -f2- | \
|
||||
// tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\", \""$2"\", \""$3"\"\}\n")}'
|
||||
want: []types.Library{{Name: "setuptools", Version: "51.3.3", License: "UNKNOWN"}},
|
||||
want: []ftypes.Package{
|
||||
{
|
||||
Name: "setuptools",
|
||||
Version: "51.3.3",
|
||||
Licenses: []string{"UNKNOWN"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "egg PKG-INFO with description containing non-RFC 7230 bytes",
|
||||
input: "testdata/unidecode-egg-info.PKG-INFO",
|
||||
want: []types.Library{
|
||||
want: []ftypes.Package{
|
||||
{
|
||||
Name: "Unidecode",
|
||||
Version: "0.4.1",
|
||||
License: "UNKNOWN",
|
||||
Name: "Unidecode",
|
||||
Version: "0.4.1",
|
||||
Licenses: []string{"UNKNOWN"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -55,7 +61,13 @@ func TestParse(t *testing.T) {
|
||||
// cd /usr/lib/python3.9/site-packages/
|
||||
// cat distlib-0.3.1-py3.9.egg-info | grep -e "^Name:" -e "^Version:" -e "^License:" | cut -d" " -f2- | \
|
||||
// tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\", \""$2"\", \""$3"\"\}\n")}'
|
||||
want: []types.Library{{Name: "distlib", Version: "0.3.1", License: "Python license"}},
|
||||
want: []ftypes.Package{
|
||||
{
|
||||
Name: "distlib",
|
||||
Version: "0.3.1",
|
||||
Licenses: []string{"Python license"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wheel METADATA",
|
||||
@@ -67,31 +79,53 @@ func TestParse(t *testing.T) {
|
||||
// find dist-infos/ | grep -v METADATA | xargs rm -R
|
||||
|
||||
// for single METADATA file with known name
|
||||
// cat "{{ libname }}.METADATA | grep -e "^Name:" -e "^Version:" -e "^License:" | cut -d" " -f2- | tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\", \""$2"\", \""$3"\"\}\n")}'
|
||||
want: []types.Library{{Name: "simple", Version: "0.1.0", License: ""}},
|
||||
// cat "{{ libname }}.METADATA | grep -e "^Name:" -e "^Version:" -e "^Licenses: []string{" | cut -d" " -f2- | tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\"}, \""$2"\", \""$3"\"\}\n")}'
|
||||
want: []ftypes.Package{
|
||||
{
|
||||
Name: "simple",
|
||||
Version: "0.1.0",
|
||||
Licenses: nil,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wheel METADATA",
|
||||
|
||||
// for single METADATA file with known name
|
||||
// cat "{{ libname }}.METADATA | grep -e "^Name:" -e "^Version:" -e "^License:" | cut -d" " -f2- | tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\", \""$2"\", \""$3"\"\}\n")}'
|
||||
// cat "{{ libname }}.METADATA | grep -e "^Name:" -e "^Version:" -e "^Licenses: []string{" | cut -d" " -f2- | tr "\n" "\t" | awk -F "\t" '{printf("\{\""$1"\"}, \""$2"\", \""$3"\"\}\n")}'
|
||||
input: "testdata/distlib-0.3.1.METADATA",
|
||||
want: []types.Library{{Name: "distlib", Version: "0.3.1", License: "Python Software Foundation License"}},
|
||||
want: []ftypes.Package{
|
||||
{
|
||||
Name: "distlib",
|
||||
Version: "0.3.1",
|
||||
Licenses: []string{"Python Software Foundation License"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wheel METADATA",
|
||||
// Input defines "Classifier: License" but it ends at "OSI Approved" which doesn't define any specific license, thus "License" field is added to results
|
||||
input: "testdata/asyncssh-2.14.2.METADATA",
|
||||
|
||||
want: []types.Library{{Name: "asyncssh", Version: "2.14.2", License: "Eclipse Public License v2.0"}},
|
||||
want: []ftypes.Package{
|
||||
{
|
||||
Name: "asyncssh",
|
||||
Version: "2.14.2",
|
||||
Licenses: []string{"Eclipse Public License v2.0"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "wheel METADATA",
|
||||
// Input defines multiple "Classifier: License"
|
||||
input: "testdata/pyphen-0.14.0.METADATA",
|
||||
|
||||
want: []types.Library{
|
||||
{Name: "pyphen", Version: "0.14.0", License: "GNU General Public License v2 or later (GPLv2+), GNU Lesser General Public License v2 or later (LGPLv2+), Mozilla Public License 1.1 (MPL 1.1)"},
|
||||
want: []ftypes.Package{
|
||||
{
|
||||
Name: "pyphen",
|
||||
Version: "0.14.0",
|
||||
Licenses: []string{"GNU General Public License v2 or later (GPLv2+), GNU Lesser General Public License v2 or later (LGPLv2+), Mozilla Public License 1.1 (MPL 1.1)"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -102,33 +136,33 @@ func TestParse(t *testing.T) {
|
||||
{
|
||||
name: "with License-Expression field",
|
||||
input: "testdata/iniconfig-2.0.0.METADATA",
|
||||
want: []types.Library{
|
||||
want: []ftypes.Package{
|
||||
{
|
||||
Name: "iniconfig",
|
||||
Version: "2.0.0",
|
||||
License: "MIT",
|
||||
Name: "iniconfig",
|
||||
Version: "2.0.0",
|
||||
Licenses: []string{"MIT"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "with an empty license field but with license in Classifier",
|
||||
input: "testdata/zipp-3.12.1.METADATA",
|
||||
want: []types.Library{
|
||||
want: []ftypes.Package{
|
||||
{
|
||||
Name: "zipp",
|
||||
Version: "3.12.1",
|
||||
License: "MIT License",
|
||||
Name: "zipp",
|
||||
Version: "3.12.1",
|
||||
Licenses: []string{"MIT License"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "without licenses, but with a license file (a license in Classifier was removed)",
|
||||
input: "testdata/networkx-3.0.METADATA",
|
||||
want: []types.Library{
|
||||
want: []ftypes.Package{
|
||||
{
|
||||
Name: "networkx",
|
||||
Version: "3.0",
|
||||
License: "file://LICENSE.txt",
|
||||
Name: "networkx",
|
||||
Version: "3.0",
|
||||
Licenses: []string{"file://LICENSE.txt"},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"golang.org/x/text/transform"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
xio "github.com/aquasecurity/trivy/pkg/x/io"
|
||||
)
|
||||
|
||||
@@ -24,11 +24,11 @@ const (
|
||||
|
||||
type Parser struct{}
|
||||
|
||||
func NewParser() types.Parser {
|
||||
func NewParser() *Parser {
|
||||
return &Parser{}
|
||||
}
|
||||
|
||||
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) {
|
||||
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) {
|
||||
// `requirements.txt` can use byte order marks (BOM)
|
||||
// e.g. on Windows `requirements.txt` can use UTF-16LE with BOM
|
||||
// We need to override them to avoid the file being read incorrectly
|
||||
@@ -36,7 +36,7 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency,
|
||||
decodedReader := transform.NewReader(r, transformer)
|
||||
|
||||
scanner := bufio.NewScanner(decodedReader)
|
||||
var libs []types.Library
|
||||
var pkgs []ftypes.Package
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
line = strings.ReplaceAll(line, " ", "")
|
||||
@@ -49,7 +49,7 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency,
|
||||
if len(s) != 2 {
|
||||
continue
|
||||
}
|
||||
libs = append(libs, types.Library{
|
||||
pkgs = append(pkgs, ftypes.Package{
|
||||
Name: s[0],
|
||||
Version: s[1],
|
||||
})
|
||||
@@ -57,7 +57,7 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency,
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, nil, xerrors.Errorf("scan error: %w", err)
|
||||
}
|
||||
return libs, nil, nil
|
||||
return pkgs, nil, nil
|
||||
}
|
||||
|
||||
func rStripByKey(line, key string) string {
|
||||
|
||||
@@ -8,13 +8,13 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
)
|
||||
|
||||
func TestParse(t *testing.T) {
|
||||
vectors := []struct {
|
||||
file string
|
||||
want []types.Library
|
||||
want []ftypes.Package
|
||||
}{
|
||||
{
|
||||
file: "testdata/requirements_flask.txt",
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
package pip
|
||||
|
||||
import "github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
import ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
|
||||
var (
|
||||
requirementsFlask = []types.Library{
|
||||
requirementsFlask = []ftypes.Package{
|
||||
{Name: "click", Version: "8.0.0"},
|
||||
{Name: "Flask", Version: "2.0.0"},
|
||||
{Name: "itsdangerous", Version: "2.0.0"},
|
||||
@@ -12,45 +12,45 @@ var (
|
||||
{Name: "Werkzeug", Version: "2.0.0"},
|
||||
}
|
||||
|
||||
requirementsComments = []types.Library{
|
||||
requirementsComments = []ftypes.Package{
|
||||
{Name: "click", Version: "8.0.0"},
|
||||
{Name: "Flask", Version: "2.0.0"},
|
||||
{Name: "Jinja2", Version: "3.0.0"},
|
||||
{Name: "MarkupSafe", Version: "2.0.0"},
|
||||
}
|
||||
|
||||
requirementsSpaces = []types.Library{
|
||||
requirementsSpaces = []ftypes.Package{
|
||||
{Name: "click", Version: "8.0.0"},
|
||||
{Name: "Flask", Version: "2.0.0"},
|
||||
{Name: "itsdangerous", Version: "2.0.0"},
|
||||
{Name: "Jinja2", Version: "3.0.0"},
|
||||
}
|
||||
|
||||
requirementsNoVersion = []types.Library{
|
||||
requirementsNoVersion = []ftypes.Package{
|
||||
{Name: "Flask", Version: "2.0.0"},
|
||||
}
|
||||
|
||||
requirementsOperator = []types.Library{
|
||||
requirementsOperator = []ftypes.Package{
|
||||
{Name: "Django", Version: "2.3.4"},
|
||||
{Name: "SomeProject", Version: "5.4"},
|
||||
}
|
||||
|
||||
requirementsHash = []types.Library{
|
||||
requirementsHash = []ftypes.Package{
|
||||
{Name: "FooProject", Version: "1.2"},
|
||||
{Name: "Jinja2", Version: "3.0.0"},
|
||||
}
|
||||
|
||||
requirementsHyphens = []types.Library{
|
||||
requirementsHyphens = []ftypes.Package{
|
||||
{Name: "oauth2-client", Version: "4.0.0"},
|
||||
{Name: "python-gitlab", Version: "2.0.0"},
|
||||
}
|
||||
|
||||
requirementsExtras = []types.Library{
|
||||
requirementsExtras = []ftypes.Package{
|
||||
{Name: "pyjwt", Version: "2.1.0"},
|
||||
{Name: "celery", Version: "4.4.7"},
|
||||
}
|
||||
|
||||
requirementsUtf16le = []types.Library{
|
||||
requirementsUtf16le = []ftypes.Package{
|
||||
{Name: "attrs", Version: "20.3.0"},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"github.com/liamg/jfather"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
xio "github.com/aquasecurity/trivy/pkg/x/io"
|
||||
)
|
||||
|
||||
@@ -22,11 +22,11 @@ type dependency struct {
|
||||
|
||||
type Parser struct{}
|
||||
|
||||
func NewParser() types.Parser {
|
||||
func NewParser() *Parser {
|
||||
return &Parser{}
|
||||
}
|
||||
|
||||
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) {
|
||||
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) {
|
||||
var lockFile lockFile
|
||||
input, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
@@ -36,20 +36,20 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency,
|
||||
return nil, nil, xerrors.Errorf("failed to decode Pipenv.lock: %w", err)
|
||||
}
|
||||
|
||||
var libs []types.Library
|
||||
for pkgName, dependency := range lockFile.Default {
|
||||
libs = append(libs, types.Library{
|
||||
var pkgs []ftypes.Package
|
||||
for pkgName, dep := range lockFile.Default {
|
||||
pkgs = append(pkgs, ftypes.Package{
|
||||
Name: pkgName,
|
||||
Version: strings.TrimLeft(dependency.Version, "="),
|
||||
Locations: []types.Location{
|
||||
Version: strings.TrimLeft(dep.Version, "="),
|
||||
Locations: []ftypes.Location{
|
||||
{
|
||||
StartLine: dependency.StartLine,
|
||||
EndLine: dependency.EndLine,
|
||||
StartLine: dep.StartLine,
|
||||
EndLine: dep.EndLine,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
return libs, nil, nil
|
||||
return pkgs, nil, nil
|
||||
}
|
||||
|
||||
// UnmarshalJSONWithMetadata needed to detect start and end lines of deps
|
||||
|
||||
@@ -4,19 +4,18 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
)
|
||||
|
||||
func TestParse(t *testing.T) {
|
||||
vectors := []struct {
|
||||
file string // Test input file
|
||||
want []types.Library
|
||||
want []ftypes.Package
|
||||
}{
|
||||
{
|
||||
file: "testdata/Pipfile_normal.lock",
|
||||
@@ -40,21 +39,8 @@ func TestParse(t *testing.T) {
|
||||
got, _, err := NewParser().Parse(f)
|
||||
require.NoError(t, err)
|
||||
|
||||
sort.Slice(got, func(i, j int) bool {
|
||||
ret := strings.Compare(got[i].Name, got[j].Name)
|
||||
if ret == 0 {
|
||||
return got[i].Version < got[j].Version
|
||||
}
|
||||
return ret < 0
|
||||
})
|
||||
|
||||
sort.Slice(v.want, func(i, j int) bool {
|
||||
ret := strings.Compare(v.want[i].Name, v.want[j].Name)
|
||||
if ret == 0 {
|
||||
return v.want[i].Version < v.want[j].Version
|
||||
}
|
||||
return ret < 0
|
||||
})
|
||||
sort.Sort(ftypes.Packages(got))
|
||||
sort.Sort(ftypes.Packages(v.want))
|
||||
|
||||
assert.Equal(t, v.want, got)
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package pipenv
|
||||
|
||||
import "github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
import ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
|
||||
var (
|
||||
// docker run --name pipenv --rm -it python:3.9-alpine sh
|
||||
@@ -11,13 +11,13 @@ var (
|
||||
// pipenv graph --json | jq -rc '.[] | "{\"\(.package.package_name | ascii_downcase)\", \"\(.package.installed_version)\", \"\"},"'
|
||||
// graph doesn't contain information about location of dependency in lock file.
|
||||
// add locations manually
|
||||
pipenvNormal = []types.Library{
|
||||
{Name: "urllib3", Version: "1.24.2", Locations: []types.Location{{StartLine: 65, EndLine: 71}}},
|
||||
{Name: "requests", Version: "2.21.0", Locations: []types.Location{{StartLine: 57, EndLine: 64}}},
|
||||
{Name: "pyyaml", Version: "5.1", Locations: []types.Location{{StartLine: 40, EndLine: 56}}},
|
||||
{Name: "idna", Version: "2.8", Locations: []types.Location{{StartLine: 33, EndLine: 39}}},
|
||||
{Name: "chardet", Version: "3.0.4", Locations: []types.Location{{StartLine: 26, EndLine: 32}}},
|
||||
{Name: "certifi", Version: "2019.3.9", Locations: []types.Location{{StartLine: 19, EndLine: 25}}},
|
||||
pipenvNormal = []ftypes.Package{
|
||||
{Name: "urllib3", Version: "1.24.2", Locations: []ftypes.Location{{StartLine: 65, EndLine: 71}}},
|
||||
{Name: "requests", Version: "2.21.0", Locations: []ftypes.Location{{StartLine: 57, EndLine: 64}}},
|
||||
{Name: "pyyaml", Version: "5.1", Locations: []ftypes.Location{{StartLine: 40, EndLine: 56}}},
|
||||
{Name: "idna", Version: "2.8", Locations: []ftypes.Location{{StartLine: 33, EndLine: 39}}},
|
||||
{Name: "chardet", Version: "3.0.4", Locations: []ftypes.Location{{StartLine: 26, EndLine: 32}}},
|
||||
{Name: "certifi", Version: "2019.3.9", Locations: []ftypes.Location{{StartLine: 19, EndLine: 25}}},
|
||||
}
|
||||
|
||||
// docker run --name pipenv --rm -it python:3.9-alpine bash
|
||||
@@ -28,17 +28,17 @@ var (
|
||||
// pipenv graph --json | jq -rc '.[] | "{\"\(.package.package_name | ascii_downcase)\", \"\(.package.installed_version)\", \"\"},"'
|
||||
// graph doesn't contain information about location of dependency in lock file.
|
||||
// add locations manually
|
||||
pipenvDjango = []types.Library{
|
||||
{Name: "urllib3", Version: "1.24.2", Locations: []types.Location{{StartLine: 95, EndLine: 101}}},
|
||||
{Name: "sqlparse", Version: "0.3.0", Locations: []types.Location{{StartLine: 88, EndLine: 94}}},
|
||||
{Name: "requests", Version: "2.21.0", Locations: []types.Location{{StartLine: 80, EndLine: 87}}},
|
||||
{Name: "pyyaml", Version: "5.1", Locations: []types.Location{{StartLine: 63, EndLine: 79}}},
|
||||
{Name: "pytz", Version: "2019.1", Locations: []types.Location{{StartLine: 56, EndLine: 62}}},
|
||||
{Name: "idna", Version: "2.8", Locations: []types.Location{{StartLine: 49, EndLine: 55}}},
|
||||
{Name: "djangorestframework", Version: "3.9.3", Locations: []types.Location{{StartLine: 41, EndLine: 48}}},
|
||||
{Name: "django", Version: "2.2", Locations: []types.Location{{StartLine: 33, EndLine: 40}}},
|
||||
{Name: "chardet", Version: "3.0.4", Locations: []types.Location{{StartLine: 26, EndLine: 32}}},
|
||||
{Name: "certifi", Version: "2019.3.9", Locations: []types.Location{{StartLine: 19, EndLine: 25}}},
|
||||
pipenvDjango = []ftypes.Package{
|
||||
{Name: "urllib3", Version: "1.24.2", Locations: []ftypes.Location{{StartLine: 95, EndLine: 101}}},
|
||||
{Name: "sqlparse", Version: "0.3.0", Locations: []ftypes.Location{{StartLine: 88, EndLine: 94}}},
|
||||
{Name: "requests", Version: "2.21.0", Locations: []ftypes.Location{{StartLine: 80, EndLine: 87}}},
|
||||
{Name: "pyyaml", Version: "5.1", Locations: []ftypes.Location{{StartLine: 63, EndLine: 79}}},
|
||||
{Name: "pytz", Version: "2019.1", Locations: []ftypes.Location{{StartLine: 56, EndLine: 62}}},
|
||||
{Name: "idna", Version: "2.8", Locations: []ftypes.Location{{StartLine: 49, EndLine: 55}}},
|
||||
{Name: "djangorestframework", Version: "3.9.3", Locations: []ftypes.Location{{StartLine: 41, EndLine: 48}}},
|
||||
{Name: "django", Version: "2.2", Locations: []ftypes.Location{{StartLine: 33, EndLine: 40}}},
|
||||
{Name: "chardet", Version: "3.0.4", Locations: []ftypes.Location{{StartLine: 26, EndLine: 32}}},
|
||||
{Name: "certifi", Version: "2019.3.9", Locations: []ftypes.Location{{StartLine: 19, EndLine: 25}}},
|
||||
}
|
||||
|
||||
// docker run --name pipenv --rm -it python:3.9-alpine bash
|
||||
@@ -49,30 +49,30 @@ var (
|
||||
// pipenv graph --json | jq -rc '.[] | "{\"\(.package.package_name | ascii_downcase)\", \"\(.package.installed_version)\", \"\"},"'
|
||||
// graph doesn't contain information about location of dependency in lock file.
|
||||
// add locations manually
|
||||
pipenvMany = []types.Library{
|
||||
{Name: "urllib3", Version: "1.24.2", Locations: []types.Location{{StartLine: 237, EndLine: 244}}},
|
||||
{Name: "sqlparse", Version: "0.3.0", Locations: []types.Location{{StartLine: 230, EndLine: 236}}},
|
||||
{Name: "six", Version: "1.12.0", Locations: []types.Location{{StartLine: 222, EndLine: 229}}},
|
||||
{Name: "simplejson", Version: "3.16.0", Locations: []types.Location{{StartLine: 204, EndLine: 221}}},
|
||||
{Name: "s3transfer", Version: "0.2.0", Locations: []types.Location{{StartLine: 197, EndLine: 203}}},
|
||||
{Name: "rsa", Version: "3.4.2", Locations: []types.Location{{StartLine: 190, EndLine: 196}}},
|
||||
{Name: "requests", Version: "2.21.0", Locations: []types.Location{{StartLine: 182, EndLine: 189}}},
|
||||
{Name: "pyyaml", Version: "3.13", Locations: []types.Location{{StartLine: 165, EndLine: 181}}},
|
||||
{Name: "pytz", Version: "2019.1", Locations: []types.Location{{StartLine: 158, EndLine: 164}}},
|
||||
{Name: "python-dateutil", Version: "2.8.0", Locations: []types.Location{{StartLine: 150, EndLine: 157}}},
|
||||
{Name: "pyasn1", Version: "0.4.5", Locations: []types.Location{{StartLine: 142, EndLine: 149}}},
|
||||
{Name: "markupsafe", Version: "1.1.1", Locations: []types.Location{{StartLine: 109, EndLine: 141}}},
|
||||
{Name: "jmespath", Version: "0.9.4", Locations: []types.Location{{StartLine: 102, EndLine: 108}}},
|
||||
{Name: "jinja2", Version: "2.10.1", Locations: []types.Location{{StartLine: 94, EndLine: 101}}},
|
||||
{Name: "idna", Version: "2.8", Locations: []types.Location{{StartLine: 87, EndLine: 93}}},
|
||||
{Name: "framework", Version: "0.1.0", Locations: []types.Location{{StartLine: 80, EndLine: 86}}},
|
||||
{Name: "docutils", Version: "0.14", Locations: []types.Location{{StartLine: 72, EndLine: 79}}},
|
||||
{Name: "djangorestframework", Version: "3.9.3", Locations: []types.Location{{StartLine: 64, EndLine: 71}}},
|
||||
{Name: "django", Version: "2.2", Locations: []types.Location{{StartLine: 56, EndLine: 63}}},
|
||||
{Name: "colorama", Version: "0.3.9", Locations: []types.Location{{StartLine: 49, EndLine: 55}}},
|
||||
{Name: "chardet", Version: "3.0.4", Locations: []types.Location{{StartLine: 42, EndLine: 48}}},
|
||||
{Name: "certifi", Version: "2019.3.9", Locations: []types.Location{{StartLine: 35, EndLine: 41}}},
|
||||
{Name: "botocore", Version: "1.12.137", Locations: []types.Location{{StartLine: 27, EndLine: 34}}},
|
||||
{Name: "awscli", Version: "1.16.147", Locations: []types.Location{{StartLine: 19, EndLine: 26}}},
|
||||
pipenvMany = []ftypes.Package{
|
||||
{Name: "urllib3", Version: "1.24.2", Locations: []ftypes.Location{{StartLine: 237, EndLine: 244}}},
|
||||
{Name: "sqlparse", Version: "0.3.0", Locations: []ftypes.Location{{StartLine: 230, EndLine: 236}}},
|
||||
{Name: "six", Version: "1.12.0", Locations: []ftypes.Location{{StartLine: 222, EndLine: 229}}},
|
||||
{Name: "simplejson", Version: "3.16.0", Locations: []ftypes.Location{{StartLine: 204, EndLine: 221}}},
|
||||
{Name: "s3transfer", Version: "0.2.0", Locations: []ftypes.Location{{StartLine: 197, EndLine: 203}}},
|
||||
{Name: "rsa", Version: "3.4.2", Locations: []ftypes.Location{{StartLine: 190, EndLine: 196}}},
|
||||
{Name: "requests", Version: "2.21.0", Locations: []ftypes.Location{{StartLine: 182, EndLine: 189}}},
|
||||
{Name: "pyyaml", Version: "3.13", Locations: []ftypes.Location{{StartLine: 165, EndLine: 181}}},
|
||||
{Name: "pytz", Version: "2019.1", Locations: []ftypes.Location{{StartLine: 158, EndLine: 164}}},
|
||||
{Name: "python-dateutil", Version: "2.8.0", Locations: []ftypes.Location{{StartLine: 150, EndLine: 157}}},
|
||||
{Name: "pyasn1", Version: "0.4.5", Locations: []ftypes.Location{{StartLine: 142, EndLine: 149}}},
|
||||
{Name: "markupsafe", Version: "1.1.1", Locations: []ftypes.Location{{StartLine: 109, EndLine: 141}}},
|
||||
{Name: "jmespath", Version: "0.9.4", Locations: []ftypes.Location{{StartLine: 102, EndLine: 108}}},
|
||||
{Name: "jinja2", Version: "2.10.1", Locations: []ftypes.Location{{StartLine: 94, EndLine: 101}}},
|
||||
{Name: "idna", Version: "2.8", Locations: []ftypes.Location{{StartLine: 87, EndLine: 93}}},
|
||||
{Name: "framework", Version: "0.1.0", Locations: []ftypes.Location{{StartLine: 80, EndLine: 86}}},
|
||||
{Name: "docutils", Version: "0.14", Locations: []ftypes.Location{{StartLine: 72, EndLine: 79}}},
|
||||
{Name: "djangorestframework", Version: "3.9.3", Locations: []ftypes.Location{{StartLine: 64, EndLine: 71}}},
|
||||
{Name: "django", Version: "2.2", Locations: []ftypes.Location{{StartLine: 56, EndLine: 63}}},
|
||||
{Name: "colorama", Version: "0.3.9", Locations: []ftypes.Location{{StartLine: 49, EndLine: 55}}},
|
||||
{Name: "chardet", Version: "3.0.4", Locations: []ftypes.Location{{StartLine: 42, EndLine: 48}}},
|
||||
{Name: "certifi", Version: "2019.3.9", Locations: []ftypes.Location{{StartLine: 35, EndLine: 41}}},
|
||||
{Name: "botocore", Version: "1.12.137", Locations: []ftypes.Location{{StartLine: 27, EndLine: 34}}},
|
||||
{Name: "awscli", Version: "1.16.147", Locations: []ftypes.Location{{StartLine: 19, EndLine: 26}}},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
|
||||
version "github.com/aquasecurity/go-pep440-version"
|
||||
"github.com/aquasecurity/trivy/pkg/dependency"
|
||||
"github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
"github.com/aquasecurity/trivy/pkg/log"
|
||||
xio "github.com/aquasecurity/trivy/pkg/x/io"
|
||||
@@ -39,61 +38,61 @@ func NewParser() *Parser {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) {
|
||||
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) {
|
||||
var lockfile Lockfile
|
||||
if _, err := toml.NewDecoder(r).Decode(&lockfile); err != nil {
|
||||
return nil, nil, xerrors.Errorf("failed to decode poetry.lock: %w", err)
|
||||
}
|
||||
|
||||
// Keep all installed versions
|
||||
libVersions := p.parseVersions(lockfile)
|
||||
pkgVersions := p.parseVersions(lockfile)
|
||||
|
||||
var libs []types.Library
|
||||
var deps []types.Dependency
|
||||
var pkgs []ftypes.Package
|
||||
var deps []ftypes.Dependency
|
||||
for _, pkg := range lockfile.Packages {
|
||||
if pkg.Category == "dev" {
|
||||
continue
|
||||
}
|
||||
|
||||
pkgID := packageID(pkg.Name, pkg.Version)
|
||||
libs = append(libs, types.Library{
|
||||
pkgs = append(pkgs, ftypes.Package{
|
||||
ID: pkgID,
|
||||
Name: pkg.Name,
|
||||
Version: pkg.Version,
|
||||
})
|
||||
|
||||
dependsOn := p.parseDependencies(pkg.Dependencies, libVersions)
|
||||
dependsOn := p.parseDependencies(pkg.Dependencies, pkgVersions)
|
||||
if len(dependsOn) != 0 {
|
||||
deps = append(deps, types.Dependency{
|
||||
deps = append(deps, ftypes.Dependency{
|
||||
ID: pkgID,
|
||||
DependsOn: dependsOn,
|
||||
})
|
||||
}
|
||||
}
|
||||
return libs, deps, nil
|
||||
return pkgs, deps, nil
|
||||
}
|
||||
|
||||
// parseVersions stores all installed versions of libraries for use in dependsOn
|
||||
// as the dependencies of libraries use version range.
|
||||
// parseVersions stores all installed versions of packages for use in dependsOn
|
||||
// as the dependencies of packages use version range.
|
||||
func (p *Parser) parseVersions(lockfile Lockfile) map[string][]string {
|
||||
libVersions := make(map[string][]string)
|
||||
pkgVersions := make(map[string][]string)
|
||||
for _, pkg := range lockfile.Packages {
|
||||
if pkg.Category == "dev" {
|
||||
continue
|
||||
}
|
||||
if vers, ok := libVersions[pkg.Name]; ok {
|
||||
libVersions[pkg.Name] = append(vers, pkg.Version)
|
||||
if vers, ok := pkgVersions[pkg.Name]; ok {
|
||||
pkgVersions[pkg.Name] = append(vers, pkg.Version)
|
||||
} else {
|
||||
libVersions[pkg.Name] = []string{pkg.Version}
|
||||
pkgVersions[pkg.Name] = []string{pkg.Version}
|
||||
}
|
||||
}
|
||||
return libVersions
|
||||
return pkgVersions
|
||||
}
|
||||
|
||||
func (p *Parser) parseDependencies(deps map[string]any, libVersions map[string][]string) []string {
|
||||
func (p *Parser) parseDependencies(deps map[string]any, pkgVersions map[string][]string) []string {
|
||||
var dependsOn []string
|
||||
for name, versRange := range deps {
|
||||
if dep, err := p.parseDependency(name, versRange, libVersions); err != nil {
|
||||
if dep, err := p.parseDependency(name, versRange, pkgVersions); err != nil {
|
||||
p.logger.Debug("Failed to parse poetry dependency", log.Err(err))
|
||||
} else if dep != "" {
|
||||
dependsOn = append(dependsOn, dep)
|
||||
@@ -105,9 +104,9 @@ func (p *Parser) parseDependencies(deps map[string]any, libVersions map[string][
|
||||
return dependsOn
|
||||
}
|
||||
|
||||
func (p *Parser) parseDependency(name string, versRange any, libVersions map[string][]string) (string, error) {
|
||||
func (p *Parser) parseDependency(name string, versRange any, pkgVersions map[string][]string) (string, error) {
|
||||
name = normalizePkgName(name)
|
||||
vers, ok := libVersions[name]
|
||||
vers, ok := pkgVersions[name]
|
||||
if !ok {
|
||||
return "", xerrors.Errorf("no version found for %q", name)
|
||||
}
|
||||
|
||||
@@ -8,34 +8,34 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
)
|
||||
|
||||
func TestParser_Parse(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
file string
|
||||
wantLibs []types.Library
|
||||
wantDeps []types.Dependency
|
||||
wantPkgs []ftypes.Package
|
||||
wantDeps []ftypes.Dependency
|
||||
wantErr assert.ErrorAssertionFunc
|
||||
}{
|
||||
{
|
||||
name: "normal",
|
||||
file: "testdata/poetry_normal.lock",
|
||||
wantLibs: poetryNormal,
|
||||
wantPkgs: poetryNormal,
|
||||
wantErr: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "many",
|
||||
file: "testdata/poetry_many.lock",
|
||||
wantLibs: poetryMany,
|
||||
wantPkgs: poetryMany,
|
||||
wantDeps: poetryManyDeps,
|
||||
wantErr: assert.NoError,
|
||||
},
|
||||
{
|
||||
name: "flask",
|
||||
file: "testdata/poetry_flask.lock",
|
||||
wantLibs: poetryFlask,
|
||||
wantPkgs: poetryFlask,
|
||||
wantDeps: poetryFlaskDeps,
|
||||
wantErr: assert.NoError,
|
||||
},
|
||||
@@ -47,11 +47,11 @@ func TestParser_Parse(t *testing.T) {
|
||||
defer f.Close()
|
||||
|
||||
p := NewParser()
|
||||
gotLibs, gotDeps, err := p.Parse(f)
|
||||
gotPkgs, gotDeps, err := p.Parse(f)
|
||||
if !tt.wantErr(t, err, fmt.Sprintf("Parse(%v)", tt.file)) {
|
||||
return
|
||||
}
|
||||
assert.Equalf(t, tt.wantLibs, gotLibs, "Parse(%v)", tt.file)
|
||||
assert.Equalf(t, tt.wantPkgs, gotPkgs, "Parse(%v)", tt.file)
|
||||
assert.Equalf(t, tt.wantDeps, gotDeps, "Parse(%v)", tt.file)
|
||||
})
|
||||
}
|
||||
@@ -62,7 +62,7 @@ func TestParseDependency(t *testing.T) {
|
||||
name string
|
||||
packageName string
|
||||
versionRange interface{}
|
||||
libsVersions map[string][]string
|
||||
pkgsVersions map[string][]string
|
||||
want string
|
||||
wantErr string
|
||||
}{
|
||||
@@ -70,7 +70,7 @@ func TestParseDependency(t *testing.T) {
|
||||
name: "handle package name",
|
||||
packageName: "Test_project.Name",
|
||||
versionRange: "*",
|
||||
libsVersions: map[string][]string{
|
||||
pkgsVersions: map[string][]string{
|
||||
"test-project-name": {"1.0.0"},
|
||||
},
|
||||
want: "test-project-name@1.0.0",
|
||||
@@ -79,7 +79,7 @@ func TestParseDependency(t *testing.T) {
|
||||
name: "version range as string",
|
||||
packageName: "test",
|
||||
versionRange: ">=1.0.0",
|
||||
libsVersions: map[string][]string{
|
||||
pkgsVersions: map[string][]string{
|
||||
"test": {"2.0.0"},
|
||||
},
|
||||
want: "test@2.0.0",
|
||||
@@ -88,7 +88,7 @@ func TestParseDependency(t *testing.T) {
|
||||
name: "version range == *",
|
||||
packageName: "test",
|
||||
versionRange: "*",
|
||||
libsVersions: map[string][]string{
|
||||
pkgsVersions: map[string][]string{
|
||||
"test": {"3.0.0"},
|
||||
},
|
||||
want: "test@3.0.0",
|
||||
@@ -100,23 +100,23 @@ func TestParseDependency(t *testing.T) {
|
||||
"version": ">=4.8.3",
|
||||
"markers": "python_version < \"3.8\"",
|
||||
},
|
||||
libsVersions: map[string][]string{
|
||||
pkgsVersions: map[string][]string{
|
||||
"test": {"5.0.0"},
|
||||
},
|
||||
want: "test@5.0.0",
|
||||
},
|
||||
{
|
||||
name: "libsVersions doesn't contain required version",
|
||||
name: "pkgsVersions doesn't contain required version",
|
||||
packageName: "test",
|
||||
versionRange: ">=1.0.0",
|
||||
libsVersions: map[string][]string{},
|
||||
pkgsVersions: map[string][]string{},
|
||||
wantErr: "no version found",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := NewParser().parseDependency(tt.packageName, tt.versionRange, tt.libsVersions)
|
||||
got, err := NewParser().parseDependency(tt.packageName, tt.versionRange, tt.pkgsVersions)
|
||||
if tt.wantErr != "" {
|
||||
assert.ErrorContains(t, err, tt.wantErr)
|
||||
return
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package poetry
|
||||
|
||||
import "github.com/aquasecurity/trivy/pkg/dependency/types"
|
||||
import ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
|
||||
|
||||
var (
|
||||
// docker run --name pipenv --rm -it python@sha256:e1141f10176d74d1a0e87a7c0a0a5a98dd98ec5ac12ce867768f40c6feae2fd9 sh
|
||||
@@ -10,7 +10,7 @@ var (
|
||||
// poetry new normal && cd normal
|
||||
// poetry add pypi@2.1
|
||||
// poetry show -a | awk '{gsub(/\(!\)/, ""); printf("{ID: \""$1"@"$2"\", Name: \""$1"\", Version: \""$2"\"},\n") }'
|
||||
poetryNormal = []types.Library{
|
||||
poetryNormal = []ftypes.Package{
|
||||
{ID: "pypi@2.1", Name: "pypi", Version: "2.1"},
|
||||
}
|
||||
|
||||
@@ -24,7 +24,7 @@ var (
|
||||
// poetry show -a | awk '{gsub(/\(!\)/, ""); printf("{ID: \""$1"@"$2"\", Name: \""$1"\", Version: \""$2"\"},\n") }'
|
||||
// `--no-dev` flag uncorrected returns deps. Then need to remove `dev` deps manually
|
||||
// list of dev deps - cat poetry.lock | grep 'category = "dev"' -B 3
|
||||
poetryMany = []types.Library{
|
||||
poetryMany = []ftypes.Package{
|
||||
{ID: "attrs@22.2.0", Name: "attrs", Version: "22.2.0"},
|
||||
{ID: "backports-cached-property@1.0.2", Name: "backports-cached-property", Version: "1.0.2"},
|
||||
{ID: "build@0.10.0", Name: "build", Version: "0.10.0"},
|
||||
@@ -82,7 +82,7 @@ var (
|
||||
}
|
||||
|
||||
// cat poetry.lock | grep "\[package.dependencies\]" -B 3 -A 8 - it might help to complete this slice
|
||||
poetryManyDeps = []types.Dependency{
|
||||
poetryManyDeps = []ftypes.Dependency{
|
||||
{ID: "build@0.10.0", DependsOn: []string{"colorama@0.4.6", "importlib-metadata@6.0.0", "packaging@23.0", "pyproject-hooks@1.0.0", "tomli@2.0.1"}},
|
||||
{ID: "cachecontrol@0.12.11", DependsOn: []string{"lockfile@0.12.2", "msgpack@1.0.4", "requests@2.28.2"}},
|
||||
{ID: "cffi@1.15.1", DependsOn: []string{"pycparser@2.21"}},
|
||||
@@ -115,7 +115,7 @@ var (
|
||||
// poetry new web && cd web
|
||||
// poetry add flask@1.0.3
|
||||
// poetry show -a | awk '{gsub(/\(!\)/, ""); printf("{ID: \""$1"@"$2"\", Name: \""$1"\", Version: \""$2"\"},\n") }'
|
||||
poetryFlask = []types.Library{
|
||||
poetryFlask = []ftypes.Package{
|
||||
{ID: "click@8.1.3", Name: "click", Version: "8.1.3"},
|
||||
{ID: "colorama@0.4.6", Name: "colorama", Version: "0.4.6"},
|
||||
{ID: "flask@1.0.3", Name: "flask", Version: "1.0.3"},
|
||||
@@ -126,7 +126,7 @@ var (
|
||||
}
|
||||
|
||||
// cat poetry.lock | grep "\[package.dependencies\]" -B 3 -A 8 - it might help to complete this slice
|
||||
poetryFlaskDeps = []types.Dependency{
|
||||
poetryFlaskDeps = []ftypes.Dependency{
|
||||
{ID: "click@8.1.3", DependsOn: []string{"colorama@0.4.6"}},
|
||||
{ID: "flask@1.0.3", DependsOn: []string{"click@8.1.3", "itsdangerous@2.1.2", "jinja2@3.1.2", "werkzeug@2.2.3"}},
|
||||
{ID: "jinja2@3.1.2", DependsOn: []string{"markupsafe@2.1.2"}},
|
||||
|
||||
Reference in New Issue
Block a user