feat: add support environment.yaml files (#6569)

Signed-off-by: knqyf263 <knqyf263@gmail.com>
Co-authored-by: knqyf263 <knqyf263@gmail.com>
This commit is contained in:
DmitriyLewen
2024-04-29 16:33:53 +06:00
committed by GitHub
parent 916f6c66f8
commit e3bef02018
21 changed files with 673 additions and 19 deletions

View File

@@ -75,6 +75,7 @@ jobs:
dart
swift
bitnami
conda
os
lang

View File

@@ -0,0 +1,36 @@
# Conda
Trivy supports the following scanners for Conda packages.
| Scanner | Supported |
|:-------------:|:---------:|
| SBOM | ✓ |
| Vulnerability | - |
| License | ✓[^1] |
## SBOM
Trivy detects packages that have been installed with `Conda`.
### `<package>.json`
Trivy parses `<conda-root>/envs/<env>/conda-meta/<package>.json` files to find the version and license for the dependencies installed in your env.
### `environment.yml`[^2]
Trivy supports parsing [environment.yml][environment.yml][^2] files to find dependency list.
!!! note
License detection is currently not supported.
`environment.yml`[^2] files supports [version range][env-version-range]. We can't be sure about versions for these dependencies.
Therefore, you need to use `conda env export` command to get dependency list in `Conda` default format before scanning `environment.yml`[^2] file.
!!! note
For dependencies in a non-Conda format, Trivy doesn't include a version of them.
[^1]: License detection is only supported for `<package>.json` files
[^2]: Trivy supports both `yaml` and `yml` extensions.
[environment.yml]: https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html#sharing-an-environment
[env-version-range]: https://docs.conda.io/projects/conda-build/en/latest/resources/package-spec.html#examples-of-package-specs

View File

@@ -9,23 +9,24 @@ Trivy supports operating systems for
## Supported OS
| OS | Supported Versions | Package Managers |
|-----------------------------------------------|-------------------------------------|------------------|
| [Alpine Linux](alpine.md) | 2.2 - 2.7, 3.0 - 3.19, edge | apk |
| [Wolfi Linux](wolfi.md) | (n/a) | apk |
| [Chainguard](chainguard.md) | (n/a) | apk |
| [Red Hat Enterprise Linux](rhel.md) | 6, 7, 8 | dnf/yum/rpm |
| [CentOS](centos.md)[^1] | 6, 7, 8 | dnf/yum/rpm |
| [AlmaLinux](alma.md) | 8, 9 | dnf/yum/rpm |
| [Rocky Linux](rocky.md) | 8, 9 | dnf/yum/rpm |
| [Oracle Linux](oracle.md) | 5, 6, 7, 8 | dnf/yum/rpm |
| [CBL-Mariner](cbl-mariner.md) | 1.0, 2.0 | dnf/yum/rpm |
| [Amazon Linux](amazon.md) | 1, 2, 2023 | dnf/yum/rpm |
| [openSUSE Leap](suse.md) | 42, 15 | zypper/rpm |
| [SUSE Enterprise Linux](suse.md) | 11, 12, 15 | zypper/rpm |
| [Photon OS](photon.md) | 1.0, 2.0, 3.0, 4.0 | tndf/yum/rpm |
| [Debian GNU/Linux](debian.md) | 7, 8, 9, 10, 11, 12 | apt/dpkg |
| [Ubuntu](ubuntu.md) | All versions supported by Canonical | apt/dpkg |
| OS | Supported Versions | Package Managers |
|--------------------------------------|-------------------------------------|------------------|
| [Alpine Linux](alpine.md) | 2.2 - 2.7, 3.0 - 3.19, edge | apk |
| [Wolfi Linux](wolfi.md) | (n/a) | apk |
| [Chainguard](chainguard.md) | (n/a) | apk |
| [Red Hat Enterprise Linux](rhel.md) | 6, 7, 8 | dnf/yum/rpm |
| [CentOS](centos.md)[^1] | 6, 7, 8 | dnf/yum/rpm |
| [AlmaLinux](alma.md) | 8, 9 | dnf/yum/rpm |
| [Rocky Linux](rocky.md) | 8, 9 | dnf/yum/rpm |
| [Oracle Linux](oracle.md) | 5, 6, 7, 8 | dnf/yum/rpm |
| [CBL-Mariner](cbl-mariner.md) | 1.0, 2.0 | dnf/yum/rpm |
| [Amazon Linux](amazon.md) | 1, 2, 2023 | dnf/yum/rpm |
| [openSUSE Leap](suse.md) | 42, 15 | zypper/rpm |
| [SUSE Enterprise Linux](suse.md) | 11, 12, 15 | zypper/rpm |
| [Photon OS](photon.md) | 1.0, 2.0, 3.0, 4.0 | tndf/yum/rpm |
| [Debian GNU/Linux](debian.md) | 7, 8, 9, 10, 11, 12 | apt/dpkg |
| [Ubuntu](ubuntu.md) | All versions supported by Canonical | apt/dpkg |
| [OSs with installed Conda](conda.md) | - | conda |
## Supported container images

View File

@@ -341,6 +341,15 @@ func TestRepository(t *testing.T) {
},
golden: "testdata/conda-cyclonedx.json.golden",
},
{
name: "conda environment.yaml generating CycloneDX SBOM",
args: args{
command: "fs",
format: "cyclonedx",
input: "testdata/fixtures/repo/conda-environment",
},
golden: "testdata/conda-environment-cyclonedx.json.golden",
},
{
name: "pom.xml generating CycloneDX SBOM (with vulnerabilities)",
args: args{

View File

@@ -0,0 +1,80 @@
{
"$schema": "http://cyclonedx.org/schema/bom-1.5.schema.json",
"bomFormat": "CycloneDX",
"specVersion": "1.5",
"serialNumber": "urn:uuid:3ff14136-e09f-4df9-80ea-000000000004",
"version": 1,
"metadata": {
"timestamp": "2021-08-25T12:20:30+00:00",
"tools": {
"components": [
{
"type": "application",
"group": "aquasecurity",
"name": "trivy",
"version": "dev"
}
]
},
"component": {
"bom-ref": "3ff14136-e09f-4df9-80ea-000000000001",
"type": "application",
"name": "testdata/fixtures/repo/conda-environment",
"properties": [
{
"name": "aquasecurity:trivy:SchemaVersion",
"value": "2"
}
]
}
},
"components": [
{
"bom-ref": "3ff14136-e09f-4df9-80ea-000000000002",
"type": "application",
"name": "environment.yaml",
"properties": [
{
"name": "aquasecurity:trivy:Class",
"value": "lang-pkgs"
},
{
"name": "aquasecurity:trivy:Type",
"value": "conda-environment"
}
]
},
{
"bom-ref": "pkg:conda/bzip2@1.0.8",
"type": "library",
"name": "bzip2",
"version": "1.0.8",
"purl": "pkg:conda/bzip2@1.0.8",
"properties": [
{
"name": "aquasecurity:trivy:PkgType",
"value": "conda-environment"
}
]
}
],
"dependencies": [
{
"ref": "3ff14136-e09f-4df9-80ea-000000000001",
"dependsOn": [
"3ff14136-e09f-4df9-80ea-000000000002"
]
},
{
"ref": "3ff14136-e09f-4df9-80ea-000000000002",
"dependsOn": [
"pkg:conda/bzip2@1.0.8"
]
},
{
"ref": "pkg:conda/bzip2@1.0.8",
"dependsOn": []
}
],
"vulnerabilities": []
}

View File

@@ -0,0 +1,6 @@
name: test-env
channels:
- defaults
dependencies:
- bzip2=1.0.8=h998d150_5
prefix: /opt/conda/envs/test-env

View File

@@ -77,6 +77,7 @@ nav:
- CBL-Mariner: docs/coverage/os/cbl-mariner.md
- CentOS: docs/coverage/os/centos.md
- Chainguard: docs/coverage/os/chainguard.md
- Conda: docs/coverage/os/conda.md
- Debian: docs/coverage/os/debian.md
- Oracle Linux: docs/coverage/os/oracle.md
- Photon OS: docs/coverage/os/photon.md

View File

@@ -0,0 +1,103 @@
package environment
import (
"sort"
"strings"
"sync"
"golang.org/x/xerrors"
"gopkg.in/yaml.v3"
"github.com/aquasecurity/go-version/pkg/version"
"github.com/aquasecurity/trivy/pkg/dependency/types"
"github.com/aquasecurity/trivy/pkg/log"
xio "github.com/aquasecurity/trivy/pkg/x/io"
)
type environment struct {
Dependencies []Dependency `yaml:"dependencies"`
}
type Dependency struct {
Value string
Line int
}
type Parser struct {
logger *log.Logger
once sync.Once
}
func NewParser() types.Parser {
return &Parser{
logger: log.WithPrefix("conda"),
once: sync.Once{},
}
}
func (p *Parser) Parse(r xio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) {
var env environment
if err := yaml.NewDecoder(r).Decode(&env); err != nil {
return nil, nil, xerrors.Errorf("unable to decode conda environment.yml file: %w", err)
}
var libs []types.Library
for _, dep := range env.Dependencies {
lib := p.toLibrary(dep)
// Skip empty libs
if lib.Name == "" {
continue
}
libs = append(libs, lib)
}
sort.Sort(types.Libraries(libs))
return libs, nil, nil
}
func (p *Parser) toLibrary(dep Dependency) types.Library {
name, ver := p.parseDependency(dep.Value)
if ver == "" {
p.once.Do(func() {
p.logger.Warn("Unable to detect the dependency versions from `environment.yml` as those versions are not pinned. Use `conda env export` to pin versions.")
})
}
return types.Library{
Name: name,
Version: ver,
Locations: types.Locations{
{
StartLine: dep.Line,
EndLine: dep.Line,
},
},
}
}
// parseDependency parses the dependency line and returns the name and the pinned version.
// The version range is not supported. It parses only the pinned version.
// e.g.
// - numpy 1.8.1
// - numpy ==1.8.1
// - numpy 1.8.1 py27_0
// - numpy=1.8.1=py27_0
//
// cf. https://docs.conda.io/projects/conda-build/en/latest/resources/package-spec.html#examples-of-package-specs
func (*Parser) parseDependency(line string) (string, string) {
line = strings.NewReplacer(">", " >", "<", " <", "=", " ").Replace(line)
parts := strings.Fields(line)
name := parts[0]
if len(parts) == 1 {
return name, ""
}
if _, err := version.Parse(parts[1]); err != nil {
return name, ""
}
return name, parts[1]
}
func (d *Dependency) UnmarshalYAML(node *yaml.Node) error {
d.Value = node.Value
d.Line = node.Line
return nil
}

View File

@@ -0,0 +1,192 @@
package environment_test
import (
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/aquasecurity/trivy/pkg/dependency/parser/conda/environment"
"github.com/aquasecurity/trivy/pkg/dependency/types"
)
func TestParse(t *testing.T) {
tests := []struct {
name string
input string
want []types.Library
wantErr string
}{
{
name: "happy path",
input: "testdata/happy.yaml",
want: []types.Library{
{
Name: "_openmp_mutex",
Locations: types.Locations{
{
StartLine: 6,
EndLine: 6,
},
},
},
{
Name: "blas",
Version: "1.0",
Locations: types.Locations{
{
StartLine: 5,
EndLine: 5,
},
},
},
{
Name: "bzip2",
Version: "1.0.8",
Locations: types.Locations{
{
StartLine: 19,
EndLine: 19,
},
},
},
{
Name: "ca-certificates",
Version: "2024.2",
Locations: types.Locations{
{
StartLine: 7,
EndLine: 7,
},
},
},
{
Name: "ld_impl_linux-aarch64",
Locations: types.Locations{
{
StartLine: 8,
EndLine: 8,
},
},
},
{
Name: "libblas",
Locations: types.Locations{
{
StartLine: 9,
EndLine: 9,
},
},
},
{
Name: "libcblas",
Locations: types.Locations{
{
StartLine: 10,
EndLine: 10,
},
},
},
{
Name: "libexpat",
Version: "2.6.2",
Locations: types.Locations{
{
StartLine: 11,
EndLine: 11,
},
},
},
{
Name: "libffi",
Version: "3.4.2",
Locations: types.Locations{
{
StartLine: 12,
EndLine: 12,
},
},
},
{
Name: "libgcc-ng",
Locations: types.Locations{
{
StartLine: 13,
EndLine: 13,
},
},
},
{
Name: "libgfortran-ng",
Locations: types.Locations{
{
StartLine: 14,
EndLine: 14,
},
},
},
{
Name: "libgfortran5",
Locations: types.Locations{
{
StartLine: 15,
EndLine: 15,
},
},
},
{
Name: "libgomp",
Version: "13.2.0",
Locations: types.Locations{
{
StartLine: 16,
EndLine: 16,
},
},
},
{
Name: "liblapack",
Locations: types.Locations{
{
StartLine: 17,
EndLine: 17,
},
},
},
{
Name: "libnsl",
Version: "2.0.1",
Locations: types.Locations{
{
StartLine: 18,
EndLine: 18,
},
},
},
},
},
{
name: "invalid_json",
input: "testdata/invalid.yaml",
wantErr: "unable to decode conda environment.yml file",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f, err := os.Open(tt.input)
require.NoError(t, err)
defer f.Close()
got, _, err := environment.NewParser().Parse(f)
if tt.wantErr != "" {
assert.ErrorContains(t, err, tt.wantErr)
return
}
assert.NoError(t, err)
assert.Equal(t, tt.want, got)
})
}
}

View File

@@ -0,0 +1,21 @@
name: test-env
channels:
- defaults
dependencies:
- blas=1.0=openblas
- _openmp_mutex
- ca-certificates=2024.2
- ld_impl_linux-aarch64=2.40.*
- libblas>=3.9
- libcblas<=3.9.0=22_linuxaarch64_openblas
- libexpat==2.6.2
- libffi==3.4.2=h3557bc0_5
- libgcc-ng 13.2|13.3
- libgfortran-ng >13.2.0,<=13.3
- libgfortran5 =>13.2.0,<13.3|13.4
- libgomp 13.2.0 hf8544c7_5
- liblapack=3.9.*=22_linuxaarch64_openblas
- libnsl=2.0.1=h31becfc_0
- bzip2=1.0.8=h998d150_5
prefix: /opt/conda/envs/test-env

View File

@@ -0,0 +1 @@
invalid

View File

@@ -72,7 +72,7 @@ func NewDriver(libType ftypes.LangType) (Driver, bool) {
// https://guides.cocoapods.org/making/making-a-cocoapod.html#cocoapods-versioning-specifics
ecosystem = vulnerability.Cocoapods
comparer = rubygems.Comparer{}
case ftypes.CondaPkg:
case ftypes.CondaPkg, ftypes.CondaEnv:
log.Warn("Conda package is supported for SBOM, not for vulnerability scanning")
return Driver{}, false
case ftypes.Bitnami:

View File

@@ -8,6 +8,7 @@ import (
_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/imgconf/dockerfile"
_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/imgconf/secret"
_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/c/conan"
_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/conda/environment"
_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/conda/meta"
_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/dart/pub"
_ "github.com/aquasecurity/trivy/pkg/fanal/analyzer/language/dotnet/deps"

View File

@@ -69,6 +69,7 @@ const (
// Conda
TypeCondaPkg Type = "conda-pkg"
TypeCondaEnv Type = "conda-environment"
// Python
TypePythonPkg Type = "python-pkg"
@@ -177,6 +178,7 @@ var (
TypeDotNetCore,
TypePackagesProps,
TypeCondaPkg,
TypeCondaEnv,
TypePythonPkg,
TypePip,
TypePipenv,
@@ -208,6 +210,7 @@ var (
TypeSwift,
TypePubSpecLock,
TypeMixLock,
TypeCondaEnv,
}
// TypeIndividualPkgs has all analyzers for individual packages

View File

@@ -0,0 +1,41 @@
package environment
import (
"context"
"os"
"path/filepath"
"golang.org/x/xerrors"
"github.com/aquasecurity/trivy/pkg/dependency/parser/conda/environment"
"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
"github.com/aquasecurity/trivy/pkg/fanal/analyzer/language"
"github.com/aquasecurity/trivy/pkg/fanal/types"
)
func init() {
analyzer.RegisterAnalyzer(&environmentAnalyzer{})
}
const version = 1
type environmentAnalyzer struct{}
func (a environmentAnalyzer) Analyze(_ context.Context, input analyzer.AnalysisInput) (*analyzer.AnalysisResult, error) {
res, err := language.Analyze(types.CondaEnv, input.FilePath, input.Content, environment.NewParser())
if err != nil {
return nil, xerrors.Errorf("unable to parse environment.yaml: %w", err)
}
return res, nil
}
func (a environmentAnalyzer) Required(filePath string, _ os.FileInfo) bool {
return filepath.Base(filePath) == types.CondaEnvYml || filepath.Base(filePath) == types.CondaEnvYaml
}
func (a environmentAnalyzer) Type() analyzer.Type {
return analyzer.TypeCondaEnv
}
func (a environmentAnalyzer) Version() int {
return version
}

View File

@@ -0,0 +1,131 @@
package environment
import (
"context"
"github.com/aquasecurity/trivy/pkg/fanal/analyzer"
"github.com/aquasecurity/trivy/pkg/fanal/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"os"
"testing"
)
func Test_environmentAnalyzer_Analyze(t *testing.T) {
tests := []struct {
name string
inputFile string
want *analyzer.AnalysisResult
wantErr string
}{
{
name: "happy path",
inputFile: "testdata/environment.yaml",
want: &analyzer.AnalysisResult{
Applications: []types.Application{
{
Type: types.CondaEnv,
FilePath: "testdata/environment.yaml",
Libraries: types.Packages{
{
Name: "_libgcc_mutex",
Locations: []types.Location{
{
StartLine: 5,
EndLine: 5,
},
},
},
{
Name: "_openmp_mutex",
Version: "5.1",
Locations: []types.Location{
{
StartLine: 6,
EndLine: 6,
},
},
},
{
Name: "blas",
Version: "1.0",
Locations: []types.Location{
{
StartLine: 7,
EndLine: 7,
},
},
},
{
Name: "bzip2",
Version: "1.0.8",
Locations: []types.Location{
{
StartLine: 8,
EndLine: 8,
},
},
},
},
},
},
},
},
{
name: "invalid",
inputFile: "testdata/invalid.yaml",
wantErr: "unable to parse environment.yaml",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f, err := os.Open(tt.inputFile)
require.NoError(t, err)
defer f.Close()
a := environmentAnalyzer{}
ctx := context.Background()
got, err := a.Analyze(ctx, analyzer.AnalysisInput{
FilePath: tt.inputFile,
Content: f,
})
if tt.wantErr != "" {
require.ErrorContains(t, err, tt.wantErr)
return
}
require.NoError(t, err)
require.Equal(t, tt.want, got)
})
}
}
func Test_environmentAnalyzer_Required(t *testing.T) {
tests := []struct {
name string
filePath string
want bool
}{
{
name: "happy path `yaml`",
filePath: "foo/environment.yaml",
want: true,
},
{
name: "happy path `yml`",
filePath: "bar/environment.yaml",
want: true,
},
{
name: "sad path `json` ",
filePath: "environment.json",
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
a := environmentAnalyzer{}
got := a.Required(tt.filePath, nil)
assert.Equal(t, tt.want, got)
})
}
}

View File

@@ -0,0 +1,9 @@
name: test-env
channels:
- defaults
dependencies:
- _libgcc_mutex
- _openmp_mutex=5.1
- blas=1.0=openblas
- bzip2=1.0.8=h998d150_5
prefix: /opt/conda/envs/test-env

View File

@@ -0,0 +1 @@
invalid

View File

@@ -55,6 +55,7 @@ const (
Pipenv LangType = "pipenv"
Poetry LangType = "poetry"
CondaPkg LangType = "conda-pkg"
CondaEnv LangType = "conda-environment"
PythonPkg LangType = "python-pkg"
NodePkg LangType = "node-pkg"
Yarn LangType = "yarn"
@@ -139,4 +140,7 @@ const (
PubSpecLock = "pubspec.lock"
MixLock = "mix.lock"
CondaEnvYaml = "environment.yaml"
CondaEnvYml = "environment.yml"
)

View File

@@ -432,7 +432,7 @@ func purlType(t ftypes.TargetType) string {
return packageurl.TypeGem
case ftypes.NuGet, ftypes.DotNetCore, ftypes.PackagesProps:
return packageurl.TypeNuget
case ftypes.CondaPkg:
case ftypes.CondaPkg, ftypes.CondaEnv:
return packageurl.TypeConda
case ftypes.PythonPkg, ftypes.Pip, ftypes.Pipenv, ftypes.Poetry:
return packageurl.TypePyPi

View File

@@ -131,6 +131,19 @@ func TestNewPackageURL(t *testing.T) {
Version: "0.4.1",
},
},
{
name: "conda environment.yaml",
typ: ftypes.CondaEnv,
pkg: ftypes.Package{
Name: "blas",
Version: "1.0",
},
want: &purl.PackageURL{
Type: packageurl.TypeConda,
Name: "blas",
Version: "1.0",
},
},
{
name: "composer package",
typ: ftypes.Composer,