From e7c16a756c5f60aa17bd3f09aaae0d8171c803de Mon Sep 17 00:00:00 2001 From: Nikita Pivkin Date: Tue, 30 Sep 2025 09:33:52 +0600 Subject: [PATCH] refactor(misconf): replace github.com/liamg/memoryfs with internal mapfs and testing/fstest (#9282) Signed-off-by: nikpivkin Co-authored-by: knqyf263 --- .golangci.yaml | 3 + go.mod | 1 - go.sum | 2 - internal/testutil/util.go | 17 ++-- .../adapters/arm/adaptertest/adaptertest.go | 2 +- .../cloudformation/testutil/testutil.go | 2 +- .../adapters/terraform/tftestutil/testutil.go | 2 +- pkg/iac/rego/scanner_test.go | 59 +++++------- .../scanners/azure/arm/parser/parser_test.go | 78 ++++++++-------- .../cloudformation/parser/parser_test.go | 18 ++-- .../scanners/cloudformation/scanner_test.go | 4 +- pkg/iac/scanners/dockerfile/scanner_test.go | 4 +- pkg/iac/scanners/generic/scanner_test.go | 6 +- pkg/iac/scanners/helm/parser/parser.go | 4 +- pkg/iac/scanners/helm/parser/parser_tar.go | 90 ++++++++----------- pkg/iac/scanners/helm/scanner.go | 29 +++--- .../scanners/terraform/deterministic_test.go | 2 +- pkg/iac/scanners/terraform/ignore_test.go | 2 +- pkg/iac/scanners/terraform/module_test.go | 2 +- .../terraform/parser/load_vars_test.go | 4 +- .../scanners/terraform/parser/modules_test.go | 2 +- .../scanners/terraform/parser/parser_test.go | 66 +++++++------- pkg/iac/scanners/terraform/scanner_test.go | 32 +++---- pkg/iac/scanners/terraform/setup_test.go | 6 +- .../terraformplan/snapshot/snapshot.go | 10 +-- .../terraformplan/tfjson/parser/parser.go | 10 +-- .../terraformplan/tfjson/scanner_test.go | 2 +- pkg/iac/types/fskey_test.go | 8 +- pkg/iac/types/range.go | 44 ++++----- 29 files changed, 227 insertions(+), 284 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 0d0b726c70..63d3d3a480 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -59,6 +59,9 @@ linters: recommendations: - github.com/aquasecurity/go-version reason: "`aquasecurity/go-version` is designed for our use-cases" + - github.com/liamg/memoryfs: + recommendations: + - github.com/aquasecurity/trivy/pkg/mapfs gosec: excludes: - G101 diff --git a/go.mod b/go.mod index 74be2e7ba1..db08400301 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,6 @@ require ( github.com/knqyf263/go-rpmdb v0.1.1 github.com/knqyf263/nested v0.0.1 github.com/kylelemons/godebug v1.1.0 - github.com/liamg/memoryfs v1.6.0 github.com/magefile/mage v1.15.0 github.com/masahiro331/go-disk v0.0.0-20240625071113-56c933208fee github.com/masahiro331/go-ebs-file v0.0.0-20240917043618-e6d2bea5c32e diff --git a/go.sum b/go.sum index 3c460b8e76..d6ed5a933e 100644 --- a/go.sum +++ b/go.sum @@ -860,8 +860,6 @@ github.com/lestrrat-go/option/v2 v2.0.0 h1:XxrcaJESE1fokHy3FpaQ/cXW8ZsIdWcdFzzLO github.com/lestrrat-go/option/v2 v2.0.0/go.mod h1:oSySsmzMoR0iRzCDCaUfsCzxQHUEuhOViQObyy7S6Vg= github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec h1:2tTW6cDth2TSgRbAhD7yjZzTQmcN25sDRPEeinR51yQ= github.com/letsencrypt/boulder v0.0.0-20240620165639-de9c06129bec/go.mod h1:TmwEoGCwIti7BCeJ9hescZgRtatxRE+A72pCoPfmcfk= -github.com/liamg/memoryfs v1.6.0 h1:jAFec2HI1PgMTem5gR7UT8zi9u4BfG5jorCRlLH06W8= -github.com/liamg/memoryfs v1.6.0/go.mod h1:z7mfqXFQS8eSeBBsFjYLlxYRMRyiPktytvYCYTb3BSk= github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= diff --git a/internal/testutil/util.go b/internal/testutil/util.go index c24f17788f..1961267416 100644 --- a/internal/testutil/util.go +++ b/internal/testutil/util.go @@ -3,11 +3,10 @@ package testutil import ( "encoding/json" "io/fs" - "path/filepath" "strings" "testing" + "testing/fstest" - "github.com/liamg/memoryfs" "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -56,16 +55,10 @@ func ruleIDInResults(ruleID string, results scan.Results) bool { return false } -func CreateFS(t *testing.T, files map[string]string) fs.FS { - memfs := memoryfs.New() - for name, content := range files { - name := strings.TrimPrefix(name, "/") - err := memfs.MkdirAll(filepath.Dir(name), 0o700) - require.NoError(t, err) - err = memfs.WriteFile(name, []byte(content), 0o644) - require.NoError(t, err) - } - return memfs +func CreateFS(files map[string]string) fs.FS { + return fstest.MapFS(lo.MapEntries(files, func(k, v string) (string, *fstest.MapFile) { + return strings.TrimPrefix(k, "/"), &fstest.MapFile{Data: []byte(v)} + })) } func AssertDefsecEqual(t *testing.T, expected, actual any) { diff --git a/pkg/iac/adapters/arm/adaptertest/adaptertest.go b/pkg/iac/adapters/arm/adaptertest/adaptertest.go index 17367d7321..d1edad127a 100644 --- a/pkg/iac/adapters/arm/adaptertest/adaptertest.go +++ b/pkg/iac/adapters/arm/adaptertest/adaptertest.go @@ -13,7 +13,7 @@ import ( type adaptFn[T any] func(deployment azure.Deployment) T func AdaptAndCompare[T any](t *testing.T, source string, expected any, fn adaptFn[T]) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "test.json": source, }) diff --git a/pkg/iac/adapters/cloudformation/testutil/testutil.go b/pkg/iac/adapters/cloudformation/testutil/testutil.go index c4f458c381..b441afab7e 100644 --- a/pkg/iac/adapters/cloudformation/testutil/testutil.go +++ b/pkg/iac/adapters/cloudformation/testutil/testutil.go @@ -12,7 +12,7 @@ import ( type adaptFn[T any] func(fctx parser.FileContext) T func AdaptAndCompare[T any](t *testing.T, source string, expected any, fn adaptFn[T]) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.yaml": source, }) diff --git a/pkg/iac/adapters/terraform/tftestutil/testutil.go b/pkg/iac/adapters/terraform/tftestutil/testutil.go index 96045b50b2..74bd828a97 100644 --- a/pkg/iac/adapters/terraform/tftestutil/testutil.go +++ b/pkg/iac/adapters/terraform/tftestutil/testutil.go @@ -11,7 +11,7 @@ import ( ) func CreateModulesFromSource(t *testing.T, source, ext string) terraform.Modules { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "source" + ext: source, }) p := parser.New(fs, "", parser.OptionStopOnHCLError(true)) diff --git a/pkg/iac/rego/scanner_test.go b/pkg/iac/rego/scanner_test.go index 2d0f65257c..9bb4a10d54 100644 --- a/pkg/iac/rego/scanner_test.go +++ b/pkg/iac/rego/scanner_test.go @@ -3,38 +3,25 @@ package rego_test import ( "bytes" "fmt" - "io/fs" "os" "path/filepath" "strings" "testing" "testing/fstest" - "github.com/liamg/memoryfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/aquasecurity/trivy/internal/testutil" "github.com/aquasecurity/trivy/pkg/iac/rego" "github.com/aquasecurity/trivy/pkg/iac/scanners/options" "github.com/aquasecurity/trivy/pkg/iac/severity" "github.com/aquasecurity/trivy/pkg/iac/types" ) -func CreateFS(t *testing.T, files map[string]string) fs.FS { - memfs := memoryfs.New() - for name, content := range files { - name := strings.TrimPrefix(name, "/") - err := memfs.MkdirAll(filepath.Dir(name), 0o700) - require.NoError(t, err) - err = memfs.WriteFile(name, []byte(content), 0o644) - require.NoError(t, err) - } - return memfs -} - func Test_RegoScanning_Deny(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": ` # METADATA # title: Custom policy @@ -128,7 +115,7 @@ deny { } func Test_RegoScanning_Allow(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # title: Custom policy # description: Custom policy for testing @@ -176,7 +163,7 @@ func Test_RegoScanning_WithRuntimeValues(t *testing.T) { t.Setenv("DEFSEC_RUNTIME_VAL", "AOK") - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # title: Custom policy # description: Custom policy for testing @@ -220,7 +207,7 @@ deny_evil { } func Test_RegoScanning_WithDenyMessage(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # title: Custom policy # description: Custom policy for testing @@ -267,7 +254,7 @@ deny[msg] { } func Test_RegoScanning_WithDenyMetadata_ImpliedPath(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": ` # METADATA # title: Custom policy @@ -322,7 +309,7 @@ deny[res] { } func Test_RegoScanning_WithDenyMetadata_PersistedPath(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": ` # METADATA # title: Custom policy @@ -378,7 +365,7 @@ deny[res] { } func Test_RegoScanning_WithStaticMetadata(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": ` package defsec.test @@ -439,7 +426,7 @@ deny[res] { } func Test_RegoScanning_WithMatchingInputSelector(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # title: Custom policy # description: Custom policy for testing @@ -487,7 +474,7 @@ deny { } func Test_RegoScanning_WithNonMatchingInputSelector(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": ` package defsec.test @@ -521,7 +508,7 @@ deny { func Test_RegoScanning_NoTracingByDefault(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # title: Custom policy # description: Custom policy for testing @@ -567,7 +554,7 @@ deny { func Test_RegoScanning_GlobalTracingEnabled(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # title: Custom policy # description: Custom policy for testing @@ -617,7 +604,7 @@ deny { func Test_RegoScanning_PerResultTracingEnabled(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # title: Custom policy # description: Custom policy for testing @@ -663,7 +650,7 @@ deny { func Test_dynamicMetadata(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": ` package defsec.test @@ -695,7 +682,7 @@ deny { func Test_staticMetadata(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": ` package defsec.test @@ -727,7 +714,7 @@ deny { func Test_annotationMetadata(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # title: i am a title # description: i am a description @@ -782,7 +769,7 @@ deny { func Test_RegoScanning_WithInvalidInputSchema(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # schemas: # - input: schema["input"] @@ -802,7 +789,7 @@ deny { func Test_RegoScanning_WithValidInputSchema(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # schemas: # - input: schema["input"] @@ -821,7 +808,7 @@ deny { } func Test_RegoScanning_WithFilepathToSchema(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # schemas: # - input: schema["dockerfile"] @@ -846,7 +833,7 @@ deny { } func Test_RegoScanning_CustomData(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # title: Custom policy # description: Custom policy for testing @@ -871,7 +858,7 @@ deny { `, }) - dataFS := CreateFS(t, map[string]string{ + dataFS := testutil.CreateFS(map[string]string{ "data/data.json": `{ "settings": { "DS123":{ @@ -899,7 +886,7 @@ deny { } func Test_RegoScanning_InvalidFS(t *testing.T) { - srcFS := CreateFS(t, map[string]string{ + srcFS := testutil.CreateFS(map[string]string{ "policies/test.rego": `# METADATA # title: Custom policy # description: Custom policy for testing @@ -924,7 +911,7 @@ deny { `, }) - dataFS := CreateFS(t, map[string]string{ + dataFS := testutil.CreateFS(map[string]string{ "data/data.json": `{ "settings": { "DS123":{ diff --git a/pkg/iac/scanners/azure/arm/parser/parser_test.go b/pkg/iac/scanners/azure/arm/parser/parser_test.go index 41e6ed9cd8..69dbb29eb2 100644 --- a/pkg/iac/scanners/azure/arm/parser/parser_test.go +++ b/pkg/iac/scanners/azure/arm/parser/parser_test.go @@ -3,8 +3,8 @@ package parser import ( "io/fs" "testing" + "testing/fstest" - "github.com/liamg/memoryfs" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -24,12 +24,10 @@ func createMetadata(targetFS fs.FS, filename string, start, end int, ref string, func TestParser_Parse(t *testing.T) { filename := "example.json" - targetFS := memoryfs.New() - tests := []struct { name string input string - want func() azure.Deployment + want func(fsys fs.FS) azure.Deployment wantDeployment bool }{ { @@ -47,10 +45,10 @@ func TestParser_Parse(t *testing.T) { }, "resources": [] }`, - want: func() azure.Deployment { - root := createMetadata(targetFS, filename, 0, 0, "", nil).WithInternal(resolver.NewResolver()) - metadata := createMetadata(targetFS, filename, 1, 13, "", &root) - storageMetadata := createMetadata(targetFS, filename, 5, 10, "parameters.storagePrefix", &metadata) + want: func(fsys fs.FS) azure.Deployment { + root := createMetadata(fsys, filename, 0, 0, "", nil).WithInternal(resolver.NewResolver()) + metadata := createMetadata(fsys, filename, 1, 13, "", &root) + storageMetadata := createMetadata(fsys, filename, 5, 10, "parameters.storagePrefix", &metadata) return azure.Deployment{ Metadata: metadata, @@ -59,9 +57,9 @@ func TestParser_Parse(t *testing.T) { { Variable: azure.Variable{ Name: "storagePrefix", - Value: azure.NewValue("x", createMetadata(targetFS, filename, 7, 7, "parameters.storagePrefix.defaultValue", &storageMetadata)), + Value: azure.NewValue("x", createMetadata(fsys, filename, 7, 7, "parameters.storagePrefix.defaultValue", &storageMetadata)), }, - Default: azure.NewValue("x", createMetadata(targetFS, filename, 7, 7, "parameters.storagePrefix.defaultValue", &storageMetadata)), + Default: azure.NewValue("x", createMetadata(fsys, filename, 7, 7, "parameters.storagePrefix.defaultValue", &storageMetadata)), Decorators: nil, }, }, @@ -117,19 +115,18 @@ func TestParser_Parse(t *testing.T) { } ] }`, - want: func() azure.Deployment { - rootMetadata := createMetadata(targetFS, filename, 0, 0, "", nil).WithInternal(resolver.NewResolver()) - fileMetadata := createMetadata(targetFS, filename, 1, 46, "", &rootMetadata) + want: func(fsys fs.FS) azure.Deployment { + rootMetadata := createMetadata(fsys, filename, 0, 0, "", nil).WithInternal(resolver.NewResolver()) + fileMetadata := createMetadata(fsys, filename, 1, 46, "", &rootMetadata) - resourceMetadata := createMetadata(targetFS, filename, 6, 44, "resources[0]", &fileMetadata) + resourceMetadata := createMetadata(fsys, filename, 6, 44, "resources[0]", &fileMetadata) - propertiesMetadata := createMetadata(targetFS, filename, 27, 43, "resources[0].properties", &resourceMetadata) + propertiesMetadata := createMetadata(fsys, filename, 27, 43, "resources[0].properties", &resourceMetadata) + customDomainMetadata := createMetadata(fsys, filename, 29, 34, "resources[0].properties.customDomain", &propertiesMetadata) + networkACLListMetadata := createMetadata(fsys, filename, 35, 42, "resources[0].properties.networkAcls", &propertiesMetadata) - customDomainMetadata := createMetadata(targetFS, filename, 29, 34, "resources[0].properties.customDomain", &propertiesMetadata) - networkACLListMetadata := createMetadata(targetFS, filename, 35, 42, "resources[0].properties.networkAcls", &propertiesMetadata) - - networkACL0Metadata := createMetadata(targetFS, filename, 36, 38, "resources[0].properties.networkAcls[0]", &networkACLListMetadata) - networkACL1Metadata := createMetadata(targetFS, filename, 39, 41, "resources[0].properties.networkAcls[1]", &networkACLListMetadata) + networkACL0Metadata := createMetadata(fsys, filename, 36, 38, "resources[0].properties.networkAcls[0]", &networkACLListMetadata) + networkACL1Metadata := createMetadata(fsys, filename, 39, 41, "resources[0].properties.networkAcls[1]", &networkACLListMetadata) return azure.Deployment{ Metadata: fileMetadata, @@ -139,45 +136,45 @@ func TestParser_Parse(t *testing.T) { Metadata: resourceMetadata, APIVersion: azure.NewValue( "2022-05-01", - createMetadata(targetFS, filename, 8, 8, "resources[0].apiVersion", &resourceMetadata), + createMetadata(fsys, filename, 8, 8, "resources[0].apiVersion", &resourceMetadata), ), Type: azure.NewValue( "Microsoft.Storage/storageAccounts", - createMetadata(targetFS, filename, 7, 7, "resources[0].type", &resourceMetadata), + createMetadata(fsys, filename, 7, 7, "resources[0].type", &resourceMetadata), ), Kind: azure.NewValue( "string", - createMetadata(targetFS, filename, 18, 18, "resources[0].kind", &resourceMetadata), + createMetadata(fsys, filename, 18, 18, "resources[0].kind", &resourceMetadata), ), Name: azure.NewValue( "myResource", - createMetadata(targetFS, filename, 9, 9, "resources[0].name", &resourceMetadata), + createMetadata(fsys, filename, 9, 9, "resources[0].name", &resourceMetadata), ), Location: azure.NewValue( "string", - createMetadata(targetFS, filename, 10, 10, "resources[0].location", &resourceMetadata), + createMetadata(fsys, filename, 10, 10, "resources[0].location", &resourceMetadata), ), Properties: azure.NewValue( map[string]azure.Value{ - "allowSharedKeyAccess": azure.NewValue(false, createMetadata(targetFS, filename, 28, 28, "resources[0].properties.allowSharedKeyAccess", &propertiesMetadata)), + "allowSharedKeyAccess": azure.NewValue(false, createMetadata(fsys, filename, 28, 28, "resources[0].properties.allowSharedKeyAccess", &propertiesMetadata)), "customDomain": azure.NewValue( map[string]azure.Value{ - "name": azure.NewValue("string", createMetadata(targetFS, filename, 30, 30, "resources[0].properties.customDomain.name", &customDomainMetadata)), - "useSubDomainName": azure.NewValue(false, createMetadata(targetFS, filename, 31, 31, "resources[0].properties.customDomain.useSubDomainName", &customDomainMetadata)), - "number": azure.NewValue(int64(123), createMetadata(targetFS, filename, 32, 32, "resources[0].properties.customDomain.number", &customDomainMetadata)), - "expr": azure.NewExprValue("toLower('Production')", createMetadata(targetFS, filename, 33, 33, "resources[0].properties.customDomain.expr", &customDomainMetadata)), + "name": azure.NewValue("string", createMetadata(fsys, filename, 30, 30, "resources[0].properties.customDomain.name", &customDomainMetadata)), + "useSubDomainName": azure.NewValue(false, createMetadata(fsys, filename, 31, 31, "resources[0].properties.customDomain.useSubDomainName", &customDomainMetadata)), + "number": azure.NewValue(int64(123), createMetadata(fsys, filename, 32, 32, "resources[0].properties.customDomain.number", &customDomainMetadata)), + "expr": azure.NewExprValue("toLower('Production')", createMetadata(fsys, filename, 33, 33, "resources[0].properties.customDomain.expr", &customDomainMetadata)), }, customDomainMetadata), "networkAcls": azure.NewValue( []azure.Value{ azure.NewValue( map[string]azure.Value{ - "bypass": azure.NewValue("AzureServices1", createMetadata(targetFS, filename, 37, 37, "resources[0].properties.networkAcls[0].bypass", &networkACL0Metadata)), + "bypass": azure.NewValue("AzureServices1", createMetadata(fsys, filename, 37, 37, "resources[0].properties.networkAcls[0].bypass", &networkACL0Metadata)), }, networkACL0Metadata, ), azure.NewValue( map[string]azure.Value{ - "bypass": azure.NewValue("AzureServices2", createMetadata(targetFS, filename, 40, 40, "resources[0].properties.networkAcls[1].bypass", &networkACL1Metadata)), + "bypass": azure.NewValue("AzureServices2", createMetadata(fsys, filename, 40, 40, "resources[0].properties.networkAcls[1].bypass", &networkACL1Metadata)), }, networkACL1Metadata, ), @@ -196,9 +193,10 @@ func TestParser_Parse(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - require.NoError(t, targetFS.WriteFile(filename, []byte(tt.input), 0o644)) - - p := New(targetFS) + fsys := fstest.MapFS{ + filename: &fstest.MapFile{Data: []byte(tt.input)}, + } + p := New(fsys) got, err := p.ParseFS(t.Context(), ".") require.NoError(t, err) @@ -208,7 +206,7 @@ func TestParser_Parse(t *testing.T) { } require.Len(t, got, 1) - want := tt.want() + want := tt.want(fsys) assert.Equal(t, want, got[0]) }) } @@ -279,11 +277,11 @@ func Test_NestedResourceParsing(t *testing.T) { } ` - targetFS := memoryfs.New() + fsys := fstest.MapFS{ + "nested.json": &fstest.MapFile{Data: []byte(input)}, + } - require.NoError(t, targetFS.WriteFile("nested.json", []byte(input), 0o644)) - - p := New(targetFS) + p := New(fsys) got, err := p.ParseFS(t.Context(), ".") require.NoError(t, err) require.Len(t, got, 1) diff --git a/pkg/iac/scanners/cloudformation/parser/parser_test.go b/pkg/iac/scanners/cloudformation/parser/parser_test.go index 8cc8031e8d..9c7e5d1400 100644 --- a/pkg/iac/scanners/cloudformation/parser/parser_test.go +++ b/pkg/iac/scanners/cloudformation/parser/parser_test.go @@ -226,7 +226,7 @@ Resources: } func TestParse_WithParameters(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.yaml": `AWSTemplateFormatVersion: 2010-09-09 Parameters: KmsMasterKeyId: @@ -259,7 +259,7 @@ Resources: } func TestParse_WithParameterFiles(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.yaml": `AWSTemplateFormatVersion: 2010-09-09 Parameters: KmsMasterKeyId: @@ -296,7 +296,7 @@ Resources: } func TestParse_WithConfigFS(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "queue.yaml": `AWSTemplateFormatVersion: 2010-09-09 Parameters: KmsMasterKeyId: @@ -321,7 +321,7 @@ Resources: `, }) - configFS := testutil.CreateFS(t, map[string]string{ + configFS := testutil.CreateFS(map[string]string{ "/workdir/parameters/queue.json": `[ { "ParameterKey": "KmsMasterKeyId", @@ -338,7 +338,7 @@ Resources: }) p := New( - WithParameterFiles("/workdir/parameters/queue.json", "/workdir/parameters/s3.json"), + WithParameterFiles("workdir/parameters/queue.json", "workdir/parameters/s3.json"), WithConfigsFS(configFS), ) @@ -391,7 +391,7 @@ func TestJsonWithNumbers(t *testing.T) { } ` - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.json": src, }) @@ -425,7 +425,7 @@ Conditions: SubscribeEmail: !Not [!Equals [ !Ref Email, ""]] ` - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.yaml": src, }) @@ -442,7 +442,7 @@ Resources: Properties: BucketName:` - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.yaml": src, }) @@ -468,7 +468,7 @@ Resources: PublicAccessBlockConfiguration: BlockPublicAcls: null` - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.yaml": src, }) diff --git a/pkg/iac/scanners/cloudformation/scanner_test.go b/pkg/iac/scanners/cloudformation/scanner_test.go index 6c330a0301..97bf3441d9 100644 --- a/pkg/iac/scanners/cloudformation/scanner_test.go +++ b/pkg/iac/scanners/cloudformation/scanner_test.go @@ -15,7 +15,7 @@ import ( func Test_BasicScan(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "/code/main.yaml": `--- Resources: S3Bucket: @@ -205,7 +205,7 @@ Resources: for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "/code/main.yaml": tt.src, }) diff --git a/pkg/iac/scanners/dockerfile/scanner_test.go b/pkg/iac/scanners/dockerfile/scanner_test.go index 4e0a42a60f..af7df23043 100644 --- a/pkg/iac/scanners/dockerfile/scanner_test.go +++ b/pkg/iac/scanners/dockerfile/scanner_test.go @@ -214,7 +214,7 @@ deny[res] { }` func Test_BasicScanLegacyRegoMetadata(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "/code/Dockerfile": `FROM ubuntu USER root `, @@ -559,7 +559,7 @@ res := true COPY --from=dep /binary /` fsysMap["/rules/rule.rego"] = tc.inputRegoPolicy fsysMap["/rules/schemas/myfancydockerfile.json"] = string(schemas.Dockerfile) // just use the same for testing - fsys := testutil.CreateFS(t, fsysMap) + fsys := testutil.CreateFS(fsysMap) var traceBuf bytes.Buffer diff --git a/pkg/iac/scanners/generic/scanner_test.go b/pkg/iac/scanners/generic/scanner_test.go index 6cc9e2d28a..2b10ce8db5 100644 --- a/pkg/iac/scanners/generic/scanner_test.go +++ b/pkg/iac/scanners/generic/scanner_test.go @@ -14,7 +14,7 @@ import ( ) func TestJsonScanner(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "/code/data.json": `{ "x": { "y": 123, "z": ["a", "b", "c"]}}`, "/rules/rule.rego": `package builtin.json.lol @@ -76,7 +76,7 @@ deny[res] { } func TestYamlScanner(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "/code/data.yaml": `--- x: y: 123 @@ -147,7 +147,7 @@ deny[res] { } func TestTomlParser(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "/code/code.toml": ` [x] y = 123 diff --git a/pkg/iac/scanners/helm/parser/parser.go b/pkg/iac/scanners/helm/parser/parser.go index 44aa73fb67..ec3fd912ef 100644 --- a/pkg/iac/scanners/helm/parser/parser.go +++ b/pkg/iac/scanners/helm/parser/parser.go @@ -13,7 +13,6 @@ import ( "strings" "github.com/google/uuid" - "github.com/liamg/memoryfs" "gopkg.in/yaml.v3" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" @@ -24,6 +23,7 @@ import ( "github.com/aquasecurity/trivy/pkg/iac/detection" "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/mapfs" ) var manifestNameRegex = regexp.MustCompile("# Source: [^/]+/(.+)") @@ -101,7 +101,7 @@ func (p *Parser) parseFS(ctx context.Context, fsys fs.FS, target string) error { } if detection.IsArchive(filePath) && !isDependencyChartArchive(fsys, filePath) { - memFS := memoryfs.New() + memFS := mapfs.New() if err := p.unpackArchive(fsys, memFS, filePath); errors.Is(err, errSkipFS) { // an unpacked Chart already exists return nil diff --git a/pkg/iac/scanners/helm/parser/parser_tar.go b/pkg/iac/scanners/helm/parser/parser_tar.go index c8d464e5a6..63da1aa97d 100644 --- a/pkg/iac/scanners/helm/parser/parser_tar.go +++ b/pkg/iac/scanners/helm/parser/parser_tar.go @@ -11,15 +11,14 @@ import ( "path" "path/filepath" - "github.com/liamg/memoryfs" - "github.com/aquasecurity/trivy/pkg/iac/detection" "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/mapfs" ) var errSkipFS = errors.New("skip parse FS") -func (p *Parser) unpackArchive(srcFS fs.FS, targetFS *memoryfs.FS, archivePath string) error { +func (p *Parser) unpackArchive(srcFS fs.FS, targetFS *mapfs.FS, archivePath string) error { file, err := srcFS.Open(archivePath) if err != nil { return fmt.Errorf("failed to open tar: %w", err) @@ -72,8 +71,13 @@ func (p *Parser) unpackArchive(srcFS fs.FS, targetFS *memoryfs.FS, archivePath s return err } case tar.TypeReg: + data, err := io.ReadAll(tr) + if err != nil { + return fmt.Errorf("read file: %w", err) + } + p.logger.Debug("Unpacking tar entry", log.FilePath(targetPath)) - if err := copyFile(targetFS, tr, targetPath); err != nil { + if err := writeFile(targetFS, data, targetPath); err != nil { return err } case tar.TypeSymlink: @@ -90,7 +94,9 @@ func (p *Parser) unpackArchive(srcFS fs.FS, targetFS *memoryfs.FS, archivePath s } for target, link := range symlinks { - if err := copySymlink(targetFS, link, target); err != nil { + p.logger.Debug("Copying symlink as file/dir", + log.String("target", target), log.String("link", link)) + if err := copyPath(targetFS, link, target); err != nil { return fmt.Errorf("copy symlink error: %w", err) } } @@ -102,43 +108,30 @@ func archiveEntryPath(archivePath, name string) string { return path.Join(path.Dir(archivePath), path.Clean(name)) } -func copySymlink(fsys *memoryfs.FS, src, dst string) error { - fi, err := fsys.Stat(src) - if err != nil { - return nil - } - if fi.IsDir() { - if err := copyDir(fsys, src, dst); err != nil { - return fmt.Errorf("copy dir error: %w", err) - } - return nil - } - - if err := copyFileLazy(fsys, src, dst); err != nil { - return fmt.Errorf("copy file error: %w", err) - } - - return nil -} - -func copyFile(fsys *memoryfs.FS, src io.Reader, dst string) error { +func writeFile(fsys *mapfs.FS, data []byte, dst string) error { if err := fsys.MkdirAll(path.Dir(dst), fs.ModePerm); err != nil && !errors.Is(err, fs.ErrExist) { return fmt.Errorf("mkdir error: %w", err) } - - b, err := io.ReadAll(src) - if err != nil { - return fmt.Errorf("read error: %w", err) - } - - if err := fsys.WriteFile(dst, b, fs.ModePerm); err != nil { - return fmt.Errorf("write file error: %w", err) - } - - return nil + return fsys.WriteVirtualFile(dst, data, fs.ModePerm) } -func copyDir(fsys *memoryfs.FS, src, dst string) error { +func copyPath(fsys *mapfs.FS, src, dst string) error { + fi, err := fsys.Stat(src) + if err != nil { + // the file is missing, just skip it + return nil + } + if fi.IsDir() { + return copyDir(fsys, src, dst) + } + data, err := fs.ReadFile(fsys, src) + if err != nil { + return fmt.Errorf("read file: %w", err) + } + return writeFile(fsys, data, dst) +} + +func copyDir(fsys *mapfs.FS, src, dst string) error { walkFn := func(filePath string, entry fs.DirEntry, err error) error { if err != nil { return err @@ -148,26 +141,13 @@ func copyDir(fsys *memoryfs.FS, src, dst string) error { return nil } - dst := path.Join(dst, filePath[len(src):]) - - if err := copyFileLazy(fsys, filePath, dst); err != nil { - return fmt.Errorf("copy file error: %w", err) + target := path.Join(dst, filePath[len(src):]) + data, err := fs.ReadFile(fsys, filePath) + if err != nil { + return fmt.Errorf("read file: %w", err) } - return nil + return writeFile(fsys, data, target) } return fs.WalkDir(fsys, src, walkFn) } - -func copyFileLazy(fsys *memoryfs.FS, src, dst string) error { - if err := fsys.MkdirAll(path.Dir(dst), fs.ModePerm); err != nil && !errors.Is(err, fs.ErrExist) { - return fmt.Errorf("mkdir error: %w", err) - } - return fsys.WriteLazyFile(dst, func() (io.Reader, error) { - f, err := fsys.Open(src) - if err != nil { - return nil, err - } - return f, nil - }, fs.ModePerm) -} diff --git a/pkg/iac/scanners/helm/scanner.go b/pkg/iac/scanners/helm/scanner.go index 037dc09bf1..3967b9db37 100644 --- a/pkg/iac/scanners/helm/scanner.go +++ b/pkg/iac/scanners/helm/scanner.go @@ -3,13 +3,11 @@ package helm import ( "context" "fmt" - "io" "io/fs" "path" "path/filepath" "strings" - "github.com/liamg/memoryfs" "helm.sh/helm/v3/pkg/chartutil" "github.com/aquasecurity/trivy/pkg/iac/detection" @@ -22,6 +20,7 @@ import ( "github.com/aquasecurity/trivy/pkg/iac/scanners/options" "github.com/aquasecurity/trivy/pkg/iac/types" "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/mapfs" ) var _ scanners.FSScanner = (*Scanner)(nil) @@ -129,29 +128,27 @@ func (s *Scanner) getScanResults(ctx context.Context, path string, target fs.FS) if err != nil { return nil, fmt.Errorf("unmarshal yaml: %w", err) } + + manifestFS := mapfs.New() + if err := manifestFS.MkdirAll(filepath.Dir(file.TemplateFilePath), fs.ModePerm); err != nil { + return nil, err + } + if err := manifestFS.WriteVirtualFile(file.TemplateFilePath, []byte(file.ManifestContent), fs.ModePerm); err != nil { + return nil, err + } + for _, manifest := range manifests { fileResults, err := rs.ScanInput(ctx, types.SourceKubernetes, rego.Input{ Path: file.TemplateFilePath, Contents: manifest, - FS: target, + FS: manifestFS, }) if err != nil { return nil, fmt.Errorf("scanning error: %w", err) } - if len(fileResults) > 0 { - renderedFS := memoryfs.New() - if err := renderedFS.MkdirAll(filepath.Dir(file.TemplateFilePath), fs.ModePerm); err != nil { - return nil, err - } - if err := renderedFS.WriteLazyFile(file.TemplateFilePath, func() (io.Reader, error) { - return strings.NewReader(file.ManifestContent), nil - }, fs.ModePerm); err != nil { - return nil, err - } - fileResults.SetSourceAndFilesystem(helmParser.ChartSource, renderedFS, detection.IsArchive(helmParser.ChartSource)) - fileResults.Ignore(ignoreRules, nil) - } + fileResults.SetSourceAndFilesystem(helmParser.ChartSource, manifestFS, detection.IsArchive(helmParser.ChartSource)) + fileResults.Ignore(ignoreRules, nil) results = append(results, fileResults...) } diff --git a/pkg/iac/scanners/terraform/deterministic_test.go b/pkg/iac/scanners/terraform/deterministic_test.go index fb2503b509..b5a21fe6ea 100644 --- a/pkg/iac/scanners/terraform/deterministic_test.go +++ b/pkg/iac/scanners/terraform/deterministic_test.go @@ -11,7 +11,7 @@ import ( ) func Test_DeterministicResults(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "first.tf": ` resource "aws_s3_bucket" "test" { for_each = other.thing diff --git a/pkg/iac/scanners/terraform/ignore_test.go b/pkg/iac/scanners/terraform/ignore_test.go index 7ef6d09d9c..8b96617371 100644 --- a/pkg/iac/scanners/terraform/ignore_test.go +++ b/pkg/iac/scanners/terraform/ignore_test.go @@ -589,7 +589,7 @@ func Test_IgnoreInlineByAVDID(t *testing.T) { func TestIgnoreRemoteTerraformResource(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.tf": `module "bucket" { source = "git::https://github.com/test/bucket" }`, diff --git a/pkg/iac/scanners/terraform/module_test.go b/pkg/iac/scanners/terraform/module_test.go index a70ec36582..71222e0190 100644 --- a/pkg/iac/scanners/terraform/module_test.go +++ b/pkg/iac/scanners/terraform/module_test.go @@ -364,7 +364,7 @@ resource "aws_s3_bucket" "test" { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fsys := testutil.CreateFS(t, tt.files) + fsys := testutil.CreateFS(tt.files) results, err := scanFS(fsys, "project", rego.WithPolicyReader(strings.NewReader(emptyBucketCheck)), rego.WithPolicyNamespaces("user"), diff --git a/pkg/iac/scanners/terraform/parser/load_vars_test.go b/pkg/iac/scanners/terraform/parser/load_vars_test.go index 384ced396e..09f256f261 100644 --- a/pkg/iac/scanners/terraform/parser/load_vars_test.go +++ b/pkg/iac/scanners/terraform/parser/load_vars_test.go @@ -12,7 +12,7 @@ import ( func Test_TFVarsFile(t *testing.T) { t.Run("tfvars file", func(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "test.tfvars": `instance_type = "t2.large"`, }) @@ -22,7 +22,7 @@ func Test_TFVarsFile(t *testing.T) { }) t.Run("tfvars json file", func(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "test.tfvars.json": `{ "variable": { "foo": { diff --git a/pkg/iac/scanners/terraform/parser/modules_test.go b/pkg/iac/scanners/terraform/parser/modules_test.go index 2377823a04..d88a873f77 100644 --- a/pkg/iac/scanners/terraform/parser/modules_test.go +++ b/pkg/iac/scanners/terraform/parser/modules_test.go @@ -70,7 +70,7 @@ module "this" { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fsys := testutil.CreateFS(t, tt.files) + fsys := testutil.CreateFS(tt.files) parser := New(fsys, "", OptionStopOnHCLError(true)) modules := lo.Map(lo.Keys(tt.files), func(p string, _ int) string { diff --git a/pkg/iac/scanners/terraform/parser/parser_test.go b/pkg/iac/scanners/terraform/parser/parser_test.go index 03bd198b44..f1de5273f9 100644 --- a/pkg/iac/scanners/terraform/parser/parser_test.go +++ b/pkg/iac/scanners/terraform/parser/parser_test.go @@ -23,7 +23,7 @@ import ( func Test_BasicParsing(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "test.tf": ` locals { @@ -154,7 +154,7 @@ check "cats_mittens_is_special" { func Test_Modules(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/test.tf": ` module "my-mod" { source = "../module" @@ -216,7 +216,7 @@ output "mod_result" { func Test_NestedParentModule(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/test.tf": ` module "my-mod" { source = "../." @@ -275,7 +275,7 @@ output "mod_result" { func Test_UndefinedModuleOutputReference(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/test.tf": ` resource "something" "blah" { value = module.x.y @@ -302,7 +302,7 @@ resource "something" "blah" { func Test_UndefinedModuleOutputReferenceInSlice(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/test.tf": ` resource "something" "blah" { value = ["first", module.x.y, "last"] @@ -340,7 +340,7 @@ resource "something" "blah" { func Test_TemplatedSliceValue(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/test.tf": ` variable "x" { @@ -384,7 +384,7 @@ resource "something" "blah" { func Test_SliceOfVars(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/test.tf": ` variable "x" { @@ -429,7 +429,7 @@ resource "something" "blah" { func Test_VarSlice(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/test.tf": ` variable "x" { @@ -473,7 +473,7 @@ resource "something" "blah" { func Test_LocalSliceNested(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/test.tf": ` variable "x" { @@ -521,7 +521,7 @@ resource "something" "blah" { func Test_FunctionCall(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/test.tf": ` variable "x" { @@ -565,7 +565,7 @@ resource "something" "blah" { } func Test_NullDefaultValueForVar(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "test.tf": ` variable "bucket_name" { type = string @@ -596,7 +596,7 @@ resource "aws_s3_bucket" "default" { } func Test_MultipleInstancesOfSameResource(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "test.tf": ` resource "aws_kms_key" "key1" { @@ -658,7 +658,7 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "this2" { } func Test_IfConfigFsIsNotSet_ThenUseModuleFsForVars(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": ` variable "bucket_name" { type = string @@ -689,7 +689,7 @@ resource "aws_s3_bucket" "main" { } func Test_ForEachRefToLocals(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": ` locals { buckets = toset([ @@ -725,7 +725,7 @@ resource "aws_s3_bucket" "this" { } func Test_ForEachRefToVariableWithDefault(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": ` variable "buckets" { type = set(string) @@ -759,7 +759,7 @@ resource "aws_s3_bucket" "this" { } func Test_ForEachRefToVariableFromFile(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": ` variable "policy_rules" { type = object({ @@ -814,7 +814,7 @@ policy_rules = { } func Test_ForEachRefersToMapThatContainsSameStringValues(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": `locals { buckets = { bucket1 = "test1" @@ -853,7 +853,7 @@ resource "aws_s3_bucket" "this" { } func TestDataSourceWithCountMetaArgument(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": ` data "http" "example" { count = 2 @@ -886,7 +886,7 @@ data "http" "example" { } func TestDataSourceWithForEachMetaArgument(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": ` locals { ports = ["80", "8080"] @@ -1120,7 +1120,7 @@ resource "aws_s3_bucket" "this" { } func TestForEachRefToResource(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": ` locals { vpcs = { @@ -1166,7 +1166,7 @@ resource "aws_internet_gateway" "example" { func TestArnAttributeOfBucketIsCorrect(t *testing.T) { t.Run("the bucket doesn't have a name", func(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": `resource "aws_s3_bucket" "this" {}`, }) parser := New(fs, "", OptionStopOnHCLError(true)) @@ -1192,7 +1192,7 @@ func TestArnAttributeOfBucketIsCorrect(t *testing.T) { }) t.Run("the bucket has a name", func(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": `resource "aws_s3_bucket" "this" { bucket = "test" } @@ -1251,7 +1251,7 @@ data "aws_iam_policy_document" "this" { } func TestForEachWithObjectsOfDifferentTypes(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": `module "backups" { bucket_name = each.key client = each.value.client @@ -1302,7 +1302,7 @@ func TestCountMetaArgument(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.tf": tt.src, }) parser := New(fsys, "", OptionStopOnHCLError(true)) @@ -1353,7 +1353,7 @@ func TestCountMetaArgumentInModule(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fsys := testutil.CreateFS(t, tt.files) + fsys := testutil.CreateFS(tt.files) parser := New(fsys, "", OptionStopOnHCLError(true)) require.NoError(t, parser.ParseFS(t.Context(), ".")) @@ -1659,7 +1659,7 @@ func TestNestedDynamicBlock(t *testing.T) { } func parse(t *testing.T, files map[string]string, opts ...Option) terraform.Modules { - fs := testutil.CreateFS(t, files) + fs := testutil.CreateFS(files) opts = append(opts, OptionStopOnHCLError(true)) parser := New(fs, "", opts...) require.NoError(t, parser.ParseFS(t.Context(), ".")) @@ -2281,7 +2281,7 @@ func TestTFVarsFileDoesNotExist(t *testing.T) { } func Test_OptionsWithTfVars(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "main.tf": `resource "test" "this" { foo = var.foo } @@ -2309,7 +2309,7 @@ variable "foo" {} func Test_AWSRegionNameDefined(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/test.tf": ` data "aws_region" "current" {} @@ -2568,7 +2568,7 @@ resource "aws_s3_bucket" "example" { } func Test_AttrIsRefToOtherBlock(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.tf": `locals { baz_idx = 0 } @@ -2746,7 +2746,7 @@ func TestInstancedLogger(t *testing.T) { }) t.Run("ModuleFetching", func(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.tf": ` module "invalid" { source = "totally.invalid" @@ -2792,7 +2792,7 @@ func TestInstancedLogger(t *testing.T) { func TestProvidedWorkingDirectory(t *testing.T) { const fakeCwd = "/some/path" - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.tf": ` resource "foo" "bar" { cwd = path.cwd @@ -2864,7 +2864,7 @@ module "test" { `, } - fsys := testutil.CreateFS(t, files) + fsys := testutil.CreateFS(files) parser := New(fsys, "", OptionWithSkipCachedModules(true), OptionStopOnHCLError(true), @@ -2898,7 +2898,7 @@ func Test_MarkedValues(t *testing.T) { `main.tf`: tt.src, } - fsys := testutil.CreateFS(t, files) + fsys := testutil.CreateFS(files) parser := New(fsys, "", OptionWithSkipCachedModules(true), OptionStopOnHCLError(true), diff --git a/pkg/iac/scanners/terraform/scanner_test.go b/pkg/iac/scanners/terraform/scanner_test.go index d4a5e153bc..37ead634b6 100644 --- a/pkg/iac/scanners/terraform/scanner_test.go +++ b/pkg/iac/scanners/terraform/scanner_test.go @@ -18,7 +18,7 @@ import ( func Test_OptionWithPolicyDirs(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "/code/main.tf": `resource "aws_s3_bucket" "my-bucket" {}`, "/rules/test.rego": emptyBucketCheck, }) @@ -113,7 +113,7 @@ func Test_OptionWithPolicyNamespaces(t *testing.T) { t.Run(strconv.Itoa(i), func(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "/code/main.tf": ` resource "aws_s3_bucket" "my-bucket" { bucket = "evil" @@ -161,7 +161,7 @@ cause := bucket.name } func Test_IAMPolicyRego(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "/code/main.tf": ` resource "aws_sqs_queue_policy" "bad_example" { queue_url = aws_sqs_queue.q.id @@ -229,7 +229,7 @@ deny[res] { } func Test_ContainerDefinitionRego(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "/code/main.tf": ` resource "aws_ecs_task_definition" "test" { family = "test" @@ -349,7 +349,7 @@ resource "aws_s3_bucket_public_access_block" "foo" { ` - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/main.tf": code, }) @@ -412,7 +412,7 @@ resource "aws_s3_bucket_public_access_block" "testB" { ` - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/main.tf": code, }) @@ -431,7 +431,7 @@ resource "aws_s3_bucket_public_access_block" "testB" { // PoC for replacing Go with Rego: AVD-AWS-0001 func Test_RegoRules(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "/code/main.tf": ` resource "aws_apigatewayv2_stage" "bad_example" { api_id = aws_apigatewayv2_api.example.id @@ -525,7 +525,7 @@ deny[res] { } func Test_OptionWithConfigsFileSystem(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/main.tf": ` variable "bucket_name" { type = string @@ -537,7 +537,7 @@ resource "aws_s3_bucket" "main" { "rules/bucket_name.rego": emptyBucketCheck, }) - configsFS := testutil.CreateFS(t, map[string]string{ + configsFS := testutil.CreateFS(map[string]string{ "main.tfvars": ` bucket_name = "test" `, @@ -562,7 +562,7 @@ bucket_name = "test" } func Test_OptionWithConfigsFileSystem_ConfigInCode(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/main.tf": ` variable "bucket_name" { type = string @@ -596,7 +596,7 @@ bucket_name = "test" } func Test_DoNotScanNonRootModules(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "/code/app1/main.tf": ` module "s3" { source = "./modules/s3" @@ -670,7 +670,7 @@ deny[res] { } func Test_RoleRefToOutput(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/main.tf": ` module "this" { source = "./modules/iam" @@ -738,7 +738,7 @@ deny[res] { } func Test_RegoRefToAwsProviderAttributes(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/providers.tf": ` provider "aws" { region = "us-east-2" @@ -811,7 +811,7 @@ deny[res] { } func TestScanModuleWithCount(t *testing.T) { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "code/main.tf": ` module "this" { count = 0 @@ -877,7 +877,7 @@ deny[res] { } func TestSkipDir(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "deployments/main.tf": ` module "use_bad_configuration" { source = "../modules" @@ -1259,7 +1259,7 @@ deny contains res if { } func Test_ScanTofuFiles(t *testing.T) { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "code/main.tofu": `resource "aws_s3_bucket" "this" {}`, "rules/check.rego": emptyBucketCheck, }) diff --git a/pkg/iac/scanners/terraform/setup_test.go b/pkg/iac/scanners/terraform/setup_test.go index c40884d045..2e6cdebbe1 100644 --- a/pkg/iac/scanners/terraform/setup_test.go +++ b/pkg/iac/scanners/terraform/setup_test.go @@ -78,7 +78,7 @@ is_group_mfa_enforced(group) if { ` func createModulesFromSource(t *testing.T, source, ext string) terraform.Modules { - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "source" + ext: source, }) @@ -107,7 +107,7 @@ func scanFS(fsys fs.FS, target string, opts ...options.ScannerOption) (scan.Resu func scanHCL(t *testing.T, source string, opts ...options.ScannerOption) scan.Results { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.tf": source, }) results, err := scanFS(fsys, ".", opts...) @@ -117,7 +117,7 @@ func scanHCL(t *testing.T, source string, opts ...options.ScannerOption) scan.Re func scanJSON(t *testing.T, source string, opts ...options.ScannerOption) scan.Results { - fsys := testutil.CreateFS(t, map[string]string{ + fsys := testutil.CreateFS(map[string]string{ "main.tf.json": source, }) diff --git a/pkg/iac/scanners/terraformplan/snapshot/snapshot.go b/pkg/iac/scanners/terraformplan/snapshot/snapshot.go index 5920a408d1..b71e35a064 100644 --- a/pkg/iac/scanners/terraformplan/snapshot/snapshot.go +++ b/pkg/iac/scanners/terraformplan/snapshot/snapshot.go @@ -12,11 +12,11 @@ import ( "slices" "strings" - "github.com/liamg/memoryfs" "github.com/zclconf/go-cty/cty" "github.com/aquasecurity/trivy/pkg/iac/scanners/terraform/parser" "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/mapfs" iox "github.com/aquasecurity/trivy/pkg/x/io" ) @@ -190,7 +190,7 @@ func (s *snapshot) getOrCreateModuleSnapshot(key string) *snapshotModule { } func (s *snapshot) toFS() (fs.FS, error) { - fsys := memoryfs.New() + fsys := mapfs.New() if err := s.writeManifest(fsys); err != nil { log.WithPrefix(log.PrefixMisconfiguration).Error("Failed to write manifest file", log.Err(err)) @@ -205,7 +205,7 @@ func (s *snapshot) toFS() (fs.FS, error) { if module.dir != "" { filePath = path.Join(module.dir, filename) } - if err := fsys.WriteFile(filePath, file, fs.ModePerm); err != nil { + if err := fsys.WriteVirtualFile(filePath, file, fs.ModePerm); err != nil { return nil, fmt.Errorf("failed to add file: %w", err) } } @@ -213,7 +213,7 @@ func (s *snapshot) toFS() (fs.FS, error) { return fsys, nil } -func (s *snapshot) writeManifest(fsys *memoryfs.FS) error { +func (s *snapshot) writeManifest(fsys *mapfs.FS) error { if err := fsys.MkdirAll(path.Dir(parser.ManifestSnapshotFile), fs.ModePerm); err != nil { return fmt.Errorf("create manifest directory: %w", err) } @@ -223,7 +223,7 @@ func (s *snapshot) writeManifest(fsys *memoryfs.FS) error { return fmt.Errorf("marshal manifest snapshot: %w", err) } - if err := fsys.WriteFile(parser.ManifestSnapshotFile, b, fs.ModePerm); err != nil { + if err := fsys.WriteVirtualFile(parser.ManifestSnapshotFile, b, fs.ModePerm); err != nil { return fmt.Errorf("write manifest snapshot: %w", err) } return nil diff --git a/pkg/iac/scanners/terraformplan/tfjson/parser/parser.go b/pkg/iac/scanners/terraformplan/tfjson/parser/parser.go index a55cfa9938..7aaad2762f 100644 --- a/pkg/iac/scanners/terraformplan/tfjson/parser/parser.go +++ b/pkg/iac/scanners/terraformplan/tfjson/parser/parser.go @@ -5,13 +5,13 @@ import ( "encoding/json" "fmt" "io" + "io/fs" "os" "strings" - "github.com/liamg/memoryfs" - "github.com/aquasecurity/trivy/pkg/iac/terraform" "github.com/aquasecurity/trivy/pkg/log" + "github.com/aquasecurity/trivy/pkg/mapfs" ) type Parser struct { @@ -50,9 +50,9 @@ func (p *Parser) Parse(reader io.Reader) (*PlanFile, error) { } -func (p *PlanFile) ToFS() (*memoryfs.FS, error) { +func (p *PlanFile) ToFS() (fs.FS, error) { - rootFS := memoryfs.New() + rootFS := mapfs.New() var fileResources []string @@ -66,7 +66,7 @@ func (p *PlanFile) ToFS() (*memoryfs.FS, error) { } fileContent := strings.Join(fileResources, "\n\n") - if err := rootFS.WriteFile("main.tf", []byte(fileContent), os.ModePerm); err != nil { + if err := rootFS.WriteVirtualFile("main.tf", []byte(fileContent), os.ModePerm); err != nil { return nil, err } return rootFS, nil diff --git a/pkg/iac/scanners/terraformplan/tfjson/scanner_test.go b/pkg/iac/scanners/terraformplan/tfjson/scanner_test.go index 88cd4e96b5..4d26738e14 100644 --- a/pkg/iac/scanners/terraformplan/tfjson/scanner_test.go +++ b/pkg/iac/scanners/terraformplan/tfjson/scanner_test.go @@ -106,7 +106,7 @@ deny[cause] { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { b, _ := os.ReadFile(tc.inputFile) - fs := testutil.CreateFS(t, map[string]string{ + fs := testutil.CreateFS(map[string]string{ "/code/main.tfplan.json": string(b), "/rules/test.rego": tc.check, }) diff --git a/pkg/iac/types/fskey_test.go b/pkg/iac/types/fskey_test.go index de91b32a04..a9288b58bc 100644 --- a/pkg/iac/types/fskey_test.go +++ b/pkg/iac/types/fskey_test.go @@ -4,10 +4,11 @@ import ( "io/fs" "os" "testing" + "testing/fstest" - "github.com/liamg/memoryfs" "github.com/stretchr/testify/assert" + "github.com/aquasecurity/trivy/pkg/mapfs" "github.com/aquasecurity/trivy/pkg/set" ) @@ -16,8 +17,9 @@ func Test_FSKey(t *testing.T) { systems := []fs.FS{ os.DirFS("."), os.DirFS(".."), - memoryfs.New(), - memoryfs.New(), + fstest.MapFS{}, + mapfs.New(), + mapfs.New(), } keys := set.New[string]() diff --git a/pkg/iac/types/range.go b/pkg/iac/types/range.go index 7bc701ae09..a013e3f8b2 100755 --- a/pkg/iac/types/range.go +++ b/pkg/iac/types/range.go @@ -8,41 +8,27 @@ import ( ) func NewRange(filename string, startLine, endLine int, sourcePrefix string, srcFS fs.FS) Range { - r := Range{ - filename: filename, - startLine: startLine, - endLine: endLine, - fs: srcFS, - fsKey: CreateFSKey(srcFS), - sourcePrefix: sourcePrefix, - } - return r + return newRange(filename, startLine, endLine, sourcePrefix, CreateFSKey(srcFS), srcFS, false) } -func NewRangeWithLogicalSource(filename string, startLine int, endLine int, sourcePrefix string, - srcFS fs.FS) Range { - r := Range{ - filename: filename, - startLine: startLine, - endLine: endLine, - fs: srcFS, - fsKey: CreateFSKey(srcFS), - sourcePrefix: sourcePrefix, - isLogicalSource: true, - } - return r +func NewRangeWithLogicalSource(filename string, startLine, endLine int, sourcePrefix string, srcFS fs.FS) Range { + return newRange(filename, startLine, endLine, sourcePrefix, CreateFSKey(srcFS), srcFS, true) } func NewRangeWithFSKey(filename string, startLine, endLine int, sourcePrefix, fsKey string, fsys fs.FS) Range { - r := Range{ - filename: filename, - startLine: startLine, - endLine: endLine, - fs: fsys, - fsKey: fsKey, - sourcePrefix: sourcePrefix, + return newRange(filename, startLine, endLine, sourcePrefix, fsKey, fsys, false) +} + +func newRange(filename string, startLine, endLine int, sourcePrefix, fsKey string, fsys fs.FS, isLogical bool) Range { + return Range{ + filename: filename, + startLine: startLine, + endLine: endLine, + fs: fsys, + fsKey: fsKey, + sourcePrefix: sourcePrefix, + isLogicalSource: isLogical, } - return r } type Range struct {