chore: replace make with mage (#3932)

This commit is contained in:
Teppei Fukuda
2023-03-30 10:40:24 +03:00
committed by GitHub
parent 67236f6aac
commit b40f60c405
9 changed files with 466 additions and 174 deletions

View File

@@ -46,13 +46,13 @@ jobs:
skip-cache: true # https://github.com/golangci/golangci-lint-action/issues/244#issuecomment-1052197778
if: matrix.operating-system == 'ubuntu-latest'
# Install tools
- uses: aquaproj/aqua-installer@v2.0.2
- name: Install tools
uses: aquaproj/aqua-installer@v2.0.2
with:
aqua_version: v1.25.0
- name: Run unit tests
run: make test
run: mage test:unit
integration:
name: Integration Test
@@ -66,8 +66,13 @@ jobs:
with:
go-version-file: go.mod
- name: Install tools
uses: aquaproj/aqua-installer@v2.0.2
with:
aqua_version: v1.25.0
- name: Run integration tests
run: make test-integration
run: mage test:integration
module-test:
name: Module Integration Test
@@ -81,15 +86,15 @@ jobs:
with:
go-version-file: go.mod
# Install tools
- uses: aquaproj/aqua-installer@v2.0.2
- name: Install tools
uses: aquaproj/aqua-installer@v2.0.2
with:
aqua_version: v1.25.0
- name: Run module integration tests
shell: bash
run: |
make test-module-integration
mage test:module
build-test:
name: Build Test

View File

@@ -27,6 +27,10 @@ jobs:
uses: actions/setup-go@v3
with:
go-version-file: go.mod
- name: Install tools
uses: aquaproj/aqua-installer@v2.0.2
with:
aqua_version: v1.25.0
- name: Run vm integration tests
run: |
make test-vm-integration
mage test:vm

View File

@@ -10,3 +10,6 @@ RUN curl --retry 5 -OL https://github.com/protocolbuffers/protobuf/releases/down
RUN go install github.com/twitchtv/twirp/protoc-gen-twirp@v8.1.0
RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.27.1
RUN go install github.com/magefile/mage@v1.14.0
ENV TRIVY_PROTOC_CONTAINER=true

143
Makefile
View File

@@ -1,143 +0,0 @@
VERSION := $(patsubst v%,%,$(shell git describe --tags --always)) #Strips the v prefix from the tag
LDFLAGS := -ldflags "-s -w -X=main.version=$(VERSION)"
GOPATH := $(firstword $(subst :, ,$(shell go env GOPATH)))
GOBIN := $(GOPATH)/bin
GOSRC := $(GOPATH)/src
TEST_MODULE_DIR := pkg/module/testdata
TEST_MODULE_SRCS := $(wildcard $(TEST_MODULE_DIR)/*/*.go)
TEST_MODULES := $(patsubst %.go,%.wasm,$(TEST_MODULE_SRCS))
EXAMPLE_MODULE_DIR := examples/module
EXAMPLE_MODULE_SRCS := $(wildcard $(EXAMPLE_MODULE_DIR)/*/*.go)
EXAMPLE_MODULES := $(patsubst %.go,%.wasm,$(EXAMPLE_MODULE_SRCS))
MKDOCS_IMAGE := aquasec/mkdocs-material:dev
MKDOCS_PORT := 8000
export CGO_ENABLED := 0
u := $(if $(update),-u)
# Tools
$(GOBIN)/wire:
go install github.com/google/wire/cmd/wire@v0.5.0
$(GOBIN)/crane:
go install github.com/google/go-containerregistry/cmd/crane@v0.9.0
$(GOBIN)/golangci-lint:
curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s -- -b $(GOBIN) v1.52.2
$(GOBIN)/labeler:
go install github.com/knqyf263/labeler@latest
$(GOBIN)/easyjson:
go install github.com/mailru/easyjson/...@v0.7.7
$(GOBIN)/goyacc:
go install golang.org/x/tools/cmd/goyacc@latest
.PHONY: wire
wire: $(GOBIN)/wire
wire gen ./pkg/commands/... ./pkg/rpc/...
.PHONY: mock
mock: $(GOBIN)/mockery
mockery -all -inpkg -case=snake -dir $(DIR)
.PHONY: deps
deps:
go get ${u} -d
go mod tidy
.PHONY: generate-test-modules
generate-test-modules: $(TEST_MODULES)
# Compile WASM modules for unit and integration tests
%.wasm:%.go
@if !(type "tinygo" > /dev/null 2>&1); then \
echo "Need to install TinyGo. Follow https://tinygo.org/getting-started/install/"; \
exit 1; \
fi
go generate $<
# Run unit tests
.PHONY: test
test: $(TEST_MODULES)
go test -v -short -coverprofile=coverage.txt -covermode=atomic ./...
integration/testdata/fixtures/images/*.tar.gz: $(GOBIN)/crane
mkdir -p integration/testdata/fixtures/images/
integration/scripts/download-images.sh
# Run integration tests
.PHONY: test-integration
test-integration: integration/testdata/fixtures/images/*.tar.gz
go test -v -tags=integration ./integration/... ./pkg/fanal/test/integration/...
# Run WASM integration tests
.PHONY: test-module-integration
test-module-integration: integration/testdata/fixtures/images/*.tar.gz $(EXAMPLE_MODULES)
go test -v -tags=module_integration ./integration/...
# Run VM integration tests
.PHONY: test-vm-integration
test-vm-integration: integration/testdata/fixtures/vm-images/*.img.gz
go test -v -tags=vm_integration ./integration/...
integration/testdata/fixtures/vm-images/*.img.gz:
integration/scripts/download-vm-images.sh
.PHONY: lint
lint: $(GOBIN)/golangci-lint
$(GOBIN)/golangci-lint run --timeout 5m
.PHONY: fmt
fmt:
find ./ -name "*.proto" | xargs clang-format -i
.PHONY: build
build:
go build $(LDFLAGS) ./cmd/trivy
.PHONY: protoc
protoc:
docker build -t trivy-protoc - < Dockerfile.protoc
docker run --rm -it -v ${PWD}:/app -w /app trivy-protoc make _$@
_protoc:
for path in `find ./rpc/ -name "*.proto" -type f`; do \
protoc --twirp_out=. --twirp_opt=paths=source_relative --go_out=. --go_opt=paths=source_relative $${path} || exit; \
done
.PHONY: install
install:
go install $(LDFLAGS) ./cmd/trivy
.PHONY: clean
clean:
rm -rf integration/testdata/fixtures/images
# Create labels on GitHub
.PHONY: label
label: $(GOBIN)/labeler
labeler apply misc/triage/labels.yaml -r aquasecurity/trivy -l 5
# Run MkDocs development server to preview the documentation page
.PHONY: mkdocs-serve
mkdocs-serve:
docker build -t $(MKDOCS_IMAGE) -f docs/build/Dockerfile docs/build
docker run --name mkdocs-serve --rm -v $(PWD):/docs -p $(MKDOCS_PORT):8000 $(MKDOCS_IMAGE)
# Generate JSON marshaler/unmarshaler for TinyGo/WebAssembly as TinyGo doesn't support encoding/json.
.PHONY: easyjson
easyjson: $(GOBIN)/easyjson
easyjson pkg/module/serialize/types.go
# Generate license parser with goyacc
.PHONY: yacc
yacc: $(GOBIN)/goyacc
go generate ./pkg/licensing/expression/...

View File

@@ -6,3 +6,4 @@ registries:
ref: v3.106.0 # renovate: depName=aquaproj/aqua-registry
packages:
- name: tinygo-org/tinygo@v0.26.0
- name: magefile/mage@v1.14.0

View File

@@ -9,11 +9,59 @@ Thank you for taking interest in contributing to Trivy!
1. Your PR is more likely to be accepted if it includes tests (We have not historically been very strict about tests, but we would like to improve this!).
1. If your PR affects the user experience in some way, please update the README.md and the CLI help accordingly.
### Title
## Development
Install the necessary tools for development by following their respective installation instructions.
- [Go](https://go.dev/doc/install)
- [Mage](https://magefile.org/)
### Build
After making changes to the Go source code, build the project with the following command:
```shell
$ mage build
$ ./trivy -h
```
### Lint
You must pass the linter checks:
```shell
$ mage lint
```
Additionally, you need to have run `go mod tidy`, so execute the following command as well:
```shell
$ mage tidy
```
### Unit tests
Your PR must pass all the unit tests. You can test it as below.
```
$ mage test:unit
```
### Integration tests
Your PR must pass all the integration tests. You can test it as below.
```
$ mage test:integration
```
### Documentation
You can build the documents as below and view it at http://localhost:8000.
```
$ mage docs:serve
```
## Title
It is not that strict, but we use the title conventions in this repository.
Each commit message doesn't have to follow the conventions as long as it is clear and descriptive since it will be squashed and merged.
#### Format of the title
### Format of the title
```
<type>(<scope>): <subject>
@@ -122,7 +170,7 @@ others:
The `<scope>` can be empty (e.g. if the change is a global or difficult to assign to a single component), in which case the parentheses are omitted.
#### Example titles
### Example titles
```
feat(alma): add support for AlmaLinux
@@ -143,33 +191,15 @@ chore(deps): bump go.uber.org/zap from 1.19.1 to 1.20.0
**NOTE**: please do not use `chore(deps): update fanal` and something like that if you add new features or fix bugs in Trivy-related projects.
The PR title should describe what the PR adds or fixes even though it just updates the dependency in Trivy.
### Unit tests
Your PR must pass all the unit tests. You can test it as below.
## Commits
```
$ make test
```
### Integration tests
Your PR must pass all the integration tests. You can test it as below.
```
$ make test-integration
```
### Documentation
You can build the documents as below and view it at http://localhost:8000.
```
$ make mkdocs-serve
```
## Understand where your pull request belongs
Trivy is composed of several repositories that work together:
- [Trivy](https://github.com/aquasecurity/trivy) is the client-side, user-facing, command line tool.
- [vuln-list](https://github.com/aquasecurity/vuln-list) is a vulnerabilities database, aggregated from different sources, and normalized for easy consumption. Think of this as the "server" side of the trivy command line tool. **There should be no pull requests to this repo**
- [vuln-list](https://github.com/aquasecurity/vuln-list) is a vulnerability database, aggregated from different sources, and normalized for easy consumption. Think of this as the "server" side of the trivy command line tool. **There should be no pull requests to this repo**
- [vuln-list-update](https://github.com/aquasecurity/vuln-list-update) is the code that maintains the vuln-list database.
- [trivy-db](https://github.com/aquasecurity/trivy-db) maintains the vulnerability database pulled by Trivy CLI.
- [go-dep-parser](https://github.com/aquasecurity/go-dep-parser) is a library for parsing lock files such as package-lock.json and Gemfile.lock.

1
go.mod
View File

@@ -58,6 +58,7 @@ require (
github.com/knqyf263/go-rpmdb v0.0.0-20230301153543-ba94b245509b
github.com/knqyf263/nested v0.0.1
github.com/kylelemons/godebug v1.1.0
github.com/magefile/mage v1.14.0
github.com/mailru/easyjson v0.7.7
github.com/masahiro331/go-disk v0.0.0-20220919035250-c8da316f91ac
github.com/masahiro331/go-ebs-file v0.0.0-20221225061409-5ef263bb2cc3

2
go.sum
View File

@@ -1245,6 +1245,8 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9
github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo=
github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 h1:EnfXoSqDfSNJv0VBNqY/88RNnhSGYkrHaO0mmFGbVsc=
github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40/go.mod h1:vy1vK6wD6j7xX6O6hXe621WabdtNkou2h7uRtTfRMyg=
github.com/magefile/mage v1.14.0 h1:6QDX3g6z1YvJ4olPhT1wksUcSa/V0a1B+pJb73fBjyo=
github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=

389
magefiles/magefile.go Normal file
View File

@@ -0,0 +1,389 @@
package main
import (
"errors"
"fmt"
"io/fs"
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"github.com/magefile/mage/target"
)
var (
GOPATH = os.Getenv("GOPATH")
GOBIN = filepath.Join(GOPATH, "bin")
ENV = map[string]string{
"CGO_ENABLED": "0",
}
)
func version() (string, error) {
if ver, err := sh.Output("git", "describe", "--tags", "--always"); err != nil {
return "", err
} else {
// Strips the v prefix from the tag
return strings.TrimPrefix(ver, "v"), nil
}
}
func buildLdflags() (string, error) {
ver, err := version()
if err != nil {
return "", err
}
return fmt.Sprintf("-s -w -X=main.version=%s", ver), nil
}
type Tool mg.Namespace
// Aqua installs aqua if not installed
func (Tool) Aqua() error {
if exists(filepath.Join(GOBIN, "aqua")) {
return nil
}
return sh.Run("go", "install", "github.com/aquaproj/aqua/v2/cmd/aqua@v2.2.1")
}
// Wire installs wire if not installed
func (Tool) Wire() error {
if installed("wire") {
return nil
}
return sh.Run("go", "install", "github.com/google/wire/cmd/wire@v0.5.0")
}
// Crane installs crane
func (Tool) Crane() error {
if exists(filepath.Join(GOBIN, "crane")) {
return nil
}
return sh.Run("go", "install", "github.com/google/go-containerregistry/cmd/crane@v0.9.0")
}
// GolangciLint installs golangci-lint
func (Tool) GolangciLint() error {
const version = "v1.52.2"
if exists(filepath.Join(GOBIN, "golangci-lint")) {
return nil
}
command := fmt.Sprintf("curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b %s %s", GOBIN, version)
return sh.Run("bash", "-c", command)
}
// Labeler installs labeler
func (Tool) Labeler() error {
if exists(filepath.Join(GOBIN, "labeler")) {
return nil
}
return sh.Run("go", "install", "github.com/knqyf263/labeler@latest")
}
// EasyJSON installs easyjson
func (Tool) EasyJSON() error {
if exists(filepath.Join(GOBIN, "easyjson")) {
return nil
}
return sh.Run("go", "install", "github.com/mailru/easyjson/...@v0.7.7")
}
// Goyacc installs goyacc
func (Tool) Goyacc() error {
if exists(filepath.Join(GOBIN, "goyacc")) {
return nil
}
return sh.Run("go", "install", "golang.org/x/tools/cmd/goyacc@v0.7.0")
}
// Mockery installs mockery
func (Tool) Mockery() error {
if exists(filepath.Join(GOBIN, "mockery")) {
return nil
}
return sh.Run("go", "install", "github.com/knqyf263/mockery/cmd/mockery@latest")
}
// Wire generates the wire_gen.go file for each package
func Wire() error {
mg.Deps(Tool{}.Wire)
return sh.RunV("wire", "gen", "./pkg/commands/...", "./pkg/rpc/...")
}
// Mock generates mocks
func Mock(dir string) error {
mg.Deps(Tool{}.Mockery)
mockeryArgs := []string{
"-all",
"-inpkg",
"-case=snake",
"-dir",
dir,
}
return sh.RunV("mockery", mockeryArgs...)
}
// Protoc parses PROTO_FILES and generates the Go code for client/server mode
func Protoc() error {
// It is called in the protoc container
if _, ok := os.LookupEnv("TRIVY_PROTOC_CONTAINER"); ok {
protoFiles, err := findProtoFiles()
if err != nil {
return err
}
for _, file := range protoFiles {
// Check if the generated Go file is up-to-date
dst := strings.TrimSuffix(file, ".proto") + ".pb.go"
if updated, err := target.Path(dst, file); err != nil {
return err
} else if !updated {
continue
}
// Generate
if err = sh.RunV("protoc", "--twirp_out", ".", "--twirp_opt", "paths=source_relative",
"--go_out", ".", "--go_opt", "paths=source_relative", file); err != nil {
return err
}
}
return nil
}
// It is called on the host
if err := sh.RunV("bash", "-c", "docker build -t trivy-protoc - < Dockerfile.protoc"); err != nil {
return err
}
return sh.Run("docker", "run", "--rm", "-it", "-v", "${PWD}:/app", "-w", "/app", "trivy-protoc", "mage", "protoc")
}
// Yacc generates parser
func Yacc() error {
mg.Deps(Tool{}.Goyacc)
return sh.Run("go", "generate", "./pkg/licensing/expression/...")
}
// Easyjson generates JSON marshaler/unmarshaler for TinyGo/WebAssembly as TinyGo doesn't support encoding/json.
func Easyjson() error {
mg.Deps(Tool{}.EasyJSON)
return sh.Run("easyjson", "./pkg/module/serialize/types.go")
}
type Test mg.Namespace
// FixtureContainerImages downloads and extracts required images
func (Test) FixtureContainerImages() error {
mg.Deps(Tool{}.Crane)
if err := os.MkdirAll(filepath.Join("integration", "testdata", "fixtures", "images"), 0750); err != nil {
return err
}
downloadScript := filepath.Join("integration", "scripts", "download-images.sh")
return sh.Run(downloadScript)
}
// FixtureVMImages downloads and extracts required VM images
func (Test) FixtureVMImages() error {
mg.Deps(Tool{}.Crane)
downloadScript := filepath.Join("integration", "scripts", "download-vm-images.sh")
return sh.Run(downloadScript)
}
// GenerateModules compiles WASM modules for unit tests
func (Test) GenerateModules() error {
pattern := filepath.Join("pkg", "module", "testdata", "*", "*.go")
if err := compileWasmModules(pattern); err != nil {
return err
}
return nil
}
// GenerateExampleModules compiles example Wasm modules for integration tests
func (Test) GenerateExampleModules() error {
pattern := filepath.Join("examples", "module", "*", "*.go")
if err := compileWasmModules(pattern); err != nil {
return err
}
return nil
}
func compileWasmModules(pattern string) error {
goFiles, err := filepath.Glob(pattern)
if err != nil {
return err
}
for _, src := range goFiles {
// e.g. examples/module/spring4shell/spring4shell.go
// => examples/module/spring4shell/spring4shell.wasm
dst := strings.TrimSuffix(src, ".go") + ".wasm"
if updated, err := target.Path(dst, src); err != nil {
return err
} else if !updated {
continue
}
// Check if TinyGo is installed
if !installed("tinygo") {
return errors.New("need to install TinyGo, follow https://tinygo.org/getting-started/install/")
}
if err = sh.Run("go", "generate", src); err != nil {
return err
}
}
return nil
}
// Unit runs unit tests
func (t Test) Unit() error {
mg.Deps(t.GenerateModules)
return sh.RunWithV(ENV, "go", "test", "-v", "-short", "-coverprofile=coverage.txt", "-covermode=atomic", "./...")
}
// Integration runs integration tests
func (t Test) Integration() error {
mg.Deps(t.FixtureContainerImages)
return sh.RunWithV(ENV, "go", "test", "-v", "-tags=integration", "./integration/...", "./pkg/fanal/test/integration/...")
}
// Module runs Wasm integration tests
func (t Test) Module() error {
mg.Deps(t.FixtureContainerImages, t.GenerateExampleModules)
return sh.RunWithV(ENV, "go", "test", "-v", "-tags=module_integration", "./integration/...")
}
// VM runs VM integration tests
func (t Test) VM() error {
mg.Deps(t.FixtureVMImages)
return sh.RunWithV(ENV, "go", "test", "-v", "-tags=vm_integration", "./integration/...")
}
// Lint runs linters
func Lint() error {
mg.Deps(Tool{}.GolangciLint)
return sh.RunV("golangci-lint", "run", "--timeout", "5m")
}
// Fmt formats Go code and proto files
func Fmt() error {
// Check if clang-format is installed
if !installed("clang-format") {
return errors.New("need to install clang-format")
}
// Format proto files
protoFiles, err := findProtoFiles()
if err != nil {
return err
}
for _, file := range protoFiles {
if err = sh.Run("clang-format", "-i", file); err != nil {
return err
}
}
// Format Go code
return sh.Run("go", "fmt", "./...")
}
// Tidy makes sure go.mod matches the source code in the module
func Tidy() error {
return sh.RunV("go", "mod", "tidy")
}
// Build builds Trivy
func Build() error {
if updated, err := target.Dir("trivy", "pkg", "cmd"); err != nil {
return err
} else if !updated {
return nil
}
ldflags, err := buildLdflags()
if err != nil {
return err
}
wd, err := os.Getwd()
if err != nil {
return err
}
return sh.RunWith(ENV, "go", "build", "-ldflags", ldflags, filepath.Join(wd, "cmd", "trivy"))
}
// Install installs Trivy
func Install() error {
ldflags, err := buildLdflags()
if err != nil {
return err
}
wd, err := os.Getwd()
if err != nil {
return err
}
return sh.RunWith(ENV, "go", "install", "-ldflags", ldflags, filepath.Join(wd, "cmd", "trivy"))
}
// Clean cleans up the fixtures
func Clean() error {
fixtureDir := filepath.Join("integration", "testdata", "fixtures")
paths := []string{
filepath.Join(fixtureDir, "images"),
filepath.Join(fixtureDir, "vm-images"),
}
for _, p := range paths {
if err := sh.Rm(p); err != nil {
return err
}
}
return nil
}
type Docs mg.Namespace
// Serve launches MkDocs development server to preview the documentation page
func (Docs) Serve() error {
const (
mkdocsImage = "aquasec/mkdocs-material:dev"
mkdocsPort = "8000"
)
if err := sh.Run("docker", "build", "-t", mkdocsImage, "-f", "docs/build/Dockerfile", "docs/build"); err != nil {
return err
}
return sh.Run("docker", "run", "--name", "mkdocs-serve", "--rm", "-v", "${PWD}:/docs", "-p", mkdocsPort+":8000", mkdocsImage)
}
// Generate generates CLI references
func (Docs) Generate() error {
// TODO
return nil
}
func findProtoFiles() ([]string, error) {
var files []string
err := filepath.WalkDir("rpc", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
} else if d.IsDir() {
return nil
} else if filepath.Ext(path) == ".proto" {
files = append(files, path)
}
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
func exists(filename string) bool {
_, err := os.Stat(filename)
return err == nil
}
func installed(cmd string) bool {
_, err := exec.LookPath(cmd)
return err == nil
}